From: Mark Wooding Date: Wed, 01 Feb 2006 11:44:26 GMT Subject: [PATCH 1/9] http-fetch: Mark slots as `watched' to stop them being reused. Message-ID: <20060201114426.5042.97599.stgit@metalzone.distorted.org.uk> In-Reply-To: <20060201112822.5042.41256.stgit@metalzone.distorted.org.uk> From: Mark Wooding Previously, it was possible for the slot passed to run_active_slot() to be reused before that function returned, rendering its completion status useless. This is particularly noticeable when fetching a repository with a wide, flat directory structure (lots of blobs in the top-level tree), a single unpacked commit which changes just one or two blobs, and the rest of the history in one pack. When fetch_indices() gets the pack-list, the slot is (usually!) reused for some blob object which is packed, so it fails with 404; fetch_indices() decides that the pack-list fetch failed, and nothing therefore fetches the pack. We fix this by adding a feature to lock slots against reuse. Signed-off-by: Mark Wooding --- http-fetch.c | 6 +++--- http.c | 43 +++++++++++++++++++++++++++++++++---------- http.h | 6 +++++- 3 files changed, 41 insertions(+), 14 deletions(-) diff --git a/http-fetch.c b/http-fetch.c index 61b2188..4b1b62d 100644 --- a/http-fetch.c +++ b/http-fetch.c @@ -465,13 +465,13 @@ static void process_alternates_response( base); curl_easy_setopt(slot->curl, CURLOPT_URL, alt_req->url); - active_requests++; - slot->in_use = 1; + if (!slot->in_use) + active_requests++; + slot->in_use |= SLOTUSE_ACTIVE; if (start_active_slot(slot)) { return; } else { got_alternates = -1; - slot->in_use = 0; return; } } diff --git a/http.c b/http.c index 75e6717..498b4ae 100644 --- a/http.c +++ b/http.c @@ -262,7 +262,7 @@ void http_cleanup(void) while (slot != NULL) { #ifdef USE_CURL_MULTI - if (slot->in_use) { + if (slot->in_use & SLOTUSE_ACTIVE) { curl_easy_getinfo(slot->curl, CURLINFO_EFFECTIVE_URL, &wait_url); @@ -311,6 +311,7 @@ struct active_request_slot *get_active_s newslot->curl = NULL; newslot->in_use = 0; newslot->next = NULL; + newslot->nrefs = 0; slot = active_queue_head; if (slot == NULL) { @@ -333,7 +334,7 @@ struct active_request_slot *get_active_s } active_requests++; - slot->in_use = 1; + slot->in_use |= SLOTUSE_ACTIVE; slot->local = NULL; slot->callback_data = NULL; slot->callback_func = NULL; @@ -351,8 +352,9 @@ int start_active_slot(struct active_requ if (curlm_result != CURLM_OK && curlm_result != CURLM_CALL_MULTI_PERFORM) { - active_requests--; - slot->in_use = 0; + slot->in_use &= ~SLOTUSE_ACTIVE; + if (!slot->in_use) + active_requests--; return 0; } #endif @@ -375,6 +377,24 @@ void step_active_slots(void) } #endif +static void watch_active_slot(struct active_request_slot *slot) +{ + if (!slot->in_use) + active_requests++; + slot->in_use |= SLOTUSE_REF; + slot->nrefs++; +} + +static void unwatch_active_slot(struct active_request_slot *slot) +{ + slot->nrefs--; + if (!slot->nrefs) { + slot->in_use &= ~SLOTUSE_REF; + if (!slot->in_use) + active_requests--; + } +} + void run_active_slot(struct active_request_slot *slot) { #ifdef USE_CURL_MULTI @@ -386,7 +406,8 @@ void run_active_slot(struct active_reque int max_fd; struct timeval select_timeout; - while (slot->in_use) { + watch_active_slot(slot); + while (slot->in_use & SLOTUSE_ACTIVE) { data_received = 0; step_active_slots(); @@ -397,7 +418,7 @@ void run_active_slot(struct active_reque last_pos = current_pos; } - if (slot->in_use && !data_received) { + if ((slot->in_use & SLOTUSE_ACTIVE) && !data_received) { max_fd = 0; FD_ZERO(&readfds); FD_ZERO(&writefds); @@ -408,8 +429,9 @@ void run_active_slot(struct active_reque &excfds, &select_timeout); } } + unwatch_active_slot(slot); #else - while (slot->in_use) { + while (slot->in_use & SLOTUSE_ACTIVE) { slot->curl_result = curl_easy_perform(slot->curl); finish_active_slot(slot); } @@ -418,8 +440,9 @@ void run_active_slot(struct active_reque static void finish_active_slot(struct active_request_slot *slot) { - active_requests--; - slot->in_use = 0; + slot->in_use &= ~SLOTUSE_ACTIVE; + if (!slot->in_use) + active_requests--; curl_easy_getinfo(slot->curl, CURLINFO_HTTP_CODE, &slot->http_code); /* Run callback if appropriate */ @@ -433,7 +456,7 @@ void finish_all_active_slots(void) struct active_request_slot *slot = active_queue_head; while (slot != NULL) - if (slot->in_use) { + if (slot->in_use & SLOTUSE_ACTIVE) { run_active_slot(slot); slot = active_queue_head; } else { diff --git a/http.h b/http.h index ed4ea33..a66e06a 100644 --- a/http.h +++ b/http.h @@ -26,7 +26,8 @@ struct active_request_slot { CURL *curl; FILE *local; - int in_use; + unsigned in_use; + int nrefs; CURLcode curl_result; long http_code; void *callback_data; @@ -34,6 +35,9 @@ struct active_request_slot struct active_request_slot *next; }; +#define SLOTUSE_ACTIVE 1u +#define SLOTUSE_REF 2u + struct buffer { size_t posn;