{"thread":{"id":"2475","subject":"[PATCH] Fix for multiple alternates requests in http-fetch","startedAt":"2005-11-12T17:11:32Z","lastAt":"2005-11-12T17:11:32Z","messageCount":1,"participants":["Nick Hengeveld"],"isPatch":true,"patchVersion":1,"patchTotal":null},"messages":[{"id":"11685","messageId":"20051112171131.GE4051@reactrix.com","threadId":"2475","inReplyTo":null,"subject":"[PATCH] Fix for multiple alternates requests in http-fetch","fromName":"Nick Hengeveld","fromEmail":"nickh@reactrix.com","sentAt":"2005-11-12T17:11:32Z","receivedAt":"2005-11-12T17:11:32Z","isPatch":true,"sender":{"key":"nickh@reactrix.com","avatar":null},"body":"Stop additional alternates requests from starting if one is already in\nprogress.  This adds an optional callback which is processed after a slot\nhas finished running.\n\nSigned-off-by: Nick Hengeveld <nickh@reactrix.com>\n\n\n---\n\n http-fetch.c |  178 ++++++++++++++++++++++++++++++++++++++--------------------\n 1 files changed, 117 insertions(+), 61 deletions(-)\n\napplies-to: 9a0f0c748316751fbf593a21f2b16bcdd975095a\n6a34e305a8c718b69604e416e0766ab60e5c1944\ndiff --git a/http-fetch.c b/http-fetch.c\nindex f39e748..99b6cc7 100644\n--- a/http-fetch.c\n+++ b/http-fetch.c\n@@ -25,7 +25,7 @@\n #define PREV_BUF_SIZE 4096\n #define RANGE_HEADER_SIZE 30\n \n-static int got_alternates = 0;\n+static int got_alternates = -1;\n static int active_requests = 0;\n static int data_received;\n \n@@ -87,9 +87,19 @@ struct active_request_slot\n \tint done;\n \tCURLcode curl_result;\n \tlong http_code;\n+\tvoid *callback_data;\n+\tvoid (*callback_func)(void *data);\n \tstruct active_request_slot *next;\n };\n \n+struct alt_request {\n+\tchar *base;\n+\tchar *url;\n+\tstruct buffer *buffer;\n+\tstruct active_request_slot *slot;\n+\tint http_specific;\n+};\n+\n static struct transfer_request *request_queue_head = NULL;\n static struct active_request_slot *active_queue_head = NULL;\n \n@@ -237,7 +247,7 @@ static size_t fwrite_sha1_file(void *ptr\n static void process_curl_messages(void);\n static void process_request_queue(void);\n #endif\n-static int fetch_alternates(char *base);\n+static void fetch_alternates(char *base);\n \n static CURL* get_curl_handle(void)\n {\n@@ -324,6 +334,8 @@ static struct active_request_slot *get_a\n \tslot->in_use = 1;\n \tslot->done = 0;\n \tslot->local = NULL;\n+\tslot->callback_data = NULL;\n+\tslot->callback_func = NULL;\n \tcurl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, pragma_header);\n \tcurl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, no_range_header);\n \tcurl_easy_setopt(slot->curl, CURLOPT_ERRORBUFFER, curl_errorstr);\n@@ -601,6 +613,12 @@ static void process_curl_messages(void)\n \t\t\t} else {\n \t\t\t\tfprintf(stderr, \"Received DONE message for unknown request!\\n\");\n \t\t\t}\n+\n+\t\t\t/* Process slot callback if appropriate */\n+\t\t\tif (slot->callback_func != NULL) {\n+\t\t\t\tslot->callback_func(slot->callback_data);\n+\t\t\t}\n+\n \t\t\tif (request != NULL) {\n \t\t\t\trequest->curl_result = curl_result;\n \t\t\t\trequest->http_code = slot->http_code;\n@@ -766,72 +784,51 @@ static int setup_index(struct alt_base *\n \treturn 0;\n }\n \n-static int fetch_alternates(char *base)\n+static void process_alternates(void *callback_data)\n {\n-\tint ret = 0;\n-\tstruct buffer buffer;\n-\tchar *url;\n-\tchar *data;\n-\tint i = 0;\n-\tint http_specific = 1;\n+\tstruct alt_request *alt_req = (struct alt_request *)callback_data;\n+\tstruct active_request_slot *slot = alt_req->slot;\n \tstruct alt_base *tail = alt;\n+\tchar *base = alt_req->base;\n \tstatic const char null_byte = '\\0';\n+\tchar *data;\n+\tint i = 0;\n \n-\tstruct active_request_slot *slot;\n-\n-\tif (got_alternates)\n-\t\treturn 0;\n-\n-\tdata = xmalloc(4096);\n-\tbuffer.size = 4096;\n-\tbuffer.posn = 0;\n-\tbuffer.buffer = data;\n-\n-\tif (get_verbosely)\n-\t\tfprintf(stderr, \"Getting alternates list for %s\\n\", base);\n-\t\n-\turl = xmalloc(strlen(base) + 31);\n-\tsprintf(url, \"%s/objects/info/http-alternates\", base);\n-\n-\tslot = get_active_slot();\n-\tcurl_easy_setopt(slot->curl, CURLOPT_FILE, &buffer);\n-\tcurl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION,\n-\t\t\t fwrite_buffer_dynamic);\n-\tcurl_easy_setopt(slot->curl, CURLOPT_URL, url);\n-\tif (start_active_slot(slot)) {\n-\t\trun_active_slot(slot);\n-\t\tif (slot->curl_result != CURLE_OK || !buffer.posn) {\n-\t\t\thttp_specific = 0;\n-\n-\t\t\tsprintf(url, \"%s/objects/info/alternates\", base);\n-\n-\t\t\tslot = get_active_slot();\n-\t\t\tcurl_easy_setopt(slot->curl, CURLOPT_FILE, &buffer);\n-\t\t\tcurl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION,\n-\t\t\t\t\t fwrite_buffer_dynamic);\n-\t\t\tcurl_easy_setopt(slot->curl, CURLOPT_URL, url);\n+\tif (alt_req->http_specific) {\n+\t\tif (slot->curl_result != CURLE_OK ||\n+\t\t    !alt_req->buffer->posn) {\n+\n+\t\t\t/* Try reusing the slot to get non-http alternates */\n+\t\t\talt_req->http_specific = 0;\n+\t\t\tsprintf(alt_req->url, \"%s/objects/info/alternates\",\n+\t\t\t\tbase);\n+\t\t\tcurl_easy_setopt(slot->curl, CURLOPT_URL,\n+\t\t\t\t\t alt_req->url);\n+\t\t\tactive_requests++;\n+\t\t\tslot->in_use = 1;\n+\t\t\tslot->done = 0;\n \t\t\tif (start_active_slot(slot)) {\n-\t\t\t\trun_active_slot(slot);\n-\t\t\t\tif (slot->curl_result != CURLE_OK) {\n-\t\t\t\t\tfree(buffer.buffer);\n-\t\t\t\t\tif (slot->http_code == 404)\n-\t\t\t\t\t\tgot_alternates = 1;\n-\t\t\t\t\treturn 0;\n-\t\t\t\t}\n+\t\t\t\treturn;\n+\t\t\t} else {\n+\t\t\t\tgot_alternates = -1;\n+\t\t\t\tslot->done = 1;\n+\t\t\t\treturn;\n \t\t\t}\n \t\t}\n-\t} else {\n-\t\tfree(buffer.buffer);\n-\t\treturn 0;\n+\t} else if (slot->curl_result != CURLE_OK) {\n+\t\tif (slot->http_code != 404) {\n+\t\t\tgot_alternates = -1;\n+\t\t\treturn;\n+\t\t}\n \t}\n \n-\tfwrite_buffer_dynamic(&null_byte, 1, 1, &buffer);\n-\tbuffer.posn--;\n-\tdata = buffer.buffer;\n+\tfwrite_buffer_dynamic(&null_byte, 1, 1, alt_req->buffer);\n+\talt_req->buffer->posn--;\n+\tdata = alt_req->buffer->buffer;\n \n-\twhile (i < buffer.posn) {\n+\twhile (i < alt_req->buffer->posn) {\n \t\tint posn = i;\n-\t\twhile (posn < buffer.posn && data[posn] != '\\n')\n+\t\twhile (posn < alt_req->buffer->posn && data[posn] != '\\n')\n \t\t\tposn++;\n \t\tif (data[posn] == '\\n') {\n \t\t\tint okay = 0;\n@@ -855,7 +852,7 @@ static int fetch_alternates(char *base)\n \t\t\t\t// If the server got removed, give up.\n \t\t\t\tokay = strchr(base, ':') - base + 3 < \n \t\t\t\t\tserverlen;\n-\t\t\t} else if (http_specific) {\n+\t\t\t} else if (alt_req->http_specific) {\n \t\t\t\tchar *colon = strchr(data + i, ':');\n \t\t\t\tchar *slash = strchr(data + i, '/');\n \t\t\t\tif (colon && slash && colon < data + posn &&\n@@ -881,15 +878,74 @@ static int fetch_alternates(char *base)\n \t\t\t\twhile (tail->next != NULL)\n \t\t\t\t\ttail = tail->next;\n \t\t\t\ttail->next = newalt;\n-\t\t\t\tret++;\n \t\t\t}\n \t\t}\n \t\ti = posn + 1;\n \t}\n \n \tgot_alternates = 1;\n-\tfree(buffer.buffer);\n-\treturn ret;\n+}\n+\n+static void fetch_alternates(char *base)\n+{\n+\tstruct buffer buffer;\n+\tchar *url;\n+\tchar *data;\n+\tstruct active_request_slot *slot;\n+\tstatic struct alt_request alt_req;\n+\tint num_transfers;\n+\n+\t/* If another request has already started fetching alternates,\n+\t   wait for them to arrive and return to processing this request's\n+\t   curl message */\n+\twhile (got_alternates == 0) {\n+\t\tcurl_multi_perform(curlm, &num_transfers);\n+\t\tprocess_curl_messages();\n+\t\tprocess_request_queue();\n+\t}\n+\n+\t/* Nothing to do if they've already been fetched */\n+\tif (got_alternates == 1)\n+\t\treturn;\n+\n+\t/* Start the fetch */\n+\tgot_alternates = 0;\n+\n+\tdata = xmalloc(4096);\n+\tbuffer.size = 4096;\n+\tbuffer.posn = 0;\n+\tbuffer.buffer = data;\n+\n+\tif (get_verbosely)\n+\t\tfprintf(stderr, \"Getting alternates list for %s\\n\", base);\n+\t\n+\turl = xmalloc(strlen(base) + 31);\n+\tsprintf(url, \"%s/objects/info/http-alternates\", base);\n+\n+\t/* Use a callback to process the result, since another request\n+\t   may fail and need to have alternates loaded before continuing */\n+\tslot = get_active_slot();\n+\tslot->callback_func = process_alternates;\n+\tslot->callback_data = &alt_req;\n+\n+\tcurl_easy_setopt(slot->curl, CURLOPT_FILE, &buffer);\n+\tcurl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION,\n+\t\t\t fwrite_buffer_dynamic);\n+\tcurl_easy_setopt(slot->curl, CURLOPT_URL, url);\n+\n+\talt_req.base = base;\n+\talt_req.url = url;\n+\talt_req.buffer = &buffer;\n+\talt_req.http_specific = 1;\n+\talt_req.slot = slot;\n+\n+\tif (start_active_slot(slot))\n+\t\trun_active_slot(slot);\n+\telse\n+\t\tgot_alternates = -1;\n+\n+\tfree(data);\n+\tfree(url);\n }\n \n static int fetch_indices(struct alt_base *repo)\n---\n0.99.9.GIT\n"}]}