{"thread":{"id":"19377","subject":"[PATCH 08/11] http*: add fetch_http_pack_index","startedAt":"2009-05-16T03:51:52Z","lastAt":"2009-05-16T03:51:52Z","messageCount":1,"participants":["Tay Ray Chuan"],"isPatch":true,"patchVersion":1,"patchTotal":11},"messages":[{"id":"114069","messageId":"20090516115152.5137d4b3.rctay89@gmail.com","threadId":"19377","inReplyTo":null,"subject":"[PATCH 08/11] http*: add fetch_http_pack_index","fromName":"Tay Ray Chuan","fromEmail":"rctay89@gmail.com","sentAt":"2009-05-16T03:51:52Z","receivedAt":"2009-05-16T03:51:52Z","isPatch":true,"sender":{"key":"rctay89@gmail.com","avatar":"https://avatars.githubusercontent.com/u/61553?v=4"},"body":"The code handling the fetching of pack indices in http-push.c and\nhttp-walker.c, namely, the fetch_index and setup_index methods,\nhave been refactored and moved to http.c.\n\nfetch_index has been moved to http.c, and is not meant to be used\noutside of http.c. It invokes end_url_with_slash with base_url;\napart from that change, the code is identical.\n\nsetup_index has been moved and renamed to fetch_http_pack_index.\n\nSigned-off-by: Tay Ray Chuan <rctay89@gmail.com>\n---\n http-push.c   |  117 +------------------------------------------------------\n http-walker.c |  119 +-------------------------------------------------------\n http.c        |  122 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++\n http.h        |    4 ++\n 4 files changed, 128 insertions(+), 234 deletions(-)\n\ndiff --git a/http-push.c b/http-push.c\nindex 7cbc8cf..ece7a43 100644\n--- a/http-push.c\n+++ b/http-push.c\n@@ -949,121 +949,6 @@ static int add_send_request(struct object *obj, struct remote_lock *lock)\n \treturn 1;\n }\n \n-static int fetch_index(unsigned char *sha1)\n-{\n-\tint ret = 0;\n-\tchar *hex = xstrdup(sha1_to_hex(sha1));\n-\tchar *filename;\n-\tchar *url;\n-\tchar tmpfile[PATH_MAX];\n-\tlong prev_posn = 0;\n-\tchar range[RANGE_HEADER_SIZE];\n-\tstruct curl_slist *range_header = NULL;\n-\n-\tFILE *indexfile;\n-\tstruct active_request_slot *slot;\n-\tstruct slot_results results;\n-\n-\t/* Don't use the index if the pack isn't there */\n-\turl = xmalloc(strlen(repo->url) + 64);\n-\tsprintf(url, \"%sobjects/pack/pack-%s.pack\", repo->url, hex);\n-\tslot = get_active_slot();\n-\tslot->results = &results;\n-\tcurl_easy_setopt(slot->curl, CURLOPT_URL, url);\n-\tcurl_easy_setopt(slot->curl, CURLOPT_NOBODY, 1);\n-\tif (start_active_slot(slot)) {\n-\t\trun_active_slot(slot);\n-\t\tif (results.curl_result != CURLE_OK) {\n-\t\t\tret = error(\"Unable to verify pack %s is available\",\n-\t\t\t\t    hex);\n-\t\t\tgoto cleanup_pack;\n-\t\t}\n-\t} else {\n-\t\tret = error(\"Unable to start request\");\n-\t\tgoto cleanup_pack;\n-\t}\n-\n-\tif (has_pack_index(sha1)) {\n-\t\tret = 0;\n-\t\tgoto cleanup_pack;\n-\t}\n-\n-\tif (push_verbosely)\n-\t\tfprintf(stderr, \"Getting index for pack %s\\n\", hex);\n-\n-\tsprintf(url, \"%sobjects/pack/pack-%s.idx\", repo->url, hex);\n-\n-\tfilename = sha1_pack_index_name(sha1);\n-\tsnprintf(tmpfile, sizeof(tmpfile), \"%s.temp\", filename);\n-\tindexfile = fopen(tmpfile, \"a\");\n-\tif (!indexfile) {\n-\t\tret = error(\"Unable to open local file %s for pack index\",\n-\t\t\t    tmpfile);\n-\t\tgoto cleanup_pack;\n-\t}\n-\n-\tslot = get_active_slot();\n-\tslot->results = &results;\n-\tcurl_easy_setopt(slot->curl, CURLOPT_NOBODY, 0);\n-\tcurl_easy_setopt(slot->curl, CURLOPT_HTTPGET, 1);\n-\tcurl_easy_setopt(slot->curl, CURLOPT_FILE, indexfile);\n-\tcurl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite);\n-\tcurl_easy_setopt(slot->curl, CURLOPT_URL, url);\n-\tcurl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, no_pragma_header);\n-\tslot->local = indexfile;\n-\n-\t/*\n-\t * If there is data present from a previous transfer attempt,\n-\t * resume where it left off\n-\t */\n-\tprev_posn = ftell(indexfile);\n-\tif (prev_posn>0) {\n-\t\tif (push_verbosely)\n-\t\t\tfprintf(stderr,\n-\t\t\t\t\"Resuming fetch of index for pack %s at byte %ld\\n\",\n-\t\t\t\thex, prev_posn);\n-\t\tsprintf(range, \"Range: bytes=%ld-\", prev_posn);\n-\t\trange_header = curl_slist_append(range_header, range);\n-\t\tcurl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, range_header);\n-\t}\n-\n-\tif (start_active_slot(slot)) {\n-\t\trun_active_slot(slot);\n-\t\tif (results.curl_result != CURLE_OK) {\n-\t\t\tret = error(\"Unable to get pack index %s\\n%s\", url,\n-\t\t\t\t    curl_errorstr);\n-\t\t\tgoto cleanup_index;\n-\t\t}\n-\t} else {\n-\t\tret = error(\"Unable to start request\");\n-\t\tgoto cleanup_index;\n-\t}\n-\n-\tret = move_temp_to_file(tmpfile, filename);\n-\n-cleanup_index:\n-\tfclose(indexfile);\n-cleanup_pack:\n-\tfree(url);\n-\tfree(hex);\n-\treturn ret;\n-}\n-\n-static int setup_index(unsigned char *sha1)\n-{\n-\tstruct packed_git *new_pack;\n-\n-\tif (fetch_index(sha1))\n-\t\treturn -1;\n-\n-\tnew_pack = parse_pack_index(sha1);\n-\tif (!new_pack)\n-\t\treturn -1; /* parse_pack_index() already issued error message */\n-\tnew_pack->next = repo->packs;\n-\trepo->packs = new_pack;\n-\treturn 0;\n-}\n-\n static int fetch_indices(void)\n {\n \tunsigned char sha1[20];\n@@ -1113,7 +998,7 @@ static int fetch_indices(void)\n \t\t\t    !prefixcmp(data + i, \" pack-\") &&\n \t\t\t    !prefixcmp(data + i + 46, \".pack\\n\")) {\n \t\t\t\tget_sha1_hex(data + i + 6, sha1);\n-\t\t\t\tsetup_index(sha1);\n+\t\t\t\tfetch_http_pack_index(&repo->packs, sha1, repo->url);\n \t\t\t\ti += 51;\n \t\t\t\tbreak;\n \t\t\t}\ndiff --git a/http-walker.c b/http-walker.c\nindex 024ed58..3a0c021 100644\n--- a/http-walker.c\n+++ b/http-walker.c\n@@ -368,123 +368,6 @@ static void prefetch(struct walker *walker, unsigned char *sha1)\n #endif\n }\n \n-static int fetch_index(struct walker *walker, struct alt_base *repo, unsigned char *sha1)\n-{\n-\tint ret = 0;\n-\tchar *hex = xstrdup(sha1_to_hex(sha1));\n-\tchar *filename;\n-\tchar *url;\n-\tchar tmpfile[PATH_MAX];\n-\tlong prev_posn = 0;\n-\tchar range[RANGE_HEADER_SIZE];\n-\tstruct curl_slist *range_header = NULL;\n-\n-\tFILE *indexfile;\n-\tstruct active_request_slot *slot;\n-\tstruct slot_results results;\n-\n-\t/* Don't use the index if the pack isn't there */\n-\turl = xmalloc(strlen(repo->base) + 64);\n-\tsprintf(url, \"%s/objects/pack/pack-%s.pack\", repo->base, hex);\n-\tslot = get_active_slot();\n-\tslot->results = &results;\n-\tcurl_easy_setopt(slot->curl, CURLOPT_URL, url);\n-\tcurl_easy_setopt(slot->curl, CURLOPT_NOBODY, 1);\n-\tif (start_active_slot(slot)) {\n-\t\trun_active_slot(slot);\n-\t\tif (results.curl_result != CURLE_OK) {\n-\t\t\tret = error(\"Unable to verify pack %s is available\",\n-\t\t\t\t     hex);\n-\t\t\tgoto cleanup_pack;\n-\t\t}\n-\t} else {\n-\t\tret = error(\"Unable to start request\");\n-\t\tgoto cleanup_pack;\n-\t}\n-\n-\tif (has_pack_index(sha1)) {\n-\t\tret = 0;\n-\t\tgoto cleanup_pack;\n-\t}\n-\n-\tif (walker->get_verbosely)\n-\t\tfprintf(stderr, \"Getting index for pack %s\\n\", hex);\n-\n-\tsprintf(url, \"%s/objects/pack/pack-%s.idx\", repo->base, hex);\n-\n-\tfilename = sha1_pack_index_name(sha1);\n-\tsnprintf(tmpfile, sizeof(tmpfile), \"%s.temp\", filename);\n-\tindexfile = fopen(tmpfile, \"a\");\n-\tif (!indexfile) {\n-\t\tret = error(\"Unable to open local file %s for pack index\",\n-\t\t\t    tmpfile);\n-\t\tgoto cleanup_pack;\n-\t}\n-\n-\tslot = get_active_slot();\n-\tslot->results = &results;\n-\tcurl_easy_setopt(slot->curl, CURLOPT_NOBODY, 0);\n-\tcurl_easy_setopt(slot->curl, CURLOPT_HTTPGET, 1);\n-\tcurl_easy_setopt(slot->curl, CURLOPT_FILE, indexfile);\n-\tcurl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite);\n-\tcurl_easy_setopt(slot->curl, CURLOPT_URL, url);\n-\tcurl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, no_pragma_header);\n-\tslot->local = indexfile;\n-\n-\t/*\n-\t * If there is data present from a previous transfer attempt,\n-\t * resume where it left off\n-\t */\n-\tprev_posn = ftell(indexfile);\n-\tif (prev_posn>0) {\n-\t\tif (walker->get_verbosely)\n-\t\t\tfprintf(stderr,\n-\t\t\t\t\"Resuming fetch of index for pack %s at byte %ld\\n\",\n-\t\t\t\thex, prev_posn);\n-\t\tsprintf(range, \"Range: bytes=%ld-\", prev_posn);\n-\t\trange_header = curl_slist_append(range_header, range);\n-\t\tcurl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, range_header);\n-\t}\n-\n-\tif (start_active_slot(slot)) {\n-\t\trun_active_slot(slot);\n-\t\tif (results.curl_result != CURLE_OK) {\n-\t\t\tret = error(\"Unable to get pack index %s\\n%s\", url,\n-\t\t\t\t    curl_errorstr);\n-\t\t\tgoto cleanup_index;\n-\t\t}\n-\t} else {\n-\t\tret = error(\"Unable to start request\");\n-\t\tgoto cleanup_index;\n-\t}\n-\n-\tret = move_temp_to_file(tmpfile, filename);\n-\n-cleanup_index:\n-\tfclose(indexfile);\n-cleanup_pack:\n-\tfree(url);\n-\tfree(hex);\n-\treturn ret;\n-}\n-\n-static int setup_index(struct walker *walker, struct alt_base *repo, unsigned char *sha1)\n-{\n-\tstruct packed_git *new_pack;\n-\tif (has_pack_file(sha1))\n-\t\treturn 0; /* don't list this as something we can get */\n-\n-\tif (fetch_index(walker, repo, sha1))\n-\t\treturn -1;\n-\n-\tnew_pack = parse_pack_index(sha1);\n-\tif (!new_pack)\n-\t\treturn -1; /* parse_pack_index() already issued error message */\n-\tnew_pack->next = repo->packs;\n-\trepo->packs = new_pack;\n-\treturn 0;\n-}\n-\n static void process_alternates_response(void *callback_data)\n {\n \tstruct alternates_request *alt_req =\n@@ -735,7 +618,7 @@ static int fetch_indices(struct walker *walker, struct alt_base *repo)\n \t\t\t    !prefixcmp(data + i, \" pack-\") &&\n \t\t\t    !prefixcmp(data + i + 46, \".pack\\n\")) {\n \t\t\t\tget_sha1_hex(data + i + 6, sha1);\n-\t\t\t\tsetup_index(walker, repo, sha1);\n+\t\t\t\tfetch_http_pack_index(&repo->packs, sha1, repo->base);\n \t\t\t\ti += 51;\n \t\t\t\tbreak;\n \t\t\t}\ndiff --git a/http.c b/http.c\nindex 75fce9e..91c6bf6 100644\n--- a/http.c\n+++ b/http.c\n@@ -703,3 +703,125 @@ int http_fetch_ref(const char *base, struct ref *ref)\n \tfree(url);\n \treturn ret;\n }\n+\n+/* Helpers for fetching packs */\n+static int fetch_index(unsigned char *sha1, const char *base_url)\n+{\n+\tint ret = 0;\n+\tchar *hex = xstrdup(sha1_to_hex(sha1));\n+\tchar *filename;\n+\tchar *url;\n+\tchar tmpfile[PATH_MAX];\n+\tlong prev_posn = 0;\n+\tchar range[RANGE_HEADER_SIZE];\n+\tstruct strbuf buf = STRBUF_INIT;\n+\tstruct curl_slist *range_header = NULL;\n+\n+\tFILE *indexfile;\n+\tstruct active_request_slot *slot;\n+\tstruct slot_results results;\n+\n+\t/* Don't use the index if the pack isn't there */\n+\tend_url_with_slash(&buf, base_url);\n+\tstrbuf_addf(&buf, \"objects/pack/pack-%s.pack\", hex);\n+\turl = strbuf_detach(&buf, 0);\n+\n+\tslot = get_active_slot();\n+\tslot->results = &results;\n+\tcurl_easy_setopt(slot->curl, CURLOPT_URL, url);\n+\tcurl_easy_setopt(slot->curl, CURLOPT_NOBODY, 1);\n+\tif (start_active_slot(slot)) {\n+\t\trun_active_slot(slot);\n+\t\tif (results.curl_result != CURLE_OK) {\n+\t\t\tret = error(\"Unable to verify pack %s is available\",\n+\t\t\t\t    hex);\n+\t\t\tgoto cleanup_pack;\n+\t\t}\n+\t} else {\n+\t\tret = error(\"Unable to start request\");\n+\t\tgoto cleanup_pack;\n+\t}\n+\n+\tif (has_pack_index(sha1)) {\n+\t\tret = 0;\n+\t\tgoto cleanup_pack;\n+\t}\n+\n+\tif (http_is_verbose)\n+\t\tfprintf(stderr, \"Getting index for pack %s\\n\", hex);\n+\n+\tend_url_with_slash(&buf, base_url);\n+\tstrbuf_addf(&buf, \"objects/pack/pack-%s.idx\", hex);\n+\turl = strbuf_detach(&buf, NULL);\n+\n+\tfilename = sha1_pack_index_name(sha1);\n+\tsnprintf(tmpfile, sizeof(tmpfile), \"%s.temp\", filename);\n+\tindexfile = fopen(tmpfile, \"a\");\n+\tif (!indexfile) {\n+\t\tret = error(\"Unable to open local file %s for pack index\",\n+\t\t\t    tmpfile);\n+\t\tgoto cleanup_pack;\n+\t}\n+\n+\tslot = get_active_slot();\n+\tslot->results = &results;\n+\tcurl_easy_setopt(slot->curl, CURLOPT_NOBODY, 0);\n+\tcurl_easy_setopt(slot->curl, CURLOPT_HTTPGET, 1);\n+\tcurl_easy_setopt(slot->curl, CURLOPT_FILE, indexfile);\n+\tcurl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite);\n+\tcurl_easy_setopt(slot->curl, CURLOPT_URL, url);\n+\tcurl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, no_pragma_header);\n+\tslot->local = indexfile;\n+\n+\t/*\n+\t * If there is data present from a previous transfer attempt,\n+\t * resume where it left off\n+\t */\n+\tprev_posn = ftell(indexfile);\n+\tif (prev_posn>0) {\n+\t\tif (http_is_verbose)\n+\t\t\tfprintf(stderr,\n+\t\t\t\t\"Resuming fetch of index for pack %s at byte %ld\\n\",\n+\t\t\t\thex, prev_posn);\n+\t\tsprintf(range, \"Range: bytes=%ld-\", prev_posn);\n+\t\trange_header = curl_slist_append(range_header, range);\n+\t\tcurl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, range_header);\n+\t}\n+\n+\tif (start_active_slot(slot)) {\n+\t\trun_active_slot(slot);\n+\t\tif (results.curl_result != CURLE_OK) {\n+\t\t\tret = error(\"Unable to get pack index %s\\n%s\",\n+\t\t\t\t    url, curl_errorstr);\n+\t\t\tgoto cleanup_index;\n+\t\t}\n+\t} else {\n+\t\tret = error(\"Unable to start request\");\n+\t\tgoto cleanup_index;\n+\t}\n+\n+\tret = move_temp_to_file(tmpfile, filename);\n+\n+cleanup_index:\n+\tfclose(indexfile);\n+cleanup_pack:\n+\tfree(hex);\n+\tfree(url);\n+\treturn ret;\n+}\n+\n+int fetch_http_pack_index(struct packed_git **packs_head, unsigned char *sha1,\n+\tconst char *base_url)\n+{\n+\tstruct packed_git *new_pack;\n+\n+\tif (fetch_index(sha1, base_url))\n+\t\treturn -1;\n+\n+\tnew_pack = parse_pack_index(sha1);\n+\tif (!new_pack)\n+\t\treturn -1; /* parse_pack_index() already issued error message */\n+\tnew_pack->next = *packs_head;\n+\t*packs_head = new_pack;\n+\treturn 0;\n+}\ndiff --git a/http.h b/http.h\nindex 1ef7dc1..cd2920b 100644\n--- a/http.h\n+++ b/http.h\n@@ -116,4 +116,8 @@ static inline int missing__target(int code, int result)\n \n extern int http_fetch_ref(const char *base, struct ref *ref);\n \n+/* Helpers for fetching packs */\n+extern int fetch_http_pack_index(struct packed_git **packs_head,\n+\tunsigned char *sha1, const char *base_url);\n+\n #endif /* HTTP_H */\n-- \n1.6.3.rc0\n"}]}