{"thread":{"id":"8995","subject":"[PATCH 3/4] Add fetch methods to transport library.","startedAt":"2007-07-12T01:37:08Z","lastAt":"2007-07-12T01:37:08Z","messageCount":1,"participants":["Daniel Barkalow"],"isPatch":true,"patchVersion":1,"patchTotal":4},"messages":[{"id":"47103","messageId":"Pine.LNX.4.64.0707112107520.6977@iabervon.org","threadId":"8995","inReplyTo":null,"subject":"[PATCH 3/4] Add fetch methods to transport library.","fromName":"Daniel Barkalow","fromEmail":"barkalow@iabervon.org","sentAt":"2007-07-12T01:37:08Z","receivedAt":"2007-07-12T01:37:08Z","isPatch":true,"sender":{"key":"barkalow@iabervon.org","avatar":"https://avatars.githubusercontent.com/u/55364219?v=4"},"body":"These don't include bundle or rsync. It would be great if people would \nwrite at least bundle; rsync may well be deprecatable at this point (since \npeople are more likely to have http than anonymous rsync and more likely \nto have ssh+git than non-anonymous rsync).\n\nIn any case, it identifies such URLs and fails cleanly on them.\n\nSigned-off-by: Daniel Barkalow <barkalow@iabervon.org>\n---\n transport.c |  286 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++-\n transport.h |   23 +++++-\n 2 files changed, 305 insertions(+), 4 deletions(-)\n\ndiff --git a/transport.c b/transport.c\nindex ce03133..487d315 100644\n--- a/transport.c\n+++ b/transport.c\n@@ -1,6 +1,39 @@\n #include \"cache.h\"\n #include \"transport.h\"\n #include \"run-command.h\"\n+#include \"http.h\"\n+#include \"pkt-line.h\"\n+#include \"fetch-pack.h\"\n+#include \"walker.h\"\n+\n+/* Generic functions for using commit walkers */\n+\n+static int fetch_objs_via_walker(const struct transport *transport,\n+\t\t\t\t int nr_objs, char **objs)\n+{\n+\tchar *dest = xstrdup(transport->url);\n+\tstruct walker *walker = transport->data;\n+\n+\twalker->get_all = 1;\n+\twalker->get_tree = 1;\n+\twalker->get_history = 1;\n+\twalker->get_verbosely = transport->verbose;\n+\twalker->get_recover = 0;\n+\n+\tif (walker_fetch(walker, nr_objs, objs, NULL, dest))\n+\t\tdie(\"Fetch failed.\");\n+\n+\tfree(dest);\n+\treturn 0;\n+}\n+\n+static int disconnect_walker(struct transport *transport)\n+{\n+\tstruct walker *walker = transport->data;\n+\tif (walker)\n+\t\twalker_free(walker);\n+\treturn 0;\n+}\n \n static const struct transport_ops rsync_transport = {\n };\n@@ -37,9 +70,119 @@ static int curl_transport_push(struct transport *transport, int refspec_nr, cons\n \treturn !!err;\n }\n \n+#ifndef NO_CURL\n+static int missing__target(int code, int result)\n+{\n+\treturn\t/* file:// URL -- do we ever use one??? */\n+\t\t(result == CURLE_FILE_COULDNT_READ_FILE) ||\n+\t\t/* http:// and https:// URL */\n+\t\t(code == 404 && result == CURLE_HTTP_RETURNED_ERROR) ||\n+\t\t/* ftp:// URL */\n+\t\t(code == 550 && result == CURLE_FTP_COULDNT_RETR_FILE)\n+\t\t;\n+}\n+\n+#define missing_target(a) missing__target((a)->http_code, (a)->curl_result)\n+\n+static struct ref *get_refs_via_curl(const struct transport *transport)\n+{\n+\tstruct buffer buffer;\n+\tchar *data, *start, *mid;\n+\tchar *ref_name;\n+\tchar *refs_url;\n+\tint i = 0;\n+\n+\tstruct active_request_slot *slot;\n+\tstruct slot_results results;\n+\n+\tstruct ref *refs = NULL;\n+\tstruct ref *ref = NULL;\n+\tstruct ref *last_ref = NULL;\n+\n+\tdata = xmalloc(4096);\n+\tbuffer.size = 4096;\n+\tbuffer.posn = 0;\n+\tbuffer.buffer = data;\n+\n+\trefs_url = xmalloc(strlen(transport->url) + 11);\n+\tsprintf(refs_url, \"%s/info/refs\", transport->url);\n+\n+\thttp_init();\n+\n+\tslot = get_active_slot();\n+\tslot->results = &results;\n+\tcurl_easy_setopt(slot->curl, CURLOPT_FILE, &buffer);\n+\tcurl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite_buffer);\n+\tcurl_easy_setopt(slot->curl, CURLOPT_URL, refs_url);\n+\tcurl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, NULL);\n+\tif (start_active_slot(slot)) {\n+\t\trun_active_slot(slot);\n+\t\tif (results.curl_result != CURLE_OK) {\n+\t\t\tif (missing_target(&results)) {\n+\t\t\t\tfree(buffer.buffer);\n+\t\t\t\treturn NULL;\n+\t\t\t} else {\n+\t\t\t\tfree(buffer.buffer);\n+\t\t\t\terror(\"%s\", curl_errorstr);\n+\t\t\t\treturn NULL;\n+\t\t\t}\n+\t\t}\n+\t} else {\n+\t\tfree(buffer.buffer);\n+\t\terror(\"Unable to start request\");\n+\t\treturn NULL;\n+\t}\n+\n+\thttp_cleanup();\n+\n+\tdata = buffer.buffer;\n+\tstart = NULL;\n+\tmid = data;\n+\twhile (i < buffer.posn) {\n+\t\tif (!start)\n+\t\t\tstart = &data[i];\n+\t\tif (data[i] == '\\t')\n+\t\t\tmid = &data[i];\n+\t\tif (data[i] == '\\n') {\n+\t\t\tdata[i] = 0;\n+\t\t\tref_name = mid + 1;\n+\t\t\tref = xmalloc(sizeof(struct ref) +\n+\t\t\t\t      strlen(ref_name) + 1);\n+\t\t\tmemset(ref, 0, sizeof(struct ref));\n+\t\t\tstrcpy(ref->name, ref_name);\n+\t\t\tget_sha1_hex(start, ref->old_sha1);\n+\t\t\tif (!refs)\n+\t\t\t\trefs = ref;\n+\t\t\tif (last_ref)\n+\t\t\t\tlast_ref->next = ref;\n+\t\t\tlast_ref = ref;\n+\t\t\tstart = NULL;\n+\t\t}\n+\t\ti++;\n+\t}\n+\n+\tfree(buffer.buffer);\n+\n+\treturn refs;\n+}\n+\n+#else\n+\n+static struct ref *get_refs_via_curl(const struct transport *transport)\n+{\n+\tdie(\"Cannot fetch from '%s' without curl ...\", transport->url);\n+\treturn NULL;\n+}\n+\n+#endif\n+\n static const struct transport_ops curl_transport = {\n \t/* set_option */\tNULL,\n-\t/* push */\t\tcurl_transport_push\n+\t/* get_refs_list */\tget_refs_via_curl,\n+\t/* fetch_refs */\tNULL,\n+\t/* fetch_objs */\tfetch_objs_via_walker,\n+\t/* push */\t\tcurl_transport_push,\n+\t/* disconnect */\tdisconnect_walker\n };\n \n static const struct transport_ops bundle_transport = {\n@@ -47,7 +190,13 @@ static const struct transport_ops bundle_transport = {\n \n struct git_transport_data {\n \tunsigned thin : 1;\n+\tunsigned keep : 1;\n \n+\tint unpacklimit;\n+\n+\tint depth;\n+\n+\tconst char *uploadpack;\n \tconst char *receivepack;\n };\n \n@@ -55,16 +204,86 @@ static int set_git_option(struct transport *connection,\n \t\t\t  const char *name, const char *value)\n {\n \tstruct git_transport_data *data = connection->data;\n-\tif (!strcmp(name, TRANS_OPT_RECEIVEPACK)) {\n+\tif (!strcmp(name, TRANS_OPT_UPLOADPACK)) {\n+\t\tdata->uploadpack = value;\n+\t\treturn 0;\n+\t} else if (!strcmp(name, TRANS_OPT_RECEIVEPACK)) {\n \t\tdata->receivepack = value;\n \t\treturn 0;\n \t} else if (!strcmp(name, TRANS_OPT_THIN)) {\n \t\tdata->thin = !!value;\n \t\treturn 0;\n+\t} else if (!strcmp(name, TRANS_OPT_KEEP)) {\n+\t\tdata->keep = !!value;\n+\t\treturn 0;\n+\t} else if (!strcmp(name, TRANS_OPT_UNPACKLIMIT)) {\n+\t\tdata->unpacklimit = atoi(value);\n+\t\treturn 0;\n+\t} else if (!strcmp(name, TRANS_OPT_DEPTH)) {\n+\t\tif (!value)\n+\t\t\tdata->depth = 0;\n+\t\telse\n+\t\t\tdata->depth = atoi(value);\n+\t\treturn 0;\n \t}\n \treturn 1;\n }\n \n+static struct ref *get_refs_via_connect(const struct transport *transport)\n+{\n+\tstruct git_transport_data *data = transport->data;\n+\tstruct ref *refs;\n+\tint fd[2];\n+\tpid_t pid;\n+\tchar *dest = xstrdup(transport->url);\n+\n+\tpid = git_connect(fd, dest, data->uploadpack, 0);\n+\n+\tif (pid < 0)\n+\t\tdie(\"Failed to connect to \\\"%s\\\"\", transport->url);\n+\n+\tget_remote_heads(fd[0], &refs, 0, NULL, 0);\n+\tpacket_flush(fd[1]);\n+\n+\tfinish_connect(pid);\n+\n+\tfree(dest);\n+\n+\treturn refs;\n+}\n+\n+static int fetch_refs_via_pack(const struct transport *transport,\n+\t\t\t       int nr_heads, char **heads)\n+{\n+\tstruct git_transport_data *data = transport->data;\n+\tstruct ref *refs;\n+\tchar *dest = xstrdup(transport->url);\n+\tstruct fetch_pack_args args;\n+\n+\targs.uploadpack = data->uploadpack;\n+\targs.quiet = 0;\n+\targs.keep_pack = data->keep;\n+\targs.unpacklimit = data->unpacklimit;\n+\targs.use_thin_pack = data->thin;\n+\targs.fetch_all = 0;\n+\targs.verbose = transport->verbose;\n+\targs.depth = data->depth;\n+\targs.no_progress = 0;\n+\n+\tsetup_fetch_pack(&args);\n+\n+\trefs = fetch_pack(dest, nr_heads, heads);\n+\n+\t// ???? check that refs got everything?\n+\n+\t/* free the memory used for the refs list ... */\n+\n+\tfree_refs(refs);\n+\n+\tfree(dest);\n+\treturn 0;\n+}\n+\n static int git_transport_push(struct transport *transport, int refspec_nr, const char **refspec, int flags) {\n \tstruct git_transport_data *data = transport->data;\n \tconst char **argv;\n@@ -111,6 +330,9 @@ static int git_transport_push(struct transport *transport, int refspec_nr, const\n \n static const struct transport_ops git_transport = {\n \t/* set_option */\tset_git_option,\n+\t/* get_refs_list */\tget_refs_via_connect,\n+\t/* fetch_refs */\tfetch_refs_via_pack,\n+\t/* fetch_objs */\tNULL,\n \t/* push */\t\tgit_transport_push\n };\n \n@@ -141,7 +363,10 @@ struct transport *transport_get(struct remote *remote, const char *url,\n \t\t   !prefixcmp(url, \"ftp://\")) {\n \t\tret = xmalloc(sizeof(*ret));\n \t\tret->ops = &curl_transport;\n-\t\tret->data = NULL;\n+\t\tif (fetch)\n+\t\t\tret->data = get_http_walker(url);\n+\t\telse\n+\t\t\tret->data = NULL;\n \t} else if (is_local(url) && is_file(url)) {\n \t\tret = xmalloc(sizeof(*ret));\n \t\tret->data = NULL;\n@@ -151,14 +376,19 @@ struct transport *transport_get(struct remote *remote, const char *url,\n \t\tret = xcalloc(1, sizeof(*ret));\n \t\tret->data = data;\n \t\tdata->thin = 1;\n+\t\tdata->uploadpack = \"git-upload-pack\";\n+\t\tif (remote->uploadpack)\n+\t\t\tdata->uploadpack = remote->uploadpack;\n \t\tdata->receivepack = \"git-receive-pack\";\n \t\tif (remote->receivepack)\n \t\t\tdata->receivepack = remote->receivepack;\n+\t\tdata->unpacklimit = -1;\n \t\tret->ops = &git_transport;\n \t}\n \tif (ret) {\n \t\tret->remote = remote;\n \t\tret->url = url;\n+\t\tret->remote_refs = NULL;\n \t\tret->fetch = !!fetch;\n \t}\n \treturn ret;\n@@ -187,6 +417,54 @@ int transport_push(struct transport *transport,\n \treturn transport->ops->push(transport, refspec_nr, refspec, flags);\n }\n \n+struct ref *transport_get_remote_refs(struct transport *transport)\n+{\n+\tif (!transport->remote_refs)\n+\t\ttransport->remote_refs =\n+\t\t\ttransport->ops->get_refs_list(transport);\n+\treturn transport->remote_refs;\n+}\n+\n+#define PACK_HEADS_CHUNK_COUNT 256\n+\n+int transport_fetch_refs(struct transport *transport, struct ref *refs)\n+{\n+\tint i;\n+\tint nr_heads = 0;\n+\tchar **heads = xmalloc(PACK_HEADS_CHUNK_COUNT * sizeof(char *));\n+\tstruct ref *rm;\n+\tint use_objs = !transport->ops->fetch_refs;\n+\n+\tfor (rm = refs; rm; rm = rm->next) {\n+\t\tif (rm->peer_ref &&\n+\t\t    !hashcmp(rm->peer_ref->old_sha1, rm->old_sha1))\n+\t\t\tcontinue;\n+\t\tif (use_objs) {\n+\t\t\theads[nr_heads++] = xstrdup(sha1_to_hex(rm->old_sha1));\n+\t\t} else {\n+\t\t\theads[nr_heads++] = xstrdup(rm->name);\n+\t\t}\n+\t\tif (nr_heads % PACK_HEADS_CHUNK_COUNT == 0)\n+\t\t\theads = xrealloc(heads,\n+\t\t\t\t\t (nr_heads + PACK_HEADS_CHUNK_COUNT) *\n+\t\t\t\t\t sizeof(char *));\n+\t}\n+\n+\tif (use_objs) {\n+\t\tif (transport->ops->fetch_objs(transport, nr_heads, heads))\n+\t\t\treturn -1;\n+\t} else {\n+\t\tif (transport->ops->fetch_refs(transport, nr_heads, heads))\n+\t\t\treturn -1;\n+\t}\n+\n+\t/* free the memory used for the heads list ... */\n+\tfor (i = 0; i < nr_heads; i++)\n+\t\tfree(heads[i]);\n+\tfree(heads);\n+\treturn 0;\n+}\n+\n int transport_disconnect(struct transport *transport)\n {\n \tint ret = 0;\n@@ -195,3 +473,5 @@ int transport_disconnect(struct transport *transport)\n \tfree(transport);\n \treturn ret;\n }\n+\n+\ndiff --git a/transport.h b/transport.h\nindex 5c2eb95..947db66 100644\n--- a/transport.h\n+++ b/transport.h\n@@ -29,6 +29,11 @@ struct transport_ops {\n \tint (*set_option)(struct transport *connection, const char *name,\n \t\t\t  const char *value);\n \n+\tstruct ref *(*get_refs_list)(const struct transport *transport);\n+\tint (*fetch_refs)(const struct transport *transport,\n+\t\t\t  int nr_refs, char **refs);\n+\tint (*fetch_objs)(const struct transport *transport,\n+\t\t\t  int nr_objs, char **objs);\n \tint (*push)(struct transport *connection, int refspec_nr, const char **refspec, int flags);\n \n \tint (*disconnect)(struct transport *connection);\n@@ -40,12 +45,24 @@ struct transport *transport_get(struct remote *remote, const char *url,\n \n /* Transport options which apply to git:// and scp-style URLs */\n \n+/* The program to use on the remote side to send a pack */\n+#define TRANS_OPT_UPLOADPACK \"uploadpack\"\n+\n /* The program to use on the remote side to receive a pack */\n #define TRANS_OPT_RECEIVEPACK \"receivepack\"\n \n /* Transfer the data as a thin pack if not null */\n #define TRANS_OPT_THIN \"thin\"\n \n+/* Keep the pack that was transferred if not null */\n+#define TRANS_OPT_KEEP \"keep\"\n+\n+/* Unpack the objects if fewer than this number of objects are fetched */\n+#define TRANS_OPT_UNPACKLIMIT \"unpacklimit\"\n+\n+/* Limit the depth of the fetch if not null */\n+#define TRANS_OPT_DEPTH \"depth\"\n+\n /**\n  * Returns 0 if the option was used, non-zero otherwise. Prints a\n  * message to stderr if the option is not used.\n@@ -53,9 +70,13 @@ struct transport *transport_get(struct remote *remote, const char *url,\n int transport_set_option(struct transport *transport, const char *name,\n \t\t\t const char *value);\n \n-int transport_push(struct transport *connection,\n+int transport_push(struct transport *connection, \n \t\t   int refspec_nr, const char **refspec, int flags);\n \n+struct ref *transport_get_remote_refs(struct transport *transport);\n+\n+int transport_fetch_refs(struct transport *transport, struct ref *refs);\n+\n int transport_disconnect(struct transport *transport);\n \n #endif\n-- \n1.5.2.2.1600.ga4e5\n"}]}