{"thread":{"id":"10096","subject":"[PATCH v4 2/2] fetch/push: readd rsync support","startedAt":"2007-09-30T23:59:39Z","lastAt":"2007-09-30T23:59:39Z","messageCount":1,"participants":["Johannes Schindelin"],"isPatch":true,"patchVersion":4,"patchTotal":2},"messages":[{"id":"54448","messageId":"Pine.LNX.4.64.0710010058340.28395@racer.site","threadId":"10096","inReplyTo":null,"subject":"[PATCH v4 2/2] fetch/push: readd rsync support","fromName":"Johannes Schindelin","fromEmail":"johannes.schindelin@gmx.de","sentAt":"2007-09-30T23:59:39Z","receivedAt":"2007-09-30T23:59:39Z","isPatch":true,"sender":{"key":"johannes.schindelin@gmx.de","avatar":"https://avatars.githubusercontent.com/u/127790?v=4"},"body":"\nWe lost rsync support when transitioning from shell to C.  Support it \nagain (even if the transport is technically deprecated, some people just \ndo not have any chance to use anything else).\n\nAlso, add a test to t5510.  Since rsync transport is not configured by \ndefault on most machines, and especially not such that you can write to \nrsync://127.0.0.1$(pwd)/, it is disabled by default; you can enable it by \nsetting the environment variable TEST_RSYNC.\n\nSigned-off-by: Johannes Schindelin <johannes.schindelin@gmx.de>\n---\n\tSeems that mkdtemp() is BSD, not POSIX, and even then Windows\n\tlacks it ;-)\n\n\tSo I'll provide a NEEDS_MKDTEMP patch tomorrow, I hope...\n\n t/t5510-fetch.sh |   35 ++++++\n transport.c      |  328 +++++++++++++++++++++++++++++++++++++++++++++++++++++-\n 2 files changed, 362 insertions(+), 1 deletions(-)\n\ndiff --git a/t/t5510-fetch.sh b/t/t5510-fetch.sh\nindex 439430f..73a4e3c 100755\n--- a/t/t5510-fetch.sh\n+++ b/t/t5510-fetch.sh\n@@ -153,4 +153,39 @@ test_expect_success 'bundle should be able to create a full history' '\n \n '\n \n+test \"$TEST_RSYNC\" && {\n+test_expect_success 'fetch via rsync' '\n+\tgit pack-refs &&\n+\tmkdir rsynced &&\n+\tcd rsynced &&\n+\tgit init &&\n+\tgit fetch rsync://127.0.0.1$(pwd)/../.git master:refs/heads/master &&\n+\tgit gc --prune &&\n+\ttest $(git rev-parse master) = $(cd .. && git rev-parse master) &&\n+\tgit fsck --full\n+'\n+\n+test_expect_success 'push via rsync' '\n+\tmkdir ../rsynced2 &&\n+\t(cd ../rsynced2 &&\n+\t git init) &&\n+\tgit push rsync://127.0.0.1$(pwd)/../rsynced2/.git master &&\n+\tcd ../rsynced2 &&\n+\tgit gc --prune &&\n+\ttest $(git rev-parse master) = $(cd .. && git rev-parse master) &&\n+\tgit fsck --full\n+'\n+\n+test_expect_success 'push via rsync' '\n+\tcd .. &&\n+\tmkdir rsynced3 &&\n+\t(cd rsynced3 &&\n+\t git init) &&\n+\tgit push --all rsync://127.0.0.1$(pwd)/rsynced3/.git &&\n+\tcd rsynced3 &&\n+\ttest $(git rev-parse master) = $(cd .. && git rev-parse master) &&\n+\tgit fsck --full\n+'\n+}\n+\n test_done\ndiff --git a/transport.c b/transport.c\nindex 3475cca..7266fd3 100644\n--- a/transport.c\n+++ b/transport.c\n@@ -6,6 +6,330 @@\n #include \"fetch-pack.h\"\n #include \"walker.h\"\n #include \"bundle.h\"\n+#include \"dir.h\"\n+#include \"refs.h\"\n+\n+/* rsync support */\n+\n+/*\n+ * We copy packed-refs and refs/ into a temporary file, then read the\n+ * loose refs recursively (sorting whenever possible), and then inserting\n+ * those packed refs that are not yet in the list (not validating, but\n+ * assuming that the file is sorted).\n+ *\n+ * Appears refactoring this from refs.c is too cumbersome.\n+ */\n+\n+static int str_cmp(const void *a, const void *b)\n+{\n+\tconst char *s1 = a;\n+\tconst char *s2 = b;\n+\n+\treturn strcmp(s1, s2);\n+}\n+\n+/* path->buf + name_offset is expected to point to \"refs/\" */\n+\n+static int read_loose_refs(struct strbuf *path, int name_offset,\n+\t\tstruct ref **tail)\n+{\n+\tDIR *dir = opendir(path->buf);\n+\tstruct dirent *de;\n+\tstruct {\n+\t\tchar **entries;\n+\t\tint nr, alloc;\n+\t} list;\n+\tint i, pathlen;\n+\n+\tif (!dir)\n+\t\treturn -1;\n+\n+\tmemset (&list, 0, sizeof(list));\n+\n+\twhile ((de = readdir(dir))) {\n+\t\tif (de->d_name[0] == '.' && (de->d_name[1] == '\\0' ||\n+\t\t\t\t(de->d_name[1] == '.' &&\n+\t\t\t\t de->d_name[2] == '\\0')))\n+\t\t\tcontinue;\n+\t\tALLOC_GROW(list.entries, list.nr + 1, list.alloc);\n+\t\tlist.entries[list.nr++] = xstrdup(de->d_name);\n+\t}\n+\tclosedir(dir);\n+\n+\t/* sort the list */\n+\n+\tqsort(list.entries, list.nr, sizeof(char *), str_cmp);\n+\n+\tpathlen = path->len;\n+\tstrbuf_addch(path, '/');\n+\n+\tfor (i = 0; i < list.nr; i++, strbuf_setlen(path, pathlen + 1)) {\n+\t\tstrbuf_addstr(path, list.entries[i]);\n+\t\tif (read_loose_refs(path, name_offset, tail)) {\n+\t\t\tint fd = open(path->buf, O_RDONLY);\n+\t\t\tchar buffer[40];\n+\t\t\tstruct ref *next;\n+\n+\t\t\tif (fd < 0)\n+\t\t\t\tcontinue;\n+\t\t\tnext = alloc_ref(path->len - name_offset + 1);\n+\t\t\tif (read_in_full(fd, buffer, 40) != 40 ||\n+\t\t\t\t\tget_sha1_hex(buffer, next->old_sha1)) {\n+\t\t\t\tclose(fd);\n+\t\t\t\tfree(next);\n+\t\t\t\tcontinue;\n+\t\t\t}\n+\t\t\tclose(fd);\n+\t\t\tstrcpy(next->name, path->buf + name_offset);\n+\t\t\t(*tail)->next = next;\n+\t\t\t*tail = next;\n+\t\t}\n+\t}\n+\tstrbuf_setlen(path, pathlen);\n+\n+\tfor (i = 0; i < list.nr; i++)\n+\t\tfree(list.entries[i]);\n+\tfree(list.entries);\n+\n+\treturn 0;\n+}\n+\n+/* insert the packed refs for which no loose refs were found */\n+\n+static void insert_packed_refs(const char *packed_refs, struct ref **list)\n+{\n+\tFILE *f = fopen(packed_refs, \"r\");\n+\tstatic char buffer[PATH_MAX];\n+\n+\tif (!f)\n+\t\treturn;\n+\n+\tfor (;;) {\n+\t\tint cmp, len;\n+\n+\t\tif (!fgets(buffer, sizeof(buffer), f)) {\n+\t\t\tfclose(f);\n+\t\t\treturn;\n+\t\t}\n+\n+\t\tif (hexval(buffer[0]) > 0xf)\n+\t\t\tcontinue;\n+\t\tlen = strlen(buffer);\n+\t\tif (buffer[len - 1] == '\\n')\n+\t\t\tbuffer[--len] = '\\0';\n+\t\tif (len < 41)\n+\t\t\tcontinue;\n+\t\twhile ((*list)->next &&\n+\t\t\t\t(cmp = strcmp(buffer + 41,\n+\t\t\t\t      (*list)->next->name)) > 0)\n+\t\t\tlist = &(*list)->next;\n+\t\tif (!(*list)->next || cmp < 0) {\n+\t\t\tstruct ref *next = alloc_ref(len - 40);\n+\t\t\tbuffer[40] = '\\0';\n+\t\t\tif (get_sha1_hex(buffer, next->old_sha1)) {\n+\t\t\t\twarning (\"invalid SHA-1: %s\", buffer);\n+\t\t\t\tfree(next);\n+\t\t\t\tcontinue;\n+\t\t\t}\n+\t\t\tstrcpy(next->name, buffer + 41);\n+\t\t\tnext->next = (*list)->next;\n+\t\t\t(*list)->next = next;\n+\t\t\tlist = &(*list)->next;\n+\t\t}\n+\t}\n+}\n+\n+static struct ref *get_refs_via_rsync(const struct transport *transport)\n+{\n+\tstruct strbuf buf = STRBUF_INIT, temp_dir = STRBUF_INIT;\n+\tstruct ref dummy, *tail = &dummy;\n+\tstruct child_process rsync;\n+\tconst char *args[5];\n+\tint temp_dir_len;\n+\n+\t/* copy the refs to the temporary directory */\n+\n+\tstrbuf_addstr(&temp_dir, git_path(\"rsync-refs-XXXXXX\"));\n+\tif (!mkdtemp(temp_dir.buf))\n+\t\tdie (\"Could not make temporary directory\");\n+\ttemp_dir_len = temp_dir.len;\n+\n+\tstrbuf_addstr(&buf, transport->url);\n+\tstrbuf_addstr(&buf, \"/refs\");\n+\n+\tmemset(&rsync, 0, sizeof(rsync));\n+\trsync.argv = args;\n+\trsync.stdout_to_stderr = 1;\n+\targs[0] = \"rsync\";\n+\targs[1] = transport->verbose ? \"-rv\" : \"-r\";\n+\targs[2] = buf.buf;\n+\targs[3] = temp_dir.buf;\n+\targs[4] = NULL;\n+\n+\tif (run_command(&rsync))\n+\t\tdie (\"Could not run rsync to get refs\");\n+\n+\tstrbuf_reset(&buf);\n+\tstrbuf_addstr(&buf, transport->url);\n+\tstrbuf_addstr(&buf, \"/packed-refs\");\n+\n+\targs[2] = buf.buf;\n+\n+\tif (run_command(&rsync))\n+\t\tdie (\"Could not run rsync to get refs\");\n+\n+\t/* read the copied refs */\n+\n+\tstrbuf_addstr(&temp_dir, \"/refs\");\n+\tread_loose_refs(&temp_dir, temp_dir_len + 1, &tail);\n+\tstrbuf_setlen(&temp_dir, temp_dir_len);\n+\n+\ttail = &dummy;\n+\tstrbuf_addstr(&temp_dir, \"/packed-refs\");\n+\tinsert_packed_refs(temp_dir.buf, &tail);\n+\tstrbuf_setlen(&temp_dir, temp_dir_len);\n+\n+\tif (remove_dir_recursively(&temp_dir, 0))\n+\t\twarning (\"Error removing temporary directory %s.\",\n+\t\t\t\ttemp_dir.buf);\n+\n+\tstrbuf_release(&buf);\n+\tstrbuf_release(&temp_dir);\n+\n+\treturn dummy.next;\n+}\n+\n+static int fetch_objs_via_rsync(struct transport *transport,\n+\t\t\t\t int nr_objs, struct ref **to_fetch)\n+{\n+\tstruct strbuf buf = STRBUF_INIT;\n+\tstruct child_process rsync;\n+\tconst char *args[8];\n+\tint result;\n+\n+\tstrbuf_addstr(&buf, transport->url);\n+\tstrbuf_addstr(&buf, \"/objects/\");\n+\n+\tmemset(&rsync, 0, sizeof(rsync));\n+\trsync.argv = args;\n+\trsync.stdout_to_stderr = 1;\n+\targs[0] = \"rsync\";\n+\targs[1] = transport->verbose ? \"-rv\" : \"-r\";\n+\targs[2] = \"--ignore-existing\";\n+\targs[3] = \"--exclude\";\n+\targs[4] = \"info\";\n+\targs[5] = buf.buf;\n+\targs[6] = get_object_directory();\n+\targs[7] = NULL;\n+\n+\t/* NEEDSWORK: handle one level of alternates */\n+\tresult = run_command(&rsync);\n+\n+\tstrbuf_release(&buf);\n+\n+\treturn result;\n+}\n+\n+static int write_one_ref(const char *name, const unsigned char *sha1,\n+\t\tint flags, void *data)\n+{\n+\tstruct strbuf *buf = data;\n+\tint len = buf->len;\n+\tFILE *f;\n+\n+\t/* when called via for_each_ref(), flags is non-zero */\n+\tif (flags && prefixcmp(name, \"refs/heads/\") &&\n+\t\t\tprefixcmp(name, \"refs/tags/\"))\n+\t\treturn 0;\n+\n+\tstrbuf_addstr(buf, name);\n+\tif (safe_create_leading_directories(buf->buf) ||\n+\t\t\t!(f = fopen(buf->buf, \"w\")) ||\n+\t\t\tfprintf(f, \"%s\\n\", sha1_to_hex(sha1)) < 0 ||\n+\t\t\tfclose(f))\n+\t\treturn error(\"problems writing temporary file %s\", buf->buf);\n+\tstrbuf_setlen(buf, len);\n+\treturn 0;\n+}\n+\n+static int write_refs_to_temp_dir(struct strbuf *temp_dir,\n+\t\tint refspec_nr, const char **refspec)\n+{\n+\tint i;\n+\n+\tfor (i = 0; i < refspec_nr; i++) {\n+\t\tunsigned char sha1[20];\n+\t\tchar *ref;\n+\n+\t\tif (dwim_ref(refspec[i], strlen(refspec[i]), sha1, &ref) != 1)\n+\t\t\treturn error(\"Could not get ref %s\", refspec[i]);\n+\n+\t\tif (write_one_ref(ref, sha1, 0, temp_dir)) {\n+\t\t\tfree(ref);\n+\t\t\treturn -1;\n+\t\t}\n+\t\tfree(ref);\n+\t}\n+\treturn 0;\n+}\n+\n+static int rsync_transport_push(struct transport *transport,\n+\t\tint refspec_nr, const char **refspec, int flags)\n+{\n+\tstruct strbuf buf = STRBUF_INIT, temp_dir = STRBUF_INIT;\n+\tint result = 0, i;\n+\tstruct child_process rsync;\n+\tconst char *args[8];\n+\n+\t/* first push the objects */\n+\n+\tstrbuf_addstr(&buf, transport->url);\n+\tstrbuf_addch(&buf, '/');\n+\n+\tmemset(&rsync, 0, sizeof(rsync));\n+\trsync.argv = args;\n+\trsync.stdout_to_stderr = 1;\n+\targs[0] = \"rsync\";\n+\targs[1] = transport->verbose ? \"-av\" : \"-a\";\n+\targs[2] = \"--ignore-existing\";\n+\targs[3] = \"--exclude\";\n+\targs[4] = \"info\";\n+\targs[5] = get_object_directory();;\n+\targs[6] = buf.buf;\n+\targs[7] = NULL;\n+\n+\tif (run_command(&rsync))\n+\t\treturn error(\"Could not push objects to %s\", transport->url);\n+\n+\t/* copy the refs to the temporary directory; they could be packed. */\n+\n+\tstrbuf_addstr(&temp_dir, git_path(\"rsync-refs-XXXXXX\"));\n+\tif (!mkdtemp(temp_dir.buf))\n+\t\tdie (\"Could not make temporary directory\");\n+\tstrbuf_addch(&temp_dir, '/');\n+\n+\tif (flags & TRANSPORT_PUSH_ALL) {\n+\t\tif (for_each_ref(write_one_ref, &temp_dir))\n+\t\t\treturn -1;\n+\t} else if (write_refs_to_temp_dir(&temp_dir, refspec_nr, refspec))\n+\t\treturn -1;\n+\n+\ti = (flags & TRANSPORT_PUSH_FORCE) ? 2 : 3;\n+\targs[i++] = temp_dir.buf;\n+\targs[i++] = transport->url;\n+\targs[i++] = NULL;\n+\tif (run_command(&rsync))\n+\t\tresult = error(\"Could not push to %s\", transport->url);\n+\n+\tif (remove_dir_recursively(&temp_dir, 0))\n+\t\twarning (\"Could not remove temporary directory %s.\",\n+\t\t\t\ttemp_dir.buf);\n+\n+\tstrbuf_release(&buf);\n+\tstrbuf_release(&temp_dir);\n+\n+\treturn result;\n+}\n \n /* Generic functions for using commit walkers */\n \n@@ -402,7 +726,9 @@ struct transport *transport_get(struct remote *remote, const char *url)\n \tret->url = url;\n \n \tif (!prefixcmp(url, \"rsync://\")) {\n-\t\t/* not supported; don't populate any ops */\n+\t\tret->get_refs_list = get_refs_via_rsync;\n+\t\tret->fetch = fetch_objs_via_rsync;\n+\t\tret->push = rsync_transport_push;\n \n \t} else if (!prefixcmp(url, \"http://\")\n \t        || !prefixcmp(url, \"https://\")\n-- \n1.5.3.2.1102.g9487\n"}]}