{"thread":{"id":"43782","subject":"[PATCH] enhance clone and fetch -k experience","startedAt":"2006-10-27T19:42:17Z","lastAt":"2006-10-27T19:42:17Z","messageCount":1,"participants":["Nicolas Pitre"],"isPatch":true,"patchVersion":1,"patchTotal":null},"messages":[{"id":"298748","messageId":"Pine.LNX.4.64.0610271502200.11384@xanadu.home","threadId":"43782","inReplyTo":null,"subject":"[PATCH] enhance clone and fetch -k experience","fromName":"Nicolas Pitre","fromEmail":"nico@cam.org","sentAt":"2006-10-27T19:42:17Z","receivedAt":"2006-10-27T19:42:17Z","isPatch":true,"sender":{"key":"nico@fluxnic.net","avatar":"https://avatars.githubusercontent.com/u/702790?v=4"},"body":"Now that index-pack can be streamed with a pack, it is probably a good \nidea to use it directly instead of creating a temporary file and running \nindex-pack afterwards.  This way index-pack can abort early whenever a \ncorruption is encountered even if the pack has not been fully \ndownloaded, it can display a progress percentage as it knows how much to \nexpects, and it is a bit faster since the pack indexing is partially \ndone as data is received. Using fetch -k doesn't need to disable thin \npack generation on the remote end either.\n\nSigned-off-by: Nicolas Pitre <nico@cam.org>\n\n---\n\ndiff --git a/fetch-clone.c b/fetch-clone.c\nindex 76b99af..96cdab4 100644\n--- a/fetch-clone.c\n+++ b/fetch-clone.c\n@@ -3,97 +3,6 @@\n #include \"pkt-line.h\"\n #include \"sideband.h\"\n #include <sys/wait.h>\n-#include <sys/time.h>\n-\n-static int finish_pack(const char *pack_tmp_name, const char *me)\n-{\n-\tint pipe_fd[2];\n-\tpid_t pid;\n-\tchar idx[PATH_MAX];\n-\tchar final[PATH_MAX];\n-\tchar hash[41];\n-\tunsigned char sha1[20];\n-\tchar *cp;\n-\tint err = 0;\n-\n-\tif (pipe(pipe_fd) < 0)\n-\t\tdie(\"%s: unable to set up pipe\", me);\n-\n-\tstrcpy(idx, pack_tmp_name); /* \".git/objects/pack-XXXXXX\" */\n-\tcp = strrchr(idx, '/');\n-\tmemcpy(cp, \"/pidx\", 5);\n-\n-\tpid = fork();\n-\tif (pid < 0)\n-\t\tdie(\"%s: unable to fork off git-index-pack\", me);\n-\tif (!pid) {\n-\t\tclose(0);\n-\t\tdup2(pipe_fd[1], 1);\n-\t\tclose(pipe_fd[0]);\n-\t\tclose(pipe_fd[1]);\n-\t\texecl_git_cmd(\"index-pack\", \"-o\", idx, pack_tmp_name, NULL);\n-\t\terror(\"cannot exec git-index-pack <%s> <%s>\",\n-\t\t      idx, pack_tmp_name);\n-\t\texit(1);\n-\t}\n-\tclose(pipe_fd[1]);\n-\tif (read(pipe_fd[0], hash, 40) != 40) {\n-\t\terror(\"%s: unable to read from git-index-pack\", me);\n-\t\terr = 1;\n-\t}\n-\tclose(pipe_fd[0]);\n-\n-\tfor (;;) {\n-\t\tint status, code;\n-\n-\t\tif (waitpid(pid, &status, 0) < 0) {\n-\t\t\tif (errno == EINTR)\n-\t\t\t\tcontinue;\n-\t\t\terror(\"waitpid failed (%s)\", strerror(errno));\n-\t\t\tgoto error_die;\n-\t\t}\n-\t\tif (WIFSIGNALED(status)) {\n-\t\t\tint sig = WTERMSIG(status);\n-\t\t\terror(\"git-index-pack died of signal %d\", sig);\n-\t\t\tgoto error_die;\n-\t\t}\n-\t\tif (!WIFEXITED(status)) {\n-\t\t\terror(\"git-index-pack died of unnatural causes %d\",\n-\t\t\t      status);\n-\t\t\tgoto error_die;\n-\t\t}\n-\t\tcode = WEXITSTATUS(status);\n-\t\tif (code) {\n-\t\t\terror(\"git-index-pack died with error code %d\", code);\n-\t\t\tgoto error_die;\n-\t\t}\n-\t\tif (err)\n-\t\t\tgoto error_die;\n-\t\tbreak;\n-\t}\n-\thash[40] = 0;\n-\tif (get_sha1_hex(hash, sha1)) {\n-\t\terror(\"git-index-pack reported nonsense '%s'\", hash);\n-\t\tgoto error_die;\n-\t}\n-\t/* Now we have pack in pack_tmp_name[], and\n-\t * idx in idx[]; rename them to their final names.\n-\t */\n-\tsnprintf(final, sizeof(final),\n-\t\t \"%s/pack/pack-%s.pack\", get_object_directory(), hash);\n-\tmove_temp_to_file(pack_tmp_name, final);\n-\tchmod(final, 0444);\n-\tsnprintf(final, sizeof(final),\n-\t\t \"%s/pack/pack-%s.idx\", get_object_directory(), hash);\n-\tmove_temp_to_file(idx, final);\n-\tchmod(final, 0444);\n-\treturn 0;\n-\n- error_die:\n-\tunlink(idx);\n-\tunlink(pack_tmp_name);\n-\texit(1);\n-}\n \n static pid_t setup_sideband(int sideband, const char *me, int fd[2], int xd[2])\n {\n@@ -128,7 +37,7 @@ static pid_t setup_sideband(int sideband\n \treturn side_pid;\n }\n \n-int receive_unpack_pack(int xd[2], const char *me, int quiet, int sideband)\n+static int get_pack(int xd[2], const char *me, int sideband, const char **argv)\n {\n \tint status;\n \tpid_t pid, side_pid;\n@@ -142,135 +51,37 @@ int receive_unpack_pack(int xd[2], const\n \t\tdup2(fd[0], 0);\n \t\tclose(fd[0]);\n \t\tclose(fd[1]);\n-\t\texecl_git_cmd(\"unpack-objects\", quiet ? \"-q\" : NULL, NULL);\n-\t\tdie(\"git-unpack-objects exec failed\");\n+\t\texecv_git_cmd(argv);\n+\t\tdie(\"%s exec failed\", argv[0]);\n \t}\n \tclose(fd[0]);\n \tclose(fd[1]);\n \twhile (waitpid(pid, &status, 0) < 0) {\n \t\tif (errno != EINTR)\n-\t\t\tdie(\"waiting for git-unpack-objects: %s\",\n-\t\t\t    strerror(errno));\n+\t\t\tdie(\"waiting for %s: %s\", argv[0], strerror(errno));\n \t}\n \tif (WIFEXITED(status)) {\n \t\tint code = WEXITSTATUS(status);\n \t\tif (code)\n-\t\t\tdie(\"git-unpack-objects died with error code %d\",\n-\t\t\t    code);\n+\t\t\tdie(\"%s died with error code %d\", argv[0], code);\n \t\treturn 0;\n \t}\n \tif (WIFSIGNALED(status)) {\n \t\tint sig = WTERMSIG(status);\n-\t\tdie(\"git-unpack-objects died of signal %d\", sig);\n+\t\tdie(\"%s died of signal %d\", argv[0], sig);\n \t}\n-\tdie(\"git-unpack-objects died of unnatural causes %d\", status);\n+\tdie(\"%s died of unnatural causes %d\", argv[0], status);\n }\n \n-/*\n- * We average out the download speed over this many \"events\", where\n- * an event is a minimum of about half a second. That way, we get\n- * a reasonably stable number.\n- */\n-#define NR_AVERAGE (4)\n-\n-/*\n- * A \"binary msec\" is a power-of-two-msec, aka 1/1024th of a second.\n- * Keeping the time in that format means that \"bytes / msecs\" means\n- * the same as kB/s (modulo rounding).\n- *\n- * 1000512 is a magic number (usecs in a second, rounded up by half\n- * of 1024, to make \"rounding\" come out right ;)\n- */\n-#define usec_to_binarymsec(x) ((int)(x) / (1000512 >> 10))\n+int receive_unpack_pack(int xd[2], const char *me, int quiet, int sideband)\n+{\n+\tconst char *argv[3] = { \"unpack-objects\", quiet ? \"-q\" : NULL, NULL };\n+\treturn get_pack(xd, me, sideband, argv);\n+}\n \n int receive_keep_pack(int xd[2], const char *me, int quiet, int sideband)\n {\n-\tchar tmpfile[PATH_MAX];\n-\tint ofd, ifd, fd[2];\n-\tunsigned long total;\n-\tstatic struct timeval prev_tv;\n-\tstruct average {\n-\t\tunsigned long bytes;\n-\t\tunsigned long time;\n-\t} download[NR_AVERAGE] = { {0, 0}, };\n-\tunsigned long avg_bytes, avg_time;\n-\tint idx = 0;\n-\n-\tsetup_sideband(sideband, me, fd, xd);\n-\n-\tifd = fd[0];\n-\tsnprintf(tmpfile, sizeof(tmpfile),\n-\t\t \"%s/pack/tmp-XXXXXX\", get_object_directory());\n-\tofd = mkstemp(tmpfile);\n-\tif (ofd < 0)\n-\t\treturn error(\"unable to create temporary file %s\", tmpfile);\n-\n-\tgettimeofday(&prev_tv, NULL);\n-\ttotal = 0;\n-\tavg_bytes = 0;\n-\tavg_time = 0;\n-\twhile (1) {\n-\t\tchar buf[8192];\n-\t\tssize_t sz, wsz, pos;\n-\t\tsz = read(ifd, buf, sizeof(buf));\n-\t\tif (sz == 0)\n-\t\t\tbreak;\n-\t\tif (sz < 0) {\n-\t\t\tif (errno != EINTR && errno != EAGAIN) {\n-\t\t\t\terror(\"error reading pack (%s)\", strerror(errno));\n-\t\t\t\tclose(ofd);\n-\t\t\t\tunlink(tmpfile);\n-\t\t\t\treturn -1;\n-\t\t\t}\n-\t\t\tsz = 0;\n-\t\t}\n-\t\tpos = 0;\n-\t\twhile (pos < sz) {\n-\t\t\twsz = write(ofd, buf + pos, sz - pos);\n-\t\t\tif (wsz < 0) {\n-\t\t\t\terror(\"error writing pack (%s)\",\n-\t\t\t\t      strerror(errno));\n-\t\t\t\tclose(ofd);\n-\t\t\t\tunlink(tmpfile);\n-\t\t\t\treturn -1;\n-\t\t\t}\n-\t\t\tpos += wsz;\n-\t\t}\n-\t\ttotal += sz;\n-\t\tif (!quiet) {\n-\t\t\tstatic unsigned long last;\n-\t\t\tstruct timeval tv;\n-\t\t\tunsigned long diff = total - last;\n-\t\t\t/* not really \"msecs\", but a power-of-two millisec (1/1024th of a sec) */\n-\t\t\tunsigned long msecs;\n-\n-\t\t\tgettimeofday(&tv, NULL);\n-\t\t\tmsecs = tv.tv_sec - prev_tv.tv_sec;\n-\t\t\tmsecs <<= 10;\n-\t\t\tmsecs += usec_to_binarymsec(tv.tv_usec - prev_tv.tv_usec);\n-\n-\t\t\tif (msecs > 500) {\n-\t\t\t\tprev_tv = tv;\n-\t\t\t\tlast = total;\n-\n-\t\t\t\t/* Update averages ..*/\n-\t\t\t\tavg_bytes += diff;\n-\t\t\t\tavg_time += msecs;\n-\t\t\t\tavg_bytes -= download[idx].bytes;\n-\t\t\t\tavg_time -= download[idx].time;\n-\t\t\t\tdownload[idx].bytes = diff;\n-\t\t\t\tdownload[idx].time = msecs;\n-\t\t\t\tidx++;\n-\t\t\t\tif (idx >= NR_AVERAGE)\n-\t\t\t\t\tidx = 0;\n-\n-\t\t\t\tfprintf(stderr, \"%4lu.%03luMB  (%lu kB/s)      \\r\",\n-\t\t\t\t\ttotal >> 20,\n-\t\t\t\t\t1000*((total >> 10) & 1023)>>10,\n-\t\t\t\t\tavg_bytes / avg_time );\n-\t\t\t}\n-\t\t}\n-\t}\n-\tclose(ofd);\n-\treturn finish_pack(tmpfile, me);\n+\tconst char *argv[5] = { \"index-pack\", \"--stdin\", \"--fix-thin\",\n+\t\t\t\tquiet ? NULL : \"-v\", NULL };\n+\treturn get_pack(xd, me, sideband, argv);\n }\ndiff --git a/fetch-pack.c b/fetch-pack.c\nindex 90b7940..36ea092 100644\n--- a/fetch-pack.c\n+++ b/fetch-pack.c\n@@ -518,8 +518,6 @@ int main(int argc, char **argv)\n \t}\n \tif (!dest)\n \t\tusage(fetch_pack_usage);\n-\tif (keep_pack)\n-\t\tuse_thin_pack = 0;\n \tpid = git_connect(fd, dest, exec);\n \tif (pid < 0)\n"}]}