{"thread":{"id":"6014","subject":"[PATCH] enable index-pack streaming capability","startedAt":"2006-10-23T18:50:18Z","lastAt":"2006-10-23T18:50:18Z","messageCount":1,"participants":["Nicolas Pitre"],"isPatch":true,"patchVersion":1,"patchTotal":null},"messages":[{"id":"29743","messageId":"Pine.LNX.4.64.0610231338590.12418@xanadu.home","threadId":"6014","inReplyTo":null,"subject":"[PATCH] enable index-pack streaming capability","fromName":"Nicolas Pitre","fromEmail":"nico@cam.org","sentAt":"2006-10-23T18:50:18Z","receivedAt":"2006-10-23T18:50:18Z","isPatch":true,"sender":{"key":"nico@fluxnic.net","avatar":"https://avatars.githubusercontent.com/u/702790?v=4"},"body":"A new flag, --stdin, allows for a pack to be received over a stream.\nWhen this flag is provided, the pack content is written to either\nthe named pack file or directly to the object repository under the\nsame name as produced by git-repack.  The pack index is written as\nwell with the corresponding base name, unless the index name is\noverriden with -o.\n\nWith this patch, git-pack-index could be used instead of\ngit-unpack-objects when fetching remote objects but only with\nnon \"thin\" packs for now.\n\nSigned-off-by: Nicolas Pitre <nico@cam.org>\n\n---\n\ndiff --git a/Documentation/git-index-pack.txt b/Documentation/git-index-pack.txt\nindex 71ce557..0799e67 100644\n--- a/Documentation/git-index-pack.txt\n+++ b/Documentation/git-index-pack.txt\n@@ -8,7 +8,7 @@ git-index-pack - Build pack index file f\n \n SYNOPSIS\n --------\n-'git-index-pack' [-o <index-file>] <pack-file>\n+'git-index-pack' [-o <index-file>] { <pack-file> | --stdin [<pack-file>] }\n \n \n DESCRIPTION\n@@ -29,7 +29,13 @@ OPTIONS\n \tfails if the name of packed archive does not end\n \twith .pack).\n \n-\n+--stdin::\n+\tWhen this flag is provided, the pack is read from stdin\n+\tinstead and a copy is then written to <pack-file>. If\n+\t<pack-file> is not specified, the pack is written to\n+\tobjects/pack/ directory of the current git repository with\n+\ta default name determined from the pack content.\n+\t\n Author\n ------\n Written by Sergey Vlasov <vsu@altlinux.ru>\ndiff --git a/index-pack.c b/index-pack.c\nindex e33f605..cecdd26 100644\n--- a/index-pack.c\n+++ b/index-pack.c\n@@ -8,7 +8,7 @@ #include \"tag.h\"\n #include \"tree.h\"\n \n static const char index_pack_usage[] =\n-\"git-index-pack [-o index-file] pack-file\";\n+\"git-index-pack [-o <index-file>] { <pack-file> | --stdin [<pack-file>] }\";\n \n struct object_entry\n {\n@@ -37,17 +37,18 @@ struct delta_entry\n \tunion delta_base base;\n };\n \n-static const char *pack_name;\n static struct object_entry *objects;\n static struct delta_entry *deltas;\n static int nr_objects;\n static int nr_deltas;\n \n+static int from_stdin;\n+\n /* We always read in 4kB chunks. */\n static unsigned char input_buffer[4096];\n static unsigned long input_offset, input_len, consumed_bytes;\n static SHA_CTX input_ctx;\n-static int input_fd;\n+static int input_fd, output_fd, mmap_fd;\n \n /*\n  * Make sure at least \"min\" bytes are available in the buffer, and\n@@ -60,6 +61,8 @@ static void * fill(int min)\n \tif (min > sizeof(input_buffer))\n \t\tdie(\"cannot fill %d bytes\", min);\n \tif (input_offset) {\n+\t\tif (output_fd >= 0)\n+\t\t\twrite_or_die(output_fd, input_buffer, input_offset);\n \t\tSHA1_Update(&input_ctx, input_buffer, input_offset);\n \t\tmemcpy(input_buffer, input_buffer + input_offset, input_len);\n \t\tinput_offset = 0;\n@@ -86,13 +89,31 @@ static void use(int bytes)\n \tconsumed_bytes += bytes;\n }\n \n-static void open_pack_file(void)\n+static const char * open_pack_file(const char *pack_name)\n {\n-\tinput_fd = open(pack_name, O_RDONLY);\n-\tif (input_fd < 0)\n-\t\tdie(\"cannot open packfile '%s': %s\", pack_name,\n-\t\t    strerror(errno));\n+\tif (from_stdin) {\n+\t\tinput_fd = 0;\n+\t\tif (!pack_name) {\n+\t\t\tstatic char tmpfile[PATH_MAX];\n+\t\t\tsnprintf(tmpfile, sizeof(tmpfile),\n+\t\t\t\t \"%s/pack_XXXXXX\", get_object_directory());\n+\t\t\toutput_fd = mkstemp(tmpfile);\n+\t\t\tpack_name = xstrdup(tmpfile);\n+\t\t} else\n+\t\t\toutput_fd = open(pack_name, O_CREAT|O_EXCL|O_RDWR, 0600);\n+\t\tif (output_fd < 0)\n+\t\t\tdie(\"unable to create %s: %s\\n\", pack_name, strerror(errno));\n+\t\tmmap_fd = output_fd;\n+\t} else {\n+\t\tinput_fd = open(pack_name, O_RDONLY);\n+\t\tif (input_fd < 0)\n+\t\t\tdie(\"cannot open packfile '%s': %s\",\n+\t\t\t    pack_name, strerror(errno));\n+\t\toutput_fd = -1;\n+\t\tmmap_fd = input_fd;\n+\t}\n \tSHA1_Init(&input_ctx);\n+\treturn pack_name;\n }\n \n static void parse_pack_header(void)\n@@ -101,10 +122,9 @@ static void parse_pack_header(void)\n \n \t/* Header consistency check */\n \tif (hdr->hdr_signature != htonl(PACK_SIGNATURE))\n-\t\tdie(\"packfile '%s' signature mismatch\", pack_name);\n+\t\tdie(\"pack signature mismatch\");\n \tif (!pack_version_ok(hdr->hdr_version))\n-\t\tdie(\"packfile '%s' version %d unsupported\",\n-\t\t    pack_name, ntohl(hdr->hdr_version));\n+\t\tdie(\"pack version %d unsupported\", ntohl(hdr->hdr_version));\n \n \tnr_objects = ntohl(hdr->hdr_entries);\n \tuse(sizeof(struct pack_header));\n@@ -122,8 +142,7 @@ static void bad_object(unsigned long off\n \tva_start(params, format);\n \tvsnprintf(buf, sizeof(buf), format, params);\n \tva_end(params);\n-\tdie(\"packfile '%s': bad object at offset %lu: %s\",\n-\t    pack_name, offset, buf);\n+\tdie(\"pack has bad object at offset %lu: %s\", offset, buf);\n }\n \n static void *unpack_entry_data(unsigned long offset, unsigned long size)\n@@ -222,9 +241,9 @@ static void * get_data_from_pack(struct \n \tint st;\n \n \tmap = mmap(NULL, len + pg_offset, PROT_READ, MAP_PRIVATE,\n-\t\t   input_fd, from - pg_offset);\n+\t\t   mmap_fd, from - pg_offset);\n \tif (map == MAP_FAILED)\n-\t\tdie(\"cannot mmap packfile '%s': %s\", pack_name, strerror(errno));\n+\t\tdie(\"cannot mmap pack file: %s\", strerror(errno));\n \tdata = xmalloc(obj->size);\n \tmemset(&stream, 0, sizeof(stream));\n \tstream.next_out = data;\n@@ -382,14 +401,16 @@ static void parse_pack_objects(unsigned \n \tSHA1_Update(&input_ctx, input_buffer, input_offset);\n \tSHA1_Final(sha1, &input_ctx);\n \tif (hashcmp(fill(20), sha1))\n-\t\tdie(\"packfile '%s' SHA1 mismatch\", pack_name);\n+\t\tdie(\"pack is corrupted (SHA1 mismatch)\");\n \tuse(20);\n+\tif (output_fd >= 0)\n+\t\twrite_or_die(output_fd, input_buffer, input_offset);\n \n \t/* If input_fd is a file, we should have reached its end now. */\n \tif (fstat(input_fd, &st))\n-\t\tdie(\"cannot fstat packfile '%s': %s\", pack_name, strerror(errno));\n+\t\tdie(\"cannot fstat packfile: %s\", strerror(errno));\n \tif (S_ISREG(st.st_mode) && st.st_size != consumed_bytes)\n-\t\tdie(\"packfile '%s' has junk at the end\", pack_name);\n+\t\tdie(\"pack has junk at the end\");\n \n \t/* Sort deltas by base SHA1/offset for fast searching */\n \tqsort(deltas, nr_deltas, sizeof(struct delta_entry),\n@@ -435,7 +456,7 @@ static void parse_pack_objects(unsigned \n \tfor (i = 0; i < nr_deltas; i++) {\n \t\tif (deltas[i].obj->real_type == OBJ_REF_DELTA ||\n \t\t    deltas[i].obj->real_type == OBJ_OFS_DELTA)\n-\t\t\tdie(\"packfile '%s' has unresolved deltas\",  pack_name);\n+\t\t\tdie(\"pack has unresolved deltas\");\n \t}\n }\n \n@@ -450,12 +471,12 @@ static int sha1_compare(const void *_a, \n  * On entry *sha1 contains the pack content SHA1 hash, on exit it is\n  * the SHA1 hash of sorted object names.\n  */\n-static void write_index_file(const char *index_name, unsigned char *sha1)\n+static const char * write_index_file(const char *index_name, unsigned char *sha1)\n {\n \tstruct sha1file *f;\n \tstruct object_entry **sorted_by_sha, **list, **last;\n \tunsigned int array[256];\n-\tint i;\n+\tint i, fd;\n \tSHA_CTX ctx;\n \n \tif (nr_objects) {\n@@ -472,8 +493,19 @@ static void write_index_file(const char \n \telse\n \t\tsorted_by_sha = list = last = NULL;\n \n-\tunlink(index_name);\n-\tf = sha1create(\"%s\", index_name);\n+\tif (!index_name) {\n+\t\tstatic char tmpfile[PATH_MAX];\n+\t\tsnprintf(tmpfile, sizeof(tmpfile),\n+\t\t\t \"%s/index_XXXXXX\", get_object_directory());\n+\t\tfd = mkstemp(tmpfile);\n+\t\tindex_name = xstrdup(tmpfile);\n+\t} else {\n+\t\tunlink(index_name);\n+\t\tfd = open(index_name, O_CREAT|O_EXCL|O_WRONLY, 0600);\n+\t}\n+\tif (fd < 0)\n+\t\tdie(\"unable to create %s: %s\", index_name, strerror(errno));\n+\tf = sha1fd(fd, index_name);\n \n \t/*\n \t * Write the first-level table (the list is sorted,\n@@ -513,12 +545,52 @@ static void write_index_file(const char \n \tsha1close(f, NULL, 1);\n \tfree(sorted_by_sha);\n \tSHA1_Final(sha1, &ctx);\n+\treturn index_name;\n+}\n+\n+static void final(const char *final_pack_name, const char *curr_pack_name,\n+\t\t  const char *final_index_name, const char *curr_index_name,\n+\t\t  unsigned char *sha1)\n+{\n+\tchar name[PATH_MAX];\n+\tint err;\n+\n+\tif (!from_stdin) {\n+\t\tclose(input_fd);\n+\t} else {\n+\t\terr = close(output_fd);\n+\t\tif (err)\n+\t\t\tdie(\"error while closing pack file: %s\", strerror(errno));\n+\t\tchmod(curr_pack_name, 0444);\n+\t}\n+\n+\tif (final_pack_name != curr_pack_name) {\n+\t\tif (!final_pack_name) {\n+\t\t\tsnprintf(name, sizeof(name), \"%s/pack/pack-%s.pack\",\n+\t\t\t\t get_object_directory(), sha1_to_hex(sha1));\n+\t\t\tfinal_pack_name = name;\n+\t\t}\n+\t\tif (move_temp_to_file(curr_pack_name, final_pack_name))\n+\t\t\tdie(\"cannot store pack file\");\n+\t}\n+\n+\tchmod(curr_index_name, 0444);\n+\tif (final_index_name != curr_index_name) {\n+\t\tif (!final_index_name) {\n+\t\t\tsnprintf(name, sizeof(name), \"%s/pack/pack-%s.idx\",\n+\t\t\t\t get_object_directory(), sha1_to_hex(sha1));\n+\t\t\tfinal_index_name = name;\n+\t\t}\n+\t\tif (move_temp_to_file(curr_index_name, final_index_name))\n+\t\t\tdie(\"cannot store index file\");\n+\t}\n }\n \n int main(int argc, char **argv)\n {\n \tint i;\n-\tchar *index_name = NULL;\n+\tconst char *curr_pack, *pack_name = NULL;\n+\tconst char *curr_index, *index_name = NULL;\n \tchar *index_name_buf = NULL;\n \tunsigned char sha1[20];\n \n@@ -526,7 +598,9 @@ int main(int argc, char **argv)\n \t\tconst char *arg = argv[i];\n \n \t\tif (*arg == '-') {\n-\t\t\tif (!strcmp(arg, \"-o\")) {\n+\t\t\tif (!strcmp(arg, \"--stdin\")) {\n+\t\t\t\tfrom_stdin = 1;\n+\t\t\t} else if (!strcmp(arg, \"-o\")) {\n \t\t\t\tif (index_name || (i+1) >= argc)\n \t\t\t\t\tusage(index_pack_usage);\n \t\t\t\tindex_name = argv[++i];\n@@ -540,9 +614,9 @@ int main(int argc, char **argv)\n \t\tpack_name = arg;\n \t}\n \n-\tif (!pack_name)\n+\tif (!pack_name && !from_stdin)\n \t\tusage(index_pack_usage);\n-\tif (!index_name) {\n+\tif (!index_name && pack_name) {\n \t\tint len = strlen(pack_name);\n \t\tif (!has_extension(pack_name, \".pack\"))\n \t\t\tdie(\"packfile name '%s' does not end with '.pack'\",\n@@ -553,13 +627,14 @@ int main(int argc, char **argv)\n \t\tindex_name = index_name_buf;\n \t}\n \n-\topen_pack_file();\n+\tcurr_pack = open_pack_file(pack_name);\n \tparse_pack_header();\n \tobjects = xcalloc(nr_objects + 1, sizeof(struct object_entry));\n \tdeltas = xcalloc(nr_objects, sizeof(struct delta_entry));\n \tparse_pack_objects(sha1);\n \tfree(deltas);\n-\twrite_index_file(index_name, sha1);\n+\tcurr_index = write_index_file(index_name, sha1);\n+\tfinal(pack_name, curr_pack, index_name, curr_index, sha1);\n \tfree(objects);\n \tfree(index_name_buf);\n \n"}]}