{"thread":{"id":"26244","subject":"[PATCH] index-pack: support --pack-stream to concatenate packs","startedAt":"2011-01-09T07:45:21Z","lastAt":"2011-01-09T07:45:21Z","messageCount":1,"participants":["Nguyễn Thái Ngọc Duy"],"isPatch":true,"patchVersion":1,"patchTotal":null},"messages":[{"id":"159240","messageId":"1294559121-19048-1-git-send-email-pclouds@gmail.com","threadId":"26244","inReplyTo":null,"subject":"[PATCH] index-pack: support --pack-stream to concatenate packs","fromName":"Nguyễn Thái Ngọc Duy","fromEmail":"pclouds@gmail.com","sentAt":"2011-01-09T07:45:21Z","receivedAt":"2011-01-09T07:45:21Z","isPatch":true,"sender":{"key":"pclouds@gmail.com","avatar":"https://avatars.githubusercontent.com/u/720?v=4"},"body":"--pack-stream --stdin assumes stdin is a stream of packs and creates\na single pack containing all objects of the stream.\n\nSigned-off-by: Nguyễn Thái Ngọc Duy <pclouds@gmail.com>\n---\n Don't know if it's useful for anybody else. Might be a cheaper way\n than git-repack to reduce the number of packs.\n\n builtin/index-pack.c |  119 ++++++++++++++++++++++++++++++++++++++++++--------\n 1 files changed, 101 insertions(+), 18 deletions(-)\n\ndiff --git a/builtin/index-pack.c b/builtin/index-pack.c\nindex 8dc5c0b..4864913 100644\n--- a/builtin/index-pack.c\n+++ b/builtin/index-pack.c\n@@ -59,6 +59,7 @@ static int nr_deltas;\n static int nr_resolved_deltas;\n \n static int from_stdin;\n+static int pack_stream;\n static int strict;\n static int verbose;\n \n@@ -68,7 +69,7 @@ static struct progress *progress;\n static unsigned char input_buffer[4096];\n static unsigned int input_offset, input_len;\n static off_t consumed_bytes;\n-static git_SHA_CTX input_ctx;\n+static git_SHA_CTX input_ctx, pack_stream_ctx;\n static uint32_t input_crc32;\n static int input_fd, output_fd, pack_fd;\n \n@@ -115,23 +116,35 @@ static void check_objects(void)\n \n \n /* Discard current buffer used content. */\n-static void flush(void)\n+static void flush_1(int in_pack_stream)\n {\n \tif (input_offset) {\n-\t\tif (output_fd >= 0)\n-\t\t\twrite_or_die(output_fd, input_buffer, input_offset);\n+\t\tif (output_fd >= 0 &&\n+\t\t    (!pack_stream || (pack_stream && in_pack_stream)))\n+\t\t\t\twrite_or_die(output_fd, input_buffer, input_offset);\n+\n \t\tgit_SHA1_Update(&input_ctx, input_buffer, input_offset);\n+\t\tif (pack_stream && in_pack_stream)\n+\t\t\tgit_SHA1_Update(&pack_stream_ctx, input_buffer, input_offset);\n \t\tmemmove(input_buffer, input_buffer + input_offset, input_len);\n \t\tinput_offset = 0;\n \t}\n }\n \n+static void flush(void)\n+{\n+\tflush_1(1);\n+}\n+\n /*\n  * Make sure at least \"min\" bytes are available in the buffer, and\n  * return the pointer to the buffer.\n  */\n-static void *fill(int min)\n+static void *fill_gently(int min, int *got_eof)\n {\n+\tif (got_eof)\n+\t\t*got_eof = 0;\n+\n \tif (min <= input_len)\n \t\treturn input_buffer + input_offset;\n \tif (min > sizeof(input_buffer))\n@@ -141,9 +154,16 @@ static void *fill(int min)\n \t\tssize_t ret = xread(input_fd, input_buffer + input_len,\n \t\t\t\tsizeof(input_buffer) - input_len);\n \t\tif (ret <= 0) {\n-\t\t\tif (!ret)\n-\t\t\t\tdie(\"early EOF\");\n-\t\t\tdie_errno(\"read error on input\");\n+\t\t\tif (!ret) {\n+\t\t\t\tif (!got_eof)\n+\t\t\t\t\tdie(\"early EOF\");\n+\t\t\t\telse {\n+\t\t\t\t\t*got_eof = 1;\n+\t\t\t\t\treturn NULL;\n+\t\t\t\t}\n+\t\t\t}\n+\t\t\telse\n+\t\t\t\tdie_errno(\"read error on input\");\n \t\t}\n \t\tinput_len += ret;\n \t\tif (from_stdin)\n@@ -152,6 +172,11 @@ static void *fill(int min)\n \treturn input_buffer;\n }\n \n+static void *fill(int min)\n+{\n+\treturn fill_gently(min, NULL);\n+}\n+\n static void use(int bytes)\n {\n \tif (bytes > input_len)\n@@ -188,6 +213,8 @@ static const char *open_pack_file(const char *pack_name)\n \t\tpack_fd = input_fd;\n \t}\n \tgit_SHA1_Init(&input_ctx);\n+\tif (pack_stream)\n+\t\tgit_SHA1_Init(&pack_stream_ctx);\n \treturn pack_name;\n }\n \n@@ -202,7 +229,7 @@ static void parse_pack_header(void)\n \t\tdie(\"pack version %\"PRIu32\" unsupported\",\n \t\t\tntohl(hdr->hdr_version));\n \n-\tnr_objects = ntohl(hdr->hdr_entries);\n+\tnr_objects += ntohl(hdr->hdr_entries);\n \tuse(sizeof(struct pack_header));\n }\n \n@@ -590,10 +617,10 @@ static int compare_delta_entry(const void *a, const void *b)\n }\n \n /* Parse all objects and return the pack content SHA1 hash */\n-static void parse_pack_objects(unsigned char *sha1)\n+static void parse_pack_objects(unsigned char *sha1, int start_object)\n {\n \tint i;\n-\tstruct delta_entry *delta = deltas;\n+\tstruct delta_entry *delta = deltas + nr_deltas;\n \tstruct stat st;\n \n \t/*\n@@ -606,7 +633,7 @@ static void parse_pack_objects(unsigned char *sha1)\n \t\tprogress = start_progress(\n \t\t\t\tfrom_stdin ? \"Receiving objects\" : \"Indexing objects\",\n \t\t\t\tnr_objects);\n-\tfor (i = 0; i < nr_objects; i++) {\n+\tfor (i = start_object; i < nr_objects; i++) {\n \t\tstruct object_entry *obj = &objects[i];\n \t\tvoid *data = unpack_raw_entry(obj, &delta->base);\n \t\tobj->real_type = obj->type;\n@@ -630,11 +657,13 @@ static void parse_pack_objects(unsigned char *sha1)\n \tuse(20);\n \n \t/* If input_fd is a file, we should have reached its end now. */\n-\tif (fstat(input_fd, &st))\n-\t\tdie_errno(\"cannot fstat packfile\");\n-\tif (S_ISREG(st.st_mode) &&\n-\t\t\tlseek(input_fd, 0, SEEK_CUR) - input_len != st.st_size)\n-\t\tdie(\"pack has junk at the end\");\n+\tif (!pack_stream) {\n+\t\tif (fstat(input_fd, &st))\n+\t\t\tdie_errno(\"cannot fstat packfile\");\n+\t\tif (S_ISREG(st.st_mode) &&\n+\t\t    lseek(input_fd, 0, SEEK_CUR) - input_len != st.st_size)\n+\t\t\tdie(\"pack has junk at the end\");\n+\t}\n \n \tif (!nr_deltas)\n \t\treturn;\n@@ -871,6 +900,54 @@ static int git_index_pack_config(const char *k, const char *v, void *cb)\n \treturn git_default_config(k, v, cb);\n }\n \n+static void parse_following_packs(unsigned char *pack_sha1,\n+\t\t\t\t  const char *curr_pack,\n+\t\t\t\t  int fix_thin_pack)\n+{\n+\tunsigned char sha1[20];\n+\n+\tfor (;;) {\n+\t\tint got_eof;\n+\t\tint last_nr_objects = nr_objects;\n+\n+\t\t/* SHA1 trailer of the previous pack */\n+\t\tconsumed_bytes -= 20;\n+\t\tflush_1(0);\n+\n+\t\tfill_gently(sizeof(struct pack_header), &got_eof);\n+\t\tif (got_eof) {\n+\t\t\t/*\n+\t\t\t * pretend we also consume the trailer like in\n+\t\t\t * single pack case, a new trailer will be\n+\t\t\t * rewritten by fixup_pack_header_footer().\n+\t\t\t */\n+\t\t\tconsumed_bytes += 20;\n+\t\t\tbreak;\n+\t\t}\n+\n+\t\tgit_SHA1_Init(&input_ctx);\n+\t\tparse_pack_header();\n+\t\t/* un-use() pack headers */\n+\t\tconsumed_bytes -= sizeof(struct pack_header);\n+\t\tflush_1(0);\n+\n+\t\tobjects = xrealloc(objects, (nr_objects + 1) * sizeof(struct object_entry));\n+\t\tdeltas = xrealloc(deltas, nr_objects * sizeof(struct delta_entry));\n+\t\tparse_pack_objects(sha1, last_nr_objects);\n+\t}\n+\tgit_SHA1_Final(pack_sha1, &pack_stream_ctx);\n+\n+\tif (!fix_thin_pack ||\n+\t    (nr_deltas - nr_resolved_deltas) == 0) {\n+\t\tunsigned char read_sha1[20];\n+\n+\t\thashcpy(read_sha1, pack_sha1);\n+\t\tfixup_pack_header_footer(output_fd, pack_sha1,\n+\t\t\t\t\t curr_pack, nr_objects,\n+\t\t\t\t\t read_sha1, consumed_bytes-20);\n+\t}\n+}\n+\n int cmd_index_pack(int argc, const char **argv, const char *prefix)\n {\n \tint i, fix_thin_pack = 0;\n@@ -898,6 +975,8 @@ int cmd_index_pack(int argc, const char **argv, const char *prefix)\n \t\t\t\tfrom_stdin = 1;\n \t\t\t} else if (!strcmp(arg, \"--fix-thin\")) {\n \t\t\t\tfix_thin_pack = 1;\n+\t\t\t} else if (!strcmp(arg, \"--pack-stream\")) {\n+\t\t\t\tpack_stream = 1;\n \t\t\t} else if (!strcmp(arg, \"--strict\")) {\n \t\t\t\tstrict = 1;\n \t\t\t} else if (!strcmp(arg, \"--keep\")) {\n@@ -946,6 +1025,8 @@ int cmd_index_pack(int argc, const char **argv, const char *prefix)\n \t\tusage(index_pack_usage);\n \tif (fix_thin_pack && !from_stdin)\n \t\tdie(\"--fix-thin cannot be used without --stdin\");\n+\tif (pack_stream && !from_stdin)\n+\t\tdie(\"--pack-stream cannot be used without --stdin\");\n \tif (!index_name && pack_name) {\n \t\tint len = strlen(pack_name);\n \t\tif (!has_extension(pack_name, \".pack\"))\n@@ -971,7 +1052,9 @@ int cmd_index_pack(int argc, const char **argv, const char *prefix)\n \tparse_pack_header();\n \tobjects = xmalloc((nr_objects + 1) * sizeof(struct object_entry));\n \tdeltas = xmalloc(nr_objects * sizeof(struct delta_entry));\n-\tparse_pack_objects(pack_sha1);\n+\tparse_pack_objects(pack_sha1, 0);\n+\tif (pack_stream)\n+\t\tparse_following_packs(pack_sha1, curr_pack, fix_thin_pack);\n \tif (nr_deltas == nr_resolved_deltas) {\n \t\tstop_progress(&progress);\n \t\t/* Flush remaining pack final 20-byte SHA1. */\n-- \n1.7.2.2\n"}]}