{"thread":{"id":"5972","subject":"[PATCH] add the capability for index-pack to read from a stream","startedAt":"2006-10-20T18:45:21Z","lastAt":"2006-10-20T18:45:21Z","messageCount":1,"participants":["Nicolas Pitre"],"isPatch":true,"patchVersion":1,"patchTotal":null},"messages":[{"id":"29402","messageId":"Pine.LNX.4.64.0610201425570.12418@xanadu.home","threadId":"5972","inReplyTo":null,"subject":"[PATCH] add the capability for index-pack to read from a stream","fromName":"Nicolas Pitre","fromEmail":"nico@cam.org","sentAt":"2006-10-20T18:45:21Z","receivedAt":"2006-10-20T18:45:21Z","isPatch":true,"sender":{"key":"nico@fluxnic.net","avatar":"https://avatars.githubusercontent.com/u/702790?v=4"},"body":"This patch only adds the streaming capability to index-pack.  Although \nthe code is different it has the exact same functionality as before to \nmake sure nothing broke.\n\nThis is in preparation for receiving packs over the net, parse them on \nthe fly, fix them up if they are \"thin\" packs, and keep the resulting \npack instead of exploding it into loose objects.  But such functionality \nshould come separately.\n\nOne immediate advantage of this patch is that index-pack can now deal \nwith packs up to 4GB in size even on 32-bit architectures since the pack \nis not entirely mmap()'d all at once anymore.\n\nSigned-off-by: Nicolas Pitre <nico@cam.org>\n\n---\n\ndiff --git a/index-pack.c b/index-pack.c\nindex 56c590e..e33f605 100644\n--- a/index-pack.c\n+++ b/index-pack.c\n@@ -13,6 +13,8 @@ static const char index_pack_usage[] =\n struct object_entry\n {\n \tunsigned long offset;\n+\tunsigned long size;\n+\tunsigned int hdr_size;\n \tenum object_type type;\n \tenum object_type real_type;\n \tunsigned char sha1[20];\n@@ -36,51 +38,68 @@ struct delta_entry\n };\n \n static const char *pack_name;\n-static unsigned char *pack_base;\n-static unsigned long pack_size;\n static struct object_entry *objects;\n static struct delta_entry *deltas;\n static int nr_objects;\n static int nr_deltas;\n \n-static void open_pack_file(void)\n+/* We always read in 4kB chunks. */\n+static unsigned char input_buffer[4096];\n+static unsigned long input_offset, input_len, consumed_bytes;\n+static SHA_CTX input_ctx;\n+static int input_fd;\n+\n+/*\n+ * Make sure at least \"min\" bytes are available in the buffer, and\n+ * return the pointer to the buffer.\n+ */\n+static void * fill(int min)\n {\n-\tint fd;\n-\tstruct stat st;\n+\tif (min <= input_len)\n+\t\treturn input_buffer + input_offset;\n+\tif (min > sizeof(input_buffer))\n+\t\tdie(\"cannot fill %d bytes\", min);\n+\tif (input_offset) {\n+\t\tSHA1_Update(&input_ctx, input_buffer, input_offset);\n+\t\tmemcpy(input_buffer, input_buffer + input_offset, input_len);\n+\t\tinput_offset = 0;\n+\t}\n+\tdo {\n+\t\tint ret = xread(input_fd, input_buffer + input_len,\n+\t\t\t\tsizeof(input_buffer) - input_len);\n+\t\tif (ret <= 0) {\n+\t\t\tif (!ret)\n+\t\t\t\tdie(\"early EOF\");\n+\t\t\tdie(\"read error on input: %s\", strerror(errno));\n+\t\t}\n+\t\tinput_len += ret;\n+\t} while (input_len < min);\n+\treturn input_buffer;\n+}\n+\n+static void use(int bytes)\n+{\n+\tif (bytes > input_len)\n+\t\tdie(\"used more bytes than were available\");\n+\tinput_len -= bytes;\n+\tinput_offset += bytes;\n+\tconsumed_bytes += bytes;\n+}\n \n-\tfd = open(pack_name, O_RDONLY);\n-\tif (fd < 0)\n+static void open_pack_file(void)\n+{\n+\tinput_fd = open(pack_name, O_RDONLY);\n+\tif (input_fd < 0)\n \t\tdie(\"cannot open packfile '%s': %s\", pack_name,\n \t\t    strerror(errno));\n-\tif (fstat(fd, &st)) {\n-\t\tint err = errno;\n-\t\tclose(fd);\n-\t\tdie(\"cannot fstat packfile '%s': %s\", pack_name,\n-\t\t    strerror(err));\n-\t}\n-\tpack_size = st.st_size;\n-\tpack_base = mmap(NULL, pack_size, PROT_READ, MAP_PRIVATE, fd, 0);\n-\tif (pack_base == MAP_FAILED) {\n-\t\tint err = errno;\n-\t\tclose(fd);\n-\t\tdie(\"cannot mmap packfile '%s': %s\", pack_name,\n-\t\t    strerror(err));\n-\t}\n-\tclose(fd);\n+\tSHA1_Init(&input_ctx);\n }\n \n static void parse_pack_header(void)\n {\n-\tconst struct pack_header *hdr;\n-\tunsigned char sha1[20];\n-\tSHA_CTX ctx;\n-\n-\t/* Ensure there are enough bytes for the header and final SHA1 */\n-\tif (pack_size < sizeof(struct pack_header) + 20)\n-\t\tdie(\"packfile '%s' is too small\", pack_name);\n+\tstruct pack_header *hdr = fill(sizeof(struct pack_header));\n \n \t/* Header consistency check */\n-\thdr = (void *)pack_base;\n \tif (hdr->hdr_signature != htonl(PACK_SIGNATURE))\n \t\tdie(\"packfile '%s' signature mismatch\", pack_name);\n \tif (!pack_version_ok(hdr->hdr_version))\n@@ -88,13 +107,8 @@ static void parse_pack_header(void)\n \t\t    pack_name, ntohl(hdr->hdr_version));\n \n \tnr_objects = ntohl(hdr->hdr_entries);\n-\n-\t/* Check packfile integrity */\n-\tSHA1_Init(&ctx);\n-\tSHA1_Update(&ctx, pack_base, pack_size - 20);\n-\tSHA1_Final(sha1, &ctx);\n-\tif (hashcmp(sha1, pack_base + pack_size - 20))\n-\t\tdie(\"packfile '%s' SHA1 mismatch\", pack_name);\n+\tuse(sizeof(struct pack_header));\n+\t/*fprintf(stderr, \"Indexing %d objects\\n\", nr_objects);*/\n }\n \n static void bad_object(unsigned long offset, const char *format,\n@@ -112,85 +126,78 @@ static void bad_object(unsigned long off\n \t    pack_name, offset, buf);\n }\n \n-static void *unpack_entry_data(unsigned long offset,\n-\t\t\t       unsigned long *current_pos, unsigned long size)\n+static void *unpack_entry_data(unsigned long offset, unsigned long size)\n {\n-\tunsigned long pack_limit = pack_size - 20;\n-\tunsigned long pos = *current_pos;\n \tz_stream stream;\n \tvoid *buf = xmalloc(size);\n \n \tmemset(&stream, 0, sizeof(stream));\n \tstream.next_out = buf;\n \tstream.avail_out = size;\n-\tstream.next_in = pack_base + pos;\n-\tstream.avail_in = pack_limit - pos;\n+\tstream.next_in = fill(1);\n+\tstream.avail_in = input_len;\n \tinflateInit(&stream);\n \n \tfor (;;) {\n \t\tint ret = inflate(&stream, 0);\n-\t\tif (ret == Z_STREAM_END)\n+\t\tuse(input_len - stream.avail_in);\n+\t\tif (stream.total_out == size && ret == Z_STREAM_END)\n \t\t\tbreak;\n \t\tif (ret != Z_OK)\n \t\t\tbad_object(offset, \"inflate returned %d\", ret);\n+\t\tstream.next_in = fill(1);\n+\t\tstream.avail_in = input_len;\n \t}\n \tinflateEnd(&stream);\n-\tif (stream.total_out != size)\n-\t\tbad_object(offset, \"size mismatch (expected %lu, got %lu)\",\n-\t\t\t   size, stream.total_out);\n-\t*current_pos = pack_limit - stream.avail_in;\n \treturn buf;\n }\n \n-static void *unpack_raw_entry(unsigned long offset,\n-\t\t\t      enum object_type *obj_type,\n-\t\t\t      unsigned long *obj_size,\n-\t\t\t      union delta_base *delta_base,\n-\t\t\t      unsigned long *next_obj_offset)\n+static void *unpack_raw_entry(struct object_entry *obj, union delta_base *delta_base)\n {\n-\tunsigned long pack_limit = pack_size - 20;\n-\tunsigned long pos = offset;\n-\tunsigned char c;\n+\tunsigned char *p, c;\n \tunsigned long size, base_offset;\n \tunsigned shift;\n-\tenum object_type type;\n-\tvoid *data;\n \n-\tc = pack_base[pos++];\n-\ttype = (c >> 4) & 7;\n+\tobj->offset = consumed_bytes;\n+\n+\tp = fill(1);\n+\tc = *p;\n+\tuse(1);\n+\tobj->type = (c >> 4) & 7;\n \tsize = (c & 15);\n \tshift = 4;\n \twhile (c & 0x80) {\n-\t\tif (pos >= pack_limit)\n-\t\t\tbad_object(offset, \"object extends past end of pack\");\n-\t\tc = pack_base[pos++];\n+\t\tp = fill(1);\n+\t\tc = *p;\n+\t\tuse(1);\n \t\tsize += (c & 0x7fUL) << shift;\n \t\tshift += 7;\n \t}\n+\tobj->size = size;\n \n-\tswitch (type) {\n+\tswitch (obj->type) {\n \tcase OBJ_REF_DELTA:\n-\t\tif (pos + 20 >= pack_limit)\n-\t\t\tbad_object(offset, \"object extends past end of pack\");\n-\t\thashcpy(delta_base->sha1, pack_base + pos);\n-\t\tpos += 20;\n+\t\thashcpy(delta_base->sha1, fill(20));\n+\t\tuse(20);\n \t\tbreak;\n \tcase OBJ_OFS_DELTA:\n \t\tmemset(delta_base, 0, sizeof(*delta_base));\n-\t\tc = pack_base[pos++];\n+\t\tp = fill(1);\n+\t\tc = *p;\n+\t\tuse(1);\n \t\tbase_offset = c & 127;\n \t\twhile (c & 128) {\n \t\t\tbase_offset += 1;\n \t\t\tif (!base_offset || base_offset & ~(~0UL >> 7))\n-\t\t\t\tbad_object(offset, \"offset value overflow for delta base object\");\n-\t\t\tif (pos >= pack_limit)\n-\t\t\t\tbad_object(offset, \"object extends past end of pack\");\n-\t\t\tc = pack_base[pos++];\n+\t\t\t\tbad_object(obj->offset, \"offset value overflow for delta base object\");\n+\t\t\tp = fill(1);\n+\t\t\tc = *p;\n+\t\t\tuse(1);\n \t\t\tbase_offset = (base_offset << 7) + (c & 127);\n \t\t}\n-\t\tdelta_base->offset = offset - base_offset;\n-\t\tif (delta_base->offset >= offset)\n-\t\t\tbad_object(offset, \"delta base offset is out of bound\");\n+\t\tdelta_base->offset = obj->offset - base_offset;\n+\t\tif (delta_base->offset >= obj->offset)\n+\t\t\tbad_object(obj->offset, \"delta base offset is out of bound\");\n \t\tbreak;\n \tcase OBJ_COMMIT:\n \tcase OBJ_TREE:\n@@ -198,13 +205,38 @@ static void *unpack_raw_entry(unsigned l\n \tcase OBJ_TAG:\n \t\tbreak;\n \tdefault:\n-\t\tbad_object(offset, \"bad object type %d\", type);\n+\t\tbad_object(obj->offset, \"bad object type %d\", obj->type);\n \t}\n+\tobj->hdr_size = consumed_bytes - obj->offset;\n+\n+\treturn unpack_entry_data(obj->offset, obj->size);\n+}\n+\n+static void * get_data_from_pack(struct object_entry *obj)\n+{\n+\tunsigned long from = obj[0].offset + obj[0].hdr_size;\n+\tunsigned long len = obj[1].offset - from;\n+\tunsigned pg_offset = from % getpagesize();\n+\tunsigned char *map, *data;\n+\tz_stream stream;\n+\tint st;\n \n-\tdata = unpack_entry_data(offset, &pos, size);\n-\t*obj_type = type;\n-\t*obj_size = size;\n-\t*next_obj_offset = pos;\n+\tmap = mmap(NULL, len + pg_offset, PROT_READ, MAP_PRIVATE,\n+\t\t   input_fd, from - pg_offset);\n+\tif (map == MAP_FAILED)\n+\t\tdie(\"cannot mmap packfile '%s': %s\", pack_name, strerror(errno));\n+\tdata = xmalloc(obj->size);\n+\tmemset(&stream, 0, sizeof(stream));\n+\tstream.next_out = data;\n+\tstream.avail_out = obj->size;\n+\tstream.next_in = map + pg_offset;\n+\tstream.avail_in = len;\n+\tinflateInit(&stream);\n+\twhile ((st = inflate(&stream, Z_FINISH)) == Z_OK);\n+\tinflateEnd(&stream);\n+\tif (st != Z_STREAM_END || stream.total_out != obj->size)\n+\t\tdie(\"serious inflate inconsistency\");\n+\tmunmap(map, len + pg_offset);\n \treturn data;\n }\n \n@@ -280,15 +312,12 @@ static void resolve_delta(struct delta_e\n \tunsigned long delta_size;\n \tvoid *result;\n \tunsigned long result_size;\n-\tenum object_type delta_type;\n \tunion delta_base delta_base;\n-\tunsigned long next_obj_offset;\n \tint j, first, last;\n \n \tobj->real_type = type;\n-\tdelta_data = unpack_raw_entry(obj->offset, &delta_type,\n-\t\t\t\t      &delta_size, &delta_base,\n-\t\t\t\t      &next_obj_offset);\n+\tdelta_data = get_data_from_pack(obj);\n+\tdelta_size = obj->size;\n \tresult = patch_delta(base_data, base_size, delta_data, delta_size,\n \t\t\t     &result_size);\n \tfree(delta_data);\n@@ -321,13 +350,13 @@ static int compare_delta_entry(const voi\n \treturn memcmp(&delta_a->base, &delta_b->base, UNION_BASE_SZ);\n }\n \n-static void parse_pack_objects(void)\n+/* Parse all objects and return the pack content SHA1 hash */\n+static void parse_pack_objects(unsigned char *sha1)\n {\n \tint i;\n-\tunsigned long offset = sizeof(struct pack_header);\n \tstruct delta_entry *delta = deltas;\n \tvoid *data;\n-\tunsigned long data_size;\n+\tstruct stat st;\n \n \t/*\n \t * First pass:\n@@ -337,19 +366,29 @@ static void parse_pack_objects(void)\n \t */\n \tfor (i = 0; i < nr_objects; i++) {\n \t\tstruct object_entry *obj = &objects[i];\n-\t\tobj->offset = offset;\n-\t\tdata = unpack_raw_entry(offset, &obj->type, &data_size,\n-\t\t\t\t\t&delta->base, &offset);\n+\t\tdata = unpack_raw_entry(obj, &delta->base);\n \t\tobj->real_type = obj->type;\n \t\tif (obj->type == OBJ_REF_DELTA || obj->type == OBJ_OFS_DELTA) {\n \t\t\tnr_deltas++;\n \t\t\tdelta->obj = obj;\n \t\t\tdelta++;\n \t\t} else\n-\t\t\tsha1_object(data, data_size, obj->type, obj->sha1);\n+\t\t\tsha1_object(data, obj->size, obj->type, obj->sha1);\n \t\tfree(data);\n \t}\n-\tif (offset != pack_size - 20)\n+\tobjects[i].offset = consumed_bytes;\n+\n+\t/* Check pack integrity */\n+\tSHA1_Update(&input_ctx, input_buffer, input_offset);\n+\tSHA1_Final(sha1, &input_ctx);\n+\tif (hashcmp(fill(20), sha1))\n+\t\tdie(\"packfile '%s' SHA1 mismatch\", pack_name);\n+\tuse(20);\n+\n+\t/* If input_fd is a file, we should have reached its end now. */\n+\tif (fstat(input_fd, &st))\n+\t\tdie(\"cannot fstat packfile '%s': %s\", pack_name, strerror(errno));\n+\tif (S_ISREG(st.st_mode) && st.st_size != consumed_bytes)\n \t\tdie(\"packfile '%s' has junk at the end\", pack_name);\n \n \t/* Sort deltas by base SHA1/offset for fast searching */\n@@ -378,18 +417,17 @@ static void parse_pack_objects(void)\n \t\tofs = !find_delta_childs(&base, &ofs_first, &ofs_last);\n \t\tif (!ref && !ofs)\n \t\t\tcontinue;\n-\t\tdata = unpack_raw_entry(obj->offset, &obj->type, &data_size,\n-\t\t\t\t\t&base, &offset);\n+\t\tdata = get_data_from_pack(obj);\n \t\tif (ref)\n \t\t\tfor (j = ref_first; j <= ref_last; j++)\n \t\t\t\tif (deltas[j].obj->type == OBJ_REF_DELTA)\n \t\t\t\t\tresolve_delta(&deltas[j], data,\n-\t\t\t\t\t\t      data_size, obj->type);\n+\t\t\t\t\t\t      obj->size, obj->type);\n \t\tif (ofs)\n \t\t\tfor (j = ofs_first; j <= ofs_last; j++)\n \t\t\t\tif (deltas[j].obj->type == OBJ_OFS_DELTA)\n \t\t\t\t\tresolve_delta(&deltas[j], data,\n-\t\t\t\t\t\t      data_size, obj->type);\n+\t\t\t\t\t\t      obj->size, obj->type);\n \t\tfree(data);\n \t}\n \n@@ -408,6 +446,10 @@ static int sha1_compare(const void *_a, \n \treturn hashcmp(a->sha1, b->sha1);\n }\n \n+/*\n+ * On entry *sha1 contains the pack content SHA1 hash, on exit it is\n+ * the SHA1 hash of sorted object names.\n+ */\n static void write_index_file(const char *index_name, unsigned char *sha1)\n {\n \tstruct sha1file *f;\n@@ -467,7 +509,7 @@ static void write_index_file(const char \n \t\tsha1write(f, obj->sha1, 20);\n \t\tSHA1_Update(&ctx, obj->sha1, 20);\n \t}\n-\tsha1write(f, pack_base + pack_size - 20, 20);\n+\tsha1write(f, sha1, 20);\n \tsha1close(f, NULL, 1);\n \tfree(sorted_by_sha);\n \tSHA1_Final(sha1, &ctx);\n@@ -513,9 +555,9 @@ int main(int argc, char **argv)\n \n \topen_pack_file();\n \tparse_pack_header();\n-\tobjects = xcalloc(nr_objects, sizeof(struct object_entry));\n+\tobjects = xcalloc(nr_objects + 1, sizeof(struct object_entry));\n \tdeltas = xcalloc(nr_objects, sizeof(struct delta_entry));\n-\tparse_pack_objects();\n+\tparse_pack_objects(sha1);\n \tfree(deltas);\n \twrite_index_file(index_name, sha1);\n \tfree(objects);\n"}]}