{"thread":{"id":"8040","subject":"[PATCH 2/4] git-repack --max-pack-size: write_{object,one}() respect pack limit","startedAt":"2007-05-08T20:03:02Z","lastAt":"2007-05-08T20:03:02Z","messageCount":1,"participants":["Dana How"],"isPatch":true,"patchVersion":1,"patchTotal":4},"messages":[{"id":"41489","messageId":"4640D776.7030607@gmail.com","threadId":"8040","inReplyTo":null,"subject":"[PATCH 2/4] git-repack --max-pack-size: write_{object,one}() respect pack limit","fromName":"Dana How","fromEmail":"danahow@gmail.com","sentAt":"2007-05-08T20:03:02Z","receivedAt":"2007-05-08T20:03:02Z","isPatch":true,"sender":{"key":"danahow@gmail.com","avatar":null},"body":"\nWith --max-pack-size,  generate the appropriate write limit\nfor each object and check against it before each group of writes.\nUpdate delta usability rules to handle base being in a previously-\nwritten pack.  Inline sha1write_compress() so we know the\nexact size of the written data when it needs to be compressed.\nDetect and return write \"failure\".\n\nSigned-off-by: Dana L. How <danahow@gmail.com>\n---\n builtin-pack-objects.c |  135 ++++++++++++++++++++++++++++++++++++++----------\n 1 files changed, 108 insertions(+), 27 deletions(-)\n\ndiff --git a/builtin-pack-objects.c b/builtin-pack-objects.c\nindex d832e2d..fbff8fa 100644\n--- a/builtin-pack-objects.c\n+++ b/builtin-pack-objects.c\n@@ -399,16 +399,31 @@ static int revalidate_loose_object(struct object_entry *entry,\n }\n \n static unsigned long write_object(struct sha1file *f,\n-\t\t\t\t  struct object_entry *entry)\n+\t\t\t\t  struct object_entry *entry,\n+\t\t\t\t  off_t write_offset)\n {\n \tunsigned long size;\n \tenum object_type type;\n \tvoid *buf;\n \tunsigned char header[10];\n+\tunsigned char dheader[10];\n \tunsigned hdrlen;\n \toff_t datalen;\n \tenum object_type obj_type;\n \tint to_reuse = 0;\n+\t/* write limit if limited packsize and not first object */\n+\tunsigned long limit = pack_size_limit && nr_written ?\n+\t\t\t\tpack_size_limit - write_offset : 0;\n+\t\t\t\t/* no if no delta */\n+\tint usable_delta =\t!entry->delta ? 0 :\n+\t\t\t\t/* yes if unlimited packfile */\n+\t\t\t\t!pack_size_limit ? 1 :\n+\t\t\t\t/* no if base written to previous pack */\n+\t\t\t\tentry->delta->offset == (off_t)-1 ? 0 :\n+\t\t\t\t/* otherwise double-check written to this\n+\t\t\t\t * pack,  like we do below\n+\t\t\t\t */\n+\t\t\t\tentry->delta->offset ? 1 : 0;\n \n \tif (!pack_to_stdout)\n \t\tcrc32_begin(f);\n@@ -417,7 +432,9 @@ static unsigned long write_object(struct sha1file *f,\n \tif (! entry->in_pack)\n \t\tto_reuse = 0;\t/* can't reuse what we don't have */\n \telse if (obj_type == OBJ_REF_DELTA || obj_type == OBJ_OFS_DELTA)\n-\t\tto_reuse = 1;\t/* check_object() decided it for us */\n+\t\t\t\t/* check_object() decided it for us ... */\n+\t\tto_reuse = usable_delta;\n+\t\t\t\t/* ... but pack split may override that */\n \telse if (obj_type != entry->in_pack_type)\n \t\tto_reuse = 0;\t/* pack has delta which is unusable */\n \telse if (entry->delta)\n@@ -436,6 +453,10 @@ static unsigned long write_object(struct sha1file *f,\n \t\t\tif (revalidate_loose_object(entry, map, mapsize))\n \t\t\t\tdie(\"corrupt loose object %s\",\n \t\t\t\t    sha1_to_hex(entry->sha1));\n+\t\t\tif (limit && mapsize + 20 >= limit) {\n+\t\t\t\tmunmap(map, mapsize);\n+\t\t\t\treturn 0;\n+\t\t\t}\n \t\t\tsha1write(f, map, mapsize);\n \t\t\tmunmap(map, mapsize);\n \t\t\twritten++;\n@@ -447,24 +468,48 @@ static unsigned long write_object(struct sha1file *f,\n \t}\n \n \tif (!to_reuse) {\n+\t\tz_stream stream;\n+\t\tunsigned long maxsize;\n+\t\tvoid *out;\n \t\tbuf = read_sha1_file(entry->sha1, &type, &size);\n \t\tif (!buf)\n \t\t\tdie(\"unable to read %s\", sha1_to_hex(entry->sha1));\n \t\tif (size != entry->size)\n \t\t\tdie(\"object %s size inconsistency (%lu vs %lu)\",\n \t\t\t    sha1_to_hex(entry->sha1), size, entry->size);\n-\t\tif (entry->delta) {\n+\t\tif (usable_delta) {\n \t\t\tbuf = delta_against(buf, size, entry);\n \t\t\tsize = entry->delta_size;\n \t\t\tobj_type = (allow_ofs_delta && entry->delta->offset) ?\n \t\t\t\tOBJ_OFS_DELTA : OBJ_REF_DELTA;\n+\t\t} else {\n+\t\t\t/*\n+\t\t\t * recover real object type in case\n+\t\t\t * check_object() wanted to re-use a delta,\n+\t\t\t * but we couldn't since base was in previous split pack\n+\t\t\t */\n+\t\t\tobj_type = type;\n \t\t}\n+\t\t/* compress the data to store and put compressed length in datalen */\n+\t\tmemset(&stream, 0, sizeof(stream));\n+\t\tdeflateInit(&stream, zlib_compression_level);\n+\t\tmaxsize = deflateBound(&stream, size);\n+\t\tout = xmalloc(maxsize);\n+\t\t/* Compress it */\n+\t\tstream.next_in = buf;\n+\t\tstream.avail_in = size;\n+\t\tstream.next_out = out;\n+\t\tstream.avail_out = maxsize;\n+\t\twhile (deflate(&stream, Z_FINISH) == Z_OK)\n+\t\t\t/* nothing */;\n+\t\tdeflateEnd(&stream);\n+\t\tdatalen = stream.total_out;\n+\t\tdeflateEnd(&stream);\n \t\t/*\n \t\t * The object header is a byte of 'type' followed by zero or\n \t\t * more bytes of length.\n \t\t */\n \t\thdrlen = encode_header(obj_type, size, header);\n-\t\tsha1write(f, header, hdrlen);\n \n \t\tif (obj_type == OBJ_OFS_DELTA) {\n \t\t\t/*\n@@ -473,21 +518,41 @@ static unsigned long write_object(struct sha1file *f,\n \t\t\t * base from this object's position in the pack.\n \t\t\t */\n \t\t\toff_t ofs = entry->offset - entry->delta->offset;\n-\t\t\tunsigned pos = sizeof(header) - 1;\n-\t\t\theader[pos] = ofs & 127;\n+\t\t\tunsigned pos = sizeof(dheader) - 1;\n+\t\t\tdheader[pos] = ofs & 127;\n \t\t\twhile (ofs >>= 7)\n-\t\t\t\theader[--pos] = 128 | (--ofs & 127);\n-\t\t\tsha1write(f, header + pos, sizeof(header) - pos);\n-\t\t\thdrlen += sizeof(header) - pos;\n+\t\t\t\tdheader[--pos] = 128 | (--ofs & 127);\n+\t\t\tif (limit && hdrlen + sizeof(dheader) - pos + datalen + 20 >= limit) {\n+\t\t\t\tfree(out);\n+\t\t\t\tfree(buf);\n+\t\t\t\treturn 0;\n+\t\t\t}\n+\t\t\tsha1write(f, header, hdrlen);\n+\t\t\tsha1write(f, dheader + pos, sizeof(dheader) - pos);\n+\t\t\thdrlen += sizeof(dheader) - pos;\n \t\t} else if (obj_type == OBJ_REF_DELTA) {\n \t\t\t/*\n \t\t\t * Deltas with a base reference contain\n \t\t\t * an additional 20 bytes for the base sha1.\n \t\t\t */\n+\t\t\tif (limit && hdrlen + 20 + datalen + 20 >= limit) {\n+\t\t\t\tfree(out);\n+\t\t\t\tfree(buf);\n+\t\t\t\treturn 0;\n+\t\t\t}\n+\t\t\tsha1write(f, header, hdrlen);\n \t\t\tsha1write(f, entry->delta->sha1, 20);\n \t\t\thdrlen += 20;\n+\t\t} else {\n+\t\t\tif (limit && hdrlen + datalen + 20 >= limit) {\n+\t\t\t\tfree(out);\n+\t\t\t\tfree(buf);\n+\t\t\t\treturn 0;\n+\t\t\t}\n+\t\t\tsha1write(f, header, hdrlen);\n \t\t}\n-\t\tdatalen = sha1write_compressed(f, buf, size);\n+\t\tsha1write(f, out, datalen);\n+\t\tfree(out);\n \t\tfree(buf);\n \t}\n \telse {\n@@ -502,20 +567,6 @@ static unsigned long write_object(struct sha1file *f,\n \t\t\treused_delta++;\n \t\t}\n \t\thdrlen = encode_header(obj_type, entry->size, header);\n-\t\tsha1write(f, header, hdrlen);\n-\t\tif (obj_type == OBJ_OFS_DELTA) {\n-\t\t\toff_t ofs = entry->offset - entry->delta->offset;\n-\t\t\tunsigned pos = sizeof(header) - 1;\n-\t\t\theader[pos] = ofs & 127;\n-\t\t\twhile (ofs >>= 7)\n-\t\t\t\theader[--pos] = 128 | (--ofs & 127);\n-\t\t\tsha1write(f, header + pos, sizeof(header) - pos);\n-\t\t\thdrlen += sizeof(header) - pos;\n-\t\t} else if (obj_type == OBJ_REF_DELTA) {\n-\t\t\tsha1write(f, entry->delta->sha1, 20);\n-\t\t\thdrlen += 20;\n-\t\t}\n-\n \t\toffset = entry->in_pack_offset;\n \t\trevidx = find_packed_object(p, offset);\n \t\tdatalen = revidx[1].offset - offset;\n@@ -524,6 +575,29 @@ static unsigned long write_object(struct sha1file *f,\n \t\t\tdie(\"bad packed object CRC for %s\", sha1_to_hex(entry->sha1));\n \t\toffset += entry->in_pack_header_size;\n \t\tdatalen -= entry->in_pack_header_size;\n+\t\tif (obj_type == OBJ_OFS_DELTA) {\n+\t\t\toff_t ofs = entry->offset - entry->delta->offset;\n+\t\t\tunsigned pos = sizeof(dheader) - 1;\n+\t\t\tdheader[pos] = ofs & 127;\n+\t\t\twhile (ofs >>= 7)\n+\t\t\t\tdheader[--pos] = 128 | (--ofs & 127);\n+\t\t\tif (limit && hdrlen + sizeof(dheader) - pos + datalen + 20 >= limit)\n+\t\t\t\treturn 0;\n+\t\t\tsha1write(f, header, hdrlen);\n+\t\t\tsha1write(f, dheader + pos, sizeof(dheader) - pos);\n+\t\t\thdrlen += sizeof(dheader) - pos;\n+\t\t} else if (obj_type == OBJ_REF_DELTA) {\n+\t\t\tif (limit && hdrlen + 20 + datalen + 20 >= limit)\n+\t\t\t\treturn 0;\n+\t\t\tsha1write(f, header, hdrlen);\n+\t\t\tsha1write(f, entry->delta->sha1, 20);\n+\t\t\thdrlen += 20;\n+\t\t} else {\n+\t\t\tif (limit && hdrlen + datalen + 20 >= limit)\n+\t\t\t\treturn 0;\n+\t\t\tsha1write(f, header, hdrlen);\n+\t\t}\n+\n \t\tif (!pack_to_stdout && p->index_version == 1 &&\n \t\t    check_pack_inflate(p, &w_curs, offset, datalen, entry->size))\n \t\t\tdie(\"corrupt packed object for %s\", sha1_to_hex(entry->sha1));\n@@ -531,7 +605,7 @@ static unsigned long write_object(struct sha1file *f,\n \t\tunuse_pack(&w_curs);\n \t\treused++;\n \t}\n-\tif (entry->delta)\n+\tif (usable_delta)\n \t\twritten_delta++;\n \twritten++;\n \tif (!pack_to_stdout)\n@@ -550,11 +624,18 @@ static off_t write_one(struct sha1file *f,\n \t\treturn offset;\n \n \t/* if we are deltified, write out base object first. */\n-\tif (e->delta)\n+\tif (e->delta) {\n \t\toffset = write_one(f, e->delta, offset);\n+\t\tif (!offset)\n+\t\t\treturn 0;\n+\t}\n \n \te->offset = offset;\n-\tsize = write_object(f, e);\n+\tsize = write_object(f, e, offset);\n+\tif (!size) {\n+\t\te->offset = 0;\n+\t\treturn 0;\n+\t}\n \n \t/* make sure off_t is sufficiently large not to wrap */\n \tif (offset > offset + size)\n-- \n1.5.2.rc0.787.g0014\n"}]}