{"thread":{"id":"7917","subject":"[PATCH 5/8] git-repack --max-pack-size: write_object() takes \"limit\" arg","startedAt":"2007-04-30T23:22:20Z","lastAt":"2007-04-30T23:22:20Z","messageCount":1,"participants":["Dana How"],"isPatch":true,"patchVersion":1,"patchTotal":8},"messages":[{"id":"40773","messageId":"46367A2C.3030804@gmail.com","threadId":"7917","inReplyTo":null,"subject":"[PATCH 5/8] git-repack --max-pack-size: write_object() takes \"limit\" arg","fromName":"Dana How","fromEmail":"danahow@gmail.com","sentAt":"2007-04-30T23:22:20Z","receivedAt":"2007-04-30T23:22:20Z","isPatch":true,"sender":{"key":"danahow@gmail.com","avatar":null},"body":"\nAccept new \"limit\" argument and check against it\nbefore each group of writes.  Update delta usability rules\nfor possibility of delta base being in a previously-\nwritten pack.  Inline sha1write_compress() so we know\nthe exact size of the written data when it needs to be compressed.\n\nSigned-off-by: Dana L. How <danahow@gmail.com>\n---\n builtin-pack-objects.c |  110 +++++++++++++++++++++++++++++++++++++----------\n 1 files changed, 86 insertions(+), 24 deletions(-)\n\ndiff --git a/builtin-pack-objects.c b/builtin-pack-objects.c\nindex 98066bf..d3ebe1d 100644\n--- a/builtin-pack-objects.c\n+++ b/builtin-pack-objects.c\n@@ -399,12 +399,14 @@ static int revalidate_loose_object(struct object_entry *entry,\n }\n \n static unsigned long write_object(struct sha1file *f,\n-\t\t\t\t  struct object_entry *entry)\n+\t\t\t\t  struct object_entry *entry,\n+\t\t\t\t  unsigned long limit)\n {\n \tunsigned long size;\n \tenum object_type type;\n \tvoid *buf;\n \tunsigned char header[10];\n+\tunsigned char dheader[10];\n \tunsigned hdrlen;\n \toff_t datalen;\n \tenum object_type obj_type;\n@@ -436,6 +438,10 @@ static unsigned long write_object(struct sha1file *f,\n \t\t\tif (revalidate_loose_object(entry, map, mapsize))\n \t\t\t\tdie(\"corrupt loose object %s\",\n \t\t\t\t    sha1_to_hex(entry->sha1));\n+\t\t\tif (limit && mapsize + 20 >= limit) {\n+\t\t\t\tmunmap(map, mapsize);\n+\t\t\t\treturn 0;\n+\t\t\t}\n \t\t\tsha1write(f, map, mapsize);\n \t\t\tmunmap(map, mapsize);\n \t\t\twritten++;\n@@ -447,24 +453,51 @@ static unsigned long write_object(struct sha1file *f,\n \t}\n \n \tif (!to_reuse) {\n+\t\tz_stream stream;\n+\t\tunsigned long maxsize;\n+\t\tvoid *out;\n+\t\t\t\t\t/* no if no delta */\n+\t\tint usable_delta =\t!entry->delta ? 0 :\n+\t\t\t\t\t/* yes if unlimited packfile */\n+\t\t\t\t\t!pack_size_limit ? 1 :\n+\t\t\t\t\t/* no if base written to previous pack */\n+\t\t\t\t\tentry->delta->offset == (off_t)-1 ? 0 :\n+\t\t\t\t\t/* otherwise double-check written to this\n+\t\t\t\t\t * pack,  like we do below\n+\t\t\t\t\t */\n+\t\t\t\t\tentry->delta->offset ? 1 : 0;\n \t\tbuf = read_sha1_file(entry->sha1, &type, &size);\n \t\tif (!buf)\n \t\t\tdie(\"unable to read %s\", sha1_to_hex(entry->sha1));\n \t\tif (size != entry->size)\n \t\t\tdie(\"object %s size inconsistency (%lu vs %lu)\",\n \t\t\t    sha1_to_hex(entry->sha1), size, entry->size);\n-\t\tif (entry->delta) {\n+\t\tif (usable_delta) {\n \t\t\tbuf = delta_against(buf, size, entry);\n \t\t\tsize = entry->delta_size;\n \t\t\tobj_type = (allow_ofs_delta && entry->delta->offset) ?\n \t\t\t\tOBJ_OFS_DELTA : OBJ_REF_DELTA;\n \t\t}\n+\t\t/* compress the data to store and put compressed length in datalen */\n+\t\tmemset(&stream, 0, sizeof(stream));\n+\t\tdeflateInit(&stream, zlib_compression_level);\n+\t\tmaxsize = deflateBound(&stream, size);\n+\t\tout = xmalloc(maxsize);\n+\t\t/* Compress it */\n+\t\tstream.next_in = buf;\n+\t\tstream.avail_in = size;\n+\t\tstream.next_out = out;\n+\t\tstream.avail_out = maxsize;\n+\t\twhile (deflate(&stream, Z_FINISH) == Z_OK)\n+\t\t\t/* nothing */;\n+\t\tdeflateEnd(&stream);\n+\t\tdatalen = stream.total_out;\n+\t\tdeflateEnd(&stream);\n \t\t/*\n \t\t * The object header is a byte of 'type' followed by zero or\n \t\t * more bytes of length.\n \t\t */\n \t\thdrlen = encode_header(obj_type, size, header);\n-\t\tsha1write(f, header, hdrlen);\n \n \t\tif (obj_type == OBJ_OFS_DELTA) {\n \t\t\t/*\n@@ -473,21 +506,41 @@ static unsigned long write_object(struct sha1file *f,\n \t\t\t * base from this object's position in the pack.\n \t\t\t */\n \t\t\toff_t ofs = entry->offset - entry->delta->offset;\n-\t\t\tunsigned pos = sizeof(header) - 1;\n-\t\t\theader[pos] = ofs & 127;\n+\t\t\tunsigned pos = sizeof(dheader) - 1;\n+\t\t\tdheader[pos] = ofs & 127;\n \t\t\twhile (ofs >>= 7)\n-\t\t\t\theader[--pos] = 128 | (--ofs & 127);\n-\t\t\tsha1write(f, header + pos, sizeof(header) - pos);\n-\t\t\thdrlen += sizeof(header) - pos;\n+\t\t\t\tdheader[--pos] = 128 | (--ofs & 127);\n+\t\t\tif (limit && hdrlen + sizeof(dheader) - pos + datalen + 20 >= limit) {\n+\t\t\t\tfree(out);\n+\t\t\t\tfree(buf);\n+\t\t\t\treturn 0;\n+\t\t\t}\n+\t\t\tsha1write(f, header, hdrlen);\n+\t\t\tsha1write(f, dheader + pos, sizeof(dheader) - pos);\n+\t\t\thdrlen += sizeof(dheader) - pos;\n \t\t} else if (obj_type == OBJ_REF_DELTA) {\n \t\t\t/*\n \t\t\t * Deltas with a base reference contain\n \t\t\t * an additional 20 bytes for the base sha1.\n \t\t\t */\n+\t\t\tif (limit && hdrlen + 20 + datalen + 20 >= limit) {\n+\t\t\t\tfree(out);\n+\t\t\t\tfree(buf);\n+\t\t\t\treturn 0;\n+\t\t\t}\n+\t\t\tsha1write(f, header, hdrlen);\n \t\t\tsha1write(f, entry->delta->sha1, 20);\n \t\t\thdrlen += 20;\n+\t\t} else {\n+\t\t\tif (limit && hdrlen + datalen + 20 >= limit) {\n+\t\t\t\tfree(out);\n+\t\t\t\tfree(buf);\n+\t\t\t\treturn 0;\n+\t\t\t}\n+\t\t\tsha1write(f, header, hdrlen);\n \t\t}\n-\t\tdatalen = sha1write_compressed(f, buf, size);\n+\t\tsha1write(f, out, datalen);\n+\t\tfree(out);\n \t\tfree(buf);\n \t}\n \telse {\n@@ -502,20 +555,6 @@ static unsigned long write_object(struct sha1file *f,\n \t\t\treused_delta++;\n \t\t}\n \t\thdrlen = encode_header(obj_type, entry->size, header);\n-\t\tsha1write(f, header, hdrlen);\n-\t\tif (obj_type == OBJ_OFS_DELTA) {\n-\t\t\toff_t ofs = entry->offset - entry->delta->offset;\n-\t\t\tunsigned pos = sizeof(header) - 1;\n-\t\t\theader[pos] = ofs & 127;\n-\t\t\twhile (ofs >>= 7)\n-\t\t\t\theader[--pos] = 128 | (--ofs & 127);\n-\t\t\tsha1write(f, header + pos, sizeof(header) - pos);\n-\t\t\thdrlen += sizeof(header) - pos;\n-\t\t} else if (obj_type == OBJ_REF_DELTA) {\n-\t\t\tsha1write(f, entry->delta->sha1, 20);\n-\t\t\thdrlen += 20;\n-\t\t}\n-\n \t\toffset = entry->in_pack_offset;\n \t\trevidx = find_packed_object(p, offset);\n \t\tdatalen = revidx[1].offset - offset;\n@@ -524,6 +563,29 @@ static unsigned long write_object(struct sha1file *f,\n \t\t\tdie(\"bad packed object CRC for %s\", sha1_to_hex(entry->sha1));\n \t\toffset += entry->in_pack_header_size;\n \t\tdatalen -= entry->in_pack_header_size;\n+\t\tif (obj_type == OBJ_OFS_DELTA) {\n+\t\t\toff_t ofs = entry->offset - entry->delta->offset;\n+\t\t\tunsigned pos = sizeof(dheader) - 1;\n+\t\t\tdheader[pos] = ofs & 127;\n+\t\t\twhile (ofs >>= 7)\n+\t\t\t\tdheader[--pos] = 128 | (--ofs & 127);\n+\t\t\tif (limit && hdrlen + sizeof(dheader) - pos + datalen + 20 >= limit)\n+\t\t\t\treturn 0;\n+\t\t\tsha1write(f, header, hdrlen);\n+\t\t\tsha1write(f, dheader + pos, sizeof(dheader) - pos);\n+\t\t\thdrlen += sizeof(dheader) - pos;\n+\t\t} else if (obj_type == OBJ_REF_DELTA) {\n+\t\t\tif (limit && hdrlen + 20 + datalen + 20 >= limit)\n+\t\t\t\treturn 0;\n+\t\t\tsha1write(f, header, hdrlen);\n+\t\t\tsha1write(f, entry->delta->sha1, 20);\n+\t\t\thdrlen += 20;\n+\t\t} else {\n+\t\t\tif (limit && hdrlen + datalen + 20 >= limit)\n+\t\t\t\treturn 0;\n+\t\t\tsha1write(f, header, hdrlen);\n+\t\t}\n+\n \t\tif (!pack_to_stdout && p->index_version == 1 &&\n \t\t    check_pack_inflate(p, &w_curs, offset, datalen, entry->size))\n \t\t\tdie(\"corrupt packed object for %s\", sha1_to_hex(entry->sha1));\n@@ -554,7 +616,7 @@ static off_t write_one(struct sha1file *f,\n \t\toffset = write_one(f, e->delta, offset);\n \n \te->offset = offset;\n-\tsize = write_object(f, e);\n+\tsize = write_object(f, e, 0);\n \n \t/* make sure off_t is sufficiently large not to wrap */\n \tif (offset > offset + size)\n-- \n1.5.2.rc0.766.gba60-dirty\n"}]}