threads / patch / 27599

patchunpack_sha1_file(): zlib can only process 4GB at a time

Subject: [PATCH] unpack_sha1_file(): zlib can only process 4GB at a time

## tl;dr

2 messages between Jun 9, 2011 and Jun 9, 2011. Diffs are folded; open one to read it.

replies: 1people: 2as markdown or json

Junio C Hamano· Jun 9, 2011, 20:15 UTC · lore
The same theme as "unpack-objects" patch.
Signed-off-by: Junio C Hamano <gitster@pobox.com>
---
 * On a beefy enough machine, you could register a large binary blob
   and run fsck, fetch, or push to play with that object with these
   patches, but there are many more places that aren't safe.
   I seem to have caught a nasty cold, so this will be the end of the
   series from me for today.  At least that is what I plan for now.
 sha1_file.c |   29 ++++++++++++++++++++---------
 1 files changed, 20 insertions(+), 9 deletions(-)
Show changes to sha1_file.c +20 −9
diff --git a/sha1_file.c b/sha1_file.c
index 12a166f..ab9e962 100644
--- a/sha1_file.c
+++ b/sha1_file.c
@@ -1257,7 +1257,7 @@ static int unpack_sha1_header(z_stream *stream, unsigned char *map, unsigned lon
 	/* Get the data stream */
 	memset(stream, 0, sizeof(*stream));
 	stream->next_in = map;
-	stream->avail_in = mapsize;
+	stream->avail_in = zlib_buf_cap(mapsize);
 	stream->next_out = buffer;
 	stream->avail_out = bufsiz;
 
@@ -1291,11 +1291,12 @@ static int unpack_sha1_header(z_stream *stream, unsigned char *map, unsigned lon
 	return 0;
 }
 
-static void *unpack_sha1_rest(z_stream *stream, void *buffer, unsigned long size, const unsigned char *sha1)
+static void *unpack_sha1_rest(z_stream *stream, void *buffer, unsigned long mapsize,
+			      unsigned long size, const unsigned char *sha1)
 {
-	int bytes = strlen(buffer) + 1;
+	unsigned long bytes = strlen(buffer) + 1;
 	unsigned char *buf = xmallocz(size);
-	unsigned long n;
+	unsigned long n, bytes_to_produce, bytes_to_consume;
 	int status = Z_OK;
 
 	n = stream->total_out - bytes;
@@ -1303,6 +1304,8 @@ static void *unpack_sha1_rest(z_stream *stream, void *buffer, unsigned long size
 		n = size;
 	memcpy(buf, (char *) buffer + bytes, n);
 	bytes = n;
+
+	bytes_to_consume = mapsize - stream->total_in;
 	if (bytes <= size) {
 		/*
 		 * The above condition must be (bytes <= size), not
@@ -1318,17 +1321,25 @@ static void *unpack_sha1_rest(z_stream *stream, void *buffer, unsigned long size
 		 * went well with status == Z_STREAM_END at the end.
 		 */
 		stream->next_out = buf + bytes;
-		stream->avail_out = size - bytes;
-		while (status == Z_OK)
+		bytes_to_produce = size - bytes;
+
+		while (status == Z_OK || status == Z_BUF_ERROR) {
+			unsigned char *in0 = stream->next_in;
+			unsigned char *out0 = stream->next_out;
+			stream->avail_in = zlib_buf_cap(bytes_to_consume);
+			stream->avail_out = zlib_buf_cap(bytes_to_produce);
 			status = git_inflate(stream, Z_FINISH);
+			bytes_to_produce -= stream->next_out - out0;
+			bytes_to_consume -= stream->next_in - in0;
+		}
 	}
-	if (status == Z_STREAM_END && !stream->avail_in) {
+	if (status == Z_STREAM_END && !bytes_to_consume) {
 		git_inflate_end(stream);
 		return buf;
 	}
 
 	if (status < 0)
-		error("corrupt loose object '%s'", sha1_to_hex(sha1));
+		error("corrupt loose object '%s' %d", sha1_to_hex(sha1), status);
 	else if (stream->avail_in)
 		error("garbage at end of loose object '%s'",
 		      sha1_to_hex(sha1));
@@ -1397,7 +1408,7 @@ static void *unpack_sha1_file(void *map, unsigned long mapsize, enum object_type
 	if (ret < Z_OK || (*type = parse_sha1_header(hdr, size)) < 0)
 		return NULL;
 
-	return unpack_sha1_rest(&stream, hdr, *size, sha1);
+	return unpack_sha1_rest(&stream, hdr, mapsize, *size, sha1);
 }
 
 unsigned long get_size_from_delta(struct packed_git *p,
-- 
1.7.6.rc1.118.ge175b4a
Sverre Rabbelier· Jun 9, 2011, 22:28 UTC · re: Junio C Hamano · lore

Re: [PATCH] unpack_sha1_file(): zlib can only process 4GB at a time

Heya,
On Thu, Jun 9, 2011 at 22:15, Junio C Hamano <gitster@pobox.com> wrote:
>  * On a beefy enough machine, you could register a large binary blob
>   and run fsck, fetch, or push to play with that object with these
>   patches, but there are many more places that aren't safe.

Do you know what those places are, and do you plan to fix them when you get better? If not, should we record those places somewhere (if nothing else just on the list)?

-- 
Cheers,

Sverre Rabbelier

← back to recent threads