git/list[1] front-page[2] threads[3] people[4] search[5] about
 

[PATCH 7/6] zlib: allow feeding more than 4GB in one go

From
Junio C Hamano <gitster@pobox.com>
Date
Jun 10, 2011, 23:47 UTC
Message-ID
<7vei31uu38.fsf@alter.siamese.dyndns.org>
In-Reply-To
<1307736948-16956-1-git-send-email-gitster@pobox.com>
Junio C Hamano <gitster@pobox.com> writes:
> The next step would be to tweak zlib_post_call(), git_inflate() and
> git_deflate() functions to internally loop and call underlying inflate()
> and deflate() when the incoming buffers are larger than 4GB, but that
> part is not done in this series (yet).
And this is such a patch.

I had a draft of this one ready when I sent the previous 6-patch series, but it took embarrassingly long time for me to figure out what went wrong in this patch before I realized that Z_FINISH should not be relayed to the underlying inflate/deflate if we are splitting the request into multiple phases.

I consider it still rough, but with this on top of v1.7.5.4, together with the previous 6 patches, it seems to pass a trivial "have a large blob, add, commit, pack, verify, index-pack, fsck" tests, without using either of the recent "let's pass large blob to fast-import" (in 'master') nor "let's stream a blob out without expanding it in core first" (in 'next') changes.

-- test script to be stored in /var/tmp/junk or somewhere -- #!/bin/sh

set -x
rm -fr .git && git init || exit

echo | dd bs=1M seek=4800 of=a.big sleep 2 git add a.big

eval $(git ls-files -s a.big |
    sed -e 's/^[0-7]* \([0-9a-f][0-9a-f]\)\([0-9a-f]*\) .*/h=\1 a=\2/')
echo $h $a
ls -l .git/objects/$h/$a || exit
git commit -m initial || exit
git fsck || exit
git repack -a -d || exit
git fsck || exit

f=$(find .git/objects/?? -type f) test -z "$f" || exit

p=$(find .git/objects/pack -type f -name '*.pack') && i=${p%.pack}.idx &&

test -f $p && test -f $i || exit

mv $p $i . && pp=$(basename $p) && ii=$(basename $i) && test -f $pp && test -f $ii || exit

git verify-pack $pp || exit
git unpack-objects <$pp || exit
ls -l .git/objects/$h/$a || exit
rm -fr .git/objects/?? || exit

git index-pack --stdin <$pp || exit test -f "$i" || exit cmp $i $ii || exit

git cat-file blob "$h$a" >b.big || exit cmp a.big b.big

echo all done. exit 0 -- test script ends here --

-- >8 -- Update zlib_post_call() that adjusts the wrapper's notion of avail_in and avail_out to what came back from zlib, so that the callers can feed buffers larger than than 4GB to the API.

When underlying inflate/deflate stopped processing because we fed a buffer larger than 4GB limit, detect that case, update the state variables, and let the zlib function work another round.

Signed-off-by: Junio C Hamano <gitster@pobox.com>
---
 zlib.c |   78 +++++++++++++++++++++++++++++++++++++++++++++++----------------
 1 files changed, 58 insertions(+), 20 deletions(-)
diff --git a/zlib.c b/zlib.c
index fe537e3..3c63d48 100644
--- a/zlib.c
+++ b/zlib.c
@@ -27,12 +27,11 @@ static const char *zerr_to_string(int status)
  * limits the size of the buffer we can use to 4GB when interacting
  * with zlib in a single call to inflate/deflate.
  */
-#define ZLIB_BUF_MAX ((uInt)-1)
+/* #define ZLIB_BUF_MAX ((uInt)-1) */
+#define ZLIB_BUF_MAX ((uInt) 1024 * 1024 * 1024) /* 1GB */
 static inline uInt zlib_buf_cap(unsigned long len)
 {
-	if (ZLIB_BUF_MAX < len)
-		die("working buffer for zlib too large");
-	return len;
+	return (ZLIB_BUF_MAX < len) ? ZLIB_BUF_MAX : len;
 }
 
 static void zlib_pre_call(git_zstream *s)
@@ -47,12 +46,22 @@ static void zlib_pre_call(git_zstream *s)
 
 static void zlib_post_call(git_zstream *s)
 {
+	unsigned long bytes_consumed;
+	unsigned long bytes_produced;
+
+	bytes_consumed = s->z.next_in - s->next_in;
+	bytes_produced = s->z.next_out - s->next_out;
+	if (s->z.total_out != s->total_out + bytes_produced)
+		die("BUG: total_out mismatch");
+	if (s->z.total_in != s->total_in + bytes_consumed)
+		die("BUG: total_in mismatch");
+
+	s->total_out = s->z.total_out;
+	s->total_in = s->z.total_in;
 	s->next_in = s->z.next_in;
 	s->next_out = s->z.next_out;
-	s->total_in = s->z.total_in;
-	s->total_out = s->z.total_out;
-	s->avail_in = s->z.avail_in;
-	s->avail_out = s->z.avail_out;
+	s->avail_in -= bytes_consumed;
+	s->avail_out -= bytes_produced;
 }
 
 void git_inflate_init(git_zstream *strm)
@@ -103,18 +112,32 @@ int git_inflate(git_zstream *strm, int flush)
 {
 	int status;
 
-	zlib_pre_call(strm);
-	status = inflate(&strm->z, flush);
-	zlib_post_call(strm);
+	for (;;) {
+		zlib_pre_call(strm);
+		/* Never say Z_FINISH unless we are feeding everything */
+		status = inflate(&strm->z,
+				 (strm->z.avail_in != strm->avail_in)
+				 ? 0 : flush);
+		if (status == Z_MEM_ERROR)
+			die("inflate: out of memory");
+		zlib_post_call(strm);
+
+		/*
+		 * Let zlib work another round, while we can still
+		 * make progress.
+		 */
+		if ((strm->avail_out && !strm->z.avail_out) &&
+		    (status == Z_OK || status == Z_BUF_ERROR))
+			continue;
+		break;
+	}
+
 	switch (status) {
 	/* Z_BUF_ERROR: normal, needs more space in the output buffer */
 	case Z_BUF_ERROR:
 	case Z_OK:
 	case Z_STREAM_END:
 		return status;
-
-	case Z_MEM_ERROR:
-		die("inflate: out of memory");
 	default:
 		break;
 	}
@@ -192,18 +215,33 @@ int git_deflate(git_zstream *strm, int flush)
 {
 	int status;
 
-	zlib_pre_call(strm);
-	status = deflate(&strm->z, flush);
-	zlib_post_call(strm);
+	for (;;) {
+		zlib_pre_call(strm);
+
+		/* Never say Z_FINISH unless we are feeding everything */
+		status = deflate(&strm->z,
+				 (strm->z.avail_in != strm->avail_in)
+				 ? 0 : flush);
+		if (status == Z_MEM_ERROR)
+			die("deflate: out of memory");
+		zlib_post_call(strm);
+
+		/*
+		 * Let zlib work another round, while we can still
+		 * make progress.
+		 */
+		if ((strm->avail_out && !strm->z.avail_out) &&
+		    (status == Z_OK || status == Z_BUF_ERROR))
+			continue;
+		break;
+	}
+
 	switch (status) {
 	/* Z_BUF_ERROR: normal, needs more space in the output buffer */
 	case Z_BUF_ERROR:
 	case Z_OK:
 	case Z_STREAM_END:
 		return status;
-
-	case Z_MEM_ERROR:
-		die("deflate: out of memory");
 	default:
 		break;
 	}

-- 
1.7.6.rc1.118.ge175b4a
Previous: Junio C Hamano
Message 16 of 16 in “zlib only processes 4GB at a time”
  1. 0/6 zlib only processes 4GB at a timeJunio C Hamano, Jun 10, 2011
  2. 1/6 zlib: refactor error message formatterJunio C Hamano, Jun 10, 2011
  3. 2/6 zlib: wrap remaining calls to direct inflate/inflateEndJunio C Hamano, Jun 10, 2011
  4. 3/6 zlib: wrap inflateInit2 used to accept only for gzip formatJunio C Hamano, Jun 10, 2011
  5. 4/6 zlib: wrap deflate side of the APIJunio C Hamano, Jun 10, 2011
  6. Thiago FarinaJun 10, 2011
  7. Junio C HamanoJun 10, 2011
  8. 5/6 zlib: wrap deflateBound() tooJunio C Hamano, Jun 10, 2011
  9. 6/6 zlib: zlib can only process 4GB at a timeJunio C Hamano, Jun 10, 2011
  10. Erik Faye-LundJun 12, 2011
  11. Junio C HamanoJun 12, 2011
  12. Matthieu MoyJun 12, 2011
  13. Erik Faye-LundJun 13, 2011
  14. Jonathan NiederJun 13, 2011
  15. Junio C HamanoJun 13, 2011
  16. 7/6 zlib: allow feeding more than 4GB in one goJunio C Hamano, Jun 10, 2011

Read the whole thread, see it on lore, or plain text.

$ cat FOOTERMessages come from the public archive at lore.kernel.org/git, fetched every hour. The front page is chosen and written each morning by an AI editor and can be wrong; the threads themselves are the record. About and API. For agents: an MCP server at https://gitlist.dev/mcp, and any thread, story or person page as Markdown by adding .md to its URL (or sending Accept: text/markdown). Details in /llms.txt.