git/list[1] front-page[2] threads[3] people[4] search[5] about
 

[PATCH v2 10/10] archive-zip: streaming for deflated files

From
Nguyễn Thái Ngọc Duy <pclouds@gmail.com>
Date
May 2, 2012, 13:25 UTC
Message-ID
<1335965122-17458-11-git-send-email-pclouds@gmail.com>
In-Reply-To
<1335965122-17458-1-git-send-email-pclouds@gmail.com>
From: René Scharfe <rene.scharfe@lsrfire.ath.cx>

After an entry has been streamed out, its CRC and sizes are written as part of a data descriptor.

For simplicity, we make the buffer for the compressed chunks twice as big as for the uncompressed ones, to be sure the result fit in even if deflate makes them bigger.

t5000 verifies output. t1050 makes sure the command always respects core.bigfilethreshold

Signed-off-by: Rene Scharfe <rene.scharfe@lsrfire.ath.cx>
Signed-off-by: Nguyễn Thái Ngọc Duy <pclouds@gmail.com>
---
 archive-zip.c       |   64 ++++++++++++++++++++++++++++++++++++++++++++++++++-
 t/t1050-large.sh    |    4 +++
 t/t5000-tar-tree.sh |    8 ++++++
 3 files changed, 75 insertions(+), 1 deletions(-)
diff --git a/archive-zip.c b/archive-zip.c
index 1c6c39d..f5af81f 100644
--- a/archive-zip.c
+++ b/archive-zip.c
@@ -211,7 +211,7 @@ static int write_zip_entry(struct archiver_args *args,
 		compressed_size = size;
 
 		if (S_ISREG(mode) && type == OBJ_BLOB && !args->convert &&
-		    size > big_file_threshold && method == 0) {
+		    size > big_file_threshold) {
 			stream = open_istream(sha1, &type, &size, NULL);
 			if (!stream)
 				return error("cannot stream blob %s",
@@ -308,6 +308,68 @@ static int write_zip_entry(struct archiver_args *args,
 		zip_offset += ZIP_DATA_DESC_SIZE;
 
 		set_zip_dir_data_desc(&dirent, size, compressed_size, crc);
+	} else if (stream && method == 8) {
+		unsigned char buf[STREAM_BUFFER_SIZE];
+		ssize_t readlen;
+		git_zstream zstream;
+		int result;
+		size_t out_len;
+		unsigned char compressed[STREAM_BUFFER_SIZE * 2];
+
+		memset(&zstream, 0, sizeof(zstream));
+		git_deflate_init(&zstream, args->compression_level);
+
+		compressed_size = 0;
+		zstream.next_out = compressed;
+		zstream.avail_out = sizeof(compressed);
+
+		for (;;) {
+			readlen = read_istream(stream, buf, sizeof(buf));
+			if (readlen <= 0)
+				break;
+			crc = crc32(crc, buf, readlen);
+
+			zstream.next_in = buf;
+			zstream.avail_in = readlen;
+			result = git_deflate(&zstream, 0);
+			if (result != Z_OK)
+				die("deflate error (%d)", result);
+			out = compressed;
+			if (!compressed_size)
+				out += 2;
+			out_len = zstream.next_out - out;
+
+			if (out_len > 0) {
+				write_or_die(1, out, out_len);
+				compressed_size += out_len;
+				zstream.next_out = compressed;
+				zstream.avail_out = sizeof(compressed);
+			}
+
+		}
+		close_istream(stream);
+		if (readlen)
+			return readlen;
+
+		zstream.next_in = buf;
+		zstream.avail_in = 0;
+		result = git_deflate(&zstream, Z_FINISH);
+		if (result != Z_STREAM_END)
+			die("deflate error (%d)", result);
+
+		git_deflate_end(&zstream);
+		out = compressed;
+		if (!compressed_size)
+			out += 2;
+		out_len = zstream.next_out - out - 4;
+		write_or_die(1, out, out_len);
+		compressed_size += out_len;
+		zip_offset += compressed_size;
+
+		write_zip_data_desc(size, compressed_size, crc);
+		zip_offset += ZIP_DATA_DESC_SIZE;
+
+		set_zip_dir_data_desc(&dirent, size, compressed_size, crc);
 	} else if (compressed_size > 0) {
 		write_or_die(1, out, compressed_size);
 		zip_offset += compressed_size;
diff --git a/t/t1050-large.sh b/t/t1050-large.sh
index 9db54b5..55ed955 100755
--- a/t/t1050-large.sh
+++ b/t/t1050-large.sh
@@ -142,4 +142,8 @@ test_expect_success 'zip achiving, store only' '
 	git archive --format=zip -0 HEAD >/dev/null
 '
 
+test_expect_success 'zip achiving, deflate' '
+	git archive --format=zip HEAD >/dev/null
+'
+
 test_done
diff --git a/t/t5000-tar-tree.sh b/t/t5000-tar-tree.sh
index 3cd2e51..ca83ac2 100755
--- a/t/t5000-tar-tree.sh
+++ b/t/t5000-tar-tree.sh
@@ -252,6 +252,14 @@ test_expect_success UNZIP 'git archive -0 --format=zip on large files' '
     (mkdir large && cd large && $UNZIP ../large.zip)
 '
 
+test_expect_success UNZIP 'git archive --format=zip on large files' '
+    git config core.bigfilethreshold 1 &&
+    git archive --format=zip HEAD >large-compressed.zip &&
+    git config --unset core.bigfilethreshold &&
+    (mkdir large-compressed && cd large-compressed && $UNZIP ../large-compressed.zip) &&
+    test_cmp large-compressed/a/bin/sh large/a/bin/sh
+'
+
 test_expect_success \
     'git archive --list outside of a git repo' \
     'GIT_DIR=some/non-existing/directory git archive --list'

-- 
1.7.8.36.g69ee2
Previous: Nguyễn Thái Ngọc Duy
Message 14 of 14 in “Large file support for git-archive”
  1. 00/10 Large file support for git-archiveNguyễn Thái Ngọc Duy, May 2, 2012
  2. 01/10 streaming: void pointer instead of char pointerNguyễn Thái Ngọc Duy, May 2, 2012
  3. 02/10 archive-tar: turn write_tar_entry into blob-writing onlyNguyễn Thái Ngọc Duy, May 2, 2012
  4. 03/10 archive-tar: unindent write_tar_entry by one levelNguyễn Thái Ngọc Duy, May 2, 2012
  5. 04/10 archive: delegate blob reading to backendNguyễn Thái Ngọc Duy, May 2, 2012
  6. 05/10 archive-tar: allow to accumulate writes before writing 512-byte blocksNguyễn Thái Ngọc Duy, May 2, 2012
  7. René ScharfeMay 2, 2012
  8. Nguyen Thai Ngoc DuyMay 2, 2012
  9. 06/10 archive-tar: stream large blobs to tar fileNguyễn Thái Ngọc Duy, May 2, 2012
  10. René ScharfeMay 2, 2012
  11. 07/10 archive-zip: remove uncompressed_sizeNguyễn Thái Ngọc Duy, May 2, 2012
  12. 08/10 archive-zip: factor out helpers for writing sizes and CRCNguyễn Thái Ngọc Duy, May 2, 2012
  13. 09/10 archive-zip: streaming for stored filesNguyễn Thái Ngọc Duy, May 2, 2012
  14. 10/10 archive-zip: streaming for deflated filesNguyễn Thái Ngọc Duy, May 2, 2012

Read the whole thread, see it on lore, or plain text.

$ cat FOOTERMessages come from the public archive at lore.kernel.org/git, fetched every hour. The front page is chosen and written each morning by an AI editor and can be wrong; the threads themselves are the record. About and API. For agents: an MCP server at https://gitlist.dev/mcp, and any thread, story or person page as Markdown by adding .md to its URL (or sending Accept: text/markdown). Details in /llms.txt.