git/list[1] front-page[2] threads[3] people[4] search[5] about
 

[PATCH v3 5/5] archive-zip: support files bigger than 4GB

From
René Scharfe <l.s.r@web.de>
Date
Apr 24, 2017, 17:33 UTC
Message-ID
<e53f1d3f-be3a-ac28-89eb-63011da64586@web.de>
In-Reply-To
<85f2b6d1-107b-0624-af82-92446f28269e@web.de>

Write a zip64 extended information extra field for big files as part of their local headers and as part of their central directory headers. Also write a zip64 version of the data descriptor in that case.

If we're streaming then we don't know the compressed size at the time we write the header. Deflate can end up making a file bigger instead of smaller if we're unlucky. Write a local zip64 header already for files with a size of 2GB or more in this case to be on the safe side.

Both sizes need to be included in the local zip64 header, but the extra field for the directory must only contain 64-bit equivalents for 32-bit values of 0xffffffff.

Signed-off-by: Rene Scharfe <l.s.r@web.de>
---
 archive-zip.c                   | 90 ++++++++++++++++++++++++++++++++++-------
 t/t5004-archive-corner-cases.sh |  2 +-
 2 files changed, 76 insertions(+), 16 deletions(-)
diff --git a/archive-zip.c b/archive-zip.c
index 7d6f2a85d0..44ed78f163 100644
--- a/archive-zip.c
+++ b/archive-zip.c
@@ -45,6 +45,14 @@ struct zip_data_desc {
 	unsigned char _end[1];
 };
 
+struct zip64_data_desc {
+	unsigned char magic[4];
+	unsigned char crc32[4];
+	unsigned char compressed_size[8];
+	unsigned char size[8];
+	unsigned char _end[1];
+};
+
 struct zip_dir_trailer {
 	unsigned char magic[4];
 	unsigned char disk[2];
@@ -65,6 +73,14 @@ struct zip_extra_mtime {
 	unsigned char _end[1];
 };
 
+struct zip64_extra {
+	unsigned char magic[2];
+	unsigned char extra_size[2];
+	unsigned char size[8];
+	unsigned char compressed_size[8];
+	unsigned char _end[1];
+};
+
 struct zip64_dir_trailer {
 	unsigned char magic[4];
 	unsigned char record_size[8];
@@ -94,11 +110,15 @@ struct zip64_dir_trailer_locator {
  */
 #define ZIP_LOCAL_HEADER_SIZE	offsetof(struct zip_local_header, _end)
 #define ZIP_DATA_DESC_SIZE	offsetof(struct zip_data_desc, _end)
+#define ZIP64_DATA_DESC_SIZE	offsetof(struct zip64_data_desc, _end)
 #define ZIP_DIR_HEADER_SIZE	offsetof(struct zip_dir_header, _end)
 #define ZIP_DIR_TRAILER_SIZE	offsetof(struct zip_dir_trailer, _end)
 #define ZIP_EXTRA_MTIME_SIZE	offsetof(struct zip_extra_mtime, _end)
 #define ZIP_EXTRA_MTIME_PAYLOAD_SIZE \
 	(ZIP_EXTRA_MTIME_SIZE - offsetof(struct zip_extra_mtime, flags))
+#define ZIP64_EXTRA_SIZE	offsetof(struct zip64_extra, _end)
+#define ZIP64_EXTRA_PAYLOAD_SIZE \
+	(ZIP64_EXTRA_SIZE - offsetof(struct zip64_extra, size))
 #define ZIP64_DIR_TRAILER_SIZE	offsetof(struct zip64_dir_trailer, _end)
 #define ZIP64_DIR_TRAILER_RECORD_SIZE \
 	(ZIP64_DIR_TRAILER_SIZE - \
@@ -202,13 +222,23 @@ static void write_zip_data_desc(unsigned long size,
 				unsigned long compressed_size,
 				unsigned long crc)
 {
-	struct zip_data_desc trailer;
-
-	copy_le32(trailer.magic, 0x08074b50);
-	copy_le32(trailer.crc32, crc);
-	copy_le32(trailer.compressed_size, compressed_size);
-	copy_le32(trailer.size, size);
-	write_or_die(1, &trailer, ZIP_DATA_DESC_SIZE);
+	if (size >= 0xffffffff || compressed_size >= 0xffffffff) {
+		struct zip64_data_desc trailer;
+		copy_le32(trailer.magic, 0x08074b50);
+		copy_le32(trailer.crc32, crc);
+		copy_le64(trailer.compressed_size, compressed_size);
+		copy_le64(trailer.size, size);
+		write_or_die(1, &trailer, ZIP64_DATA_DESC_SIZE);
+		zip_offset += ZIP64_DATA_DESC_SIZE;
+	} else {
+		struct zip_data_desc trailer;
+		copy_le32(trailer.magic, 0x08074b50);
+		copy_le32(trailer.crc32, crc);
+		copy_le32(trailer.compressed_size, compressed_size);
+		copy_le32(trailer.size, size);
+		write_or_die(1, &trailer, ZIP_DATA_DESC_SIZE);
+		zip_offset += ZIP_DATA_DESC_SIZE;
+	}
 }
 
 static void set_zip_header_data_desc(struct zip_local_header *header,
@@ -252,6 +282,9 @@ static int write_zip_entry(struct archiver_args *args,
 	struct zip_local_header header;
 	uintmax_t offset = zip_offset;
 	struct zip_extra_mtime extra;
+	struct zip64_extra extra64;
+	size_t header_extra_size = ZIP_EXTRA_MTIME_SIZE;
+	int need_zip64_extra = 0;
 	unsigned long attr2;
 	unsigned long compressed_size;
 	unsigned long crc;
@@ -344,21 +377,40 @@ static int write_zip_entry(struct archiver_args *args,
 	extra.flags[0] = 1;	/* just mtime */
 	copy_le32(extra.mtime, args->time);
 
+	if (size > 0xffffffff || compressed_size > 0xffffffff)
+		need_zip64_extra = 1;
+	if (stream && size > 0x7fffffff)
+		need_zip64_extra = 1;
+
 	copy_le32(header.magic, 0x04034b50);
 	copy_le16(header.version, 10);
 	copy_le16(header.flags, flags);
 	copy_le16(header.compression_method, method);
 	copy_le16(header.mtime, zip_time);
 	copy_le16(header.mdate, zip_date);
-	set_zip_header_data_desc(&header, size, compressed_size, crc);
+	if (need_zip64_extra) {
+		set_zip_header_data_desc(&header, 0xffffffff, 0xffffffff, crc);
+		header_extra_size += ZIP64_EXTRA_SIZE;
+	} else {
+		set_zip_header_data_desc(&header, size, compressed_size, crc);
+	}
 	copy_le16(header.filename_length, pathlen);
-	copy_le16(header.extra_length, ZIP_EXTRA_MTIME_SIZE);
+	copy_le16(header.extra_length, header_extra_size);
 	write_or_die(1, &header, ZIP_LOCAL_HEADER_SIZE);
 	zip_offset += ZIP_LOCAL_HEADER_SIZE;
 	write_or_die(1, path, pathlen);
 	zip_offset += pathlen;
 	write_or_die(1, &extra, ZIP_EXTRA_MTIME_SIZE);
 	zip_offset += ZIP_EXTRA_MTIME_SIZE;
+	if (need_zip64_extra) {
+		copy_le16(extra64.magic, 0x0001);
+		copy_le16(extra64.extra_size, ZIP64_EXTRA_PAYLOAD_SIZE);
+		copy_le64(extra64.size, size);
+		copy_le64(extra64.compressed_size, compressed_size);
+		write_or_die(1, &extra64, ZIP64_EXTRA_SIZE);
+		zip_offset += ZIP64_EXTRA_SIZE;
+	}
+
 	if (stream && method == 0) {
 		unsigned char buf[STREAM_BUFFER_SIZE];
 		ssize_t readlen;
@@ -381,7 +433,6 @@ static int write_zip_entry(struct archiver_args *args,
 		zip_offset += compressed_size;
 
 		write_zip_data_desc(size, compressed_size, crc);
-		zip_offset += ZIP_DATA_DESC_SIZE;
 	} else if (stream && method == 8) {
 		unsigned char buf[STREAM_BUFFER_SIZE];
 		ssize_t readlen;
@@ -437,7 +488,6 @@ static int write_zip_entry(struct archiver_args *args,
 		zip_offset += compressed_size;
 
 		write_zip_data_desc(size, compressed_size, crc);
-		zip_offset += ZIP_DATA_DESC_SIZE;
 	} else if (compressed_size > 0) {
 		write_or_die(1, out, compressed_size);
 		zip_offset += compressed_size;
@@ -446,8 +496,14 @@ static int write_zip_entry(struct archiver_args *args,
 	free(deflated);
 	free(buffer);
 
-	if (offset > 0xffffffff) {
-		zip64_dir_extra_payload_size += 8;
+	if (compressed_size > 0xffffffff || size > 0xffffffff ||
+	    offset > 0xffffffff) {
+		if (compressed_size >= 0xffffffff)
+			zip64_dir_extra_payload_size += 8;
+		if (size >= 0xffffffff)
+			zip64_dir_extra_payload_size += 8;
+		if (offset >= 0xffffffff)
+			zip64_dir_extra_payload_size += 8;
 		zip_dir_extra_size += 2 + 2 + zip64_dir_extra_payload_size;
 	}
 
@@ -459,8 +515,8 @@ static int write_zip_entry(struct archiver_args *args,
 	strbuf_add_le(&zip_dir, 2, zip_time);
 	strbuf_add_le(&zip_dir, 2, zip_date);
 	strbuf_add_le(&zip_dir, 4, crc);
-	strbuf_add_le(&zip_dir, 4, compressed_size);
-	strbuf_add_le(&zip_dir, 4, size);
+	strbuf_add_le(&zip_dir, 4, clamp32(compressed_size));
+	strbuf_add_le(&zip_dir, 4, clamp32(size));
 	strbuf_add_le(&zip_dir, 2, pathlen);
 	strbuf_add_le(&zip_dir, 2, zip_dir_extra_size);
 	strbuf_add_le(&zip_dir, 2, 0);		/* comment length */
@@ -473,6 +529,10 @@ static int write_zip_entry(struct archiver_args *args,
 	if (zip64_dir_extra_payload_size) {
 		strbuf_add_le(&zip_dir, 2, 0x0001);	/* magic */
 		strbuf_add_le(&zip_dir, 2, zip64_dir_extra_payload_size);
+		if (size >= 0xffffffff)
+			strbuf_add_le(&zip_dir, 8, size);
+		if (compressed_size >= 0xffffffff)
+			strbuf_add_le(&zip_dir, 8, compressed_size);
 		if (offset >= 0xffffffff)
 			strbuf_add_le(&zip_dir, 8, offset);
 	}
diff --git a/t/t5004-archive-corner-cases.sh b/t/t5004-archive-corner-cases.sh
index 0ac94b5cc9..a6875dfdb1 100755
--- a/t/t5004-archive-corner-cases.sh
+++ b/t/t5004-archive-corner-cases.sh
@@ -178,7 +178,7 @@ test_expect_success EXPENSIVE,UNZIP 'zip archive bigger than 4GB' '
 	"$GIT_UNZIP" -t many-big.zip
 '
 
-test_expect_failure EXPENSIVE,UNZIP 'zip archive with files bigger than 4GB' '
+test_expect_success EXPENSIVE,UNZIP,ZIPINFO 'zip archive with files bigger than 4GB' '
 	# Pack created with:
 	#   dd if=/dev/zero of=file bs=1M count=4100 && git hash-object -w file
 	mkdir -p .git/objects/pack &&

-- 
2.12.2
Previous: Peter KreftingNext: Keith Goldfarb
Message 29 of 44 in “Git archive doesn't fully support zip64”
  1. Keith GoldfarbApr 21, 2017
  2. archive-zip: Add zip64 headers when file size is too large for 32 bitsPeter Krefting, Apr 22, 2017
  3. Johannes SixtApr 22, 2017
  4. archive-zip: Add zip64 headers when file size is too large for 32 bitsPeter Krefting, Apr 22, 2017
  5. Johannes SixtApr 23, 2017
  6. Peter KreftingApr 23, 2017
  7. Johannes SixtApr 23, 2017
  8. Peter KreftingApr 24, 2017
  9. René ScharfeApr 24, 2017
  10. 0/5 archive-zip: support files and archives bigger than 4GBRené Scharfe, Apr 24, 2017
  11. 1/5 archive-zip: add tests for big ZIP archivesRené Scharfe, Apr 24, 2017
  12. 2/5 archive-zip: use strbuf for ZIP directoryRené Scharfe, Apr 24, 2017
  13. Junio C HamanoApr 25, 2017
  14. René ScharfeApr 25, 2017
  15. 3/5 archive-zip: write ZIP dir entry directly to strbufRené Scharfe, Apr 24, 2017
  16. 4/5 archive-zip: support archives bigger than 4GBRené Scharfe, Apr 24, 2017
  17. Peter KreftingApr 24, 2017
  18. René ScharfeApr 24, 2017
  19. René ScharfeApr 24, 2017
  20. Johannes SixtApr 24, 2017
  21. René ScharfeApr 24, 2017
  22. Peter KreftingApr 25, 2017
  23. René ScharfeApr 25, 2017
  24. Peter KreftingApr 26, 2017
  25. René ScharfeApr 26, 2017
  26. Peter KreftingApr 27, 2017
  27. René ScharfeApr 27, 2017
  28. Peter KreftingApr 28, 2017
  29. 5/5 archive-zip: support files bigger than 4GBRené Scharfe, Apr 24, 2017
  30. Keith GoldfarbApr 24, 2017
  31. Junio C HamanoApr 25, 2017
  32. René ScharfeApr 25, 2017
  33. Torsten BögershausenApr 29, 2017
  34. René ScharfeApr 29, 2017
  35. Torsten BögershausenApr 30, 2017
  36. René ScharfeApr 30, 2017
  37. Torsten BögershausenApr 30, 2017
  38. Johannes SixtApr 30, 2017
  39. René ScharfeApr 30, 2017
  40. Junio C HamanoApr 30, 2017
  41. René ScharfeMay 1, 2017
  42. René ScharfeApr 23, 2017
  43. Peter KreftingApr 23, 2017
  44. Johannes SixtApr 23, 2017

Read the whole thread, see it on lore, or plain text.

$ cat FOOTERMessages come from the public archive at lore.kernel.org/git, fetched every hour. The front page is chosen and written each morning by an AI editor and can be wrong; the threads themselves are the record. About and API. For agents: an MCP server at https://gitlist.dev/mcp, and any thread, story or person page as Markdown by adding .md to its URL (or sending Accept: text/markdown). Details in /llms.txt.