git/list[1] front-page[2] threads[3] people[4] search[5] about
 

[PATCH 3/3] archive-zip: support more than 65535 entries

From
René Scharfe <l.s.r@web.de>
Date
Aug 22, 2015, 19:06 UTC
Message-ID
<55D8C845.3090908@web.de>
In-Reply-To
<20150813022545.30116.44787@localhost>

Support more than 65535 entries cleanly by writing a "zip64 end of central directory record" (with a 64-bit field for the number of entries) before the usual "end of central directory record" (which contains only a 16-bit field). InfoZIP's zip does the same. Archives with 65535 or less entries are not affected.

Programs that extract all files like InfoZIP's zip and 7-Zip ignored the field and could extract all files already. Software that relies on the ZIP file directory to show a list of contained files quickly to simulate to normal directory like Windows' built-in ZIP functionality only saw a subset of the included files.

Windows supports ZIP64 since Vista according to https://en.wikipedia.org/wiki/Zip_%28file_format%29#ZIP64.

Suggested-by: Johannes Schauer <josch@debian.org>
Signed-off-by: Rene Scharfe <l.s.r@web.de>
---
 archive-zip.c                   | 93 +++++++++++++++++++++++++++++++++++++++--
 t/t5004-archive-corner-cases.sh |  2 +-
 2 files changed, 91 insertions(+), 4 deletions(-)
diff --git a/archive-zip.c b/archive-zip.c
index 2a76156..9db4735 100644
--- a/archive-zip.c
+++ b/archive-zip.c
@@ -16,7 +16,9 @@ static unsigned int zip_dir_size;
 
 static unsigned int zip_offset;
 static unsigned int zip_dir_offset;
-static unsigned int zip_dir_entries;
+static uint64_t zip_dir_entries;
+
+static unsigned int max_creator_version;
 
 #define ZIP_DIRECTORY_MIN_SIZE	(1024 * 1024)
 #define ZIP_STREAM	(1 <<  3)
@@ -86,6 +88,28 @@ struct zip_extra_mtime {
 	unsigned char _end[1];
 };
 
+struct zip64_dir_trailer {
+	unsigned char magic[4];
+	unsigned char record_size[8];
+	unsigned char creator_version[2];
+	unsigned char version[2];
+	unsigned char disk[4];
+	unsigned char directory_start_disk[4];
+	unsigned char entries_on_this_disk[8];
+	unsigned char entries[8];
+	unsigned char size[8];
+	unsigned char offset[8];
+	unsigned char _end[1];
+};
+
+struct zip64_dir_trailer_locator {
+	unsigned char magic[4];
+	unsigned char disk[4];
+	unsigned char offset[8];
+	unsigned char number_of_disks[4];
+	unsigned char _end[1];
+};
+
 /*
  * On ARM, padding is added at the end of the struct, so a simple
  * sizeof(struct ...) reports two bytes more than the payload size
@@ -98,6 +122,12 @@ struct zip_extra_mtime {
 #define ZIP_EXTRA_MTIME_SIZE	offsetof(struct zip_extra_mtime, _end)
 #define ZIP_EXTRA_MTIME_PAYLOAD_SIZE \
 	(ZIP_EXTRA_MTIME_SIZE - offsetof(struct zip_extra_mtime, flags))
+#define ZIP64_DIR_TRAILER_SIZE	offsetof(struct zip64_dir_trailer, _end)
+#define ZIP64_DIR_TRAILER_RECORD_SIZE \
+	(ZIP64_DIR_TRAILER_SIZE - \
+	 offsetof(struct zip64_dir_trailer, creator_version))
+#define ZIP64_DIR_TRAILER_LOCATOR_SIZE \
+	offsetof(struct zip64_dir_trailer_locator, _end)
 
 static void copy_le16(unsigned char *dest, unsigned int n)
 {
@@ -113,6 +143,31 @@ static void copy_le32(unsigned char *dest, unsigned int n)
 	dest[3] = 0xff & (n >> 030);
 }
 
+static void copy_le64(unsigned char *dest, uint64_t n)
+{
+	dest[0] = 0xff & n;
+	dest[1] = 0xff & (n >> 010);
+	dest[2] = 0xff & (n >> 020);
+	dest[3] = 0xff & (n >> 030);
+	dest[4] = 0xff & (n >> 040);
+	dest[5] = 0xff & (n >> 050);
+	dest[6] = 0xff & (n >> 060);
+	dest[7] = 0xff & (n >> 070);
+}
+
+static uint64_t clamp_max(uint64_t n, uint64_t max, int *clamped)
+{
+	if (n <= max)
+		return n;
+	*clamped = 1;
+	return max;
+}
+
+static void copy_le16_clamp(unsigned char *dest, uint64_t n, int *clamped)
+{
+	copy_le16(dest, clamp_max(n, 0xffff, clamped));
+}
+
 static void *zlib_deflate_raw(void *data, unsigned long size,
 			      int compression_level,
 			      unsigned long *compressed_size)
@@ -282,6 +337,9 @@ static int write_zip_entry(struct archiver_args *args,
 				sha1_to_hex(sha1));
 	}
 
+	if (creator_version > max_creator_version)
+		max_creator_version = creator_version;
+
 	if (buffer && method == 8) {
 		out = deflated = zlib_deflate_raw(buffer, size,
 						  args->compression_level,
@@ -439,20 +497,49 @@ static int write_zip_entry(struct archiver_args *args,
 	return 0;
 }
 
+static void write_zip64_trailer(void)
+{
+	struct zip64_dir_trailer trailer64;
+	struct zip64_dir_trailer_locator locator64;
+
+	copy_le32(trailer64.magic, 0x06064b50);
+	copy_le64(trailer64.record_size, ZIP64_DIR_TRAILER_RECORD_SIZE);
+	copy_le16(trailer64.creator_version, max_creator_version);
+	copy_le16(trailer64.version, 45);
+	copy_le32(trailer64.disk, 0);
+	copy_le32(trailer64.directory_start_disk, 0);
+	copy_le64(trailer64.entries_on_this_disk, zip_dir_entries);
+	copy_le64(trailer64.entries, zip_dir_entries);
+	copy_le64(trailer64.size, zip_dir_offset);
+	copy_le64(trailer64.offset, zip_offset);
+
+	copy_le32(locator64.magic, 0x07064b50);
+	copy_le32(locator64.disk, 0);
+	copy_le64(locator64.offset, zip_offset + zip_dir_offset);
+	copy_le32(locator64.number_of_disks, 1);
+
+	write_or_die(1, &trailer64, ZIP64_DIR_TRAILER_SIZE);
+	write_or_die(1, &locator64, ZIP64_DIR_TRAILER_LOCATOR_SIZE);
+}
+
 static void write_zip_trailer(const unsigned char *sha1)
 {
 	struct zip_dir_trailer trailer;
+	int clamped = 0;
 
 	copy_le32(trailer.magic, 0x06054b50);
 	copy_le16(trailer.disk, 0);
 	copy_le16(trailer.directory_start_disk, 0);
-	copy_le16(trailer.entries_on_this_disk, zip_dir_entries);
-	copy_le16(trailer.entries, zip_dir_entries);
+	copy_le16_clamp(trailer.entries_on_this_disk, zip_dir_entries,
+			&clamped);
+	copy_le16_clamp(trailer.entries, zip_dir_entries, &clamped);
 	copy_le32(trailer.size, zip_dir_offset);
 	copy_le32(trailer.offset, zip_offset);
 	copy_le16(trailer.comment_length, sha1 ? GIT_SHA1_HEXSZ : 0);
 
 	write_or_die(1, zip_dir, zip_dir_offset);
+	if (clamped)
+		write_zip64_trailer();
 	write_or_die(1, &trailer, ZIP_DIR_TRAILER_SIZE);
 	if (sha1)
 		write_or_die(1, sha1_to_hex(sha1), GIT_SHA1_HEXSZ);
diff --git a/t/t5004-archive-corner-cases.sh b/t/t5004-archive-corner-cases.sh
index c6bd729..cca2338 100755
--- a/t/t5004-archive-corner-cases.sh
+++ b/t/t5004-archive-corner-cases.sh
@@ -122,7 +122,7 @@ test_lazy_prereq ZIPINFO '
 	test "x$n" = "x0"
 '
 
-test_expect_failure ZIPINFO 'zip archive with many entries' '
+test_expect_success ZIPINFO 'zip archive with many entries' '
 	# add a directory with 256 files
 	mkdir 00 &&
 	for a in 0 1 2 3 4 5 6 7 8 9 a b c d e f

-- 
2.5.0
Previous: René ScharfeNext: Duy Nguyen
Message 15 of 18 in “bug: git-archive does not use the zip64 extension for archives with more than 16k entries”
  1. Johannes SchauerAug 11, 2015
  2. René ScharfeAug 12, 2015
  3. Johannes SchauerAug 13, 2015
  4. 1/3 t5004: test ZIP archives with many entriesRené Scharfe, Aug 22, 2015
  5. Eric SunshineAug 23, 2015
  6. René ScharfeAug 23, 2015
  7. Eric Sunshine mail delivery failureRené Scharfe, Aug 23, 2015
  8. Johannes LöthbergAug 23, 2015
  9. Eric SunshineAug 23, 2015
  10. Eric SunshineAug 23, 2015
  11. Junio C HamanoAug 28, 2015
  12. Junio C HamanoAug 28, 2015
  13. Eric SunshineAug 28, 2015
  14. 2/3 archive-zip: use a local variable to store the creator versionRené Scharfe, Aug 22, 2015
  15. 3/3 archive-zip: support more than 65535 entriesRené Scharfe, Aug 22, 2015
  16. Duy NguyenAug 15, 2015
  17. Eric SunshineAug 23, 2015
  18. Eric SunshineAug 23, 2015

Read the whole thread, see it on lore, or plain text.

$ cat FOOTERMessages come from the public archive at lore.kernel.org/git, fetched every hour. The front page is chosen and written each morning by an AI editor and can be wrong; the threads themselves are the record. About and API. For agents: an MCP server at https://gitlist.dev/mcp, and any thread, story or person page as Markdown by adding .md to its URL (or sending Accept: text/markdown). Details in /llms.txt.