git/list[1] front-page[2] threads[3] people[4] search[5] about
 

[PATCH 2/3] sha1_file: add the ability to parse objects in "pack file format"

From
Linus Torvalds <torvalds@osdl.org>
Date
Jul 11, 2006, 17:12 UTC
Message-ID
<Pine.LNX.4.64.0607111010320.5623@g5.osdl.org>
In-Reply-To
<Pine.LNX.4.64.0607111004360.5623@g5.osdl.org>

The pack-file format is slightly different from the traditional git object format, in that it has a much denser binary header encoding.

The traditional format uses an ASCII string with type and length information, which is somewhat wasteful.

Signed-off-by: Linus Torvalds <torvalds@osdl.org>
---

This should probably be applied to the main tree asap if we think this is at all a worthwhile exercise. But somebody should verify that I got the format right first!

 sha1_file.c |   66 +++++++++++++++++++++++++++++++++++++++++++++++++++--------
 1 files changed, 57 insertions(+), 9 deletions(-)
diff --git a/sha1_file.c b/sha1_file.c
index 8734d50..ca5f0c0 100644
--- a/sha1_file.c
+++ b/sha1_file.c
@@ -697,9 +697,9 @@ static int unpack_sha1_header(z_stream *
 	return inflate(stream, 0);
 }
 
-static void *unpack_sha1_rest(z_stream *stream, void *buffer, unsigned long size)
+static void *unpack_sha1_rest(z_stream *stream, void *buffer, unsigned long size, unsigned int hdrlen)
 {
-	int bytes = strlen(buffer) + 1;
+	int bytes = hdrlen;
 	unsigned char *buf = xmalloc(1+size);
 
 	memcpy(buf, (char *) buffer + bytes, stream->total_out - bytes);
@@ -720,9 +720,9 @@ static void *unpack_sha1_rest(z_stream *
  * too permissive for what we want to check. So do an anal
  * object header parse by hand.
  */
-static int parse_sha1_header(char *hdr, char *type, unsigned long *sizep)
+static int parse_ascii_sha1_header(char *hdr, char *type, unsigned long *sizep)
 {
-	int i;
+	int i, bytes = 0;
 	unsigned long size;
 
 	/*
@@ -733,6 +733,7 @@ static int parse_sha1_header(char *hdr, 
 	i = 10;
 	for (;;) {
 		char c = *hdr++;
+		bytes++;
 		if (c == ' ')
 			break;
 		if (!--i)
@@ -746,6 +747,7 @@ static int parse_sha1_header(char *hdr, 
 	 * decimal format (ie "010" is not valid).
 	 */
 	size = *hdr++ - '0';
+	bytes++;
 	if (size > 9)
 		return -1;
 	if (size) {
@@ -754,6 +756,7 @@ static int parse_sha1_header(char *hdr, 
 			if (c > 9)
 				break;
 			hdr++;
+			bytes++;
 			size = size * 10 + c;
 		}
 	}
@@ -762,20 +765,65 @@ static int parse_sha1_header(char *hdr, 
 	/*
 	 * The length must be followed by a zero byte
 	 */
-	return *hdr ? -1 : 0;
+	bytes++;
+	if (*hdr)
+		bytes = -1;
+	return bytes;
+}
+
+static int parse_binary_sha1_header(char *hdr, char *type, unsigned long *sizep)
+{
+	unsigned char c;
+	int bytes = 1;
+	unsigned long size;
+	unsigned object_type, bits;
+	static const char *typename[8] = {
+		NULL,	/* OBJ_EXT */
+		"commit", "tree", "blob", "tag",
+		NULL, NULL, NULL 
+	};	
+
+	c = *hdr++;
+	object_type = (c >> 4) & 7;
+	if (!typename[object_type])
+		return -1;
+	strcpy(type, typename[object_type]);
+	size = c & 15;
+	bits = 4;
+	while (!(c & 0x80)) {
+		if (bits >= 8*sizeof(unsigned long))
+			return -1;
+		c = *hdr++;
+		size += (unsigned long) (c & 0x7f) << bits;
+		bytes++;
+		bits += 7;
+	}
+	*sizep = size;
+	return bytes;
+}
+
+static int parse_sha1_header(char *hdr, char *type, unsigned long *sizep)
+{
+	int retval = parse_ascii_sha1_header(hdr, type, sizep);
+	if (retval < 0)
+		retval = parse_binary_sha1_header(hdr, type, sizep);
+	return retval;
 }
 
 void * unpack_sha1_file(void *map, unsigned long mapsize, char *type, unsigned long *size)
 {
-	int ret;
+	int ret, hdrlen;
 	z_stream stream;
 	char hdr[8192];
 
 	ret = unpack_sha1_header(&stream, map, mapsize, hdr, sizeof(hdr));
-	if (ret < Z_OK || parse_sha1_header(hdr, type, size) < 0)
+	if (ret < Z_OK)
+		return NULL;
+	hdrlen = parse_sha1_header(hdr, type, size);
+	if (hdrlen < 0)
 		return NULL;
 
-	return unpack_sha1_rest(&stream, hdr, *size);
+	return unpack_sha1_rest(&stream, hdr, *size, hdrlen);
 }
 
 /* forward declaration for a mutually recursive function */
@@ -1192,7 +1240,7 @@ struct packed_git *find_sha1_pack(const 
 
 int sha1_object_info(const unsigned char *sha1, char *type, unsigned long *sizep)
 {
-	int status;
+	int status, hdrlen;
 	unsigned long mapsize, size;
 	void *map;
 	z_stream stream;
Previous: Linus TorvaldsNext: Johannes Schindelin
Message 13 of 37 in “Revisiting large binary files issue.”
  1. Carl BaldwinJul 10, 2006
  2. Junio C HamanoJul 10, 2006
  3. Peter BaumannJul 11, 2006
  4. Linus TorvaldsJul 10, 2006
  5. [RFC]: Pack-file object format for individual objects (Was: Revisiting large binary files issue.)sf, Jul 11, 2006
  6. Linus TorvaldsJul 11, 2006
  7. sfJul 11, 2006
  8. Linus TorvaldsJul 11, 2006
  9. Linus TorvaldsJul 11, 2006
  10. Carl BaldwinJul 11, 2006
  11. Linus TorvaldsJul 11, 2006
  12. 1/3 Make the unpacked object header functions static to sha1_file.cLinus Torvalds, Jul 11, 2006
  13. 2/3 sha1_file: add the ability to parse objects in "pack file format"Linus Torvalds, Jul 11, 2006
  14. Johannes SchindelinJul 11, 2006
  15. Linus TorvaldsJul 11, 2006
  16. Johannes SchindelinJul 11, 2006
  17. Linus TorvaldsJul 11, 2006
  18. Johannes SchindelinJul 11, 2006
  19. Junio C HamanoJul 11, 2006
  20. sfJul 11, 2006
  21. Linus TorvaldsJul 11, 2006
  22. sfJul 11, 2006
  23. Junio C HamanoJul 11, 2006
  24. Linus TorvaldsJul 12, 2006
  25. Johannes SchindelinJul 12, 2006
  26. Linus TorvaldsJul 12, 2006
  27. Linus TorvaldsJul 12, 2006
  28. Junio C HamanoJul 12, 2006
  29. Linus TorvaldsJul 12, 2006
  30. Junio C HamanoJul 12, 2006
  31. Linus TorvaldsJul 12, 2006
  32. Peter BaumannJul 12, 2006
  33. Junio C HamanoJul 12, 2006
  34. Peter BaumannJul 12, 2006
  35. Linus TorvaldsJul 12, 2006
  36. Junio C HamanoJul 12, 2006
  37. 3/3 Enable the new binary header format for unpacked objectsLinus Torvalds, Jul 11, 2006

Read the whole thread, see it on lore, or plain text.

$ cat FOOTERMessages come from the public archive at lore.kernel.org/git, fetched every hour. The front page is chosen and written each morning by an AI editor and can be wrong; the threads themselves are the record. About and API. For agents: an MCP server at https://gitlist.dev/mcp, and any thread, story or person page as Markdown by adding .md to its URL (or sending Accept: text/markdown). Details in /llms.txt.