git/list[1] front-page[2] threads[3] people[4] search[5] about
 

[PATCH 13/21] unpack-objects: read v4 dictionaries

From
Nguyễn Thái Ngọc Duy <pclouds@gmail.com>
Date
Sep 11, 2013, 06:06 UTC
Message-ID
<1378879582-15372-14-git-send-email-pclouds@gmail.com>
In-Reply-To
<1378879582-15372-1-git-send-email-pclouds@gmail.com>
Signed-off-by: Nguyễn Thái Ngọc Duy <pclouds@gmail.com>
---
 builtin/unpack-objects.c | 50 ++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 50 insertions(+)
diff --git a/builtin/unpack-objects.c b/builtin/unpack-objects.c
index c9eb31d..1a3c30e 100644
--- a/builtin/unpack-objects.c
+++ b/builtin/unpack-objects.c
@@ -10,6 +10,7 @@
 #include "tree-walk.h"
 #include "progress.h"
 #include "decorate.h"
+#include "packv4-parse.h"
 #include "fsck.h"
 
 static int dry_run, quiet, recover, has_errors, strict;
@@ -20,7 +21,10 @@ static unsigned char buffer[4096];
 static unsigned int offset, len;
 static off_t consumed_bytes;
 static git_SHA_CTX ctx;
+
 static int packv4;
+static unsigned char *sha1_table;
+static struct packv4_dict *name_dict, *path_dict;
 
 /*
  * When running under --strict mode, objects whose reachability are
@@ -89,6 +93,28 @@ static void use(int bytes)
 	consumed_bytes += bytes;
 }
 
+static inline void *fill_and_use(int bytes)
+{
+	void *p = fill(bytes);
+	use(bytes);
+	return p;
+}
+
+static uintmax_t read_varint(void)
+{
+	unsigned char c = *(char*)fill_and_use(1);
+	uintmax_t val = c & 127;
+	while (c & 128) {
+		val += 1;
+		if (!val || MSB(val, 7))
+			die("offset overflow in read_varint at %lu",
+			    (unsigned long)consumed_bytes);
+		c = *(char*)fill_and_use(1);
+		val = (val << 7) + (c & 127);
+	}
+	return val;
+}
+
 static void *get_data(unsigned long size)
 {
 	git_zstream stream;
@@ -470,6 +496,20 @@ static int unpack_one(unsigned nr)
 	return 0;
 }
 
+static struct packv4_dict *read_dict(void)
+{
+	unsigned long size;
+	unsigned char *data;
+	struct packv4_dict *dict;
+
+	size = read_varint();
+	data = get_data(size);
+	dict = pv4_create_dict(data, size);
+	if (!dict)
+		die("unable to parse dictionary");
+	return dict;
+}
+
 static void unpack_all(void)
 {
 	int i;
@@ -486,6 +526,16 @@ static void unpack_all(void)
 	packv4 = ntohl(hdr->hdr_version) == 4;
 	use(sizeof(struct pack_header));
 
+	if (packv4) {
+		sha1_table = xmalloc(20 * nr_objects);
+		for (i = 0; i < nr_objects; i++) {
+			unsigned char *p = sha1_table + i * 20;
+			hashcpy(p, fill_and_use(20));
+		}
+		name_dict = read_dict();
+		path_dict = read_dict();
+	}
+
 	if (!quiet)
 		progress = start_progress("Unpacking objects", nr_objects);
 	obj_list = xcalloc(nr_objects, sizeof(*obj_list));
-- 
1.8.2.82.gc24b958
Previous: Nguyễn Thái Ngọc DuyNext: Nguyễn Thái Ngọc Duy
Message 20 of 36 in “[PULL REQUEST] initial pack v4 support”
  1. Nicolas PitreSep 9, 2013
  2. Junio C HamanoSep 9, 2013
  3. Junio C HamanoSep 10, 2013
  4. Nicolas PitreSep 10, 2013
  5. Junio C HamanoSep 10, 2013
  6. Nicolas PitreSep 10, 2013
  7. 00/21 np/pack-v4 updatesNguyễn Thái Ngọc Duy, Sep 11, 2013
  8. 01/21 fixup! pack-objects: prepare SHA-1 table in v4Nguyễn Thái Ngọc Duy, Sep 11, 2013
  9. 02/21 fixup! pack-objects: support writing pack v4Nguyễn Thái Ngọc Duy, Sep 11, 2013
  10. 03/21 fixup! pack v4: support "end-of-pack" indicator in index-pack and pack-objectsNguyễn Thái Ngọc Duy, Sep 11, 2013
  11. 04/21 fixup! index-pack: parse v4 header and dictionariesNguyễn Thái Ngọc Duy, Sep 11, 2013
  12. 05/21 fixup! index-pack: record all delta bases in v4 (tree and ref-delta)Nguyễn Thái Ngọc Duy, Sep 11, 2013
  13. 06/21 pack v4: lift dict size check in load_dict()Nguyễn Thái Ngọc Duy, Sep 11, 2013
  14. 07/21 pack v4: move pv4 objhdr parsing code to packv4-parse.cNguyễn Thái Ngọc Duy, Sep 11, 2013
  15. 08/21 pack-objects: respect compression level in v4Nguyễn Thái Ngọc Duy, Sep 11, 2013
  16. 09/21 pack-objects: recognize v4 as pack sourceNguyễn Thái Ngọc Duy, Sep 11, 2013
  17. 10/21 pack v4: add a note that streaming does not support OBJ_PV4_*Nguyễn Thái Ngọc Duy, Sep 11, 2013
  18. 11/21 unpack-objects: report missing object nameNguyễn Thái Ngọc Duy, Sep 11, 2013
  19. 12/21 unpack-objects: recognize end-of-pack in v4 thin packNguyễn Thái Ngọc Duy, Sep 11, 2013
  20. 13/21 unpack-objects: read v4 dictionariesNguyễn Thái Ngọc Duy, Sep 11, 2013
  21. 14/21 unpack-objects: decode v4 object headerNguyễn Thái Ngọc Duy, Sep 11, 2013
  22. 15/21 unpack-objects: decode v4 ref-deltaNguyễn Thái Ngọc Duy, Sep 11, 2013
  23. 16/21 unpack-objects: decode v4 commitsNguyễn Thái Ngọc Duy, Sep 11, 2013
  24. 17/21 unpack-objects: allow to save processed bytes to a bufferNguyễn Thái Ngọc Duy, Sep 11, 2013
  25. 18/21 unpack-objects: decode v4 treesNguyễn Thái Ngọc Duy, Sep 11, 2013
  26. 19/21 index-pack, pack-objects: allow creating .idx v2 with .pack v4Nguyễn Thái Ngọc Duy, Sep 11, 2013
  27. Nicolas PitreSep 11, 2013
  28. 20/21 show-index: acknowledge that it does not read .idx v3Nguyễn Thái Ngọc Duy, Sep 11, 2013
  29. Nicolas PitreSep 11, 2013
  30. 21/21 t1050, t5500: replace the use of "show-index|wc -l" with verify-packNguyễn Thái Ngọc Duy, Sep 11, 2013
  31. Duy NguyenSep 11, 2013
  32. Nicolas PitreSep 11, 2013
  33. Duy NguyenSep 12, 2013
  34. Nicolas PitreSep 12, 2013
  35. Duy NguyenSep 13, 2013
  36. Nicolas PitreSep 11, 2013

Read the whole thread, see it on lore, or plain text.

$ cat FOOTERMessages come from the public archive at lore.kernel.org/git, fetched every hour. The front page is chosen and written each morning by an AI editor and can be wrong; the threads themselves are the record. About and API. For agents: an MCP server at https://gitlist.dev/mcp, and any thread, story or person page as Markdown by adding .md to its URL (or sending Accept: text/markdown). Details in /llms.txt.