git/list[1] front-page[2] threads[3] people[4] search[5] about
 

[PATCH 04/38] pack v4: add tree entry mode support to dictionary entries

From
Nicolas Pitre <nico@fluxnic.net>
Date
Sep 5, 2013, 06:19 UTC
Message-ID
<1378362001-1738-5-git-send-email-nico@fluxnic.net>
In-Reply-To
<1378362001-1738-1-git-send-email-nico@fluxnic.net>

Augment dict entries with a 16-bit prefix in order to store the file mode value of tree entries.

Signed-off-by: Nicolas Pitre <nico@fluxnic.net>
---
 packv4-create.c | 56 ++++++++++++++++++++++++++++++++++++--------------------
 1 file changed, 36 insertions(+), 20 deletions(-)
diff --git a/packv4-create.c b/packv4-create.c
index 00762a5..eccd9fc 100644
--- a/packv4-create.c
+++ b/packv4-create.c
@@ -14,11 +14,12 @@
 
 struct data_entry {
 	unsigned offset;
+	unsigned size;
 	unsigned hits;
 };
 
 struct dict_table {
-	char *data;
+	unsigned char *data;
 	unsigned ptr;
 	unsigned size;
 	struct data_entry *entry;
@@ -41,18 +42,19 @@ void destroy_dict_table(struct dict_table *t)
 	free(t);
 }
 
-static int locate_entry(struct dict_table *t, const char *str)
+static int locate_entry(struct dict_table *t, const void *data, int size)
 {
-	int i = 0;
-	const unsigned char *s = (const unsigned char *) str;
+	int i = 0, len = size;
+	const unsigned char *p = data;
 
-	while (*s)
-		i = i * 111 + *s++;
+	while (len--)
+		i = i * 111 + *p++;
 	i = (unsigned)i % t->hash_size;
 
 	while (t->hash[i]) {
 		unsigned n = t->hash[i] - 1;
-		if (!strcmp(str, t->data + t->entry[n].offset))
+		if (t->entry[n].size == size &&
+		    memcmp(t->data + t->entry[n].offset, data, size) == 0)
 			return n;
 		if (++i >= t->hash_size)
 			i = 0;
@@ -71,23 +73,28 @@ static void rehash_entries(struct dict_table *t)
 	memset(t->hash, 0, t->hash_size * sizeof(*t->hash));
 
 	for (n = 0; n < t->nb_entries; n++) {
-		int i = locate_entry(t, t->data + t->entry[n].offset);
+		int i = locate_entry(t, t->data + t->entry[n].offset,
+					t->entry[n].size);
 		if (i < 0)
 			t->hash[-1 - i] = n + 1;
 	}
 }
 
-int dict_add_entry(struct dict_table *t, const char *str)
+int dict_add_entry(struct dict_table *t, int val, const char *str)
 {
-	int i, len = strlen(str) + 1;
+	int i, val_len = 2, str_len = strlen(str) + 1;
 
-	if (t->ptr + len >= t->size) {
-		t->size = (t->size + len + 1024) * 3 / 2;
+	if (t->ptr + val_len + str_len > t->size) {
+		t->size = (t->size + val_len + str_len + 1024) * 3 / 2;
 		t->data = xrealloc(t->data, t->size);
 	}
-	memcpy(t->data + t->ptr, str, len);
 
-	i = (t->nb_entries) ? locate_entry(t, t->data + t->ptr) : -1;
+	t->data[t->ptr] = val >> 8;
+	t->data[t->ptr + 1] = val;
+	memcpy(t->data + t->ptr + val_len, str, str_len);
+
+	i = (t->nb_entries) ?
+		locate_entry(t, t->data + t->ptr, val_len + str_len) : -1;
 	if (i >= 0) {
 		t->entry[i].hits++;
 		return i;
@@ -98,8 +105,9 @@ int dict_add_entry(struct dict_table *t, const char *str)
 		t->entry = xrealloc(t->entry, t->max_entries * sizeof(*t->entry));
 	}
 	t->entry[t->nb_entries].offset = t->ptr;
+	t->entry[t->nb_entries].size = val_len + str_len;
 	t->entry[t->nb_entries].hits = 1;
-	t->ptr += len + 1;
+	t->ptr += val_len + str_len;
 	t->nb_entries++;
 
 	if (t->hash_size * 3 <= t->nb_entries * 4)
@@ -139,7 +147,8 @@ static int add_tree_dict_entries(void *buf, unsigned long size)
 
 	init_tree_desc(&desc, buf, size);
 	while (tree_entry(&desc, &name_entry))
-		dict_add_entry(tree_path_table, name_entry.path);
+		dict_add_entry(tree_path_table, name_entry.mode,
+			       name_entry.path);
 	return 0;
 }
 
@@ -148,10 +157,16 @@ void dict_dump(struct dict_table *t)
 	int i;
 
 	sort_dict_entries_by_hits(t);
-	for (i = 0; i < t->nb_entries; i++)
-		printf("%d\t%s\n",
-			t->entry[i].hits,
-			t->data + t->entry[i].offset);
+	for (i = 0; i < t->nb_entries; i++) {
+		int16_t val;
+		uint16_t uval;
+		val = t->data[t->entry[i].offset] << 8;
+		val |= t->data[t->entry[i].offset + 1];
+		uval = val;
+		printf("%d\t%d\t%o\t%s\n",
+			t->entry[i].hits, val, uval,
+			t->data + t->entry[i].offset + 2);
+	}
 }
 
 struct idx_entry
@@ -170,6 +185,7 @@ static int sort_by_offset(const void *e1, const void *e2)
 		return 1;
 	return 0;
 }
+
 static int create_pack_dictionaries(struct packed_git *p)
 {
 	uint32_t nr_objects, i;
-- 
1.8.4.38.g317e65b
Previous: Nicolas PitreNext: Nicolas Pitre
Message 5 of 124 in “pack version 4 basic functionalities”
  1. 00/38 pack version 4 basic functionalitiesNicolas Pitre, Sep 5, 2013
  2. 01/38 pack v4: initial pack dictionary structure and codeNicolas Pitre, Sep 5, 2013
  3. 02/38 export packed_object_info()Nicolas Pitre, Sep 5, 2013
  4. 03/38 pack v4: scan tree objectsNicolas Pitre, Sep 5, 2013
  5. 04/38 pack v4: add tree entry mode support to dictionary entriesNicolas Pitre, Sep 5, 2013
  6. 05/38 pack v4: add commit object parsingNicolas Pitre, Sep 5, 2013
  7. SZEDER GáborSep 5, 2013
  8. Nicolas PitreSep 5, 2013
  9. 06/38 pack v4: split the object list and dictionary creationNicolas Pitre, Sep 5, 2013
  10. 07/38 pack v4: move to struct pack_idx_entry and get rid of our own struct idx_entryNicolas Pitre, Sep 5, 2013
  11. 08/38 pack v4: basic SHA1 reference encodingNicolas Pitre, Sep 5, 2013
  12. 09/38 introduce get_sha1_lowhex()Nicolas Pitre, Sep 5, 2013
  13. 10/38 pack v4: commit object encodingNicolas Pitre, Sep 5, 2013
  14. Junio C HamanoSep 6, 2013
  15. Nicolas PitreSep 6, 2013
  16. Junio C HamanoSep 6, 2013
  17. Nicolas PitreSep 7, 2013
  18. 11/38 pack v4: tree object encodingNicolas Pitre, Sep 5, 2013
  19. 12/38 pack v4: dictionary table outputNicolas Pitre, Sep 5, 2013
  20. 13/38 pack v4: creation codeNicolas Pitre, Sep 5, 2013
  21. 14/38 pack v4: object headersNicolas Pitre, Sep 5, 2013
  22. 15/38 pack v4: object data copyNicolas Pitre, Sep 5, 2013
  23. 16/38 pack v4: object writingNicolas Pitre, Sep 5, 2013
  24. 17/38 pack v4: tree object delta encodingNicolas Pitre, Sep 5, 2013
  25. 18/38 pack v4: load delta candidate for encoding tree objectsNicolas Pitre, Sep 5, 2013
  26. 19/38 packv4-create: optimize delta encodingNicolas Pitre, Sep 5, 2013
  27. 20/38 pack v4: honor pack.compression config optionNicolas Pitre, Sep 5, 2013
  28. 21/38 pack v4: relax commit parsing a bitNicolas Pitre, Sep 5, 2013
  29. 22/38 pack index v3Nicolas Pitre, Sep 5, 2013
  30. 23/38 packv4-create: normalize pack name to properly generate the pack index file nameNicolas Pitre, Sep 5, 2013
  31. 24/38 packv4-create: add progress displayNicolas Pitre, Sep 5, 2013
  32. 25/38 pack v4: initial pack index v3 support on the read sideNicolas Pitre, Sep 5, 2013
  33. 26/38 pack v4: object header decodeNicolas Pitre, Sep 5, 2013
  34. 27/38 pack v4: code to obtain a SHA1 from a sha1refNicolas Pitre, Sep 5, 2013
  35. 28/38 pack v4: code to load and prepare a pack dictionary table for useNicolas Pitre, Sep 5, 2013
  36. 29/38 pack v4: code to retrieve a nameNicolas Pitre, Sep 5, 2013
  37. 30/38 pack v4: code to recreate a canonical commit objectNicolas Pitre, Sep 5, 2013
  38. 31/38 sha1_file.c: make use of decode_varint()Nicolas Pitre, Sep 5, 2013
  39. SZEDER GáborSep 5, 2013
  40. 32/38 pack v4: parse delta base referenceNicolas Pitre, Sep 5, 2013
  41. 33/38 pack v4: we can read commit objects nowNicolas Pitre, Sep 5, 2013
  42. 34/38 pack v4: code to retrieve a path componentNicolas Pitre, Sep 5, 2013
  43. 35/38 pack v4: decode tree objectsNicolas Pitre, Sep 5, 2013
  44. 36/38 pack v4: get tree objectsNicolas Pitre, Sep 5, 2013
  45. 37/38 pack v4: introduce "escape hatches" in the name and path indexesNicolas Pitre, Sep 5, 2013
  46. Nicolas PitreSep 5, 2013
  47. Nicolas PitreSep 5, 2013
  48. Duy NguyenSep 5, 2013
  49. 38/38 packv4-create: add a command line argument to limit tree copy sequencesNicolas Pitre, Sep 5, 2013
  50. 00/12 pack v4 support in index-packNguyễn Thái Ngọc Duy, Sep 7, 2013
  51. 01/12 pack v4: split pv4_create_dict() out of load_dict()Nguyễn Thái Ngọc Duy, Sep 7, 2013
  52. 02/12 index-pack: split out varint decoding codeNguyễn Thái Ngọc Duy, Sep 7, 2013
  53. 03/12 index-pack: do not allocate buffer for unpacking deltas in the first passNguyễn Thái Ngọc Duy, Sep 7, 2013
  54. 04/12 index-pack: split inflate/digest code out of unpack_entry_dataNguyễn Thái Ngọc Duy, Sep 7, 2013
  55. 05/12 index-pack: parse v4 header and dictionariesNguyễn Thái Ngọc Duy, Sep 7, 2013
  56. Nicolas PitreSep 8, 2013
  57. 06/12 index-pack: make sure all objects are registered in v4's SHA-1 tableNguyễn Thái Ngọc Duy, Sep 7, 2013
  58. 07/12 index-pack: parse v4 commit formatNguyễn Thái Ngọc Duy, Sep 7, 2013
  59. 08/12 index-pack: parse v4 tree formatNguyễn Thái Ngọc Duy, Sep 7, 2013
  60. Nicolas PitreSep 8, 2013
  61. 09/12 index-pack: move delta base queuing code to unpack_raw_entryNguyễn Thái Ngọc Duy, Sep 7, 2013
  62. 10/12 index-pack: record all delta bases in v4 (tree and ref-delta)Nguyễn Thái Ngọc Duy, Sep 7, 2013
  63. 11/12 index-pack: skip looking for ofs-deltas in v4 as they are not allowedNguyễn Thái Ngọc Duy, Sep 7, 2013
  64. 12/12 index-pack: resolve v4 one-base treesNguyễn Thái Ngọc Duy, Sep 7, 2013
  65. Nicolas PitreSep 8, 2013
  66. Duy NguyenSep 8, 2013
  67. 00/14 pack v4 support in index-packNguyễn Thái Ngọc Duy, Sep 8, 2013
  68. 01/14 pack v4: split pv4_create_dict() out of load_dict()Nguyễn Thái Ngọc Duy, Sep 8, 2013
  69. 02/14 pack v4: add pv4_free_dict()Nguyễn Thái Ngọc Duy, Sep 8, 2013
  70. 03/14 index-pack: add more comments on some big functionsNguyễn Thái Ngọc Duy, Sep 8, 2013
  71. 04/14 index-pack: split out varint decoding codeNguyễn Thái Ngọc Duy, Sep 8, 2013
  72. 05/14 index-pack: do not allocate buffer for unpacking deltas in the first passNguyễn Thái Ngọc Duy, Sep 8, 2013
  73. 06/14 index-pack: split inflate/digest code out of unpack_entry_dataNguyễn Thái Ngọc Duy, Sep 8, 2013
  74. 07/14 index-pack: parse v4 header and dictionariesNguyễn Thái Ngọc Duy, Sep 8, 2013
  75. 08/14 index-pack: make sure all objects are registered in v4's SHA-1 tableNguyễn Thái Ngọc Duy, Sep 8, 2013
  76. 09/14 index-pack: parse v4 commit formatNguyễn Thái Ngọc Duy, Sep 8, 2013
  77. 10/14 index-pack: parse v4 tree formatNguyễn Thái Ngọc Duy, Sep 8, 2013
  78. 11/14 index-pack: move delta base queuing code to unpack_raw_entryNguyễn Thái Ngọc Duy, Sep 8, 2013
  79. 12/14 index-pack: record all delta bases in v4 (tree and ref-delta)Nguyễn Thái Ngọc Duy, Sep 8, 2013
  80. 13/14 index-pack: skip looking for ofs-deltas in v4 as they are not allowedNguyễn Thái Ngọc Duy, Sep 8, 2013
  81. 14/14 index-pack: resolve v4 one-base treesNguyễn Thái Ngọc Duy, Sep 8, 2013
  82. 00/11 pack v4 support in pack-objectsNguyễn Thái Ngọc Duy, Sep 8, 2013
  83. 01/11 pack v4: allocate dicts from the beginningNguyễn Thái Ngọc Duy, Sep 8, 2013
  84. 02/11 pack v4: stop using static/global variables in packv4-create.cNguyễn Thái Ngọc Duy, Sep 8, 2013
  85. 03/11 pack v4: move packv4-create.c to libgit.aNguyễn Thái Ngọc Duy, Sep 8, 2013
  86. Nicolas PitreSep 8, 2013
  87. 04/11 pack v4: add version argument to write_pack_headerNguyễn Thái Ngọc Duy, Sep 8, 2013
  88. 05/11 pack-write.c: add pv4_encode_in_pack_object_headerNguyễn Thái Ngọc Duy, Sep 8, 2013
  89. Nicolas PitreSep 8, 2013
  90. 06/11 pack-objects: add --version to specify written pack versionNguyễn Thái Ngọc Duy, Sep 8, 2013
  91. 07/11 list-objects.c: add show_tree_entry callback to traverse_commit_listNguyễn Thái Ngọc Duy, Sep 8, 2013
  92. 08/11 pack-objects: create pack v4 tablesNguyễn Thái Ngọc Duy, Sep 8, 2013
  93. Duy NguyenSep 9, 2013
  94. Nicolas PitreSep 9, 2013
  95. Junio C HamanoSep 9, 2013
  96. 09/11 pack-objects: do not cache delta for v4 treesNguyễn Thái Ngọc Duy, Sep 8, 2013
  97. 10/11 pack-objects: exclude commits out of delta objects in v4Nguyễn Thái Ngọc Duy, Sep 8, 2013
  98. 11/11 pack-objects: support writing pack v4Nguyễn Thái Ngọc Duy, Sep 8, 2013
  99. 00/16 pack v4 support in pack-objectsNguyễn Thái Ngọc Duy, Sep 9, 2013
  100. 01/16 pack v4: allocate dicts from the beginningNguyễn Thái Ngọc Duy, Sep 9, 2013
  101. 02/16 pack v4: stop using static/global variables in packv4-create.cNguyễn Thái Ngọc Duy, Sep 9, 2013
  102. 03/16 pack v4: move packv4-create.c to libgit.aNguyễn Thái Ngọc Duy, Sep 9, 2013
  103. 04/16 pack v4: add version argument to write_pack_headerNguyễn Thái Ngọc Duy, Sep 9, 2013
  104. 05/16 pack_write: tighten valid object type check in encode_in_pack_object_headerNguyễn Thái Ngọc Duy, Sep 9, 2013
  105. 06/16 pack-write.c: add pv4_encode_object_headerNguyễn Thái Ngọc Duy, Sep 9, 2013
  106. 07/16 pack-objects: add --version to specify written pack versionNguyễn Thái Ngọc Duy, Sep 9, 2013
  107. 08/16 list-objects.c: add show_tree_entry callback to traverse_commit_listNguyễn Thái Ngọc Duy, Sep 9, 2013
  108. 09/16 pack-objects: do not cache delta for v4 treesNguyễn Thái Ngọc Duy, Sep 9, 2013
  109. 10/16 pack-objects: exclude commits out of delta objects in v4Nguyễn Thái Ngọc Duy, Sep 9, 2013
  110. 11/16 pack-objects: create pack v4 tablesNguyễn Thái Ngọc Duy, Sep 9, 2013
  111. 12/16 pack-objects: prepare SHA-1 table in v4Nguyễn Thái Ngọc Duy, Sep 9, 2013
  112. 13/16 pack-objects: support writing pack v4Nguyễn Thái Ngọc Duy, Sep 9, 2013
  113. 14/16 pack v4: support "end-of-pack" indicator in index-pack and pack-objectsNguyễn Thái Ngọc Duy, Sep 9, 2013
  114. 15/16 index-pack: use nr_objects_final as sha1_table sizeNguyễn Thái Ngọc Duy, Sep 9, 2013
  115. Nicolas PitreSep 9, 2013
  116. Junio C HamanoSep 9, 2013
  117. Nicolas PitreSep 9, 2013
  118. Junio C HamanoSep 9, 2013
  119. Nicolas PitreSep 9, 2013
  120. Junio C HamanoSep 9, 2013
  121. Nicolas PitreSep 9, 2013
  122. Duy NguyenSep 10, 2013
  123. Nicolas PitreSep 12, 2013
  124. 16/16 index-pack: support completing thin packs v4Nguyễn Thái Ngọc Duy, Sep 9, 2013

Read the whole thread, see it on lore, or plain text.

$ cat FOOTERMessages come from the public archive at lore.kernel.org/git, fetched every hour. The front page is chosen and written each morning by an AI editor and can be wrong; the threads themselves are the record. About and API. For agents: an MCP server at https://gitlist.dev/mcp, and any thread, story or person page as Markdown by adding .md to its URL (or sending Accept: text/markdown). Details in /llms.txt.