git/list[1] front-page[2] threads[3] people[4] search[5] about
 

[PATCH 10/10] pack-redundant.c: learn about index v2

From
Nicolas Pitre <nico@cam.org>
Date
Apr 9, 2007, 05:06 UTC
Message-ID
<11760952033684-git-send-email-nico@cam.org>
In-Reply-To
<11760952021074-git-send-email-nico@cam.org>

Initially the conversion was made using nth_packed_object_sha1() which made this file completely index version agnostic. Unfortunately the overhead was quite significant so I went back to raw index walking but with selectable base and step values which brought back similar performances as the original.

Signed-off-by: Nicolas Pitre <nico@cam.org>
---
 pack-redundant.c |   47 +++++++++++++++++++++++++++--------------------
 1 files changed, 27 insertions(+), 20 deletions(-)
diff --git a/pack-redundant.c b/pack-redundant.c
index 40e579b..87077e1 100644
--- a/pack-redundant.c
+++ b/pack-redundant.c
@@ -247,16 +247,19 @@ static struct pack_list * pack_list_difference(const struct pack_list *A,
 
 static void cmp_two_packs(struct pack_list *p1, struct pack_list *p2)
 {
-	int p1_off, p2_off;
+	unsigned long p1_off = 0, p2_off = 0, p1_step, p2_step;
 	const unsigned char *p1_base, *p2_base;
 	struct llist_item *p1_hint = NULL, *p2_hint = NULL;
 
-	p1_off = p2_off = 256 * 4 + 4;
 	p1_base = p1->pack->index_data;
 	p2_base = p2->pack->index_data;
+	p1_base += 256 * 4 + ((p1->pack->index_version < 2) ? 4 : 8);
+	p2_base += 256 * 4 + ((p2->pack->index_version < 2) ? 4 : 8);
+	p1_step = (p1->pack->index_version < 2) ? 24 : 20;
+	p2_step = (p2->pack->index_version < 2) ? 24 : 20;
 
-	while (p1_off <= p1->pack->index_size - 3 * 20 &&
-	       p2_off <= p2->pack->index_size - 3 * 20)
+	while (p1_off < p1->pack->num_objects * p1_step &&
+	       p2_off < p2->pack->num_objects * p2_step)
 	{
 		int cmp = hashcmp(p1_base + p1_off, p2_base + p2_off);
 		/* cmp ~ p1 - p2 */
@@ -265,14 +268,14 @@ static void cmp_two_packs(struct pack_list *p1, struct pack_list *p2)
 					p1_base + p1_off, p1_hint);
 			p2_hint = llist_sorted_remove(p2->unique_objects,
 					p1_base + p1_off, p2_hint);
-			p1_off+=24;
-			p2_off+=24;
+			p1_off += p1_step;
+			p2_off += p2_step;
 			continue;
 		}
 		if (cmp < 0) { /* p1 has the object, p2 doesn't */
-			p1_off+=24;
+			p1_off += p1_step;
 		} else { /* p2 has the object, p1 doesn't */
-			p2_off+=24;
+			p2_off += p2_step;
 		}
 	}
 }
@@ -352,28 +355,31 @@ static int is_superset(struct pack_list *pl, struct llist *list)
 static size_t sizeof_union(struct packed_git *p1, struct packed_git *p2)
 {
 	size_t ret = 0;
-	int p1_off, p2_off;
+	unsigned long p1_off = 0, p2_off = 0, p1_step, p2_step;
 	const unsigned char *p1_base, *p2_base;
 
-	p1_off = p2_off = 256 * 4 + 4;
 	p1_base = p1->index_data;
 	p2_base = p2->index_data;
+	p1_base += 256 * 4 + ((p1->index_version < 2) ? 4 : 8);
+	p2_base += 256 * 4 + ((p2->index_version < 2) ? 4 : 8);
+	p1_step = (p1->index_version < 2) ? 24 : 20;
+	p2_step = (p2->index_version < 2) ? 24 : 20;
 
-	while (p1_off <= p1->index_size - 3 * 20 &&
-	       p2_off <= p2->index_size - 3 * 20)
+	while (p1_off < p1->num_objects * p1_step &&
+	       p2_off < p2->num_objects * p2_step)
 	{
 		int cmp = hashcmp(p1_base + p1_off, p2_base + p2_off);
 		/* cmp ~ p1 - p2 */
 		if (cmp == 0) {
 			ret++;
-			p1_off+=24;
-			p2_off+=24;
+			p1_off += p1_step;
+			p2_off += p2_step;
 			continue;
 		}
 		if (cmp < 0) { /* p1 has the object, p2 doesn't */
-			p1_off+=24;
+			p1_off += p1_step;
 		} else { /* p2 has the object, p1 doesn't */
-			p2_off+=24;
+			p2_off += p2_step;
 		}
 	}
 	return ret;
@@ -535,7 +541,7 @@ static void scan_alt_odb_packs(void)
 static struct pack_list * add_pack(struct packed_git *p)
 {
 	struct pack_list l;
-	size_t off;
+	unsigned long off = 0, step;
 	const unsigned char *base;
 
 	if (!p->pack_local && !(alt_odb || verbose))
@@ -544,11 +550,12 @@ static struct pack_list * add_pack(struct packed_git *p)
 	l.pack = p;
 	llist_init(&l.all_objects);
 
-	off = 256 * 4 + 4;
 	base = p->index_data;
-	while (off <= p->index_size - 3 * 20) {
+	base += 256 * 4 + ((p->index_version < 2) ? 4 : 8);
+	step = (p->index_version < 2) ? 24 : 20;
+	while (off < p->num_objects * step) {
 		llist_insert_back(l.all_objects, base + off);
-		off += 24;
+		off += step;
 	}
 	/* this list will be pruned in cmp_two_packs later */
 	l.unique_objects = llist_copy(l.all_objects);
-- 
1.5.1.696.g6d352-dirty
Previous: Nicolas PitreNext: Junio C Hamano
Message 11 of 24 in “support for large packs and 64-bit offsets”
  1. Nicolas PitreApr 9, 2007
  2. 01/10 get rid of num_packed_objects()Nicolas Pitre, Apr 9, 2007
  3. 02/10 make overflow test on delta base offset work regardless of variable sizeNicolas Pitre, Apr 9, 2007
  4. 03/10 add overflow tests on pack offset variablesNicolas Pitre, Apr 9, 2007
  5. 04/10 compute a CRC32 for each object as stored in a packNicolas Pitre, Apr 9, 2007
  6. 05/10 compute object CRC32 with index-packNicolas Pitre, Apr 9, 2007
  7. 06/10 pack-objects: learn about pack index version 2Nicolas Pitre, Apr 9, 2007
  8. 07/10 index-pack: learn about pack index version 2Nicolas Pitre, Apr 9, 2007
  9. 08/10 sha1_file.c: learn about index version 2Nicolas Pitre, Apr 9, 2007
  10. 09/10 show-index.c: learn about index v2Nicolas Pitre, Apr 9, 2007
  11. 10/10 pack-redundant.c: learn about index v2Nicolas Pitre, Apr 9, 2007
  12. Junio C HamanoApr 9, 2007
  13. Nicolas PitreApr 9, 2007
  14. Shawn O. PearceApr 9, 2007
  15. Nicolas PitreApr 9, 2007
  16. Shawn O. PearceApr 9, 2007
  17. Junio C HamanoApr 9, 2007
  18. Shawn O. PearceApr 9, 2007
  19. Nicolas PitreApr 9, 2007
  20. Junio C HamanoApr 9, 2007
  21. Linus TorvaldsApr 9, 2007
  22. Nicolas PitreApr 9, 2007
  23. Shawn O. PearceApr 9, 2007
  24. Nicolas PitreApr 9, 2007

Read the whole thread, see it on lore, or plain text.

$ cat FOOTERMessages come from the public archive at lore.kernel.org/git, fetched every hour. The front page is chosen and written each morning by an AI editor and can be wrong; the threads themselves are the record. About and API. For agents: an MCP server at https://gitlist.dev/mcp, and any thread, story or person page as Markdown by adding .md to its URL (or sending Accept: text/markdown). Details in /llms.txt.