git/list[1] front-page[2] threads[3] people[4] search[5] about
 

[PATCH 3/3] packfile: use oidset for bad objects

From
René Scharfe <l.s.r@web.de>
Date
Sep 11, 2021, 08:01 UTC
Message-ID
<14d48124-d8bb-aa34-aad0-4203d699e17e@web.de>
In-Reply-To
<4a702bfe-afd0-669a-c893-0262289c24b8@web.de>

Store the object ID of broken pack entries in an oidset instead of keeping only their hashes in an unsorted array. The resulting code is shorter and easier to read. It also handles the (hopefully) very rare case of having a high number of bad objects better.

Signed-off-by: René Scharfe <l.s.r@web.de>
---
 midx.c         | 13 ++++---------
 object-store.h |  4 ++--
 packfile.c     | 27 +++++----------------------
 3 files changed, 11 insertions(+), 33 deletions(-)
diff --git a/midx.c b/midx.c
index 321c6fdd2f..01623fb339 100644
--- a/midx.c
+++ b/midx.c
@@ -283,6 +283,7 @@ static int nth_midxed_pack_entry(struct repository *r,
 {
 	uint32_t pack_int_id;
 	struct packed_git *p;
+	struct object_id oid;

 	if (pos >= m->num_objects)
 		return 0;
@@ -303,15 +304,9 @@ static int nth_midxed_pack_entry(struct repository *r,
 	if (!is_pack_valid(p))
 		return 0;

-	if (p->num_bad_objects) {
-		uint32_t i;
-		struct object_id oid;
-		nth_midxed_object_oid(&oid, m, pos);
-		for (i = 0; i < p->num_bad_objects; i++)
-			if (hasheq(oid.hash,
-				   p->bad_object_sha1 + the_hash_algo->rawsz * i))
-				return 0;
-	}
+	nth_midxed_object_oid(&oid, m, pos);
+	if (oidset_contains(&p->bad_objects, &oid))
+		return 0;

 	e->offset = nth_midxed_offset(m, pos);
 	e->p = p;
diff --git a/object-store.h b/object-store.h
index b4dc6668aa..c7bead66f6 100644
--- a/object-store.h
+++ b/object-store.h
@@ -10,6 +10,7 @@
 #include "khash.h"
 #include "dir.h"
 #include "oidtree.h"
+#include "oidset.h"

 struct object_directory {
 	struct object_directory *next;
@@ -75,9 +76,8 @@ struct packed_git {
 	const void *index_data;
 	size_t index_size;
 	uint32_t num_objects;
-	uint32_t num_bad_objects;
 	uint32_t crc_offset;
-	unsigned char *bad_object_sha1;
+	struct oidset bad_objects;
 	int index_version;
 	time_t mtime;
 	int pack_fd;
diff --git a/packfile.c b/packfile.c
index 04080a558b..8f6d1d6328 100644
--- a/packfile.c
+++ b/packfile.c
@@ -1163,29 +1163,17 @@ int unpack_object_header(struct packed_git *p,

 void mark_bad_packed_object(struct packed_git *p, const struct object_id *oid)
 {
-	unsigned i;
-	const unsigned hashsz = the_hash_algo->rawsz;
-	for (i = 0; i < p->num_bad_objects; i++)
-		if (hasheq(oid->hash, p->bad_object_sha1 + hashsz * i))
-			return;
-	p->bad_object_sha1 = xrealloc(p->bad_object_sha1,
-				      st_mult(GIT_MAX_RAWSZ,
-					      st_add(p->num_bad_objects, 1)));
-	hashcpy(p->bad_object_sha1 + hashsz * p->num_bad_objects, oid->hash);
-	p->num_bad_objects++;
+	oidset_insert(&p->bad_objects, oid);
 }

 const struct packed_git *has_packed_and_bad(struct repository *r,
 					    const struct object_id *oid)
 {
 	struct packed_git *p;
-	unsigned i;

 	for (p = r->objects->packed_git; p; p = p->next)
-		for (i = 0; i < p->num_bad_objects; i++)
-			if (hasheq(oid->hash,
-				   p->bad_object_sha1 + the_hash_algo->rawsz * i))
-				return p;
+		if (oidset_contains(&p->bad_objects, oid))
+			return p;
 	return NULL;
 }

@@ -2016,13 +2004,8 @@ static int fill_pack_entry(const struct object_id *oid,
 {
 	off_t offset;

-	if (p->num_bad_objects) {
-		unsigned i;
-		for (i = 0; i < p->num_bad_objects; i++)
-			if (hasheq(oid->hash,
-				   p->bad_object_sha1 + the_hash_algo->rawsz * i))
-				return 0;
-	}
+	if (oidset_contains(&p->bad_objects, oid))
+		return 0;

 	offset = find_pack_entry_one(oid->hash, p);
 	if (!offset)
--
2.33.0
Previous: René ScharfeNext: Jeff King
Message 4 of 28 in “packfile: use oidset for bad objects”
  1. 0/3 packfile: use oidset for bad objectsRené Scharfe, Sep 11, 2021
  2. 1/3 packfile: convert mark_bad_packed_object() to object_idRené Scharfe, Sep 11, 2021
  3. 2/3 packfile: convert has_packed_and_bad() to object_idRené Scharfe, Sep 11, 2021
  4. 3/3 packfile: use oidset for bad objectsRené Scharfe, Sep 11, 2021
  5. Jeff KingSep 11, 2021
  6. René ScharfeSep 11, 2021
  7. Jeff KingSep 11, 2021
  8. René ScharfeSep 11, 2021
  9. Jeff KingSep 11, 2021
  10. 4/3 midx: inline nth_midxed_pack_entry()René Scharfe, Sep 11, 2021
  11. René ScharfeSep 11, 2021
  12. Jeff KingSep 11, 2021
  13. René ScharfeSep 11, 2021
  14. Jeff KingSep 11, 2021
  15. René ScharfeSep 11, 2021
  16. 0/5 packfile: use oidset for bad objectsRené Scharfe, Sep 11, 2021
  17. 1/5 oidset: make oidset_size() an inline functionRené Scharfe, Sep 11, 2021
  18. 2/5 midx: inline nth_midxed_pack_entry()René Scharfe, Sep 11, 2021
  19. 3/5 packfile: convert mark_bad_packed_object() to object_idRené Scharfe, Sep 11, 2021
  20. 4/5 packfile: convert has_packed_and_bad() to object_idRené Scharfe, Sep 11, 2021
  21. 5/5 packfile: use oidset for bad objectsRené Scharfe, Sep 11, 2021
  22. René ScharfeSep 11, 2021
  23. Jeff KingSep 11, 2021
  24. Derrick StoleeSep 11, 2021
  25. Taylor BlauSep 12, 2021
  26. Taylor BlauSep 12, 2021
  27. Derrick StoleeSep 12, 2021
  28. Taylor BlauSep 12, 2021

Read the whole thread, see it on lore, or plain text.

$ cat FOOTERMessages come from the public archive at lore.kernel.org/git, fetched every hour. The front page is chosen and written each morning by an AI editor and can be wrong; the threads themselves are the record. About and API. For agents: an MCP server at https://gitlist.dev/mcp, and any thread, story or person page as Markdown by adding .md to its URL (or sending Accept: text/markdown). Details in /llms.txt.