git/list[1] front-page[2] threads[3] people[4] search[5] about
 

[PATCH 3/6] object-file: move iteration over loose files into backend

From
Patrick Steinhardt <ps@pks.im>
Date
Nov 2, 2026, 11:20 UTC
Message-ID
<20261102-pks-odb-loose-cleanups-v1-3-fbbc86c29693@pks.im>
In-Reply-To
<20261102-pks-odb-loose-cleanups-v1-0-fbbc86c29693@pks.im>

The logic to iterate over files part of the "loose" backend is contained in "object-file.c" even though it's an implementation detail of the backend itself. Move the logic into the backend accordingly.

Note that the function is a bit on the weird side: it accepts a generic `struct odb_source` as input and doesn't cast it down to its backend at all. This is because all it requires access to is the `path` member of the source, and from thereon it knows how to iterate the subdirectories of the "loose" backend.

We should adapt this function to accept `struct odb_source_loose` to properly reflect that it expects a path to a loose object database. But that would require changes to a bunch of callers without much benefit. So for now, the function signature is left as-is. But eventually we may want to change it once we have reworked a couple of its callers.

Signed-off-by: Patrick Steinhardt <ps@pks.im>
---
 builtin/count-objects.c |  1 +
 builtin/gc.c            |  1 +
 builtin/prune.c         |  1 +
 object-file.c           | 92 -------------------------------------------------
 object-file.h           | 44 -----------------------
 odb/source-loose.c      | 92 +++++++++++++++++++++++++++++++++++++++++++++++++
 odb/source-loose.h      | 33 ++++++++++++++++++
 prune-packed.c          |  1 +
 8 files changed, 129 insertions(+), 136 deletions(-)
diff --git a/builtin/count-objects.c b/builtin/count-objects.c
index 3f7b20fde53..aae040567e3 100644
--- a/builtin/count-objects.c
+++ b/builtin/count-objects.c
@@ -14,6 +14,7 @@
 #include "quote.h"
 #include "packfile.h"
 #include "object-file.h"
+#include "odb/source-loose.h"
 
 static unsigned long garbage;
 static off_t size_garbage;
diff --git a/builtin/gc.c b/builtin/gc.c
index 81cecf6b6c3..d3ba6f0c9e9 100644
--- a/builtin/gc.c
+++ b/builtin/gc.c
@@ -31,6 +31,7 @@
 #include "packfile.h"
 #include "object-file.h"
 #include "odb.h"
+#include "odb/source-loose.h"
 #include "path.h"
 #include "reflog.h"
 #include "rerere.h"
diff --git a/builtin/prune.c b/builtin/prune.c
index 97f4c1418bc..af2a547d931 100644
--- a/builtin/prune.c
+++ b/builtin/prune.c
@@ -17,6 +17,7 @@
 #include "object-file.h"
 #include "object-name.h"
 #include "odb.h"
+#include "odb/source-loose.h"
 #include "shallow.h"
 
 static const char * const prune_usage[] = {
diff --git a/object-file.c b/object-file.c
index ab919f03fef..d3ddcbf309c 100644
--- a/object-file.c
+++ b/object-file.c
@@ -12,7 +12,6 @@
 #include "git-compat-util.h"
 #include "config.h"
 #include "convert.h"
-#include "dir.h"
 #include "environment.h"
 #include "fsck.h"
 #include "gettext.h"
@@ -838,97 +837,6 @@ int read_pack_header(int fd, struct pack_header *header)
 	return 0;
 }
 
-int for_each_file_in_obj_subdir(unsigned int subdir_nr,
-				struct strbuf *path,
-				const struct git_hash_algo *algop,
-				each_loose_object_fn obj_cb,
-				each_loose_cruft_fn cruft_cb,
-				each_loose_subdir_fn subdir_cb,
-				void *data)
-{
-	size_t origlen, baselen;
-	DIR *dir;
-	struct dirent *de;
-	int r = 0;
-	struct object_id oid;
-
-	if (subdir_nr > 0xff)
-		BUG("invalid loose object subdirectory: %x", subdir_nr);
-
-	origlen = path->len;
-	strbuf_complete(path, '/');
-	strbuf_addf(path, "%02x", subdir_nr);
-
-	dir = opendir(path->buf);
-	if (!dir) {
-		if (errno != ENOENT)
-			r = error_errno(_("unable to open %s"), path->buf);
-		strbuf_setlen(path, origlen);
-		return r;
-	}
-
-	oid.hash[0] = subdir_nr;
-	strbuf_addch(path, '/');
-	baselen = path->len;
-
-	while ((de = readdir_skip_dot_and_dotdot(dir))) {
-		size_t namelen;
-
-		namelen = strlen(de->d_name);
-		strbuf_setlen(path, baselen);
-		strbuf_add(path, de->d_name, namelen);
-		if (namelen == algop->hexsz - 2 &&
-		    !hex_to_bytes(oid.hash + 1, de->d_name,
-				  algop->rawsz - 1)) {
-			oid_set_algo(&oid, algop);
-			memset(oid.hash + algop->rawsz, 0,
-			       GIT_MAX_RAWSZ - algop->rawsz);
-			if (obj_cb) {
-				r = obj_cb(&oid, path->buf, data);
-				if (r)
-					break;
-			}
-			continue;
-		}
-
-		if (cruft_cb) {
-			r = cruft_cb(de->d_name, path->buf, data);
-			if (r)
-				break;
-		}
-	}
-	closedir(dir);
-
-	strbuf_setlen(path, baselen - 1);
-	if (!r && subdir_cb)
-		r = subdir_cb(subdir_nr, path->buf, data);
-
-	strbuf_setlen(path, origlen);
-
-	return r;
-}
-
-int for_each_loose_file_in_source(struct odb_source *source,
-				  each_loose_object_fn obj_cb,
-				  each_loose_cruft_fn cruft_cb,
-				  each_loose_subdir_fn subdir_cb,
-				  void *data)
-{
-	struct strbuf buf = STRBUF_INIT;
-	int r;
-
-	strbuf_addstr(&buf, source->path);
-	for (int i = 0; i < 256; i++) {
-		r = for_each_file_in_obj_subdir(i, &buf, source->odb->repo->hash_algo,
-						obj_cb, cruft_cb, subdir_cb, data);
-		if (r)
-			break;
-	}
-
-	strbuf_release(&buf);
-	return r;
-}
-
 static int write_pack_lockfile(struct odb_source *source, const char *pack_name,
 			       const char *keep_msg, struct tempfile **out)
 {
diff --git a/object-file.h b/object-file.h
index 3cfcaefa420..83d3f38a540 100644
--- a/object-file.h
+++ b/object-file.h
@@ -23,50 +23,6 @@ int index_path(struct index_state *istate, struct object_id *oid, const char *pa
 
 struct odb_source;
 
-/*
- * Iterate over the files in the loose-object parts of the object
- * directory "path", triggering the following callbacks:
- *
- *  - loose_object is called for each loose object we find.
- *
- *  - loose_cruft is called for any files that do not appear to be
- *    loose objects. Note that we only look in the loose object
- *    directories "objects/[0-9a-f]{2}/", so we will not report
- *    "objects/foobar" as cruft.
- *
- *  - loose_subdir is called for each top-level hashed subdirectory
- *    of the object directory (e.g., "$OBJDIR/f0"). It is called
- *    after the objects in the directory are processed.
- *
- * Any callback that is NULL will be ignored. Callbacks returning non-zero
- * will end the iteration.
- *
- * In the "buf" variant, "path" is a strbuf which will also be used as a
- * scratch buffer, but restored to its original contents before
- * the function returns.
- */
-typedef int each_loose_object_fn(const struct object_id *oid,
-				 const char *path,
-				 void *data);
-typedef int each_loose_cruft_fn(const char *basename,
-				const char *path,
-				void *data);
-typedef int each_loose_subdir_fn(unsigned int nr,
-				 const char *path,
-				 void *data);
-int for_each_loose_file_in_source(struct odb_source *source,
-				  each_loose_object_fn obj_cb,
-				  each_loose_cruft_fn cruft_cb,
-				  each_loose_subdir_fn subdir_cb,
-				  void *data);
-int for_each_file_in_obj_subdir(unsigned int subdir_nr,
-				struct strbuf *path,
-				const struct git_hash_algo *algop,
-				each_loose_object_fn obj_cb,
-				each_loose_cruft_fn cruft_cb,
-				each_loose_subdir_fn subdir_cb,
-				void *data);
-
 /**
  * format_object_header() is a thin wrapper around s xsnprintf() that
  * writes the initial "<type> <obj-len>" part of the loose object
diff --git a/odb/source-loose.c b/odb/source-loose.c
index 0f29162d752..a90268e200a 100644
--- a/odb/source-loose.c
+++ b/odb/source-loose.c
@@ -1,6 +1,7 @@
 #include "git-compat-util.h"
 #include "abspath.h"
 #include "chdir-notify.h"
+#include "dir.h"
 #include "gettext.h"
 #include "hex.h"
 #include "loose.h"
@@ -39,6 +40,97 @@ const char *odb_loose_path(struct odb_source_loose *loose,
 	return buf->buf;
 }
 
+static int for_each_file_in_obj_subdir(unsigned int subdir_nr,
+				       struct strbuf *path,
+				       const struct git_hash_algo *algop,
+				       each_loose_object_fn obj_cb,
+				       each_loose_cruft_fn cruft_cb,
+				       each_loose_subdir_fn subdir_cb,
+				       void *data)
+{
+	size_t origlen, baselen;
+	DIR *dir;
+	struct dirent *de;
+	int r = 0;
+	struct object_id oid;
+
+	if (subdir_nr > 0xff)
+		BUG("invalid loose object subdirectory: %x", subdir_nr);
+
+	origlen = path->len;
+	strbuf_complete(path, '/');
+	strbuf_addf(path, "%02x", subdir_nr);
+
+	dir = opendir(path->buf);
+	if (!dir) {
+		if (errno != ENOENT)
+			r = error_errno(_("unable to open %s"), path->buf);
+		strbuf_setlen(path, origlen);
+		return r;
+	}
+
+	oid.hash[0] = subdir_nr;
+	strbuf_addch(path, '/');
+	baselen = path->len;
+
+	while ((de = readdir_skip_dot_and_dotdot(dir))) {
+		size_t namelen;
+
+		namelen = strlen(de->d_name);
+		strbuf_setlen(path, baselen);
+		strbuf_add(path, de->d_name, namelen);
+		if (namelen == algop->hexsz - 2 &&
+		    !hex_to_bytes(oid.hash + 1, de->d_name,
+				  algop->rawsz - 1)) {
+			oid_set_algo(&oid, algop);
+			memset(oid.hash + algop->rawsz, 0,
+			       GIT_MAX_RAWSZ - algop->rawsz);
+			if (obj_cb) {
+				r = obj_cb(&oid, path->buf, data);
+				if (r)
+					break;
+			}
+			continue;
+		}
+
+		if (cruft_cb) {
+			r = cruft_cb(de->d_name, path->buf, data);
+			if (r)
+				break;
+		}
+	}
+	closedir(dir);
+
+	strbuf_setlen(path, baselen - 1);
+	if (!r && subdir_cb)
+		r = subdir_cb(subdir_nr, path->buf, data);
+
+	strbuf_setlen(path, origlen);
+
+	return r;
+}
+
+int for_each_loose_file_in_source(struct odb_source *source,
+				  each_loose_object_fn obj_cb,
+				  each_loose_cruft_fn cruft_cb,
+				  each_loose_subdir_fn subdir_cb,
+				  void *data)
+{
+	struct strbuf buf = STRBUF_INIT;
+	int r;
+
+	strbuf_addstr(&buf, source->path);
+	for (int i = 0; i < 256; i++) {
+		r = for_each_file_in_obj_subdir(i, &buf, source->odb->repo->hash_algo,
+						obj_cb, cruft_cb, subdir_cb, data);
+		if (r)
+			break;
+	}
+
+	strbuf_release(&buf);
+	return r;
+}
+
 static int append_loose_object(const struct object_id *oid,
 			       const char *path UNUSED,
 			       void *data)
diff --git a/odb/source-loose.h b/odb/source-loose.h
index 8d4cdac0d50..20f6ec7fa08 100644
--- a/odb/source-loose.h
+++ b/odb/source-loose.h
@@ -57,4 +57,37 @@ const char *odb_loose_path(struct odb_source_loose *source,
 			   struct strbuf *buf,
 			   const struct object_id *oid);
 
+/*
+ * Iterate over the files in the loose-object parts of the object
+ * directory "path", triggering the following callbacks:
+ *
+ *  - loose_object is called for each loose object we find.
+ *
+ *  - loose_cruft is called for any files that do not appear to be
+ *    loose objects. Note that we only look in the loose object
+ *    directories "objects/[0-9a-f]{2}/", so we will not report
+ *    "objects/foobar" as cruft.
+ *
+ *  - loose_subdir is called for each top-level hashed subdirectory
+ *    of the object directory (e.g., "$OBJDIR/f0"). It is called
+ *    after the objects in the directory are processed.
+ *
+ * Any callback that is NULL will be ignored. Callbacks returning non-zero
+ * will end the iteration.
+ */
+typedef int each_loose_object_fn(const struct object_id *oid,
+				 const char *path,
+				 void *data);
+typedef int each_loose_cruft_fn(const char *basename,
+				const char *path,
+				void *data);
+typedef int each_loose_subdir_fn(unsigned int nr,
+				 const char *path,
+				 void *data);
+int for_each_loose_file_in_source(struct odb_source *source,
+				  each_loose_object_fn obj_cb,
+				  each_loose_cruft_fn cruft_cb,
+				  each_loose_subdir_fn subdir_cb,
+				  void *data);
+
 #endif
diff --git a/prune-packed.c b/prune-packed.c
index b6749b8d132..d0a8a062b97 100644
--- a/prune-packed.c
+++ b/prune-packed.c
@@ -3,6 +3,7 @@
 #include "git-compat-util.h"
 #include "gettext.h"
 #include "object-file.h"
+#include "odb/source-loose.h"
 #include "packfile.h"
 #include "progress.h"
 #include "prune-packed.h"

-- 
2.56.0.500.g5d06dc02cd.dirty
Previous: Patrick SteinhardtNext: Patrick Steinhardt
Message 4 of 7 in “object-file: move logic related to the loose backend”
  1. 0/6 object-file: move logic related to the loose backendPatrick Steinhardt, Nov 2, 2026
  2. 1/6 object-file: move logic to read loose objects into backendPatrick Steinhardt, Nov 2, 2026
  3. 2/6 object-file: move logic to derive loose object paths into backendPatrick Steinhardt, Nov 2, 2026
  4. 3/6 object-file: move iteration over loose files into backendPatrick Steinhardt, Nov 2, 2026
  5. 4/6 odb/source-loose: deduplicate logic to mmap objectsPatrick Steinhardt, Nov 2, 2026
  6. 5/6 object-file: stop including "odb/source-loose.h"Patrick Steinhardt, Nov 2, 2026
  7. 6/6 object-file: stop including "git-zlib.h"Patrick Steinhardt, Nov 2, 2026

Read the whole thread, see it on lore, or plain text.

$ cat FOOTERMessages come from the public archive at lore.kernel.org/git, fetched every hour. The front page is chosen and written each morning by an AI editor and can be wrong; the threads themselves are the record. About and API. For agents: an MCP server at https://gitlist.dev/mcp, and any thread, story or person page as Markdown by adding .md to its URL (or sending Accept: text/markdown). Details in /llms.txt.