git/list[1] front-page[2] threads[3] people[4] search[5] about
 

[PATCH v2 06/13] odb/source-files: add the ability to have multiple object dirs

From
Patrick Steinhardt <ps@pks.im>
Date
Oct 8, 2026, 08:35 UTC
Message-ID
<20261008-pks-odb-move-alternates-v2-6-b47e8189baa5@pks.im>
In-Reply-To
<20261008-pks-odb-move-alternates-v2-0-b47e8189baa5@pks.im>

In a subsequent commit we'll migrate handling of alternates into the "files" backend. As part of that, the backend will need to grow the ability to handle multiple object directories for alternates and for transactions.

Introduce a `struct odb_files_dir::next` pointer so that we can have multiple directories. Adapt the backend to loop through this list as necessary. In general:

  - For reading paths we will loop through all object directories until
    we have found the object.
  - For writing paths we will typically write to the first of our object
    directories.

This mechanism isn't used yet as we still only ever have a single object directory at the current point in time. But that will change over subsequent commits.

Signed-off-by: Patrick Steinhardt <ps@pks.im>
---
 odb/source-files.c | 147 +++++++++++++++++++++++++++++++++--------------------
 odb/source-files.h |   9 ++++
 2 files changed, 100 insertions(+), 56 deletions(-)
diff --git a/odb/source-files.c b/odb/source-files.c
index e5e43b1543..9389546b3e 100644
--- a/odb/source-files.c
+++ b/odb/source-files.c
@@ -63,8 +63,15 @@ static void odb_source_files_reparent(const char *old_cwd,
 static void odb_source_files_free(struct odb_source *source)
 {
 	struct odb_source_files *files = odb_source_files_downcast(source);
+
 	chdir_notify_unregister(odb_source_files_reparent, files);
-	odb_files_dir_free(files->dirs);
+
+	while (files->dirs) {
+		struct odb_files_dir *next = files->dirs->next;
+		odb_files_dir_free(files->dirs);
+		files->dirs = next;
+	}
+
 	odb_source_release(&files->base);
 	free(files);
 }
@@ -72,8 +79,11 @@ static void odb_source_files_free(struct odb_source *source)
 static void odb_source_files_close(struct odb_source *source)
 {
 	struct odb_source_files *files = odb_source_files_downcast(source);
-	odb_source_close(&files->dirs->loose->base);
-	odb_source_close(&files->dirs->packed->base);
+
+	for (struct odb_files_dir *dir = files->dirs; dir; dir = dir->next) {
+		odb_source_close(&dir->loose->base);
+		odb_source_close(&dir->packed->base);
+	}
 }
 
 static int odb_source_files_create_on_disk(struct odb_source *source,
@@ -168,8 +178,11 @@ static void odb_source_files_prepare(struct odb_source *source,
 				     enum odb_prepare_flags flags)
 {
 	struct odb_source_files *files = odb_source_files_downcast(source);
-	odb_source_prepare(&files->dirs->loose->base, flags);
-	odb_source_prepare(&files->dirs->packed->base, flags);
+
+	for (struct odb_files_dir *dir = files->dirs; dir; dir = dir->next) {
+		odb_source_prepare(&dir->loose->base, flags);
+		odb_source_prepare(&dir->packed->base, flags);
+	}
 }
 
 static enum odb_read_status odb_source_files_read_object_info(struct odb_source *source,
@@ -179,28 +192,33 @@ static enum odb_read_status odb_source_files_read_object_info(struct odb_source
 							      struct strbuf *errmsg)
 {
 	struct odb_source_files *files = odb_source_files_downcast(source);
-	enum odb_read_status ret_packed, ret_loose;
-
-	ret_packed = odb_source_read_object_info(&files->dirs->packed->base, oid, oi,
-						 flags, errmsg);
-	if (!ret_packed)
-		return 0;
-
-	ret_loose = odb_source_read_object_info(&files->dirs->loose->base, oid, oi, flags,
-						ret_packed == ODB_READ_NOT_FOUND ? errmsg : NULL);
-	if (!ret_loose)
-		return 0;
+	enum odb_read_status status = ODB_READ_NOT_FOUND;
 
 	/*
-	 * Reading the packed object may have failed even though the object
-	 * exists, for example because it is corrupt. Report this failure to
-	 * the caller in case neither of the sources was able to read the
-	 * object, and prefer the error of the packed source in case both
-	 * reads have failed.
+	 * Reading an object may fail even though the object exists, for
+	 * example because it is corrupt. Report this failure to the caller in
+	 * case none of the directories was able to read the object, and
+	 * prefer the first such error in case multiple reads have failed.
 	 */
-	if (ret_packed != ODB_READ_NOT_FOUND)
-		return ret_packed;
-	return ret_loose;
+	for (struct odb_files_dir *dir = files->dirs; dir; dir = dir->next) {
+		enum odb_read_status ret;
+
+		ret = odb_source_read_object_info(&dir->packed->base, oid, oi, flags,
+						  status == ODB_READ_NOT_FOUND ? errmsg : NULL);
+		if (!ret)
+			return 0;
+		if (ret != ODB_READ_NOT_FOUND && status == ODB_READ_NOT_FOUND)
+			status = ret;
+
+		ret = odb_source_read_object_info(&dir->loose->base, oid, oi, flags,
+						  status == ODB_READ_NOT_FOUND ? errmsg : NULL);
+		if (!ret)
+			return 0;
+		if (ret != ODB_READ_NOT_FOUND && status == ODB_READ_NOT_FOUND)
+			status = ret;
+	}
+
+	return status;
 }
 
 static int odb_source_files_read_object_stream(struct odb_stream **out,
@@ -208,9 +226,12 @@ static int odb_source_files_read_object_stream(struct odb_stream **out,
 					       const struct object_id *oid)
 {
 	struct odb_source_files *files = odb_source_files_downcast(source);
-	if (!odb_source_read_object_stream(out, &files->dirs->packed->base, oid) ||
-	    !odb_source_read_object_stream(out, &files->dirs->loose->base, oid))
-		return 0;
+
+	for (struct odb_files_dir *dir = files->dirs; dir; dir = dir->next)
+		if (!odb_source_read_object_stream(out, &dir->packed->base, oid) ||
+		    !odb_source_read_object_stream(out, &dir->loose->base, oid))
+			return 0;
+
 	return -1;
 }
 
@@ -223,16 +244,21 @@ static int odb_source_files_for_each_object(struct odb_source *source,
 	struct odb_source_files *files = odb_source_files_downcast(source);
 	int ret;
 
-	if (!(opts->flags & ODB_FOR_EACH_OBJECT_PROMISOR_ONLY)) {
-		ret = odb_source_for_each_object(&files->dirs->loose->base, request, cb, cb_data, opts);
+	for (struct odb_files_dir *dir = files->dirs; dir; dir = dir->next) {
+		if (opts->flags & ODB_FOR_EACH_OBJECT_LOCAL_ONLY && !dir->local)
+			continue;
+
+		if (!(opts->flags & ODB_FOR_EACH_OBJECT_PROMISOR_ONLY)) {
+			ret = odb_source_for_each_object(&dir->loose->base, request, cb, cb_data, opts);
+			if (ret)
+				return ret;
+		}
+
+		ret = odb_source_for_each_object(&dir->packed->base, request, cb, cb_data, opts);
 		if (ret)
 			return ret;
 	}
 
-	ret = odb_source_for_each_object(&files->dirs->packed->base, request, cb, cb_data, opts);
-	if (ret)
-		return ret;
-
 	return 0;
 }
 
@@ -241,21 +267,23 @@ static int odb_source_files_count_objects(struct odb_source *source,
 					  unsigned long *out)
 {
 	struct odb_source_files *files = odb_source_files_downcast(source);
-	unsigned long count;
+	unsigned long count = 0;
 	int ret;
 
-	ret = odb_source_count_objects(&files->dirs->packed->base, flags, &count);
-	if (ret < 0)
-		goto out;
+	for (struct odb_files_dir *dir = files->dirs; dir; dir = dir->next) {
+		unsigned long dir_count;
 
-	if (!(flags & ODB_COUNT_OBJECTS_APPROXIMATE)) {
-		unsigned long loose_count;
-
-		ret = odb_source_count_objects(&files->dirs->loose->base, flags, &loose_count);
+		ret = odb_source_count_objects(&dir->packed->base, flags, &dir_count);
 		if (ret < 0)
 			goto out;
+		count += dir_count;
 
-		count += loose_count;
+		if (!(flags & ODB_COUNT_OBJECTS_APPROXIMATE)) {
+			ret = odb_source_count_objects(&dir->loose->base, flags, &dir_count);
+			if (ret < 0)
+				goto out;
+			count += dir_count;
+		}
 	}
 
 	*out = count;
@@ -272,15 +300,17 @@ static int odb_source_files_find_abbrev_len(struct odb_source *source,
 {
 	struct odb_source_files *files = odb_source_files_downcast(source);
 	unsigned len = min_len;
-	int ret;
+	int ret = 0;
 
-	ret = odb_source_find_abbrev_len(&files->dirs->packed->base, oid, len, &len);
-	if (ret < 0)
-		goto out;
+	for (struct odb_files_dir *dir = files->dirs; dir; dir = dir->next) {
+		ret = odb_source_find_abbrev_len(&dir->packed->base, oid, len, &len);
+		if (ret < 0)
+			goto out;
 
-	ret = odb_source_find_abbrev_len(&files->dirs->loose->base, oid, len, &len);
-	if (ret < 0)
-		goto out;
+		ret = odb_source_find_abbrev_len(&dir->loose->base, oid, len, &len);
+		if (ret < 0)
+			goto out;
+	}
 
 	*out = len;
 	ret = 0;
@@ -294,9 +324,12 @@ static int odb_source_files_freshen_object(struct odb_source *source,
 					   const time_t *mtime)
 {
 	struct odb_source_files *files = odb_source_files_downcast(source);
-	if (odb_source_freshen_object(&files->dirs->packed->base, oid, mtime) ||
-	    odb_source_freshen_object(&files->dirs->loose->base, oid, mtime))
-		return 1;
+
+	for (struct odb_files_dir *dir = files->dirs; dir; dir = dir->next)
+		if (odb_source_freshen_object(&dir->packed->base, oid, mtime) ||
+		    odb_source_freshen_object(&dir->loose->base, oid, mtime))
+			return 1;
+
 	return 0;
 }
 
@@ -958,11 +991,13 @@ static int odb_source_files_fsck(struct odb_source *source,
 	struct odb_source_files *files = odb_source_files_downcast(source);
 	int ret = 0;
 
-	if (!(opts->flags & ODB_FSCK_FULL) && !source->local)
-		return 0;
+	for (struct odb_files_dir *dir = files->dirs; dir; dir = dir->next) {
+		if (!(opts->flags & ODB_FSCK_FULL) && !dir->local)
+			continue;
 
-	ret |= odb_source_fsck(&files->dirs->loose->base, opts);
-	ret |= odb_source_fsck(&files->dirs->packed->base, opts);
+		ret |= odb_source_fsck(&dir->loose->base, opts);
+		ret |= odb_source_fsck(&dir->packed->base, opts);
+	}
 
 	return ret;
 }
diff --git a/odb/source-files.h b/odb/source-files.h
index 77f4d842e0..36af0c1b8b 100644
--- a/odb/source-files.h
+++ b/odb/source-files.h
@@ -15,6 +15,9 @@ struct odb_files_dir {
 	/* Absolute path to the object directory. */
 	char *abspath;
 
+	/* List of alternate object directories. */
+	struct odb_files_dir *next;
+
 	/* The two sources derived from this object directory. */
 	struct odb_source_loose *loose;
 	struct odb_source_packed *packed;
@@ -36,6 +39,12 @@ void odb_files_dir_free(struct odb_files_dir *dir);
  */
 struct odb_source_files {
 	struct odb_source base;
+
+	/*
+	 * List of all object directories; the main directory is first (and
+	 * cannot be NULL after initialization). Subsequent directories are
+	 * alternates.
+	 */
 	struct odb_files_dir *dirs;
 };
 

-- 
2.56.0.406.ga2d225a756.dirty
Previous: Patrick SteinhardtNext: Patrick Steinhardt
Message 33 of 41 in “odb/source-files: move alternates into the backend”
  1. 00/13 odb/source-files: move alternates into the backendPatrick Steinhardt, Oct 2, 2026
  2. 01/13 commit-graph: require resolved packfile paths for `stdin_packs`Patrick Steinhardt, Oct 2, 2026
  3. Karthik NayakOct 5, 2026
  4. Patrick SteinhardtOct 6, 2026
  5. 02/13 commit-graph: stop depending on `struct odb_source`Patrick Steinhardt, Oct 2, 2026
  6. Karthik NayakOct 5, 2026
  7. Patrick SteinhardtOct 6, 2026
  8. Karthik NayakOct 6, 2026
  9. 03/13 odb/source-files: introduce `struct odb_files_dir`Patrick Steinhardt, Oct 2, 2026
  10. 04/13 odb: refactor `odb_for_each_alternate()` to yield dirsPatrick Steinhardt, Oct 2, 2026
  11. Karthik NayakOct 6, 2026
  12. 05/13 odb: refactor `odb_find_source()` to yield dirsPatrick Steinhardt, Oct 2, 2026
  13. 06/13 odb/source-files: add the ability to have multiple object dirsPatrick Steinhardt, Oct 2, 2026
  14. 07/13 tmp-objdir: absorb logic to set and restore primary sourcesPatrick Steinhardt, Oct 2, 2026
  15. 08/13 tmp-objdir: manage quarantine as an object directoryPatrick Steinhardt, Oct 2, 2026
  16. Karthik NayakOct 6, 2026
  17. Karthik NayakOct 6, 2026
  18. 09/13 tmp-objdir: replace primary source at creation timePatrick Steinhardt, Oct 2, 2026
  19. 10/13 odb/source: make `will_destroy` an implementation detailPatrick Steinhardt, Oct 2, 2026
  20. 11/13 odb/source-files: extract reading alternatesPatrick Steinhardt, Oct 2, 2026
  21. 12/13 odb/source-files: move alternates into the backendPatrick Steinhardt, Oct 2, 2026
  22. Karthik NayakOct 6, 2026
  23. Patrick SteinhardtOct 7, 2026
  24. 13/13 odb/source: drop `read_alternates` callbackPatrick Steinhardt, Oct 2, 2026
  25. Karthik NayakOct 6, 2026
  26. Patrick SteinhardtOct 7, 2026
  27. 00/13 odb/source-files: move alternates into the backendPatrick Steinhardt, Oct 8, 2026
  28. 01/13 commit-graph: require resolved packfile paths for `stdin_packs`Patrick Steinhardt, Oct 8, 2026
  29. 02/13 commit-graph: stop depending on `struct odb_source`Patrick Steinhardt, Oct 8, 2026
  30. 03/13 odb/source-files: introduce `struct odb_files_dir`Patrick Steinhardt, Oct 8, 2026
  31. 04/13 odb: refactor `odb_for_each_alternate()` to yield dirsPatrick Steinhardt, Oct 8, 2026
  32. 05/13 odb: refactor `odb_find_source()` to yield dirsPatrick Steinhardt, Oct 8, 2026
  33. 06/13 odb/source-files: add the ability to have multiple object dirsPatrick Steinhardt, Oct 8, 2026
  34. 07/13 tmp-objdir: absorb logic to set and restore primary sourcesPatrick Steinhardt, Oct 8, 2026
  35. 08/13 tmp-objdir: manage quarantine as an object directoryPatrick Steinhardt, Oct 8, 2026
  36. 09/13 tmp-objdir: replace primary source at creation timePatrick Steinhardt, Oct 8, 2026
  37. 10/13 odb/source: make `will_destroy` an implementation detailPatrick Steinhardt, Oct 8, 2026
  38. 11/13 odb/source-files: extract reading alternatesPatrick Steinhardt, Oct 8, 2026
  39. 12/13 odb/source-files: move alternates into the backendPatrick Steinhardt, Oct 8, 2026
  40. 13/13 odb/source: drop `read_alternates` callbackPatrick Steinhardt, Oct 8, 2026
  41. Karthik NayakOct 8, 2026

Read the whole thread, see it on lore, or plain text.

$ cat FOOTERMessages come from the public archive at lore.kernel.org/git, fetched every hour. The front page is chosen and written each morning by an AI editor and can be wrong; the threads themselves are the record. About and API. For agents: an MCP server at https://gitlist.dev/mcp, and any thread, story or person page as Markdown by adding .md to its URL (or sending Accept: text/markdown). Details in /llms.txt.