git/list[1] front-page[2] threads[3] people[4] search[5] about
 

[PATCH v2 01/17] odb: introduce "in-memory" source

From
Patrick Steinhardt <ps@pks.im>
Date
Apr 9, 2026, 07:24 UTC
Message-ID
<20260409-b4-pks-odb-source-inmemory-v2-1-f02b4f1c0f13@pks.im>
In-Reply-To
<20260409-b4-pks-odb-source-inmemory-v2-0-f02b4f1c0f13@pks.im>

Next to our typical object database sources, each object database also has an implicit source of "cached" objects. These cached objects only exist in memory and some use cases:

  - They contain evergreen objects that we expect to always exist, like
    for example the empty tree.
  - They can be used to store temporary objects that we don't want to
    persist to disk, which is used by git-blame(1) to create a fake
    worktree commit.

Overall, their use is somewhat restricted though. For example, we don't provide the ability to use it as a temporary object database source that allows the user to write objects, but discard them after Git exists. So while these cached objects behave almost like a source, they aren't used as one.

This is about to change over the following commits, where we will turn cached objects into a new "in-memory" source. This will allow us to use it exactly the same as any other source by providing the same common interface as the "files" source.

For now, the in-memory source only hosts the cached objects and doesn't provide any logic yet. This will change with subsequent commits, where we move respective functionality into the source.

Signed-off-by: Patrick Steinhardt <ps@pks.im>
---
 Makefile              |  1 +
 meson.build           |  1 +
 odb.c                 | 21 +++++++++++++--------
 odb.h                 |  4 ++--
 odb/source-inmemory.c | 12 ++++++++++++
 odb/source-inmemory.h | 35 +++++++++++++++++++++++++++++++++++
 odb/source.h          |  3 +++
 7 files changed, 67 insertions(+), 10 deletions(-)
diff --git a/Makefile b/Makefile
index 22a8993482..3cda12c455 100644
--- a/Makefile
+++ b/Makefile
@@ -1218,6 +1218,7 @@ LIB_OBJS += object.o
 LIB_OBJS += odb.o
 LIB_OBJS += odb/source.o
 LIB_OBJS += odb/source-files.o
+LIB_OBJS += odb/source-inmemory.o
 LIB_OBJS += odb/streaming.o
 LIB_OBJS += odb/transaction.o
 LIB_OBJS += oid-array.o
diff --git a/meson.build b/meson.build
index 6dc23b3af2..ffa73ce7ce 100644
--- a/meson.build
+++ b/meson.build
@@ -404,6 +404,7 @@ libgit_sources = [
   'odb.c',
   'odb/source.c',
   'odb/source-files.c',
+  'odb/source-inmemory.c',
   'odb/streaming.c',
   'odb/transaction.c',
   'oid-array.c',
diff --git a/odb.c b/odb.c
index 40a5e9c4e0..60e1eead25 100644
--- a/odb.c
+++ b/odb.c
@@ -14,6 +14,7 @@
 #include "object-file.h"
 #include "object-name.h"
 #include "odb.h"
+#include "odb/source-inmemory.h"
 #include "packfile.h"
 #include "path.h"
 #include "promisor-remote.h"
@@ -53,9 +54,9 @@ static const struct cached_object *find_cached_object(struct object_database *ob
 		.type = OBJ_TREE,
 		.buf = "",
 	};
-	const struct cached_object_entry *co = object_store->cached_objects;
+	const struct cached_object_entry *co = object_store->inmemory_objects->objects;
 
-	for (size_t i = 0; i < object_store->cached_object_nr; i++, co++)
+	for (size_t i = 0; i < object_store->inmemory_objects->objects_nr; i++, co++)
 		if (oideq(&co->oid, oid))
 			return &co->value;
 
@@ -792,9 +793,10 @@ int odb_pretend_object(struct object_database *odb,
 	    find_cached_object(odb, oid))
 		return 0;
 
-	ALLOC_GROW(odb->cached_objects,
-		   odb->cached_object_nr + 1, odb->cached_object_alloc);
-	co = &odb->cached_objects[odb->cached_object_nr++];
+	ALLOC_GROW(odb->inmemory_objects->objects,
+		   odb->inmemory_objects->objects_nr + 1,
+		   odb->inmemory_objects->objects_alloc);
+	co = &odb->inmemory_objects->objects[odb->inmemory_objects->objects_nr++];
 	co->value.size = len;
 	co->value.type = type;
 	co_buf = xmalloc(len);
@@ -1083,6 +1085,7 @@ struct object_database *odb_new(struct repository *repo,
 	o->sources = odb_source_new(o, primary_source, true);
 	o->sources_tail = &o->sources->next;
 	o->alternate_db = xstrdup_or_null(secondary_sources);
+	o->inmemory_objects = odb_source_inmemory_new(o);
 
 	free(to_free);
 
@@ -1123,9 +1126,11 @@ void odb_free(struct object_database *o)
 	odb_close(o);
 	odb_free_sources(o);
 
-	for (size_t i = 0; i < o->cached_object_nr; i++)
-		free((char *) o->cached_objects[i].value.buf);
-	free(o->cached_objects);
+	for (size_t i = 0; i < o->inmemory_objects->objects_nr; i++)
+		free((char *) o->inmemory_objects->objects[i].value.buf);
+	free(o->inmemory_objects->objects);
+	free(o->inmemory_objects->base.path);
+	free(o->inmemory_objects);
 
 	string_list_clear(&o->submodule_source_paths, 0);
 
diff --git a/odb.h b/odb.h
index 9eb8355aca..c3a7edf9c8 100644
--- a/odb.h
+++ b/odb.h
@@ -8,6 +8,7 @@
 #include "thread-utils.h"
 
 struct cached_object_entry;
+struct odb_source_inmemory;
 struct packed_git;
 struct repository;
 struct strbuf;
@@ -80,8 +81,7 @@ struct object_database {
 	 * to write them into the object store (e.g. a browse-only
 	 * application).
 	 */
-	struct cached_object_entry *cached_objects;
-	size_t cached_object_nr, cached_object_alloc;
+	struct odb_source_inmemory *inmemory_objects;
 
 	/*
 	 * A fast, rough count of the number of objects in the repository.
diff --git a/odb/source-inmemory.c b/odb/source-inmemory.c
new file mode 100644
index 0000000000..c7ac5c24f0
--- /dev/null
+++ b/odb/source-inmemory.c
@@ -0,0 +1,12 @@
+#include "git-compat-util.h"
+#include "odb/source-inmemory.h"
+
+struct odb_source_inmemory *odb_source_inmemory_new(struct object_database *odb)
+{
+	struct odb_source_inmemory *source;
+
+	CALLOC_ARRAY(source, 1);
+	odb_source_init(&source->base, odb, ODB_SOURCE_INMEMORY, "source", false);
+
+	return source;
+}
diff --git a/odb/source-inmemory.h b/odb/source-inmemory.h
new file mode 100644
index 0000000000..95477bf36d
--- /dev/null
+++ b/odb/source-inmemory.h
@@ -0,0 +1,35 @@
+#ifndef ODB_SOURCE_INMEMORY_H
+#define ODB_SOURCE_INMEMORY_H
+
+#include "odb/source.h"
+
+struct cached_object_entry;
+
+/*
+ * An inmemory source that you can write objects to that shall be made
+ * available for reading, but that shouldn't ever be persisted to disk. Note
+ * that any objects written to this source will be stored in memory, so the
+ * number of objects you can store is limited by available system memory.
+ */
+struct odb_source_inmemory {
+	struct odb_source base;
+
+	struct cached_object_entry *objects;
+	size_t objects_nr, objects_alloc;
+};
+
+/* Create a new in-memory object database source. */
+struct odb_source_inmemory *odb_source_inmemory_new(struct object_database *odb);
+
+/*
+ * Cast the given object database source to the inmemory backend. This will
+ * cause a BUG in case the source doesn't use this backend.
+ */
+static inline struct odb_source_inmemory *odb_source_inmemory_downcast(struct odb_source *source)
+{
+	if (source->type != ODB_SOURCE_INMEMORY)
+		BUG("trying to downcast source of type '%d' to inmemory", source->type);
+	return container_of(source, struct odb_source_inmemory, base);
+}
+
+#endif
diff --git a/odb/source.h b/odb/source.h
index f706e0608a..cd14f9e046 100644
--- a/odb/source.h
+++ b/odb/source.h
@@ -13,6 +13,9 @@ enum odb_source_type {
 
 	/* The "files" backend that uses loose objects and packfiles. */
 	ODB_SOURCE_FILES,
+
+	/* The "inmemory" backend that stores objects in memory. */
+	ODB_SOURCE_INMEMORY,
 };
 
 struct object_id;
-- 
2.54.0.rc0.680.geaeac8ef83.dirty
Previous: Patrick SteinhardtNext: Karthik Nayak
Message 34 of 85 in “odb: introduce "inmemory" source”
  1. 00/16 odb: introduce "inmemory" sourcePatrick Steinhardt, Apr 3, 2026
  2. 01/16 odb: introduce "inmemory" sourcePatrick Steinhardt, Apr 3, 2026
  3. Justin ToblerApr 8, 2026
  4. Patrick SteinhardtApr 9, 2026
  5. 02/16 odb/source-inmemory: implement `free()` callbackPatrick Steinhardt, Apr 3, 2026
  6. Justin ToblerApr 8, 2026
  7. 03/16 odb: fix unnecessary call to `find_cached_object()`Patrick Steinhardt, Apr 3, 2026
  8. Justin ToblerApr 8, 2026
  9. Patrick SteinhardtApr 9, 2026
  10. 04/16 odb/source-inmemory: implement `read_object_info()` callbackPatrick Steinhardt, Apr 3, 2026
  11. 05/16 odb/source-inmemory: implement `read_object_stream()` callbackPatrick Steinhardt, Apr 3, 2026
  12. Justin ToblerApr 8, 2026
  13. Patrick SteinhardtApr 9, 2026
  14. 06/16 odb/source-inmemory: implement `write_object()` callbackPatrick Steinhardt, Apr 3, 2026
  15. 07/16 odb/source-inmemory: implement `write_object_stream()` callbackPatrick Steinhardt, Apr 3, 2026
  16. Junio C HamanoApr 3, 2026
  17. Patrick SteinhardtApr 8, 2026
  18. 08/16 cbtree: allow using arbitrary wrapper structures for nodesPatrick Steinhardt, Apr 3, 2026
  19. 09/16 oidtree: add ability to store dataPatrick Steinhardt, Apr 3, 2026
  20. 10/16 odb/source-inmemory: convert to use oidtreePatrick Steinhardt, Apr 3, 2026
  21. 11/16 odb/source-inmemory: implement `for_each_object()` callbackPatrick Steinhardt, Apr 3, 2026
  22. 12/16 odb/source-inmemory: implement `find_abbrev_len()` callbackPatrick Steinhardt, Apr 3, 2026
  23. 13/16 odb/source-inmemory: implement `count_objects()` callbackPatrick Steinhardt, Apr 3, 2026
  24. 14/16 odb/source-inmemory: implement `freshen_object()` callbackPatrick Steinhardt, Apr 3, 2026
  25. 15/16 odb/source-inmemory: stub out remaining functionsPatrick Steinhardt, Apr 3, 2026
  26. 16/16 odb: generic inmemory sourcePatrick Steinhardt, Apr 3, 2026
  27. Junio C HamanoApr 3, 2026
  28. Patrick SteinhardtApr 8, 2026
  29. Junio C HamanoApr 8, 2026
  30. Patrick SteinhardtApr 9, 2026
  31. Junio C HamanoApr 9, 2026
  32. Patrick SteinhardtApr 10, 2026
  33. 00/17 odb: introduce "in-memory" sourcePatrick Steinhardt, Apr 9, 2026
  34. 01/17 odb: introduce "in-memory" sourcePatrick Steinhardt, Apr 9, 2026
  35. Karthik NayakApr 9, 2026
  36. Patrick SteinhardtApr 9, 2026
  37. 02/17 odb/source-inmemory: implement `free()` callbackPatrick Steinhardt, Apr 9, 2026
  38. 03/17 odb: fix unnecessary call to `find_cached_object()`Patrick Steinhardt, Apr 9, 2026
  39. 04/17 odb/source-inmemory: implement `read_object_info()` callbackPatrick Steinhardt, Apr 9, 2026
  40. Karthik NayakApr 9, 2026
  41. Patrick SteinhardtApr 9, 2026
  42. Karthik NayakApr 9, 2026
  43. 05/17 odb/source-inmemory: implement `read_object_stream()` callbackPatrick Steinhardt, Apr 9, 2026
  44. Karthik NayakApr 9, 2026
  45. Patrick SteinhardtApr 9, 2026
  46. 06/17 odb/source-inmemory: implement `write_object()` callbackPatrick Steinhardt, Apr 9, 2026
  47. 07/17 odb/source-inmemory: implement `write_object()` callbackPatrick Steinhardt, Apr 9, 2026
  48. Karthik NayakApr 9, 2026
  49. Patrick SteinhardtApr 9, 2026
  50. 08/17 odb/source-inmemory: implement `write_object_stream()` callbackPatrick Steinhardt, Apr 9, 2026
  51. 09/17 cbtree: allow using arbitrary wrapper structures for nodesPatrick Steinhardt, Apr 9, 2026
  52. Karthik NayakApr 9, 2026
  53. Patrick SteinhardtApr 9, 2026
  54. 10/17 oidtree: add ability to store dataPatrick Steinhardt, Apr 9, 2026
  55. 11/17 odb/source-inmemory: convert to use oidtreePatrick Steinhardt, Apr 9, 2026
  56. 12/17 odb/source-inmemory: implement `for_each_object()` callbackPatrick Steinhardt, Apr 9, 2026
  57. 13/17 odb/source-inmemory: implement `find_abbrev_len()` callbackPatrick Steinhardt, Apr 9, 2026
  58. 14/17 odb/source-inmemory: implement `count_objects()` callbackPatrick Steinhardt, Apr 9, 2026
  59. 15/17 odb/source-inmemory: implement `freshen_object()` callbackPatrick Steinhardt, Apr 9, 2026
  60. 16/17 odb/source-inmemory: stub out remaining functionsPatrick Steinhardt, Apr 9, 2026
  61. Junio C HamanoApr 9, 2026
  62. Patrick SteinhardtApr 10, 2026
  63. 17/17 odb: generic in-memory sourcePatrick Steinhardt, Apr 9, 2026
  64. Karthik NayakApr 9, 2026
  65. Patrick SteinhardtApr 9, 2026
  66. 00/17 odb: introduce "in-memory" sourcePatrick Steinhardt, Apr 10, 2026
  67. 01/17 odb: introduce "in-memory" sourcePatrick Steinhardt, Apr 10, 2026
  68. 02/17 odb/source-inmemory: implement `free()` callbackPatrick Steinhardt, Apr 10, 2026
  69. 03/17 odb: fix unnecessary call to `find_cached_object()`Patrick Steinhardt, Apr 10, 2026
  70. 04/17 odb/source-inmemory: implement `read_object_info()` callbackPatrick Steinhardt, Apr 10, 2026
  71. 05/17 odb/source-inmemory: implement `read_object_stream()` callbackPatrick Steinhardt, Apr 10, 2026
  72. 06/17 odb/source-inmemory: implement `write_object()` callbackPatrick Steinhardt, Apr 10, 2026
  73. 07/17 odb/source-inmemory: implement `write_object_stream()` callbackPatrick Steinhardt, Apr 10, 2026
  74. 08/17 cbtree: allow using arbitrary wrapper structures for nodesPatrick Steinhardt, Apr 10, 2026
  75. 09/17 oidtree: add ability to store dataPatrick Steinhardt, Apr 10, 2026
  76. 10/17 odb/source-inmemory: convert to use oidtreePatrick Steinhardt, Apr 10, 2026
  77. 11/17 odb/source-inmemory: implement `for_each_object()` callbackPatrick Steinhardt, Apr 10, 2026
  78. 12/17 odb/source-inmemory: implement `find_abbrev_len()` callbackPatrick Steinhardt, Apr 10, 2026
  79. 13/17 odb/source-inmemory: implement `count_objects()` callbackPatrick Steinhardt, Apr 10, 2026
  80. 14/17 odb/source-inmemory: implement `freshen_object()` callbackPatrick Steinhardt, Apr 10, 2026
  81. 15/17 odb/source-inmemory: stub out remaining functionsPatrick Steinhardt, Apr 10, 2026
  82. 16/17 odb: generic in-memory sourcePatrick Steinhardt, Apr 10, 2026
  83. 17/17 t/unit-tests: add tests for the in-memory object sourcePatrick Steinhardt, Apr 10, 2026
  84. Karthik NayakApr 14, 2026
  85. Karthik NayakApr 14, 2026

Read the whole thread, see it on lore, or plain text.

$ cat FOOTERMessages come from the public archive at lore.kernel.org/git, fetched every hour. The front page is chosen and written each morning by an AI editor and can be wrong; the threads themselves are the record. About and API. For agents: an MCP server at https://gitlist.dev/mcp, and any thread, story or person page as Markdown by adding .md to its URL (or sending Accept: text/markdown). Details in /llms.txt.