git/list[1] front-page[2] threads[3] people[4] search[5] about
 

Re: [PATCH v2 01/17] odb: introduce "in-memory" source

From
Karthik Nayak <karthik.188@gmail.com>
Date
Apr 9, 2026, 09:26 UTC
Message-ID
<CAOLa=ZRkctXNkpTqiTSTkvskajPZTid9WTG3fKr0YV641_5qrw@mail.gmail.com>
In-Reply-To
<20260409-b4-pks-odb-source-inmemory-v2-1-f02b4f1c0f13@pks.im>
Patrick Steinhardt <ps@pks.im> writes:
Show 25 quoted lines
> Next to our typical object database sources, each object database also
> has an implicit source of "cached" objects. These cached objects only
> exist in memory and some use cases:
>
>   - They contain evergreen objects that we expect to always exist, like
>     for example the empty tree.
>
>   - They can be used to store temporary objects that we don't want to
>     persist to disk, which is used by git-blame(1) to create a fake
>     worktree commit.
>
> Overall, their use is somewhat restricted though. For example, we don't
> provide the ability to use it as a temporary object database source that
> allows the user to write objects, but discard them after Git exists. So
> while these cached objects behave almost like a source, they aren't used
> as one.
>
> This is about to change over the following commits, where we will turn
> cached objects into a new "in-memory" source. This will allow us to use
> it exactly the same as any other source by providing the same common
> interface as the "files" source.
>
> For now, the in-memory source only hosts the cached objects and doesn't
> provide any logic yet. This will change with subsequent commits, where
> we move respective functionality into the source.
[snip]
Show 35 quoted lines
> diff --git a/odb.c b/odb.c
> index 40a5e9c4e0..60e1eead25 100644
> --- a/odb.c
> +++ b/odb.c
> @@ -14,6 +14,7 @@
>  #include "object-file.h"
>  #include "object-name.h"
>  #include "odb.h"
> +#include "odb/source-inmemory.h"
>  #include "packfile.h"
>  #include "path.h"
>  #include "promisor-remote.h"
> @@ -53,9 +54,9 @@ static const struct cached_object *find_cached_object(struct object_database *ob
>  		.type = OBJ_TREE,
>  		.buf = "",
>  	};
> -	const struct cached_object_entry *co = object_store->cached_objects;
> +	const struct cached_object_entry *co = object_store->inmemory_objects->objects;
>
> -	for (size_t i = 0; i < object_store->cached_object_nr; i++, co++)
> +	for (size_t i = 0; i < object_store->inmemory_objects->objects_nr; i++, co++)
>  		if (oideq(&co->oid, oid))
>  			return &co->value;
>
> @@ -792,9 +793,10 @@ int odb_pretend_object(struct object_database *odb,
>  	    find_cached_object(odb, oid))
>  		return 0;
>
> -	ALLOC_GROW(odb->cached_objects,
> -		   odb->cached_object_nr + 1, odb->cached_object_alloc);
> -	co = &odb->cached_objects[odb->cached_object_nr++];
> +	ALLOC_GROW(odb->inmemory_objects->objects,
> +		   odb->inmemory_objects->objects_nr + 1,
> +		   odb->inmemory_objects->objects_alloc);
> +	co = &odb->inmemory_objects->objects[odb->inmemory_objects->objects_nr++];

Okay so we introduce the inmemory object storage and directly write objects to it. I guess in the upcoming commits, we'll swap to using the API as we implement them.

Makes sense for now.
Show 106 quoted lines
>  	co->value.size = len;
>  	co->value.type = type;
>  	co_buf = xmalloc(len);
> @@ -1083,6 +1085,7 @@ struct object_database *odb_new(struct repository *repo,
>  	o->sources = odb_source_new(o, primary_source, true);
>  	o->sources_tail = &o->sources->next;
>  	o->alternate_db = xstrdup_or_null(secondary_sources);
> +	o->inmemory_objects = odb_source_inmemory_new(o);
>
>  	free(to_free);
>
> @@ -1123,9 +1126,11 @@ void odb_free(struct object_database *o)
>  	odb_close(o);
>  	odb_free_sources(o);
>
> -	for (size_t i = 0; i < o->cached_object_nr; i++)
> -		free((char *) o->cached_objects[i].value.buf);
> -	free(o->cached_objects);
> +	for (size_t i = 0; i < o->inmemory_objects->objects_nr; i++)
> +		free((char *) o->inmemory_objects->objects[i].value.buf);
> +	free(o->inmemory_objects->objects);
> +	free(o->inmemory_objects->base.path);
> +	free(o->inmemory_objects);
>
>  	string_list_clear(&o->submodule_source_paths, 0);
>
> diff --git a/odb.h b/odb.h
> index 9eb8355aca..c3a7edf9c8 100644
> --- a/odb.h
> +++ b/odb.h
> @@ -8,6 +8,7 @@
>  #include "thread-utils.h"
>
>  struct cached_object_entry;
> +struct odb_source_inmemory;
>  struct packed_git;
>  struct repository;
>  struct strbuf;
> @@ -80,8 +81,7 @@ struct object_database {
>  	 * to write them into the object store (e.g. a browse-only
>  	 * application).
>  	 */
> -	struct cached_object_entry *cached_objects;
> -	size_t cached_object_nr, cached_object_alloc;
> +	struct odb_source_inmemory *inmemory_objects;
>
>  	/*
>  	 * A fast, rough count of the number of objects in the repository.
> diff --git a/odb/source-inmemory.c b/odb/source-inmemory.c
> new file mode 100644
> index 0000000000..c7ac5c24f0
> --- /dev/null
> +++ b/odb/source-inmemory.c
> @@ -0,0 +1,12 @@
> +#include "git-compat-util.h"
> +#include "odb/source-inmemory.h"
> +
> +struct odb_source_inmemory *odb_source_inmemory_new(struct object_database *odb)
> +{
> +	struct odb_source_inmemory *source;
> +
> +	CALLOC_ARRAY(source, 1);
> +	odb_source_init(&source->base, odb, ODB_SOURCE_INMEMORY, "source", false);
> +
> +	return source;
> +}
> diff --git a/odb/source-inmemory.h b/odb/source-inmemory.h
> new file mode 100644
> index 0000000000..95477bf36d
> --- /dev/null
> +++ b/odb/source-inmemory.h
> @@ -0,0 +1,35 @@
> +#ifndef ODB_SOURCE_INMEMORY_H
> +#define ODB_SOURCE_INMEMORY_H
> +
> +#include "odb/source.h"
> +
> +struct cached_object_entry;
> +
> +/*
> + * An inmemory source that you can write objects to that shall be made
> + * available for reading, but that shouldn't ever be persisted to disk. Note
> + * that any objects written to this source will be stored in memory, so the
> + * number of objects you can store is limited by available system memory.
> + */
> +struct odb_source_inmemory {
> +	struct odb_source base;
> +
> +	struct cached_object_entry *objects;
> +	size_t objects_nr, objects_alloc;
> +};
> +
> +/* Create a new in-memory object database source. */
> +struct odb_source_inmemory *odb_source_inmemory_new(struct object_database *odb);
> +
> +/*
> + * Cast the given object database source to the inmemory backend. This will
> + * cause a BUG in case the source doesn't use this backend.
> + */
> +static inline struct odb_source_inmemory *odb_source_inmemory_downcast(struct odb_source *source)
> +{
> +	if (source->type != ODB_SOURCE_INMEMORY)
> +		BUG("trying to downcast source of type '%d' to inmemory", source->type);
> +	return container_of(source, struct odb_source_inmemory, base);
> +}
> +

Interesting, in the refs namespace the downcast functions are added to the source file (.c). This works too, is there any reason though?

[snip]
Previous: Patrick SteinhardtNext: Patrick Steinhardt
Message 35 of 85 in “odb: introduce "inmemory" source”
  1. 00/16 odb: introduce "inmemory" sourcePatrick Steinhardt, Apr 3, 2026
  2. 01/16 odb: introduce "inmemory" sourcePatrick Steinhardt, Apr 3, 2026
  3. Justin ToblerApr 8, 2026
  4. Patrick SteinhardtApr 9, 2026
  5. 02/16 odb/source-inmemory: implement `free()` callbackPatrick Steinhardt, Apr 3, 2026
  6. Justin ToblerApr 8, 2026
  7. 03/16 odb: fix unnecessary call to `find_cached_object()`Patrick Steinhardt, Apr 3, 2026
  8. Justin ToblerApr 8, 2026
  9. Patrick SteinhardtApr 9, 2026
  10. 04/16 odb/source-inmemory: implement `read_object_info()` callbackPatrick Steinhardt, Apr 3, 2026
  11. 05/16 odb/source-inmemory: implement `read_object_stream()` callbackPatrick Steinhardt, Apr 3, 2026
  12. Justin ToblerApr 8, 2026
  13. Patrick SteinhardtApr 9, 2026
  14. 06/16 odb/source-inmemory: implement `write_object()` callbackPatrick Steinhardt, Apr 3, 2026
  15. 07/16 odb/source-inmemory: implement `write_object_stream()` callbackPatrick Steinhardt, Apr 3, 2026
  16. Junio C HamanoApr 3, 2026
  17. Patrick SteinhardtApr 8, 2026
  18. 08/16 cbtree: allow using arbitrary wrapper structures for nodesPatrick Steinhardt, Apr 3, 2026
  19. 09/16 oidtree: add ability to store dataPatrick Steinhardt, Apr 3, 2026
  20. 10/16 odb/source-inmemory: convert to use oidtreePatrick Steinhardt, Apr 3, 2026
  21. 11/16 odb/source-inmemory: implement `for_each_object()` callbackPatrick Steinhardt, Apr 3, 2026
  22. 12/16 odb/source-inmemory: implement `find_abbrev_len()` callbackPatrick Steinhardt, Apr 3, 2026
  23. 13/16 odb/source-inmemory: implement `count_objects()` callbackPatrick Steinhardt, Apr 3, 2026
  24. 14/16 odb/source-inmemory: implement `freshen_object()` callbackPatrick Steinhardt, Apr 3, 2026
  25. 15/16 odb/source-inmemory: stub out remaining functionsPatrick Steinhardt, Apr 3, 2026
  26. 16/16 odb: generic inmemory sourcePatrick Steinhardt, Apr 3, 2026
  27. Junio C HamanoApr 3, 2026
  28. Patrick SteinhardtApr 8, 2026
  29. Junio C HamanoApr 8, 2026
  30. Patrick SteinhardtApr 9, 2026
  31. Junio C HamanoApr 9, 2026
  32. Patrick SteinhardtApr 10, 2026
  33. 00/17 odb: introduce "in-memory" sourcePatrick Steinhardt, Apr 9, 2026
  34. 01/17 odb: introduce "in-memory" sourcePatrick Steinhardt, Apr 9, 2026
  35. Karthik NayakApr 9, 2026
  36. Patrick SteinhardtApr 9, 2026
  37. 02/17 odb/source-inmemory: implement `free()` callbackPatrick Steinhardt, Apr 9, 2026
  38. 03/17 odb: fix unnecessary call to `find_cached_object()`Patrick Steinhardt, Apr 9, 2026
  39. 04/17 odb/source-inmemory: implement `read_object_info()` callbackPatrick Steinhardt, Apr 9, 2026
  40. Karthik NayakApr 9, 2026
  41. Patrick SteinhardtApr 9, 2026
  42. Karthik NayakApr 9, 2026
  43. 05/17 odb/source-inmemory: implement `read_object_stream()` callbackPatrick Steinhardt, Apr 9, 2026
  44. Karthik NayakApr 9, 2026
  45. Patrick SteinhardtApr 9, 2026
  46. 06/17 odb/source-inmemory: implement `write_object()` callbackPatrick Steinhardt, Apr 9, 2026
  47. 07/17 odb/source-inmemory: implement `write_object()` callbackPatrick Steinhardt, Apr 9, 2026
  48. Karthik NayakApr 9, 2026
  49. Patrick SteinhardtApr 9, 2026
  50. 08/17 odb/source-inmemory: implement `write_object_stream()` callbackPatrick Steinhardt, Apr 9, 2026
  51. 09/17 cbtree: allow using arbitrary wrapper structures for nodesPatrick Steinhardt, Apr 9, 2026
  52. Karthik NayakApr 9, 2026
  53. Patrick SteinhardtApr 9, 2026
  54. 10/17 oidtree: add ability to store dataPatrick Steinhardt, Apr 9, 2026
  55. 11/17 odb/source-inmemory: convert to use oidtreePatrick Steinhardt, Apr 9, 2026
  56. 12/17 odb/source-inmemory: implement `for_each_object()` callbackPatrick Steinhardt, Apr 9, 2026
  57. 13/17 odb/source-inmemory: implement `find_abbrev_len()` callbackPatrick Steinhardt, Apr 9, 2026
  58. 14/17 odb/source-inmemory: implement `count_objects()` callbackPatrick Steinhardt, Apr 9, 2026
  59. 15/17 odb/source-inmemory: implement `freshen_object()` callbackPatrick Steinhardt, Apr 9, 2026
  60. 16/17 odb/source-inmemory: stub out remaining functionsPatrick Steinhardt, Apr 9, 2026
  61. Junio C HamanoApr 9, 2026
  62. Patrick SteinhardtApr 10, 2026
  63. 17/17 odb: generic in-memory sourcePatrick Steinhardt, Apr 9, 2026
  64. Karthik NayakApr 9, 2026
  65. Patrick SteinhardtApr 9, 2026
  66. 00/17 odb: introduce "in-memory" sourcePatrick Steinhardt, Apr 10, 2026
  67. 01/17 odb: introduce "in-memory" sourcePatrick Steinhardt, Apr 10, 2026
  68. 02/17 odb/source-inmemory: implement `free()` callbackPatrick Steinhardt, Apr 10, 2026
  69. 03/17 odb: fix unnecessary call to `find_cached_object()`Patrick Steinhardt, Apr 10, 2026
  70. 04/17 odb/source-inmemory: implement `read_object_info()` callbackPatrick Steinhardt, Apr 10, 2026
  71. 05/17 odb/source-inmemory: implement `read_object_stream()` callbackPatrick Steinhardt, Apr 10, 2026
  72. 06/17 odb/source-inmemory: implement `write_object()` callbackPatrick Steinhardt, Apr 10, 2026
  73. 07/17 odb/source-inmemory: implement `write_object_stream()` callbackPatrick Steinhardt, Apr 10, 2026
  74. 08/17 cbtree: allow using arbitrary wrapper structures for nodesPatrick Steinhardt, Apr 10, 2026
  75. 09/17 oidtree: add ability to store dataPatrick Steinhardt, Apr 10, 2026
  76. 10/17 odb/source-inmemory: convert to use oidtreePatrick Steinhardt, Apr 10, 2026
  77. 11/17 odb/source-inmemory: implement `for_each_object()` callbackPatrick Steinhardt, Apr 10, 2026
  78. 12/17 odb/source-inmemory: implement `find_abbrev_len()` callbackPatrick Steinhardt, Apr 10, 2026
  79. 13/17 odb/source-inmemory: implement `count_objects()` callbackPatrick Steinhardt, Apr 10, 2026
  80. 14/17 odb/source-inmemory: implement `freshen_object()` callbackPatrick Steinhardt, Apr 10, 2026
  81. 15/17 odb/source-inmemory: stub out remaining functionsPatrick Steinhardt, Apr 10, 2026
  82. 16/17 odb: generic in-memory sourcePatrick Steinhardt, Apr 10, 2026
  83. 17/17 t/unit-tests: add tests for the in-memory object sourcePatrick Steinhardt, Apr 10, 2026
  84. Karthik NayakApr 14, 2026
  85. Karthik NayakApr 14, 2026

Read the whole thread, see it on lore, or plain text.

$ cat FOOTERMessages come from the public archive at lore.kernel.org/git, fetched every hour. The front page is chosen and written each morning by an AI editor and can be wrong; the threads themselves are the record. About and API. For agents: an MCP server at https://gitlist.dev/mcp, and any thread, story or person page as Markdown by adding .md to its URL (or sending Accept: text/markdown). Details in /llms.txt.