git/list[1] front-page[2] threads[3] people[4] search[5] about
 

Re: [PATCH v3 00/17] odb: introduce "in-memory" source

From
Karthik Nayak <karthik.188@gmail.com>
Date
Apr 14, 2026, 08:27 UTC
Message-ID
<CAOLa=ZSrThty13-C_WVa5dvakZAtidwOXWnUrOA4LGX93DvmGQ@mail.gmail.com>
In-Reply-To
<20260410-b4-pks-odb-source-inmemory-v3-0-22fd0fad58fe@pks.im>

Patrick Steinhardt <ps@pks.im> writes: [snip]

Show 53 quoted lines
> Range-diff versus v2:
>
>  1:  b18e427c69 !  1:  155b2cdf81 odb: introduce "in-memory" source
>     @@ odb/source-inmemory.h (new)
>      +struct cached_object_entry;
>      +
>      +/*
>     -+ * An inmemory source that you can write objects to that shall be made
>     ++ * An in-memory source that you can write objects to that shall be made
>      + * available for reading, but that shouldn't ever be persisted to disk. Note
>      + * that any objects written to this source will be stored in memory, so the
>      + * number of objects you can store is limited by available system memory.
>     @@ odb/source-inmemory.h (new)
>      +struct odb_source_inmemory *odb_source_inmemory_new(struct object_database *odb);
>      +
>      +/*
>     -+ * Cast the given object database source to the inmemory backend. This will
>     ++ * Cast the given object database source to the in-memory backend. This will
>      + * cause a BUG in case the source doesn't use this backend.
>      + */
>      +static inline struct odb_source_inmemory *odb_source_inmemory_downcast(struct odb_source *source)
>      +{
>      +	if (source->type != ODB_SOURCE_INMEMORY)
>     -+		BUG("trying to downcast source of type '%d' to inmemory", source->type);
>     ++		BUG("trying to downcast source of type '%d' to in-memory", source->type);
>      +	return container_of(source, struct odb_source_inmemory, base);
>      +}
>      +
>     @@ odb/source.h: enum odb_source_type {
>       	/* The "files" backend that uses loose objects and packfiles. */
>       	ODB_SOURCE_FILES,
>      +
>     -+	/* The "inmemory" backend that stores objects in memory. */
>     ++	/* The "in-memory" backend that stores objects in memory. */
>      +	ODB_SOURCE_INMEMORY,
>       };
>
>  2:  8fd337da90 !  2:  c66edd10a8 odb/source-inmemory: implement `free()` callback
>     @@ odb/source-inmemory.h
>      +};
>
>       /*
>     -  * An inmemory source that you can write objects to that shall be made
>     +  * An in-memory source that you can write objects to that shall be made
>  3:  f4ae2a2bde =  3:  a86549f39c odb: fix unnecessary call to `find_cached_object()`
>  4:  8600b88530 =  4:  49ac739dd2 odb/source-inmemory: implement `read_object_info()` callback
>  5:  ab33c0b7ee !  5:  321ef11be3 odb/source-inmemory: implement `read_object_stream()` callback
>     @@ odb/source-inmemory.c: static int odb_source_inmemory_read_object_info(struct od
>
>      +struct odb_read_stream_inmemory {
>      +	struct odb_read_stream base;
>     -+	const void *buf;
>     ++	const unsigned char *buf;
Okay this does make more sense.
Show 11 quoted lines
>      +	size_t offset;
>      +};
>      +
>     @@ odb/source-inmemory.c: static int odb_source_inmemory_read_object_info(struct od
>      +
>      +	if (buf_len > inmemory->base.size - inmemory->offset)
>      +		bytes = inmemory->base.size - inmemory->offset;
>     -+	memcpy(buf, inmemory->buf, bytes);
>     ++
>     ++	memcpy(buf, inmemory->buf + inmemory->offset, bytes);
>     ++	inmemory->offset += bytes;
Now, we also use the offset correctly.
Show 54 quoted lines
>      +
>      +	return bytes;
>      +}
>  6:  983f886eeb !  6:  506df5e488 odb/source-inmemory: implement `write_object()` callback
>     @@ odb.c: int odb_pretend_object(struct object_database *odb,
>       void *odb_read_object(struct object_database *odb,
>
>       ## odb/source-inmemory.c ##
>     +@@
>     + #include "git-compat-util.h"
>     ++#include "object-file.h"
>     + #include "odb.h"
>     + #include "odb/source-inmemory.h"
>     + #include "odb/streaming.h"
>      @@ odb/source-inmemory.c: static int odb_source_inmemory_read_object_stream(struct odb_read_stream **out,
>       	return 0;
>       }
>     @@ odb/source-inmemory.c: static int odb_source_inmemory_read_object_stream(struct
>      +	struct odb_source_inmemory *inmemory = odb_source_inmemory_downcast(source);
>      +	struct cached_object_entry *object;
>      +
>     ++	hash_object_file(source->odb->repo->hash_algo, buf, len, type, oid);
>     ++
>      +	ALLOC_GROW(inmemory->objects, inmemory->objects_nr + 1,
>      +		   inmemory->objects_alloc);
>      +	object = &inmemory->objects[inmemory->objects_nr++];
>  7:  68edefa269 <  -:  ---------- odb/source-inmemory: implement `write_object()` callback
>  8:  18d451152b !  7:  21eef34c1b odb/source-inmemory: implement `write_object_stream()` callback
>     @@ odb/source-inmemory.c: static int odb_source_inmemory_write_object(struct odb_so
>      +			goto out;
>      +		}
>      +
>     -+		memcpy(data, buf, bytes_read);
>     ++		memcpy(data + total_read, buf, bytes_read);
>      +		total_read += bytes_read;
>      +	}
>      +
>  9:  cee53b9853 !  8:  504e34d116 cbtree: allow using arbitrary wrapper structures for nodes
>     @@ cbtree.c: int cb_each(struct cb_tree *t, const uint8_t *kpfx, size_t klen,
>
>
>       ## cbtree.h ##
>     +@@
>     +  *
>     +  * This is adapted to store arbitrary data (not just NUL-terminated C strings
>     +  * and allocates no memory internally.  The user needs to allocate
>     +- * "struct cb_node" and fill cb_node.k[] with arbitrary match data
>     +- * for memcmp.
>     +- * If "klen" is variable, then it should be embedded into "c_node.k[]"
>     ++ * "struct cb_node" and provide `key_offset` to indicate where the key can be
>     ++ * found relative to the `struct cb_node` for memcmp.
>     ++ * If "klen" is variable, then it should be embedded into the key.
>     +  * Recursion is bound by the maximum value of "klen" used.
>     +  */
We fix up the comments here also.
Show 49 quoted lines
>     + #ifndef CBTREE_H
>      @@ cbtree.h: struct cb_node {
>       	 */
>       	uint32_t byte;
> 10:  8ad5b81b13 =  9:  9bdd475a92 oidtree: add ability to store data
> 11:  1ed2d23137 ! 10:  956b989529 odb/source-inmemory: convert to use oidtree
>     @@ odb/source-inmemory.h
>      +struct oidtree;
>
>       /*
>     -  * An inmemory source that you can write objects to that shall be made
>     +  * An in-memory source that you can write objects to that shall be made
>      @@ odb/source-inmemory.h: struct cached_object_entry {
>        */
>       struct odb_source_inmemory {
> 12:  99fbb1cc35 ! 11:  bec1428116 odb/source-inmemory: implement `for_each_object()` callback
>     @@ odb/source-inmemory.c: static int odb_source_inmemory_read_object_stream(struct
>      +	if ((opts->flags & ODB_FOR_EACH_OBJECT_PROMISOR_ONLY) ||
>      +	    (opts->flags & ODB_FOR_EACH_OBJECT_LOCAL_ONLY && !source->local))
>      +		return 0;
>     ++	if (!inmemory->objects)
>     ++		return 0;
>      +
>      +	return oidtree_each(inmemory->objects,
>      +			    opts->prefix ? opts->prefix : &null_oid, opts->prefix_hex_len,
> 13:  c87a621f39 = 12:  32dada3c27 odb/source-inmemory: implement `find_abbrev_len()` callback
> 14:  9b88f0c07b = 13:  43127840c0 odb/source-inmemory: implement `count_objects()` callback
> 15:  3c9493f2bb = 14:  439acbd068 odb/source-inmemory: implement `freshen_object()` callback
> 16:  f2b6317104 ! 15:  12c1b6ffd2 odb/source-inmemory: stub out remaining functions
>     @@ odb/source-inmemory.c: static int odb_source_inmemory_freshen_object(struct odb_
>      +static int odb_source_inmemory_begin_transaction(struct odb_source *source UNUSED,
>      +						 struct odb_transaction **out UNUSED)
>      +{
>     -+	return error("inmemory source does not support transactions");
>     ++	return error("in-memory source does not support transactions");
>      +}
>      +
>      +static int odb_source_inmemory_read_alternates(struct odb_source *source UNUSED,
>     @@ odb/source-inmemory.c: static int odb_source_inmemory_freshen_object(struct odb_
>      +static int odb_source_inmemory_write_alternate(struct odb_source *source UNUSED,
>      +					       const char *alternate UNUSED)
>      +{
>     -+	return error("inmemory source does not support alternates");
>     ++	return error("in-memory source does not support alternates");
>      +}
>      +
>      +static void odb_source_inmemory_close(struct odb_source *source UNUSED)
> 17:  81da5d5048 = 16:  ef37a61e7f odb: generic in-memory source
>  -:  ---------- > 17:  51b51e0382 t/unit-tests: add tests for the in-memory object source

The range diff looks good. I'll have a look at the unit test patch independently. Thanks

Previous: Karthik Nayak
Message 85 of 85 in “odb: introduce "inmemory" source”
  1. 00/16 odb: introduce "inmemory" sourcePatrick Steinhardt, Apr 3, 2026
  2. 01/16 odb: introduce "inmemory" sourcePatrick Steinhardt, Apr 3, 2026
  3. Justin ToblerApr 8, 2026
  4. Patrick SteinhardtApr 9, 2026
  5. 02/16 odb/source-inmemory: implement `free()` callbackPatrick Steinhardt, Apr 3, 2026
  6. Justin ToblerApr 8, 2026
  7. 03/16 odb: fix unnecessary call to `find_cached_object()`Patrick Steinhardt, Apr 3, 2026
  8. Justin ToblerApr 8, 2026
  9. Patrick SteinhardtApr 9, 2026
  10. 04/16 odb/source-inmemory: implement `read_object_info()` callbackPatrick Steinhardt, Apr 3, 2026
  11. 05/16 odb/source-inmemory: implement `read_object_stream()` callbackPatrick Steinhardt, Apr 3, 2026
  12. Justin ToblerApr 8, 2026
  13. Patrick SteinhardtApr 9, 2026
  14. 06/16 odb/source-inmemory: implement `write_object()` callbackPatrick Steinhardt, Apr 3, 2026
  15. 07/16 odb/source-inmemory: implement `write_object_stream()` callbackPatrick Steinhardt, Apr 3, 2026
  16. Junio C HamanoApr 3, 2026
  17. Patrick SteinhardtApr 8, 2026
  18. 08/16 cbtree: allow using arbitrary wrapper structures for nodesPatrick Steinhardt, Apr 3, 2026
  19. 09/16 oidtree: add ability to store dataPatrick Steinhardt, Apr 3, 2026
  20. 10/16 odb/source-inmemory: convert to use oidtreePatrick Steinhardt, Apr 3, 2026
  21. 11/16 odb/source-inmemory: implement `for_each_object()` callbackPatrick Steinhardt, Apr 3, 2026
  22. 12/16 odb/source-inmemory: implement `find_abbrev_len()` callbackPatrick Steinhardt, Apr 3, 2026
  23. 13/16 odb/source-inmemory: implement `count_objects()` callbackPatrick Steinhardt, Apr 3, 2026
  24. 14/16 odb/source-inmemory: implement `freshen_object()` callbackPatrick Steinhardt, Apr 3, 2026
  25. 15/16 odb/source-inmemory: stub out remaining functionsPatrick Steinhardt, Apr 3, 2026
  26. 16/16 odb: generic inmemory sourcePatrick Steinhardt, Apr 3, 2026
  27. Junio C HamanoApr 3, 2026
  28. Patrick SteinhardtApr 8, 2026
  29. Junio C HamanoApr 8, 2026
  30. Patrick SteinhardtApr 9, 2026
  31. Junio C HamanoApr 9, 2026
  32. Patrick SteinhardtApr 10, 2026
  33. 00/17 odb: introduce "in-memory" sourcePatrick Steinhardt, Apr 9, 2026
  34. 01/17 odb: introduce "in-memory" sourcePatrick Steinhardt, Apr 9, 2026
  35. Karthik NayakApr 9, 2026
  36. Patrick SteinhardtApr 9, 2026
  37. 02/17 odb/source-inmemory: implement `free()` callbackPatrick Steinhardt, Apr 9, 2026
  38. 03/17 odb: fix unnecessary call to `find_cached_object()`Patrick Steinhardt, Apr 9, 2026
  39. 04/17 odb/source-inmemory: implement `read_object_info()` callbackPatrick Steinhardt, Apr 9, 2026
  40. Karthik NayakApr 9, 2026
  41. Patrick SteinhardtApr 9, 2026
  42. Karthik NayakApr 9, 2026
  43. 05/17 odb/source-inmemory: implement `read_object_stream()` callbackPatrick Steinhardt, Apr 9, 2026
  44. Karthik NayakApr 9, 2026
  45. Patrick SteinhardtApr 9, 2026
  46. 06/17 odb/source-inmemory: implement `write_object()` callbackPatrick Steinhardt, Apr 9, 2026
  47. 07/17 odb/source-inmemory: implement `write_object()` callbackPatrick Steinhardt, Apr 9, 2026
  48. Karthik NayakApr 9, 2026
  49. Patrick SteinhardtApr 9, 2026
  50. 08/17 odb/source-inmemory: implement `write_object_stream()` callbackPatrick Steinhardt, Apr 9, 2026
  51. 09/17 cbtree: allow using arbitrary wrapper structures for nodesPatrick Steinhardt, Apr 9, 2026
  52. Karthik NayakApr 9, 2026
  53. Patrick SteinhardtApr 9, 2026
  54. 10/17 oidtree: add ability to store dataPatrick Steinhardt, Apr 9, 2026
  55. 11/17 odb/source-inmemory: convert to use oidtreePatrick Steinhardt, Apr 9, 2026
  56. 12/17 odb/source-inmemory: implement `for_each_object()` callbackPatrick Steinhardt, Apr 9, 2026
  57. 13/17 odb/source-inmemory: implement `find_abbrev_len()` callbackPatrick Steinhardt, Apr 9, 2026
  58. 14/17 odb/source-inmemory: implement `count_objects()` callbackPatrick Steinhardt, Apr 9, 2026
  59. 15/17 odb/source-inmemory: implement `freshen_object()` callbackPatrick Steinhardt, Apr 9, 2026
  60. 16/17 odb/source-inmemory: stub out remaining functionsPatrick Steinhardt, Apr 9, 2026
  61. Junio C HamanoApr 9, 2026
  62. Patrick SteinhardtApr 10, 2026
  63. 17/17 odb: generic in-memory sourcePatrick Steinhardt, Apr 9, 2026
  64. Karthik NayakApr 9, 2026
  65. Patrick SteinhardtApr 9, 2026
  66. 00/17 odb: introduce "in-memory" sourcePatrick Steinhardt, Apr 10, 2026
  67. 01/17 odb: introduce "in-memory" sourcePatrick Steinhardt, Apr 10, 2026
  68. 02/17 odb/source-inmemory: implement `free()` callbackPatrick Steinhardt, Apr 10, 2026
  69. 03/17 odb: fix unnecessary call to `find_cached_object()`Patrick Steinhardt, Apr 10, 2026
  70. 04/17 odb/source-inmemory: implement `read_object_info()` callbackPatrick Steinhardt, Apr 10, 2026
  71. 05/17 odb/source-inmemory: implement `read_object_stream()` callbackPatrick Steinhardt, Apr 10, 2026
  72. 06/17 odb/source-inmemory: implement `write_object()` callbackPatrick Steinhardt, Apr 10, 2026
  73. 07/17 odb/source-inmemory: implement `write_object_stream()` callbackPatrick Steinhardt, Apr 10, 2026
  74. 08/17 cbtree: allow using arbitrary wrapper structures for nodesPatrick Steinhardt, Apr 10, 2026
  75. 09/17 oidtree: add ability to store dataPatrick Steinhardt, Apr 10, 2026
  76. 10/17 odb/source-inmemory: convert to use oidtreePatrick Steinhardt, Apr 10, 2026
  77. 11/17 odb/source-inmemory: implement `for_each_object()` callbackPatrick Steinhardt, Apr 10, 2026
  78. 12/17 odb/source-inmemory: implement `find_abbrev_len()` callbackPatrick Steinhardt, Apr 10, 2026
  79. 13/17 odb/source-inmemory: implement `count_objects()` callbackPatrick Steinhardt, Apr 10, 2026
  80. 14/17 odb/source-inmemory: implement `freshen_object()` callbackPatrick Steinhardt, Apr 10, 2026
  81. 15/17 odb/source-inmemory: stub out remaining functionsPatrick Steinhardt, Apr 10, 2026
  82. 16/17 odb: generic in-memory sourcePatrick Steinhardt, Apr 10, 2026
  83. 17/17 t/unit-tests: add tests for the in-memory object sourcePatrick Steinhardt, Apr 10, 2026
  84. Karthik NayakApr 14, 2026
  85. Karthik NayakApr 14, 2026

Read the whole thread, see it on lore, or plain text.

$ cat FOOTERMessages come from the public archive at lore.kernel.org/git, fetched every hour. The front page is chosen and written each morning by an AI editor and can be wrong; the threads themselves are the record. About and API. For agents: an MCP server at https://gitlist.dev/mcp, and any thread, story or person page as Markdown by adding .md to its URL (or sending Accept: text/markdown). Details in /llms.txt.