From: Karthik Nayak Date: Tue, 14 Apr 2026 08:27:26 GMT Subject: Re: [PATCH v3 00/17] odb: introduce "in-memory" source Message-ID: In-Reply-To: <20260410-b4-pks-odb-source-inmemory-v3-0-22fd0fad58fe@pks.im> Patrick Steinhardt writes: [snip] > Range-diff versus v2: > > 1: b18e427c69 ! 1: 155b2cdf81 odb: introduce "in-memory" source > @@ odb/source-inmemory.h (new) > +struct cached_object_entry; > + > +/* > -+ * An inmemory source that you can write objects to that shall be made > ++ * An in-memory source that you can write objects to that shall be made > + * available for reading, but that shouldn't ever be persisted to disk. Note > + * that any objects written to this source will be stored in memory, so the > + * number of objects you can store is limited by available system memory. > @@ odb/source-inmemory.h (new) > +struct odb_source_inmemory *odb_source_inmemory_new(struct object_database *odb); > + > +/* > -+ * Cast the given object database source to the inmemory backend. This will > ++ * Cast the given object database source to the in-memory backend. This will > + * cause a BUG in case the source doesn't use this backend. > + */ > +static inline struct odb_source_inmemory *odb_source_inmemory_downcast(struct odb_source *source) > +{ > + if (source->type != ODB_SOURCE_INMEMORY) > -+ BUG("trying to downcast source of type '%d' to inmemory", source->type); > ++ BUG("trying to downcast source of type '%d' to in-memory", source->type); > + return container_of(source, struct odb_source_inmemory, base); > +} > + > @@ odb/source.h: enum odb_source_type { > /* The "files" backend that uses loose objects and packfiles. */ > ODB_SOURCE_FILES, > + > -+ /* The "inmemory" backend that stores objects in memory. */ > ++ /* The "in-memory" backend that stores objects in memory. */ > + ODB_SOURCE_INMEMORY, > }; > > 2: 8fd337da90 ! 2: c66edd10a8 odb/source-inmemory: implement `free()` callback > @@ odb/source-inmemory.h > +}; > > /* > - * An inmemory source that you can write objects to that shall be made > + * An in-memory source that you can write objects to that shall be made > 3: f4ae2a2bde = 3: a86549f39c odb: fix unnecessary call to `find_cached_object()` > 4: 8600b88530 = 4: 49ac739dd2 odb/source-inmemory: implement `read_object_info()` callback > 5: ab33c0b7ee ! 5: 321ef11be3 odb/source-inmemory: implement `read_object_stream()` callback > @@ odb/source-inmemory.c: static int odb_source_inmemory_read_object_info(struct od > > +struct odb_read_stream_inmemory { > + struct odb_read_stream base; > -+ const void *buf; > ++ const unsigned char *buf; Okay this does make more sense. > + size_t offset; > +}; > + > @@ odb/source-inmemory.c: static int odb_source_inmemory_read_object_info(struct od > + > + if (buf_len > inmemory->base.size - inmemory->offset) > + bytes = inmemory->base.size - inmemory->offset; > -+ memcpy(buf, inmemory->buf, bytes); > ++ > ++ memcpy(buf, inmemory->buf + inmemory->offset, bytes); > ++ inmemory->offset += bytes; Now, we also use the offset correctly. > + > + return bytes; > +} > 6: 983f886eeb ! 6: 506df5e488 odb/source-inmemory: implement `write_object()` callback > @@ odb.c: int odb_pretend_object(struct object_database *odb, > void *odb_read_object(struct object_database *odb, > > ## odb/source-inmemory.c ## > +@@ > + #include "git-compat-util.h" > ++#include "object-file.h" > + #include "odb.h" > + #include "odb/source-inmemory.h" > + #include "odb/streaming.h" > @@ odb/source-inmemory.c: static int odb_source_inmemory_read_object_stream(struct odb_read_stream **out, > return 0; > } > @@ odb/source-inmemory.c: static int odb_source_inmemory_read_object_stream(struct > + struct odb_source_inmemory *inmemory = odb_source_inmemory_downcast(source); > + struct cached_object_entry *object; > + > ++ hash_object_file(source->odb->repo->hash_algo, buf, len, type, oid); > ++ > + ALLOC_GROW(inmemory->objects, inmemory->objects_nr + 1, > + inmemory->objects_alloc); > + object = &inmemory->objects[inmemory->objects_nr++]; > 7: 68edefa269 < -: ---------- odb/source-inmemory: implement `write_object()` callback > 8: 18d451152b ! 7: 21eef34c1b odb/source-inmemory: implement `write_object_stream()` callback > @@ odb/source-inmemory.c: static int odb_source_inmemory_write_object(struct odb_so > + goto out; > + } > + > -+ memcpy(data, buf, bytes_read); > ++ memcpy(data + total_read, buf, bytes_read); > + total_read += bytes_read; > + } > + > 9: cee53b9853 ! 8: 504e34d116 cbtree: allow using arbitrary wrapper structures for nodes > @@ cbtree.c: int cb_each(struct cb_tree *t, const uint8_t *kpfx, size_t klen, > > > ## cbtree.h ## > +@@ > + * > + * This is adapted to store arbitrary data (not just NUL-terminated C strings > + * and allocates no memory internally. The user needs to allocate > +- * "struct cb_node" and fill cb_node.k[] with arbitrary match data > +- * for memcmp. > +- * If "klen" is variable, then it should be embedded into "c_node.k[]" > ++ * "struct cb_node" and provide `key_offset` to indicate where the key can be > ++ * found relative to the `struct cb_node` for memcmp. > ++ * If "klen" is variable, then it should be embedded into the key. > + * Recursion is bound by the maximum value of "klen" used. > + */ We fix up the comments here also. > + #ifndef CBTREE_H > @@ cbtree.h: struct cb_node { > */ > uint32_t byte; > 10: 8ad5b81b13 = 9: 9bdd475a92 oidtree: add ability to store data > 11: 1ed2d23137 ! 10: 956b989529 odb/source-inmemory: convert to use oidtree > @@ odb/source-inmemory.h > +struct oidtree; > > /* > - * An inmemory source that you can write objects to that shall be made > + * An in-memory source that you can write objects to that shall be made > @@ odb/source-inmemory.h: struct cached_object_entry { > */ > struct odb_source_inmemory { > 12: 99fbb1cc35 ! 11: bec1428116 odb/source-inmemory: implement `for_each_object()` callback > @@ odb/source-inmemory.c: static int odb_source_inmemory_read_object_stream(struct > + if ((opts->flags & ODB_FOR_EACH_OBJECT_PROMISOR_ONLY) || > + (opts->flags & ODB_FOR_EACH_OBJECT_LOCAL_ONLY && !source->local)) > + return 0; > ++ if (!inmemory->objects) > ++ return 0; > + > + return oidtree_each(inmemory->objects, > + opts->prefix ? opts->prefix : &null_oid, opts->prefix_hex_len, > 13: c87a621f39 = 12: 32dada3c27 odb/source-inmemory: implement `find_abbrev_len()` callback > 14: 9b88f0c07b = 13: 43127840c0 odb/source-inmemory: implement `count_objects()` callback > 15: 3c9493f2bb = 14: 439acbd068 odb/source-inmemory: implement `freshen_object()` callback > 16: f2b6317104 ! 15: 12c1b6ffd2 odb/source-inmemory: stub out remaining functions > @@ odb/source-inmemory.c: static int odb_source_inmemory_freshen_object(struct odb_ > +static int odb_source_inmemory_begin_transaction(struct odb_source *source UNUSED, > + struct odb_transaction **out UNUSED) > +{ > -+ return error("inmemory source does not support transactions"); > ++ return error("in-memory source does not support transactions"); > +} > + > +static int odb_source_inmemory_read_alternates(struct odb_source *source UNUSED, > @@ odb/source-inmemory.c: static int odb_source_inmemory_freshen_object(struct odb_ > +static int odb_source_inmemory_write_alternate(struct odb_source *source UNUSED, > + const char *alternate UNUSED) > +{ > -+ return error("inmemory source does not support alternates"); > ++ return error("in-memory source does not support alternates"); > +} > + > +static void odb_source_inmemory_close(struct odb_source *source UNUSED) > 17: 81da5d5048 = 16: ef37a61e7f odb: generic in-memory source > -: ---------- > 17: 51b51e0382 t/unit-tests: add tests for the in-memory object source The range diff looks good. I'll have a look at the unit test patch independently. Thanks