Re: [PATCH v3 00/17] odb: introduce "in-memory" source
- From
Karthik Nayak <karthik.188@gmail.com>
- Date
- Apr 14, 2026, 08:27 UTC
- Message-ID
- <CAOLa=ZSrThty13-C_WVa5dvakZAtidwOXWnUrOA4LGX93DvmGQ@mail.gmail.com>
- In-Reply-To
- <20260410-b4-pks-odb-source-inmemory-v3-0-22fd0fad58fe@pks.im>
Patrick Steinhardt <ps@pks.im> writes: [snip]
Show 53 quoted lines
> Range-diff versus v2:
>
> 1: b18e427c69 ! 1: 155b2cdf81 odb: introduce "in-memory" source
> @@ odb/source-inmemory.h (new)
> +struct cached_object_entry;
> +
> +/*
> -+ * An inmemory source that you can write objects to that shall be made
> ++ * An in-memory source that you can write objects to that shall be made
> + * available for reading, but that shouldn't ever be persisted to disk. Note
> + * that any objects written to this source will be stored in memory, so the
> + * number of objects you can store is limited by available system memory.
> @@ odb/source-inmemory.h (new)
> +struct odb_source_inmemory *odb_source_inmemory_new(struct object_database *odb);
> +
> +/*
> -+ * Cast the given object database source to the inmemory backend. This will
> ++ * Cast the given object database source to the in-memory backend. This will
> + * cause a BUG in case the source doesn't use this backend.
> + */
> +static inline struct odb_source_inmemory *odb_source_inmemory_downcast(struct odb_source *source)
> +{
> + if (source->type != ODB_SOURCE_INMEMORY)
> -+ BUG("trying to downcast source of type '%d' to inmemory", source->type);
> ++ BUG("trying to downcast source of type '%d' to in-memory", source->type);
> + return container_of(source, struct odb_source_inmemory, base);
> +}
> +
> @@ odb/source.h: enum odb_source_type {
> /* The "files" backend that uses loose objects and packfiles. */
> ODB_SOURCE_FILES,
> +
> -+ /* The "inmemory" backend that stores objects in memory. */
> ++ /* The "in-memory" backend that stores objects in memory. */
> + ODB_SOURCE_INMEMORY,
> };
>
> 2: 8fd337da90 ! 2: c66edd10a8 odb/source-inmemory: implement `free()` callback
> @@ odb/source-inmemory.h
> +};
>
> /*
> - * An inmemory source that you can write objects to that shall be made
> + * An in-memory source that you can write objects to that shall be made
> 3: f4ae2a2bde = 3: a86549f39c odb: fix unnecessary call to `find_cached_object()`
> 4: 8600b88530 = 4: 49ac739dd2 odb/source-inmemory: implement `read_object_info()` callback
> 5: ab33c0b7ee ! 5: 321ef11be3 odb/source-inmemory: implement `read_object_stream()` callback
> @@ odb/source-inmemory.c: static int odb_source_inmemory_read_object_info(struct od
>
> +struct odb_read_stream_inmemory {
> + struct odb_read_stream base;
> -+ const void *buf;
> ++ const unsigned char *buf;Okay this does make more sense.
Show 11 quoted lines
> + size_t offset; > +}; > + > @@ odb/source-inmemory.c: static int odb_source_inmemory_read_object_info(struct od > + > + if (buf_len > inmemory->base.size - inmemory->offset) > + bytes = inmemory->base.size - inmemory->offset; > -+ memcpy(buf, inmemory->buf, bytes); > ++ > ++ memcpy(buf, inmemory->buf + inmemory->offset, bytes); > ++ inmemory->offset += bytes;
Now, we also use the offset correctly.
Show 54 quoted lines
> + > + return bytes; > +} > 6: 983f886eeb ! 6: 506df5e488 odb/source-inmemory: implement `write_object()` callback > @@ odb.c: int odb_pretend_object(struct object_database *odb, > void *odb_read_object(struct object_database *odb, > > ## odb/source-inmemory.c ## > +@@ > + #include "git-compat-util.h" > ++#include "object-file.h" > + #include "odb.h" > + #include "odb/source-inmemory.h" > + #include "odb/streaming.h" > @@ odb/source-inmemory.c: static int odb_source_inmemory_read_object_stream(struct odb_read_stream **out, > return 0; > } > @@ odb/source-inmemory.c: static int odb_source_inmemory_read_object_stream(struct > + struct odb_source_inmemory *inmemory = odb_source_inmemory_downcast(source); > + struct cached_object_entry *object; > + > ++ hash_object_file(source->odb->repo->hash_algo, buf, len, type, oid); > ++ > + ALLOC_GROW(inmemory->objects, inmemory->objects_nr + 1, > + inmemory->objects_alloc); > + object = &inmemory->objects[inmemory->objects_nr++]; > 7: 68edefa269 < -: ---------- odb/source-inmemory: implement `write_object()` callback > 8: 18d451152b ! 7: 21eef34c1b odb/source-inmemory: implement `write_object_stream()` callback > @@ odb/source-inmemory.c: static int odb_source_inmemory_write_object(struct odb_so > + goto out; > + } > + > -+ memcpy(data, buf, bytes_read); > ++ memcpy(data + total_read, buf, bytes_read); > + total_read += bytes_read; > + } > + > 9: cee53b9853 ! 8: 504e34d116 cbtree: allow using arbitrary wrapper structures for nodes > @@ cbtree.c: int cb_each(struct cb_tree *t, const uint8_t *kpfx, size_t klen, > > > ## cbtree.h ## > +@@ > + * > + * This is adapted to store arbitrary data (not just NUL-terminated C strings > + * and allocates no memory internally. The user needs to allocate > +- * "struct cb_node" and fill cb_node.k[] with arbitrary match data > +- * for memcmp. > +- * If "klen" is variable, then it should be embedded into "c_node.k[]" > ++ * "struct cb_node" and provide `key_offset` to indicate where the key can be > ++ * found relative to the `struct cb_node` for memcmp. > ++ * If "klen" is variable, then it should be embedded into the key. > + * Recursion is bound by the maximum value of "klen" used. > + */
We fix up the comments here also.
Show 49 quoted lines
> + #ifndef CBTREE_H
> @@ cbtree.h: struct cb_node {
> */
> uint32_t byte;
> 10: 8ad5b81b13 = 9: 9bdd475a92 oidtree: add ability to store data
> 11: 1ed2d23137 ! 10: 956b989529 odb/source-inmemory: convert to use oidtree
> @@ odb/source-inmemory.h
> +struct oidtree;
>
> /*
> - * An inmemory source that you can write objects to that shall be made
> + * An in-memory source that you can write objects to that shall be made
> @@ odb/source-inmemory.h: struct cached_object_entry {
> */
> struct odb_source_inmemory {
> 12: 99fbb1cc35 ! 11: bec1428116 odb/source-inmemory: implement `for_each_object()` callback
> @@ odb/source-inmemory.c: static int odb_source_inmemory_read_object_stream(struct
> + if ((opts->flags & ODB_FOR_EACH_OBJECT_PROMISOR_ONLY) ||
> + (opts->flags & ODB_FOR_EACH_OBJECT_LOCAL_ONLY && !source->local))
> + return 0;
> ++ if (!inmemory->objects)
> ++ return 0;
> +
> + return oidtree_each(inmemory->objects,
> + opts->prefix ? opts->prefix : &null_oid, opts->prefix_hex_len,
> 13: c87a621f39 = 12: 32dada3c27 odb/source-inmemory: implement `find_abbrev_len()` callback
> 14: 9b88f0c07b = 13: 43127840c0 odb/source-inmemory: implement `count_objects()` callback
> 15: 3c9493f2bb = 14: 439acbd068 odb/source-inmemory: implement `freshen_object()` callback
> 16: f2b6317104 ! 15: 12c1b6ffd2 odb/source-inmemory: stub out remaining functions
> @@ odb/source-inmemory.c: static int odb_source_inmemory_freshen_object(struct odb_
> +static int odb_source_inmemory_begin_transaction(struct odb_source *source UNUSED,
> + struct odb_transaction **out UNUSED)
> +{
> -+ return error("inmemory source does not support transactions");
> ++ return error("in-memory source does not support transactions");
> +}
> +
> +static int odb_source_inmemory_read_alternates(struct odb_source *source UNUSED,
> @@ odb/source-inmemory.c: static int odb_source_inmemory_freshen_object(struct odb_
> +static int odb_source_inmemory_write_alternate(struct odb_source *source UNUSED,
> + const char *alternate UNUSED)
> +{
> -+ return error("inmemory source does not support alternates");
> ++ return error("in-memory source does not support alternates");
> +}
> +
> +static void odb_source_inmemory_close(struct odb_source *source UNUSED)
> 17: 81da5d5048 = 16: ef37a61e7f odb: generic in-memory source
> -: ---------- > 17: 51b51e0382 t/unit-tests: add tests for the in-memory object sourceThe range diff looks good. I'll have a look at the unit test patch independently. Thanks