[PATCH v3 00/10] midx: stop duplicating info redundant with their sources
- From
Patrick Steinhardt <ps@pks.im>
- Date
- Aug 11, 2025, 13:46 UTC
- Message-ID
- <20250811-b4-pks-midx-deduplicate-source-info-v3-0-e442bdf2b4ad@pks.im>
- In-Reply-To
- <20250729-b4-pks-midx-deduplicate-source-info-v1-0-748db2eda3b5@pks.im>
Hi,
this patch series is a follow up to the recent patch series that started to move the MIDX into its object source. It refactors the MIDX subsystem so that we always read and write them via an ODB source. This ultimately allows us to store a pointer to the owning ODB source in the MIDX and thus dedup some redundant information.
This series is built on top of e813a0200a7 (The fifteenth batch, 2025-07-28) with ps/object-store-midx at ec865d94d46 (midx: remove now-unused linked list of multi-pack indices, 2025-07-15) merged into it.
Changes in v2:
- Fix a comment typo.
- Introduce another commit that simplifies the calling convention for
`link_alt_odb_entry()`.
- Link to v1: https://lore.kernel.org/r/20250729-b4-pks-midx-deduplicate-source-info-v1-0-748db2eda3b5@pks.imChanges in v3:
- Introduce `odb_find_source_or_die()` so that we don't have to repeat
the calls to `die()`, as suggested by Taylor.
- Split out a patch to adapt `link_alt_odb_entry()` and friends to
consistently name the parameter that refers to the alternate object
directory's path.
- Link to v2: https://lore.kernel.org/r/20250807-b4-pks-midx-deduplicate-source-info-v2-0-bcffb8fc119c@pks.imThanks!
Patrick
---
Patrick Steinhardt (10):
odb: store locality in object database sources
odb: allow `odb_find_source()` to fail
odb: consistently use "dir" to refer to alternate's directory
odb: return newly created in-memory sources
odb: simplify calling `link_alt_odb_entry()`
midx: drop redundant `struct repository` parameter
midx: load multi-pack indices via their source
midx: write multi-pack indices via their source
midx: stop duplicating info redundant with its owning source
midx: compute paths via their sourcebuiltin/commit-graph.c | 4 +- builtin/multi-pack-index.c | 31 +++++++--- builtin/pack-objects.c | 2 +- builtin/repack.c | 7 ++- midx-write.c | 116 +++++++++++++++++-------------------- midx.c | 135 +++++++++++++++++++++----------------------- midx.h | 42 ++++++-------- odb.c | 64 ++++++++++----------- odb.h | 19 +++++-- pack-bitmap.c | 15 +++-- pack-revindex.c | 14 ++--- packfile.c | 13 ++--- repository.c | 1 + t/helper/test-read-midx.c | 31 +++++----- t/t5319-multi-pack-index.sh | 8 +-- 15 files changed, 254 insertions(+), 248 deletions(-)
Range-diff versus v2:
1: 937fe12914 = 1: 51817fdd71 odb: store locality in object database sources
2: c1a382e2e5 < -: ---------- odb: allow `odb_find_source()` to fail
-: ---------- > 2: 37da6f7908 odb: allow `odb_find_source()` to fail
-: ---------- > 3: b0f3c8af7b odb: consistently use "dir" to refer to alternate's directory
3: 31fa81ce51 ! 4: 88af2d77bd odb: return newly created in-memory sources
@@ odb.c: static void read_info_alternates(struct object_database *odb,
int depth);
-static int link_alt_odb_entry(struct object_database *odb,
-- const struct strbuf *entry,
+- const char *dir,
- const char *relative_base,
- int depth,
- const char *normalized_objdir)
+static struct odb_source *link_alt_odb_entry(struct object_database *odb,
-+ const char *entry,
++ const char *dir,
+ const char *relative_base,
+ int depth,
+ const char *normalized_objdir)
@@ odb.c: static void read_info_alternates(struct object_database *odb,
khiter_t pos;
- int ret = -1;
-- if (!is_absolute_path(entry->buf) && relative_base) {
-+ if (!is_absolute_path(entry) && relative_base) {
+ if (!is_absolute_path(dir) && relative_base) {
strbuf_realpath(&pathbuf, relative_base, 1);
- strbuf_addch(&pathbuf, '/');
- }
-- strbuf_addbuf(&pathbuf, entry);
-+ strbuf_addstr(&pathbuf, entry);
-
- if (!strbuf_realpath(&tmp, pathbuf.buf, 0)) {
- error(_("unable to normalize alternate object path: %s"),
@@ odb.c: static int link_alt_odb_entry(struct object_database *odb,
/* recursively add alternates */
@@ odb.c: static int link_alt_odb_entry(struct object_database *odb,
}
static const char *parse_alt_odb_entry(const char *string,
-@@ odb.c: static void link_alt_odb_entries(struct object_database *odb, const char *alt,
- alt = parse_alt_odb_entry(alt, sep, &entry);
- if (!entry.len)
- continue;
-- link_alt_odb_entry(odb, &entry,
-+ link_alt_odb_entry(odb, entry.buf,
- relative_base, depth, objdirbuf.buf);
- }
- strbuf_release(&entry);
@@ odb.c: void odb_add_to_alternates_file(struct object_database *odb,
free(alts);
}
-void odb_add_to_alternates_memory(struct object_database *odb,
-- const char *reference)
+- const char *dir)
+struct odb_source *odb_add_to_alternates_memory(struct object_database *odb,
-+ const char *reference)
++ const char *dir)
{
+ struct odb_source *alternate;
+ char *objdir;
@@ odb.c: void odb_add_to_alternates_file(struct object_database *odb,
*/
odb_prepare_alternates(odb);
-- link_alt_odb_entries(odb, reference,
-- '\n', NULL, 0);
+- link_alt_odb_entries(odb, dir, '\n', NULL, 0);
+ objdir = real_pathdup(odb->sources->path, 1);
-+ alternate = link_alt_odb_entry(odb, reference, NULL, 0, objdir);
++ alternate = link_alt_odb_entry(odb, dir, NULL, 0, objdir);
+
+ free(objdir);
+ return alternate;
4: a94136e0a6 ! 5: 4f002abcee odb: simplify calling `link_alt_odb_entry()`
@@ Commit message
## odb.c ##
@@ odb.c: static void read_info_alternates(struct object_database *odb,
static struct odb_source *link_alt_odb_entry(struct object_database *odb,
- const char *entry,
+ const char *dir,
const char *relative_base,
- int depth,
- const char *normalized_objdir)
@@ odb.c: static const char *parse_alt_odb_entry(const char *string,
int sep, const char *relative_base, int depth)
{
- struct strbuf objdirbuf = STRBUF_INIT;
- struct strbuf entry = STRBUF_INIT;
+ struct strbuf dir = STRBUF_INIT;
if (!alt || !*alt)
@@ odb.c: static void link_alt_odb_entries(struct object_database *odb, const char *alt,
@@ odb.c: static void link_alt_odb_entries(struct object_database *odb, const char
- strbuf_realpath(&objdirbuf, odb->sources->path, 1);
-
while (*alt) {
- alt = parse_alt_odb_entry(alt, sep, &entry);
- if (!entry.len)
+ alt = parse_alt_odb_entry(alt, sep, &dir);
+ if (!dir.len)
continue;
-- link_alt_odb_entry(odb, entry.buf,
+- link_alt_odb_entry(odb, dir.buf,
- relative_base, depth, objdirbuf.buf);
-+ link_alt_odb_entry(odb, entry.buf, relative_base, depth);
++ link_alt_odb_entry(odb, dir.buf, relative_base, depth);
}
- strbuf_release(&entry);
+ strbuf_release(&dir);
- strbuf_release(&objdirbuf);
}
static void read_info_alternates(struct object_database *odb,
@@ odb.c: void odb_add_to_alternates_file(struct object_database *odb,
struct odb_source *odb_add_to_alternates_memory(struct object_database *odb,
- const char *reference)
+ const char *dir)
{
- struct odb_source *alternate;
- char *objdir;
@@ odb.c: void odb_add_to_alternates_file(struct object_database *odb,
odb_prepare_alternates(odb);
-
- objdir = real_pathdup(odb->sources->path, 1);
-- alternate = link_alt_odb_entry(odb, reference, NULL, 0, objdir);
+- alternate = link_alt_odb_entry(odb, dir, NULL, 0, objdir);
-
- free(objdir);
- return alternate;
-+ return link_alt_odb_entry(odb, reference, NULL, 0);
++ return link_alt_odb_entry(odb, dir, NULL, 0);
}
struct odb_source *odb_set_temporary_primary_source(struct object_database *odb,
5: eaf7cc324a = 6: 91cf3fc19a midx: drop redundant `struct repository` parameter
6: 64d1e33a88 = 7: eac8ce2f0d midx: load multi-pack indices via their source
7: 094fb9fd83 ! 8: 634ccedddf midx: write multi-pack indices via their source
@@ midx-write.c: static int write_midx_bitmap(struct write_midx_context *ctx,
-static struct multi_pack_index *lookup_multi_pack_index(struct repository *r,
- const char *object_dir)
-{
-- struct odb_source *source = odb_find_source(r->objects, object_dir);
-- if (!source)
-- die(_("could not find object directory matching %s"), object_dir);
+- struct odb_source *source = odb_find_source_or_die(r->objects, object_dir);
- return get_multi_pack_index(source);
-}
-
8: 2d6c844618 = 9: 91a15ad630 midx: stop duplicating info redundant with its owning source
9: 93f60b02b6 = 10: 357f6a4986 midx: compute paths via their source--- base-commit: ee2fd3d4b224c8145eeff04101ea6075560973e1 change-id: 20250729-b4-pks-midx-deduplicate-source-info-fcf3c08bfd53