Re: [PATCH v4 03/13] pack-bitmap.c: open and store incremental bitmap layers

>
> Prepare the pack-bitmap machinery to work with incremental MIDXs by
> adding a new "base" field to keep track of the bitmap index associated
> with the previous MIDX layer.
>
> The changes in this commit are mostly boilerplate to open the correct
> bitmap(s), add them to the chain bitmap layers along the "base" pointer,

> across MIDX layers, etc.
>
> While we're at it, keep track of a base_nr field to indicate how many
> bitmap layers (including the current bitmap) exist. This will be used in
> a future commit to allocate an array of 'struct ewah_bitmap' pointers to
> collect all of the respective type bitmaps among all layers to
> initialize a multi-EWAH iterator.
>
> Subsequent commits will teach the functions within the pack-bitmap
> machinery how to interact with these new fields.
>
> Signed-off-by: Taylor Blau <me@ttaylorr.com>
> ---
>  pack-bitmap.c | 62 +++++++++++++++++++++++++++++++++++++++------------
>  1 file changed, 48 insertions(+), 14 deletions(-)
>
> diff --git a/pack-bitmap.c b/pack-bitmap.c
> index c26d85b5db..72fb11d014 100644
> --- a/pack-bitmap.c
> +++ b/pack-bitmap.c
> @@ -54,6 +54,16 @@ struct bitmap_index {
>         struct packed_git *pack;
>         struct multi_pack_index *midx;
>
> +       /*
> +        * If using a multi-pack index chain, 'base' points to the
> +        * bitmap index corresponding to this bitmap's midx->base_midx.
> +        *
> +        * base_nr indicates how many layers precede this one, and is
> +        * zero when base is NULL.
> +        */
> +       struct bitmap_index *base;
> +       uint32_t base_nr;
> +
>         /* mmapped buffer of the whole bitmap index */
>         unsigned char *map;
>         size_t map_size; /* size of the mmaped buffer */
> @@ -386,8 +396,15 @@ static int load_bitmap_entries_v1(struct bitmap_index *index)
>  char *midx_bitmap_filename(struct multi_pack_index *midx)
>  {
>         struct strbuf buf = STRBUF_INIT;
> -       get_midx_filename_ext(midx->repo->hash_algo, &buf, midx->object_dir,
> -                             get_midx_checksum(midx), MIDX_EXT_BITMAP);
> +       if (midx->has_chain)
> +               get_split_midx_filename_ext(midx->repo->hash_algo, &buf,
> +                                           midx->object_dir,
> +                                           get_midx_checksum(midx),
> +                                           MIDX_EXT_BITMAP);
> +       else
> +               get_midx_filename_ext(midx->repo->hash_algo, &buf,
> +                                     midx->object_dir, get_midx_checksum(midx),
> +                                     MIDX_EXT_BITMAP);
>
>         return strbuf_detach(&buf, NULL);
>  }
> @@ -454,16 +471,21 @@ static int open_midx_bitmap_1(struct bitmap_index *bitmap_git,
>                 goto cleanup;
>         }
>
> -       for (i = 0; i < bitmap_git->midx->num_packs; i++) {
> -               if (prepare_midx_pack(bitmap_repo(bitmap_git),
> -                                     bitmap_git->midx,
> -                                     i)) {
> +       for (i = 0; i < bitmap_git->midx->num_packs + bitmap_git->midx->num_packs_in_base; i++) {
> +               if (prepare_midx_pack(bitmap_repo(bitmap_git), bitmap_git->midx, i)) {
>                         warning(_("could not open pack %s"),
>                                 bitmap_git->midx->pack_names[i]);
>                         goto cleanup;
>                 }
>         }
>
> +       if (midx->base_midx) {
> +               bitmap_git->base = prepare_midx_bitmap_git(midx->base_midx);
> +               bitmap_git->base_nr = bitmap_git->base->base_nr + 1;
> +       } else {
> +               bitmap_git->base_nr = 0;
> +       }
> +
>         return 0;
>
>  cleanup:
> @@ -515,6 +537,7 @@ static int open_pack_bitmap_1(struct bitmap_index *bitmap_git, struct packed_git
>         bitmap_git->map_size = xsize_t(st.st_size);
>         bitmap_git->map = xmmap(NULL, bitmap_git->map_size, PROT_READ, MAP_PRIVATE, fd, 0);
>         bitmap_git->map_pos = 0;
> +       bitmap_git->base_nr = 0;
>         close(fd);
>
>         if (load_bitmap_header(bitmap_git) < 0) {
> @@ -534,8 +557,7 @@ static int open_pack_bitmap_1(struct bitmap_index *bitmap_git, struct packed_git
>  static int load_reverse_index(struct repository *r, struct bitmap_index *bitmap_git)
>  {
>         if (bitmap_is_midx(bitmap_git)) {
> -               uint32_t i;
> -               int ret;
> +               struct multi_pack_index *m;
>
>                 /*
>                  * The multi-pack-index's .rev file is already loaded via
> @@ -544,10 +566,15 @@ static int load_reverse_index(struct repository *r, struct bitmap_index *bitmap_
>                  * But we still need to open the individual pack .rev files,
>                  * since we will need to make use of them in pack-objects.
>                  */
> -               for (i = 0; i < bitmap_git->midx->num_packs; i++) {
> -                       ret = load_pack_revindex(r, bitmap_git->midx->packs[i]);
> -                       if (ret)
> -                               return ret;
> +               for (m = bitmap_git->midx; m; m = m->base_midx) {
> +                       uint32_t i;
> +                       int ret;
> +
> +                       for (i = 0; i < m->num_packs; i++) {
> +                               ret = load_pack_revindex(r, m->packs[i]);
> +                               if (ret)
> +                                       return ret;
> +                       }
>                 }
>                 return 0;
>         }
> @@ -573,6 +600,13 @@ static int load_bitmap(struct repository *r, struct bitmap_index *bitmap_git)
>         if (!bitmap_git->table_lookup && load_bitmap_entries_v1(bitmap_git) < 0)
>                 goto failed;
>
> +       if (bitmap_git->base) {
> +               if (!bitmap_is_midx(bitmap_git))
> +                       BUG("non-MIDX bitmap has non-NULL base bitmap index");
> +               if (load_bitmap(r, bitmap_git->base) < 0)
> +                       goto failed;
> +       }
> +
>         return 0;
>
>  failed:
> @@ -657,10 +691,9 @@ struct bitmap_index *prepare_bitmap_git(struct repository *r)
>
>  struct bitmap_index *prepare_midx_bitmap_git(struct multi_pack_index *midx)
>  {
> -       struct repository *r = midx->repo;
>         struct bitmap_index *bitmap_git = xcalloc(1, sizeof(*bitmap_git));
>
> -       if (!open_midx_bitmap_1(bitmap_git, midx) && !load_bitmap(r, bitmap_git))
> +       if (!open_midx_bitmap_1(bitmap_git, midx))
>                 return bitmap_git;
>
>         free_bitmap_index(bitmap_git);
> @@ -2899,6 +2932,7 @@ void free_bitmap_index(struct bitmap_index *b)
>                 close_midx_revindex(b->midx);
>         }
>         free_pseudo_merge_map(&b->pseudo_merges);
> +       free_bitmap_index(b->base);
>         free(b);
>  }
>
> --
> 2.49.0.13.gd0d564685b
>