git/list[1] front-page[2] threads[3] people[4] search[5] about
 

[RFC PATCH 07/18] midx: teach midx --write to update midx-head

From
Derrick Stolee <stolee@gmail.com>
Date
Jan 7, 2018, 18:14 UTC
Message-ID
<20180107181459.222909-8-dstolee@microsoft.com>
In-Reply-To
<20180107181459.222909-1-dstolee@microsoft.com>

There may be multiple MIDX files in a single pack directory. The primary file is pointed to by a pointer file "midx-head" that contains an OID. The MIDX file to load is then given by "midx-<OID>.midx".

This head file will be especially important when the MIDX files are extended to be incremental and we expect multiple MIDX files at any point.

Signed-off-by: Derrick Stolee <dstolee@microsoft.com>
---
 Documentation/git-midx.txt | 19 ++++++++++++++++++-
 builtin/midx.c             | 32 ++++++++++++++++++++++++++++++--
 midx.c                     | 31 +++++++++++++++++++++++++++++++
 midx.h                     |  3 +++
 t/t5318-midx.sh            | 33 ++++++++++++++++++++++-----------
 5 files changed, 104 insertions(+), 14 deletions(-)
diff --git a/Documentation/git-midx.txt b/Documentation/git-midx.txt
index 17464222c1..01f79cbba5 100644
--- a/Documentation/git-midx.txt
+++ b/Documentation/git-midx.txt
@@ -9,7 +9,7 @@ git-midx - Write and verify multi-pack-indexes (MIDX files).
 SYNOPSIS
 --------
 [verse]
-'git midx' --write [--pack-dir <pack_dir>]
+'git midx' --write <options> [--pack-dir <pack_dir>]
 
 DESCRIPTION
 -----------
@@ -26,15 +26,32 @@ OPTIONS
 	If specified, write a new midx file to the pack directory using
 	the packfiles present. Outputs the hash of the result midx file.
 
+--update-head::
+	If specified with --write, update the midx-head file to point to
+	the written midx file.
+
 EXAMPLES
 --------
 
+* Read the midx-head file and output the OID of the head MIDX file.
++
+------------------------------------------------
+$ git midx
+------------------------------------------------
+
 * Write a MIDX file for the packfiles in your local .git folder.
 +
 ------------------------------------------------
 $ git midx --write
 ------------------------------------------------
 
+* Write a MIDX file for the packfiles in your local .git folder and
+* update the midx-head file.
++
+------------------------------------------------
+$ git midx --write --update-head
+------------------------------------------------
+
 * Write a MIDX file for the packfiles in a different folder
 +
 ---------------------------------------------------------
diff --git a/builtin/midx.c b/builtin/midx.c
index 4aae14cf8e..84ce6588a2 100644
--- a/builtin/midx.c
+++ b/builtin/midx.c
@@ -9,13 +9,17 @@
 #include "midx.h"
 
 static char const * const builtin_midx_usage[] = {
-	N_("git midx --write [--pack-dir <packdir>]"),
+	N_("git midx [--pack-dir <packdir>]"),
+	N_("git midx --write [--update-head] [--pack-dir <packdir>]"),
 	NULL
 };
 
 static struct opts_midx {
 	const char *pack_dir;
 	int write;
+	int update_head;
+	int has_existing;
+	struct object_id old_midx_oid;
 } opts;
 
 static int build_midx_from_packs(
@@ -109,6 +113,22 @@ static int build_midx_from_packs(
 	return 0;
 }
 
+static void update_head_file(const char *pack_dir, const char *midx_id)
+{
+	int fd;
+	struct lock_file lk = LOCK_INIT;
+	char *head_path = get_midx_head_filename(pack_dir);
+
+	fd = hold_lock_file_for_update(&lk, head_path, LOCK_DIE_ON_ERROR);
+	FREE_AND_NULL(head_path);
+
+	if (fd < 0)
+		die_errno("unable to open midx-head");
+
+	write_in_full(fd, midx_id, GIT_MAX_HEXSZ);
+	commit_lock_file(&lk);
+}
+
 static int midx_write(void)
 {
 	const char **pack_names = NULL;
@@ -152,6 +172,9 @@ static int midx_write(void)
 
 	printf("%s\n", midx_id);
 
+	if (opts.update_head)
+		update_head_file(opts.pack_dir, midx_id);
+
 cleanup:
 	if (pack_names)
 		FREE_AND_NULL(pack_names);
@@ -166,6 +189,8 @@ int cmd_midx(int argc, const char **argv, const char *prefix)
 			N_("The pack directory containing set of packfile and pack-index pairs.") },
 		OPT_BOOL('w', "write", &opts.write,
 			N_("write midx file")),
+		OPT_BOOL('u', "update-head", &opts.update_head,
+			N_("update midx-head to written midx file")),
 		OPT_END(),
 	};
 
@@ -187,9 +212,12 @@ int cmd_midx(int argc, const char **argv, const char *prefix)
 		opts.pack_dir = strbuf_detach(&path, NULL);
 	}
 
+	opts.has_existing = !!get_midx_head_oid(opts.pack_dir, &opts.old_midx_oid);
+
 	if (opts.write)
 		return midx_write();
 
-	usage_with_options(builtin_midx_usage, builtin_midx_options);
+	if (opts.has_existing)
+		printf("%s\n", oid_to_hex(&opts.old_midx_oid));
 	return 0;
 }
diff --git a/midx.c b/midx.c
index 5c320726ed..f4178c1b81 100644
--- a/midx.c
+++ b/midx.c
@@ -34,6 +34,37 @@ char* get_midx_filename_oid(const char *pack_dir,
 	return strbuf_detach(&head_path, NULL);
 }
 
+char *get_midx_head_filename(const char *pack_dir)
+{
+	struct strbuf head_filename = STRBUF_INIT;
+	strbuf_addstr(&head_filename, pack_dir);
+	strbuf_addstr(&head_filename, "/midx-head");
+	return strbuf_detach(&head_filename, NULL);
+}
+
+struct object_id *get_midx_head_oid(const char *pack_dir,
+				    struct object_id *oid)
+{
+	char oid_hex[GIT_MAX_HEXSZ + 1];
+	FILE *f;
+	char *head_filename = get_midx_head_filename(pack_dir);
+
+	f = fopen(head_filename, "r");
+	FREE_AND_NULL(head_filename);
+
+	if (!f)
+		return 0;
+
+	if (!fgets(oid_hex, sizeof(oid_hex), f))
+		die("failed to read midx-head");
+
+	fclose(f);
+
+	if (get_oid_hex(oid_hex, oid))
+		return 0;
+	return oid;
+}
+
 struct pack_midx_details_internal {
 	uint32_t pack_int_id;
 	uint32_t internal_offset;
diff --git a/midx.h b/midx.h
index 4b00463651..9d9ab85261 100644
--- a/midx.h
+++ b/midx.h
@@ -7,6 +7,9 @@
 
 extern char *get_midx_filename_oid(const char *pack_dir,
 				   struct object_id *oid);
+extern char *get_midx_head_filename(const char *pack_dir);
+
+extern struct object_id *get_midx_head_oid(const char *pack_dir, struct object_id *oid);
 
 struct pack_midx_entry {
 	struct object_id oid;
diff --git a/t/t5318-midx.sh b/t/t5318-midx.sh
index 869bbea29c..b66efcdce9 100755
--- a/t/t5318-midx.sh
+++ b/t/t5318-midx.sh
@@ -29,13 +29,16 @@ test_expect_success 'create objects' \
 test_expect_success 'write-midx from index version 1' \
     'pack1=$(git rev-list --all --objects | git pack-objects --index-version=1 ${packdir}/test-1) &&
      midx1=$(git midx --write) &&
-     test_path_is_file ${packdir}/midx-${midx1}.midx'
+     test_path_is_file ${packdir}/midx-${midx1}.midx &&
+     test_path_is_missing ${packdir}/midx-head'
 
 test_expect_success 'write-midx from index version 2' \
     'rm "${packdir}/test-1-${pack1}.pack" &&
      pack2=$(git rev-list --all --objects | git pack-objects --index-version=2 ${packdir}/test-2) &&
-     midx2=$(git midx --write) &&
-     test_path_is_file ${packdir}/midx-${midx2}.midx'
+     midx2=$(git midx --write --update-head) &&
+     test_path_is_file ${packdir}/midx-${midx2}.midx &&
+     test_path_is_file ${packdir}/midx-head &&
+     test $(cat ${packdir}/midx-head) = "$midx2"'
 
 test_expect_success 'Create more objects' \
     'for i in $(test_seq 100)
@@ -49,8 +52,10 @@ test_expect_success 'Create more objects' \
 
 test_expect_success 'write-midx with two packs' \
     'pack3=$(git rev-list --objects commit2 ^commit1 | git pack-objects --index-version=2 ${packdir}/test-3) &&
-     midx3=$(git midx --write) &&
-     test_path_is_file ${packdir}/midx-${midx3}.midx'
+     midx3=$(git midx --write --update-head) &&
+     test_path_is_file ${packdir}/midx-${midx3}.midx &&
+     test_path_is_file ${packdir}/midx-head &&
+     test $(cat ${packdir}/midx-head) = "$midx3"'
 
 test_expect_success 'Add more packs' \
     'for j in $(test_seq 10)
@@ -77,13 +82,17 @@ test_expect_success 'Add more packs' \
      done'
 
 test_expect_success 'write-midx with twelve packs' \
-    'midx4=$(git midx --write) &&
-     test_path_is_file ${packdir}/midx-${midx4}.midx'
+    'midx4=$(git midx --write --update-head) &&
+     test_path_is_file ${packdir}/midx-${midx4}.midx &&
+     test_path_is_file ${packdir}/midx-head &&
+     test $(cat ${packdir}/midx-head) = "$midx4"'
 
 test_expect_success 'write-midx with no new packs' \
-    'midx5=$(git midx --write) &&
+    'midx5=$(git midx --write --update-head) &&
      test_path_is_file ${packdir}/midx-${midx5}.midx &&
-     test "a$midx4" = "a$midx5"'
+     test "a$midx4" = "a$midx5" &&
+     test_path_is_file ${packdir}/midx-head &&
+     test $(cat ${packdir}/midx-head) = "$midx4"'
 
 test_expect_success 'create bare repo' \
     'cd .. &&
@@ -94,7 +103,9 @@ test_expect_success 'create bare repo' \
      baredir=objects/pack'
 
 test_expect_success 'write-midx in bare repo' \
-    'midxbare=$(git midx --write) &&
-     test_path_is_file ${baredir}/midx-${midxbare}.midx'
+    'midxbare=$(git midx --write --update-head) &&
+     test_path_is_file ${baredir}/midx-${midxbare}.midx  &&
+     test_path_is_file ${baredir}/midx-head &&
+     test $(cat ${baredir}/midx-head) = "$midxbare"'
 
 test_done

-- 
2.15.0
Previous: Derrick StoleeNext: Derrick Stolee
Message 20 of 48 in “Multi-pack index (MIDX)”
  1. 00/18 Multi-pack index (MIDX)Derrick Stolee, Jan 7, 2018
  2. 01/18 docs: Multi-Pack Index (MIDX) Design NotesDerrick Stolee, Jan 7, 2018
  3. Jonathan TanJan 8, 2018
  4. Derrick StoleeJan 8, 2018
  5. Jonathan TanJan 8, 2018
  6. 02/18 midx: specify midx file formatDerrick Stolee, Jan 7, 2018
  7. 06/18 midx: add t5318-midx.sh test scriptDerrick Stolee, Jan 7, 2018
  8. 09/18 midx: find details of nth object in midxDerrick Stolee, Jan 7, 2018
  9. 11/18 midx: teach git-midx to clear midx filesDerrick Stolee, Jan 7, 2018
  10. 12/18 midx: teach git-midx to delete expired filesDerrick Stolee, Jan 7, 2018
  11. 13/18 t5318-midx.h: confirm git actions are stableDerrick Stolee, Jan 7, 2018
  12. 15/18 midx: use midx for approximate object countDerrick Stolee, Jan 7, 2018
  13. 18/18 packfile: use midx for object loadsDerrick Stolee, Jan 7, 2018
  14. 17/18 sha1_name: use midx for abbreviationsDerrick Stolee, Jan 7, 2018
  15. 08/18 midx: teach git-midx to read midx file detailsDerrick Stolee, Jan 7, 2018
  16. 14/18 midx: load midx files when loading packsDerrick Stolee, Jan 7, 2018
  17. 10/18 midx: use existing midx when writingDerrick Stolee, Jan 7, 2018
  18. 16/18 midx: nth_midxed_object_oid() and bsearch_midx()Derrick Stolee, Jan 7, 2018
  19. 03/18 midx: create core.midx config settingDerrick Stolee, Jan 7, 2018
  20. 07/18 midx: teach midx --write to update midx-headDerrick Stolee, Jan 7, 2018
  21. 04/18 midx: write multi-pack indexes for an object listDerrick Stolee, Jan 7, 2018
  22. 05/18 midx: create midx builtin with --write modeDerrick Stolee, Jan 7, 2018
  23. Ævar Arnfjörð BjarmasonJan 7, 2018
  24. Derrick StoleeJan 8, 2018
  25. Jeff KingJan 8, 2018
  26. Jeff KingJan 8, 2018
  27. Ævar Arnfjörð BjarmasonJan 8, 2018
  28. Johannes SchindelinJan 8, 2018
  29. Jeff KingJan 9, 2018
  30. Johannes SchindelinJan 9, 2018
  31. Stefan BellerJan 9, 2018
  32. Junio C HamanoJan 9, 2018
  33. Stefan BellerJan 9, 2018
  34. Junio C HamanoJan 9, 2018
  35. Johannes SchindelinJan 10, 2018
  36. Jeff KingJan 10, 2018
  37. Derrick StoleeJan 8, 2018
  38. Jeff KingJan 9, 2018
  39. Ævar Arnfjörð BjarmasonJan 8, 2018
  40. Ævar Arnfjörð BjarmasonJun 6, 2018
  41. 0/2 unconditional O(1) SHA-1 abbreviationÆvar Arnfjörð Bjarmason, Jun 6, 2018
  42. 1/2 config.c: use braces on multiple conditional armsÆvar Arnfjörð Bjarmason, Jun 6, 2018
  43. 2/2 sha1-name: add core.validateAbbrev & relative core.abbrevÆvar Arnfjörð Bjarmason, Jun 6, 2018
  44. Christian CouderJun 6, 2018
  45. Derrick StoleeJun 6, 2018
  46. Martin FickJan 10, 2018
  47. Derrick StoleeJan 10, 2018
  48. Martin FickJan 10, 2018

Read the whole thread, see it on lore, or plain text.

$ cat FOOTERMessages come from the public archive at lore.kernel.org/git, fetched every hour. The front page is chosen and written each morning by an AI editor and can be wrong; the threads themselves are the record. About and API. For agents: an MCP server at https://gitlist.dev/mcp, and any thread, story or person page as Markdown by adding .md to its URL (or sending Accept: text/markdown). Details in /llms.txt.