git/list[1] front-page[2] threads[3] people[4] search[5] about
 

[RFC PATCH 12/18] midx: teach git-midx to delete expired files

From
Derrick Stolee <stolee@gmail.com>
Date
Jan 7, 2018, 18:14 UTC
Message-ID
<20180107181459.222909-13-dstolee@microsoft.com>
In-Reply-To
<20180107181459.222909-1-dstolee@microsoft.com>

As we write new MIDX files, the existing files are probably not needed. Supply the "--delete-expired" flag to remove these files during the "--write" sub- command.

Signed-off-by: Derrick Stolee <dstolee@microsoft.com>
---
 Documentation/git-midx.txt |  4 ++++
 builtin/midx.c             | 15 ++++++++++++++-
 midx.c                     | 26 ++++++++++++++++++++++++++
 midx.h                     |  2 ++
 packfile.c                 |  2 +-
 packfile.h                 |  1 +
 t/t5318-midx.sh            |  9 ++++++---
 7 files changed, 54 insertions(+), 5 deletions(-)
diff --git a/Documentation/git-midx.txt b/Documentation/git-midx.txt
index c184d3a593..4635247d0d 100644
--- a/Documentation/git-midx.txt
+++ b/Documentation/git-midx.txt
@@ -43,6 +43,10 @@ OPTIONS
 	If specified with --write, update the midx-head file to point to
 	the written midx file.
 
+--delete-expired::
+	If specified with --write and --update-head, delete the midx file
+	previously pointed to by midx-head (if changed).
+
 EXAMPLES
 --------
 
diff --git a/builtin/midx.c b/builtin/midx.c
index b30ef36ff8..6f56f39390 100644
--- a/builtin/midx.c
+++ b/builtin/midx.c
@@ -10,7 +10,7 @@
 
 static char const * const builtin_midx_usage[] = {
 	N_("git midx [--pack-dir <packdir>]"),
-	N_("git midx --write [--update-head] [--pack-dir <packdir>]"),
+	N_("git midx --write [--update-head [--delete-expired]] [--pack-dir <packdir>]"),
 	N_("git midx --clear [--pack-dir <packdir>]"),
 	NULL
 };
@@ -22,6 +22,7 @@ static struct opts_midx {
 	const char *midx_id;
 	int write;
 	int update_head;
+	int delete_expired;
 	int has_existing;
 	struct object_id old_midx_oid;
 } opts;
@@ -276,6 +277,16 @@ static int midx_write(void)
 	if (opts.update_head)
 		update_head_file(opts.pack_dir, midx_id);
 
+	if (opts.delete_expired && opts.update_head && opts.has_existing &&
+	    strcmp(midx_id, oid_to_hex(&opts.old_midx_oid))) {
+		char *old_path = get_midx_filename_oid(opts.pack_dir, &opts.old_midx_oid);
+		close_midx(midx);
+		if (remove_path(old_path))
+			die("failed to remove path %s", old_path);
+
+		free(old_path);
+	}
+
 cleanup:
 	if (pack_names)
 		FREE_AND_NULL(pack_names);
@@ -300,6 +311,8 @@ int cmd_midx(int argc, const char **argv, const char *prefix)
 			N_("write midx file")),
 		OPT_BOOL('u', "update-head", &opts.update_head,
 			N_("update midx-head to written midx file")),
+		OPT_BOOL('d', "delete-expired", &opts.delete_expired,
+			N_("delete expired head midx file")),
 		OPT_END(),
 	};
 
diff --git a/midx.c b/midx.c
index 53eb29dac3..3ce2b736ea 100644
--- a/midx.c
+++ b/midx.c
@@ -651,3 +651,29 @@ const char *write_midx_file(const char *pack_dir,
 
 	return final_hex;
 }
+
+int close_midx(struct midxed_git *m)
+{
+	int i;
+	if (m->midx_fd < 0)
+		return 0;
+
+	for (i = 0; i < m->num_packs; i++) {
+		if (m->packs[i]) {
+			close_pack(m->packs[i]);
+			free(m->packs[i]);
+			m->packs[i] = NULL;
+		}
+	}
+
+	munmap((void *)m->data, m->data_len);
+	m->data = 0;
+
+	close(m->midx_fd);
+	m->midx_fd = -1;
+
+	free(m->packs);
+	free(m->pack_names);
+
+	return 1;
+}
diff --git a/midx.h b/midx.h
index 1e7a94651c..27d48163e9 100644
--- a/midx.h
+++ b/midx.h
@@ -117,4 +117,6 @@ extern const char *write_midx_file(const char *pack_dir,
 				   struct pack_midx_entry **objects,
 				   uint32_t nr_objects);
 
+extern int close_midx(struct midxed_git *m);
+
 #endif
diff --git a/packfile.c b/packfile.c
index 4a5fe7ab18..c36420b33f 100644
--- a/packfile.c
+++ b/packfile.c
@@ -299,7 +299,7 @@ void close_pack_index(struct packed_git *p)
 	}
 }
 
-static void close_pack(struct packed_git *p)
+void close_pack(struct packed_git *p)
 {
 	close_pack_windows(p);
 	close_pack_fd(p);
diff --git a/packfile.h b/packfile.h
index 0cdeb54dcd..7cf4771029 100644
--- a/packfile.h
+++ b/packfile.h
@@ -61,6 +61,7 @@ extern void close_pack_index(struct packed_git *);
 
 extern unsigned char *use_pack(struct packed_git *, struct pack_window **, off_t, unsigned long *);
 extern void close_pack_windows(struct packed_git *);
+extern void close_pack(struct packed_git *p);
 extern void close_all_packs(void);
 extern void unuse_pack(struct pack_window **);
 extern void clear_delta_base_cache(void);
diff --git a/t/t5318-midx.sh b/t/t5318-midx.sh
index 9337355ab3..42d103c879 100755
--- a/t/t5318-midx.sh
+++ b/t/t5318-midx.sh
@@ -75,6 +75,7 @@ test_expect_success 'write-midx with two packs' \
     'pack3=$(git rev-list --objects commit2 ^commit1 | git pack-objects --index-version=2 ${packdir}/test-3) &&
      midx3=$(git midx --write --update-head) &&
      test_path_is_file ${packdir}/midx-${midx3}.midx &&
+     test_path_is_file ${packdir}/midx-${midx2}.midx &&
      test_path_is_file ${packdir}/midx-head &&
      test $(cat ${packdir}/midx-head) = "$midx3" &&
      _midx_read_expect \
@@ -107,8 +108,10 @@ test_expect_success 'Add more packs' \
      done'
 
 test_expect_success 'write-midx with twelve packs' \
-    'midx4=$(git midx --write --update-head) &&
+    'midx4=$(git midx --write --update-head --delete-expired) &&
      test_path_is_file ${packdir}/midx-${midx4}.midx &&
+     test_path_is_missing ${packdir}/midx-${midx3}.midx &&
+     test_path_is_file ${packdir}/midx-${midx2}.midx &&
      test_path_is_file ${packdir}/midx-head &&
      test $(cat ${packdir}/midx-head) = "$midx4" &&
      _midx_read_expect \
@@ -118,7 +121,7 @@ test_expect_success 'write-midx with twelve packs' \
      cmp output expect'
 
 test_expect_success 'write-midx with no new packs' \
-    'midx5=$(git midx --write --update-head) &&
+    'midx5=$(git midx --write --update-head --delete-expired) &&
      test_path_is_file ${packdir}/midx-${midx5}.midx &&
      test "a$midx4" = "a$midx5" &&
      test_path_is_file ${packdir}/midx-head &&
@@ -133,7 +136,7 @@ test_expect_success 'create bare repo' \
      baredir=./objects/pack'
 
 test_expect_success 'write-midx in bare repo' \
-    'midxbare=$(git midx --write --update-head) &&
+    'midxbare=$(git midx --write --update-head --delete-expired) &&
      test_path_is_file ${baredir}/midx-${midxbare}.midx  &&
      test_path_is_file ${baredir}/midx-head &&
      test $(cat ${baredir}/midx-head) = "$midxbare" &&

-- 
2.15.0
Previous: Derrick StoleeNext: Derrick Stolee
Message 10 of 48 in “Multi-pack index (MIDX)”
  1. 00/18 Multi-pack index (MIDX)Derrick Stolee, Jan 7, 2018
  2. 01/18 docs: Multi-Pack Index (MIDX) Design NotesDerrick Stolee, Jan 7, 2018
  3. Jonathan TanJan 8, 2018
  4. Derrick StoleeJan 8, 2018
  5. Jonathan TanJan 8, 2018
  6. 02/18 midx: specify midx file formatDerrick Stolee, Jan 7, 2018
  7. 06/18 midx: add t5318-midx.sh test scriptDerrick Stolee, Jan 7, 2018
  8. 09/18 midx: find details of nth object in midxDerrick Stolee, Jan 7, 2018
  9. 11/18 midx: teach git-midx to clear midx filesDerrick Stolee, Jan 7, 2018
  10. 12/18 midx: teach git-midx to delete expired filesDerrick Stolee, Jan 7, 2018
  11. 13/18 t5318-midx.h: confirm git actions are stableDerrick Stolee, Jan 7, 2018
  12. 15/18 midx: use midx for approximate object countDerrick Stolee, Jan 7, 2018
  13. 18/18 packfile: use midx for object loadsDerrick Stolee, Jan 7, 2018
  14. 17/18 sha1_name: use midx for abbreviationsDerrick Stolee, Jan 7, 2018
  15. 08/18 midx: teach git-midx to read midx file detailsDerrick Stolee, Jan 7, 2018
  16. 14/18 midx: load midx files when loading packsDerrick Stolee, Jan 7, 2018
  17. 10/18 midx: use existing midx when writingDerrick Stolee, Jan 7, 2018
  18. 16/18 midx: nth_midxed_object_oid() and bsearch_midx()Derrick Stolee, Jan 7, 2018
  19. 03/18 midx: create core.midx config settingDerrick Stolee, Jan 7, 2018
  20. 07/18 midx: teach midx --write to update midx-headDerrick Stolee, Jan 7, 2018
  21. 04/18 midx: write multi-pack indexes for an object listDerrick Stolee, Jan 7, 2018
  22. 05/18 midx: create midx builtin with --write modeDerrick Stolee, Jan 7, 2018
  23. Ævar Arnfjörð BjarmasonJan 7, 2018
  24. Derrick StoleeJan 8, 2018
  25. Jeff KingJan 8, 2018
  26. Jeff KingJan 8, 2018
  27. Ævar Arnfjörð BjarmasonJan 8, 2018
  28. Johannes SchindelinJan 8, 2018
  29. Jeff KingJan 9, 2018
  30. Johannes SchindelinJan 9, 2018
  31. Stefan BellerJan 9, 2018
  32. Junio C HamanoJan 9, 2018
  33. Stefan BellerJan 9, 2018
  34. Junio C HamanoJan 9, 2018
  35. Johannes SchindelinJan 10, 2018
  36. Jeff KingJan 10, 2018
  37. Derrick StoleeJan 8, 2018
  38. Jeff KingJan 9, 2018
  39. Ævar Arnfjörð BjarmasonJan 8, 2018
  40. Ævar Arnfjörð BjarmasonJun 6, 2018
  41. 0/2 unconditional O(1) SHA-1 abbreviationÆvar Arnfjörð Bjarmason, Jun 6, 2018
  42. 1/2 config.c: use braces on multiple conditional armsÆvar Arnfjörð Bjarmason, Jun 6, 2018
  43. 2/2 sha1-name: add core.validateAbbrev & relative core.abbrevÆvar Arnfjörð Bjarmason, Jun 6, 2018
  44. Christian CouderJun 6, 2018
  45. Derrick StoleeJun 6, 2018
  46. Martin FickJan 10, 2018
  47. Derrick StoleeJan 10, 2018
  48. Martin FickJan 10, 2018

Read the whole thread, see it on lore, or plain text.

$ cat FOOTERMessages come from the public archive at lore.kernel.org/git, fetched every hour. The front page is chosen and written each morning by an AI editor and can be wrong; the threads themselves are the record. About and API. For agents: an MCP server at https://gitlist.dev/mcp, and any thread, story or person page as Markdown by adding .md to its URL (or sending Accept: text/markdown). Details in /llms.txt.