git/list[1] front-page[2] threads[3] people[4] search[5] about
 

[PATCH v4 3/7] gc: add --keep-largest-pack option

From
Nguyễn Thái Ngọc Duy <pclouds@gmail.com>
Date
Mar 24, 2018, 07:42 UTC
Message-ID
<20180324074308.18934-6-pclouds@gmail.com>
In-Reply-To
<20180324074308.18934-1-pclouds@gmail.com>

This adds a new repack mode that combines everything into a secondary pack, leaving the largest pack alone.

This could help reduce memory pressure. On linux-2.6.git, valgrind massif reports 1.6GB heap in "pack all" case, and 535MB in "pack all except the base pack" case. We save roughly 1GB memory by excluding the base pack.

This should also lower I/O because we don't have to rewrite a giant pack every time (e.g. for linux-2.6.git that's a 1.4GB pack file)..

PS. The use of string_list here seems overkill, but we'll need it in the next patch...

Signed-off-by: Nguyễn Thái Ngọc Duy <pclouds@gmail.com>
---
 Documentation/git-gc.txt |  6 ++++-
 builtin/gc.c             | 47 ++++++++++++++++++++++++++++++++++++----
 t/t6500-gc.sh            | 25 +++++++++++++++++++++
 3 files changed, 73 insertions(+), 5 deletions(-)
diff --git a/Documentation/git-gc.txt b/Documentation/git-gc.txt
index 571b5a7e3c..bf81b8de30 100644
--- a/Documentation/git-gc.txt
+++ b/Documentation/git-gc.txt
@@ -9,7 +9,7 @@ git-gc - Cleanup unnecessary files and optimize the local repository
 SYNOPSIS
 --------
 [verse]
-'git gc' [--aggressive] [--auto] [--quiet] [--prune=<date> | --no-prune] [--force]
+'git gc' [--aggressive] [--auto] [--quiet] [--prune=<date> | --no-prune] [--force] [--keep-largest-pack]
 
 DESCRIPTION
 -----------
@@ -78,6 +78,10 @@ automatic consolidation of packs.
 	Force `git gc` to run even if there may be another `git gc`
 	instance running on this repository.
 
+--keep-largest-pack::
+	All packs except the largest pack and those marked with a
+	`.keep` files are consolidated into a single pack.
+
 Configuration
 -------------
 
diff --git a/builtin/gc.c b/builtin/gc.c
index 77fa720bd0..9a09cf53b0 100644
--- a/builtin/gc.c
+++ b/builtin/gc.c
@@ -164,6 +164,24 @@ static int too_many_loose_objects(void)
 	return needed;
 }
 
+static void find_base_packs(struct string_list *packs)
+{
+	struct packed_git *p, *base = NULL;
+
+	prepare_packed_git();
+
+	for (p = packed_git; p; p = p->next) {
+		if (!p->pack_local)
+			continue;
+		if (!base || base->pack_size < p->pack_size) {
+			base = p;
+		}
+	}
+
+	if (base)
+		string_list_append(packs, base->pack_name);
+}
+
 static int too_many_packs(void)
 {
 	struct packed_git *p;
@@ -187,7 +205,13 @@ static int too_many_packs(void)
 	return gc_auto_pack_limit < cnt;
 }
 
-static void add_repack_all_option(void)
+static int keep_one_pack(struct string_list_item *item, void *data)
+{
+	argv_array_pushf(&repack, "--keep-pack=%s", basename(item->string));
+	return 0;
+}
+
+static void add_repack_all_option(struct string_list *keep_pack)
 {
 	if (prune_expire && !strcmp(prune_expire, "now"))
 		argv_array_push(&repack, "-a");
@@ -196,6 +220,9 @@ static void add_repack_all_option(void)
 		if (prune_expire)
 			argv_array_pushf(&repack, "--unpack-unreachable=%s", prune_expire);
 	}
+
+	if (keep_pack)
+		for_each_string_list(keep_pack, keep_one_pack, NULL);
 }
 
 static void add_repack_incremental_option(void)
@@ -219,7 +246,7 @@ static int need_to_gc(void)
 	 * there is no need.
 	 */
 	if (too_many_packs())
-		add_repack_all_option();
+		add_repack_all_option(NULL);
 	else if (too_many_loose_objects())
 		add_repack_incremental_option();
 	else
@@ -353,6 +380,7 @@ int cmd_gc(int argc, const char **argv, const char *prefix)
 	const char *name;
 	pid_t pid;
 	int daemonized = 0;
+	int keep_base_pack = -1;
 
 	struct option builtin_gc_options[] = {
 		OPT__QUIET(&quiet, N_("suppress progress reporting")),
@@ -362,6 +390,8 @@ int cmd_gc(int argc, const char **argv, const char *prefix)
 		OPT_BOOL(0, "aggressive", &aggressive, N_("be more thorough (increased runtime)")),
 		OPT_BOOL(0, "auto", &auto_gc, N_("enable auto-gc mode")),
 		OPT_BOOL(0, "force", &force, N_("force running gc even if there may be another gc running")),
+		OPT_BOOL(0, "keep-largest-pack", &keep_base_pack,
+			 N_("repack all other packs except the largest pack")),
 		OPT_END()
 	};
 
@@ -427,8 +457,17 @@ int cmd_gc(int argc, const char **argv, const char *prefix)
 			 */
 			daemonized = !daemonize();
 		}
-	} else
-		add_repack_all_option();
+	} else {
+		struct string_list keep_pack = STRING_LIST_INIT_NODUP;
+
+		if (keep_base_pack != -1) {
+			if (keep_base_pack)
+				find_base_packs(&keep_pack);
+		}
+
+		add_repack_all_option(&keep_pack);
+		string_list_clear(&keep_pack, 0);
+	}
 
 	name = lock_repo_for_gc(force, &pid);
 	if (name) {
diff --git a/t/t6500-gc.sh b/t/t6500-gc.sh
index 41b0be575d..a95969af2a 100755
--- a/t/t6500-gc.sh
+++ b/t/t6500-gc.sh
@@ -43,6 +43,31 @@ test_expect_success 'gc is not aborted due to a stale symref' '
 	)
 '
 
+test_expect_success 'gc --keep-largest-pack' '
+	test_create_repo keep-pack &&
+	(
+		cd keep-pack &&
+		test_commit one &&
+		test_commit two &&
+		test_commit three &&
+		git gc &&
+		( cd .git/objects/pack && ls *.pack ) >pack-list &&
+		test_line_count = 1 pack-list &&
+		BASE_PACK=.git/objects/pack/pack-*.pack &&
+		test_commit four &&
+		git repack -d &&
+		test_commit five &&
+		git repack -d &&
+		( cd .git/objects/pack && ls *.pack ) >pack-list &&
+		test_line_count = 3 pack-list &&
+		git gc --keep-largest-pack &&
+		( cd .git/objects/pack && ls *.pack ) >pack-list &&
+		test_line_count = 2 pack-list &&
+		test_path_is_file $BASE_PACK &&
+		git fsck
+	)
+'
+
 test_expect_success 'auto gc with too many loose objects does not attempt to create bitmaps' '
 	test_config gc.auto 3 &&
 	test_config gc.autodetach false &&
-- 
2.17.0.rc0.348.gd5a49e0b6f
Previous: Nguyễn Thái Ngọc DuyNext: Ævar Arnfjörð Bjarmason
Message 58 of 110 in “Combine t/helper binaries into a single one”
  1. 00/36 Combine t/helper binaries into a single oneNguyễn Thái Ngọc Duy, Mar 17, 2018
  2. 01/36 t/helper: add an empty test-tool programNguyễn Thái Ngọc Duy, Mar 17, 2018
  3. 03/36 t/helper: merge test-sha1 into test-toolNguyễn Thái Ngọc Duy, Mar 17, 2018
  4. 10/36 t/helper: merge test-dump-cache-tree into test-toolNguyễn Thái Ngọc Duy, Mar 17, 2018
  5. 13/36 t/helper: merge test-genrandom into test-toolNguyễn Thái Ngọc Duy, Mar 17, 2018
  6. 11/36 t/helper: merge test-dump-split-index into test-toolNguyễn Thái Ngọc Duy, Mar 17, 2018
  7. 19/36 t/helper: merge test-online-cpus into test-toolNguyễn Thái Ngọc Duy, Mar 17, 2018
  8. 22/36 t/helepr: merge test-read-cache into test-toolNguyễn Thái Ngọc Duy, Mar 17, 2018
  9. Eric SunshineMar 18, 2018
  10. 26/36 t/helper: merge test-run-command into test-toolNguyễn Thái Ngọc Duy, Mar 17, 2018
  11. 28/36 t/helper: merge test-sha1-array into test-toolNguyễn Thái Ngọc Duy, Mar 17, 2018
  12. 31/36 t/helper: merge test-string-list into test-toolNguyễn Thái Ngọc Duy, Mar 17, 2018
  13. 33/36 t/helper: merge test-subprocess into test-toolNguyễn Thái Ngọc Duy, Mar 17, 2018
  14. 35/36 t/helper: merge test-wildmatch into test-toolNguyễn Thái Ngọc Duy, Mar 17, 2018
  15. 34/36 t/helper: merge test-urlmatch-.. into test-toolNguyễn Thái Ngọc Duy, Mar 17, 2018
  16. Eric SunshineMar 18, 2018
  17. 36/36 t/helper: merge test-write-cache into test-toolNguyễn Thái Ngọc Duy, Mar 17, 2018
  18. 32/36 t/helper: merge test-submodule-config into test-toolNguyễn Thái Ngọc Duy, Mar 17, 2018
  19. 30/36 t/helper: merge test-strcmp-offset into test-toolNguyễn Thái Ngọc Duy, Mar 17, 2018
  20. 27/36 t/helper: merge test-scrap-cache-tree into test-toolNguyễn Thái Ngọc Duy, Mar 17, 2018
  21. 29/36 t/helper: merge test-sigchain into test-toolNguyễn Thái Ngọc Duy, Mar 17, 2018
  22. 18/36 t/helper: merge test-mktemp into test-toolNguyễn Thái Ngọc Duy, Mar 17, 2018
  23. 25/36 t/helper: merge test-revision-walking into test-toolNguyễn Thái Ngọc Duy, Mar 17, 2018
  24. 21/36 t/helper: merge test-prio-queue into test-toolNguyễn Thái Ngọc Duy, Mar 17, 2018
  25. 24/36 t/helper: merge test-regex into test-toolNguyễn Thái Ngọc Duy, Mar 17, 2018
  26. 20/36 t/helper: merge test-path-utils into test-toolNguyễn Thái Ngọc Duy, Mar 17, 2018
  27. 23/36 t/helper: merge test-ref-store into test-toolNguyễn Thái Ngọc Duy, Mar 17, 2018
  28. 12/36 t/helper: merge test-example-decorate into test-toolNguyễn Thái Ngọc Duy, Mar 17, 2018
  29. 14/36 t/helper: merge test-hashmap into test-toolNguyễn Thái Ngọc Duy, Mar 17, 2018
  30. 17/36 t/helper: merge (unused) test-mergesort into test-toolNguyễn Thái Ngọc Duy, Mar 17, 2018
  31. 15/36 t/helper: merge test-index-version into test-toolNguyễn Thái Ngọc Duy, Mar 17, 2018
  32. 16/36 t/helper: merge (unused) test-match-trees into test-toolNguyễn Thái Ngọc Duy, Mar 17, 2018
  33. 08/36 t/helper: merge (unused) test-delta into test-toolNguyễn Thái Ngọc Duy, Mar 17, 2018
  34. 05/36 t/helper: merge test-config into test-toolNguyễn Thái Ngọc Duy, Mar 17, 2018
  35. 02/36 t/helper: merge test-chmtime into test-toolNguyễn Thái Ngọc Duy, Mar 17, 2018
  36. 09/36 t/helper: merge test-drop-caches into test-toolNguyễn Thái Ngọc Duy, Mar 17, 2018
  37. 04/36 t/helper: merge test-lazy-init-name-hash into test-toolNguyễn Thái Ngọc Duy, Mar 17, 2018
  38. Eric SunshineMar 18, 2018
  39. Duy NguyenMar 18, 2018
  40. Eric SunshineMar 18, 2018
  41. Jeff HostetlerMar 19, 2018
  42. Duy NguyenMar 19, 2018
  43. 07/36 t/helper: merge test-date into test-toolNguyễn Thái Ngọc Duy, Mar 17, 2018
  44. 06/36 t/helper: merge test-ctype into test-toolNguyễn Thái Ngọc Duy, Mar 17, 2018
  45. 00/36 Combine t/helper binaries into a single oneNguyễn Thái Ngọc Duy, Mar 24, 2018
  46. 1/7 t7700: have closing quote of a test at the beginning of lineNguyễn Thái Ngọc Duy, Mar 24, 2018
  47. 01/36 t/helper: add an empty test-tool programNguyễn Thái Ngọc Duy, Mar 24, 2018
  48. 2/7 repack: add --keep-pack optionNguyễn Thái Ngọc Duy, Mar 24, 2018
  49. 03/36 t/helper: merge test-sha1 into test-toolNguyễn Thái Ngọc Duy, Mar 24, 2018
  50. 05/36 t/helper: merge test-config into test-toolNguyễn Thái Ngọc Duy, Mar 24, 2018
  51. 4/7 gc: add gc.bigPackThreshold configNguyễn Thái Ngọc Duy, Mar 24, 2018
  52. 06/36 t/helper: merge test-ctype into test-toolNguyễn Thái Ngọc Duy, Mar 24, 2018
  53. 04/36 t/helper: merge test-lazy-init-name-hash into test-toolNguyễn Thái Ngọc Duy, Mar 24, 2018
  54. 07/36 t/helper: merge test-date into test-toolNguyễn Thái Ngọc Duy, Mar 24, 2018
  55. 5/7 gc: handle a corner case in gc.bigPackThresholdNguyễn Thái Ngọc Duy, Mar 24, 2018
  56. 6/7 gc --auto: exclude base pack if not enough mem to "repack -ad"Nguyễn Thái Ngọc Duy, Mar 24, 2018
  57. 7/7 pack-objects: show some progress when counting kept objectsNguyễn Thái Ngọc Duy, Mar 24, 2018
  58. 3/7 gc: add --keep-largest-pack optionNguyễn Thái Ngọc Duy, Mar 24, 2018
  59. Ævar Arnfjörð BjarmasonMar 24, 2018
  60. Duy NguyenMar 24, 2018
  61. Junio C HamanoMar 26, 2018
  62. 02/36 t/helper: merge test-chmtime into test-toolNguyễn Thái Ngọc Duy, Mar 24, 2018
  63. 00/36 Combine t/helper binaries into a single oneNguyễn Thái Ngọc Duy, Mar 24, 2018
  64. 01/36 t/helper: add an empty test-tool programNguyễn Thái Ngọc Duy, Mar 24, 2018
  65. Johannes SchindelinMar 26, 2018
  66. Duy NguyenMar 26, 2018
  67. Johannes SchindelinMar 26, 2018
  68. SZEDER GáborMar 26, 2018
  69. Johannes SchindelinMar 27, 2018
  70. 02/36 t/helper: merge test-chmtime into test-toolNguyễn Thái Ngọc Duy, Mar 24, 2018
  71. 06/36 t/helper: merge test-ctype into test-toolNguyễn Thái Ngọc Duy, Mar 24, 2018
  72. 08/36 t/helper: merge (unused) test-delta into test-toolNguyễn Thái Ngọc Duy, Mar 24, 2018
  73. 07/36 t/helper: merge test-date into test-toolNguyễn Thái Ngọc Duy, Mar 24, 2018
  74. 09/36 t/helper: merge test-drop-caches into test-toolNguyễn Thái Ngọc Duy, Mar 24, 2018
  75. 13/36 t/helper: merge test-genrandom into test-toolNguyễn Thái Ngọc Duy, Mar 24, 2018
  76. 14/36 t/helper: merge test-hashmap into test-toolNguyễn Thái Ngọc Duy, Mar 24, 2018
  77. 16/36 t/helper: merge (unused) test-match-trees into test-toolNguyễn Thái Ngọc Duy, Mar 24, 2018
  78. 12/36 t/helper: merge test-example-decorate into test-toolNguyễn Thái Ngọc Duy, Mar 24, 2018
  79. 26/36 t/helper: merge test-run-command into test-toolNguyễn Thái Ngọc Duy, Mar 24, 2018
  80. 27/36 t/helper: merge test-scrap-cache-tree into test-toolNguyễn Thái Ngọc Duy, Mar 24, 2018
  81. 29/36 t/helper: merge test-sigchain into test-toolNguyễn Thái Ngọc Duy, Mar 24, 2018
  82. 28/36 t/helper: merge test-sha1-array into test-toolNguyễn Thái Ngọc Duy, Mar 24, 2018
  83. 30/36 t/helper: merge test-strcmp-offset into test-toolNguyễn Thái Ngọc Duy, Mar 24, 2018
  84. 33/36 t/helper: merge test-subprocess into test-toolNguyễn Thái Ngọc Duy, Mar 24, 2018
  85. 35/36 t/helper: merge test-wildmatch into test-toolNguyễn Thái Ngọc Duy, Mar 24, 2018
  86. 34/36 t/helper: merge test-urlmatch-normalization into test-toolNguyễn Thái Ngọc Duy, Mar 24, 2018
  87. 36/36 t/helper: merge test-write-cache into test-toolNguyễn Thái Ngọc Duy, Mar 24, 2018
  88. 32/36 t/helper: merge test-submodule-config into test-toolNguyễn Thái Ngọc Duy, Mar 24, 2018
  89. 31/36 t/helper: merge test-string-list into test-toolNguyễn Thái Ngọc Duy, Mar 24, 2018
  90. 15/36 t/helper: merge test-index-version into test-toolNguyễn Thái Ngọc Duy, Mar 24, 2018
  91. 25/36 t/helper: merge test-revision-walking into test-toolNguyễn Thái Ngọc Duy, Mar 24, 2018
  92. 18/36 t/helper: merge test-mktemp into test-toolNguyễn Thái Ngọc Duy, Mar 24, 2018
  93. 24/36 t/helper: merge test-regex into test-toolNguyễn Thái Ngọc Duy, Mar 24, 2018
  94. 23/36 t/helper: merge test-ref-store into test-toolNguyễn Thái Ngọc Duy, Mar 24, 2018
  95. 17/36 t/helper: merge (unused) test-mergesort into test-toolNguyễn Thái Ngọc Duy, Mar 24, 2018
  96. 22/36 t/helper: merge test-read-cache into test-toolNguyễn Thái Ngọc Duy, Mar 24, 2018
  97. 19/36 t/helper: merge test-online-cpus into test-toolNguyễn Thái Ngọc Duy, Mar 24, 2018
  98. 10/36 t/helper: merge test-dump-cache-tree into test-toolNguyễn Thái Ngọc Duy, Mar 24, 2018
  99. 11/36 t/helper: merge test-dump-split-index into test-toolNguyễn Thái Ngọc Duy, Mar 24, 2018
  100. 20/36 t/helper: merge test-path-utils into test-toolNguyễn Thái Ngọc Duy, Mar 24, 2018
  101. 21/36 t/helper: merge test-prio-queue into test-toolNguyễn Thái Ngọc Duy, Mar 24, 2018
  102. 03/36 t/helper: merge test-sha1 into test-toolNguyễn Thái Ngọc Duy, Mar 24, 2018
  103. 05/36 t/helper: merge test-config into test-toolNguyễn Thái Ngọc Duy, Mar 24, 2018
  104. 04/36 t/helper: merge test-lazy-init-name-hash into test-toolNguyễn Thái Ngọc Duy, Mar 24, 2018
  105. Ævar Arnfjörð BjarmasonMar 24, 2018
  106. Duy NguyenMar 24, 2018
  107. Junio C HamanoMar 26, 2018
  108. Johannes SchindelinMar 27, 2018
  109. Junio C HamanoMar 27, 2018
  110. Johannes SchindelinMar 27, 2018

Read the whole thread, see it on lore, or plain text.

$ cat FOOTERMessages come from the public archive at lore.kernel.org/git, fetched every hour. The front page is chosen and written each morning by an AI editor and can be wrong; the threads themselves are the record. About and API. For agents: an MCP server at https://gitlist.dev/mcp, and any thread, story or person page as Markdown by adding .md to its URL (or sending Accept: text/markdown). Details in /llms.txt.