git/list[1] front-page[2] threads[3] people[4] search[5] about
 

[PATCH 15/16] pack-objects: match prune logic for discarding objects

From
Jeff King <peff@peff.net>
Date
Oct 3, 2014, 20:40 UTC
Message-ID
<20141003204016.GO16293@peff.net>
In-Reply-To
<20141003202045.GA15205@peff.net>

A recent commit taught git-prune to keep non-recent objects that are reachable from recent ones. However, pack-objects, when loosening unreachable objects, tries to optimize out the write in the case that the object will be immediately pruned. It now gets this wrong, since its rule does not reflect the new prune code (and this can be seen by running t6501 with a strategically placed repack).

Let's teach pack-objects similar logic.
Signed-off-by: Jeff King <peff@peff.net>
---
The test changes look big because of the indentation. View with "-w" for
a sane diff.
 builtin/pack-objects.c     | 38 +++++++++++++++++++
 reachable.c                |  4 +-
 reachable.h                |  2 +
 t/t6501-freshen-objects.sh | 93 +++++++++++++++++++++++++++-------------------
 4 files changed, 97 insertions(+), 40 deletions(-)
diff --git a/builtin/pack-objects.c b/builtin/pack-objects.c
index 2fe2ab0..1db5ea5 100644
--- a/builtin/pack-objects.c
+++ b/builtin/pack-objects.c
@@ -20,6 +20,8 @@
 #include "streaming.h"
 #include "thread-utils.h"
 #include "pack-bitmap.h"
+#include "reachable.h"
+#include "sha1-array.h"
 
 static const char *pack_usage[] = {
 	N_("git pack-objects --stdout [options...] [< ref-list | < object-list]"),
@@ -2407,6 +2409,15 @@ static int has_sha1_pack_kept_or_nonlocal(const unsigned char *sha1)
 	return 0;
 }
 
+/*
+ * Store a list of sha1s that are should not be discarded
+ * because they are either written too recently, or are
+ * reachable from another object that was.
+ *
+ * This is filled by get_object_list.
+ */
+static struct sha1_array recent_objects;
+
 static int loosened_object_can_be_discarded(const unsigned char *sha1,
 					    unsigned long mtime)
 {
@@ -2414,6 +2425,8 @@ static int loosened_object_can_be_discarded(const unsigned char *sha1,
 		return 0;
 	if (mtime > unpack_unreachable_expiration)
 		return 0;
+	if (sha1_array_lookup(&recent_objects, sha1) >= 0)
+		return 0;
 	return 1;
 }
 
@@ -2470,6 +2483,19 @@ static int get_object_list_from_bitmap(struct rev_info *revs)
 	return 0;
 }
 
+static void record_recent_object(struct object *obj,
+				 const struct name_path *path,
+				 const char *last,
+				 void *data)
+{
+	sha1_array_append(&recent_objects, obj->sha1);
+}
+
+static void record_recent_commit(struct commit *commit, void *data)
+{
+	sha1_array_append(&recent_objects, commit->object.sha1);
+}
+
 static void get_object_list(int ac, const char **av)
 {
 	struct rev_info revs;
@@ -2517,10 +2543,22 @@ static void get_object_list(int ac, const char **av)
 	mark_edges_uninteresting(&revs, show_edge);
 	traverse_commit_list(&revs, show_commit, show_object, NULL);
 
+	if (unpack_unreachable_expiration) {
+		if (add_unseen_recent_objects_to_traversal(&revs,
+				unpack_unreachable_expiration))
+			die("unable to add recent objects");
+		if (prepare_revision_walk(&revs))
+			die("revision walk setup failed");
+		traverse_commit_list(&revs, record_recent_commit,
+				     record_recent_object, NULL);
+	}
+
 	if (keep_unreachable)
 		add_objects_in_unpacked_packs(&revs);
 	if (unpack_unreachable)
 		loosen_unused_packed_objects(&revs);
+
+	sha1_array_clear(&recent_objects);
 }
 
 static int option_parse_index_version(const struct option *opt,
diff --git a/reachable.c b/reachable.c
index f443265..5e35f21 100644
--- a/reachable.c
+++ b/reachable.c
@@ -298,8 +298,8 @@ static int add_recent_packed(const unsigned char *sha1,
 	return 0;
 }
 
-static int add_unseen_recent_objects_to_traversal(struct rev_info *revs,
-						  unsigned long timestamp)
+int add_unseen_recent_objects_to_traversal(struct rev_info *revs,
+					   unsigned long timestamp)
 {
 	struct recent_data data;
 	int r;
diff --git a/reachable.h b/reachable.h
index 141fe30..d23efc3 100644
--- a/reachable.h
+++ b/reachable.h
@@ -2,6 +2,8 @@
 #define REACHEABLE_H
 
 struct progress;
+extern int add_unseen_recent_objects_to_traversal(struct rev_info *revs,
+						  unsigned long timestamp);
 extern void mark_reachable_objects(struct rev_info *revs, int mark_reflog,
 				   unsigned long mark_recent, struct progress *);
 
diff --git a/t/t6501-freshen-objects.sh b/t/t6501-freshen-objects.sh
index de941c2..e25c47d 100755
--- a/t/t6501-freshen-objects.sh
+++ b/t/t6501-freshen-objects.sh
@@ -39,50 +39,67 @@ commit () {
 	git commit -m "$1"
 }
 
-test_expect_success 'disable reflogs' '
-	git config core.logallrefupdates false &&
-	rm -rf .git/logs
-'
+maybe_repack () {
+	if test -n "$repack"; then
+		git repack -ad
+	fi
+}
+
+for repack in '' true; do
+	title=${repack:+repack}
+	title=${title:-loose}
+
+	test_expect_success "make repo completely empty ($title)" '
+		rm -rf .git &&
+		git init
+	'
+
+	test_expect_success "disable reflogs ($title)" '
+		git config core.logallrefupdates false &&
+		rm -rf .git/logs
+	'
 
-test_expect_success 'setup basic history' '
-	commit base
-'
+	test_expect_success "setup basic history ($title)" '
+		commit base
+	'
 
-test_expect_success 'create and abandon some objects' '
-	git checkout -b experiment &&
-	commit abandon &&
-	git checkout master &&
-	git branch -D experiment
-'
+	test_expect_success "create and abandon some objects ($title)" '
+		git checkout -b experiment &&
+		commit abandon &&
+		maybe_repack &&
+		git checkout master &&
+		git branch -D experiment
+	'
 
-test_expect_success 'simulate time passing' '
-	find .git/objects -type f |
-	xargs test-chmtime -v -86400
-'
+	test_expect_success "simulate time passing ($title)" '
+		find .git/objects -type f |
+		xargs test-chmtime -v -86400
+	'
 
-test_expect_success 'start writing new commit with old blob' '
-	tree=$(
-		GIT_INDEX_FILE=index.tmp &&
-		export GIT_INDEX_FILE &&
-		git read-tree HEAD &&
-		add unrelated &&
-		add abandon &&
-		git write-tree
-	)
-'
+	test_expect_success "start writing new commit with old blob ($title)" '
+		tree=$(
+			GIT_INDEX_FILE=index.tmp &&
+			export GIT_INDEX_FILE &&
+			git read-tree HEAD &&
+			add unrelated &&
+			add abandon &&
+			git write-tree
+		)
+	'
 
-test_expect_success 'simultaneous gc' '
-	git gc --prune=12.hours.ago
-'
+	test_expect_success "simultaneous gc ($title)" '
+		git gc --prune=12.hours.ago
+	'
 
-test_expect_success 'finish writing out commit' '
-	commit=$(echo foo | git commit-tree -p HEAD $tree) &&
-	git update-ref HEAD $commit
-'
+	test_expect_success "finish writing out commit ($title)" '
+		commit=$(echo foo | git commit-tree -p HEAD $tree) &&
+		git update-ref HEAD $commit
+	'
 
-# "abandon" blob should have been rescued by reference from new tree
-test_expect_success 'repository passes fsck' '
-	git fsck
-'
+	# "abandon" blob should have been rescued by reference from new tree
+	test_expect_success "repository passes fsck ($title)" '
+		git fsck
+	'
+done
 
 test_done

-- 
2.1.1.566.gdb1f904
Previous: Jeff KingNext: Jeff King
Message 48 of 58 in “make prune mtime-checking more careful”
  1. 0/16 make prune mtime-checking more carefulJeff King, Oct 3, 2014
  2. 01/16 foreach_alt_odb: propagate return value from callbackJeff King, Oct 3, 2014
  3. René ScharfeOct 3, 2014
  4. Jeff KingOct 4, 2014
  5. 02/16 isxdigit: cast input to unsigned charJeff King, Oct 3, 2014
  6. 03/16 object_array: factor out slopbuf-freeing logicJeff King, Oct 3, 2014
  7. Michael HaggertyOct 7, 2014
  8. Jeff KingOct 8, 2014
  9. Michael HaggertyOct 8, 2014
  10. Jeff KingOct 8, 2014
  11. 04/16 object_array: add a "clear" functionJeff King, Oct 3, 2014
  12. 05/16 clean up name allocation in prepare_revision_walkJeff King, Oct 3, 2014
  13. 06/16 reachable: clear pending array after walking itJeff King, Oct 3, 2014
  14. 07/16 t5304: use test_path_is_* instead of "test -f"Jeff King, Oct 3, 2014
  15. Junio C HamanoOct 3, 2014
  16. 08/16 t5304: use helper to report failure of "test foo = bar"Jeff King, Oct 3, 2014
  17. Junio C HamanoOct 3, 2014
  18. Jeff KingOct 4, 2014
  19. Michael HaggertyOct 7, 2014
  20. Junio C HamanoOct 7, 2014
  21. Jeff KingOct 7, 2014
  22. Junio C HamanoOct 7, 2014
  23. Junio C HamanoOct 7, 2014
  24. Jeff KingOct 7, 2014
  25. Junio C HamanoOct 7, 2014
  26. Michael HaggertyOct 7, 2014
  27. Jeff KingOct 8, 2014
  28. Junio C HamanoOct 8, 2014
  29. 09/16 prune: factor out loose-object directory traversalJeff King, Oct 3, 2014
  30. Junio C HamanoOct 3, 2014
  31. Jeff KingOct 4, 2014
  32. Michael HaggertyOct 7, 2014
  33. Jeff KingOct 8, 2014
  34. 10/16 count-objects: do not use xsize_t when counting object sizeJeff King, Oct 3, 2014
  35. 11/16 count-objects: use for_each_loose_file_in_objdirJeff King, Oct 3, 2014
  36. 12/16 sha1_file: add for_each iterators for loose and packed objectsJeff King, Oct 3, 2014
  37. René ScharfeOct 5, 2014
  38. Ramsay JonesOct 5, 2014
  39. 13/16 prune: keep objects reachable from recent objectsJeff King, Oct 3, 2014
  40. Junio C HamanoOct 3, 2014
  41. Jeff KingOct 4, 2014
  42. Jeff KingOct 4, 2014
  43. Junio C HamanoOct 4, 2014
  44. Michael HaggertyOct 7, 2014
  45. Jeff KingOct 8, 2014
  46. Michael HaggertyOct 8, 2014
  47. 14/16 pack-objects: refactor unpack-unreachable expiration checkJeff King, Oct 3, 2014
  48. 15/16 pack-objects: match prune logic for discarding objectsJeff King, Oct 3, 2014
  49. 16/16 write_sha1_file: freshen existing objectsJeff King, Oct 3, 2014
  50. Junio C HamanoOct 3, 2014
  51. Jeff KingOct 4, 2014
  52. René ScharfeOct 5, 2014
  53. Junio C HamanoOct 3, 2014
  54. Junio C HamanoOct 4, 2014
  55. René ScharfeOct 5, 2014
  56. Jeff KingOct 6, 2014
  57. Jeff KingOct 8, 2014
  58. Junio C HamanoOct 8, 2014

Read the whole thread, see it on lore, or plain text.

$ cat FOOTERMessages come from the public archive at lore.kernel.org/git, fetched every hour. The front page is chosen and written each morning by an AI editor and can be wrong; the threads themselves are the record. About and API. For agents: an MCP server at https://gitlist.dev/mcp, and any thread, story or person page as Markdown by adding .md to its URL (or sending Accept: text/markdown). Details in /llms.txt.