git/list[1] front-page[2] threads[3] people[4] search[5] about
 

[RFC PATCH v3 7/8] Support sparse checkout in unpack_trees() and read-tree

From
Nguyễn Thái Ngọc Duy <pclouds@gmail.com>
Date
Aug 11, 2009, 15:44 UTC
Message-ID
<1250005446-12047-8-git-send-email-pclouds@gmail.com>
In-Reply-To
<1250005446-12047-7-git-send-email-pclouds@gmail.com>

This patch makes unpack_trees() look at .git/info/sparse [1] to determine which files should stay in working directory, after merging, by:

 - setting CE_VALID properly so that other operations correctly ignore
   missing files
 - driving check_updates() to add/remove files in accordance to
   CE_VALID
The feature is disabled by default. Use "read-tree --sparse" to enable it.

[1] .git/info/sparse has the same syntax as .git/info/exclude. Files that match the patterns will be set as CE_VALID.

Signed-off-by: Nguyễn Thái Ngọc Duy <pclouds@gmail.com>
---
 builtin-read-tree.c         |    4 +-
 cache.h                     |    3 +
 t/t1009-read-tree-sparse.sh |   47 ++++++++++++++++++++
 unpack-trees.c              |   98 ++++++++++++++++++++++++++++++++++++++++++-
 unpack-trees.h              |    3 +
 5 files changed, 153 insertions(+), 2 deletions(-)
 create mode 100755 t/t1009-read-tree-sparse.sh
diff --git a/builtin-read-tree.c b/builtin-read-tree.c
index 9c2d634..888f136 100644
--- a/builtin-read-tree.c
+++ b/builtin-read-tree.c
@@ -31,7 +31,7 @@ static int list_tree(unsigned char *sha1)
 }
 
 static const char * const read_tree_usage[] = {
-	"git read-tree [[-m [--trivial] [--aggressive] | --reset | --prefix=<prefix>] [-u [--exclude-per-directory=<gitignore>] | -i]]  [--index-output=<file>] <tree-ish1> [<tree-ish2> [<tree-ish3>]]",
+	"git read-tree [[-m [--trivial] [--aggressive] | --reset | --prefix=<prefix>] [-u [--exclude-per-directory=<gitignore>] | -i]] [--sparse] [--index-output=<file>] <tree-ish1> [<tree-ish2> [<tree-ish3>]]",
 	NULL
 };
 
@@ -98,6 +98,8 @@ int cmd_read_tree(int argc, const char **argv, const char *unused_prefix)
 		  PARSE_OPT_NONEG, exclude_per_directory_cb },
 		OPT_SET_INT('i', NULL, &opts.index_only,
 			    "don't check the working tree after merging", 1),
+		OPT_SET_INT(0, "sparse", &opts.apply_sparse,
+			    "apply sparse checkout filter", 1),
 		OPT_END()
 	};
 
diff --git a/cache.h b/cache.h
index 1a2a3c9..dfad54a 100644
--- a/cache.h
+++ b/cache.h
@@ -177,6 +177,9 @@ struct cache_entry {
 #define CE_HASHED    (0x100000)
 #define CE_UNHASHED  (0x200000)
 
+/* Only remove in work directory, not index */
+#define CE_WT_REMOVE (0x400000)
+
 /*
  * Extended on-disk flags
  */
diff --git a/t/t1009-read-tree-sparse.sh b/t/t1009-read-tree-sparse.sh
new file mode 100755
index 0000000..f70852c
--- /dev/null
+++ b/t/t1009-read-tree-sparse.sh
@@ -0,0 +1,47 @@
+#!/bin/sh
+
+test_description='sparse checkout tests'
+
+. ./test-lib.sh
+
+test_expect_success 'setup' '
+	test_commit one &&
+	mkdir two &&
+	test_commit two two/two.t two.t
+'
+
+test_expect_success 'read-tree without .git/info/sparse' '
+	git read-tree --sparse -m -u HEAD &&
+	test -f one.t &&
+	test -f two/two.t
+'
+
+test_expect_success 'read-tree with empty .git/info/sparse' '
+	echo > .git/info/sparse &&
+	git read-tree --sparse -m -u HEAD &&
+	test -f one.t &&
+	test -f two/two.t
+'
+
+test_expect_success 'read-tree --sparse' '
+	echo "one.t" > .git/info/sparse &&
+	git read-tree --sparse -m -u HEAD &&
+	test ! -f one.t &&
+	test -f two/two.t
+'
+
+test_expect_success 'read-tree --sparse foo where foo is "directory"' '
+	echo "two" > .git/info/sparse &&
+	git read-tree --sparse -m -u HEAD &&
+	test -f one.t &&
+	test -f two/two.t
+'
+
+test_expect_success 'read-tree --sparse foo/' '
+	echo "two/" > .git/info/sparse &&
+	git read-tree --sparse -m -u HEAD &&
+	test -f one.t &&
+	test ! -f two/two.t
+'
+
+test_done
diff --git a/unpack-trees.c b/unpack-trees.c
index 02ea236..d18d333 100644
--- a/unpack-trees.c
+++ b/unpack-trees.c
@@ -32,6 +32,12 @@ static struct unpack_trees_error_msgs unpack_plumbing_errors = {
 
 	/* bind_overlap */
 	"Entry '%s' overlaps with '%s'.  Cannot bind.",
+
+	/* sparse_not_uptodate_file */
+	"Entry '%s' not uptodate. Cannot update sparse checkout.",
+
+	/* would_lose_orphaned */
+	"Working tree file '%s' would be %s by sparse checkout update.",
 };
 
 #define ERRORMSG(o,fld) \
@@ -78,7 +84,7 @@ static int check_updates(struct unpack_trees_options *o)
 	if (o->update && o->verbose_update) {
 		for (total = cnt = 0; cnt < index->cache_nr; cnt++) {
 			struct cache_entry *ce = index->cache[cnt];
-			if (ce->ce_flags & (CE_UPDATE | CE_REMOVE))
+			if (ce->ce_flags & (CE_UPDATE | CE_REMOVE | CE_WT_REMOVE))
 				total++;
 		}
 
@@ -92,6 +98,13 @@ static int check_updates(struct unpack_trees_options *o)
 	for (i = 0; i < index->cache_nr; i++) {
 		struct cache_entry *ce = index->cache[i];
 
+		if (ce->ce_flags & CE_WT_REMOVE) {
+			display_progress(progress, ++cnt);
+			if (o->update)
+				unlink_entry(ce);
+			continue;
+		}
+
 		if (ce->ce_flags & CE_REMOVE) {
 			display_progress(progress, ++cnt);
 			if (o->update)
@@ -118,6 +131,74 @@ static int check_updates(struct unpack_trees_options *o)
 	return errs != 0;
 }
 
+static int verify_uptodate_sparse(struct cache_entry *ce, struct unpack_trees_options *o);
+static int verify_absent_sparse(struct cache_entry *ce, const char *action, struct unpack_trees_options *o);
+static int apply_sparse_checkout(struct unpack_trees_options *o)
+{
+	struct index_state *index = &o->result;
+	struct exclude_list el;
+	int i, ret = 0;
+
+	memset(&el, 0, sizeof(el));
+	if (add_excludes_from_file_to_list(git_path("info/sparse"), "", 0, NULL, &el, 0) < 0)
+		return 0;
+
+	for (i = 0; i < index->cache_nr; i++) {
+		struct cache_entry *ce = index->cache[i];
+		const char *basename;
+		int was_valid = ce->ce_flags & CE_VALID;
+
+		if (ce_stage(ce))
+			continue;
+
+		basename = strrchr(ce->name, '/');
+		basename = basename ? basename+1 : ce->name;
+		if (excluded_from_list(ce->name, ce_namelen(ce), basename, NULL, &el) > 0)
+			ce->ce_flags |= CE_VALID;
+		else
+			ce->ce_flags &= ~CE_VALID;
+
+		/*
+		 * We only care about files getting into the checkout area
+		 * If merge strategies want to remove some, go ahead
+		 */
+		if (ce->ce_flags & CE_REMOVE)
+			continue;
+
+		if (!was_valid && (ce->ce_flags & CE_VALID)) {
+			/*
+			 * If CE_UPDATE is set, verify_uptodate() must be called already
+			 * also stat info may have lost after merged_entry() so calling
+			 * verify_uptodate() again may fail
+			 */
+			if (!(ce->ce_flags & CE_UPDATE) && verify_uptodate_sparse(ce, o)) {
+				ret = -1;
+				break;
+			}
+			ce->ce_flags |= CE_WT_REMOVE;
+		}
+		if (was_valid && !(ce->ce_flags & CE_VALID)) {
+			if (verify_absent_sparse(ce, "overwritten", o)) {
+				ret = -1;
+				break;
+			}
+			ce->ce_flags |= CE_UPDATE;
+		}
+
+		/* merge strategies may set CE_UPDATE outside checkout area */
+		if (ce->ce_flags & CE_VALID)
+			ce->ce_flags &= ~CE_UPDATE;
+
+	}
+
+	for (i = 0;i < el.nr;i++)
+		free(el.excludes[i]);
+	if (el.excludes)
+		free(el.excludes);
+
+	return ret;
+}
+
 static inline int call_unpack_fn(struct cache_entry **src, struct unpack_trees_options *o)
 {
 	int ret = o->fn(src, o);
@@ -416,6 +497,9 @@ int unpack_trees(unsigned len, struct tree_desc *t, struct unpack_trees_options
 	if (o->trivial_merges_only && o->nontrivial_merge)
 		return unpack_failed(o, "Merge requires file-level merging");
 
+	if (o->apply_sparse && apply_sparse_checkout(o))
+		return unpack_failed(o, NULL);
+
 	o->src_index = NULL;
 	ret = check_updates(o) ? (-2) : 0;
 	if (o->dst_index)
@@ -481,6 +565,12 @@ static int verify_uptodate(struct cache_entry *ce,
 	return verify_uptodate_1(ce, o, ERRORMSG(o, not_uptodate_file));
 }
 
+static int verify_uptodate_sparse(struct cache_entry *ce,
+				  struct unpack_trees_options *o)
+{
+	return verify_uptodate_1(ce, o, ERRORMSG(o, sparse_not_uptodate_file));
+}
+
 static void invalidate_ce_path(struct cache_entry *ce, struct unpack_trees_options *o)
 {
 	if (ce)
@@ -674,6 +764,12 @@ static int verify_absent(struct cache_entry *ce, const char *action,
 	return verify_absent_1(ce, action, o, ERRORMSG(o, would_lose_untracked));
 }
 
+static int verify_absent_sparse(struct cache_entry *ce, const char *action,
+			 struct unpack_trees_options *o)
+{
+	return verify_absent_1(ce, action, o, ERRORMSG(o, would_lose_orphaned));
+}
+
 static int merged_entry(struct cache_entry *merge, struct cache_entry *old,
 		struct unpack_trees_options *o)
 {
diff --git a/unpack-trees.h b/unpack-trees.h
index d19df44..a09077b 100644
--- a/unpack-trees.h
+++ b/unpack-trees.h
@@ -14,6 +14,8 @@ struct unpack_trees_error_msgs {
 	const char *not_uptodate_dir;
 	const char *would_lose_untracked;
 	const char *bind_overlap;
+	const char *sparse_not_uptodate_file;
+	const char *would_lose_orphaned;
 };
 
 struct unpack_trees_options {
@@ -28,6 +30,7 @@ struct unpack_trees_options {
 		     skip_unmerged,
 		     initial_checkout,
 		     diff_index_cached,
+		     apply_sparse,
 		     gently;
 	const char *prefix;
 	int pos;

-- 
1.6.3.GIT
Previous: Nguyễn Thái Ngọc DuyNext: Nguyễn Thái Ngọc Duy
Message 8 of 53 in “Sparse checkout”
  1. 0/8 Sparse checkoutNguyễn Thái Ngọc Duy, Aug 11, 2009
  2. 1/8 Prevent diff machinery from examining assume-unchanged entries on worktreeNguyễn Thái Ngọc Duy, Aug 11, 2009
  3. 2/8 Avoid writing to buffer in add_excludes_from_file_1()Nguyễn Thái Ngọc Duy, Aug 11, 2009
  4. 3/8 Read .gitignore from index if it is assume-unchangedNguyễn Thái Ngọc Duy, Aug 11, 2009
  5. 4/8 excluded_1(): support exclude "directories" in indexNguyễn Thái Ngọc Duy, Aug 11, 2009
  6. 5/8 dir.c: export excluded_1() and add_excludes_from_file_1()Nguyễn Thái Ngọc Duy, Aug 11, 2009
  7. 6/8 unpack-trees.c: generalize verify_* functionsNguyễn Thái Ngọc Duy, Aug 11, 2009
  8. 7/8 Support sparse checkout in unpack_trees() and read-treeNguyễn Thái Ngọc Duy, Aug 11, 2009
  9. 8/8 --sparse for porcelainsNguyễn Thái Ngọc Duy, Aug 11, 2009
  10. Junio C HamanoAug 12, 2009
  11. Nguyen Thai Ngoc DuyAug 12, 2009
  12. Nguyen Thai Ngoc DuyAug 13, 2009
  13. Jakub NarebskiAug 13, 2009
  14. Nguyen Thai Ngoc DuyAug 13, 2009
  15. Jakub NarebskiAug 14, 2009
  16. Junio C HamanoAug 15, 2009
  17. Jakub NarebskiAug 15, 2009
  18. Johannes SchindelinAug 16, 2009
  19. Johannes SchindelinAug 17, 2009
  20. Nguyen Thai Ngoc DuyAug 17, 2009
  21. Johannes SchindelinAug 17, 2009
  22. Nguyen Thai Ngoc DuyAug 17, 2009
  23. Johannes SchindelinAug 17, 2009
  24. Nguyen Thai Ngoc DuyAug 17, 2009
  25. Junio C HamanoAug 17, 2009
  26. Nguyen Thai Ngoc DuyAug 17, 2009
  27. Johannes SchindelinAug 17, 2009
  28. Junio C HamanoAug 17, 2009
  29. Johannes SchindelinAug 17, 2009
  30. skillzero@gmail.comAug 17, 2009
  31. Johannes SchindelinAug 17, 2009
  32. Jakub NarebskiAug 18, 2009
  33. skillzero@gmail.comAug 18, 2009
  34. Nguyen Thai Ngoc DuyAug 18, 2009
  35. git find (was: [RFC PATCH v3 8/8] --sparse for porcelains)Jakub Narebski, Aug 18, 2009
  36. Nguyen Thai Ngoc DuyAug 18, 2009
  37. Jakub NarebskiAug 18, 2009
  38. Jakub NarebskiAug 18, 2009
  39. skillzero@gmail.comAug 18, 2009
  40. Junio C HamanoAug 17, 2009
  41. Johannes SchindelinAug 17, 2009
  42. Jakub NarebskiAug 17, 2009
  43. Johannes SixtAug 12, 2009
  44. Nguyen Thai Ngoc DuyAug 12, 2009
  45. Raja R HarinathAug 12, 2009
  46. Johannes SixtAug 13, 2009
  47. skillzero@gmail.comAug 11, 2009
  48. Jakub NarebskiAug 11, 2009
  49. skillzero@gmail.comAug 11, 2009
  50. Nguyen Thai Ngoc DuyAug 12, 2009
  51. skillzero@gmail.comAug 12, 2009
  52. Junio C HamanoAug 12, 2009
  53. Nguyen Thai Ngoc DuyAug 13, 2009

Read the whole thread, see it on lore, or plain text.

$ cat FOOTERMessages come from the public archive at lore.kernel.org/git, fetched every hour. The front page is chosen and written each morning by an AI editor and can be wrong; the threads themselves are the record. About and API. For agents: an MCP server at https://gitlist.dev/mcp, and any thread, story or person page as Markdown by adding .md to its URL (or sending Accept: text/markdown). Details in /llms.txt.