git/list[1] front-page[2] threads[3] people[4] search[5] about
 

[PATCH v3 03/23] untracked cache: initial untracked cache validation

From
Nguyễn Thái Ngọc Duy <pclouds@gmail.com>
Date
Dec 8, 2014, 14:04 UTC
Message-ID
<1418047507-22892-5-git-send-email-pclouds@gmail.com>
In-Reply-To
<1418047507-22892-1-git-send-email-pclouds@gmail.com>

Make sure the starting conditions and all global exclude files are good to go. If not, either disable untracked cache completely, or wipe out the cache and start fresh.

Signed-off-by: Nguyễn Thái Ngọc Duy <pclouds@gmail.com>
---
 dir.c | 113 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++--
 dir.h |   4 +++
 2 files changed, 114 insertions(+), 3 deletions(-)
diff --git a/dir.c b/dir.c
index 27734f0..44ed9f2 100644
--- a/dir.c
+++ b/dir.c
@@ -582,6 +582,22 @@ static struct untracked_cache_dir *lookup_untracked(struct untracked_cache *uc,
 	return d;
 }
 
+static void do_invalidate_gitignore(struct untracked_cache_dir *dir)
+{
+	int i;
+	dir->valid = 0;
+	dir->untracked_nr = 0;
+	for (i = 0; i < dir->dirs_nr; i++)
+		do_invalidate_gitignore(dir->dirs[i]);
+}
+
+static void invalidate_gitignore(struct untracked_cache *uc,
+				 struct untracked_cache_dir *dir)
+{
+	uc->gitignore_invalidated++;
+	do_invalidate_gitignore(dir);
+}
+
 /*
  * Given a file with name "fname", read it (either from disk, or from
  * the index if "check_index" is non-zero), parse it and store the
@@ -697,6 +713,13 @@ static void add_excludes_from_file_1(struct dir_struct *dir, const char *fname,
 				     struct sha1_stat *sha1_stat)
 {
 	struct exclude_list *el;
+	/*
+	 * catch setup_standard_excludes() that's called before
+	 * dir->untracked is assigned. That function behaves
+	 * differently when dir->untracked is non-NULL.
+	 */
+	if (!dir->untracked)
+		dir->unmanaged_exclude_files++;
 	el = add_exclude_list(dir, EXC_FILE, fname);
 	if (add_excludes(fname, "", 0, el, 0, sha1_stat) < 0)
 		die("cannot use %s as an exclude file", fname);
@@ -704,6 +727,7 @@ static void add_excludes_from_file_1(struct dir_struct *dir, const char *fname,
 
 void add_excludes_from_file(struct dir_struct *dir, const char *fname)
 {
+	dir->unmanaged_exclude_files++; /* see validate_untracked_cache() */
 	add_excludes_from_file_1(dir, fname, NULL);
 }
 
@@ -1572,9 +1596,87 @@ static int treat_leading_path(struct dir_struct *dir,
 	return rc;
 }
 
+static struct untracked_cache_dir *validate_untracked_cache(struct dir_struct *dir,
+						      int base_len,
+						      const struct pathspec *pathspec)
+{
+	struct untracked_cache_dir *root;
+
+	if (!dir->untracked)
+		return NULL;
+
+	/*
+	 * We only support $GIT_DIR/info/exclude and core.excludesfile
+	 * as the global ignore rule files. Any other additions
+	 * (e.g. from command line) invalidate the cache. This
+	 * condition also catches running setup_standard_excludes()
+	 * before setting dir->untracked!
+	 */
+	if (dir->unmanaged_exclude_files)
+		return NULL;
+
+	/*
+	 * Optimize for the main use case only: whole-tree git
+	 * status. More work involved in treat_leading_path() if we
+	 * use cache on just a subset of the worktree. pathspec
+	 * support could make the matter even worse.
+	 */
+	if (base_len || (pathspec && pathspec->nr))
+		return NULL;
+
+	/* Different set of flags may produce different results */
+	if (dir->flags != dir->untracked->dir_flags ||
+	    /*
+	     * See treat_directory(), case index_nonexistent. Without
+	     * this flag, we may need to also cache .git file content
+	     * for the resolve_gitlink_ref() call, which we don't.
+	     */
+	    !(dir->flags & DIR_SHOW_OTHER_DIRECTORIES) ||
+	    /* We don't support collecting ignore files */
+	    (dir->flags & (DIR_SHOW_IGNORED | DIR_SHOW_IGNORED_TOO |
+			   DIR_COLLECT_IGNORED)))
+		return NULL;
+
+	/*
+	 * If we use .gitignore in the cache and now you change it to
+	 * .gitexclude, everything will go wrong.
+	 */
+	if (dir->exclude_per_dir != dir->untracked->exclude_per_dir &&
+	    strcmp(dir->exclude_per_dir, dir->untracked->exclude_per_dir))
+		return NULL;
+
+	/*
+	 * EXC_CMDL is not considered in the cache. If people set it,
+	 * skip the cache.
+	 */
+	if (dir->exclude_list_group[EXC_CMDL].nr)
+		return NULL;
+
+	if (!dir->untracked->root) {
+		const int len = sizeof(*dir->untracked->root);
+		dir->untracked->root = xmalloc(len);
+		memset(dir->untracked->root, 0, len);
+	}
+
+	/* Validate $GIT_DIR/info/exclude and core.excludesfile */
+	root = dir->untracked->root;
+	if (hashcmp(dir->ss_info_exclude.sha1,
+		    dir->untracked->ss_info_exclude.sha1)) {
+		invalidate_gitignore(dir->untracked, root);
+		dir->untracked->ss_info_exclude = dir->ss_info_exclude;
+	}
+	if (hashcmp(dir->ss_excludes_file.sha1,
+		    dir->untracked->ss_excludes_file.sha1)) {
+		invalidate_gitignore(dir->untracked, root);
+		dir->untracked->ss_excludes_file = dir->ss_excludes_file;
+	}
+	return root;
+}
+
 int read_directory(struct dir_struct *dir, const char *path, int len, const struct pathspec *pathspec)
 {
 	struct path_simplify *simplify;
+	struct untracked_cache_dir *untracked;
 
 	/*
 	 * Check out create_simplify()
@@ -1598,10 +1700,15 @@ int read_directory(struct dir_struct *dir, const char *path, int len, const stru
 	 * create_simplify().
 	 */
 	simplify = create_simplify(pathspec ? pathspec->_raw : NULL);
+	untracked = validate_untracked_cache(dir, len, pathspec);
+	if (!untracked)
+		/*
+		 * make sure untracked cache code path is disabled,
+		 * e.g. prep_exclude()
+		 */
+		dir->untracked = NULL;
 	if (!len || treat_leading_path(dir, path, len, simplify))
-		read_directory_recursive(dir, path, len,
-					 dir->untracked ? dir->untracked->root : NULL,
-					 0, simplify);
+		read_directory_recursive(dir, path, len, untracked, 0, simplify);
 	free_simplify(simplify);
 	qsort(dir->entries, dir->nr, sizeof(struct dir_entry *), cmp_name);
 	qsort(dir->ignored, dir->ignored_nr, sizeof(struct dir_entry *), cmp_name);
diff --git a/dir.h b/dir.h
index 9ab74b4..1d7a958 100644
--- a/dir.h
+++ b/dir.h
@@ -115,6 +115,8 @@ struct untracked_cache_dir {
 	unsigned int untracked_alloc, dirs_nr, dirs_alloc;
 	unsigned int untracked_nr;
 	unsigned int check_only : 1;
+	/* all data in this struct are good */
+	unsigned int valid : 1;
 	/* null SHA-1 means this directory does not have .gitignore */
 	unsigned char exclude_sha1[20];
 	char name[FLEX_ARRAY];
@@ -132,6 +134,7 @@ struct untracked_cache {
 	struct untracked_cache_dir *root;
 	/* Statistics */
 	int dir_created;
+	int gitignore_invalidated;
 };
 
 struct dir_struct {
@@ -186,6 +189,7 @@ struct dir_struct {
 	struct untracked_cache *untracked;
 	struct sha1_stat ss_info_exclude;
 	struct sha1_stat ss_excludes_file;
+	unsigned unmanaged_exclude_files;
 };
 
 /*

-- 
2.2.0.60.gb7b3c64
Previous: Nguyễn Thái Ngọc DuyNext: Nguyễn Thái Ngọc Duy
Message 6 of 35 in “nd/untracked-cache updates”
  1. 00/23 nd/untracked-cache updatesNguyễn Thái Ngọc Duy, Dec 8, 2014
  2. checkout: add --ignore-other-wortreesNguyễn Thái Ngọc Duy, Dec 8, 2014
  3. Duy NguyenDec 8, 2014
  4. 01/23 dir.c: optionally compute sha-1 of a .gitignore fileNguyễn Thái Ngọc Duy, Dec 8, 2014
  5. 02/23 untracked cache: record .gitignore information and dir hierarchyNguyễn Thái Ngọc Duy, Dec 8, 2014
  6. 03/23 untracked cache: initial untracked cache validationNguyễn Thái Ngọc Duy, Dec 8, 2014
  7. 04/23 untracked cache: invalidate dirs recursively if .gitignore changesNguyễn Thái Ngọc Duy, Dec 8, 2014
  8. 05/23 untracked cache: make a wrapper around {open,read,close}dir()Nguyễn Thái Ngọc Duy, Dec 8, 2014
  9. 06/23 untracked cache: record/validate dir mtime and reuse cached outputNguyễn Thái Ngọc Duy, Dec 8, 2014
  10. 07/23 untracked cache: mark what dirs should be recursed/savedNguyễn Thái Ngọc Duy, Dec 8, 2014
  11. 08/23 untracked cache: don't open non-existent .gitignoreNguyễn Thái Ngọc Duy, Dec 8, 2014
  12. 09/23 ewah: add convenient wrapper ewah_serialize_strbuf()Nguyễn Thái Ngọc Duy, Dec 8, 2014
  13. 10/23 untracked cache: save to an index extensionNguyễn Thái Ngọc Duy, Dec 8, 2014
  14. Eric SunshineDec 9, 2014
  15. Duy NguyenDec 10, 2014
  16. 11/23 untracked cache: load from UNTR index extensionNguyễn Thái Ngọc Duy, Dec 8, 2014
  17. Junio C HamanoDec 15, 2014
  18. 12/23 untracked cache: invalidate at index addition or removalNguyễn Thái Ngọc Duy, Dec 8, 2014
  19. 13/23 read-cache.c: split racy stat test to a separate functionNguyễn Thái Ngọc Duy, Dec 8, 2014
  20. 14/23 untracked cache: avoid racy timestampsNguyễn Thái Ngọc Duy, Dec 8, 2014
  21. 15/23 untracked cache: print stats with $GIT_TRACE_UNTRACKED_STATSNguyễn Thái Ngọc Duy, Dec 8, 2014
  22. 16/23 untracked cache: mark index dirty if untracked cache is updatedNguyễn Thái Ngọc Duy, Dec 8, 2014
  23. 17/23 untracked-cache: temporarily disable with $GIT_DISABLE_UNTRACKED_CACHENguyễn Thái Ngọc Duy, Dec 8, 2014
  24. 18/23 status: enable untracked cacheNguyễn Thái Ngọc Duy, Dec 8, 2014
  25. 19/23 update-index: manually enable or disable untracked cacheNguyễn Thái Ngọc Duy, Dec 8, 2014
  26. 20/23 update-index: test the system before enabling untracked cacheNguyễn Thái Ngọc Duy, Dec 8, 2014
  27. 21/23 t7063: tests for untracked cacheNguyễn Thái Ngọc Duy, Dec 8, 2014
  28. 22/23 mingw32: add uname()Nguyễn Thái Ngọc Duy, Dec 8, 2014
  29. 23/23 untracked cache: guard and disable on system changesNguyễn Thái Ngọc Duy, Dec 8, 2014
  30. brian m. carlsonDec 9, 2014
  31. Duy NguyenDec 9, 2014
  32. Torsten BögershausenDec 10, 2014
  33. Duy NguyenDec 10, 2014
  34. Torsten BögershausenDec 11, 2014
  35. Duy NguyenDec 15, 2014

Read the whole thread, see it on lore, or plain text.

$ cat FOOTERMessages come from the public archive at lore.kernel.org/git, fetched every hour. The front page is chosen and written each morning by an AI editor and can be wrong; the threads themselves are the record. About and API. For agents: an MCP server at https://gitlist.dev/mcp, and any thread, story or person page as Markdown by adding .md to its URL (or sending Accept: text/markdown). Details in /llms.txt.