git/list[1] front-page[2] threads[3] people[4] search[5] about
 

[PATCH 06/20] untracked cache: initial untracked cache validation

From
Nguyễn Thái Ngọc Duy <pclouds@gmail.com>
Date
May 7, 2014, 14:51 UTC
Message-ID
<1399474320-6840-7-git-send-email-pclouds@gmail.com>
In-Reply-To
<1399474320-6840-1-git-send-email-pclouds@gmail.com>
---
 dir.c | 115 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++--
 dir.h |   3 ++
 2 files changed, 115 insertions(+), 3 deletions(-)
diff --git a/dir.c b/dir.c
index 34a10b2..a198aa8 100644
--- a/dir.c
+++ b/dir.c
@@ -568,6 +568,22 @@ static struct untracked_cache_dir *lookup_untracked(struct untracked_cache *uc,
 	return d;
 }
 
+static void do_invalidate_gitignore(struct untracked_cache_dir *dir)
+{
+	int i;
+	dir->valid = 0;
+	dir->untracked_nr = 0;
+	for (i = 0; i < dir->dirs_nr; i++)
+		do_invalidate_gitignore(dir->dirs[i]);
+}
+
+static void invalidate_gitignore(struct untracked_cache *uc,
+				 struct untracked_cache_dir *dir)
+{
+	uc->gitignore_invalidated++;
+	do_invalidate_gitignore(dir);
+}
+
 static int add_excludes(const char *fname,
 			const char *base,
 			int baselen,
@@ -685,6 +701,13 @@ void add_excludes_from_file_1(struct dir_struct *dir, const char *fname,
 			      const unsigned char *ref_sha1)
 {
 	struct exclude_list *el;
+	/*
+	 * catch setup_standard_excludes() that's called before
+	 * dir->untracked is assigned. That function behaves
+	 * differently when dir->untracked is non-NULL.
+	 */
+	if (!dir->untracked)
+		dir->unmanaged_exclude_files++;
 	el = add_exclude_list(dir, EXC_FILE, fname);
 	if (add_excludes(fname, "", 0, el, 0, sha1, ref_stat, ref_sha1) < 0)
 		die("cannot use %s as an exclude file", fname);
@@ -692,6 +715,7 @@ void add_excludes_from_file_1(struct dir_struct *dir, const char *fname,
 
 void add_excludes_from_file(struct dir_struct *dir, const char *fname)
 {
+	dir->unmanaged_exclude_files++; /* see validate_untracked_cache() */
 	add_excludes_from_file_1(dir, fname, NULL, NULL, NULL);
 }
 
@@ -1570,9 +1594,89 @@ static int treat_leading_path(struct dir_struct *dir,
 	return rc;
 }
 
+static struct untracked_cache_dir *validate_untracked_cache(struct dir_struct *dir,
+						      int base_len,
+						      const struct pathspec *pathspec)
+{
+	struct untracked_cache_dir *root;
+
+	if (!dir->untracked)
+		return NULL;
+
+	/*
+	 * We only support $GIT_DIR/info/exclude and core.excludesfile
+	 * as the global ignore rule files. Any other additions
+	 * (e.g. from command line) invalidate the cache. This
+	 * condition also catches running setup_standard_excludes()
+	 * before setting dir->untracked!
+	 */
+	if (dir->unmanaged_exclude_files)
+		return NULL;
+
+	/*
+	 * Optimize for the main use case only: whole-tree git
+	 * status. More work involved in treat_leading_path() if we
+	 * use cache on just a subset of the worktree. pathspec
+	 * support could make the matter even worse.
+	 */
+	if (base_len || (pathspec && pathspec->nr))
+		return NULL;
+
+	/* Different set of flags may produce different results */
+	if (dir->flags != dir->untracked->dir_flags ||
+	    /*
+	     * See treat_directory(), case index_nonexistent. Without
+	     * this flag, we may need to also cache .git file content
+	     * for the resolve_gitlink_ref() call, which we don't.
+	     */
+	    !(dir->flags & DIR_SHOW_OTHER_DIRECTORIES) ||
+	    /* We don't support collecting ignore files */
+	    (dir->flags & (DIR_SHOW_IGNORED | DIR_SHOW_IGNORED_TOO |
+			   DIR_COLLECT_IGNORED)))
+		return NULL;
+
+	/*
+	 * If we use .gitignore in the cache and now you change it to
+	 * .gitexclude, everything will go wrong.
+	 */
+	if (dir->exclude_per_dir != dir->untracked->exclude_per_dir &&
+	    strcmp(dir->exclude_per_dir, dir->untracked->exclude_per_dir))
+		return NULL;
+
+	/*
+	 * EXC_CMDL is not considered in the cache. If people set it,
+	 * skip the cache.
+	 */
+	if (dir->exclude_list_group[EXC_CMDL].nr)
+		return NULL;
+
+	if (!dir->untracked->root) {
+		const int len = sizeof(*dir->untracked->root);
+		dir->untracked->root = xmalloc(len);
+		memset(dir->untracked->root, 0, len);
+	}
+
+	/* Validate $GIT_DIR/info/exclude and core.excludesfile */
+	root = dir->untracked->root;
+	if (hashcmp(dir->info_exclude_sha1,
+		    dir->untracked->info_exclude_sha1)) {
+		invalidate_gitignore(dir->untracked, root);
+		hashcpy(dir->untracked->info_exclude_sha1,
+			dir->info_exclude_sha1);
+	}
+	if (hashcmp(dir->excludes_file_sha1,
+		    dir->untracked->excludes_file_sha1)) {
+		invalidate_gitignore(dir->untracked, root);
+		hashcpy(dir->untracked->excludes_file_sha1,
+			dir->excludes_file_sha1);
+	}
+	return root;
+}
+
 int read_directory(struct dir_struct *dir, const char *path, int len, const struct pathspec *pathspec)
 {
 	struct path_simplify *simplify;
+	struct untracked_cache_dir *untracked;
 
 	/*
 	 * Check out create_simplify()
@@ -1596,10 +1700,15 @@ int read_directory(struct dir_struct *dir, const char *path, int len, const stru
 	 * create_simplify().
 	 */
 	simplify = create_simplify(pathspec ? pathspec->_raw : NULL);
+	untracked = validate_untracked_cache(dir, len, pathspec);
+	if (!untracked)
+		/*
+		 * make sure untracked cache code path is disabled,
+		 * e.g. prep_exclude()
+		 */
+		dir->untracked = NULL;
 	if (!len || treat_leading_path(dir, path, len, simplify))
-		read_directory_recursive(dir, path, len,
-					 dir->untracked ? dir->untracked->root : NULL,
-					 0, simplify);
+		read_directory_recursive(dir, path, len, untracked, 0, simplify);
 	free_simplify(simplify);
 	qsort(dir->entries, dir->nr, sizeof(struct dir_entry *), cmp_name);
 	qsort(dir->ignored, dir->ignored_nr, sizeof(struct dir_entry *), cmp_name);
diff --git a/dir.h b/dir.h
index bce7055..ded251e 100644
--- a/dir.h
+++ b/dir.h
@@ -109,6 +109,7 @@ struct untracked_cache_dir {
 	unsigned char exclude_sha1[20];
 	struct stat_data stat_data;
 	unsigned int check_only : 1;
+	unsigned int valid : 1;
 	unsigned int untracked_nr : 29;
 	unsigned int untracked_alloc, dirs_nr, dirs_alloc;
 	char name[1];
@@ -128,6 +129,7 @@ struct untracked_cache {
 	struct untracked_cache_dir *root;
 	/* Statistics */
 	int dir_created;
+	int gitignore_invalidated;
 };
 
 struct dir_struct {
@@ -180,6 +182,7 @@ struct dir_struct {
 
 	/* Enable untracked file cache if set */
 	struct untracked_cache *untracked;
+	unsigned unmanaged_exclude_files;
 	struct stat_data info_exclude_stat;
 	struct stat_data excludes_file_stat;
 	unsigned char info_exclude_sha1[20];
-- 
1.9.1.346.ga2b5940
Previous: Nguyễn Thái Ngọc DuyNext: Nguyễn Thái Ngọc Duy
Message 7 of 21 in “Untracked cache to speed up "git status"”
  1. 00/20 Untracked cache to speed up "git status"Nguyễn Thái Ngọc Duy, May 7, 2014
  2. 01/20 dir.c: coding style fixNguyễn Thái Ngọc Duy, May 7, 2014
  3. 02/20 dir.h: move struct exclude declaration to top levelNguyễn Thái Ngọc Duy, May 7, 2014
  4. 03/20 prep_exclude: remove the artificial PATH_MAX limitNguyễn Thái Ngọc Duy, May 7, 2014
  5. 04/20 dir.c: optionally compute sha-1 of a .gitignore fileNguyễn Thái Ngọc Duy, May 7, 2014
  6. 05/20 untracked cache: record .gitignore information and dir hierarchyNguyễn Thái Ngọc Duy, May 7, 2014
  7. 06/20 untracked cache: initial untracked cache validationNguyễn Thái Ngọc Duy, May 7, 2014
  8. 07/20 untracked cache: invalidate dirs recursively if .gitignore changesNguyễn Thái Ngọc Duy, May 7, 2014
  9. 08/20 untracked cache: record/validate dir mtime and reuse cached outputNguyễn Thái Ngọc Duy, May 7, 2014
  10. 09/20 untracked cache: mark what dirs should be recursed/savedNguyễn Thái Ngọc Duy, May 7, 2014
  11. 10/20 untracked cache: don't open non-existent .gitignoreNguyễn Thái Ngọc Duy, May 7, 2014
  12. 11/20 untracked cache: save to an index extensionNguyễn Thái Ngọc Duy, May 7, 2014
  13. 12/20 untracked cache: load from UNTR index extensionNguyễn Thái Ngọc Duy, May 7, 2014
  14. 13/20 untracked cache: invalidate at index addition or removalNguyễn Thái Ngọc Duy, May 7, 2014
  15. 14/20 untracked cache: print untracked statistics with $GIT_TRACE_UNTRACKEDNguyễn Thái Ngọc Duy, May 7, 2014
  16. 15/20 read-cache.c: split racy stat test to a separate functionNguyễn Thái Ngọc Duy, May 7, 2014
  17. 16/20 untracked cache: avoid racy timestampsNguyễn Thái Ngọc Duy, May 7, 2014
  18. 17/20 status: support untracked cacheNguyễn Thái Ngọc Duy, May 7, 2014
  19. 18/20 update-index: manually enable or disable untracked cacheNguyễn Thái Ngọc Duy, May 7, 2014
  20. 19/20 update-index: test the system before enabling untracked cacheNguyễn Thái Ngọc Duy, May 7, 2014
  21. 20/20 t7063: tests for untracked cacheNguyễn Thái Ngọc Duy, May 7, 2014

Read the whole thread, see it on lore, or plain text.

$ cat FOOTERMessages come from the public archive at lore.kernel.org/git, fetched every hour. The front page is chosen and written each morning by an AI editor and can be wrong; the threads themselves are the record. About and API. For agents: an MCP server at https://gitlist.dev/mcp, and any thread, story or person page as Markdown by adding .md to its URL (or sending Accept: text/markdown). Details in /llms.txt.