git/list[1] front-page[2] threads[3] people[4] search[5] about
 

[PATCH v3] index: add trace2 region for clear skip worktree

From
Anh Le via GitGitGadget <gitgitgadget@gmail.com>
Date
Oct 31, 2022, 00:56 UTC
Message-ID
<pull.1368.v3.git.git.1667177791591.gitgitgadget@gmail.com>
In-Reply-To
<pull.1368.v2.git.git.1666917961644.gitgitgadget@gmail.com>
From: Anh Le <anh@canva.com>

In a large repository using sparse checkout, checking whether a file marked with skip worktree is present on disk and its skip worktree bit should be cleared can take a considerable amount of time. Add a trace2 region to surface this information, keeping a count of how many paths have been checked and separately keep counts for after a full index is materialised.

Signed-off-by: Anh Le <anh@canva.com>
---
    index: add trace2 region for clear skip worktree
    
    In large repository using sparse checkout, checking whether a file
    marked with skip worktree is present on disk and its skip worktree bit
    should be cleared can take a considerable amount of time. Add a trace2
    region to surface this information.
Published-As: https://github.com/gitgitgadget/git/releases/tag/pr-git-1368%2FHaizzz%2Fmaster-v3
Fetch-It-Via: git fetch https://github.com/gitgitgadget/git pr-git-1368/Haizzz/master-v3
Pull-Request: https://github.com/git/git/pull/1368
Range-diff vs v2:
 1:  effe6b5b912 ! 1:  d0d9f258b08 index: add trace2 region for clear skip worktree
     @@ Metadata
       ## Commit message ##
          index: add trace2 region for clear skip worktree
      
     -    In a large repository using sparse checkout, checking
     -    whether a file marked with skip worktree is present
     -    on disk and its skip worktree bit should be cleared
     -    can take a considerable amount of time. Add a trace2
     -    region to surface this information, keeping a count of how many paths
     -    have been checked and separately
     -    keep counts for after a full index is
     -    materialised.
     +    In a large repository using sparse checkout, checking whether a file marked
     +    with skip worktree is present on disk and its skip worktree bit should be
     +    cleared can take a considerable amount of time. Add a trace2 region to
     +    surface this information, keeping a count of how many paths have been
     +    checked and separately keep counts for after a full index is materialised.
      
          Signed-off-by: Anh Le <anh@canva.com>
      
     @@ sparse-index.c: void clear_skip_worktree_from_present_files(struct index_state *
       	int dir_found = 1;
       
       	int i;
     -+	int path_counts[2] = {0, 0};
     ++	int path_count[2] = {0, 0};
      +	int restarted = 0;
       
       	if (!core_apply_sparse_checkout ||
       	    sparse_expect_files_outside_of_patterns)
       		return;
       
     -+	trace2_region_enter("index", "clear_skip_worktree_from_present_files", istate->repo);
     ++	trace2_region_enter("index", "clear_skip_worktree_from_present_files",
     ++											istate->repo);
       restart:
       	for (i = 0; i < istate->cache_nr; i++) {
       		struct cache_entry *ce = istate->cache[i];
     @@ sparse-index.c: void clear_skip_worktree_from_present_files(struct index_state *
      -				ensure_full_index(istate);
      -				goto restart;
      +		if (ce_skip_worktree(ce)) {
     -+			path_counts[restarted]++;
     ++			path_count[restarted]++;
      +			if (path_found(ce->name, &last_dirname, &dir_len, &dir_found)) {
      +				if (S_ISSPARSEDIR(ce->ce_mode)) {
     ++					if (restarted)
     ++						BUG("ensure-full-index did not fully flatten?");
      +					ensure_full_index(istate);
      +					restarted = 1;
      +					goto restart;
     @@ sparse-index.c: void clear_skip_worktree_from_present_files(struct index_state *
       		}
       	}
      +
     -+	if (path_counts[0] > 0) {
     -+		trace2_data_intmax("index", istate->repo, "clear_skip_worktree_from_present_files/path_count", path_counts[0]);
     -+	}
     -+	if (restarted) {
     -+		trace2_data_intmax("index", istate->repo, "clear_skip_worktree_from_present_files/full_index/path_count", path_counts[1]);
     -+	}
     -+	trace2_region_leave("index", "clear_skip_worktree_from_present_files", istate->repo);
     ++	if (path_count[0])
     ++		trace2_data_intmax("index", istate->repo,
     ++											 "sparse_path_count", path_count[0]);
     ++	if (restarted)
     ++		trace2_data_intmax("index", istate->repo,
     ++											 "sparse_path_count_full", path_count[1]);
     ++	trace2_region_leave("index", "clear_skip_worktree_from_present_files",
     ++										  istate->repo);
       }
       
       /*
 sparse-index.c | 30 ++++++++++++++++++++++++------
 1 file changed, 24 insertions(+), 6 deletions(-)
diff --git a/sparse-index.c b/sparse-index.c
index e4a54ce1943..8301129bf8f 100644
--- a/sparse-index.c
+++ b/sparse-index.c
@@ -493,24 +493,42 @@ void clear_skip_worktree_from_present_files(struct index_state *istate)
 	int dir_found = 1;
 
 	int i;
+	int path_count[2] = {0, 0};
+	int restarted = 0;
 
 	if (!core_apply_sparse_checkout ||
 	    sparse_expect_files_outside_of_patterns)
 		return;
 
+	trace2_region_enter("index", "clear_skip_worktree_from_present_files",
+											istate->repo);
 restart:
 	for (i = 0; i < istate->cache_nr; i++) {
 		struct cache_entry *ce = istate->cache[i];
 
-		if (ce_skip_worktree(ce) &&
-		    path_found(ce->name, &last_dirname, &dir_len, &dir_found)) {
-			if (S_ISSPARSEDIR(ce->ce_mode)) {
-				ensure_full_index(istate);
-				goto restart;
+		if (ce_skip_worktree(ce)) {
+			path_count[restarted]++;
+			if (path_found(ce->name, &last_dirname, &dir_len, &dir_found)) {
+				if (S_ISSPARSEDIR(ce->ce_mode)) {
+					if (restarted)
+						BUG("ensure-full-index did not fully flatten?");
+					ensure_full_index(istate);
+					restarted = 1;
+					goto restart;
+				}
+				ce->ce_flags &= ~CE_SKIP_WORKTREE;
 			}
-			ce->ce_flags &= ~CE_SKIP_WORKTREE;
 		}
 	}
+
+	if (path_count[0])
+		trace2_data_intmax("index", istate->repo,
+											 "sparse_path_count", path_count[0]);
+	if (restarted)
+		trace2_data_intmax("index", istate->repo,
+											 "sparse_path_count_full", path_count[1]);
+	trace2_region_leave("index", "clear_skip_worktree_from_present_files",
+										  istate->repo);
 }
 
 /*

base-commit: 1fc3c0ad407008c2f71dd9ae1241d8b75f8ef886
-- 
gitgitgadget
Previous: Jeff HostetlerNext: Taylor Blau
Message 12 of 18 in “index: add trace2 region for clear skip worktree”
  1. index: add trace2 region for clear skip worktreeAnh Le via GitGitGadget, Oct 26, 2022
  2. Junio C HamanoOct 26, 2022
  3. Jeff HostetlerOct 26, 2022
  4. Junio C HamanoOct 26, 2022
  5. Jeff HostetlerOct 26, 2022
  6. Anh LeOct 27, 2022
  7. index: add trace2 region for clear skip worktreeAnh Le via GitGitGadget, Oct 28, 2022
  8. Derrick StoleeOct 28, 2022
  9. Junio C HamanoOct 28, 2022
  10. Anh LeOct 30, 2022
  11. Jeff HostetlerOct 28, 2022
  12. index: add trace2 region for clear skip worktreeAnh Le via GitGitGadget, Oct 31, 2022
  13. Taylor BlauOct 31, 2022
  14. 0/2 index: add trace2 region for clear skip worktreeAnh Le via GitGitGadget, Nov 3, 2022
  15. 2/2 index: raise a bug if the index is materialised more than onceAnh Le via GitGitGadget, Nov 3, 2022
  16. 1/2 index: add trace2 region for clear skip worktreeAnh Le via GitGitGadget, Nov 3, 2022
  17. Taylor BlauNov 5, 2022
  18. Derrick StoleeNov 7, 2022

Read the whole thread, see it on lore, or plain text.

$ cat FOOTERMessages come from the public archive at lore.kernel.org/git, fetched every hour. The front page is chosen and written each morning by an AI editor and can be wrong; the threads themselves are the record. About and API. For agents: an MCP server at https://gitlist.dev/mcp, and any thread, story or person page as Markdown by adding .md to its URL (or sending Accept: text/markdown). Details in /llms.txt.