git/list[1] front-page[2] threads[3] people[4] search[5] about
 

[PATCH 17/30] survey: add report of "largest" paths

From
Derrick Stolee via GitGitGadget <gitgitgadget@gmail.com>
Date
Sep 10, 2024, 02:28 UTC
Message-ID
<9e95914d393ff83054ee419b58b9db4d3560a36c.1725935335.git.gitgitgadget@gmail.com>
In-Reply-To
<pull.1786.git.1725935335.gitgitgadget@gmail.com>
From: Derrick Stolee <stolee@gmail.com>

Since we are already walking our reachable objects using the path-walk API, let's now collect lists of the paths that contribute most to different metrics. Specifically, we care about

 * Number of versions.
 * Total size on disk.
 * Total inflated size (no delta or zlib compression).

This information can be critical to discovering which parts of the repository are causing the most growth, especially on-disk size. Different packing strategies might help compress data more efficiently, but the toal inflated size is a representation of the raw size of all snapshots of those paths. Even when stored efficiently on disk, that size represents how much information must be processed to complete a command such as 'git blame'.

Since the on-disk size is likely to be fragile, stop testing the exact output of 'git survey' and check that the correct set of headers is output.

Signed-off-by: Derrick Stolee <stolee@gmail.com>
---
 builtin/survey.c      | 90 +++++++++++++++++++++++++++++++++++++------
 t/t8100-git-survey.sh | 12 +++++-
 2 files changed, 90 insertions(+), 12 deletions(-)
diff --git a/builtin/survey.c b/builtin/survey.c
index ad467e9a88c..90b041967c8 100644
--- a/builtin/survey.c
+++ b/builtin/survey.c
@@ -80,7 +80,6 @@ struct survey_report_object_size_summary {
 typedef int (*survey_top_size_cmp)(struct survey_report_object_size_summary *s1,
 				   struct survey_report_object_size_summary *s2);
 
-MAYBE_UNUSED
 static int cmp_by_nr(struct survey_report_object_size_summary *s1,
 		     struct survey_report_object_size_summary *s2)
 {
@@ -91,7 +90,6 @@ static int cmp_by_nr(struct survey_report_object_size_summary *s1,
 	return 0;
 }
 
-MAYBE_UNUSED
 static int cmp_by_disk_size(struct survey_report_object_size_summary *s1,
 			    struct survey_report_object_size_summary *s2)
 {
@@ -102,7 +100,6 @@ static int cmp_by_disk_size(struct survey_report_object_size_summary *s1,
 	return 0;
 }
 
-MAYBE_UNUSED
 static int cmp_by_inflated_size(struct survey_report_object_size_summary *s1,
 				struct survey_report_object_size_summary *s2)
 {
@@ -126,7 +123,6 @@ struct survey_report_top_sizes {
 	size_t alloc;
 };
 
-MAYBE_UNUSED
 static void init_top_sizes(struct survey_report_top_sizes *top,
 			   size_t limit, const char *name,
 			   survey_top_size_cmp cmp)
@@ -146,7 +142,6 @@ static void clear_top_sizes(struct survey_report_top_sizes *top)
 	free(top->data);
 }
 
-MAYBE_UNUSED
 static void maybe_insert_into_top_size(struct survey_report_top_sizes *top,
 				       struct survey_report_object_size_summary *summary)
 {
@@ -182,6 +177,10 @@ struct survey_report {
 	struct survey_report_object_summary reachable_objects;
 
 	struct survey_report_object_size_summary *by_type;
+
+	struct survey_report_top_sizes *top_paths_by_count;
+	struct survey_report_top_sizes *top_paths_by_disk;
+	struct survey_report_top_sizes *top_paths_by_inflate;
 };
 
 #define REPORT_TYPE_COMMIT 0
@@ -423,6 +422,13 @@ static void survey_report_object_sizes(const char *title,
 	clear_table(&table);
 }
 
+static void survey_report_plaintext_sorted_size(
+		struct survey_report_top_sizes *top)
+{
+	survey_report_object_sizes(top->name,  _("Path"),
+				   top->data, top->nr);
+}
+
 static void survey_report_plaintext(struct survey_context *ctx)
 {
 	printf("GIT SURVEY for \"%s\"\n", ctx->repo->worktree);
@@ -433,6 +439,21 @@ static void survey_report_plaintext(struct survey_context *ctx)
 				   _("Object Type"),
 				   ctx->report.by_type,
 				   REPORT_TYPE_COUNT);
+
+	survey_report_plaintext_sorted_size(
+		&ctx->report.top_paths_by_count[REPORT_TYPE_TREE]);
+	survey_report_plaintext_sorted_size(
+		&ctx->report.top_paths_by_count[REPORT_TYPE_BLOB]);
+
+	survey_report_plaintext_sorted_size(
+		&ctx->report.top_paths_by_disk[REPORT_TYPE_TREE]);
+	survey_report_plaintext_sorted_size(
+		&ctx->report.top_paths_by_disk[REPORT_TYPE_BLOB]);
+
+	survey_report_plaintext_sorted_size(
+		&ctx->report.top_paths_by_inflate[REPORT_TYPE_TREE]);
+	survey_report_plaintext_sorted_size(
+		&ctx->report.top_paths_by_inflate[REPORT_TYPE_BLOB]);
 }
 
 static void survey_report_json(struct survey_context *ctx)
@@ -673,7 +694,8 @@ static void increment_totals(struct survey_context *ctx,
 
 static void increment_object_totals(struct survey_context *ctx,
 				    struct oid_array *oids,
-				    enum object_type type)
+				    enum object_type type,
+				    const char *path)
 {
 	struct survey_report_object_size_summary *total;
 	struct survey_report_object_size_summary summary = { 0 };
@@ -701,6 +723,27 @@ static void increment_object_totals(struct survey_context *ctx,
 	total->disk_size += summary.disk_size;
 	total->inflated_size += summary.inflated_size;
 	total->num_missing += summary.num_missing;
+
+	if (type == OBJ_TREE || type == OBJ_BLOB) {
+		int index = type == OBJ_TREE ?
+			    REPORT_TYPE_TREE : REPORT_TYPE_BLOB;
+		struct survey_report_top_sizes *top;
+
+		/*
+		 * Temporarily store (const char *) here, but it will
+		 * be duped if inserted and will not be freed.
+		 */
+		summary.label = (char *)path;
+
+		top = ctx->report.top_paths_by_count;
+		maybe_insert_into_top_size(&top[index], &summary);
+
+		top = ctx->report.top_paths_by_disk;
+		maybe_insert_into_top_size(&top[index], &summary);
+
+		top = ctx->report.top_paths_by_inflate;
+		maybe_insert_into_top_size(&top[index], &summary);
+	}
 }
 
 static int survey_objects_path_walk_fn(const char *path,
@@ -712,7 +755,7 @@ static int survey_objects_path_walk_fn(const char *path,
 
 	increment_object_counts(&ctx->report.reachable_objects,
 				type, oids->nr);
-	increment_object_totals(ctx, oids, type);
+	increment_object_totals(ctx, oids, type, path);
 
 	ctx->progress_nr += oids->nr;
 	display_progress(ctx->progress, ctx->progress_nr);
@@ -757,6 +800,34 @@ static int iterate_tag_chain(struct survey_context *ctx,
 	return -1;
 }
 
+static void initialize_report(struct survey_context *ctx)
+{
+	const int top_limit = 100;
+
+	CALLOC_ARRAY(ctx->report.by_type, REPORT_TYPE_COUNT);
+	ctx->report.by_type[REPORT_TYPE_COMMIT].label = xstrdup(_("Commits"));
+	ctx->report.by_type[REPORT_TYPE_TREE].label = xstrdup(_("Trees"));
+	ctx->report.by_type[REPORT_TYPE_BLOB].label = xstrdup(_("Blobs"));
+
+	CALLOC_ARRAY(ctx->report.top_paths_by_count, REPORT_TYPE_COUNT);
+	init_top_sizes(&ctx->report.top_paths_by_count[REPORT_TYPE_TREE],
+		       top_limit, _("TOP DIRECTORIES BY COUNT"), cmp_by_nr);
+	init_top_sizes(&ctx->report.top_paths_by_count[REPORT_TYPE_BLOB],
+		       top_limit, _("TOP FILES BY COUNT"), cmp_by_nr);
+
+	CALLOC_ARRAY(ctx->report.top_paths_by_disk, REPORT_TYPE_COUNT);
+	init_top_sizes(&ctx->report.top_paths_by_disk[REPORT_TYPE_TREE],
+		       top_limit, _("TOP DIRECTORIES BY DISK SIZE"), cmp_by_disk_size);
+	init_top_sizes(&ctx->report.top_paths_by_disk[REPORT_TYPE_BLOB],
+		       top_limit, _("TOP FILES BY DISK SIZE"), cmp_by_disk_size);
+
+	CALLOC_ARRAY(ctx->report.top_paths_by_inflate, REPORT_TYPE_COUNT);
+	init_top_sizes(&ctx->report.top_paths_by_inflate[REPORT_TYPE_TREE],
+		       top_limit, _("TOP DIRECTORIES BY INFLATED SIZE"), cmp_by_inflated_size);
+	init_top_sizes(&ctx->report.top_paths_by_inflate[REPORT_TYPE_BLOB],
+		       top_limit, _("TOP FILES BY INFLATED SIZE"), cmp_by_inflated_size);
+}
+
 static void survey_phase_objects(struct survey_context *ctx)
 {
 	struct rev_info revs = REV_INFO_INIT;
@@ -774,10 +845,7 @@ static void survey_phase_objects(struct survey_context *ctx)
 	info.blobs = 1;
 	info.tags = 1;
 
-	CALLOC_ARRAY(ctx->report.by_type, REPORT_TYPE_COUNT);
-	ctx->report.by_type[REPORT_TYPE_COMMIT].label = xstrdup(_("Commits"));
-	ctx->report.by_type[REPORT_TYPE_TREE].label = xstrdup(_("Trees"));
-	ctx->report.by_type[REPORT_TYPE_BLOB].label = xstrdup(_("Blobs"));
+	initialize_report(ctx);
 
 	repo_init_revisions(ctx->repo, &revs, "");
 
diff --git a/t/t8100-git-survey.sh b/t/t8100-git-survey.sh
index f8af9601214..c2dab0033f9 100755
--- a/t/t8100-git-survey.sh
+++ b/t/t8100-git-survey.sh
@@ -60,7 +60,17 @@ test_expect_success 'git survey (default)' '
 	      Blobs |    10 |       191 |           101
 	EOF
 
-	test_cmp expect out
+	lines=$(wc -l <expect) &&
+	head -n $lines out >out-trimmed &&
+	test_cmp expect out-trimmed &&
+
+	for type in "DIRECTORIES" "FILES"
+	do
+		for metric in "COUNT" "DISK SIZE" "INFLATED SIZE"
+		do
+			grep "TOP $type BY $metric" out || return 1
+		done || return 1
+	done
 '
 
 test_done

-- 
gitgitgadget
Previous: Derrick Stolee via GitGitGadgetNext: Derrick Stolee via GitGitGadget
Message 18 of 38 in “[RFC] Path-walk API and applications”
  1. 00/30 [RFC] Path-walk API and applicationsDerrick Stolee via GitGitGadget, Sep 10, 2024
  2. 01/30 path-walk: introduce an object walk by pathDerrick Stolee via GitGitGadget, Sep 10, 2024
  3. 02/30 backfill: add builtin boilerplateDerrick Stolee via GitGitGadget, Sep 10, 2024
  4. 03/30 backfill: basic functionality and testsDerrick Stolee via GitGitGadget, Sep 10, 2024
  5. 04/30 backfill: add --batch-size=<n> optionDerrick Stolee via GitGitGadget, Sep 10, 2024
  6. 06/30 backfill: assume --sparse when sparse-checkout is enabledDerrick Stolee via GitGitGadget, Sep 10, 2024
  7. 05/30 backfill: add --sparse optionDerrick Stolee via GitGitGadget, Sep 10, 2024
  8. 07/30 path-walk: allow consumer to specify object typesDerrick Stolee via GitGitGadget, Sep 10, 2024
  9. 08/30 path-walk: allow visiting tagsDerrick Stolee via GitGitGadget, Sep 10, 2024
  10. 09/30 survey: stub in new experimental `git-survey` commandJeff Hostetler via GitGitGadget, Sep 10, 2024
  11. 10/30 survey: add command line opts to select referencesJeff Hostetler via GitGitGadget, Sep 10, 2024
  12. 11/30 survey: collect the set of requested refsJeff Hostetler via GitGitGadget, Sep 10, 2024
  13. 12/30 survey: start pretty printing data in table formDerrick Stolee via GitGitGadget, Sep 10, 2024
  14. 13/30 survey: add object count summaryDerrick Stolee via GitGitGadget, Sep 10, 2024
  15. 14/30 survey: summarize total sizes by object typeDerrick Stolee via GitGitGadget, Sep 10, 2024
  16. 15/30 survey: show progress during object walkDerrick Stolee via GitGitGadget, Sep 10, 2024
  17. 16/30 survey: add ability to track prioritized listsDerrick Stolee via GitGitGadget, Sep 10, 2024
  18. 17/30 survey: add report of "largest" pathsDerrick Stolee via GitGitGadget, Sep 10, 2024
  19. 18/30 revision: create mark_trees_uninteresting_dense()Derrick Stolee via GitGitGadget, Sep 10, 2024
  20. 19/30 path-walk: add prune_all_uninteresting optionDerrick Stolee via GitGitGadget, Sep 10, 2024
  21. 20/30 pack-objects: add --path-walk optionDerrick Stolee via GitGitGadget, Sep 10, 2024
  22. 21/30 pack-objects: extract should_attempt_deltas()Derrick Stolee via GitGitGadget, Sep 10, 2024
  23. 22/30 pack-objects: introduce GIT_TEST_PACK_PATH_WALKDerrick Stolee via GitGitGadget, Sep 10, 2024
  24. 23/30 p5313: add size comparison testDerrick Stolee via GitGitGadget, Sep 10, 2024
  25. 24/30 repack: add --path-walk optionDerrick Stolee via GitGitGadget, Sep 10, 2024
  26. 25/30 pack-objects: enable --path-walk via configDerrick Stolee via GitGitGadget, Sep 10, 2024
  27. 26/30 scalar: enable path-walk during push via configDerrick Stolee via GitGitGadget, Sep 10, 2024
  28. 27/30 pack-objects: add --full-name-hash optionDerrick Stolee via GitGitGadget, Sep 10, 2024
  29. 28/30 test-name-hash: add helper to compute name-hash functionsDerrick Stolee via GitGitGadget, Sep 10, 2024
  30. 29/30 p5314: add a size test for name-hash collisionsDerrick Stolee via GitGitGadget, Sep 10, 2024
  31. 30/30 pack-objects: output debug info about deltasDerrick Stolee via GitGitGadget, Sep 10, 2024
  32. Junio C HamanoSep 11, 2024
  33. Christian CouderSep 17, 2024
  34. Derrick StoleeSep 18, 2024
  35. Junio C HamanoSep 22, 2024
  36. Derrick StoleeSep 23, 2024
  37. Junio C HamanoSep 23, 2024
  38. Kristoffer HaugsbakkSep 22, 2024

Read the whole thread, see it on lore, or plain text.

$ cat FOOTERMessages come from the public archive at lore.kernel.org/git, fetched every hour. The front page is chosen and written each morning by an AI editor and can be wrong; the threads themselves are the record. About and API. For agents: an MCP server at https://gitlist.dev/mcp, and any thread, story or person page as Markdown by adding .md to its URL (or sending Accept: text/markdown). Details in /llms.txt.