git/list[1] front-page[2] threads[3] people[4] search[5] about
 

[PATCH 07/12] bloom: split 'get_bloom_filter()' in two

From
Taylor Blau <me@ttaylorr.com>
Date
Sep 9, 2020, 15:23 UTC
Message-ID
<7475ce47ebcdb9c666e658ab8727b2d1df384c8d.1599664389.git.me@ttaylorr.com>
In-Reply-To
<cover.1599664389.git.me@ttaylorr.com>

'get_bloom_filter' takes a flag to control whether it will compute a Bloom filter if the requested one is missing. In the next patch, we'll add yet another parameter to this method, which would force all but one caller to specify an extra 'NULL' parameter at the end.

Instead of doing this, split 'get_bloom_filter' into two functions: 'get_bloom_filter' and 'get_or_compute_bloom_filter'. The former only looks up a Bloom filter (and does not compute one if it's missing, thus dropping the 'compute_if_not_present' flag). The latter does compute missing Bloom filters, with an additional parameter to store whether or not it needed to do so.

This simplifies many call-sites, since the majority of existing callers to 'get_bloom_filter' do not want missing Bloom filters to be computed (so they can drop the parameter entirely and use the simpler version of the function).

While we're at it, instrument the new 'get_or_compute_bloom_filter()' with counters in the 'write_commit_graph_context' struct which store the number of filters that we did and didn't compute, as well as filters that were truncated.

It would be nice to drop the 'compute_if_not_present' flag entirely, since all remaining callers of 'get_or_compute_bloom_filter' pass it as '1', but this will change in a future patch and hence cannot be removed.

Signed-off-by: Taylor Blau <me@ttaylorr.com>
---
 blame.c               |  2 +-
 bloom.c               | 16 +++++++++++++---
 bloom.h               | 16 +++++++++++++---
 commit-graph.c        | 42 +++++++++++++++++++++++++++++++++++++++---
 line-log.c            |  2 +-
 revision.c            |  2 +-
 t/helper/test-bloom.c |  3 ++-
 7 files changed, 70 insertions(+), 13 deletions(-)
diff --git a/blame.c b/blame.c
index 903e23af23..e5ba35dbd1 100644
--- a/blame.c
+++ b/blame.c
@@ -1276,7 +1276,7 @@ static int maybe_changed_path(struct repository *r,
 	if (commit_graph_generation(origin->commit) == GENERATION_NUMBER_INFINITY)
 		return 1;
 
-	filter = get_bloom_filter(r, origin->commit, 0);
+	filter = get_bloom_filter(r, origin->commit);
 
 	if (!filter)
 		return 1;
diff --git a/bloom.c b/bloom.c
index cd9380ac62..393c61b4bc 100644
--- a/bloom.c
+++ b/bloom.c
@@ -177,9 +177,10 @@ static int pathmap_cmp(const void *hashmap_cmp_fn_data,
 	return strcmp(e1->path, e2->path);
 }
 
-struct bloom_filter *get_bloom_filter(struct repository *r,
-				      struct commit *c,
-				      int compute_if_not_present)
+struct bloom_filter *get_or_compute_bloom_filter(struct repository *r,
+						 struct commit *c,
+						 int compute_if_not_present,
+						 enum bloom_filter_computed *computed)
 {
 	struct bloom_filter *filter;
 	struct bloom_filter_settings settings = DEFAULT_BLOOM_FILTER_SETTINGS;
@@ -187,6 +188,9 @@ struct bloom_filter *get_bloom_filter(struct repository *r,
 	struct diff_options diffopt;
 	int max_changes = 512;
 
+	if (computed)
+		*computed = BLOOM_NOT_COMPUTED;
+
 	if (!bloom_filters.slab_size)
 		return NULL;
 
@@ -271,8 +275,14 @@ struct bloom_filter *get_bloom_filter(struct repository *r,
 			diff_free_filepair(diff_queued_diff.queue[i]);
 		filter->data = NULL;
 		filter->len = 0;
+
+		if (computed)
+			*computed |= BLOOM_TRUNC_LARGE;
 	}
 
+	if (computed)
+		*computed |= BLOOM_COMPUTED;
+
 	free(diff_queued_diff.queue);
 	DIFF_QUEUE_CLEAR(&diff_queued_diff);
 
diff --git a/bloom.h b/bloom.h
index 0b9b59a6fe..e2e035ad14 100644
--- a/bloom.h
+++ b/bloom.h
@@ -89,9 +89,19 @@ void add_key_to_filter(const struct bloom_key *key,
 
 void init_bloom_filters(void);
 
-struct bloom_filter *get_bloom_filter(struct repository *r,
-				      struct commit *c,
-				      int compute_if_not_present);
+enum bloom_filter_computed {
+	BLOOM_NOT_COMPUTED = (1 << 0),
+	BLOOM_COMPUTED     = (1 << 1),
+	BLOOM_TRUNC_LARGE  = (1 << 2),
+};
+
+struct bloom_filter *get_or_compute_bloom_filter(struct repository *r,
+						 struct commit *c,
+						 int compute_if_not_present,
+						 enum bloom_filter_computed *computed);
+
+#define get_bloom_filter(r, c) get_or_compute_bloom_filter( \
+	(r), (c), 0, NULL)
 
 int bloom_filter_contains(const struct bloom_filter *filter,
 			  const struct bloom_key *key,
diff --git a/commit-graph.c b/commit-graph.c
index 55af498aa0..b8b2c7ca65 100644
--- a/commit-graph.c
+++ b/commit-graph.c
@@ -971,6 +971,10 @@ struct write_commit_graph_context {
 	const struct split_commit_graph_opts *split_opts;
 	size_t total_bloom_filter_data_size;
 	const struct bloom_filter_settings *bloom_settings;
+
+	int count_bloom_filter_computed;
+	int count_bloom_filter_not_computed;
+	int count_bloom_filter_trunc_large;
 };
 
 static int write_graph_chunk_fanout(struct hashfile *f,
@@ -1182,7 +1186,7 @@ static int write_graph_chunk_bloom_indexes(struct hashfile *f,
 	uint32_t cur_pos = 0;
 
 	while (list < last) {
-		struct bloom_filter *filter = get_bloom_filter(ctx->r, *list, 0);
+		struct bloom_filter *filter = get_bloom_filter(ctx->r, *list);
 		size_t len = filter ? filter->len : 0;
 		cur_pos += len;
 		display_progress(ctx->progress, ++ctx->progress_cnt);
@@ -1222,7 +1226,7 @@ static int write_graph_chunk_bloom_data(struct hashfile *f,
 	hashwrite_be32(f, ctx->bloom_settings->bits_per_entry);
 
 	while (list < last) {
-		struct bloom_filter *filter = get_bloom_filter(ctx->r, *list, 0);
+		struct bloom_filter *filter = get_bloom_filter(ctx->r, *list);
 		size_t len = filter ? filter->len : 0;
 
 		display_progress(ctx->progress, ++ctx->progress_cnt);
@@ -1392,6 +1396,24 @@ static void compute_generation_numbers(struct write_commit_graph_context *ctx)
 	stop_progress(&ctx->progress);
 }
 
+static void trace2_bloom_filter_write_statistics(struct write_commit_graph_context *ctx)
+{
+	struct json_writer jw = JSON_WRITER_INIT;
+
+	jw_object_begin(&jw, 0);
+	jw_object_intmax(&jw, "filter_computed",
+			 ctx->count_bloom_filter_computed);
+	jw_object_intmax(&jw, "filter_not_computed",
+			 ctx->count_bloom_filter_not_computed);
+	jw_object_intmax(&jw, "filter_trunc_large",
+			 ctx->count_bloom_filter_trunc_large);
+	jw_end(&jw);
+
+	trace2_data_json("commit-graph", the_repository, "bloom_statistics", &jw);
+
+	jw_release(&jw);
+}
+
 static void compute_bloom_filters(struct write_commit_graph_context *ctx)
 {
 	int i;
@@ -1414,12 +1436,26 @@ static void compute_bloom_filters(struct write_commit_graph_context *ctx)
 		QSORT(sorted_commits, ctx->commits.nr, commit_gen_cmp);
 
 	for (i = 0; i < ctx->commits.nr; i++) {
+		enum bloom_filter_computed computed = 0;
 		struct commit *c = sorted_commits[i];
-		struct bloom_filter *filter = get_bloom_filter(ctx->r, c, 1);
+		struct bloom_filter *filter = get_or_compute_bloom_filter(
+			ctx->r,
+			c,
+			1,
+			&computed);
+		if (computed & BLOOM_COMPUTED) {
+			ctx->count_bloom_filter_computed++;
+			if (computed & BLOOM_TRUNC_LARGE)
+				ctx->count_bloom_filter_trunc_large++;
+		} else if (computed & BLOOM_NOT_COMPUTED)
+			ctx->count_bloom_filter_not_computed++;
 		ctx->total_bloom_filter_data_size += sizeof(unsigned char) * filter->len;
 		display_progress(progress, i + 1);
 	}
 
+	if (trace2_is_enabled())
+		trace2_bloom_filter_write_statistics(ctx);
+
 	free(sorted_commits);
 	stop_progress(&progress);
 }
diff --git a/line-log.c b/line-log.c
index bf73ea95ac..68eeb425f8 100644
--- a/line-log.c
+++ b/line-log.c
@@ -1159,7 +1159,7 @@ static int bloom_filter_check(struct rev_info *rev,
 		return 1;
 
 	if (!rev->bloom_filter_settings ||
-	    !(filter = get_bloom_filter(rev->repo, commit, 0)))
+	    !(filter = get_bloom_filter(rev->repo, commit)))
 		return 1;
 
 	if (!range)
diff --git a/revision.c b/revision.c
index 857274408c..f4be5d1650 100644
--- a/revision.c
+++ b/revision.c
@@ -751,7 +751,7 @@ static int check_maybe_different_in_bloom_filter(struct rev_info *revs,
 	if (commit_graph_generation(commit) == GENERATION_NUMBER_INFINITY)
 		return -1;
 
-	filter = get_bloom_filter(revs->repo, commit, 0);
+	filter = get_bloom_filter(revs->repo, commit);
 
 	if (!filter) {
 		count_bloom_filter_not_present++;
diff --git a/t/helper/test-bloom.c b/t/helper/test-bloom.c
index 5e77d56f59..9f7bb729fc 100644
--- a/t/helper/test-bloom.c
+++ b/t/helper/test-bloom.c
@@ -39,7 +39,8 @@ static void get_bloom_filter_for_commit(const struct object_id *commit_oid)
 	struct bloom_filter *filter;
 	setup_git_directory();
 	c = lookup_commit(the_repository, commit_oid);
-	filter = get_bloom_filter(the_repository, c, 1);
+	filter = get_or_compute_bloom_filter(the_repository, c, 1,
+					     NULL);
 	print_bloom_filter(filter);
 }
 

-- 
2.28.0.462.g4ff11cec37
Previous: Derrick StoleeNext: Taylor Blau
Message 26 of 75 in “more miscellaneous Bloom filter improvements, redux”
  1. 00/12 more miscellaneous Bloom filter improvements, reduxTaylor Blau, Sep 9, 2020
  2. 03/12 commit-graph: pass a 'struct repository *' in more placesTaylor Blau, Sep 9, 2020
  3. 11/12 commit-graph: rename 'split_commit_graph_opts'Taylor Blau, Sep 9, 2020
  4. 12/12 builtin/commit-graph.c: introduce '--max-new-filters=<n>'Taylor Blau, Sep 9, 2020
  5. Jeff KingSep 11, 2020
  6. Taylor BlauSep 11, 2020
  7. Taylor BlauSep 11, 2020
  8. Taylor BlauSep 14, 2020
  9. Derrick StoleeSep 14, 2020
  10. Taylor BlauSep 14, 2020
  11. Derrick StoleeSep 15, 2020
  12. Taylor BlauSep 15, 2020
  13. Junio C HamanoSep 15, 2020
  14. Taylor BlauSep 15, 2020
  15. Jeff KingSep 11, 2020
  16. Junio C HamanoSep 11, 2020
  17. 04/12 t/helper/test-read-graph.c: prepare repo settingsTaylor Blau, Sep 9, 2020
  18. 05/12 commit-graph: respect 'commitGraph.readChangedPaths'Taylor Blau, Sep 9, 2020
  19. 08/12 bloom: use provided 'struct bloom_filter_settings'Taylor Blau, Sep 9, 2020
  20. 06/12 commit-graph.c: store maximum changed pathsTaylor Blau, Sep 9, 2020
  21. 09/12 bloom/diff: properly short-circuit on max_changesTaylor Blau, Sep 9, 2020
  22. 10/12 bloom: encode out-of-bounds filters as non-emptyTaylor Blau, Sep 9, 2020
  23. Taylor BlauSep 10, 2020
  24. Taylor BlauSep 10, 2020
  25. Derrick StoleeSep 11, 2020
  26. 07/12 bloom: split 'get_bloom_filter()' in twoTaylor Blau, Sep 9, 2020
  27. 02/12 t4216: use an '&&'-chainTaylor Blau, Sep 9, 2020
  28. 01/12 commit-graph: introduce 'get_bloom_filter_settings()'Taylor Blau, Sep 9, 2020
  29. 00/13 more miscellaneous Bloom filter improvements, reduxTaylor Blau, Sep 16, 2020
  30. 04/13 t/helper/test-read-graph.c: prepare repo settingsTaylor Blau, Sep 16, 2020
  31. 03/13 commit-graph: pass a 'struct repository *' in more placesTaylor Blau, Sep 16, 2020
  32. 05/13 commit-graph: respect 'commitGraph.readChangedPaths'Taylor Blau, Sep 16, 2020
  33. 06/13 commit-graph.c: store maximum changed pathsTaylor Blau, Sep 16, 2020
  34. 02/13 t4216: use an '&&'-chainTaylor Blau, Sep 16, 2020
  35. 01/13 commit-graph: introduce 'get_bloom_filter_settings()'Taylor Blau, Sep 16, 2020
  36. 07/13 bloom: split 'get_bloom_filter()' in twoTaylor Blau, Sep 16, 2020
  37. 08/13 bloom: use provided 'struct bloom_filter_settings'Taylor Blau, Sep 16, 2020
  38. 10/13 bloom: encode out-of-bounds filters as non-emptyTaylor Blau, Sep 16, 2020
  39. SZEDER GáborSep 17, 2020
  40. Taylor BlauSep 17, 2020
  41. Junio C HamanoSep 18, 2020
  42. Taylor BlauSep 18, 2020
  43. 11/13 commit-graph: rename 'split_commit_graph_opts'Taylor Blau, Sep 16, 2020
  44. 12/13 builtin/commit-graph.c: introduce '--max-new-filters=<n>'Taylor Blau, Sep 16, 2020
  45. SZEDER GáborSep 18, 2020
  46. Taylor BlauSep 18, 2020
  47. 09/13 bloom/diff: properly short-circuit on max_changesTaylor Blau, Sep 16, 2020
  48. 13/13 commit-graph: introduce 'commitGraph.maxNewFilters'Taylor Blau, Sep 16, 2020
  49. Derrick StoleeSep 16, 2020
  50. Junio C HamanoSep 16, 2020
  51. Taylor BlauSep 17, 2020
  52. Junio C HamanoSep 17, 2020
  53. Taylor BlauSep 17, 2020
  54. Taylor BlauSep 17, 2020
  55. Derrick StoleeSep 17, 2020
  56. 00/13 more miscellaneous Bloom filter improvements, reduxTaylor Blau, Sep 18, 2020
  57. 01/13 commit-graph: introduce 'get_bloom_filter_settings()'Taylor Blau, Sep 18, 2020
  58. 03/13 commit-graph: pass a 'struct repository *' in more placesTaylor Blau, Sep 18, 2020
  59. 02/13 t4216: use an '&&'-chainTaylor Blau, Sep 18, 2020
  60. 05/13 commit-graph: respect 'commitGraph.readChangedPaths'Taylor Blau, Sep 18, 2020
  61. 04/13 t/helper/test-read-graph.c: prepare repo settingsTaylor Blau, Sep 18, 2020
  62. 08/13 bloom: use provided 'struct bloom_filter_settings'Taylor Blau, Sep 18, 2020
  63. SZEDER GáborSep 18, 2020
  64. Taylor BlauSep 18, 2020
  65. 07/13 bloom: split 'get_bloom_filter()' in twoTaylor Blau, Sep 18, 2020
  66. 06/13 commit-graph.c: store maximum changed pathsTaylor Blau, Sep 18, 2020
  67. 10/13 bloom: encode out-of-bounds filters as non-emptyTaylor Blau, Sep 18, 2020
  68. 11/13 commit-graph: rename 'split_commit_graph_opts'Taylor Blau, Sep 18, 2020
  69. 13/13 commit-graph: introduce 'commitGraph.maxNewFilters'Taylor Blau, Sep 18, 2020
  70. Taylor BlauSep 18, 2020
  71. Junio C HamanoSep 18, 2020
  72. 12/13 builtin/commit-graph.c: introduce '--max-new-filters=<n>'Taylor Blau, Sep 18, 2020
  73. 09/13 bloom/diff: properly short-circuit on max_changesTaylor Blau, Sep 18, 2020
  74. Taylor BlauSep 18, 2020
  75. Taylor BlauSep 18, 2020

Read the whole thread, see it on lore, or plain text.

$ cat FOOTERMessages come from the public archive at lore.kernel.org/git, fetched every hour. The front page is chosen and written each morning by an AI editor and can be wrong; the threads themselves are the record. About and API. For agents: an MCP server at https://gitlist.dev/mcp, and any thread, story or person page as Markdown by adding .md to its URL (or sending Accept: text/markdown). Details in /llms.txt.