git/list[1] front-page[2] threads[3] people[4] search[5] about
 

[PATCH v2 14/14] ref-filter: parse objects on demand

From
Patrick Steinhardt <ps@pks.im>
Date
Oct 8, 2025, 15:50 UTC
Message-ID
<20251008-b4-pks-ref-filter-skip-parsing-objects-v2-14-76e30d5c9542@pks.im>
In-Reply-To
<20251008-b4-pks-ref-filter-skip-parsing-objects-v2-0-76e30d5c9542@pks.im>

When formatting an arbitrary object we parse that object regardless of whether or not we actually need any parsed data. In fact, many of the atoms we have don't require any.

Refactor the code so that we parse the data on demand when we see an atom that wants to access the objects. This leads to a small speedup, for example in the Chromium repository with around 40000 refs:

    Benchmark 1: for-each-ref --format='%(raw)' (HEAD~)
      Time (mean ± σ):     388.7 ms ±   1.1 ms    [User: 322.2 ms, System: 65.0 ms]
      Range (min … max):   387.3 ms … 390.8 ms    10 runs
    Benchmark 2: for-each-ref --format='%(raw)' (HEAD)
      Time (mean ± σ):     344.7 ms ±   0.7 ms    [User: 287.8 ms, System: 55.1 ms]
      Range (min … max):   343.9 ms … 345.7 ms    10 runs
    Summary
      for-each-ref --format='%(raw)' (HEAD) ran
        1.13 ± 0.00 times faster than for-each-ref --format='%(raw)' (HEAD~)

With this change, we now spend ~90% of the time decompressing objects, which is almost as good as it gets regarding git-for-each-ref(1)'s own infrastructure.

Signed-off-by: Patrick Steinhardt <ps@pks.im>
---
 ref-filter.c | 156 ++++++++++++++++++++++++++++++++++++++++++++---------------
 1 file changed, 117 insertions(+), 39 deletions(-)
diff --git a/ref-filter.c b/ref-filter.c
index 72cf85c8c6..537c7babac 100644
--- a/ref-filter.c
+++ b/ref-filter.c
@@ -91,6 +91,7 @@ static struct expand_data {
 	struct object_id delta_base_oid;
 	void *content;
 
+	struct object *maybe_object;
 	struct object_info info;
 } oi, oi_deref;
 
@@ -1475,11 +1476,28 @@ static void grab_common_values(struct atom_value *val, int deref, struct expand_
 	}
 }
 
+static int get_or_parse_object(struct expand_data *data, const char *refname,
+			       struct object **object, struct strbuf *err, int *eaten)
+{
+	if (!data->maybe_object) {
+		data->maybe_object = parse_object_buffer(the_repository, &data->oid, data->type,
+							 data->size, data->content, eaten);
+		if (!data->maybe_object)
+			return strbuf_addf_ret(err, -1, _("parse_object_buffer failed on %s for %s"),
+					       oid_to_hex(&data->oid), refname);
+	}
+
+	*object = data->maybe_object;
+	return 0;
+}
+
 /* See grab_values */
-static void grab_tag_values(struct atom_value *val, int deref, struct object *obj)
+static int grab_tag_values(struct atom_value *val, int deref,
+			   struct expand_data *data, const char *refname,
+			   struct strbuf *err, int *eaten)
 {
-	int i;
-	struct tag *tag = (struct tag *) obj;
+	struct tag *tag = NULL;
+	int i, ret;
 
 	for (i = 0; i < used_atom_cnt; i++) {
 		const char *name = used_atom[i].name;
@@ -1487,6 +1505,17 @@ static void grab_tag_values(struct atom_value *val, int deref, struct object *ob
 		struct atom_value *v = &val[i];
 		if (!!deref != (*name == '*'))
 			continue;
+
+		if (!tag) {
+			struct object *object;
+
+			ret = get_or_parse_object(data, refname, &object, err, eaten);
+			if (ret < 0)
+				return ret;
+
+			tag = (struct tag *) object;
+		}
+
 		if (deref)
 			name++;
 		if (atom_type == ATOM_TAG)
@@ -1496,22 +1525,38 @@ static void grab_tag_values(struct atom_value *val, int deref, struct object *ob
 		else if (atom_type == ATOM_OBJECT && tag->tagged)
 			v->s = xstrdup(oid_to_hex(&tag->tagged->oid));
 	}
+
+	return 0;
 }
 
 /* See grab_values */
-static void grab_commit_values(struct atom_value *val, int deref, struct object *obj)
+static int grab_commit_values(struct atom_value *val, int deref,
+			      struct expand_data *data, const char *refname,
+			      struct strbuf *err, int *eaten)
 {
-	int i;
-	struct commit *commit = (struct commit *) obj;
+	int i, ret;
+	struct commit *commit = NULL;
 
 	for (i = 0; i < used_atom_cnt; i++) {
 		const char *name = used_atom[i].name;
 		enum atom_type atom_type = used_atom[i].atom_type;
 		struct atom_value *v = &val[i];
+
 		if (!!deref != (*name == '*'))
 			continue;
 		if (deref)
 			name++;
+
+		if (!commit) {
+			struct object *object;
+
+			ret = get_or_parse_object(data, refname, &object, err, eaten);
+			if (ret < 0)
+				return ret;
+
+			commit = (struct commit *) object;
+		}
+
 		if (atom_type == ATOM_TREE &&
 		    grab_oid(name, "tree", get_commit_tree_oid(commit), v, &used_atom[i]))
 			continue;
@@ -1531,6 +1576,8 @@ static void grab_commit_values(struct atom_value *val, int deref, struct object
 			v->s = strbuf_detach(&s, NULL);
 		}
 	}
+
+	return 0;
 }
 
 static const char *find_wholine(const char *who, int wholen, const char *buf)
@@ -1759,10 +1806,12 @@ static void grab_person(const char *who, struct atom_value *val, int deref, void
 	}
 }
 
-static void grab_signature(struct atom_value *val, int deref, struct object *obj)
+static int grab_signature(struct atom_value *val, int deref,
+			  struct expand_data *data, const char *refname,
+			  struct strbuf *err, int *eaten)
 {
-	int i;
-	struct commit *commit = (struct commit *) obj;
+	int i, ret;
+	struct commit *commit = NULL;
 	struct signature_check sigc = { 0 };
 	int signature_checked = 0;
 
@@ -1790,6 +1839,16 @@ static void grab_signature(struct atom_value *val, int deref, struct object *obj
 			continue;
 
 		if (!signature_checked) {
+			if (!commit) {
+				struct object *object;
+
+				ret = get_or_parse_object(data, refname, &object, err, eaten);
+				if (ret < 0)
+					return ret;
+
+				commit = (struct commit *) object;
+			}
+
 			check_commit_signature(commit, &sigc);
 			signature_checked = 1;
 		}
@@ -1843,6 +1902,8 @@ static void grab_signature(struct atom_value *val, int deref, struct object *obj
 
 	if (signature_checked)
 		signature_check_clear(&sigc);
+
+	return 0;
 }
 
 static void find_subpos(const char *buf,
@@ -1920,9 +1981,8 @@ static void append_lines(struct strbuf *out, const char *buf, unsigned long size
 }
 
 static void grab_describe_values(struct atom_value *val, int deref,
-				 struct object *obj)
+				 struct expand_data *data)
 {
-	struct commit *commit = (struct commit *)obj;
 	int i;
 
 	for (i = 0; i < used_atom_cnt; i++) {
@@ -1944,7 +2004,7 @@ static void grab_describe_values(struct atom_value *val, int deref,
 		cmd.git_cmd = 1;
 		strvec_push(&cmd.args, "describe");
 		strvec_pushv(&cmd.args, atom->u.describe_args.v);
-		strvec_push(&cmd.args, oid_to_hex(&commit->object.oid));
+		strvec_push(&cmd.args, oid_to_hex(&data->oid));
 		if (pipe_command(&cmd, NULL, 0, &out, 0, &err, 0) < 0) {
 			error(_("failed to run 'describe'"));
 			v->s = xstrdup("");
@@ -2066,24 +2126,36 @@ static void fill_missing_values(struct atom_value *val)
  * pointed at by the ref itself; otherwise it is the object the
  * ref (which is a tag) refers to.
  */
-static void grab_values(struct atom_value *val, int deref, struct object *obj, struct expand_data *data)
+static int grab_values(struct atom_value *val, int deref, struct expand_data *data,
+		       const char *refname, struct strbuf *err, int *eaten)
 {
 	void *buf = data->content;
+	int ret;
 
-	switch (obj->type) {
+	switch (data->type) {
 	case OBJ_TAG:
-		grab_tag_values(val, deref, obj);
+		ret = grab_tag_values(val, deref, data, refname, err, eaten);
+		if (ret < 0)
+			goto out;
+
 		grab_sub_body_contents(val, deref, data);
 		grab_person("tagger", val, deref, buf);
-		grab_describe_values(val, deref, obj);
+		grab_describe_values(val, deref, data);
 		break;
 	case OBJ_COMMIT:
-		grab_commit_values(val, deref, obj);
+		ret = grab_commit_values(val, deref, data, refname, err, eaten);
+		if (ret < 0)
+			goto out;
+
 		grab_sub_body_contents(val, deref, data);
 		grab_person("author", val, deref, buf);
 		grab_person("committer", val, deref, buf);
-		grab_signature(val, deref, obj);
-		grab_describe_values(val, deref, obj);
+
+		ret = grab_signature(val, deref, data, refname, err, eaten);
+		if (ret < 0)
+			goto out;
+
+		grab_describe_values(val, deref, data);
 		break;
 	case OBJ_TREE:
 		/* grab_tree_values(val, deref, obj, buf, sz); */
@@ -2094,8 +2166,12 @@ static void grab_values(struct atom_value *val, int deref, struct object *obj, s
 		grab_sub_body_contents(val, deref, data);
 		break;
 	default:
-		die("Eh?  Object of type %d?", obj->type);
+		die("Eh?  Object of type %d?", data->type);
 	}
+
+	ret = 0;
+out:
+	return ret;
 }
 
 static inline char *copy_advance(char *dst, const char *src)
@@ -2292,38 +2368,41 @@ static const char *get_refname(struct used_atom *atom, struct ref_array_item *re
 	return show_ref(&atom->u.refname, ref->refname);
 }
 
-static int get_object(struct ref_array_item *ref, int deref, struct object **obj,
+static int get_object(struct ref_array_item *ref, int deref,
 		      struct expand_data *oi, struct strbuf *err)
 {
-	/* parse_object_buffer() will set eaten to 0 if free() will be needed */
-	int eaten = 1;
+	/* parse_object_buffer() will set eaten to 1 if free() will be needed */
+	int eaten = 0;
+	int ret;
+
 	if (oi->info.contentp) {
 		/* We need to know that to use parse_object_buffer properly */
 		oi->info.sizep = &oi->size;
 		oi->info.typep = &oi->type;
 	}
+
 	if (odb_read_object_info_extended(the_repository->objects, &oi->oid, &oi->info,
-					  OBJECT_INFO_LOOKUP_REPLACE))
-		return strbuf_addf_ret(err, -1, _("missing object %s for %s"),
-				       oid_to_hex(&oi->oid), ref->refname);
+					  OBJECT_INFO_LOOKUP_REPLACE)) {
+		ret = strbuf_addf_ret(err, -1, _("missing object %s for %s"),
+				      oid_to_hex(&oi->oid), ref->refname);
+		goto out;
+	}
 	if (oi->info.disk_sizep && oi->disk_size < 0)
 		BUG("Object size is less than zero.");
 
 	if (oi->info.contentp) {
-		*obj = parse_object_buffer(the_repository, &oi->oid, oi->type, oi->size, oi->content, &eaten);
-		if (!*obj) {
-			if (!eaten)
-				free(oi->content);
-			return strbuf_addf_ret(err, -1, _("parse_object_buffer failed on %s for %s"),
-					       oid_to_hex(&oi->oid), ref->refname);
-		}
-		grab_values(ref->value, deref, *obj, oi);
+		ret = grab_values(ref->value, deref, oi, ref->refname, err, &eaten);
+		if (ret < 0)
+			goto out;
 	}
 
 	grab_common_values(ref->value, deref, oi);
+	ret = 0;
+
+out:
 	if (!eaten)
 		free(oi->content);
-	return 0;
+	return ret;
 }
 
 static void populate_worktree_map(struct hashmap *map, struct worktree **worktrees)
@@ -2376,7 +2455,6 @@ static char *get_worktree_path(const struct ref_array_item *ref)
  */
 static int populate_value(struct ref_array_item *ref, struct strbuf *err)
 {
-	struct object *obj;
 	int i;
 	struct object_info empty = OBJECT_INFO_INIT;
 	int ahead_behind_atoms = 0;
@@ -2564,14 +2642,14 @@ static int populate_value(struct ref_array_item *ref, struct strbuf *err)
 
 
 	oi.oid = ref->objectname;
-	if (get_object(ref, 0, &obj, &oi, err))
+	if (get_object(ref, 0, &oi, err))
 		return -1;
 
 	/*
 	 * If there is no atom that wants to know about tagged
 	 * object, we are done.
 	 */
-	if (!need_tagged || (obj->type != OBJ_TAG))
+	if (!need_tagged || (oi.type != OBJ_TAG))
 		return 0;
 
 	/*
@@ -2589,7 +2667,7 @@ static int populate_value(struct ref_array_item *ref, struct strbuf *err)
 		}
 	}
 
-	return get_object(ref, 1, &obj, &oi_deref, err);
+	return get_object(ref, 1, &oi_deref, err);
 }
 
 /*

-- 
2.51.0.764.g787ff6f08a.dirty
Previous: Patrick SteinhardtNext: Jeff King
Message 54 of 106 in “refs: improvements and fixes for peeling tags”
  1. 00/13 refs: improvements and fixes for peeling tagsPatrick Steinhardt, Oct 7, 2025
  2. 01/13 refs: introduce wrapper struct for `each_ref_fn`Patrick Steinhardt, Oct 7, 2025
  3. Justin ToblerOct 7, 2025
  4. Patrick SteinhardtOct 8, 2025
  5. Taylor BlauOct 7, 2025
  6. shejialuoOct 8, 2025
  7. Patrick SteinhardtOct 9, 2025
  8. 02/13 refs: introduce `.ref` field for the base iteratorPatrick Steinhardt, Oct 7, 2025
  9. Karthik NayakOct 7, 2025
  10. Patrick SteinhardtOct 8, 2025
  11. Patrick SteinhardtOct 8, 2025
  12. Justin ToblerOct 7, 2025
  13. Taylor BlauOct 7, 2025
  14. 03/13 refs: refactor reference status flagsPatrick Steinhardt, Oct 7, 2025
  15. Karthik NayakOct 7, 2025
  16. Patrick SteinhardtOct 8, 2025
  17. 04/13 refs: expose peeled object ID via the iteratorPatrick Steinhardt, Oct 7, 2025
  18. Karthik NayakOct 7, 2025
  19. Patrick SteinhardtOct 8, 2025
  20. Karthik NayakOct 15, 2025
  21. 05/13 upload-pack: convert to use `reference_get_peeled_oid()`Patrick Steinhardt, Oct 7, 2025
  22. Karthik NayakOct 7, 2025
  23. Patrick SteinhardtOct 8, 2025
  24. 06/13 ref-filter: propagate peeled object IDPatrick Steinhardt, Oct 7, 2025
  25. 07/13 builtin/show-ref: convert to use `reference_get_peeled_oid()`Patrick Steinhardt, Oct 7, 2025
  26. 08/13 refs: drop `current_ref_iter` hackPatrick Steinhardt, Oct 7, 2025
  27. 09/13 refs: drop infrastructure to peel via iteratorsPatrick Steinhardt, Oct 7, 2025
  28. 10/13 object: add flag to `peel_object()` to verify object typePatrick Steinhardt, Oct 7, 2025
  29. Kristoffer HaugsbakkOct 8, 2025
  30. 11/13 refs: don't store peeled object IDs for invalid tagsPatrick Steinhardt, Oct 7, 2025
  31. 12/13 ref-filter: detect broken tags when dereferencing themPatrick Steinhardt, Oct 7, 2025
  32. 13/13 ref-filter: parse objects on demandPatrick Steinhardt, Oct 7, 2025
  33. Kristoffer HaugsbakkOct 8, 2025
  34. Patrick SteinhardtOct 8, 2025
  35. Junio C HamanoOct 7, 2025
  36. Taylor BlauOct 7, 2025
  37. Junio C HamanoOct 7, 2025
  38. 00/14 refs: improvements and fixes for peeling tagsPatrick Steinhardt, Oct 8, 2025
  39. 01/14 refs: introduce wrapper struct for `each_ref_fn`Patrick Steinhardt, Oct 8, 2025
  40. 02/14 refs: introduce `.ref` field for the base iteratorPatrick Steinhardt, Oct 8, 2025
  41. 03/14 refs: fully reset `struct ref_iterator::ref` on iterationPatrick Steinhardt, Oct 8, 2025
  42. 04/14 refs: refactor reference status flagsPatrick Steinhardt, Oct 8, 2025
  43. 05/14 refs: expose peeled object ID via the iteratorPatrick Steinhardt, Oct 8, 2025
  44. 06/14 upload-pack: convert to use `reference_get_peeled_oid()`Patrick Steinhardt, Oct 8, 2025
  45. 07/14 ref-filter: propagate peeled object IDPatrick Steinhardt, Oct 8, 2025
  46. 08/14 builtin/show-ref: convert to use `reference_get_peeled_oid()`Patrick Steinhardt, Oct 8, 2025
  47. 09/14 refs: drop `current_ref_iter` hackPatrick Steinhardt, Oct 8, 2025
  48. 10/14 refs: drop infrastructure to peel via iteratorsPatrick Steinhardt, Oct 8, 2025
  49. 11/14 object: add flag to `peel_object()` to verify object typePatrick Steinhardt, Oct 8, 2025
  50. 12/14 refs: don't store peeled object IDs for invalid tagsPatrick Steinhardt, Oct 8, 2025
  51. shejialuoOct 8, 2025
  52. Patrick SteinhardtOct 9, 2025
  53. 13/14 ref-filter: detect broken tags when dereferencing themPatrick Steinhardt, Oct 8, 2025
  54. 14/14 ref-filter: parse objects on demandPatrick Steinhardt, Oct 8, 2025
  55. Jeff KingOct 9, 2025
  56. Patrick SteinhardtOct 9, 2025
  57. Jeff KingOct 9, 2025
  58. Patrick SteinhardtOct 9, 2025
  59. Jeff KingOct 10, 2025
  60. Patrick SteinhardtOct 10, 2025
  61. Jeff KingOct 10, 2025
  62. Junio C HamanoOct 10, 2025
  63. Patrick SteinhardtOct 14, 2025
  64. Junio C HamanoOct 14, 2025
  65. Toon ClaesOct 9, 2025
  66. Junio C HamanoOct 9, 2025
  67. 00/14 refs: improvements and fixes for peeling tagsPatrick Steinhardt, Oct 22, 2025
  68. 01/14 refs: introduce wrapper struct for `each_ref_fn`Patrick Steinhardt, Oct 22, 2025
  69. 02/14 refs: introduce `.ref` field for the base iteratorPatrick Steinhardt, Oct 22, 2025
  70. 03/14 refs: fully reset `struct ref_iterator::ref` on iterationPatrick Steinhardt, Oct 22, 2025
  71. 04/14 refs: refactor reference status flagsPatrick Steinhardt, Oct 22, 2025
  72. 05/14 refs: expose peeled object ID via the iteratorPatrick Steinhardt, Oct 22, 2025
  73. 06/14 upload-pack: convert to use `reference_get_peeled_oid()`Patrick Steinhardt, Oct 22, 2025
  74. 07/14 ref-filter: propagate peeled object IDPatrick Steinhardt, Oct 22, 2025
  75. 08/14 builtin/show-ref: convert to use `reference_get_peeled_oid()`Patrick Steinhardt, Oct 22, 2025
  76. 09/14 refs: drop `current_ref_iter` hackPatrick Steinhardt, Oct 22, 2025
  77. 10/14 refs: drop infrastructure to peel via iteratorsPatrick Steinhardt, Oct 22, 2025
  78. 11/14 object: add flag to `peel_object()` to verify object typePatrick Steinhardt, Oct 22, 2025
  79. 12/14 refs: don't store peeled object IDs for invalid tagsPatrick Steinhardt, Oct 22, 2025
  80. 13/14 ref-filter: detect broken tags when dereferencing themPatrick Steinhardt, Oct 22, 2025
  81. 14/14 ref-filter: parse objects on demandPatrick Steinhardt, Oct 22, 2025
  82. Junio C HamanoOct 22, 2025
  83. Patrick SteinhardtOct 23, 2025
  84. Karthik NayakOct 22, 2025
  85. Junio C HamanoOct 22, 2025
  86. Patrick SteinhardtOct 23, 2025
  87. 00/14 refs: improvements and fixes for peeling tagsPatrick Steinhardt, Oct 23, 2025
  88. 01/14 refs: introduce wrapper struct for `each_ref_fn`Patrick Steinhardt, Oct 23, 2025
  89. 02/14 refs: introduce `.ref` field for the base iteratorPatrick Steinhardt, Oct 23, 2025
  90. 03/14 refs: fully reset `struct ref_iterator::ref` on iterationPatrick Steinhardt, Oct 23, 2025
  91. 04/14 refs: refactor reference status flagsPatrick Steinhardt, Oct 23, 2025
  92. 05/14 refs: expose peeled object ID via the iteratorPatrick Steinhardt, Oct 23, 2025
  93. 06/14 upload-pack: convert to use `reference_get_peeled_oid()`Patrick Steinhardt, Oct 23, 2025
  94. 07/14 ref-filter: propagate peeled object IDPatrick Steinhardt, Oct 23, 2025
  95. 08/14 builtin/show-ref: convert to use `reference_get_peeled_oid()`Patrick Steinhardt, Oct 23, 2025
  96. 09/14 refs: drop `current_ref_iter` hackPatrick Steinhardt, Oct 23, 2025
  97. 10/14 refs: drop infrastructure to peel via iteratorsPatrick Steinhardt, Oct 23, 2025
  98. 11/14 object: add flag to `peel_object()` to verify object typePatrick Steinhardt, Oct 23, 2025
  99. 12/14 refs: don't store peeled object IDs for invalid tagsPatrick Steinhardt, Oct 23, 2025
  100. 13/14 ref-filter: detect broken tags when dereferencing themPatrick Steinhardt, Oct 23, 2025
  101. 14/14 ref-filter: parse objects on demandPatrick Steinhardt, Oct 23, 2025
  102. Jeff KingNov 4, 2025
  103. Junio C HamanoNov 4, 2025
  104. Jeff KingNov 4, 2025
  105. Junio C HamanoOct 23, 2025
  106. Patrick SteinhardtOct 24, 2025

Read the whole thread, see it on lore, or plain text.

$ cat FOOTERMessages come from the public archive at lore.kernel.org/git, fetched every hour. The front page is chosen and written each morning by an AI editor and can be wrong; the threads themselves are the record. About and API. For agents: an MCP server at https://gitlist.dev/mcp, and any thread, story or person page as Markdown by adding .md to its URL (or sending Accept: text/markdown). Details in /llms.txt.