git/list[1] front-page[2] threads[3] people[4] search[5] about
 

[PATCH 14/14] packed-graph: teach git-graph to read commits

From
Derrick Stolee <stolee@gmail.com>
Date
Jan 25, 2018, 14:02 UTC
Message-ID
<20180125140231.65604-15-dstolee@microsoft.com>
In-Reply-To
<20180125140231.65604-1-dstolee@microsoft.com>

Teach git-graph to read commits from stdin when the --stdin-commits flag is specified. Commits reachable from these commits are added to the graph. This is a much faster way to construct the graph than inspecting all packed objects, but is restricted to known tips.

For the Linux repository, 700,000+ commits were added to the graph file starting from 'master' in 7-9 seconds, depending on the number of packfiles in the repo (1, 24, or 120).

Signed-off-by: Derrick Stolee <dstolee@microsoft.com>
---
 builtin/graph.c  | 33 +++++++++++++++++++++++++--------
 packed-graph.c   | 18 +++++++++++++++---
 packed-graph.h   |  3 ++-
 t/t5319-graph.sh | 18 ++++++++++++++++++
 4 files changed, 60 insertions(+), 12 deletions(-)
diff --git a/builtin/graph.c b/builtin/graph.c
index 3cace3a18c..708889677b 100644
--- a/builtin/graph.c
+++ b/builtin/graph.c
@@ -12,7 +12,7 @@ static char const * const builtin_graph_usage[] ={
 	N_("git graph [--pack-dir <packdir>]"),
 	N_("git graph --clear [--pack-dir <packdir>]"),
 	N_("git graph --read [--graph-id=<oid>]"),
-	N_("git graph --write [--pack-dir <packdir>] [--update-head] [--delete-expired] [--stdin-packs]"),
+	N_("git graph --write [--pack-dir <packdir>] [--update-head] [--delete-expired] [--stdin-packs|--stdin-commits]"),
 	NULL
 };
 
@@ -25,6 +25,7 @@ static struct opts_graph {
 	int update_head;
 	int delete_expired;
 	int stdin_packs;
+	int stdin_commits;
 	int has_existing;
 	struct object_id old_graph_oid;
 } opts;
@@ -116,22 +117,36 @@ static int graph_write(void)
 {
 	struct object_id *graph_id;
 	char **pack_indexes = NULL;
+	char **commits = NULL;
 	int num_packs = 0;
-	int size_packs = 0;
+	int num_commits = 0;
+	char **lines = NULL;
+	int num_lines = 0;
+	int size_lines = 0;
 
-	if (opts.stdin_packs) {
+	if (opts.stdin_packs || opts.stdin_commits) {
 		struct strbuf buf = STRBUF_INIT;
-		size_packs = 128;
-		ALLOC_ARRAY(pack_indexes, size_packs);
+		size_lines = 128;
+		ALLOC_ARRAY(lines, size_lines);
 
 		while (strbuf_getline(&buf, stdin) != EOF) {
-			ALLOC_GROW(pack_indexes, num_packs + 1, size_packs);
-			pack_indexes[num_packs++] = buf.buf;
+			ALLOC_GROW(lines, num_lines + 1, size_lines);
+			lines[num_lines++] = buf.buf;
 			strbuf_detach(&buf, NULL);
 		}
+
+		if (opts.stdin_packs) {
+			pack_indexes = lines;
+			num_packs = num_lines;
+		}
+		if (opts.stdin_commits) {
+			commits = lines;
+			num_commits = num_lines;
+		}
 	}
 
-	graph_id = construct_graph(opts.pack_dir, pack_indexes, num_packs);
+	graph_id = construct_graph(opts.pack_dir, pack_indexes, num_packs,
+				   commits, num_commits);
 
 	if (opts.update_head)
 		update_head_file(opts.pack_dir, graph_id);
@@ -170,6 +185,8 @@ int cmd_graph(int argc, const char **argv, const char *prefix)
 			N_("delete expired head graph file")),
 		OPT_BOOL('s', "stdin-packs", &opts.stdin_packs,
 			N_("only scan packfiles listed by stdin")),
+		OPT_BOOL('C', "stdin-commits", &opts.stdin_commits,
+			N_("start walk at commits listed by stdin")),
 		{ OPTION_STRING, 'G', "graph-id", &opts.graph_id,
 			N_("oid"),
 			N_("An OID for a specific graph file in the pack-dir."),
diff --git a/packed-graph.c b/packed-graph.c
index c93515f18e..94e1a97000 100644
--- a/packed-graph.c
+++ b/packed-graph.c
@@ -662,7 +662,8 @@ static void close_reachable(struct packed_oid_list *oids)
 	}
 }
 
-struct object_id *construct_graph(const char *pack_dir, char **pack_indexes, int nr_packs)
+struct object_id *construct_graph(const char *pack_dir, char **pack_indexes, int nr_packs,
+				  char **commit_hex, int nr_commits)
 {
 	// Find a list of oids, adding the pointer to a list.
 	struct packed_oid_list oids;
@@ -719,10 +720,21 @@ struct object_id *construct_graph(const char *pack_dir, char **pack_indexes, int
 			for_each_object_in_pack(p, if_packed_commit_add_to_list, &oids);
 			close_pack(p);
 		}
-	} else {
-		for_each_packed_object(if_packed_commit_add_to_list, &oids, 0);
 	}
 
+	if (commit_hex) {
+		for (i = 0; i < nr_commits; i++) {
+			const char *end;
+			ALLOC_GROW(oids.list, oids.num + 1, oids.size);
+			oids.list[oids.num] = malloc(sizeof(struct object_id));
+			parse_oid_hex(commit_hex[i], oids.list[oids.num], &end);
+			oids.num++;
+		}
+	}
+
+	if (!pack_indexes && !commit_hex)
+		for_each_packed_object(if_packed_commit_add_to_list, &oids, 0);
+
 	close_reachable(&oids);
 	QSORT(oids.list, oids.num, commit_compare);
 
diff --git a/packed-graph.h b/packed-graph.h
index 97ce1e2652..9c766411be 100644
--- a/packed-graph.h
+++ b/packed-graph.h
@@ -58,7 +58,8 @@ extern struct object_id *get_nth_commit_oid(struct packed_graph *g,
 					    uint32_t n,
 					    struct object_id *oid);
 
-extern struct object_id *construct_graph(const char *pack_dir, char **pack_indexes, int nr_packs);
+extern struct object_id *construct_graph(const char *pack_dir, char **pack_indexes, int nr_packs,
+					 char **commits, int nr_commits);
 extern int close_graph(struct packed_graph *g);
 
 #endif
diff --git a/t/t5319-graph.sh b/t/t5319-graph.sh
index 8bf5a0c993..b12d2477ba 100755
--- a/t/t5319-graph.sh
+++ b/t/t5319-graph.sh
@@ -226,6 +226,24 @@ test_expect_success 'build graph from latest pack with closure' \
 _graph_git_behavior commits/20 merge/1
 _graph_git_behavior commits/20 merge/2
 
+test_expect_success 'build graph from commits with closure' \
+    'git rev-parse commits/20 >commits-in &&
+     git rev-parse merge/1 >>commits-in &&
+     git rev-parse merge/2 >>commits-in &&
+     graph6=$(cat commits-in | git graph --write --update-head --delete-expired --stdin-commits) &&
+     test_path_is_file ${packdir}/graph-${graph6}.graph &&
+     test_path_is_missing ${packdir}/graph-${graph5}.graph &&
+     test_path_is_file ${packdir}/graph-${graph1}.graph &&
+     test_path_is_file ${packdir}/graph-head &&
+     echo ${graph6} >expect &&
+     cmp -n 40 expect ${packdir}/graph-head &&
+     git graph --read --graph-id=${graph6} >output &&
+     _graph_read_expect "23" "${packdir}" &&
+     cmp expect output'
+
+_graph_git_behavior commits/20 merge/1
+_graph_git_behavior commits/20 merge/2
+
 test_expect_success 'setup bare repo' \
     'cd .. &&
      git clone --bare full bare &&

-- 
2.16.0
Previous: Derrick StoleeNext: Derrick Stolee
Message 38 of 49 in “Serialized Commit Graph”
  1. 00/14 Serialized Commit GraphDerrick Stolee, Jan 25, 2018
  2. 01/14 graph: add packed graph design documentDerrick Stolee, Jan 25, 2018
  3. Stefan BellerJan 25, 2018
  4. Derrick StoleeJan 26, 2018
  5. Stefan BellerJan 26, 2018
  6. Junio C HamanoJan 25, 2018
  7. Derrick StoleeJan 26, 2018
  8. Duy NguyenJan 26, 2018
  9. 03/14 packed-graph: create git-graph builtinDerrick Stolee, Jan 25, 2018
  10. Stefan BellerJan 25, 2018
  11. Derrick StoleeJan 26, 2018
  12. Junio C HamanoJan 25, 2018
  13. Derrick StoleeJan 26, 2018
  14. Duy NguyenJan 26, 2018
  15. 02/14 packed-graph: add core.graph settingDerrick Stolee, Jan 25, 2018
  16. Stefan BellerJan 25, 2018
  17. Derrick StoleeJan 25, 2018
  18. Junio C HamanoJan 25, 2018
  19. Derrick StoleeJan 26, 2018
  20. 04/14 packed-graph: add format documentDerrick Stolee, Jan 25, 2018
  21. Junio C HamanoJan 25, 2018
  22. Stefan BellerJan 25, 2018
  23. Junio C HamanoJan 25, 2018
  24. Derrick StoleeJan 26, 2018
  25. Stefan BellerJan 25, 2018
  26. Derrick StoleeJan 26, 2018
  27. 09/14 packed-graph: implement git-graph --clearDerrick Stolee, Jan 25, 2018
  28. Stefan BellerJan 25, 2018
  29. 05/14 packed-graph: implement construct_graph()Derrick Stolee, Jan 25, 2018
  30. Stefan BellerJan 25, 2018
  31. Junio C HamanoJan 26, 2018
  32. Junio C HamanoJan 26, 2018
  33. Andreas SchwabJan 26, 2018
  34. Junio C HamanoJan 26, 2018
  35. 06/14 packed-graph: implement git-graph --writeDerrick Stolee, Jan 25, 2018
  36. Stefan BellerJan 25, 2018
  37. Derrick StoleeJan 26, 2018
  38. 14/14 packed-graph: teach git-graph to read commitsDerrick Stolee, Jan 25, 2018
  39. 08/14 graph: implement git-graph --update-headDerrick Stolee, Jan 25, 2018
  40. 07/14 packed-graph: implement git-graph --readDerrick Stolee, Jan 25, 2018
  41. 10/14 packed-graph: teach git-graph --delete-expiredDerrick Stolee, Jan 25, 2018
  42. 12/14 packed-graph: read only from specific pack-indexesDerrick Stolee, Jan 25, 2018
  43. 13/14 packed-graph: close under reachabilityDerrick Stolee, Jan 25, 2018
  44. 11/14 commit: integrate packed graph with commit parsingDerrick Stolee, Jan 25, 2018
  45. Stefan BellerJan 26, 2018
  46. Ævar Arnfjörð BjarmasonJan 25, 2018
  47. Derrick StoleeJan 25, 2018
  48. Ævar Arnfjörð BjarmasonJan 25, 2018
  49. Derrick StoleeJan 26, 2018

Read the whole thread, see it on lore, or plain text.

$ cat FOOTERMessages come from the public archive at lore.kernel.org/git, fetched every hour. The front page is chosen and written each morning by an AI editor and can be wrong; the threads themselves are the record. About and API. For agents: an MCP server at https://gitlist.dev/mcp, and any thread, story or person page as Markdown by adding .md to its URL (or sending Accept: text/markdown). Details in /llms.txt.