git/list[1] front-page[2] threads[3] people[4] search[5] about
 

Re: [PATCH v2 06/14] commit-graph: implement git-commit-graph --read

From
SZEDER Gábor <szeder.dev@gmail.com>
Date
Feb 2, 2018, 00:02 UTC
Message-ID
<20180202000225.9648-1-szeder.dev@gmail.com>
In-Reply-To
<1517348383-112294-7-git-send-email-dstolee@microsoft.com>
Show 23 quoted lines
> Teach git-commit-graph to read commit graph files and summarize their contents.
> 
> Use the --read option to verify the contents of a commit graph file in the
> tests.
> 
> Signed-off-by: Derrick Stolee <dstolee@microsoft.com>
> ---
>  Documentation/git-commit-graph.txt |   7 ++
>  builtin/commit-graph.c             |  55 +++++++++++++++
>  commit-graph.c                     | 138 ++++++++++++++++++++++++++++++++++++-
>  commit-graph.h                     |  25 +++++++
>  t/t5318-commit-graph.sh            |  28 ++++++--
>  5 files changed, 247 insertions(+), 6 deletions(-)
> 
> diff --git a/Documentation/git-commit-graph.txt b/Documentation/git-commit-graph.txt
> index 3f3790d9a8..09aeaf6c82 100644
> --- a/Documentation/git-commit-graph.txt
> +++ b/Documentation/git-commit-graph.txt
> @@ -10,6 +10,7 @@ SYNOPSIS
>  --------
>  [verse]
>  'git commit-graph' --write <options> [--pack-dir <pack_dir>]
> +'git commit-graph' --read <options> [--pack-dir <pack_dir>]
Again, what does this option do?
Show 19 quoted lines
>  EXAMPLES
>  --------
> @@ -20,6 +21,12 @@ EXAMPLES
>  $ git commit-graph --write
>  ------------------------------------------------
>  
> +* Read basic information from a graph file.
> ++
> +------------------------------------------------
> +$ git commit-graph --read --graph-hash=<hash>
> +------------------------------------------------
> +
>  GIT
>  ---
>  Part of the linkgit:git[1] suite
> diff --git a/builtin/commit-graph.c b/builtin/commit-graph.c
> index 7affd512f1..218740b1f8 100644
> --- a/builtin/commit-graph.c
> +++ b/builtin/commit-graph.c
> +int close_commit_graph(struct commit_graph *g)

static, perhaps? I see it's declared as extern in the headeer file below, but I don't see it called outside of this source file by the end of the patch series.

Show 81 quoted lines
> +{
> +	if (g->graph_fd < 0)
> +		return 0;
> +
> +	munmap((void *)g->data, g->data_len);
> +	g->data = 0;
> +
> +	close(g->graph_fd);
> +	g->graph_fd = -1;
> +
> +	return 1;
> +}
> +
> +static void free_commit_graph(struct commit_graph **g)
> +{
> +	if (!g || !*g)
> +		return;
> +
> +	close_commit_graph(*g);
> +
> +	free(*g);
> +	*g = NULL;
> +}
> +
> +struct commit_graph *load_commit_graph_one(const char *graph_file, const char *pack_dir)
> +{
> +	void *graph_map;
> +	const unsigned char *data;
> +	struct commit_graph_header *hdr;
> +	size_t graph_size;
> +	struct stat st;
> +	uint32_t i;
> +	struct commit_graph *graph;
> +	int fd = git_open(graph_file);
> +	uint64_t last_chunk_offset;
> +	uint32_t last_chunk_id;
> +
> +	if (fd < 0)
> +		return 0;
> +	if (fstat(fd, &st)) {
> +		close(fd);
> +		return 0;
> +	}
> +	graph_size = xsize_t(st.st_size);
> +
> +	if (graph_size < GRAPH_MIN_SIZE) {
> +		close(fd);
> +		die("graph file %s is too small", graph_file);
> +	}
> +	graph_map = xmmap(NULL, graph_size, PROT_READ, MAP_PRIVATE, fd, 0);
> +	data = (const unsigned char *)graph_map;
> +
> +	hdr = graph_map;
> +	if (ntohl(hdr->graph_signature) != GRAPH_SIGNATURE) {
> +		uint32_t signature = ntohl(hdr->graph_signature);
> +		munmap(graph_map, graph_size);
> +		close(fd);
> +		die("graph signature %X does not match signature %X",
> +			signature, GRAPH_SIGNATURE);
> +	}
> +	if (hdr->graph_version != GRAPH_VERSION) {
> +		unsigned char version = hdr->graph_version;
> +		munmap(graph_map, graph_size);
> +		close(fd);
> +		die("graph version %X does not match version %X",
> +			version, GRAPH_VERSION);
> +	}
> +
> +	graph = alloc_commit_graph(strlen(pack_dir) + 1);
> +
> +	graph->hdr = hdr;
> +	graph->graph_fd = fd;
> +	graph->data = graph_map;
> +	graph->data_len = graph_size;
> +
> +	last_chunk_id = 0;
> +	last_chunk_offset = (uint64_t)sizeof(*hdr);
> +	for (i = 0; i < hdr->num_chunks; i++) {
> +		uint32_t chunk_id = ntohl(*(uint32_t*)(data + sizeof(*hdr) + 12 * i));
> +		uint64_t chunk_offset1 = ntohl(*(uint32_t*)(data + sizeof(*hdr) + 12 * i + 4));
> +		uint32_t chunk_offset2 = ntohl(*(uint32_t*)(data + sizeof(*hdr) + 12 * i + 8));

There are a lot of magic number in these three lines, but at least they are all multiples of 4.

Show 54 quoted lines
> +		uint64_t chunk_offset = (chunk_offset1 << 32) | chunk_offset2;
> +
> +		if (chunk_offset > graph_size - GIT_MAX_RAWSZ)
> +			die("improper chunk offset %08x%08x", (uint32_t)(chunk_offset >> 32),
> +			    (uint32_t)chunk_offset);
> +
> +		switch (chunk_id) {
> +			case GRAPH_CHUNKID_OIDFANOUT:
> +				graph->chunk_oid_fanout = data + chunk_offset;
> +				break;
> +
> +			case GRAPH_CHUNKID_OIDLOOKUP:
> +				graph->chunk_oid_lookup = data + chunk_offset;
> +				break;
> +
> +			case GRAPH_CHUNKID_DATA:
> +				graph->chunk_commit_data = data + chunk_offset;
> +				break;
> +
> +			case GRAPH_CHUNKID_LARGEEDGES:
> +				graph->chunk_large_edges = data + chunk_offset;
> +				break;
> +
> +			case 0:
> +				break;
> +
> +			default:
> +				free_commit_graph(&graph);
> +				die("unrecognized graph chunk id: %08x", chunk_id);
> +		}
> +
> +		if (last_chunk_id == GRAPH_CHUNKID_OIDLOOKUP)
> +		{
> +			graph->num_commits = (chunk_offset - last_chunk_offset)
> +					     / hdr->hash_len;
> +		}
> +
> +		last_chunk_id = chunk_id;
> +		last_chunk_offset = chunk_offset;
> +	}
> +
> +	strcpy(graph->pack_dir, pack_dir);
> +	return graph;
> +}
> +
>  static void write_graph_chunk_fanout(struct sha1file *f,
>  				     struct commit **commits,
>  				     int nr_commits)
> @@ -361,7 +497,7 @@ struct object_id *construct_commit_graph(const char *pack_dir)
>  	sha1close(f, final_hash, CSUM_CLOSE | CSUM_FSYNC);
>  
>  	f_hash = (struct object_id *)malloc(sizeof(struct object_id));
> -	memcpy(f_hash->hash, final_hash, GIT_MAX_RAWSZ);
> +	hashcpy(f_hash->hash, final_hash);
Oh, look, I told you it's hashcpy()! ;)
Show 55 quoted lines
>  	fname = get_commit_graph_filename_hash(pack_dir, f_hash);
>  
>  	if (rename(graph_name, fname))
> diff --git a/commit-graph.h b/commit-graph.h
> index 7b3469a7df..e046ae575c 100644
> --- a/commit-graph.h
> +++ b/commit-graph.h
> @@ -15,6 +15,31 @@ struct commit_graph_header {
>  	unsigned char num_chunks;
>  };
>  
> +extern struct commit_graph {
> +	int graph_fd;
> +
> +	const unsigned char *data;
> +	size_t data_len;
> +
> +	const struct commit_graph_header *hdr;
> +
> +	struct object_id oid;
> +
> +	uint32_t num_commits;
> +
> +	const unsigned char *chunk_oid_fanout;
> +	const unsigned char *chunk_oid_lookup;
> +	const unsigned char *chunk_commit_data;
> +	const unsigned char *chunk_large_edges;
> +
> +	/* something like ".git/objects/pack" */
> +	char pack_dir[FLEX_ARRAY]; /* more */
> +} *commit_graph;
> +
> +extern int close_commit_graph(struct commit_graph *g);
> +
> +extern struct commit_graph *load_commit_graph_one(const char *graph_file, const char *pack_dir);
> +
>  extern struct object_id *construct_commit_graph(const char *pack_dir);
>  
>  #endif
> diff --git a/t/t5318-commit-graph.sh b/t/t5318-commit-graph.sh
> index 6bcd1cc264..da565624e3 100755
> --- a/t/t5318-commit-graph.sh
> +++ b/t/t5318-commit-graph.sh
> @@ -25,11 +25,23 @@ test_expect_success 'create commits and repack' \
>       done &&
>       git repack'
>  
> +_graph_read_expect() {
> +    cat >expect <<- EOF
> +header: 43475048 01 01 14 04
> +num_commits: $1
> +chunks: oid_fanout oid_lookup commit_metadata large_edges
> +pack_dir: $2
> +EOF
> +}

Style nit: since you are already using the '<<-' operator for the here-doc, you could indent it with tabs.

Show 8 quoted lines
> +
>  test_expect_success 'write graph' \
>      'graph1=$(git commit-graph --write) &&
> -     test_path_is_file ${packdir}/graph-${graph1}.graph'
> +     test_path_is_file ${packdir}/graph-${graph1}.graph &&
> +     git commit-graph --read --graph-hash=${graph1} >output &&
> +     _graph_read_expect "5" "${packdir}" &&
> +     cmp expect output'

Please use the 'test_cmp' helper throughout the tests instead. If the two files don't match, 'cmp' will only tell you where they start to differ, whereas 'test_cmp' will actually show the difference.

> -t_expect_success 'Add more commits' \
> +test_expect_success 'Add more commits' \
This should be squashed into the earlier commit.
Show 21 quoted lines
>      'git reset --hard commits/3 &&
>       for i in $(test_seq 6 10)
>       do
> @@ -79,7 +91,10 @@ t_expect_success 'Add more commits' \
>  
>  test_expect_success 'write graph with merges' \
>      'graph2=$(git commit-graph --write) &&
> -     test_path_is_file ${packdir}/graph-${graph2}.graph'
> +     test_path_is_file ${packdir}/graph-${graph2}.graph &&
> +     git commit-graph --read --graph-hash=${graph2} >output &&
> +     _graph_read_expect "18" "${packdir}" &&
> +     cmp expect output'
>  
>  test_expect_success 'setup bare repo' \
>      'cd .. &&
> @@ -87,10 +102,13 @@ test_expect_success 'setup bare repo' \
>       cd bare &&
>       git config core.graph true &&
>       git config pack.threads 1 &&
> -     baredir="objects/pack"'
> +     baredir="./objects/pack"'

Is this change really necessary? If it is, then perhaps it should have been written this way upon its introduction.

Show 11 quoted lines
>  test_expect_success 'write graph in bare repo' \
>      'graphbare=$(git commit-graph --write) &&
> -     test_path_is_file ${baredir}/graph-${graphbare}.graph'
> +     test_path_is_file ${baredir}/graph-${graphbare}.graph &&
> +     git commit-graph --read --graph-hash=${graphbare} >output &&
> +     _graph_read_expect "18" "${baredir}" &&
> +     cmp expect output'
>  
>  test_done
> -- 
> 2.16.0.15.g9c3cf44.dirty
Previous: Stefan BellerNext: Jonathan Tan
Message 8 of 146 in “Serialized Git Commit Graph”
  1. 00/14 Serialized Git Commit GraphDerrick Stolee, Jan 30, 2018
  2. 01/14 commit-graph: add format documentDerrick Stolee, Jan 30, 2018
  3. Jonathan TanFeb 1, 2018
  4. 03/14 commit-graph: create git-commit-graph builtinDerrick Stolee, Jan 30, 2018
  5. SZEDER GáborFeb 2, 2018
  6. 06/14 commit-graph: implement git-commit-graph --readDerrick Stolee, Jan 30, 2018
  7. Stefan BellerJan 31, 2018
  8. SZEDER GáborFeb 2, 2018
  9. Jonathan TanFeb 2, 2018
  10. Derrick StoleeFeb 5, 2018
  11. 08/14 commit-graph: implement git-commit-graph --clearDerrick Stolee, Jan 30, 2018
  12. SZEDER GáborFeb 2, 2018
  13. 11/14 commit: integrate commit graph with commit parsingDerrick Stolee, Jan 30, 2018
  14. Jonathan TanFeb 2, 2018
  15. Derrick StoleeFeb 6, 2018
  16. 13/14 commit-graph: close under reachabilityDerrick Stolee, Jan 30, 2018
  17. 09/14 commit-graph: teach git-commit-graph --delete-expiredDerrick Stolee, Jan 30, 2018
  18. SZEDER GáborFeb 2, 2018
  19. 12/14 commit-graph: read only from specific pack-indexesDerrick Stolee, Jan 30, 2018
  20. 14/14 commit-graph: build graph from starting commitsDerrick Stolee, Jan 30, 2018
  21. 10/14 commit-graph: add core.commitgraph settingDerrick Stolee, Jan 30, 2018
  22. Igor DjordjevicJan 31, 2018
  23. SZEDER GáborFeb 2, 2018
  24. 02/14 graph: add commit graph design documentDerrick Stolee, Jan 30, 2018
  25. Stefan BellerJan 31, 2018
  26. 05/14 commit-graph: implement git-commit-graph --writeDerrick Stolee, Jan 30, 2018
  27. Jonathan TanFeb 1, 2018
  28. Stefan BellerFeb 2, 2018
  29. Junio C HamanoFeb 2, 2018
  30. Derrick StoleeFeb 3, 2018
  31. Jeff KingFeb 3, 2018
  32. Junio C HamanoFeb 5, 2018
  33. Derrick StoleeFeb 6, 2018
  34. SZEDER GáborFeb 1, 2018
  35. Derrick StoleeFeb 5, 2018
  36. SZEDER GáborFeb 2, 2018
  37. 07/14 commit-graph: implement git-commit-graph --update-headDerrick Stolee, Jan 30, 2018
  38. SZEDER GáborFeb 2, 2018
  39. Derrick StoleeFeb 5, 2018
  40. SZEDER GáborFeb 2, 2018
  41. 04/14 commit-graph: implement construct_commit_graph()Derrick Stolee, Jan 30, 2018
  42. Jonathan TanFeb 1, 2018
  43. SZEDER GáborFeb 1, 2018
  44. SZEDER GáborFeb 2, 2018
  45. Derrick StoleeFeb 5, 2018
  46. SZEDER GáborFeb 7, 2018
  47. Derrick StoleeFeb 7, 2018
  48. Stefan BellerJan 30, 2018
  49. Stefan BellerFeb 1, 2018
  50. 00/14 Serialized Git Commit GraphDerrick Stolee, Feb 8, 2018
  51. 01/14 commit-graph: add format documentDerrick Stolee, Feb 8, 2018
  52. Junio C HamanoFeb 8, 2018
  53. Derrick StoleeFeb 8, 2018
  54. Junio C HamanoFeb 8, 2018
  55. 03/14 commit-graph: create git-commit-graph builtinDerrick Stolee, Feb 8, 2018
  56. Junio C HamanoFeb 8, 2018
  57. Derrick StoleeFeb 8, 2018
  58. Junio C HamanoFeb 8, 2018
  59. 07/14 commit-graph: update graph-head during writeDerrick Stolee, Feb 8, 2018
  60. Junio C HamanoFeb 12, 2018
  61. Junio C HamanoFeb 12, 2018
  62. Derrick StoleeFeb 12, 2018
  63. Jonathan TanFeb 13, 2018
  64. 10/14 commit-graph: add core.commitGraph settingDerrick Stolee, Feb 8, 2018
  65. 12/14 commit-graph: close under reachabilityDerrick Stolee, Feb 8, 2018
  66. 11/14 commit: integrate commit graph with commit parsingDerrick Stolee, Feb 8, 2018
  67. Jonathan TanFeb 14, 2018
  68. Derrick StoleeFeb 14, 2018
  69. Junio C HamanoFeb 15, 2018
  70. 05/14 commit-graph: implement 'git-commit-graph write'Derrick Stolee, Feb 8, 2018
  71. Jonathan TanFeb 13, 2018
  72. 08/14 commit-graph: implement 'git-commit-graph clear'Derrick Stolee, Feb 8, 2018
  73. Jonathan TanFeb 13, 2018
  74. 09/14 commit-graph: implement --delete-expiredDerrick Stolee, Feb 8, 2018
  75. 13/14 commit-graph: read only from specific pack-indexesDerrick Stolee, Feb 8, 2018
  76. 14/14 commit-graph: build graph from starting commitsDerrick Stolee, Feb 8, 2018
  77. SZEDER GáborFeb 9, 2018
  78. Derrick StoleeFeb 9, 2018
  79. 06/14 commit-graph: implement 'git-commit-graph read'Derrick Stolee, Feb 8, 2018
  80. Junio C HamanoFeb 8, 2018
  81. 02/14 graph: add commit graph design documentDerrick Stolee, Feb 8, 2018
  82. 04/14 commit-graph: implement write_commit_graph()Derrick Stolee, Feb 8, 2018
  83. Junio C HamanoFeb 8, 2018
  84. Junio C HamanoFeb 15, 2018
  85. Derrick StoleeFeb 15, 2018
  86. Derrick StoleeFeb 14, 2018
  87. Stefan BellerFeb 14, 2018
  88. Derrick StoleeFeb 14, 2018
  89. 00/13 Serialized Git Commit GraphDerrick Stolee, Feb 19, 2018
  90. 01/13 commit-graph: add format documentDerrick Stolee, Feb 19, 2018
  91. Junio C HamanoFeb 20, 2018
  92. Stefan BellerFeb 21, 2018
  93. Derrick StoleeFeb 21, 2018
  94. Stefan BellerFeb 21, 2018
  95. Jakub NarebskiMar 30, 2018
  96. Derrick StoleeApr 2, 2018
  97. Jakub NarebskiApr 2, 2018
  98. 03/13 commit-graph: create git-commit-graph builtinDerrick Stolee, Feb 19, 2018
  99. Junio C HamanoFeb 20, 2018
  100. Junio C HamanoFeb 21, 2018
  101. Derrick StoleeFeb 23, 2018
  102. SZEDER GáborFeb 26, 2018
  103. Derrick StoleeFeb 26, 2018
  104. 06/13 commit-graph: implement git commit-graph readDerrick Stolee, Feb 19, 2018
  105. Junio C HamanoFeb 21, 2018
  106. Junio C HamanoFeb 22, 2018
  107. 08/13 commit-graph: implement --delete-expiredDerrick Stolee, Feb 19, 2018
  108. Stefan BellerFeb 21, 2018
  109. Derrick StoleeFeb 23, 2018
  110. Junio C HamanoFeb 22, 2018
  111. Derrick StoleeFeb 23, 2018
  112. Junio C HamanoFeb 23, 2018
  113. Derrick StoleeFeb 23, 2018
  114. Junio C HamanoFeb 23, 2018
  115. 09/13 commit-graph: add core.commitGraph settingDerrick Stolee, Feb 19, 2018
  116. 10/13 commit-graph: close under reachabilityDerrick Stolee, Feb 19, 2018
  117. 11/13 commit: integrate commit graph with commit parsingDerrick Stolee, Feb 19, 2018
  118. 12/13 commit-graph: read only from specific pack-indexesDerrick Stolee, Feb 19, 2018
  119. Stefan BellerFeb 21, 2018
  120. Derrick StoleeFeb 23, 2018
  121. 13/13 commit-graph: build graph from starting commitsDerrick Stolee, Feb 19, 2018
  122. 07/13 commit-graph: implement --set-latestDerrick Stolee, Feb 19, 2018
  123. Junio C HamanoFeb 22, 2018
  124. Derrick StoleeFeb 23, 2018
  125. 02/13 graph: add commit graph design documentDerrick Stolee, Feb 19, 2018
  126. Junio C HamanoFeb 20, 2018
  127. Derrick StoleeFeb 23, 2018
  128. Stefan BellerFeb 21, 2018
  129. 04/13 commit-graph: implement write_commit_graph()Derrick Stolee, Feb 19, 2018
  130. Junio C HamanoFeb 20, 2018
  131. Derrick StoleeFeb 23, 2018
  132. Junio C HamanoFeb 23, 2018
  133. Junio C HamanoFeb 23, 2018
  134. Derrick StoleeFeb 23, 2018
  135. SZEDER GáborFeb 26, 2018
  136. Junio C HamanoFeb 28, 2018
  137. 05/13 commit-graph: implement 'git-commit-graph write'Derrick Stolee, Feb 19, 2018
  138. Junio C HamanoFeb 21, 2018
  139. Jakub NarebskiMar 30, 2018
  140. Derrick StoleeApr 2, 2018
  141. Jakub NarebskiApr 2, 2018
  142. Derrick StoleeApr 2, 2018
  143. Stefan BellerApr 2, 2018
  144. Derrick StoleeApr 2, 2018
  145. Stefan BellerApr 2, 2018
  146. Jakub NarebskiApr 7, 2018

Read the whole thread, see it on lore, or plain text.

$ cat FOOTERMessages come from the public archive at lore.kernel.org/git, fetched every hour. The front page is chosen and written each morning by an AI editor and can be wrong; the threads themselves are the record. About and API. For agents: an MCP server at https://gitlist.dev/mcp, and any thread, story or person page as Markdown by adding .md to its URL (or sending Accept: text/markdown). Details in /llms.txt.