git/list[1] front-page[2] threads[3] people[4] search[5] about
 

[PATCH 06/17] midx: add entries to write_midx_context

From
Derrick Stolee via GitGitGadget <gitgitgadget@gmail.com>
Date
Jan 26, 2021, 16:01 UTC
Message-ID
<79f479ef7d16bd5fd4be058eed5fe9d06291fd65.1611676886.git.gitgitgadget@gmail.com>
In-Reply-To
<pull.848.git.1611676886.gitgitgadget@gmail.com>
From: Derrick Stolee <dstolee@microsoft.com>

In an effort to align write_midx_internal() with the chunk-format API, continue to group necessary data into "struct write_midx_context". This change collects the "struct pack_midx_entry *entries" list and its count into the context.

Update write_midx_oid_fanout() and write_midx_oid_lookup() to take the context directly, as these are easy conversions with this new data.

Only the callers of write_midx_object_offsets() and write_midx_large_offsets() are updated here, since additional data in the context before those methods can match chunk_write_fn.

Signed-off-by: Derrick Stolee <dstolee@microsoft.com>
---
 midx.c | 49 ++++++++++++++++++++++++++-----------------------
 1 file changed, 26 insertions(+), 23 deletions(-)
diff --git a/midx.c b/midx.c
index f348a70e018..ee6f3504c6a 100644
--- a/midx.c
+++ b/midx.c
@@ -458,6 +458,9 @@ struct write_midx_context {
 	struct multi_pack_index *m;
 	struct progress *progress;
 	unsigned pack_paths_checked;
+
+	struct pack_midx_entry *entries;
+	uint32_t entries_nr;
 };
 
 static void add_pack_to_midx(const char *full_path, size_t full_path_len,
@@ -678,11 +681,11 @@ static size_t write_midx_pack_names(struct hashfile *f, void *data)
 }
 
 static size_t write_midx_oid_fanout(struct hashfile *f,
-				    struct pack_midx_entry *objects,
-				    uint32_t nr_objects)
+				    void *data)
 {
-	struct pack_midx_entry *list = objects;
-	struct pack_midx_entry *last = objects + nr_objects;
+	struct write_midx_context *ctx = (struct write_midx_context *)data;
+	struct pack_midx_entry *list = ctx->entries;
+	struct pack_midx_entry *last = ctx->entries + ctx->entries_nr;
 	uint32_t count = 0;
 	uint32_t i;
 
@@ -706,18 +709,19 @@ static size_t write_midx_oid_fanout(struct hashfile *f,
 	return MIDX_CHUNK_FANOUT_SIZE;
 }
 
-static size_t write_midx_oid_lookup(struct hashfile *f, unsigned char hash_len,
-				    struct pack_midx_entry *objects,
-				    uint32_t nr_objects)
+static size_t write_midx_oid_lookup(struct hashfile *f,
+				    void *data)
 {
-	struct pack_midx_entry *list = objects;
+	struct write_midx_context *ctx = (struct write_midx_context *)data;
+	unsigned char hash_len = the_hash_algo->rawsz;
+	struct pack_midx_entry *list = ctx->entries;
 	uint32_t i;
 	size_t written = 0;
 
-	for (i = 0; i < nr_objects; i++) {
+	for (i = 0; i < ctx->entries_nr; i++) {
 		struct pack_midx_entry *obj = list++;
 
-		if (i < nr_objects - 1) {
+		if (i < ctx->entries_nr - 1) {
 			struct pack_midx_entry *next = list;
 			if (oidcmp(&obj->oid, &next->oid) >= 0)
 				BUG("OIDs not in order: %s >= %s",
@@ -805,8 +809,7 @@ static int write_midx_internal(const char *object_dir, struct multi_pack_index *
 	uint64_t written = 0;
 	uint32_t chunk_ids[MIDX_MAX_CHUNKS + 1];
 	uint64_t chunk_offsets[MIDX_MAX_CHUNKS + 1];
-	uint32_t nr_entries, num_large_offsets = 0;
-	struct pack_midx_entry *entries = NULL;
+	uint32_t num_large_offsets = 0;
 	struct progress *progress = NULL;
 	int large_offsets_needed = 0;
 	int pack_name_concat_len = 0;
@@ -852,12 +855,12 @@ static int write_midx_internal(const char *object_dir, struct multi_pack_index *
 	if (ctx.m && ctx.nr == ctx.m->num_packs && !packs_to_drop)
 		goto cleanup;
 
-	entries = get_sorted_entries(ctx.m, ctx.info, ctx.nr, &nr_entries);
+	ctx.entries = get_sorted_entries(ctx.m, ctx.info, ctx.nr, &ctx.entries_nr);
 
-	for (i = 0; i < nr_entries; i++) {
-		if (entries[i].offset > 0x7fffffff)
+	for (i = 0; i < ctx.entries_nr; i++) {
+		if (ctx.entries[i].offset > 0x7fffffff)
 			num_large_offsets++;
-		if (entries[i].offset > 0xffffffff)
+		if (ctx.entries[i].offset > 0xffffffff)
 			large_offsets_needed = 1;
 	}
 
@@ -947,10 +950,10 @@ static int write_midx_internal(const char *object_dir, struct multi_pack_index *
 
 	cur_chunk++;
 	chunk_ids[cur_chunk] = MIDX_CHUNKID_OBJECTOFFSETS;
-	chunk_offsets[cur_chunk] = chunk_offsets[cur_chunk - 1] + nr_entries * the_hash_algo->rawsz;
+	chunk_offsets[cur_chunk] = chunk_offsets[cur_chunk - 1] + ctx.entries_nr * the_hash_algo->rawsz;
 
 	cur_chunk++;
-	chunk_offsets[cur_chunk] = chunk_offsets[cur_chunk - 1] + nr_entries * MIDX_CHUNK_OFFSET_WIDTH;
+	chunk_offsets[cur_chunk] = chunk_offsets[cur_chunk - 1] + ctx.entries_nr * MIDX_CHUNK_OFFSET_WIDTH;
 	if (large_offsets_needed) {
 		chunk_ids[cur_chunk] = MIDX_CHUNKID_LARGEOFFSETS;
 
@@ -993,19 +996,19 @@ static int write_midx_internal(const char *object_dir, struct multi_pack_index *
 				break;
 
 			case MIDX_CHUNKID_OIDFANOUT:
-				written += write_midx_oid_fanout(f, entries, nr_entries);
+				written += write_midx_oid_fanout(f, &ctx);
 				break;
 
 			case MIDX_CHUNKID_OIDLOOKUP:
-				written += write_midx_oid_lookup(f, the_hash_algo->rawsz, entries, nr_entries);
+				written += write_midx_oid_lookup(f, &ctx);
 				break;
 
 			case MIDX_CHUNKID_OBJECTOFFSETS:
-				written += write_midx_object_offsets(f, large_offsets_needed, pack_perm, entries, nr_entries);
+				written += write_midx_object_offsets(f, large_offsets_needed, pack_perm, ctx.entries, ctx.entries_nr);
 				break;
 
 			case MIDX_CHUNKID_LARGEOFFSETS:
-				written += write_midx_large_offsets(f, num_large_offsets, entries, nr_entries);
+				written += write_midx_large_offsets(f, num_large_offsets, ctx.entries, ctx.entries_nr);
 				break;
 
 			default:
@@ -1035,7 +1038,7 @@ static int write_midx_internal(const char *object_dir, struct multi_pack_index *
 	}
 
 	free(ctx.info);
-	free(entries);
+	free(ctx.entries);
 	free(pack_perm);
 	free(midx_name);
 	return result;
-- 
gitgitgadget
Previous: Derrick Stolee via GitGitGadgetNext: Derrick Stolee via GitGitGadget
Message 9 of 120 in “Refactor chunk-format into an API”
  1. 00/17 Refactor chunk-format into an APIDerrick Stolee via GitGitGadget, Jan 26, 2021
  2. 02/17 chunk-format: create chunk format write APIDerrick Stolee via GitGitGadget, Jan 26, 2021
  3. Taylor BlauJan 27, 2021
  4. Derrick StoleeJan 27, 2021
  5. 01/17 commit-graph: anonymize data in chunk_write_fnDerrick Stolee via GitGitGadget, Jan 26, 2021
  6. Chris TorekJan 27, 2021
  7. Taylor BlauJan 27, 2021
  8. 05/17 midx: use context in write_midx_pack_names()Derrick Stolee via GitGitGadget, Jan 26, 2021
  9. 06/17 midx: add entries to write_midx_contextDerrick Stolee via GitGitGadget, Jan 26, 2021
  10. 04/17 midx: rename pack_info to write_midx_contextDerrick Stolee via GitGitGadget, Jan 26, 2021
  11. Taylor BlauJan 27, 2021
  12. 03/17 commit-graph: use chunk-format write APIDerrick Stolee via GitGitGadget, Jan 26, 2021
  13. Taylor BlauJan 27, 2021
  14. 14/17 midx: use chunk-format read APIDerrick Stolee via GitGitGadget, Jan 26, 2021
  15. Taylor BlauJan 27, 2021
  16. Derrick StoleeJan 27, 2021
  17. 17/17 chunk-format: add technical docsDerrick Stolee via GitGitGadget, Jan 26, 2021
  18. 11/17 midx: use chunk-format API in write_midx_internal()Derrick Stolee via GitGitGadget, Jan 26, 2021
  19. 16/17 chunk-format: restore duplicate chunk checksDerrick Stolee via GitGitGadget, Jan 26, 2021
  20. 15/17 midx: use 64-bit multiplication for chunk sizesDerrick Stolee via GitGitGadget, Jan 26, 2021
  21. 07/17 midx: add pack_perm to write_midx_contextDerrick Stolee via GitGitGadget, Jan 26, 2021
  22. 10/17 midx: drop chunk progress during writeDerrick Stolee via GitGitGadget, Jan 26, 2021
  23. 09/17 midx: return success/failure in chunk write methodsDerrick Stolee via GitGitGadget, Jan 26, 2021
  24. 08/17 midx: add num_large_offsets to write_midx_contextDerrick Stolee via GitGitGadget, Jan 26, 2021
  25. 13/17 commit-graph: use chunk-format read APIDerrick Stolee via GitGitGadget, Jan 26, 2021
  26. 12/17 chunk-format: create read chunk APIDerrick Stolee via GitGitGadget, Jan 26, 2021
  27. Taylor BlauJan 27, 2021
  28. Taylor BlauJan 27, 2021
  29. Junio C HamanoJan 26, 2021
  30. 00/17 Refactor chunk-format into an APIDerrick Stolee via GitGitGadget, Jan 27, 2021
  31. 05/17 midx: use context in write_midx_pack_names()Derrick Stolee via GitGitGadget, Jan 27, 2021
  32. 03/17 commit-graph: use chunk-format write APIDerrick Stolee via GitGitGadget, Jan 27, 2021
  33. 07/17 midx: add pack_perm to write_midx_contextDerrick Stolee via GitGitGadget, Jan 27, 2021
  34. 02/17 chunk-format: create chunk format write APIDerrick Stolee via GitGitGadget, Jan 27, 2021
  35. Junio C HamanoFeb 4, 2021
  36. Junio C HamanoFeb 4, 2021
  37. Derrick StoleeFeb 5, 2021
  38. Junio C HamanoFeb 5, 2021
  39. 15/17 midx: use 64-bit multiplication for chunk sizesDerrick Stolee via GitGitGadget, Jan 27, 2021
  40. Junio C HamanoFeb 5, 2021
  41. Chris TorekFeb 5, 2021
  42. Junio C HamanoFeb 5, 2021
  43. Chris TorekFeb 6, 2021
  44. Derrick StoleeFeb 5, 2021
  45. Junio C HamanoFeb 5, 2021
  46. SZEDER GáborFeb 7, 2021
  47. Junio C HamanoFeb 8, 2021
  48. 17/17 chunk-format: add technical docsDerrick Stolee via GitGitGadget, Jan 27, 2021
  49. Junio C HamanoFeb 5, 2021
  50. 14/17 midx: use chunk-format read APIDerrick Stolee via GitGitGadget, Jan 27, 2021
  51. 13/17 commit-graph: use chunk-format read APIDerrick Stolee via GitGitGadget, Jan 27, 2021
  52. 16/17 chunk-format: restore duplicate chunk checksDerrick Stolee via GitGitGadget, Jan 27, 2021
  53. Junio C HamanoFeb 5, 2021
  54. Derrick StoleeFeb 5, 2021
  55. 11/17 midx: use chunk-format API in write_midx_internal()Derrick Stolee via GitGitGadget, Jan 27, 2021
  56. 12/17 chunk-format: create read chunk APIDerrick Stolee via GitGitGadget, Jan 27, 2021
  57. Junio C HamanoFeb 4, 2021
  58. Derrick StoleeFeb 5, 2021
  59. Junio C HamanoFeb 5, 2021
  60. Junio C HamanoFeb 8, 2021
  61. Derrick StoleeFeb 9, 2021
  62. Junio C HamanoFeb 9, 2021
  63. 09/17 midx: return success/failure in chunk write methodsDerrick Stolee via GitGitGadget, Jan 27, 2021
  64. Junio C HamanoFeb 4, 2021
  65. Derrick StoleeFeb 5, 2021
  66. 10/17 midx: drop chunk progress during writeDerrick Stolee via GitGitGadget, Jan 27, 2021
  67. 06/17 midx: add entries to write_midx_contextDerrick Stolee via GitGitGadget, Jan 27, 2021
  68. 08/17 midx: add num_large_offsets to write_midx_contextDerrick Stolee via GitGitGadget, Jan 27, 2021
  69. 04/17 midx: rename pack_info to write_midx_contextDerrick Stolee via GitGitGadget, Jan 27, 2021
  70. 01/17 commit-graph: anonymize data in chunk_write_fnDerrick Stolee via GitGitGadget, Jan 27, 2021
  71. Taylor BlauJan 27, 2021
  72. Junio C HamanoFeb 5, 2021
  73. Derrick StoleeFeb 5, 2021
  74. 00/17 Refactor chunk-format into an APIDerrick Stolee via GitGitGadget, Feb 5, 2021
  75. 10/17 midx: drop chunk progress during writeDerrick Stolee via GitGitGadget, Feb 5, 2021
  76. 17/17 chunk-format: add technical docsDerrick Stolee via GitGitGadget, Feb 5, 2021
  77. 09/17 midx: return success/failure in chunk write methodsDerrick Stolee via GitGitGadget, Feb 5, 2021
  78. 15/17 midx: use 64-bit multiplication for chunk sizesDerrick Stolee via GitGitGadget, Feb 5, 2021
  79. 02/17 chunk-format: create chunk format write APIDerrick Stolee via GitGitGadget, Feb 5, 2021
  80. SZEDER GáborFeb 7, 2021
  81. Derrick StoleeFeb 8, 2021
  82. SZEDER GáborFeb 11, 2021
  83. 08/17 midx: add num_large_offsets to write_midx_contextDerrick Stolee via GitGitGadget, Feb 5, 2021
  84. 05/17 midx: use context in write_midx_pack_names()Derrick Stolee via GitGitGadget, Feb 5, 2021
  85. 14/17 midx: use chunk-format read APIDerrick Stolee via GitGitGadget, Feb 5, 2021
  86. 07/17 midx: add pack_perm to write_midx_contextDerrick Stolee via GitGitGadget, Feb 5, 2021
  87. 06/17 midx: add entries to write_midx_contextDerrick Stolee via GitGitGadget, Feb 5, 2021
  88. 16/17 chunk-format: restore duplicate chunk checksDerrick Stolee via GitGitGadget, Feb 5, 2021
  89. 13/17 commit-graph: use chunk-format read APIDerrick Stolee via GitGitGadget, Feb 5, 2021
  90. 01/17 commit-graph: anonymize data in chunk_write_fnDerrick Stolee via GitGitGadget, Feb 5, 2021
  91. 03/17 commit-graph: use chunk-format write APIDerrick Stolee via GitGitGadget, Feb 5, 2021
  92. 04/17 midx: rename pack_info to write_midx_contextDerrick Stolee via GitGitGadget, Feb 5, 2021
  93. 11/17 midx: use chunk-format API in write_midx_internal()Derrick Stolee via GitGitGadget, Feb 5, 2021
  94. 12/17 chunk-format: create read chunk APIDerrick Stolee via GitGitGadget, Feb 5, 2021
  95. SZEDER GáborFeb 7, 2021
  96. Derrick StoleeFeb 8, 2021
  97. 00/17 Refactor chunk-format into an APIDerrick Stolee via GitGitGadget, Feb 18, 2021
  98. 07/17 midx: add pack_perm to write_midx_contextDerrick Stolee via GitGitGadget, Feb 18, 2021
  99. 05/17 midx: use context in write_midx_pack_names()Derrick Stolee via GitGitGadget, Feb 18, 2021
  100. 03/17 commit-graph: use chunk-format write APIDerrick Stolee via GitGitGadget, Feb 18, 2021
  101. SZEDER GáborFeb 24, 2021
  102. Taylor BlauFeb 24, 2021
  103. Derrick StoleeFeb 24, 2021
  104. Junio C HamanoFeb 24, 2021
  105. 04/17 midx: rename pack_info to write_midx_contextDerrick Stolee via GitGitGadget, Feb 18, 2021
  106. 06/17 midx: add entries to write_midx_contextDerrick Stolee via GitGitGadget, Feb 18, 2021
  107. 02/17 chunk-format: create chunk format write APIDerrick Stolee via GitGitGadget, Feb 18, 2021
  108. 01/17 commit-graph: anonymize data in chunk_write_fnDerrick Stolee via GitGitGadget, Feb 18, 2021
  109. 08/17 midx: add num_large_offsets to write_midx_contextDerrick Stolee via GitGitGadget, Feb 18, 2021
  110. 09/17 midx: return success/failure in chunk write methodsDerrick Stolee via GitGitGadget, Feb 18, 2021
  111. 10/17 midx: drop chunk progress during writeDerrick Stolee via GitGitGadget, Feb 18, 2021
  112. 11/17 midx: use chunk-format API in write_midx_internal()Derrick Stolee via GitGitGadget, Feb 18, 2021
  113. 12/17 chunk-format: create read chunk APIDerrick Stolee via GitGitGadget, Feb 18, 2021
  114. 17/17 chunk-format: add technical docsDerrick Stolee via GitGitGadget, Feb 18, 2021
  115. Junio C HamanoFeb 18, 2021
  116. Derrick StoleeFeb 19, 2021
  117. 15/17 midx: use 64-bit multiplication for chunk sizesDerrick Stolee via GitGitGadget, Feb 18, 2021
  118. 14/17 midx: use chunk-format read APIDerrick Stolee via GitGitGadget, Feb 18, 2021
  119. 13/17 commit-graph: use chunk-format read APIDerrick Stolee via GitGitGadget, Feb 18, 2021
  120. 16/17 chunk-format: restore duplicate chunk checksDerrick Stolee via GitGitGadget, Feb 18, 2021

Read the whole thread, see it on lore, or plain text.

$ cat FOOTERMessages come from the public archive at lore.kernel.org/git, fetched every hour. The front page is chosen and written each morning by an AI editor and can be wrong; the threads themselves are the record. About and API. For agents: an MCP server at https://gitlist.dev/mcp, and any thread, story or person page as Markdown by adding .md to its URL (or sending Accept: text/markdown). Details in /llms.txt.