git/list[1] front-page[2] threads[3] people[4] search[5] about
 

[PATCH v3 07/17] midx: add pack_perm to write_midx_context

From
Derrick Stolee via GitGitGadget <gitgitgadget@gmail.com>
Date
Feb 5, 2021, 14:30 UTC
Message-ID
<b283a38fb775614280cb1330029aafcd8ae8dfe8.1612535453.git.gitgitgadget@gmail.com>
In-Reply-To
<pull.848.v3.git.1612535452.gitgitgadget@gmail.com>
From: Derrick Stolee <dstolee@microsoft.com>

In an effort to align write_midx_internal() with the chunk-format API, continue to group necessary data into "struct write_midx_context". This change collects the "uint32_t *pack_perm" and large_offsets_needed bit into the context.

Update write_midx_object_offsets() to match chunk_write_fn.
Signed-off-by: Derrick Stolee <dstolee@microsoft.com>
---
 midx.c | 40 +++++++++++++++++++++-------------------
 1 file changed, 21 insertions(+), 19 deletions(-)
diff --git a/midx.c b/midx.c
index 4520ef82b91b..cd994e333ecb 100644
--- a/midx.c
+++ b/midx.c
@@ -461,6 +461,9 @@ struct write_midx_context {
 
 	struct pack_midx_entry *entries;
 	uint32_t entries_nr;
+
+	uint32_t *pack_perm;
+	unsigned large_offsets_needed:1;
 };
 
 static void add_pack_to_midx(const char *full_path, size_t full_path_len,
@@ -736,27 +739,27 @@ static size_t write_midx_oid_lookup(struct hashfile *f,
 	return written;
 }
 
-static size_t write_midx_object_offsets(struct hashfile *f, int large_offset_needed,
-					uint32_t *perm,
-					struct pack_midx_entry *objects, uint32_t nr_objects)
+static size_t write_midx_object_offsets(struct hashfile *f,
+					void *data)
 {
-	struct pack_midx_entry *list = objects;
+	struct write_midx_context *ctx = data;
+	struct pack_midx_entry *list = ctx->entries;
 	uint32_t i, nr_large_offset = 0;
 	size_t written = 0;
 
-	for (i = 0; i < nr_objects; i++) {
+	for (i = 0; i < ctx->entries_nr; i++) {
 		struct pack_midx_entry *obj = list++;
 
-		if (perm[obj->pack_int_id] == PACK_EXPIRED)
+		if (ctx->pack_perm[obj->pack_int_id] == PACK_EXPIRED)
 			BUG("object %s is in an expired pack with int-id %d",
 			    oid_to_hex(&obj->oid),
 			    obj->pack_int_id);
 
-		hashwrite_be32(f, perm[obj->pack_int_id]);
+		hashwrite_be32(f, ctx->pack_perm[obj->pack_int_id]);
 
-		if (large_offset_needed && obj->offset >> 31)
+		if (ctx->large_offsets_needed && obj->offset >> 31)
 			hashwrite_be32(f, MIDX_LARGE_OFFSET_NEEDED | nr_large_offset++);
-		else if (!large_offset_needed && obj->offset >> 32)
+		else if (!ctx->large_offsets_needed && obj->offset >> 32)
 			BUG("object %s requires a large offset (%"PRIx64") but the MIDX is not writing large offsets!",
 			    oid_to_hex(&obj->oid),
 			    obj->offset);
@@ -805,13 +808,11 @@ static int write_midx_internal(const char *object_dir, struct multi_pack_index *
 	struct hashfile *f = NULL;
 	struct lock_file lk;
 	struct write_midx_context ctx = { 0 };
-	uint32_t *pack_perm = NULL;
 	uint64_t written = 0;
 	uint32_t chunk_ids[MIDX_MAX_CHUNKS + 1];
 	uint64_t chunk_offsets[MIDX_MAX_CHUNKS + 1];
 	uint32_t num_large_offsets = 0;
 	struct progress *progress = NULL;
-	int large_offsets_needed = 0;
 	int pack_name_concat_len = 0;
 	int dropped_packs = 0;
 	int result = 0;
@@ -857,11 +858,12 @@ static int write_midx_internal(const char *object_dir, struct multi_pack_index *
 
 	ctx.entries = get_sorted_entries(ctx.m, ctx.info, ctx.nr, &ctx.entries_nr);
 
+	ctx.large_offsets_needed = 0;
 	for (i = 0; i < ctx.entries_nr; i++) {
 		if (ctx.entries[i].offset > 0x7fffffff)
 			num_large_offsets++;
 		if (ctx.entries[i].offset > 0xffffffff)
-			large_offsets_needed = 1;
+			ctx.large_offsets_needed = 1;
 	}
 
 	QSORT(ctx.info, ctx.nr, pack_info_compare);
@@ -900,13 +902,13 @@ static int write_midx_internal(const char *object_dir, struct multi_pack_index *
 	 *
 	 * pack_perm[old_id] = new_id
 	 */
-	ALLOC_ARRAY(pack_perm, ctx.nr);
+	ALLOC_ARRAY(ctx.pack_perm, ctx.nr);
 	for (i = 0; i < ctx.nr; i++) {
 		if (ctx.info[i].expired) {
 			dropped_packs++;
-			pack_perm[ctx.info[i].orig_pack_int_id] = PACK_EXPIRED;
+			ctx.pack_perm[ctx.info[i].orig_pack_int_id] = PACK_EXPIRED;
 		} else {
-			pack_perm[ctx.info[i].orig_pack_int_id] = i - dropped_packs;
+			ctx.pack_perm[ctx.info[i].orig_pack_int_id] = i - dropped_packs;
 		}
 	}
 
@@ -927,7 +929,7 @@ static int write_midx_internal(const char *object_dir, struct multi_pack_index *
 		close_midx(ctx.m);
 
 	cur_chunk = 0;
-	num_chunks = large_offsets_needed ? 5 : 4;
+	num_chunks = ctx.large_offsets_needed ? 5 : 4;
 
 	if (ctx.nr - dropped_packs == 0) {
 		error(_("no pack files to index."));
@@ -954,7 +956,7 @@ static int write_midx_internal(const char *object_dir, struct multi_pack_index *
 
 	cur_chunk++;
 	chunk_offsets[cur_chunk] = chunk_offsets[cur_chunk - 1] + ctx.entries_nr * MIDX_CHUNK_OFFSET_WIDTH;
-	if (large_offsets_needed) {
+	if (ctx.large_offsets_needed) {
 		chunk_ids[cur_chunk] = MIDX_CHUNKID_LARGEOFFSETS;
 
 		cur_chunk++;
@@ -1004,7 +1006,7 @@ static int write_midx_internal(const char *object_dir, struct multi_pack_index *
 				break;
 
 			case MIDX_CHUNKID_OBJECTOFFSETS:
-				written += write_midx_object_offsets(f, large_offsets_needed, pack_perm, ctx.entries, ctx.entries_nr);
+				written += write_midx_object_offsets(f, &ctx);
 				break;
 
 			case MIDX_CHUNKID_LARGEOFFSETS:
@@ -1039,7 +1041,7 @@ static int write_midx_internal(const char *object_dir, struct multi_pack_index *
 
 	free(ctx.info);
 	free(ctx.entries);
-	free(pack_perm);
+	free(ctx.pack_perm);
 	free(midx_name);
 	return result;
 }
-- 
gitgitgadget
Previous: Derrick Stolee via GitGitGadgetNext: Derrick Stolee via GitGitGadget
Message 86 of 120 in “Refactor chunk-format into an API”
  1. 00/17 Refactor chunk-format into an APIDerrick Stolee via GitGitGadget, Jan 26, 2021
  2. 02/17 chunk-format: create chunk format write APIDerrick Stolee via GitGitGadget, Jan 26, 2021
  3. Taylor BlauJan 27, 2021
  4. Derrick StoleeJan 27, 2021
  5. 01/17 commit-graph: anonymize data in chunk_write_fnDerrick Stolee via GitGitGadget, Jan 26, 2021
  6. Chris TorekJan 27, 2021
  7. Taylor BlauJan 27, 2021
  8. 05/17 midx: use context in write_midx_pack_names()Derrick Stolee via GitGitGadget, Jan 26, 2021
  9. 06/17 midx: add entries to write_midx_contextDerrick Stolee via GitGitGadget, Jan 26, 2021
  10. 04/17 midx: rename pack_info to write_midx_contextDerrick Stolee via GitGitGadget, Jan 26, 2021
  11. Taylor BlauJan 27, 2021
  12. 03/17 commit-graph: use chunk-format write APIDerrick Stolee via GitGitGadget, Jan 26, 2021
  13. Taylor BlauJan 27, 2021
  14. 14/17 midx: use chunk-format read APIDerrick Stolee via GitGitGadget, Jan 26, 2021
  15. Taylor BlauJan 27, 2021
  16. Derrick StoleeJan 27, 2021
  17. 17/17 chunk-format: add technical docsDerrick Stolee via GitGitGadget, Jan 26, 2021
  18. 11/17 midx: use chunk-format API in write_midx_internal()Derrick Stolee via GitGitGadget, Jan 26, 2021
  19. 16/17 chunk-format: restore duplicate chunk checksDerrick Stolee via GitGitGadget, Jan 26, 2021
  20. 15/17 midx: use 64-bit multiplication for chunk sizesDerrick Stolee via GitGitGadget, Jan 26, 2021
  21. 07/17 midx: add pack_perm to write_midx_contextDerrick Stolee via GitGitGadget, Jan 26, 2021
  22. 10/17 midx: drop chunk progress during writeDerrick Stolee via GitGitGadget, Jan 26, 2021
  23. 09/17 midx: return success/failure in chunk write methodsDerrick Stolee via GitGitGadget, Jan 26, 2021
  24. 08/17 midx: add num_large_offsets to write_midx_contextDerrick Stolee via GitGitGadget, Jan 26, 2021
  25. 13/17 commit-graph: use chunk-format read APIDerrick Stolee via GitGitGadget, Jan 26, 2021
  26. 12/17 chunk-format: create read chunk APIDerrick Stolee via GitGitGadget, Jan 26, 2021
  27. Taylor BlauJan 27, 2021
  28. Taylor BlauJan 27, 2021
  29. Junio C HamanoJan 26, 2021
  30. 00/17 Refactor chunk-format into an APIDerrick Stolee via GitGitGadget, Jan 27, 2021
  31. 05/17 midx: use context in write_midx_pack_names()Derrick Stolee via GitGitGadget, Jan 27, 2021
  32. 03/17 commit-graph: use chunk-format write APIDerrick Stolee via GitGitGadget, Jan 27, 2021
  33. 07/17 midx: add pack_perm to write_midx_contextDerrick Stolee via GitGitGadget, Jan 27, 2021
  34. 02/17 chunk-format: create chunk format write APIDerrick Stolee via GitGitGadget, Jan 27, 2021
  35. Junio C HamanoFeb 4, 2021
  36. Junio C HamanoFeb 4, 2021
  37. Derrick StoleeFeb 5, 2021
  38. Junio C HamanoFeb 5, 2021
  39. 15/17 midx: use 64-bit multiplication for chunk sizesDerrick Stolee via GitGitGadget, Jan 27, 2021
  40. Junio C HamanoFeb 5, 2021
  41. Chris TorekFeb 5, 2021
  42. Junio C HamanoFeb 5, 2021
  43. Chris TorekFeb 6, 2021
  44. Derrick StoleeFeb 5, 2021
  45. Junio C HamanoFeb 5, 2021
  46. SZEDER GáborFeb 7, 2021
  47. Junio C HamanoFeb 8, 2021
  48. 17/17 chunk-format: add technical docsDerrick Stolee via GitGitGadget, Jan 27, 2021
  49. Junio C HamanoFeb 5, 2021
  50. 14/17 midx: use chunk-format read APIDerrick Stolee via GitGitGadget, Jan 27, 2021
  51. 13/17 commit-graph: use chunk-format read APIDerrick Stolee via GitGitGadget, Jan 27, 2021
  52. 16/17 chunk-format: restore duplicate chunk checksDerrick Stolee via GitGitGadget, Jan 27, 2021
  53. Junio C HamanoFeb 5, 2021
  54. Derrick StoleeFeb 5, 2021
  55. 11/17 midx: use chunk-format API in write_midx_internal()Derrick Stolee via GitGitGadget, Jan 27, 2021
  56. 12/17 chunk-format: create read chunk APIDerrick Stolee via GitGitGadget, Jan 27, 2021
  57. Junio C HamanoFeb 4, 2021
  58. Derrick StoleeFeb 5, 2021
  59. Junio C HamanoFeb 5, 2021
  60. Junio C HamanoFeb 8, 2021
  61. Derrick StoleeFeb 9, 2021
  62. Junio C HamanoFeb 9, 2021
  63. 09/17 midx: return success/failure in chunk write methodsDerrick Stolee via GitGitGadget, Jan 27, 2021
  64. Junio C HamanoFeb 4, 2021
  65. Derrick StoleeFeb 5, 2021
  66. 10/17 midx: drop chunk progress during writeDerrick Stolee via GitGitGadget, Jan 27, 2021
  67. 06/17 midx: add entries to write_midx_contextDerrick Stolee via GitGitGadget, Jan 27, 2021
  68. 08/17 midx: add num_large_offsets to write_midx_contextDerrick Stolee via GitGitGadget, Jan 27, 2021
  69. 04/17 midx: rename pack_info to write_midx_contextDerrick Stolee via GitGitGadget, Jan 27, 2021
  70. 01/17 commit-graph: anonymize data in chunk_write_fnDerrick Stolee via GitGitGadget, Jan 27, 2021
  71. Taylor BlauJan 27, 2021
  72. Junio C HamanoFeb 5, 2021
  73. Derrick StoleeFeb 5, 2021
  74. 00/17 Refactor chunk-format into an APIDerrick Stolee via GitGitGadget, Feb 5, 2021
  75. 10/17 midx: drop chunk progress during writeDerrick Stolee via GitGitGadget, Feb 5, 2021
  76. 17/17 chunk-format: add technical docsDerrick Stolee via GitGitGadget, Feb 5, 2021
  77. 09/17 midx: return success/failure in chunk write methodsDerrick Stolee via GitGitGadget, Feb 5, 2021
  78. 15/17 midx: use 64-bit multiplication for chunk sizesDerrick Stolee via GitGitGadget, Feb 5, 2021
  79. 02/17 chunk-format: create chunk format write APIDerrick Stolee via GitGitGadget, Feb 5, 2021
  80. SZEDER GáborFeb 7, 2021
  81. Derrick StoleeFeb 8, 2021
  82. SZEDER GáborFeb 11, 2021
  83. 08/17 midx: add num_large_offsets to write_midx_contextDerrick Stolee via GitGitGadget, Feb 5, 2021
  84. 05/17 midx: use context in write_midx_pack_names()Derrick Stolee via GitGitGadget, Feb 5, 2021
  85. 14/17 midx: use chunk-format read APIDerrick Stolee via GitGitGadget, Feb 5, 2021
  86. 07/17 midx: add pack_perm to write_midx_contextDerrick Stolee via GitGitGadget, Feb 5, 2021
  87. 06/17 midx: add entries to write_midx_contextDerrick Stolee via GitGitGadget, Feb 5, 2021
  88. 16/17 chunk-format: restore duplicate chunk checksDerrick Stolee via GitGitGadget, Feb 5, 2021
  89. 13/17 commit-graph: use chunk-format read APIDerrick Stolee via GitGitGadget, Feb 5, 2021
  90. 01/17 commit-graph: anonymize data in chunk_write_fnDerrick Stolee via GitGitGadget, Feb 5, 2021
  91. 03/17 commit-graph: use chunk-format write APIDerrick Stolee via GitGitGadget, Feb 5, 2021
  92. 04/17 midx: rename pack_info to write_midx_contextDerrick Stolee via GitGitGadget, Feb 5, 2021
  93. 11/17 midx: use chunk-format API in write_midx_internal()Derrick Stolee via GitGitGadget, Feb 5, 2021
  94. 12/17 chunk-format: create read chunk APIDerrick Stolee via GitGitGadget, Feb 5, 2021
  95. SZEDER GáborFeb 7, 2021
  96. Derrick StoleeFeb 8, 2021
  97. 00/17 Refactor chunk-format into an APIDerrick Stolee via GitGitGadget, Feb 18, 2021
  98. 07/17 midx: add pack_perm to write_midx_contextDerrick Stolee via GitGitGadget, Feb 18, 2021
  99. 05/17 midx: use context in write_midx_pack_names()Derrick Stolee via GitGitGadget, Feb 18, 2021
  100. 03/17 commit-graph: use chunk-format write APIDerrick Stolee via GitGitGadget, Feb 18, 2021
  101. SZEDER GáborFeb 24, 2021
  102. Taylor BlauFeb 24, 2021
  103. Derrick StoleeFeb 24, 2021
  104. Junio C HamanoFeb 24, 2021
  105. 04/17 midx: rename pack_info to write_midx_contextDerrick Stolee via GitGitGadget, Feb 18, 2021
  106. 06/17 midx: add entries to write_midx_contextDerrick Stolee via GitGitGadget, Feb 18, 2021
  107. 02/17 chunk-format: create chunk format write APIDerrick Stolee via GitGitGadget, Feb 18, 2021
  108. 01/17 commit-graph: anonymize data in chunk_write_fnDerrick Stolee via GitGitGadget, Feb 18, 2021
  109. 08/17 midx: add num_large_offsets to write_midx_contextDerrick Stolee via GitGitGadget, Feb 18, 2021
  110. 09/17 midx: return success/failure in chunk write methodsDerrick Stolee via GitGitGadget, Feb 18, 2021
  111. 10/17 midx: drop chunk progress during writeDerrick Stolee via GitGitGadget, Feb 18, 2021
  112. 11/17 midx: use chunk-format API in write_midx_internal()Derrick Stolee via GitGitGadget, Feb 18, 2021
  113. 12/17 chunk-format: create read chunk APIDerrick Stolee via GitGitGadget, Feb 18, 2021
  114. 17/17 chunk-format: add technical docsDerrick Stolee via GitGitGadget, Feb 18, 2021
  115. Junio C HamanoFeb 18, 2021
  116. Derrick StoleeFeb 19, 2021
  117. 15/17 midx: use 64-bit multiplication for chunk sizesDerrick Stolee via GitGitGadget, Feb 18, 2021
  118. 14/17 midx: use chunk-format read APIDerrick Stolee via GitGitGadget, Feb 18, 2021
  119. 13/17 commit-graph: use chunk-format read APIDerrick Stolee via GitGitGadget, Feb 18, 2021
  120. 16/17 chunk-format: restore duplicate chunk checksDerrick Stolee via GitGitGadget, Feb 18, 2021

Read the whole thread, see it on lore, or plain text.

$ cat FOOTERMessages come from the public archive at lore.kernel.org/git, fetched every hour. The front page is chosen and written each morning by an AI editor and can be wrong; the threads themselves are the record. About and API. For agents: an MCP server at https://gitlist.dev/mcp, and any thread, story or person page as Markdown by adding .md to its URL (or sending Accept: text/markdown). Details in /llms.txt.