git/list[1] front-page[2] threads[3] people[4] search[5] about
 

[PATCH 30/30] pack-objects: output debug info about deltas

From
Derrick Stolee via GitGitGadget <gitgitgadget@gmail.com>
Date
Sep 10, 2024, 02:28 UTC
Message-ID
<460feef90fdd869b42e3663a1a1336a8ae663bc0.1725935335.git.gitgitgadget@gmail.com>
In-Reply-To
<pull.1786.git.1725935335.gitgitgadget@gmail.com>
From: Derrick Stolee <stolee@gmail.com>

In order to debug what is going on during delta calculations, add a --debug-file=<file> option to 'git pack-objects'. This leads to sending a JSON-formatted description of the delta information to that file.

Signed-off-by: Derrick Stolee <stolee@gmail.com>
---
 builtin/pack-objects.c | 69 ++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 69 insertions(+)
diff --git a/builtin/pack-objects.c b/builtin/pack-objects.c
index 5d5a57e6b1f..7d1dd5a6557 100644
--- a/builtin/pack-objects.c
+++ b/builtin/pack-objects.c
@@ -50,6 +50,9 @@
  */
 static struct packing_data to_pack;
 
+static FILE *delta_file;
+static int delta_file_nr;
+
 static inline struct object_entry *oe_delta(
 		const struct packing_data *pack,
 		const struct object_entry *e)
@@ -516,6 +519,14 @@ static unsigned long write_no_reuse_object(struct hashfile *f, struct object_ent
 	hdrlen = encode_in_pack_object_header(header, sizeof(header),
 					      type, size);
 
+	if (delta_file) {
+		if (delta_file_nr++)
+			fprintf(delta_file, ",\n");
+		fprintf(delta_file, "\t\t{\n");
+		fprintf(delta_file, "\t\t\t\"oid\" : \"%s\",\n", oid_to_hex(&entry->idx.oid));
+		fprintf(delta_file, "\t\t\t\"size\" : %"PRIuMAX",\n", datalen);
+	}
+
 	if (type == OBJ_OFS_DELTA) {
 		/*
 		 * Deltas with relative base contain an additional
@@ -536,6 +547,11 @@ static unsigned long write_no_reuse_object(struct hashfile *f, struct object_ent
 		hashwrite(f, header, hdrlen);
 		hashwrite(f, dheader + pos, sizeof(dheader) - pos);
 		hdrlen += sizeof(dheader) - pos;
+		if (delta_file) {
+			fprintf(delta_file, "\t\t\t\"delta_type\" : \"OFS\",\n");
+			fprintf(delta_file, "\t\t\t\"offset\" : %"PRIuMAX",\n", ofs);
+			fprintf(delta_file, "\t\t\t\"delta_base\" : \"%s\",\n", oid_to_hex(&DELTA(entry)->idx.oid));
+		}
 	} else if (type == OBJ_REF_DELTA) {
 		/*
 		 * Deltas with a base reference contain
@@ -550,6 +566,10 @@ static unsigned long write_no_reuse_object(struct hashfile *f, struct object_ent
 		hashwrite(f, header, hdrlen);
 		hashwrite(f, DELTA(entry)->idx.oid.hash, hashsz);
 		hdrlen += hashsz;
+		if (delta_file) {
+			fprintf(delta_file, "\t\t\t\"delta_type\" : \"REF\",\n");
+			fprintf(delta_file, "\t\t\t\"delta_base\" : \"%s\",\n", oid_to_hex(&DELTA(entry)->idx.oid));
+		}
 	} else {
 		if (limit && hdrlen + datalen + hashsz >= limit) {
 			if (st)
@@ -559,6 +579,10 @@ static unsigned long write_no_reuse_object(struct hashfile *f, struct object_ent
 		}
 		hashwrite(f, header, hdrlen);
 	}
+
+	if (delta_file)
+		fprintf(delta_file, "\t\t\t\"reused\" : false\n\t\t}");
+
 	if (st) {
 		datalen = write_large_blob_data(st, f, &entry->idx.oid);
 		close_istream(st);
@@ -619,6 +643,14 @@ static off_t write_reuse_object(struct hashfile *f, struct object_entry *entry,
 		return write_no_reuse_object(f, entry, limit, usable_delta);
 	}
 
+	if (delta_file) {
+		if (delta_file_nr++)
+			fprintf(delta_file, ",\n");
+		fprintf(delta_file, "\t\t{\n");
+		fprintf(delta_file, "\t\t\t\"oid\" : \"%s\",\n", oid_to_hex(&entry->idx.oid));
+		fprintf(delta_file, "\t\t\t\"size\" : %"PRIuMAX",\n", entry_size);
+	}
+
 	if (type == OBJ_OFS_DELTA) {
 		off_t ofs = entry->idx.offset - DELTA(entry)->idx.offset;
 		unsigned pos = sizeof(dheader) - 1;
@@ -633,6 +665,12 @@ static off_t write_reuse_object(struct hashfile *f, struct object_entry *entry,
 		hashwrite(f, dheader + pos, sizeof(dheader) - pos);
 		hdrlen += sizeof(dheader) - pos;
 		reused_delta++;
+
+		if (delta_file) {
+			fprintf(delta_file, "\t\t\t\"delta_type\" : \"OFS\",\n");
+			fprintf(delta_file, "\t\t\t\"offset\" : %"PRIuMAX",\n", ofs);
+			fprintf(delta_file, "\t\t\t\"delta_base\" : \"%s\",\n", oid_to_hex(&DELTA(entry)->idx.oid));
+		}
 	} else if (type == OBJ_REF_DELTA) {
 		if (limit && hdrlen + hashsz + datalen + hashsz >= limit) {
 			unuse_pack(&w_curs);
@@ -642,6 +680,10 @@ static off_t write_reuse_object(struct hashfile *f, struct object_entry *entry,
 		hashwrite(f, DELTA(entry)->idx.oid.hash, hashsz);
 		hdrlen += hashsz;
 		reused_delta++;
+		if (delta_file) {
+			fprintf(delta_file, "\t\t\t\"delta_type\" : \"REF\",\n");
+			fprintf(delta_file, "\t\t\t\"delta_base\" : \"%s\",\n", oid_to_hex(&DELTA(entry)->idx.oid));
+		}
 	} else {
 		if (limit && hdrlen + datalen + hashsz >= limit) {
 			unuse_pack(&w_curs);
@@ -652,6 +694,10 @@ static off_t write_reuse_object(struct hashfile *f, struct object_entry *entry,
 	copy_pack_data(f, p, &w_curs, offset, datalen);
 	unuse_pack(&w_curs);
 	reused++;
+
+	if (delta_file)
+		fprintf(delta_file, "\t\t\t\"reused\" : true\n\t\t}");
+
 	return hdrlen + datalen;
 }
 
@@ -1264,6 +1310,11 @@ static void write_pack_file(void)
 	ALLOC_ARRAY(written_list, to_pack.nr_objects);
 	write_order = compute_write_order();
 
+	if (delta_file) {
+		fprintf(delta_file, "{\n\t\"num_objects\" : %"PRIu32",\n", to_pack.nr_objects);
+		fprintf(delta_file, "\t\"objects\" : [\n");
+	}
+
 	do {
 		unsigned char hash[GIT_MAX_RAWSZ];
 		char *pack_tmp_name = NULL;
@@ -1412,6 +1463,9 @@ static void write_pack_file(void)
 		    written, nr_result);
 	trace2_data_intmax("pack-objects", the_repository,
 			   "write_pack_file/wrote", nr_result);
+
+	if (delta_file)
+		fprintf(delta_file, "\n\t]\n}");
 }
 
 static int no_try_delta(const char *path)
@@ -4430,6 +4484,7 @@ int cmd_pack_objects(int argc, const char **argv, const char *prefix)
 	struct string_list keep_pack_list = STRING_LIST_INIT_NODUP;
 	struct list_objects_filter_options filter_options =
 		LIST_OBJECTS_FILTER_INIT;
+	const char *delta_file_name = NULL;
 
 	struct option pack_objects_options[] = {
 		OPT_CALLBACK_F('q', "quiet", &progress, NULL,
@@ -4536,6 +4591,9 @@ int cmd_pack_objects(int argc, const char **argv, const char *prefix)
 				N_("exclude any configured uploadpack.blobpackfileuri with this protocol")),
 		OPT_BOOL(0, "full-name-hash", &use_full_name_hash,
 			 N_("optimize delta compression across identical path names over time")),
+		OPT_STRING(0, "delta-file", &delta_file_name,
+				N_("filename"),
+				N_("output delta compression details to the given file")),
 		OPT_END(),
 	};
 
@@ -4573,6 +4631,12 @@ int cmd_pack_objects(int argc, const char **argv, const char *prefix)
 	if (pack_to_stdout != !base_name || argc)
 		usage_with_options(pack_usage, pack_objects_options);
 
+	if (delta_file_name) {
+		delta_file = fopen(delta_file_name, "w");
+		if (!delta_file)
+			die_errno("failed to open '%s'", delta_file_name);
+		trace2_printf("opened '%s' for writing deltas", delta_file_name);
+	}
 	if (depth < 0)
 		depth = 0;
 	if (depth >= (1 << OE_DEPTH_BITS)) {
@@ -4796,5 +4860,10 @@ cleanup:
 	list_objects_filter_release(&filter_options);
 	strvec_clear(&rp);
 
+	if (delta_file) {
+		fflush(delta_file);
+		fclose(delta_file);
+	}
+
 	return 0;
 }

-- 
gitgitgadget
Previous: Derrick Stolee via GitGitGadgetNext: Junio C Hamano
Message 31 of 38 in “[RFC] Path-walk API and applications”
  1. 00/30 [RFC] Path-walk API and applicationsDerrick Stolee via GitGitGadget, Sep 10, 2024
  2. 01/30 path-walk: introduce an object walk by pathDerrick Stolee via GitGitGadget, Sep 10, 2024
  3. 02/30 backfill: add builtin boilerplateDerrick Stolee via GitGitGadget, Sep 10, 2024
  4. 03/30 backfill: basic functionality and testsDerrick Stolee via GitGitGadget, Sep 10, 2024
  5. 04/30 backfill: add --batch-size=<n> optionDerrick Stolee via GitGitGadget, Sep 10, 2024
  6. 06/30 backfill: assume --sparse when sparse-checkout is enabledDerrick Stolee via GitGitGadget, Sep 10, 2024
  7. 05/30 backfill: add --sparse optionDerrick Stolee via GitGitGadget, Sep 10, 2024
  8. 07/30 path-walk: allow consumer to specify object typesDerrick Stolee via GitGitGadget, Sep 10, 2024
  9. 08/30 path-walk: allow visiting tagsDerrick Stolee via GitGitGadget, Sep 10, 2024
  10. 09/30 survey: stub in new experimental `git-survey` commandJeff Hostetler via GitGitGadget, Sep 10, 2024
  11. 10/30 survey: add command line opts to select referencesJeff Hostetler via GitGitGadget, Sep 10, 2024
  12. 11/30 survey: collect the set of requested refsJeff Hostetler via GitGitGadget, Sep 10, 2024
  13. 12/30 survey: start pretty printing data in table formDerrick Stolee via GitGitGadget, Sep 10, 2024
  14. 13/30 survey: add object count summaryDerrick Stolee via GitGitGadget, Sep 10, 2024
  15. 14/30 survey: summarize total sizes by object typeDerrick Stolee via GitGitGadget, Sep 10, 2024
  16. 15/30 survey: show progress during object walkDerrick Stolee via GitGitGadget, Sep 10, 2024
  17. 16/30 survey: add ability to track prioritized listsDerrick Stolee via GitGitGadget, Sep 10, 2024
  18. 17/30 survey: add report of "largest" pathsDerrick Stolee via GitGitGadget, Sep 10, 2024
  19. 18/30 revision: create mark_trees_uninteresting_dense()Derrick Stolee via GitGitGadget, Sep 10, 2024
  20. 19/30 path-walk: add prune_all_uninteresting optionDerrick Stolee via GitGitGadget, Sep 10, 2024
  21. 20/30 pack-objects: add --path-walk optionDerrick Stolee via GitGitGadget, Sep 10, 2024
  22. 21/30 pack-objects: extract should_attempt_deltas()Derrick Stolee via GitGitGadget, Sep 10, 2024
  23. 22/30 pack-objects: introduce GIT_TEST_PACK_PATH_WALKDerrick Stolee via GitGitGadget, Sep 10, 2024
  24. 23/30 p5313: add size comparison testDerrick Stolee via GitGitGadget, Sep 10, 2024
  25. 24/30 repack: add --path-walk optionDerrick Stolee via GitGitGadget, Sep 10, 2024
  26. 25/30 pack-objects: enable --path-walk via configDerrick Stolee via GitGitGadget, Sep 10, 2024
  27. 26/30 scalar: enable path-walk during push via configDerrick Stolee via GitGitGadget, Sep 10, 2024
  28. 27/30 pack-objects: add --full-name-hash optionDerrick Stolee via GitGitGadget, Sep 10, 2024
  29. 28/30 test-name-hash: add helper to compute name-hash functionsDerrick Stolee via GitGitGadget, Sep 10, 2024
  30. 29/30 p5314: add a size test for name-hash collisionsDerrick Stolee via GitGitGadget, Sep 10, 2024
  31. 30/30 pack-objects: output debug info about deltasDerrick Stolee via GitGitGadget, Sep 10, 2024
  32. Junio C HamanoSep 11, 2024
  33. Christian CouderSep 17, 2024
  34. Derrick StoleeSep 18, 2024
  35. Junio C HamanoSep 22, 2024
  36. Derrick StoleeSep 23, 2024
  37. Junio C HamanoSep 23, 2024
  38. Kristoffer HaugsbakkSep 22, 2024

Read the whole thread, see it on lore, or plain text.

$ cat FOOTERMessages come from the public archive at lore.kernel.org/git, fetched every hour. The front page is chosen and written each morning by an AI editor and can be wrong; the threads themselves are the record. About and API. For agents: an MCP server at https://gitlist.dev/mcp, and any thread, story or person page as Markdown by adding .md to its URL (or sending Accept: text/markdown). Details in /llms.txt.