git/list[1] front-page[2] threads[3] people[4] search[5] about
 

[PATCH 1/8] pack-objects: add --skip and --skip-hash

From
Nguyễn Thái Ngọc Duy <pclouds@gmail.com>
Date
Feb 5, 2016, 08:57 UTC
Message-ID
<1454662677-15137-2-git-send-email-pclouds@gmail.com>
In-Reply-To
<1454662677-15137-1-git-send-email-pclouds@gmail.com>

The idea is, a pack is requested the first time with --skip=0. If pack transfer is interrupted, the client will ask for the same pack again, but this time it asks the server not to send what it already has. The client hashes what it has and sends the SHA-1 to the server. If the server finds out the skipped part does not match, it can abort early.

Signed-off-by: Nguyễn Thái Ngọc Duy <pclouds@gmail.com>
---
 builtin/pack-objects.c | 21 ++++++++++++++++++++
 csum-file.c            | 52 ++++++++++++++++++++++++++++++++++++++++++++++----
 csum-file.h            |  3 +++
 3 files changed, 72 insertions(+), 4 deletions(-)
diff --git a/builtin/pack-objects.c b/builtin/pack-objects.c
index 4dae5b1..417c830 100644
--- a/builtin/pack-objects.c
+++ b/builtin/pack-objects.c
@@ -75,6 +75,9 @@ static unsigned long cache_max_small_delta_size = 1000;
 
 static unsigned long window_memory_limit = 0;
 
+static struct object_id skip_hash;
+static int skip_opt = -1;
+
 /*
  * stats
  */
@@ -781,6 +784,11 @@ static void write_pack_file(void)
 		else
 			f = create_tmp_packfile(&pack_tmp_name);
 
+		if (skip_opt > 0) {
+			f->skip = skip_opt;
+			hashcpy(f->skip_hash, skip_hash.hash);
+		}
+
 		offset = write_pack_header(f, nr_remaining);
 
 		if (reuse_packfile) {
@@ -2597,6 +2605,7 @@ int cmd_pack_objects(int argc, const char **argv, const char *prefix)
 	struct argv_array rp = ARGV_ARRAY_INIT;
 	int rev_list_unpacked = 0, rev_list_all = 0, rev_list_reflog = 0;
 	int rev_list_index = 0;
+	const char *skip_hash_hex = NULL;
 	struct option pack_objects_options[] = {
 		OPT_SET_INT('q', "quiet", &progress,
 			    N_("do not show progress meter"), 0),
@@ -2669,6 +2678,10 @@ int cmd_pack_objects(int argc, const char **argv, const char *prefix)
 			 N_("use a bitmap index if available to speed up counting objects")),
 		OPT_BOOL(0, "write-bitmap-index", &write_bitmap_index,
 			 N_("write a bitmap index together with the pack index")),
+		OPT_STRING(0, "skip-hash", &skip_hash_hex, "hash",
+			   N_("hash of the skipped part of the pack")),
+		OPT_INTEGER(0, "skip", &skip_opt,
+			   N_("do not produce the first <n> bytes of the pack")),
 		OPT_END(),
 	};
 
@@ -2689,6 +2702,14 @@ int cmd_pack_objects(int argc, const char **argv, const char *prefix)
 	}
 	if (pack_to_stdout != !base_name || argc)
 		usage_with_options(pack_usage, pack_objects_options);
+	if (skip_opt >= 0) {
+		if (skip_opt > 0) {
+			if (!skip_hash_hex)
+				die(_("--skip-hash is required if --skip is non-zero"));
+			if (get_oid_hex(skip_hash_hex, &skip_hash))
+				die(_("%s is not SHA-1"), skip_hash_hex);
+		}
+	}
 
 	argv_array_push(&rp, "pack-objects");
 	if (thin) {
diff --git a/csum-file.c b/csum-file.c
index a172199..284847f 100644
--- a/csum-file.c
+++ b/csum-file.c
@@ -11,8 +11,27 @@
 #include "progress.h"
 #include "csum-file.h"
 
+static void check_skip_hash(struct sha1file *f, off_t old_total)
+{
+	if (old_total < f->skip && f->total > f->skip)
+		die("BUG: flush() must stop at skip boundary");
+
+	if (f->total == f->skip) {
+		git_SHA_CTX ctx;
+		unsigned char hash[20];
+
+		ctx = f->ctx;
+		git_SHA1_Final(hash, &ctx);
+		if (hashcmp(hash, f->skip_hash))
+			die("skip hash does not match, expected %s got %s",
+			    sha1_to_hex(f->skip_hash), sha1_to_hex(hash));
+	}
+}
+
 static void flush(struct sha1file *f, const void *buf, unsigned int count)
 {
+	off_t old_total = f->total;
+
 	if (0 <= f->check_fd && count)  {
 		unsigned char check_buffer[8192];
 		ssize_t ret = read_in_full(f->check_fd, check_buffer, count);
@@ -26,7 +45,11 @@ static void flush(struct sha1file *f, const void *buf, unsigned int count)
 	}
 
 	for (;;) {
-		int ret = xwrite(f->fd, buf, count);
+		int ret;
+		if (f->total + count <= f->skip)
+			ret = count;
+		else
+			ret = xwrite(f->fd, buf, count);
 		if (ret > 0) {
 			f->total += ret;
 			display_throughput(f->tp, f->total);
@@ -34,6 +57,8 @@ static void flush(struct sha1file *f, const void *buf, unsigned int count)
 			count -= ret;
 			if (count)
 				continue;
+			if (f->skip > 0)
+				check_skip_hash(f, old_total);
 			return;
 		}
 		if (!ret)
@@ -45,12 +70,25 @@ static void flush(struct sha1file *f, const void *buf, unsigned int count)
 void sha1flush(struct sha1file *f)
 {
 	unsigned offset = f->offset;
+	const unsigned char *buffer = f->buffer;
+
+	if (!offset)
+		return;
+
+	if (f->total < f->skip && f->skip < f->total + offset) {
+		unsigned size = f->skip - f->total;
+		git_SHA1_Update(&f->ctx, buffer, size);
+		flush(f, buffer, size);
+		buffer += size;
+		offset -= size;
+	}
 
 	if (offset) {
-		git_SHA1_Update(&f->ctx, f->buffer, offset);
-		flush(f, f->buffer, offset);
-		f->offset = 0;
+		git_SHA1_Update(&f->ctx, buffer, offset);
+		flush(f, buffer, offset);
 	}
+
+	f->offset = 0;
 }
 
 int sha1close(struct sha1file *f, unsigned char *result, unsigned int flags)
@@ -62,6 +100,8 @@ int sha1close(struct sha1file *f, unsigned char *result, unsigned int flags)
 	if (result)
 		hashcpy(result, f->buffer);
 	if (flags & (CSUM_CLOSE | CSUM_FSYNC)) {
+		if (f->skip > f->total)
+			die(_("can't skip in the middle of, or beyond the trailing SHA-1"));
 		/* write checksum and close fd */
 		flush(f, f->buffer, 20);
 		if (flags & CSUM_FSYNC)
@@ -94,6 +134,9 @@ void sha1write(struct sha1file *f, const void *buf, unsigned int count)
 		unsigned nr = count > left ? left : count;
 		const void *data;
 
+		if (f->total < f->skip && f->skip - f->total < nr)
+			nr = f->skip - f->total;
+
 		if (f->do_crc)
 			f->crc32 = crc32(f->crc32, buf, nr);
 
@@ -149,6 +192,7 @@ struct sha1file *sha1fd_throughput(int fd, const char *name, struct progress *tp
 	f->tp = tp;
 	f->name = name;
 	f->do_crc = 0;
+	f->skip = 0;
 	git_SHA1_Init(&f->ctx);
 	return f;
 }
diff --git a/csum-file.h b/csum-file.h
index 7530927..5a9475f 100644
--- a/csum-file.h
+++ b/csum-file.h
@@ -15,6 +15,9 @@ struct sha1file {
 	int do_crc;
 	uint32_t crc32;
 	unsigned char buffer[8192];
+
+	off_t skip;
+	unsigned char skip_hash[20];
 };
 
 /* Checkpoint */

-- 
2.7.0.377.g4cd97dd
Previous: Nguyễn Thái Ngọc DuyNext: Johannes Schindelin
Message 2 of 22 in “Resumable clone revisited, proof of concept”
  1. 0/8 Resumable clone revisited, proof of conceptNguyễn Thái Ngọc Duy, Feb 5, 2016
  2. 1/8 pack-objects: add --skip and --skip-hashNguyễn Thái Ngọc Duy, Feb 5, 2016
  3. Johannes SchindelinFeb 5, 2016
  4. 2/8 pack-objects: produce a stable pack when --skip is givenNguyễn Thái Ngọc Duy, Feb 5, 2016
  5. Junio C HamanoFeb 5, 2016
  6. Duy NguyenFeb 5, 2016
  7. Junio C HamanoFeb 6, 2016
  8. Duy NguyenFeb 8, 2016
  9. 3/8 index-pack: add --append-pack=<path>Nguyễn Thái Ngọc Duy, Feb 5, 2016
  10. 4/8 upload-pack: new capability to pass --skip* to pack-objectsNguyễn Thái Ngọc Duy, Feb 5, 2016
  11. Johannes SchindelinFeb 5, 2016
  12. 5/8 fetch-pack.c: send "skip" line to pack-objectsNguyễn Thái Ngọc Duy, Feb 5, 2016
  13. Eric SunshineFeb 7, 2016
  14. 6/8 fetch: add --resume-pack=<path>Nguyễn Thái Ngọc Duy, Feb 5, 2016
  15. 7/8 index-pack: --append-pack implies --strictNguyễn Thái Ngọc Duy, Feb 5, 2016
  16. Eric SunshineFeb 7, 2016
  17. 8/8 one ugly test to verify basic functionalityNguyễn Thái Ngọc Duy, Feb 5, 2016
  18. Elia PintoFeb 5, 2016
  19. Duy NguyenFeb 5, 2016
  20. Elia PintoFeb 5, 2016
  21. Johannes SchindelinFeb 5, 2016
  22. Elia PintoFeb 5, 2016

Read the whole thread, see it on lore, or plain text.

$ cat FOOTERMessages come from the public archive at lore.kernel.org/git, fetched every hour. The front page is chosen and written each morning by an AI editor and can be wrong; the threads themselves are the record. About and API. For agents: an MCP server at https://gitlist.dev/mcp, and any thread, story or person page as Markdown by adding .md to its URL (or sending Accept: text/markdown). Details in /llms.txt.