git/list[1] front-page[2] threads[3] people[4] search[5] about
 

[PATCH v2 06/16] {send,receive}-pack: support pushing from a shallow clone

From
Nguyễn Thái Ngọc Duy <pclouds@gmail.com>
Date
Jul 20, 2013, 09:58 UTC
Message-ID
<1374314290-5976-7-git-send-email-pclouds@gmail.com>
In-Reply-To
<1374314290-5976-1-git-send-email-pclouds@gmail.com>

Pushing from a shallow clone using today's send-pack and receive-pack may work, if the transferred pack does not ends up at any graft points. If it does, recent receive-pack that does connectivity check will reject the push. If receive-pack is old, the upstream repo becomes corrupt.

The pack protocol is updated and send-pack now sends all shallow grafts before it sends the commands, if the repo is shallow. This protocol extension will break current receive-pack, which is intended, mostly to stop corrupting the upstream repo.

The receiver end, the newreceive-pack, does something similar to fetch-pack: it creates a temporary shallow file with grafts from send-pack, then receives the pack, and finally writes the refined shallow file down.

shadow file is not cleaned up after deleting (or force updating) a ref if that ref is the only way to reach the graft points. The reason is once we delete graft points, we can't recover. That may make reflog entries on server useless. Leave that for the administrators to decide when to clean up shadow file (maybe at repack/gc time).

Signed-off-by: Nguyễn Thái Ngọc Duy <pclouds@gmail.com>
---
 Documentation/technical/pack-protocol.txt |  4 +-
 builtin/receive-pack.c                    | 66 ++++++++++++++++++++++-----
 builtin/send-pack.c                       |  2 +-
 send-pack.c                               |  3 ++
 t/t5537-push-shallow.sh (new +x)          | 74 +++++++++++++++++++++++++++++++
 5 files changed, 137 insertions(+), 12 deletions(-)
 create mode 100755 t/t5537-push-shallow.sh
diff --git a/Documentation/technical/pack-protocol.txt b/Documentation/technical/pack-protocol.txt
index eb8edd1..c73b62f 100644
--- a/Documentation/technical/pack-protocol.txt
+++ b/Documentation/technical/pack-protocol.txt
@@ -464,7 +464,9 @@ contain all the objects that the server will need to complete the new
 references.
 
 ----
-  update-request    =  command-list [pack-file]
+  update-request    =  *shallow command-list [pack-file]
+
+  shallow           =  PKT-LINE("shallow" SP obj-id)
 
   command-list      =  PKT-LINE(command NUL capability-list LF)
 		       *PKT-LINE(command LF)
diff --git a/builtin/receive-pack.c b/builtin/receive-pack.c
index 6ffe526..cbb2025 100644
--- a/builtin/receive-pack.c
+++ b/builtin/receive-pack.c
@@ -41,6 +41,10 @@ static int auto_gc = 1;
 static const char *head_name;
 static void *head_name_to_free;
 static int sent_capabilities;
+static int shallow_changed;
+static const char* alternate_shallow_file;
+static struct lock_file shallow_lock;
+static struct extra_have_objects shallow;
 
 static enum deny_action parse_deny_action(const char *var, const char *value)
 {
@@ -751,6 +755,13 @@ static void execute_commands(struct command *commands, const char *unpacker_erro
 	}
 }
 
+static void add_extra_have(struct extra_have_objects *extra, unsigned char *sha1)
+{
+	ALLOC_GROW(extra->array, extra->nr + 1, extra->alloc);
+	hashcpy(&(extra->array[extra->nr][0]), sha1);
+	extra->nr++;
+}
+
 static struct command *read_head_info(void)
 {
 	struct command *commands = NULL;
@@ -765,6 +776,17 @@ static struct command *read_head_info(void)
 		line = packet_read_line(0, &len);
 		if (!line)
 			break;
+
+		if (len == 48 && !prefixcmp(line, "shallow ")) {
+			if (get_sha1_hex(line + 8, old_sha1))
+				die("protocol error: expected shallow sha, got '%s'", line + 8);
+			if (!has_sha1_file(old_sha1)) {
+				add_extra_have(&shallow, old_sha1);
+				shallow_changed = 1;
+			}
+			continue;
+		}
+
 		if (len < 83 ||
 		    line[40] != ' ' ||
 		    line[81] != ' ' ||
@@ -827,6 +849,13 @@ static const char *unpack(int err_fd)
 			    ? transfer_fsck_objects
 			    : 0);
 
+	if (shallow_changed)
+		setup_alternate_shallow(&shallow_lock,
+					&alternate_shallow_file,
+					&shallow,
+					WRITE_SHALLOW_NO_CUT);
+
+
 	hdr_err = parse_pack_header(&hdr);
 	if (hdr_err) {
 		if (err_fd > 0)
@@ -840,7 +869,12 @@ static const char *unpack(int err_fd)
 	if (ntohl(hdr.hdr_entries) < unpack_limit) {
 		int code, i = 0;
 		struct child_process child;
-		const char *unpacker[5];
+		const char *unpacker[7];
+		if (alternate_shallow_file &&
+		    *alternate_shallow_file) {
+			unpacker[i++] = "--shallow-file";
+			unpacker[i++] = alternate_shallow_file;
+		}
 		unpacker[i++] = "unpack-objects";
 		if (quiet)
 			unpacker[i++] = "-q";
@@ -854,15 +888,19 @@ static const char *unpack(int err_fd)
 		child.err = err_fd;
 		child.git_cmd = 1;
 		code = run_command(&child);
-		if (!code)
-			return NULL;
-		return "unpack-objects abnormal exit";
+		if (code)
+			return "unpack-objects abnormal exit";
 	} else {
-		const char *keeper[7];
+		const char *keeper[9];
 		int s, status, i = 0;
 		char keep_arg[256];
 		struct child_process ip;
 
+		if (alternate_shallow_file &&
+		    *alternate_shallow_file) {
+			keeper[i++] = "--shallow-file";
+			keeper[i++] = alternate_shallow_file;
+		}
 		s = sprintf(keep_arg, "--keep=receive-pack %"PRIuMAX" on ", (uintmax_t) getpid());
 		if (gethostname(keep_arg + s, sizeof(keep_arg) - s))
 			strcpy(keep_arg + s, "localhost");
@@ -887,12 +925,20 @@ static const char *unpack(int err_fd)
 		pack_lockfile = index_pack_lockfile(ip.out);
 		close(ip.out);
 		status = finish_command(&ip);
-		if (!status) {
-			reprepare_packed_git();
-			return NULL;
-		}
-		return "index-pack abnormal exit";
+		if (status)
+			return "index-pack abnormal exit";
+		reprepare_packed_git();
+	}
+
+	if (shallow_changed) {
+		setup_alternate_shallow(&shallow_lock,
+					&alternate_shallow_file,
+					&shallow,
+					WRITE_SHALLOW_NO_CUT |
+					WRITE_SHALLOW_REWRITE);
+		commit_lock_file(&shallow_lock);
 	}
+	return NULL;
 }
 
 static const char *unpack_with_sideband(void)
diff --git a/builtin/send-pack.c b/builtin/send-pack.c
index 4be5931..7dd254f 100644
--- a/builtin/send-pack.c
+++ b/builtin/send-pack.c
@@ -184,7 +184,7 @@ int cmd_send_pack(int argc, const char **argv, const char *prefix)
 	    (send_all && args.send_mirror))
 		usage(send_pack_usage);
 
-	if (is_repository_shallow())
+	if (is_repository_shallow() && args.stateless_rpc)
 		die("attempt to push from a shallow repository");
 
 	if (remote_name) {
diff --git a/send-pack.c b/send-pack.c
index 7d172ef..dcd3345 100644
--- a/send-pack.c
+++ b/send-pack.c
@@ -212,6 +212,9 @@ int send_pack(struct send_pack_args *args,
 		return 0;
 	}
 
+	if (!args->dry_run)
+		advertise_shallow_grafts(out);
+
 	/*
 	 * Finally, tell the other end!
 	 */
diff --git a/t/t5537-push-shallow.sh b/t/t5537-push-shallow.sh
new file mode 100755
index 0000000..37ad69b
--- /dev/null
+++ b/t/t5537-push-shallow.sh
@@ -0,0 +1,74 @@
+#!/bin/sh
+
+test_description='push from/to a shallow clone'
+
+. ./test-lib.sh
+
+commit() {
+	echo "$1" >tracked &&
+	git add tracked &&
+	git commit -m "$1"
+}
+
+test_expect_success 'setup' '
+	git config --global transfer.fsckObjects true &&
+	commit 1 &&
+	commit 2 &&
+	commit 3 &&
+	commit 4 &&
+	(
+	git init full-abc &&
+	cd full-abc &&
+	commit a &&
+	commit b &&
+	commit c
+	) &&
+	git clone --no-local --depth=2 .git shallow &&
+	git --git-dir=shallow/.git log --format=%s >actual &&
+	cat <<EOF >expect &&
+4
+3
+EOF
+	test_cmp expect actual &&
+	git clone --no-local --depth=2 full-abc/.git shallow2 &&
+	git --git-dir=shallow2/.git log --format=%s >actual &&
+	cat <<EOF >expect &&
+c
+b
+EOF
+	test_cmp expect actual
+'
+
+test_expect_success 'push from shallow clone' '
+	(
+	cd shallow &&
+	commit 5 &&
+	git push ../.git +master:refs/remotes/shallow/master
+	) &&
+	git log --format=%s shallow/master >actual &&
+	git fsck &&
+	cat <<EOF >expect &&
+5
+4
+3
+2
+1
+EOF
+	test_cmp expect actual
+'
+
+test_expect_success 'push from shallow clone, with grafted roots' '
+	(
+	cd shallow2 &&
+	git push ../.git +master:refs/remotes/shallow2/master
+	) &&
+	git log --format=%s shallow2/master >actual &&
+	git fsck &&
+	cat <<EOF >expect &&
+c
+b
+EOF
+	test_cmp expect actual
+'
+
+test_done

-- 
1.8.2.83.gc99314b
Previous: Duy NguyenNext: Nguyễn Thái Ngọc Duy
Message 20 of 44 in “First class shallow clone”
  1. 0/7 First class shallow cloneNguyễn Thái Ngọc Duy, Jul 17, 2013
  2. 1/7 transport.h: remove send_pack prototype, already defined in send-pack.hNguyễn Thái Ngọc Duy, Jul 17, 2013
  3. 2/7 {receive,upload}-pack: advertise shallow graft informationNguyễn Thái Ngọc Duy, Jul 17, 2013
  4. 3/7 connect.c: teach get_remote_heads to parse "shallow" linesNguyễn Thái Ngọc Duy, Jul 17, 2013
  5. Junio C HamanoJul 20, 2013
  6. 4/7 Move setup_alternate_shallow and write_shallow_commits to shallow.cNguyễn Thái Ngọc Duy, Jul 17, 2013
  7. 5/7 fetch-pack: support fetching from a shallow repositoryNguyễn Thái Ngọc Duy, Jul 17, 2013
  8. Junio C HamanoJul 20, 2013
  9. 6/7 {send,receive}-pack: support pushing from a shallow cloneNguyễn Thái Ngọc Duy, Jul 17, 2013
  10. 7/7 send-pack: support pushing to a shallow cloneNguyễn Thái Ngọc Duy, Jul 17, 2013
  11. 00/16 First class shallow cloneNguyễn Thái Ngọc Duy, Jul 20, 2013
  12. 01/16 send-pack: forbid pushing from a shallow repositoryNguyễn Thái Ngọc Duy, Jul 20, 2013
  13. 02/16 {receive,upload}-pack: advertise shallow graft informationNguyễn Thái Ngọc Duy, Jul 20, 2013
  14. 03/16 connect.c: teach get_remote_heads to parse "shallow" linesNguyễn Thái Ngọc Duy, Jul 20, 2013
  15. 04/16 Move setup_alternate_shallow and write_shallow_commits to shallow.cNguyễn Thái Ngọc Duy, Jul 20, 2013
  16. 05/16 fetch-pack: support fetching from a shallow repositoryNguyễn Thái Ngọc Duy, Jul 20, 2013
  17. Philip OakleyJul 22, 2013
  18. Duy NguyenJul 23, 2013
  19. Duy NguyenJul 23, 2013
  20. 06/16 {send,receive}-pack: support pushing from a shallow cloneNguyễn Thái Ngọc Duy, Jul 20, 2013
  21. 07/16 send-pack: support pushing to a shallow cloneNguyễn Thái Ngọc Duy, Jul 20, 2013
  22. 08/16 upload-pack: let pack-objects do the object counting in shallow caseNguyễn Thái Ngọc Duy, Jul 20, 2013
  23. 09/16 pack-protocol.txt: a bit about smart httpNguyễn Thái Ngọc Duy, Jul 20, 2013
  24. 10/16 Add document for command arguments for supporting smart httpNguyễn Thái Ngọc Duy, Jul 20, 2013
  25. 11/16 {fetch,upload}-pack: support fetching from a shallow clone via smart httpNguyễn Thái Ngọc Duy, Jul 20, 2013
  26. 12/16 receive-pack: support pushing to a shallow clone via httpNguyễn Thái Ngọc Duy, Jul 20, 2013
  27. 13/16 send-pack: support pushing from a shallow clone via httpNguyễn Thái Ngọc Duy, Jul 20, 2013
  28. 14/16 git-clone.txt: remove shallow clone limitationsNguyễn Thái Ngọc Duy, Jul 20, 2013
  29. 15/16 config: add core.noshallow to prevent turning a repo into a shallow oneNguyễn Thái Ngọc Duy, Jul 20, 2013
  30. Philip OakleyJul 22, 2013
  31. Duy NguyenJul 23, 2013
  32. Junio C HamanoJul 23, 2013
  33. Philip OakleyJul 23, 2013
  34. 16/16 clone: use git protocol for cloning shallow repo locallyNguyễn Thái Ngọc Duy, Jul 20, 2013
  35. Philip OakleyJul 22, 2013
  36. Duy NguyenJul 23, 2013
  37. Junio C HamanoJul 23, 2013
  38. Duy NguyenJul 23, 2013
  39. Philip OakleyJul 23, 2013
  40. Duy NguyenJul 24, 2013
  41. Philip OakleyJul 24, 2013
  42. Piotr KrukowieckiJul 24, 2013
  43. Duy NguyenJul 24, 2013
  44. Piotr KrukowieckiJul 24, 2013

Read the whole thread, see it on lore, or plain text.

$ cat FOOTERMessages come from the public archive at lore.kernel.org/git, fetched every hour. The front page is chosen and written each morning by an AI editor and can be wrong; the threads themselves are the record. About and API. For agents: an MCP server at https://gitlist.dev/mcp, and any thread, story or person page as Markdown by adding .md to its URL (or sending Accept: text/markdown). Details in /llms.txt.