git/list[1] front-page[2] threads[3] people[4] search[5] about
 

[RFC PATCH v3 16/17] Smart fetch over HTTP: client side

From
Shawn O. Pearce <spearce@spearce.org>
Date
Oct 15, 2009, 03:36 UTC
Message-ID
<1255577814-14745-17-git-send-email-spearce@spearce.org>
In-Reply-To
<1255577814-14745-1-git-send-email-spearce@spearce.org>

The git-remote-curl backend detects if the remote server supports the git-upload-pack service, and if so, runs git-fetch-pack locally in a pipe to generate the want/have commands.

The advertisements from the server that were obtained during the discovery are passed into git-fetch-pack before the POST request starts, permitting server capability discovery and enablement.

Common objects that are discovered are appended onto the request as have lines and are sent again on the next request. This allows the remote side to reinitialize its in-memory list of common objects during the next request.

Because all requests are relatively short, below git-remote-curl's 1 MiB buffer limit, requests will use the standard Content-Length header and be valid HTTP/1.0 POST requests. This makes the fetch client more tolerant of proxy servers which don't support HTTP/1.1 or the chunked transfer encoding.

Signed-off-by: Shawn O. Pearce <spearce@spearce.org>
CC: Daniel Barkalow <barkalow@iabervon.org>
---
 builtin-fetch-pack.c |  110 ++++++++++++++++++++++++++++++++++++++++++--------
 fetch-pack.h         |    3 +-
 remote-curl.c        |   69 +++++++++++++++++++++++++++++--
 3 files changed, 160 insertions(+), 22 deletions(-)
diff --git a/builtin-fetch-pack.c b/builtin-fetch-pack.c
index 615f549..8ed4a6f 100644
--- a/builtin-fetch-pack.c
+++ b/builtin-fetch-pack.c
@@ -165,6 +165,24 @@ enum ack_type {
 	ACK_ready
 };
 
+static void consume_shallow_list(int fd)
+{
+	if (args.stateless_rpc && args.depth > 0) {
+		/* If we sent a depth we will get back "duplicate"
+		 * shallow and unshallow commands every time there
+		 * is a block of have lines exchanged.
+		 */
+		char line[1000];
+		while (packet_read_line(fd, line, sizeof(line))) {
+			if (!prefixcmp(line, "shallow "))
+				continue;
+			if (!prefixcmp(line, "unshallow "))
+				continue;
+			die("git fetch-pack: expected shallow list");
+		}
+	}
+}
+
 static enum ack_type get_ack(int fd, unsigned char *result_sha1)
 {
 	static char line[1000];
@@ -190,6 +208,15 @@ static enum ack_type get_ack(int fd, unsigned char *result_sha1)
 	die("git fetch_pack: expected ACK/NAK, got '%s'", line);
 }
 
+static void send_request(int fd, struct strbuf *buf)
+{
+	if (args.stateless_rpc) {
+		send_sideband(fd, -1, buf->buf, buf->len, LARGE_PACKET_MAX);
+		packet_flush(fd);
+	} else
+		safe_write(fd, buf->buf, buf->len);
+}
+
 static int find_common(int fd[2], unsigned char *result_sha1,
 		       struct ref *refs)
 {
@@ -199,7 +226,10 @@ static int find_common(int fd[2], unsigned char *result_sha1,
 	unsigned in_vain = 0;
 	int got_continue = 0;
 	struct strbuf req_buf = STRBUF_INIT;
+	size_t state_len = 0;
 
+	if (args.stateless_rpc && multi_ack == 1)
+		die("--stateless-rpc requires multi_ack_detailed");
 	if (marked)
 		for_each_ref(clear_marks, NULL);
 	marked = 1;
@@ -256,13 +286,13 @@ static int find_common(int fd[2], unsigned char *result_sha1,
 	if (args.depth > 0)
 		packet_buf_write(&req_buf, "deepen %d", args.depth);
 	packet_buf_flush(&req_buf);
-
-	safe_write(fd[1], req_buf.buf, req_buf.len);
+	state_len = req_buf.len;
 
 	if (args.depth > 0) {
 		char line[1024];
 		unsigned char sha1[20];
 
+		send_request(fd[1], &req_buf);
 		while (packet_read_line(fd[0], line, sizeof(line))) {
 			if (!prefixcmp(line, "shallow ")) {
 				if (get_sha1_hex(line + 8, sha1))
@@ -284,28 +314,40 @@ static int find_common(int fd[2], unsigned char *result_sha1,
 			}
 			die("expected shallow/unshallow, got %s", line);
 		}
+	} else if (!args.stateless_rpc)
+		send_request(fd[1], &req_buf);
+
+	if (!args.stateless_rpc) {
+		/* If we aren't using the stateless-rpc interface
+		 * we don't need to retain the headers.
+		 */
+		strbuf_setlen(&req_buf, 0);
+		state_len = 0;
 	}
 
 	flushes = 0;
 	retval = -1;
 	while ((sha1 = get_rev())) {
-		packet_write(fd[1], "have %s\n", sha1_to_hex(sha1));
+		packet_buf_write(&req_buf, "have %s\n", sha1_to_hex(sha1));
 		if (args.verbose)
 			fprintf(stderr, "have %s\n", sha1_to_hex(sha1));
 		in_vain++;
 		if (!(31 & ++count)) {
 			int ack;
 
-			packet_flush(fd[1]);
+			packet_buf_flush(&req_buf);
+			send_request(fd[1], &req_buf);
+			strbuf_setlen(&req_buf, state_len);
 			flushes++;
 
 			/*
 			 * We keep one window "ahead" of the other side, and
 			 * will wait for an ACK only on the next one
 			 */
-			if (count == 32)
+			if (!args.stateless_rpc && count == 32)
 				continue;
 
+			consume_shallow_list(fd[0]);
 			do {
 				ack = get_ack(fd[0], result_sha1);
 				if (args.verbose && ack)
@@ -322,6 +364,17 @@ static int find_common(int fd[2], unsigned char *result_sha1,
 				case ACK_continue: {
 					struct commit *commit =
 						lookup_commit(result_sha1);
+					if (args.stateless_rpc
+					 && ack == ACK_common
+					 && !(commit->object.flags & COMMON)) {
+						/* We need to replay the have for this object
+						 * on the next RPC request so the peer knows
+						 * it is in common with us.
+						 */
+						const char *hex = sha1_to_hex(result_sha1);
+						packet_buf_write(&req_buf, "have %s\n", hex);
+						state_len = req_buf.len;
+					}
 					mark_common(commit, 0, 1);
 					retval = 0;
 					in_vain = 0;
@@ -339,7 +392,8 @@ static int find_common(int fd[2], unsigned char *result_sha1,
 		}
 	}
 done:
-	packet_write(fd[1], "done\n");
+	packet_buf_write(&req_buf, "done\n");
+	send_request(fd[1], &req_buf);
 	if (args.verbose)
 		fprintf(stderr, "done\n");
 	if (retval != 0) {
@@ -348,6 +402,7 @@ done:
 	}
 	strbuf_release(&req_buf);
 
+	consume_shallow_list(fd[0]);
 	while (flushes || multi_ack) {
 		int ack = get_ack(fd[0], result_sha1);
 		if (ack) {
@@ -672,6 +727,8 @@ static struct ref *do_fetch_pack(int fd[2],
 			 */
 			warning("no common commits");
 
+	if (args.stateless_rpc)
+		packet_flush(fd[1]);
 	if (get_pack(fd, pack_lockfile))
 		die("git fetch-pack: fetch failed.");
 
@@ -742,6 +799,8 @@ int cmd_fetch_pack(int argc, const char **argv, const char *prefix)
 	struct ref *ref = NULL;
 	char *dest = NULL, **heads;
 	int fd[2];
+	char *pack_lockfile = NULL;
+	char **pack_lockfile_ptr = NULL;
 	struct child_process *conn;
 
 	nr_heads = 0;
@@ -791,6 +850,15 @@ int cmd_fetch_pack(int argc, const char **argv, const char *prefix)
 				args.no_progress = 1;
 				continue;
 			}
+			if (!strcmp("--stateless-rpc", arg)) {
+				args.stateless_rpc = 1;
+				continue;
+			}
+			if (!strcmp("--lock-pack", arg)) {
+				args.lock_pack = 1;
+				pack_lockfile_ptr = &pack_lockfile;
+				continue;
+			}
 			usage(fetch_pack_usage);
 		}
 		dest = (char *)arg;
@@ -801,19 +869,27 @@ int cmd_fetch_pack(int argc, const char **argv, const char *prefix)
 	if (!dest)
 		usage(fetch_pack_usage);
 
-	conn = git_connect(fd, (char *)dest, args.uploadpack,
-			   args.verbose ? CONNECT_VERBOSE : 0);
-	if (conn) {
-		get_remote_heads(fd[0], &ref, 0, NULL, 0, NULL);
-
-		ref = fetch_pack(&args, fd, conn, ref, dest, nr_heads, heads, NULL);
-		close(fd[0]);
-		close(fd[1]);
-		if (finish_connect(conn))
-			ref = NULL;
+	if (args.stateless_rpc) {
+		conn = NULL;
+		fd[0] = 0;
+		fd[1] = 1;
 	} else {
-		ref = NULL;
+		conn = git_connect(fd, (char *)dest, args.uploadpack,
+				   args.verbose ? CONNECT_VERBOSE : 0);
+	}
+
+	get_remote_heads(fd[0], &ref, 0, NULL, 0, NULL);
+
+	ref = fetch_pack(&args, fd, conn, ref, dest,
+		nr_heads, heads, pack_lockfile_ptr);
+	if (pack_lockfile) {
+		printf("lock %s\n", pack_lockfile);
+		fflush(stdout);
 	}
+	close(fd[0]);
+	close(fd[1]);
+	if (finish_connect(conn))
+		ref = NULL;
 	ret = !ref;
 
 	if (!ret && nr_heads) {
diff --git a/fetch-pack.h b/fetch-pack.h
index 8bd9c32..fbe85ac 100644
--- a/fetch-pack.h
+++ b/fetch-pack.h
@@ -13,7 +13,8 @@ struct fetch_pack_args
 		fetch_all:1,
 		verbose:1,
 		no_progress:1,
-		include_tag:1;
+		include_tag:1,
+		stateless_rpc:1;
 };
 
 struct ref *fetch_pack(struct fetch_pack_args *args,
diff --git a/remote-curl.c b/remote-curl.c
index 45436d1..eb71ffd 100644
--- a/remote-curl.c
+++ b/remote-curl.c
@@ -45,7 +45,7 @@ static int set_option(const char *name, const char *value)
 			options.progress = 0;
 		else
 			return -1;
-		return 1 /* TODO implement later */;
+		return 0;
 	}
 	else if (!strcmp(name, "depth")) {
 		char *end;
@@ -53,7 +53,7 @@ static int set_option(const char *name, const char *value)
 		if (value == end || *end)
 			return -1;
 		options.depth = v;
-		return 1 /* TODO implement later */;
+		return 0;
 	}
 	else if (!strcmp(name, "followtags")) {
 		if (!strcmp(value, "true"))
@@ -62,7 +62,7 @@ static int set_option(const char *name, const char *value)
 			options.followtags = 0;
 		else
 			return -1;
-		return 1 /* TODO implement later */;
+		return 0;
 	}
 	else if (!strcmp(name, "dry-run")) {
 		if (!strcmp(value, "true"))
@@ -462,6 +462,8 @@ static int fetch_dumb(int nr_heads, struct ref **to_fetch)
 	char **targets = xmalloc(nr_heads * sizeof(char*));
 	int ret, i;
 
+	if (options.depth)
+		die("dumb http transport does not support --depth");
 	for (i = 0; i < nr_heads; i++)
 		targets[i] = xstrdup(sha1_to_hex(to_fetch[i]->old_sha1));
 
@@ -480,6 +482,65 @@ static int fetch_dumb(int nr_heads, struct ref **to_fetch)
 	return ret ? error("Fetch failed.") : 0;
 }
 
+static int fetch_git(struct discovery *heads,
+	int nr_heads, struct ref **to_fetch)
+{
+	struct rpc_state rpc;
+	char *depth_arg = NULL;
+	const char **argv;
+	int argc = 0, i, err;
+
+	argv = xmalloc((15 + nr_heads) * sizeof(char*));
+	argv[argc++] = "fetch-pack";
+	argv[argc++] = "--stateless-rpc";
+	argv[argc++] = "--lock-pack";
+	if (options.followtags)
+		argv[argc++] = "--include-tag";
+	if (options.thin)
+		argv[argc++] = "--thin";
+	if (options.verbosity >= 3) {
+		argv[argc++] = "-v";
+		argv[argc++] = "-v";
+	}
+	if (!options.progress)
+		argv[argc++] = "--no-progress";
+	if (options.depth) {
+		struct strbuf buf = STRBUF_INIT;
+		strbuf_addf(&buf, "--depth=%lu", options.depth);
+		depth_arg = strbuf_detach(&buf, NULL);
+		argv[argc++] = depth_arg;
+	}
+	argv[argc++] = url;
+	for (i = 0; i < nr_heads; i++) {
+		struct ref *ref = to_fetch[i];
+		if (!ref->name || !*ref->name)
+			die("cannot fetch by sha1 over smart http");
+		argv[argc++] = ref->name;
+	}
+	argv[argc++] = NULL;
+
+	memset(&rpc, 0, sizeof(rpc));
+	rpc.service_name = "git-upload-pack",
+	rpc.argv = argv;
+
+	err = rpc_service(&rpc, heads);
+	if (rpc.result.len)
+		safe_write(1, rpc.result.buf, rpc.result.len);
+	strbuf_release(&rpc.result);
+	free(argv);
+	free(depth_arg);
+	return err;
+}
+
+static int fetch(int nr_heads, struct ref **to_fetch)
+{
+	struct discovery *d = discover_refs("git-upload-pack");
+	if (d->proto_git)
+		return fetch_git(d, nr_heads, to_fetch);
+	else
+		return fetch_dumb(nr_heads, to_fetch);
+}
+
 static void parse_fetch(struct strbuf *buf)
 {
 	struct ref **to_fetch = NULL;
@@ -522,7 +583,7 @@ static void parse_fetch(struct strbuf *buf)
 			break;
 	} while (1);
 
-	if (fetch_dumb(nr_heads, to_fetch))
+	if (fetch(nr_heads, to_fetch))
 		exit(128); /* error already reported */
 	free_refs(list_head);
 	free(to_fetch);

-- 
1.6.5.52.g0ff2e
Previous: Shawn O. PearceNext: Shawn O. Pearce
Message 20 of 52 in “Return of smart HTTP”
  1. 00/17 Return of smart HTTPShawn O. Pearce, Oct 15, 2009
  2. 01/17 pkt-line: Add strbuf based functionsShawn O. Pearce, Oct 15, 2009
  3. 02/17 pkt-line: Make packet_read_line easier to debugShawn O. Pearce, Oct 15, 2009
  4. 03/17 fetch-pack: Use a strbuf to compose the want listShawn O. Pearce, Oct 15, 2009
  5. 04/17 Move "get_ack()" back to fetch-packShawn O. Pearce, Oct 15, 2009
  6. 05/17 Add multi_ack_detailed capability to fetch-pack/upload-packShawn O. Pearce, Oct 15, 2009
  7. 06/17 remote-curl: Refactor walker initializationShawn O. Pearce, Oct 15, 2009
  8. 07/17 fetch: Allow transport -v -v -v to set verbosity to 3Shawn O. Pearce, Oct 15, 2009
  9. 08/17 remote-helpers: Fetch more than one ref in a batchShawn O. Pearce, Oct 15, 2009
  10. 09/17 remote-helpers: Support custom transport optionsShawn O. Pearce, Oct 15, 2009
  11. 10/17 Move WebDAV HTTP push under remote-curlShawn O. Pearce, Oct 15, 2009
  12. Tay Ray ChuanOct 19, 2009
  13. Shawn O. PearceOct 28, 2009
  14. Tay Ray ChuanOct 28, 2009
  15. 11/17 Git-aware CGI to provide dumb HTTP transportShawn O. Pearce, Oct 15, 2009
  16. 12/17 Add stateless RPC options to upload-pack, receive-packShawn O. Pearce, Oct 15, 2009
  17. 13/17 Smart fetch and push over HTTP: server sideShawn O. Pearce, Oct 15, 2009
  18. 14/17 Discover refs via smart HTTP server when availableShawn O. Pearce, Oct 15, 2009
  19. 15/17 Smart push over HTTP: client sideShawn O. Pearce, Oct 15, 2009
  20. 16/17 Smart fetch over HTTP: client sideShawn O. Pearce, Oct 15, 2009
  21. 17/17 Smart HTTP fetch: gzip requestsShawn O. Pearce, Oct 15, 2009
  22. Junio C HamanoOct 15, 2009
  23. Nanako ShiraishiOct 15, 2009
  24. Shawn O. PearceOct 15, 2009
  25. Johan HerlandOct 15, 2009
  26. Shawn O. PearceOct 15, 2009
  27. Junio C HamanoOct 15, 2009
  28. Shawn O. PearceOct 15, 2009
  29. Nanako ShiraishiOct 22, 2009
  30. Daniel BarkalowOct 22, 2009
  31. Fix memory leak in transport-helperDaniel Barkalow, Oct 27, 2009
  32. Johannes SchindelinOct 27, 2009
  33. Daniel BarkalowOct 27, 2009
  34. Jeff KingOct 27, 2009
  35. Johannes SchindelinOct 27, 2009
  36. Daniel BarkalowOct 27, 2009
  37. Junio C HamanoOct 28, 2009
  38. Mark LodatoOct 16, 2009
  39. Shawn O. PearceOct 16, 2009
  40. Mark LodatoOct 16, 2009
  41. Shawn O. PearceOct 16, 2009
  42. Marcus CamenOct 22, 2009
  43. 0/6 http: push and test fixesTay Ray Chuan, Oct 25, 2009
  44. 1/7 http-push: fix check condition on http.c::finish_http_pack_request()Tay Ray Chuan, Oct 25, 2009
  45. 2/7 http-push: allow stderr messages to appear alongside helper_status onesTay Ray Chuan, Oct 25, 2009
  46. 3/7 http-push: add more 'error <dst> <why>' status reportsTay Ray Chuan, Oct 25, 2009
  47. 4/7 t5540-http-push: expect success when pushing without argumentsTay Ray Chuan, Oct 25, 2009
  48. 5/7 t5540-http-push: check existence of fetched filesTay Ray Chuan, Oct 25, 2009
  49. 6/7 t5540-http-push: when deleting remote refs, don't need to branch -d -rTay Ray Chuan, Oct 25, 2009
  50. 7/7 t5540-http-push: remove redundant fetchesTay Ray Chuan, Oct 25, 2009
  51. Clemens BuchacherOct 25, 2009
  52. Clemens BuchacherOct 25, 2009

Read the whole thread, see it on lore, or plain text.

$ cat FOOTERMessages come from the public archive at lore.kernel.org/git, fetched every hour. The front page is chosen and written each morning by an AI editor and can be wrong; the threads themselves are the record. About and API. For agents: an MCP server at https://gitlist.dev/mcp, and any thread, story or person page as Markdown by adding .md to its URL (or sending Accept: text/markdown). Details in /llms.txt.