git/list[1] front-page[2] threads[3] people[4] search[5] about
 

[RFC PATCH v2 07/16] remote-helpers: Fetch more than one ref in a batch

From
Shawn O. Pearce <spearce@spearce.org>
Date
Oct 13, 2009, 02:25 UTC
Message-ID
<1255400715-10508-8-git-send-email-spearce@spearce.org>
In-Reply-To
<1255400715-10508-1-git-send-email-spearce@spearce.org>

Some network protocols (e.g. native git://) are able to fetch more than one ref at a time and reduce the overall transfer cost by combining the requests into a single exchange. Instead of feeding each fetch request one at a time to the helper, feed all of them at once so the helper can decide whether or not it should batch them.

Because 'fetch' was already released in 1.6.5 we introduce the new fetch-multiple capability/command to signal that the helper wants to use batch oriented approach to fetching refs.

Signed-off-by: Shawn O. Pearce <spearce@spearce.org>
CC: Daniel Barkalow <barkalow@iabervon.org>
---
 Documentation/git-remote-helpers.txt |   18 ++++++
 remote-curl.c                        |   98 ++++++++++++++++++++++++++++++----
 transport-helper.c                   |   58 +++++++++++++++-----
 3 files changed, 149 insertions(+), 25 deletions(-)
diff --git a/Documentation/git-remote-helpers.txt b/Documentation/git-remote-helpers.txt
index 173ee23..e10ce99 100644
--- a/Documentation/git-remote-helpers.txt
+++ b/Documentation/git-remote-helpers.txt
@@ -43,6 +43,15 @@ Commands are given by the caller on the helper's standard input, one per line.
 +
 Supported if the helper has the "fetch" capability.
 
+'fetch-multiple'::
+	Fetches multiple objects at once.  The fetch-multiple
+	command is followed by one or more 'fetch' lines as above,
+	and then a blank line to terminate the batch.  Outputs a
+	single blank line when the entire batch is complete.
+	Optionally may output a 'lock <file>' line indicating a
+	file under GIT_DIR/objects/pack which is keeping a pack
+	until refs can be suitably updated.
+
 If a fatal error occurs, the program writes the error message to
 stderr and exits. The caller should expect that a suitable error
 message has been printed if the child closes the connection without
@@ -57,11 +66,20 @@ CAPABILITIES
 'fetch'::
 	This helper supports the 'fetch' command.
 
+'fetch-multiple'::
+	This helper supports the 'fetch-multiple' command.
+
 REF LIST ATTRIBUTES
 -------------------
 
 None are defined yet, but the caller must accept any which are supplied.
 
+FETCH OPTIONS
+-------------
+
+'option verbose'::
+	Print more verbose activity messages to stderr.
+
 Documentation
 -------------
 Documentation by Daniel Barkalow.
diff --git a/remote-curl.c b/remote-curl.c
index 4628ee8..34ca4e7 100644
--- a/remote-curl.c
+++ b/remote-curl.c
@@ -86,6 +86,84 @@ static struct ref *get_refs(void)
 	return refs;
 }
 
+static int fetch_dumb(int nr_heads, struct ref **to_fetch)
+{
+	char **targets = xmalloc(nr_heads * sizeof(char*));
+	int ret, i;
+
+	for (i = 0; i < nr_heads; i++)
+		targets[i] = xstrdup(sha1_to_hex(to_fetch[i]->old_sha1));
+
+	init_walker();
+	walker->get_all = 1;
+	walker->get_tree = 1;
+	walker->get_history = 1;
+	walker->get_verbosely = 0;
+	walker->get_recover = 0;
+	ret = walker_fetch(walker, nr_heads, targets, NULL, NULL);
+
+	for (i = 0; i < nr_heads; i++)
+		free(targets[i]);
+	free(targets);
+
+	return ret ? error("Fetch failed.") : 0;
+}
+
+static void parse_fetch(struct strbuf *buf, int multiple)
+{
+	struct ref **to_fetch = NULL;
+	struct ref *list_head = NULL;
+	struct ref **list = &list_head;
+	int alloc_heads = 0, nr_heads = 0;
+
+	do {
+		if (!prefixcmp(buf->buf, "fetch ")) {
+			char *p = buf->buf + strlen("fetch ");
+			char *name;
+			struct ref *ref;
+			unsigned char old_sha1[20];
+
+			if (strlen(p) < 40 || get_sha1_hex(p, old_sha1))
+				die("protocol error: expected sha/ref, got %s'", p);
+			if (p[40] == ' ')
+				name = p + 41;
+			else if (!p[40])
+				name = "";
+			else
+				die("protocol error: expected sha/ref, got %s'", p);
+
+			ref = alloc_ref(name);
+			hashcpy(ref->old_sha1, old_sha1);
+
+			*list = ref;
+			list = &ref->next;
+
+			ALLOC_GROW(to_fetch, nr_heads + 1, alloc_heads);
+			to_fetch[nr_heads++] = ref;
+
+			if (!multiple)
+				break;
+		}
+		else
+			die("http transport does not support %s", buf->buf);
+
+		strbuf_reset(buf);
+		if (strbuf_getline(buf, stdin, '\n') == EOF)
+			return;
+		if (!*buf->buf)
+			break;
+	} while (1);
+
+	if (fetch_dumb(nr_heads, to_fetch))
+		exit(128); /* error already reported */
+	free_refs(list_head);
+	free(to_fetch);
+
+	printf("\n");
+	fflush(stdout);
+	strbuf_reset(buf);
+}
+
 int main(int argc, const char **argv)
 {
 	struct strbuf buf = STRBUF_INIT;
@@ -108,17 +186,14 @@ int main(int argc, const char **argv)
 		if (strbuf_getline(&buf, stdin, '\n') == EOF)
 			break;
 		if (!prefixcmp(buf.buf, "fetch ")) {
-			char *obj = buf.buf + strlen("fetch ");
-			init_walker();
-			walker->get_all = 1;
-			walker->get_tree = 1;
-			walker->get_history = 1;
-			walker->get_verbosely = 0;
-			walker->get_recover = 0;
-			if (walker_fetch(walker, 1, &obj, NULL, NULL))
-				die("Fetch failed.");
-			printf("\n");
-			fflush(stdout);
+			parse_fetch(&buf, 0);
+
+		} else if (!strcmp(buf.buf, "fetch-multiple")) {
+			strbuf_reset(&buf);
+			if (strbuf_getline(&buf, stdin, '\n') == EOF)
+				break;
+			parse_fetch(&buf, 1);
+
 		} else if (!strcmp(buf.buf, "list")) {
 			struct ref *refs = get_refs();
 			struct ref *posn;
@@ -132,6 +207,7 @@ int main(int argc, const char **argv)
 			fflush(stdout);
 		} else if (!strcmp(buf.buf, "capabilities")) {
 			printf("fetch\n");
+			printf("fetch-multiple\n");
 			printf("\n");
 			fflush(stdout);
 		} else {
diff --git a/transport-helper.c b/transport-helper.c
index f57e84c..eb66e0c 100644
--- a/transport-helper.c
+++ b/transport-helper.c
@@ -10,7 +10,9 @@ struct helper_data
 {
 	const char *name;
 	struct child_process *helper;
-	unsigned fetch : 1;
+	FILE *out;
+	unsigned fetch : 1,
+		fetch_multiple : 1;
 };
 
 static struct child_process *get_helper(struct transport *transport)
@@ -18,7 +20,6 @@ static struct child_process *get_helper(struct transport *transport)
 	struct helper_data *data = transport->data;
 	struct strbuf buf = STRBUF_INIT;
 	struct child_process *helper;
-	FILE *file;
 
 	if (data->helper)
 		return data->helper;
@@ -39,15 +40,17 @@ static struct child_process *get_helper(struct transport *transport)
 
 	write_str_in_full(helper->in, "capabilities\n");
 
-	file = xfdopen(helper->out, "r");
+	data->out = xfdopen(helper->out, "r");
 	while (1) {
-		if (strbuf_getline(&buf, file, '\n') == EOF)
+		if (strbuf_getline(&buf, data->out, '\n') == EOF)
 			exit(128); /* child died, message supplied already */
 
 		if (!*buf.buf)
 			break;
 		if (!strcmp(buf.buf, "fetch"))
 			data->fetch = 1;
+		if (!strcmp(buf.buf, "fetch-multiple"))
+			data->fetch_multiple = 1;
 	}
 	return data->helper;
 }
@@ -58,6 +61,7 @@ static int disconnect_helper(struct transport *transport)
 	if (data->helper) {
 		write_str_in_full(data->helper->in, "\n");
 		close(data->helper->in);
+		fclose(data->out);
 		finish_command(data->helper);
 		free((char *)data->helper->argv[0]);
 		free(data->helper->argv);
@@ -67,14 +71,37 @@ static int disconnect_helper(struct transport *transport)
 	return 0;
 }
 
+static void perform_fetch_command(struct transport *transport,
+	struct strbuf *buf)
+{
+	struct helper_data *data = transport->data;
+	size_t n = buf->len;
+
+	if (write_in_full(data->helper->in, buf->buf, n) != n)
+		exit(128);
+
+	while (1) {
+		strbuf_reset(buf);
+		if (strbuf_getline(buf, data->out, '\n') == EOF)
+			exit(128); /* child died, message supplied already */
+		if (!prefixcmp(buf->buf, "lock "))
+			transport->pack_lockfile = xstrdup(buf->buf + 5);
+		else if (!buf->len)
+			break;
+	}
+	strbuf_reset(buf);
+}
+
 static int fetch_with_fetch(struct transport *transport,
 			    int nr_heads, const struct ref **to_fetch)
 {
-	struct child_process *helper = get_helper(transport);
-	FILE *file = xfdopen(helper->out, "r");
+	struct helper_data *data = transport->data;
 	int i;
 	struct strbuf buf = STRBUF_INIT;
 
+	if (data->fetch_multiple)
+		strbuf_addstr(&buf, "fetch-multiple\n");
+
 	for (i = 0; i < nr_heads; i++) {
 		const struct ref *posn = to_fetch[i];
 		if (posn->status & REF_STATUS_UPTODATE)
@@ -82,12 +109,16 @@ static int fetch_with_fetch(struct transport *transport,
 
 		strbuf_addf(&buf, "fetch %s %s\n",
 			    sha1_to_hex(posn->old_sha1), posn->name);
-		write_in_full(helper->in, buf.buf, buf.len);
-		strbuf_reset(&buf);
+		if (!data->fetch_multiple)
+			perform_fetch_command(transport, &buf);
+	}
 
-		if (strbuf_getline(&buf, file, '\n') == EOF)
-			exit(128); /* child died, message supplied already */
+	if (data->fetch_multiple) {
+		strbuf_addch(&buf, '\n');
+		perform_fetch_command(transport, &buf);
 	}
+
+	strbuf_release(&buf);
 	return 0;
 }
 
@@ -105,7 +136,7 @@ static int fetch(struct transport *transport,
 	if (!count)
 		return 0;
 
-	if (data->fetch)
+	if (data->fetch || data->fetch_multiple)
 		return fetch_with_fetch(transport, nr_heads, to_fetch);
 
 	return -1;
@@ -113,21 +144,20 @@ static int fetch(struct transport *transport,
 
 static struct ref *get_refs_list(struct transport *transport, int for_push)
 {
+	struct helper_data *data = transport->data;
 	struct child_process *helper;
 	struct ref *ret = NULL;
 	struct ref **tail = &ret;
 	struct ref *posn;
 	struct strbuf buf = STRBUF_INIT;
-	FILE *file;
 
 	helper = get_helper(transport);
 
 	write_str_in_full(helper->in, "list\n");
 
-	file = xfdopen(helper->out, "r");
 	while (1) {
 		char *eov, *eon;
-		if (strbuf_getline(&buf, file, '\n') == EOF)
+		if (strbuf_getline(&buf, data->out, '\n') == EOF)
 			exit(128); /* child died, message supplied already */
 
 		if (!*buf.buf)

-- 
1.6.5.52.g0ff2e
Previous: Shawn O. PearceNext: Daniel Barkalow
Message 12 of 38 in “Return of smart HTTP”
  1. 00/16 Return of smart HTTPShawn O. Pearce, Oct 13, 2009
  2. 01/16 pkt-line: Add strbuf based functionsShawn O. Pearce, Oct 13, 2009
  3. Johannes SixtOct 13, 2009
  4. Shawn O. PearceOct 13, 2009
  5. 02/16 pkt-line: Make packet_read_line easier to debugShawn O. Pearce, Oct 13, 2009
  6. 03/16 fetch-pack: Use a strbuf to compose the want listShawn O. Pearce, Oct 13, 2009
  7. 04/16 Move "get_ack()" back to fetch-packShawn O. Pearce, Oct 13, 2009
  8. 05/16 Add multi_ack_2 capability to fetch-pack/upload-packShawn O. Pearce, Oct 13, 2009
  9. Jakub NarebskiOct 13, 2009
  10. Shawn O. PearceOct 13, 2009
  11. 06/16 remote-curl: Refactor walker initializationShawn O. Pearce, Oct 13, 2009
  12. 07/16 remote-helpers: Fetch more than one ref in a batchShawn O. Pearce, Oct 13, 2009
  13. Daniel BarkalowOct 13, 2009
  14. Shawn O. PearceOct 13, 2009
  15. 08/16 remote-helpers: Support custom transport optionsShawn O. Pearce, Oct 13, 2009
  16. Daniel BarkalowOct 13, 2009
  17. Shawn O. PearceOct 13, 2009
  18. Daniel BarkalowOct 13, 2009
  19. Shawn O. PearceOct 13, 2009
  20. Daniel BarkalowOct 13, 2009
  21. Shawn O. PearceOct 13, 2009
  22. 09/16 Move WebDAV HTTP push under remote-curlShawn O. Pearce, Oct 13, 2009
  23. Mike HommeyOct 13, 2009
  24. Johannes SchindelinOct 13, 2009
  25. 10/16 Git-aware CGI to provide dumb HTTP transportShawn O. Pearce, Oct 13, 2009
  26. Johannes SixtOct 13, 2009
  27. 11/16 Add one shot RPC options to upload-pack, receive-packShawn O. Pearce, Oct 13, 2009
  28. 12/16 Smart fetch and push over HTTP: server sideShawn O. Pearce, Oct 13, 2009
  29. Johannes SixtOct 13, 2009
  30. Shawn O. PearceOct 13, 2009
  31. 13/16 Discover refs via smart HTTP server when availableShawn O. Pearce, Oct 13, 2009
  32. 14/16 Smart push over HTTP: client sideShawn O. Pearce, Oct 13, 2009
  33. Felipe ContrerasOct 13, 2009
  34. 15/16 Smart fetch over HTTP: client sideShawn O. Pearce, Oct 13, 2009
  35. 16/16 Smart HTTP fetch: gzip requestsShawn O. Pearce, Oct 13, 2009
  36. Junio C HamanoOct 13, 2009
  37. eduard stefanOct 13, 2009
  38. Junio C HamanoOct 13, 2009

Read the whole thread, see it on lore, or plain text.

$ cat FOOTERMessages come from the public archive at lore.kernel.org/git, fetched every hour. The front page is chosen and written each morning by an AI editor and can be wrong; the threads themselves are the record. About and API. For agents: an MCP server at https://gitlist.dev/mcp, and any thread, story or person page as Markdown by adding .md to its URL (or sending Accept: text/markdown). Details in /llms.txt.