git/list[1] front-page[2] threads[3] people[4] search[5] about
 

[RFC PATCH v4 09/26] remote-helpers: Fetch more than one ref in a batch

From
Shawn O. Pearce <spearce@spearce.org>
Date
Oct 29, 2009, 00:00 UTC
Message-ID
<1256774448-7625-10-git-send-email-spearce@spearce.org>
In-Reply-To
<1256774448-7625-1-git-send-email-spearce@spearce.org>

Some network protocols (e.g. native git://) are able to fetch more than one ref at a time and reduce the overall transfer cost by combining the requests into a single exchange. Instead of feeding each fetch request one at a time to the helper, feed all of them at once so the helper can decide whether or not it should batch them.

Signed-off-by: Shawn O. Pearce <spearce@spearce.org>
CC: Daniel Barkalow <barkalow@iabervon.org>
---
 Documentation/git-remote-helpers.txt |   14 ++++--
 remote-curl.c                        |   88 +++++++++++++++++++++++++++++----
 transport-helper.c                   |   39 +++++++++++----
 3 files changed, 115 insertions(+), 26 deletions(-)
diff --git a/Documentation/git-remote-helpers.txt b/Documentation/git-remote-helpers.txt
index 173ee23..e44d821 100644
--- a/Documentation/git-remote-helpers.txt
+++ b/Documentation/git-remote-helpers.txt
@@ -36,10 +36,16 @@ Commands are given by the caller on the helper's standard input, one per line.
 	complete list, outputs a blank line.
 
 'fetch' <sha1> <name>::
-	Fetches the given object, writing the necessary objects to the
-	database. Outputs a blank line when the fetch is
-	complete. Only objects which were reported in the ref list
-	with a sha1 may be fetched this way.
+	Fetches the given object, writing the necessary objects
+	to the database.  Fetch commands are sent in a batch, one
+	per line, and the batch is terminated with a blank line.
+	Outputs a single blank line when all fetch commands in the
+	same batch are complete. Only objects which were reported
+	in the ref list with a sha1 may be fetched this way.
++
+Optionally may output a 'lock <file>' line indicating a file under
+GIT_DIR/objects/pack which is keeping a pack until refs can be
+suitably updated.
 +
 Supported if the helper has the "fetch" capability.
 
diff --git a/remote-curl.c b/remote-curl.c
index 478f3ea..22cd5c5 100644
--- a/remote-curl.c
+++ b/remote-curl.c
@@ -87,6 +87,81 @@ static struct ref *get_refs(void)
 	return refs;
 }
 
+static int fetch_dumb(int nr_heads, struct ref **to_fetch)
+{
+	char **targets = xmalloc(nr_heads * sizeof(char*));
+	int ret, i;
+
+	for (i = 0; i < nr_heads; i++)
+		targets[i] = xstrdup(sha1_to_hex(to_fetch[i]->old_sha1));
+
+	init_walker();
+	walker->get_all = 1;
+	walker->get_tree = 1;
+	walker->get_history = 1;
+	walker->get_verbosely = 0;
+	walker->get_recover = 0;
+	ret = walker_fetch(walker, nr_heads, targets, NULL, NULL);
+
+	for (i = 0; i < nr_heads; i++)
+		free(targets[i]);
+	free(targets);
+
+	return ret ? error("Fetch failed.") : 0;
+}
+
+static void parse_fetch(struct strbuf *buf)
+{
+	struct ref **to_fetch = NULL;
+	struct ref *list_head = NULL;
+	struct ref **list = &list_head;
+	int alloc_heads = 0, nr_heads = 0;
+
+	do {
+		if (!prefixcmp(buf->buf, "fetch ")) {
+			char *p = buf->buf + strlen("fetch ");
+			char *name;
+			struct ref *ref;
+			unsigned char old_sha1[20];
+
+			if (strlen(p) < 40 || get_sha1_hex(p, old_sha1))
+				die("protocol error: expected sha/ref, got %s'", p);
+			if (p[40] == ' ')
+				name = p + 41;
+			else if (!p[40])
+				name = "";
+			else
+				die("protocol error: expected sha/ref, got %s'", p);
+
+			ref = alloc_ref(name);
+			hashcpy(ref->old_sha1, old_sha1);
+
+			*list = ref;
+			list = &ref->next;
+
+			ALLOC_GROW(to_fetch, nr_heads + 1, alloc_heads);
+			to_fetch[nr_heads++] = ref;
+		}
+		else
+			die("http transport does not support %s", buf->buf);
+
+		strbuf_reset(buf);
+		if (strbuf_getline(buf, stdin, '\n') == EOF)
+			return;
+		if (!*buf->buf)
+			break;
+	} while (1);
+
+	if (fetch_dumb(nr_heads, to_fetch))
+		exit(128); /* error already reported */
+	free_refs(list_head);
+	free(to_fetch);
+
+	printf("\n");
+	fflush(stdout);
+	strbuf_reset(buf);
+}
+
 int main(int argc, const char **argv)
 {
 	struct strbuf buf = STRBUF_INIT;
@@ -110,17 +185,8 @@ int main(int argc, const char **argv)
 		if (strbuf_getline(&buf, stdin, '\n') == EOF)
 			break;
 		if (!prefixcmp(buf.buf, "fetch ")) {
-			char *obj = buf.buf + strlen("fetch ");
-			init_walker();
-			walker->get_all = 1;
-			walker->get_tree = 1;
-			walker->get_history = 1;
-			walker->get_verbosely = 0;
-			walker->get_recover = 0;
-			if (walker_fetch(walker, 1, &obj, NULL, NULL))
-				die("Fetch failed.");
-			printf("\n");
-			fflush(stdout);
+			parse_fetch(&buf);
+
 		} else if (!strcmp(buf.buf, "list")) {
 			struct ref *refs = get_refs();
 			struct ref *posn;
diff --git a/transport-helper.c b/transport-helper.c
index f57e84c..9de3408 100644
--- a/transport-helper.c
+++ b/transport-helper.c
@@ -10,6 +10,7 @@ struct helper_data
 {
 	const char *name;
 	struct child_process *helper;
+	FILE *out;
 	unsigned fetch : 1;
 };
 
@@ -18,7 +19,6 @@ static struct child_process *get_helper(struct transport *transport)
 	struct helper_data *data = transport->data;
 	struct strbuf buf = STRBUF_INIT;
 	struct child_process *helper;
-	FILE *file;
 
 	if (data->helper)
 		return data->helper;
@@ -39,9 +39,9 @@ static struct child_process *get_helper(struct transport *transport)
 
 	write_str_in_full(helper->in, "capabilities\n");
 
-	file = xfdopen(helper->out, "r");
+	data->out = xfdopen(helper->out, "r");
 	while (1) {
-		if (strbuf_getline(&buf, file, '\n') == EOF)
+		if (strbuf_getline(&buf, data->out, '\n') == EOF)
 			exit(128); /* child died, message supplied already */
 
 		if (!*buf.buf)
@@ -58,6 +58,7 @@ static int disconnect_helper(struct transport *transport)
 	if (data->helper) {
 		write_str_in_full(data->helper->in, "\n");
 		close(data->helper->in);
+		fclose(data->out);
 		finish_command(data->helper);
 		free((char *)data->helper->argv[0]);
 		free(data->helper->argv);
@@ -70,8 +71,7 @@ static int disconnect_helper(struct transport *transport)
 static int fetch_with_fetch(struct transport *transport,
 			    int nr_heads, const struct ref **to_fetch)
 {
-	struct child_process *helper = get_helper(transport);
-	FILE *file = xfdopen(helper->out, "r");
+	struct helper_data *data = transport->data;
 	int i;
 	struct strbuf buf = STRBUF_INIT;
 
@@ -82,12 +82,30 @@ static int fetch_with_fetch(struct transport *transport,
 
 		strbuf_addf(&buf, "fetch %s %s\n",
 			    sha1_to_hex(posn->old_sha1), posn->name);
-		write_in_full(helper->in, buf.buf, buf.len);
-		strbuf_reset(&buf);
+	}
 
-		if (strbuf_getline(&buf, file, '\n') == EOF)
+	strbuf_addch(&buf, '\n');
+	if (write_in_full(data->helper->in, buf.buf, buf.len) != buf.len)
+		die_errno("cannot send fetch to %s", data->name);
+
+	while (1) {
+		strbuf_reset(&buf);
+		if (strbuf_getline(&buf, data->out, '\n') == EOF)
 			exit(128); /* child died, message supplied already */
+
+		if (!prefixcmp(buf.buf, "lock ")) {
+			const char *name = buf.buf + 5;
+			if (transport->pack_lockfile)
+				warning("%s also locked %s", data->name, name);
+			else
+				transport->pack_lockfile = xstrdup(name);
+		}
+		else if (!buf.len)
+			break;
+		else
+			warning("%s unexpectedly said: '%s'", data->name, buf.buf);
 	}
+	strbuf_release(&buf);
 	return 0;
 }
 
@@ -113,21 +131,20 @@ static int fetch(struct transport *transport,
 
 static struct ref *get_refs_list(struct transport *transport, int for_push)
 {
+	struct helper_data *data = transport->data;
 	struct child_process *helper;
 	struct ref *ret = NULL;
 	struct ref **tail = &ret;
 	struct ref *posn;
 	struct strbuf buf = STRBUF_INIT;
-	FILE *file;
 
 	helper = get_helper(transport);
 
 	write_str_in_full(helper->in, "list\n");
 
-	file = xfdopen(helper->out, "r");
 	while (1) {
 		char *eov, *eon;
-		if (strbuf_getline(&buf, file, '\n') == EOF)
+		if (strbuf_getline(&buf, data->out, '\n') == EOF)
 			exit(128); /* child died, message supplied already */
 
 		if (!*buf.buf)

-- 
1.6.5.2.181.gd6f41
Previous: Shawn O. PearceNext: Shawn O. Pearce
Message 23 of 60 in “Return of smart HTTP”
  1. 00/26 Return of smart HTTPShawn O. Pearce, Oct 29, 2009
  2. 01/26 http-push: fix check condition on http.c::finish_http_pack_request()Shawn O. Pearce, Oct 29, 2009
  3. 02/26 pkt-line: Add strbuf based functionsShawn O. Pearce, Oct 29, 2009
  4. 03/26 pkt-line: Make packet_read_line easier to debugShawn O. Pearce, Oct 29, 2009
  5. Junio C HamanoOct 29, 2009
  6. Shawn O. PearceOct 29, 2009
  7. Junio C HamanoOct 29, 2009
  8. Shawn O. PearceOct 29, 2009
  9. Jeff KingOct 30, 2009
  10. Shawn O. PearceOct 30, 2009
  11. Jeff KingOct 30, 2009
  12. David BrownOct 30, 2009
  13. Junio C HamanoOct 30, 2009
  14. 04/26 fetch-pack: Use a strbuf to compose the want listShawn O. Pearce, Oct 29, 2009
  15. 05/26 Move "get_ack()" back to fetch-packShawn O. Pearce, Oct 29, 2009
  16. Junio C HamanoOct 29, 2009
  17. Shawn O. PearceOct 29, 2009
  18. 06/26 Add multi_ack_detailed capability to fetch-pack/upload-packShawn O. Pearce, Oct 29, 2009
  19. Junio C HamanoOct 29, 2009
  20. Shawn O. PearceOct 29, 2009
  21. 07/26 remote-curl: Refactor walker initializationShawn O. Pearce, Oct 29, 2009
  22. 08/26 fetch: Allow transport -v -v -v to set verbosity to 3Shawn O. Pearce, Oct 29, 2009
  23. 09/26 remote-helpers: Fetch more than one ref in a batchShawn O. Pearce, Oct 29, 2009
  24. 10/26 remote-helpers: Support custom transport optionsShawn O. Pearce, Oct 29, 2009
  25. 11/26 Move WebDAV HTTP push under remote-curlShawn O. Pearce, Oct 29, 2009
  26. Tay Ray ChuanOct 30, 2009
  27. Shawn O. PearceOct 31, 2009
  28. Tay Ray ChuanOct 30, 2009
  29. Clemens BuchacherOct 30, 2009
  30. Tay Ray ChuanOct 30, 2009
  31. 12/26 remote-helpers: return successfully if everything up-to-dateShawn O. Pearce, Oct 29, 2009
  32. 13/26 Git-aware CGI to provide dumb HTTP transportShawn O. Pearce, Oct 29, 2009
  33. 14/26 Add stateless RPC options to upload-pack, receive-packShawn O. Pearce, Oct 29, 2009
  34. Junio C HamanoOct 29, 2009
  35. Shawn O. PearceOct 29, 2009
  36. Junio C HamanoOct 29, 2009
  37. Shawn O. PearceOct 30, 2009
  38. 15/26 Smart fetch and push over HTTP: server sideShawn O. Pearce, Oct 29, 2009
  39. 16/26 http-backend: add GIT_PROJECT_ROOT environment varShawn O. Pearce, Oct 29, 2009
  40. 17/26 http-backend: reword some documentationShawn O. Pearce, Oct 29, 2009
  41. 18/26 http-backend: use mod_alias instead of mod_rewriteShawn O. Pearce, Oct 29, 2009
  42. 19/26 http-backend: add example for gitweb on same URLShawn O. Pearce, Oct 29, 2009
  43. 20/26 http-backend: more explict LocationMatchShawn O. Pearce, Oct 29, 2009
  44. 21/26 Discover refs via smart HTTP server when availableShawn O. Pearce, Oct 29, 2009
  45. 22/26 Smart push over HTTP: client sideShawn O. Pearce, Oct 29, 2009
  46. 23/26 Smart fetch over HTTP: client sideShawn O. Pearce, Oct 29, 2009
  47. 24/26 Smart HTTP fetch: gzip requestsShawn O. Pearce, Oct 29, 2009
  48. 25/26 t5540-http-push: remove redundant fetchesShawn O. Pearce, Oct 29, 2009
  49. 26/26 test smart http fetch and pushShawn O. Pearce, Oct 29, 2009
  50. Clemens BuchacherOct 29, 2009
  51. Shawn O. PearceOct 29, 2009
  52. Junio C HamanoOct 29, 2009
  53. Shawn O. PearceOct 29, 2009
  54. Shawn O. PearceOct 29, 2009
  55. Junio C HamanoOct 30, 2009
  56. Shawn O. PearceOct 30, 2009
  57. Tay Ray ChuanOct 30, 2009
  58. Shawn O. PearceOct 31, 2009
  59. Jakub NarebskiOct 29, 2009
  60. Shawn O. PearceOct 29, 2009

Read the whole thread, see it on lore, or plain text.

$ cat FOOTERMessages come from the public archive at lore.kernel.org/git, fetched every hour. The front page is chosen and written each morning by an AI editor and can be wrong; the threads themselves are the record. About and API. For agents: an MCP server at https://gitlist.dev/mcp, and any thread, story or person page as Markdown by adding .md to its URL (or sending Accept: text/markdown). Details in /llms.txt.