git/list[1] front-page[2] threads[3] people[4] search[5] about
 

[RFC PATCH 7/7] promisor-remote: use subprocess to fetch

From
JTJonathan Tan <jonathantanmy@google.com>
Date
Aug 5, 2020, 01:20 UTC
Message-ID
<38bbad333e3eda1322943e29efabad9d9c6a996a.1596590295.git.jonathantanmy@google.com>
In-Reply-To
<cover.1596590295.git.jonathantanmy@google.com>
---
 promisor-remote.c             | 46 ++++++++++++++++-------------------
 t/t0410-partial-clone.sh      |  2 +-
 t/t4067-diff-partial-clone.sh |  8 +++---
 t/t5601-clone.sh              |  2 +-
 4 files changed, 27 insertions(+), 31 deletions(-)
diff --git a/promisor-remote.c b/promisor-remote.c
index baaea12fd6..6e647610e9 100644
--- a/promisor-remote.c
+++ b/promisor-remote.c
@@ -3,6 +3,7 @@
 #include "promisor-remote.h"
 #include "config.h"
 #include "transport.h"
+#include "argv-array.h"
 
 static char *repository_format_partial_clone;
 static const char *core_partial_clone_filter_default;
@@ -12,39 +13,34 @@ void set_repository_format_partial_clone(char *partial_clone)
 	repository_format_partial_clone = xstrdup_or_null(partial_clone);
 }
 
-static int fetch_refs(const char *remote_name, struct ref *ref)
-{
-	struct remote *remote;
-	struct transport *transport;
-	int res;
-
-	remote = remote_get(remote_name);
-	if (!remote->url[0])
-		die(_("Remote with no URL"));
-	transport = transport_get(remote, remote->url[0]);
-
-	transport_set_option(transport, TRANS_OPT_FROM_PROMISOR, "1");
-	transport_set_option(transport, TRANS_OPT_NO_DEPENDENTS, "1");
-	res = transport_fetch_refs(transport, ref);
-
-	return res;
-}
-
 static int fetch_objects(const char *remote_name,
 			 const struct object_id *oids,
 			 int oid_nr)
 {
-	struct ref *ref = NULL;
+	struct child_process child = CHILD_PROCESS_INIT;
 	int i;
+	FILE *child_in;
+
+	child.git_cmd = 1;
+	child.in = -1;
+	argv_array_pushl(&child.args, "-c", "fetch.negotiationAlgorithm=null",
+			 "fetch", remote_name, "--no-tags",
+			 "--no-write-fetch-head", "--recurse-submodules=no",
+			 "--filter=blob:none", "--stdin", NULL);
+	if (start_command(&child))
+		die(_("promisor-remote: unable to fork off fetch subprocess"));
+	child_in = xfdopen(child.in, "w");
 
 	for (i = 0; i < oid_nr; i++) {
-		struct ref *new_ref = alloc_ref(oid_to_hex(&oids[i]));
-		oidcpy(&new_ref->old_oid, &oids[i]);
-		new_ref->exact_oid = 1;
-		new_ref->next = ref;
-		ref = new_ref;
+		if (fputs(oid_to_hex(&oids[i]), child_in) < 0)
+			die_errno(_("promisor-remote: could not write to fetch subprocess"));
+		if (fputc('\n', child_in) < 0)
+			die_errno(_("promisor-remote: could not write to fetch subprocess"));
 	}
-	return fetch_refs(remote_name, ref);
+
+	if (fclose(child_in) < 0)
+		die_errno(_("promisor-remote: could not close stdin to fetch subprocess"));
+	return finish_command(&child) ? -1 : 0;
 }
 
 static struct promisor_remote *promisors;
diff --git a/t/t0410-partial-clone.sh b/t/t0410-partial-clone.sh
index 463dc3a8be..3e454f934e 100755
--- a/t/t0410-partial-clone.sh
+++ b/t/t0410-partial-clone.sh
@@ -203,7 +203,7 @@ test_expect_success 'fetching of missing objects works with ref-in-want enabled'
 	rm -rf repo/.git/objects/* &&
 	rm -f trace &&
 	GIT_TRACE_PACKET="$(pwd)/trace" git -C repo cat-file -p "$HASH" &&
-	grep "git< fetch=.*ref-in-want" trace
+	grep "fetch< fetch=.*ref-in-want" trace
 '
 
 test_expect_success 'fetching of missing objects from another promisor remote' '
diff --git a/t/t4067-diff-partial-clone.sh b/t/t4067-diff-partial-clone.sh
index ef8e0e9cb0..804f2a82e8 100755
--- a/t/t4067-diff-partial-clone.sh
+++ b/t/t4067-diff-partial-clone.sh
@@ -20,7 +20,7 @@ test_expect_success 'git show batches blobs' '
 	# Ensure that there is exactly 1 negotiation by checking that there is
 	# only 1 "done" line sent. ("done" marks the end of negotiation.)
 	GIT_TRACE_PACKET="$(pwd)/trace" git -C client show HEAD &&
-	grep "git> done" trace >done_lines &&
+	grep "fetch> done" trace >done_lines &&
 	test_line_count = 1 done_lines
 '
 
@@ -44,7 +44,7 @@ test_expect_success 'diff batches blobs' '
 	# Ensure that there is exactly 1 negotiation by checking that there is
 	# only 1 "done" line sent. ("done" marks the end of negotiation.)
 	GIT_TRACE_PACKET="$(pwd)/trace" git -C client diff HEAD^ HEAD &&
-	grep "git> done" trace >done_lines &&
+	grep "fetch> done" trace >done_lines &&
 	test_line_count = 1 done_lines
 '
 
@@ -127,7 +127,7 @@ test_expect_success 'diff with rename detection batches blobs' '
 	# only 1 "done" line sent. ("done" marks the end of negotiation.)
 	GIT_TRACE_PACKET="$(pwd)/trace" git -C client diff --raw -M HEAD^ HEAD >out &&
 	grep ":100644 100644.*R[0-9][0-9][0-9].*b.*c" out &&
-	grep "git> done" trace >done_lines &&
+	grep "fetch> done" trace >done_lines &&
 	test_line_count = 1 done_lines
 '
 
@@ -175,7 +175,7 @@ test_expect_success 'diff --break-rewrites fetches only if necessary, and batche
 	# by checking that there is only 1 "done" line sent. ("done" marks the
 	# end of negotiation.)
 	GIT_TRACE_PACKET="$(pwd)/trace" git -C client diff --break-rewrites --raw -M HEAD^ HEAD &&
-	grep "git> done" trace >done_lines &&
+	grep "fetch> done" trace >done_lines &&
 	test_line_count = 1 done_lines
 '
 
diff --git a/t/t5601-clone.sh b/t/t5601-clone.sh
index 84ea2a3eb7..f82b0dbb5a 100755
--- a/t/t5601-clone.sh
+++ b/t/t5601-clone.sh
@@ -702,7 +702,7 @@ test_expect_success 'batch missing blob request during checkout' '
 	# Ensure that there is only one negotiation by checking that there is
 	# only "done" line sent. ("done" marks the end of negotiation.)
 	GIT_TRACE_PACKET="$(pwd)/trace" git -C client checkout HEAD^ &&
-	grep "git> done" trace >done_lines &&
+	grep "fetch> done" trace >done_lines &&
 	test_line_count = 1 done_lines
 '
 
-- 
2.28.0.163.g6104cc2f0b6-goog
Previous: Jonathan TanNext: Jonathan Tan
Message 28 of 57 in “Modify fetch-pack to no longer die on error?”
  1. Modify fetch-pack to no longer die on error?Jonathan Tan, Jul 24, 2020
  2. Junio C HamanoJul 24, 2020
  3. Junio C HamanoJul 24, 2020
  4. Jeff KingJul 25, 2020
  5. Junio C HamanoJul 25, 2020
  6. Jeff KingJul 27, 2020
  7. Jonathan TanJul 28, 2020
  8. Jeff KingJul 28, 2020
  9. Jonathan TanJul 29, 2020
  10. Jeff KingJul 29, 2020
  11. Junio C HamanoJul 29, 2020
  12. Jonathan TanJul 29, 2020
  13. 0/7 Lazy fetch with subprocessJonathan Tan, Aug 5, 2020
  14. 1/7 fetch-pack: allow NULL negotiator->add_tipJonathan Tan, Aug 5, 2020
  15. Junio C HamanoAug 5, 2020
  16. Jonathan TanAug 7, 2020
  17. 3/7 negotiator/null: add null fetch negotiatorJonathan Tan, Aug 5, 2020
  18. 2/7 fetch-pack: allow NULL negotiator->known_commonJonathan Tan, Aug 5, 2020
  19. Junio C HamanoAug 5, 2020
  20. Junio C HamanoAug 5, 2020
  21. Jonathan TanAug 7, 2020
  22. 4/7 fetch: --stdinJonathan Tan, Aug 5, 2020
  23. Junio C HamanoAug 5, 2020
  24. Jonathan TanAug 7, 2020
  25. Junio C HamanoAug 7, 2020
  26. Jonathan TanAug 7, 2020
  27. 5/7 fetch: submodule configJonathan Tan, Aug 5, 2020
  28. 7/7 promisor-remote: use subprocess to fetchJonathan Tan, Aug 5, 2020
  29. 6/7 fetch: only populate existing_refs if neededJonathan Tan, Aug 5, 2020
  30. 0/7 Lazy fetch with subprocessJonathan Tan, Aug 11, 2020
  31. 1/7 negotiator/null: add null fetch negotiatorJonathan Tan, Aug 11, 2020
  32. Derrick StoleeAug 12, 2020
  33. Junio C HamanoAug 12, 2020
  34. Jonathan TanAug 12, 2020
  35. 2/7 fetch: allow refspecs specified through stdinJonathan Tan, Aug 11, 2020
  36. 3/7 fetch: avoid reading submodule config until neededJonathan Tan, Aug 11, 2020
  37. Junio C HamanoAug 12, 2020
  38. 4/7 fetch: only populate existing_refs if neededJonathan Tan, Aug 11, 2020
  39. Junio C HamanoAug 12, 2020
  40. 5/7 fetch-pack: do not lazy-fetch during ref iterationJonathan Tan, Aug 11, 2020
  41. Junio C HamanoAug 12, 2020
  42. 6/7 promisor-remote: lazy-fetch objects in subprocessJonathan Tan, Aug 11, 2020
  43. Junio C HamanoAug 12, 2020
  44. 7/7 fetch-pack: remove no_dependents codeJonathan Tan, Aug 11, 2020
  45. Derrick StoleeAug 12, 2020
  46. 0/7 Lazy fetch with subprocessJonathan Tan, Aug 18, 2020
  47. 1/7 negotiator/noop: add noop fetch negotiatorJonathan Tan, Aug 18, 2020
  48. 2/7 fetch: allow refspecs specified through stdinJonathan Tan, Aug 18, 2020
  49. 4/7 fetch: only populate existing_refs if neededJonathan Tan, Aug 18, 2020
  50. 5/7 fetch-pack: do not lazy-fetch during ref iterationJonathan Tan, Aug 18, 2020
  51. 3/7 fetch: avoid reading submodule config until neededJonathan Tan, Aug 18, 2020
  52. 6/7 promisor-remote: lazy-fetch objects in subprocessJonathan Tan, Aug 18, 2020
  53. 7/7 fetch-pack: remove no_dependents codeJonathan Tan, Aug 18, 2020
  54. Junio C HamanoAug 18, 2020
  55. Junio C HamanoAug 18, 2020
  56. fixup! promisor-remote: lazy-fetch objects in subprocessJonathan Tan, Aug 18, 2020
  57. Junio C HamanoAug 18, 2020

Read the whole thread, see it on lore, or plain text.

$ cat FOOTERMessages come from the public archive at lore.kernel.org/git, fetched every hour. The front page is chosen and written each morning by an AI editor and can be wrong; the threads themselves are the record. About and API. For agents: an MCP server at https://gitlist.dev/mcp, and any thread, story or person page as Markdown by adding .md to its URL (or sending Accept: text/markdown). Details in /llms.txt.