git/list[1] front-page[2] threads[3] people[4] search[5] about
 

[PATCH 4/4] promisor-remote: teach lazy-fetch in any repo

From
JTJonathan Tan <jonathantanmy@google.com>
Date
Jun 1, 2021, 21:34 UTC
Message-ID
<b70a00b9b06e5142bb95891cfc2faa87d708ef0d.1622580781.git.jonathantanmy@google.com>
In-Reply-To
<cover.1622580781.git.jonathantanmy@google.com>
This is one step towards supporting partial clone submodules.

Even after this patch, we will still lack partial clone submodules support, primarily because a lot of Git code that accesses submodule objects does so by adding their object stores as alternates, meaning that any lazy fetches that would occur in the submodule would be done based on the config of the superproject, not of the submodule. This also prevents testing of the functionality in this patch by user-facing commands. So for now, test this mechanism using a test helper.

Signed-off-by: Jonathan Tan <jonathantanmy@google.com>
---
 Makefile                      |  1 +
 object-file.c                 |  7 ++-----
 promisor-remote.c             | 14 +++++++++-----
 t/helper/test-partial-clone.c | 34 ++++++++++++++++++++++++++++++++++
 t/helper/test-tool.c          |  1 +
 t/helper/test-tool.h          |  1 +
 t/t0410-partial-clone.sh      | 24 ++++++++++++++++++++++++
 7 files changed, 72 insertions(+), 10 deletions(-)
 create mode 100644 t/helper/test-partial-clone.c
diff --git a/Makefile b/Makefile
index c3565fc0f8..f6653bcd5e 100644
--- a/Makefile
+++ b/Makefile
@@ -725,6 +725,7 @@ TEST_BUILTINS_OBJS += test-oidmap.o
 TEST_BUILTINS_OBJS += test-online-cpus.o
 TEST_BUILTINS_OBJS += test-parse-options.o
 TEST_BUILTINS_OBJS += test-parse-pathspec-file.o
+TEST_BUILTINS_OBJS += test-partial-clone.o
 TEST_BUILTINS_OBJS += test-path-utils.o
 TEST_BUILTINS_OBJS += test-pcre2-config.o
 TEST_BUILTINS_OBJS += test-pkt-line.o
diff --git a/object-file.c b/object-file.c
index f233b440b2..ebf273e9e7 100644
--- a/object-file.c
+++ b/object-file.c
@@ -1570,15 +1570,12 @@ static int do_oid_object_info_extended(struct repository *r,
 		}
 
 		/* Check if it is a missing object */
-		if (fetch_if_missing && has_promisor_remote() &&
-		    !already_retried && r == the_repository &&
+		if (fetch_if_missing && repo_has_promisor_remote(r) &&
+		    !already_retried &&
 		    !(flags & OBJECT_INFO_SKIP_FETCH_OBJECT)) {
 			/*
 			 * TODO Investigate checking promisor_remote_get_direct()
 			 * TODO return value and stopping on error here.
-			 * TODO Pass a repository struct through
-			 * promisor_remote_get_direct(), such that arbitrary
-			 * repositories work.
 			 */
 			promisor_remote_get_direct(r, real, 1);
 			already_retried = 1;
diff --git a/promisor-remote.c b/promisor-remote.c
index 5819d2cf28..1601f05d79 100644
--- a/promisor-remote.c
+++ b/promisor-remote.c
@@ -11,7 +11,8 @@ struct promisor_remote_config {
 	struct promisor_remote **promisors_tail;
 };
 
-static int fetch_objects(const char *remote_name,
+static int fetch_objects(struct repository *repo,
+			 const char *remote_name,
 			 const struct object_id *oids,
 			 int oid_nr)
 {
@@ -21,6 +22,11 @@ static int fetch_objects(const char *remote_name,
 
 	child.git_cmd = 1;
 	child.in = -1;
+	if (repo != the_repository) {
+		prepare_other_repo_env(&child.env_array);
+		strvec_pushf(&child.env_array, "%s=%s", GIT_DIR_ENVIRONMENT,
+			     repo->gitdir);
+	}
 	strvec_pushl(&child.args, "-c", "fetch.negotiationAlgorithm=noop",
 		     "fetch", remote_name, "--no-tags",
 		     "--no-write-fetch-head", "--recurse-submodules=no",
@@ -150,7 +156,7 @@ static void promisor_remote_init(struct repository *r)
 		xcalloc(sizeof(*r->promisor_remote_config), 1);
 	config->promisors_tail = &config->promisors;
 
-	git_config(promisor_remote_config, config);
+	repo_config(r, promisor_remote_config, config);
 
 	if (config->repository_format_partial_clone) {
 		struct promisor_remote *o, *previous;
@@ -246,10 +252,8 @@ int promisor_remote_get_direct(struct repository *repo,
 
 	promisor_remote_init(repo);
 
-	if (repo != the_repository)
-		BUG("only the_repository is supported for now");
 	for (r = repo->promisor_remote_config->promisors; r; r = r->next) {
-		if (fetch_objects(r->name, remaining_oids, remaining_nr) < 0) {
+		if (fetch_objects(repo, r->name, remaining_oids, remaining_nr) < 0) {
 			if (remaining_nr == 1)
 				continue;
 			remaining_nr = remove_fetched_oids(repo, &remaining_oids,
diff --git a/t/helper/test-partial-clone.c b/t/helper/test-partial-clone.c
new file mode 100644
index 0000000000..e7bc7eb21f
--- /dev/null
+++ b/t/helper/test-partial-clone.c
@@ -0,0 +1,34 @@
+#include "cache.h"
+#include "test-tool.h"
+#include "repository.h"
+#include "object-store.h"
+
+static void object_info(const char *gitdir, const char *oid_hex)
+{
+	struct repository r;
+	struct object_id oid;
+	unsigned long size;
+	struct object_info oi = {.sizep = &size};
+	const char *p;
+
+	if (repo_init(&r, gitdir, NULL))
+		die("could not init repo");
+	if (parse_oid_hex(oid_hex, &oid, &p))
+		die("could not parse oid");
+	if (oid_object_info_extended(&r, &oid, &oi, 0))
+		die("could not obtain object info");
+	printf("%d\n", (int) size);
+}
+
+int cmd__partial_clone(int argc, const char **argv)
+{
+	if (argc < 4)
+		die("too few arguments");
+
+	if (!strcmp(argv[1], "object-info"))
+		object_info(argv[2], argv[3]);
+	else
+		die("invalid argument '%s'", argv[1]);
+
+	return 0;
+}
diff --git a/t/helper/test-tool.c b/t/helper/test-tool.c
index c5bd0c6d4c..b21e8f1519 100644
--- a/t/helper/test-tool.c
+++ b/t/helper/test-tool.c
@@ -46,6 +46,7 @@ static struct test_cmd cmds[] = {
 	{ "online-cpus", cmd__online_cpus },
 	{ "parse-options", cmd__parse_options },
 	{ "parse-pathspec-file", cmd__parse_pathspec_file },
+	{ "partial-clone", cmd__partial_clone },
 	{ "path-utils", cmd__path_utils },
 	{ "pcre2-config", cmd__pcre2_config },
 	{ "pkt-line", cmd__pkt_line },
diff --git a/t/helper/test-tool.h b/t/helper/test-tool.h
index e8069a3b22..f845ced4b3 100644
--- a/t/helper/test-tool.h
+++ b/t/helper/test-tool.h
@@ -35,6 +35,7 @@ int cmd__oidmap(int argc, const char **argv);
 int cmd__online_cpus(int argc, const char **argv);
 int cmd__parse_options(int argc, const char **argv);
 int cmd__parse_pathspec_file(int argc, const char** argv);
+int cmd__partial_clone(int argc, const char **argv);
 int cmd__path_utils(int argc, const char **argv);
 int cmd__pcre2_config(int argc, const char **argv);
 int cmd__pkt_line(int argc, const char **argv);
diff --git a/t/t0410-partial-clone.sh b/t/t0410-partial-clone.sh
index 584a039b85..e804d267e6 100755
--- a/t/t0410-partial-clone.sh
+++ b/t/t0410-partial-clone.sh
@@ -604,6 +604,30 @@ test_expect_success 'do not fetch when checking existence of tree we construct o
 	git -C repo cherry-pick side1
 '
 
+test_expect_success 'lazy-fetch when accessing object not in the_repository' '
+	rm -rf full partial.git &&
+	test_create_repo full &&
+	printf 12345 >full/file.txt &&
+	git -C full add file.txt &&
+	git -C full commit -m "first commit" &&
+
+	test_config -C full uploadpack.allowfilter 1 &&
+	test_config -C full uploadpack.allowanysha1inwant 1 &&
+	git clone --filter=blob:none --bare "file://$(pwd)/full" partial.git &&
+	FILE_HASH=$(git hash-object --stdin <full/file.txt) &&
+
+	# Sanity check that the file is missing
+	git -C partial.git rev-list --objects --missing=print HEAD >out &&
+	grep "[?]$FILE_HASH" out &&
+
+	OUT=$(test-tool partial-clone object-info partial.git "$FILE_HASH") &&
+	test "$OUT" -eq 5 &&
+
+	# Sanity check that the file is now present
+	git -C partial.git rev-list --objects --missing=print HEAD >out &&
+	! grep "[?]$FILE_HASH" out
+'
+
 . "$TEST_DIRECTORY"/lib-httpd.sh
 start_httpd
 

-- 
2.32.0.rc0.204.g9fa02ecfa5-goog
Previous: Emily ShafferNext: Taylor Blau
Message 16 of 77 in “First steps towards partial clone submodules”
  1. 0/4 First steps towards partial clone submodulesJonathan Tan, Jun 1, 2021
  2. 1/4 promisor-remote: read partialClone config hereJonathan Tan, Jun 1, 2021
  3. Taylor BlauJun 4, 2021
  4. Jonathan TanJun 5, 2021
  5. Emily ShafferJun 7, 2021
  6. 2/4 promisor-remote: support per-repository configJonathan Tan, Jun 1, 2021
  7. Taylor BlauJun 4, 2021
  8. Jonathan TanJun 5, 2021
  9. Elijah NewrenJun 4, 2021
  10. Jonathan TanJun 5, 2021
  11. Emily ShafferJun 8, 2021
  12. 3/4 run-command: move envvar-resetting functionJonathan Tan, Jun 1, 2021
  13. Taylor BlauJun 4, 2021
  14. Jonathan TanJun 5, 2021
  15. Emily ShafferJun 8, 2021
  16. 4/4 promisor-remote: teach lazy-fetch in any repoJonathan Tan, Jun 1, 2021
  17. Taylor BlauJun 4, 2021
  18. Jonathan TanJun 5, 2021
  19. Elijah NewrenJun 4, 2021
  20. Jonathan TanJun 5, 2021
  21. Elijah NewrenJun 5, 2021
  22. Elijah NewrenJun 5, 2021
  23. Jonathan TanJun 5, 2021
  24. Emily ShafferJun 8, 2021
  25. Jonathan TanJun 9, 2021
  26. 0/4 First steps towards partial clone submodulesJonathan Tan, Jun 8, 2021
  27. 3/4 run-command: move envvar-resetting functionJonathan Tan, Jun 8, 2021
  28. Junio C HamanoJun 8, 2021
  29. Jonathan TanJun 9, 2021
  30. Junio C HamanoJun 9, 2021
  31. Jonathan TanJun 9, 2021
  32. 2/4 promisor-remote: support per-repository configJonathan Tan, Jun 8, 2021
  33. Junio C HamanoJun 8, 2021
  34. Jonathan TanJun 9, 2021
  35. 1/4 promisor-remote: read partialClone config hereJonathan Tan, Jun 8, 2021
  36. Junio C HamanoJun 8, 2021
  37. Jonathan TanJun 9, 2021
  38. Junio C HamanoJun 9, 2021
  39. Jonathan TanJun 9, 2021
  40. Elijah NewrenJun 8, 2021
  41. Jonathan TanJun 9, 2021
  42. Elijah NewrenJun 9, 2021
  43. Jonathan TanJun 10, 2021
  44. 4/4 promisor-remote: teach lazy-fetch in any repoJonathan Tan, Jun 8, 2021
  45. Junio C HamanoJun 8, 2021
  46. Jonathan TanJun 9, 2021
  47. Junio C HamanoJun 9, 2021
  48. Jonathan TanJun 9, 2021
  49. Junio C HamanoJun 10, 2021
  50. Elijah NewrenJun 8, 2021
  51. Jonathan TanJun 9, 2021
  52. Elijah NewrenJun 8, 2021
  53. Junio C HamanoJun 8, 2021
  54. Elijah NewrenJun 9, 2021
  55. Junio C HamanoJun 9, 2021
  56. Jonathan TanJun 9, 2021
  57. Emily ShafferJun 8, 2021
  58. 0/5 First steps towards partial clone submodulesJonathan Tan, Jun 10, 2021
  59. 2/5 promisor-remote: support per-repository configJonathan Tan, Jun 10, 2021
  60. 5/5 promisor-remote: teach lazy-fetch in any repoJonathan Tan, Jun 10, 2021
  61. 1/5 repository: move global r_f_p_c to repo structJonathan Tan, Jun 10, 2021
  62. Elijah NewrenJun 10, 2021
  63. 3/5 submodule: refrain from filtering GIT_CONFIG_COUNTJonathan Tan, Jun 10, 2021
  64. Elijah NewrenJun 10, 2021
  65. Jeff KingJun 10, 2021
  66. Jonathan TanJun 11, 2021
  67. 4/5 run-command: refactor subprocess env preparationJonathan Tan, Jun 10, 2021
  68. Elijah NewrenJun 10, 2021
  69. Elijah NewrenJun 10, 2021
  70. Elijah NewrenJun 15, 2021
  71. 0/5 First steps towards partial clone submodulesJonathan Tan, Jun 17, 2021
  72. 1/5 repository: move global r_f_p_c to repo structJonathan Tan, Jun 17, 2021
  73. 2/5 promisor-remote: support per-repository configJonathan Tan, Jun 17, 2021
  74. 3/5 submodule: refrain from filtering GIT_CONFIG_COUNTJonathan Tan, Jun 17, 2021
  75. 4/5 run-command: refactor subprocess env preparationJonathan Tan, Jun 17, 2021
  76. 5/5 promisor-remote: teach lazy-fetch in any repoJonathan Tan, Jun 17, 2021
  77. Elijah NewrenJun 19, 2021

Read the whole thread, see it on lore, or plain text.

$ cat FOOTERMessages come from the public archive at lore.kernel.org/git, fetched every hour. The front page is chosen and written each morning by an AI editor and can be wrong; the threads themselves are the record. About and API. For agents: an MCP server at https://gitlist.dev/mcp, and any thread, story or person page as Markdown by adding .md to its URL (or sending Accept: text/markdown). Details in /llms.txt.