git/list[1] front-page[2] threads[3] people[4] search[5] about
 

[PATCH v4 7/8] fetch-pack: put shallow info in output parameter

From
BWBrandon Williams <bmwill@google.com>
Date
Jun 25, 2018, 18:53 UTC
Message-ID
<20180625185332.164142-8-bmwill@google.com>
In-Reply-To
<20180625185332.164142-1-bmwill@google.com>

Expand the transport fetch method signature, by adding an output parameter, to allow transports to return information about the refs they have fetched. Then communicate shallow status information through this mechanism instead of by modifying the input list of refs.

This does require clients to sometimes generate the ref map twice: once from the list of refs provided by the remote (as is currently done) and potentially once from the new list of refs that the fetch mechanism provides.

Signed-off-by: Brandon Williams <bmwill@google.com>
---
 builtin/clone.c      |  4 ++--
 builtin/fetch.c      | 28 ++++++++++++++++++++++++----
 fetch-object.c       |  2 +-
 fetch-pack.c         | 15 ++++++++-------
 transport-helper.c   |  6 ++++--
 transport-internal.h |  9 ++++++++-
 transport.c          | 34 ++++++++++++++++++++++++++++------
 transport.h          |  3 ++-
 8 files changed, 77 insertions(+), 24 deletions(-)
diff --git a/builtin/clone.c b/builtin/clone.c
index 99e73dae8..8f86d99c5 100644
--- a/builtin/clone.c
+++ b/builtin/clone.c
@@ -1155,7 +1155,7 @@ int cmd_clone(int argc, const char **argv, const char *prefix)
 			}
 
 		if (!is_local && !complete_refs_before_fetch)
-			transport_fetch_refs(transport, mapped_refs);
+			transport_fetch_refs(transport, mapped_refs, NULL);
 
 		remote_head = find_ref_by_name(refs, "HEAD");
 		remote_head_points_at =
@@ -1197,7 +1197,7 @@ int cmd_clone(int argc, const char **argv, const char *prefix)
 	if (is_local)
 		clone_local(path, git_dir);
 	else if (refs && complete_refs_before_fetch)
-		transport_fetch_refs(transport, mapped_refs);
+		transport_fetch_refs(transport, mapped_refs, NULL);
 
 	update_remote_refs(refs, mapped_refs, remote_head_points_at,
 			   branch_top.buf, reflog_msg.buf, transport,
diff --git a/builtin/fetch.c b/builtin/fetch.c
index bda00e826..0347cf016 100644
--- a/builtin/fetch.c
+++ b/builtin/fetch.c
@@ -946,11 +946,13 @@ static int quickfetch(struct ref *ref_map)
 	return check_connected(iterate_ref_map, &rm, &opt);
 }
 
-static int fetch_refs(struct transport *transport, struct ref *ref_map)
+static int fetch_refs(struct transport *transport, struct ref *ref_map,
+		      struct ref **updated_remote_refs)
 {
 	int ret = quickfetch(ref_map);
 	if (ret)
-		ret = transport_fetch_refs(transport, ref_map);
+		ret = transport_fetch_refs(transport, ref_map,
+					   updated_remote_refs);
 	if (!ret)
 		/*
 		 * Keep the new pack's ".keep" file around to allow the caller
@@ -1112,7 +1114,7 @@ static void backfill_tags(struct transport *transport, struct ref *ref_map)
 	transport_set_option(transport, TRANS_OPT_FOLLOWTAGS, NULL);
 	transport_set_option(transport, TRANS_OPT_DEPTH, "0");
 	transport_set_option(transport, TRANS_OPT_DEEPEN_RELATIVE, NULL);
-	if (!fetch_refs(transport, ref_map))
+	if (!fetch_refs(transport, ref_map, NULL))
 		consume_refs(transport, ref_map);
 
 	if (gsecondary) {
@@ -1128,6 +1130,7 @@ static int do_fetch(struct transport *transport,
 	int autotags = (transport->remote->fetch_tags == 1);
 	int retcode = 0;
 	const struct ref *remote_refs;
+	struct ref *updated_remote_refs = NULL;
 	struct argv_array ref_prefixes = ARGV_ARRAY_INIT;
 
 	if (tags == TAGS_DEFAULT) {
@@ -1178,7 +1181,24 @@ static int do_fetch(struct transport *transport,
 				   transport->url);
 		}
 	}
-	if (fetch_refs(transport, ref_map) || consume_refs(transport, ref_map)) {
+
+	if (fetch_refs(transport, ref_map, &updated_remote_refs)) {
+		free_refs(ref_map);
+		retcode = 1;
+		goto cleanup;
+	}
+	if (updated_remote_refs) {
+		/*
+		 * Regenerate ref_map using the updated remote refs.  This is
+		 * to account for additional information which may be provided
+		 * by the transport (e.g. shallow info).
+		 */
+		free_refs(ref_map);
+		ref_map = get_ref_map(transport->remote, updated_remote_refs, rs,
+				      tags, &autotags);
+		free_refs(updated_remote_refs);
+	}
+	if (consume_refs(transport, ref_map)) {
 		free_refs(ref_map);
 		retcode = 1;
 		goto cleanup;
diff --git a/fetch-object.c b/fetch-object.c
index 853624f81..48fe63dd6 100644
--- a/fetch-object.c
+++ b/fetch-object.c
@@ -19,7 +19,7 @@ static void fetch_refs(const char *remote_name, struct ref *ref)
 
 	transport_set_option(transport, TRANS_OPT_FROM_PROMISOR, "1");
 	transport_set_option(transport, TRANS_OPT_NO_DEPENDENTS, "1");
-	transport_fetch_refs(transport, ref);
+	transport_fetch_refs(transport, ref, NULL);
 	fetch_if_missing = original_fetch_if_missing;
 }
 
diff --git a/fetch-pack.c b/fetch-pack.c
index a320ce987..73890b894 100644
--- a/fetch-pack.c
+++ b/fetch-pack.c
@@ -1470,12 +1470,13 @@ static int remove_duplicates_in_refs(struct ref **ref, int nr)
 }
 
 static void update_shallow(struct fetch_pack_args *args,
-			   struct ref **sought, int nr_sought,
+			   struct ref *refs,
 			   struct shallow_info *si)
 {
 	struct oid_array ref = OID_ARRAY_INIT;
 	int *status;
 	int i;
+	struct ref *r;
 
 	if (args->deepen && alternate_shallow_file) {
 		if (*alternate_shallow_file == '\0') { /* --unshallow */
@@ -1517,8 +1518,8 @@ static void update_shallow(struct fetch_pack_args *args,
 	remove_nonexistent_theirs_shallow(si);
 	if (!si->nr_ours && !si->nr_theirs)
 		return;
-	for (i = 0; i < nr_sought; i++)
-		oid_array_append(&ref, &sought[i]->old_oid);
+	for (r = refs; r; r = r->next)
+		oid_array_append(&ref, &r->old_oid);
 	si->ref = &ref;
 
 	if (args->update_shallow) {
@@ -1552,12 +1553,12 @@ static void update_shallow(struct fetch_pack_args *args,
 	 * remote is also shallow, check what ref is safe to update
 	 * without updating .git/shallow
 	 */
-	status = xcalloc(nr_sought, sizeof(*status));
+	status = xcalloc(ref.nr, sizeof(*status));
 	assign_shallow_commits_to_refs(si, NULL, status);
 	if (si->nr_ours || si->nr_theirs) {
-		for (i = 0; i < nr_sought; i++)
+		for (r = refs, i = 0; r; r = r->next, i++)
 			if (status[i])
-				sought[i]->status = REF_STATUS_REJECT_SHALLOW;
+				r->status = REF_STATUS_REJECT_SHALLOW;
 	}
 	free(status);
 	oid_array_clear(&ref);
@@ -1591,7 +1592,7 @@ struct ref *fetch_pack(struct fetch_pack_args *args,
 		ref_cpy = do_fetch_pack(args, fd, ref, sought, nr_sought,
 					&si, pack_lockfile);
 	reprepare_packed_git(the_repository);
-	update_shallow(args, sought, nr_sought, &si);
+	update_shallow(args, ref_cpy, &si);
 	clear_shallow_info(&si);
 	return ref_cpy;
 }
diff --git a/transport-helper.c b/transport-helper.c
index 1f8ff7e94..8b5abca29 100644
--- a/transport-helper.c
+++ b/transport-helper.c
@@ -651,14 +651,16 @@ static int connect_helper(struct transport *transport, const char *name,
 }
 
 static int fetch(struct transport *transport,
-		 int nr_heads, struct ref **to_fetch)
+		 int nr_heads, struct ref **to_fetch,
+		 struct ref **fetched_refs)
 {
 	struct helper_data *data = transport->data;
 	int i, count;
 
 	if (process_connect(transport, 0)) {
 		do_take_over(transport);
-		return transport->vtable->fetch(transport, nr_heads, to_fetch);
+		return transport->vtable->fetch(transport, nr_heads, to_fetch,
+						fetched_refs);
 	}
 
 	count = 0;
diff --git a/transport-internal.h b/transport-internal.h
index 1cde6258a..eeb6c340e 100644
--- a/transport-internal.h
+++ b/transport-internal.h
@@ -36,11 +36,18 @@ struct transport_vtable {
 	 * Fetch the objects for the given refs. Note that this gets
 	 * an array, and should ignore the list structure.
 	 *
+	 * The transport *may* provide, in fetched_refs, the list of refs that
+	 * it fetched.  If the transport knows anything about the fetched refs
+	 * that the caller does not know (for example, shallow status), it
+	 * should provide that list of refs and include that information in the
+	 * list.
+	 *
 	 * If the transport did not get hashes for refs in
 	 * get_refs_list(), it should set the old_sha1 fields in the
 	 * provided refs now.
 	 **/
-	int (*fetch)(struct transport *transport, int refs_nr, struct ref **refs);
+	int (*fetch)(struct transport *transport, int refs_nr, struct ref **refs,
+		     struct ref **fetched_refs);
 
 	/**
 	 * Push the objects and refs. Send the necessary objects, and
diff --git a/transport.c b/transport.c
index a32da30de..39d8c2fa5 100644
--- a/transport.c
+++ b/transport.c
@@ -151,7 +151,8 @@ static struct ref *get_refs_from_bundle(struct transport *transport,
 }
 
 static int fetch_refs_from_bundle(struct transport *transport,
-			       int nr_heads, struct ref **to_fetch)
+			       int nr_heads, struct ref **to_fetch,
+			       struct ref **fetched_refs)
 {
 	struct bundle_transport_data *data = transport->data;
 	return unbundle(&data->header, data->fd,
@@ -287,7 +288,8 @@ static struct ref *get_refs_via_connect(struct transport *transport, int for_pus
 }
 
 static int fetch_refs_via_pack(struct transport *transport,
-			       int nr_heads, struct ref **to_fetch)
+			       int nr_heads, struct ref **to_fetch,
+			       struct ref **fetched_refs)
 {
 	int ret = 0;
 	struct git_transport_data *data = transport->data;
@@ -354,8 +356,12 @@ static int fetch_refs_via_pack(struct transport *transport,
 	if (report_unmatched_refs(to_fetch, nr_heads))
 		ret = -1;
 
+	if (fetched_refs)
+		*fetched_refs = refs;
+	else
+		free_refs(refs);
+
 	free_refs(refs_tmp);
-	free_refs(refs);
 	free(dest);
 	return ret;
 }
@@ -1215,19 +1221,31 @@ const struct ref *transport_get_remote_refs(struct transport *transport,
 	return transport->remote_refs;
 }
 
-int transport_fetch_refs(struct transport *transport, struct ref *refs)
+int transport_fetch_refs(struct transport *transport, struct ref *refs,
+			 struct ref **fetched_refs)
 {
 	int rc;
 	int nr_heads = 0, nr_alloc = 0, nr_refs = 0;
 	struct ref **heads = NULL;
+	struct ref *nop_head = NULL, **nop_tail = &nop_head;
 	struct ref *rm;
 
 	for (rm = refs; rm; rm = rm->next) {
 		nr_refs++;
 		if (rm->peer_ref &&
 		    !is_null_oid(&rm->old_oid) &&
-		    !oidcmp(&rm->peer_ref->old_oid, &rm->old_oid))
+		    !oidcmp(&rm->peer_ref->old_oid, &rm->old_oid)) {
+			/*
+			 * These need to be reported as fetched, but we don't
+			 * actually need to fetch them.
+			 */
+			if (fetched_refs) {
+				struct ref *nop_ref = copy_ref(rm);
+				*nop_tail = nop_ref;
+				nop_tail = &nop_ref->next;
+			}
 			continue;
+		}
 		ALLOC_GROW(heads, nr_heads + 1, nr_alloc);
 		heads[nr_heads++] = rm;
 	}
@@ -1245,7 +1263,11 @@ int transport_fetch_refs(struct transport *transport, struct ref *refs)
 			heads[nr_heads++] = rm;
 	}
 
-	rc = transport->vtable->fetch(transport, nr_heads, heads);
+	rc = transport->vtable->fetch(transport, nr_heads, heads, fetched_refs);
+	if (fetched_refs && nop_head) {
+		*nop_tail = *fetched_refs;
+		*fetched_refs = nop_head;
+	}
 
 	free(heads);
 	return rc;
diff --git a/transport.h b/transport.h
index 7792b0858..3dff767a8 100644
--- a/transport.h
+++ b/transport.h
@@ -218,7 +218,8 @@ int transport_push(struct transport *connection,
 const struct ref *transport_get_remote_refs(struct transport *transport,
 					    const struct argv_array *ref_prefixes);
 
-int transport_fetch_refs(struct transport *transport, struct ref *refs);
+int transport_fetch_refs(struct transport *transport, struct ref *refs,
+			 struct ref **fetched_refs);
 void transport_unlock_pack(struct transport *transport);
 int transport_disconnect(struct transport *transport);
 char *transport_anonymize_url(const char *url);

-- 
2.18.0.rc2.346.g013aa6912e-goog
Previous: Jonathan TanNext: Brandon Williams
Message 71 of 122 in “ref-in-want”
  1. 0/8 ref-in-wantBrandon Williams, Jun 5, 2018
  2. 3/8 upload-pack: test negotiation with changing repositoryBrandon Williams, Jun 5, 2018
  3. 4/8 fetch: refactor the population of peer ref OIDsBrandon Williams, Jun 5, 2018
  4. 6/8 fetch: refactor to make function args narrowerBrandon Williams, Jun 5, 2018
  5. 8/8 fetch-pack: implement ref-in-wantBrandon Williams, Jun 5, 2018
  6. 7/8 fetch-pack: put shallow info in output parameterBrandon Williams, Jun 5, 2018
  7. 5/8 fetch: refactor fetch_refs into two functionsBrandon Williams, Jun 5, 2018
  8. 1/8 test-pkt-line: add unpack-sideband subcommandBrandon Williams, Jun 5, 2018
  9. 2/8 upload-pack: implement ref-in-wantBrandon Williams, Jun 5, 2018
  10. Ramsay JonesJun 5, 2018
  11. Ævar Arnfjörð BjarmasonJun 5, 2018
  12. Brandon WilliamsJun 6, 2018
  13. Ævar Arnfjörð BjarmasonJun 6, 2018
  14. Brandon WilliamsJun 6, 2018
  15. 0/8 ref-in-wantBrandon Williams, Jun 13, 2018
  16. 1/8 test-pkt-line: add unpack-sideband subcommandBrandon Williams, Jun 13, 2018
  17. Stefan BellerJun 14, 2018
  18. Brandon WilliamsJun 14, 2018
  19. 2/8 upload-pack: implement ref-in-wantBrandon Williams, Jun 13, 2018
  20. Stefan BellerJun 14, 2018
  21. Brandon WilliamsJun 14, 2018
  22. Junio C HamanoJun 15, 2018
  23. Junio C HamanoJun 15, 2018
  24. Brandon WilliamsJun 19, 2018
  25. Junio C HamanoJun 19, 2018
  26. Brandon WilliamsJun 19, 2018
  27. Junio C HamanoJun 21, 2018
  28. 4/8 fetch: refactor the population of peer ref OIDsBrandon Williams, Jun 13, 2018
  29. 3/8 upload-pack: test negotiation with changing repositoryBrandon Williams, Jun 13, 2018
  30. Stefan BellerJun 14, 2018
  31. 5/8 fetch: refactor fetch_refs into two functionsBrandon Williams, Jun 13, 2018
  32. 8/8 fetch-pack: implement ref-in-wantBrandon Williams, Jun 13, 2018
  33. Stefan BellerJun 14, 2018
  34. Brandon WilliamsJun 14, 2018
  35. Jonathan NiederJun 22, 2018
  36. 7/8 fetch-pack: put shallow info in output parameterBrandon Williams, Jun 13, 2018
  37. Stefan BellerJun 14, 2018
  38. Jonathan TanJun 14, 2018
  39. Brandon WilliamsJun 19, 2018
  40. 6/8 fetch: refactor to make function args narrowerBrandon Williams, Jun 13, 2018
  41. Stefan BellerJun 14, 2018
  42. Junio C HamanoJun 15, 2018
  43. Brandon WilliamsJun 18, 2018
  44. 0/8 ref-in-wantBrandon Williams, Jun 20, 2018
  45. 1/8 test-pkt-line: add unpack-sideband subcommandBrandon Williams, Jun 20, 2018
  46. Jonathan NiederJun 22, 2018
  47. 2/8 upload-pack: implement ref-in-wantBrandon Williams, Jun 20, 2018
  48. Jonathan TanJun 25, 2018
  49. Jonathan TanJun 25, 2018
  50. Brandon WilliamsJun 25, 2018
  51. 3/8 upload-pack: test negotiation with changing repositoryBrandon Williams, Jun 20, 2018
  52. 4/8 fetch: refactor the population of peer ref OIDsBrandon Williams, Jun 20, 2018
  53. Jonathan TanJun 25, 2018
  54. 5/8 fetch: refactor fetch_refs into two functionsBrandon Williams, Jun 20, 2018
  55. Jonathan NiederJun 22, 2018
  56. Jonathan NiederJun 22, 2018
  57. 6/8 fetch: refactor to make function args narrowerBrandon Williams, Jun 20, 2018
  58. 8/8 fetch-pack: implement ref-in-wantBrandon Williams, Jun 20, 2018
  59. Jonathan NiederJun 22, 2018
  60. Brandon WilliamsJun 25, 2018
  61. 7/8 fetch-pack: put shallow info in output parameterBrandon Williams, Jun 20, 2018
  62. Jonathan TanJun 25, 2018
  63. Brandon WilliamsJun 25, 2018
  64. 0/8 ref-in-wantBrandon Williams, Jun 25, 2018
  65. 1/8 test-pkt-line: add unpack-sideband subcommandBrandon Williams, Jun 25, 2018
  66. 2/8 upload-pack: implement ref-in-wantBrandon Williams, Jun 25, 2018
  67. 4/8 fetch: refactor the population of peer ref OIDsBrandon Williams, Jun 25, 2018
  68. 5/8 fetch: refactor fetch_refs into two functionsBrandon Williams, Jun 25, 2018
  69. 6/8 fetch: refactor to make function args narrowerBrandon Williams, Jun 25, 2018
  70. Jonathan TanJun 25, 2018
  71. 7/8 fetch-pack: put shallow info in output parameterBrandon Williams, Jun 25, 2018
  72. 8/8 fetch-pack: implement ref-in-wantBrandon Williams, Jun 25, 2018
  73. 3/8 upload-pack: test negotiation with changing repositoryBrandon Williams, Jun 25, 2018
  74. Jonathan TanJun 25, 2018
  75. Jonathan TanJun 25, 2018
  76. 0/8 ref-in-wantBrandon Williams, Jun 26, 2018
  77. 1/8 test-pkt-line: add unpack-sideband subcommandBrandon Williams, Jun 26, 2018
  78. 2/8 upload-pack: implement ref-in-wantBrandon Williams, Jun 26, 2018
  79. Junio C HamanoJun 26, 2018
  80. Brandon WilliamsJun 27, 2018
  81. Junio C HamanoJun 27, 2018
  82. Brandon WilliamsJun 27, 2018
  83. Stefan BellerJun 27, 2018
  84. Jonathan TanJun 27, 2018
  85. 5/8 fetch: refactor fetch_refs into two functionsBrandon Williams, Jun 26, 2018
  86. 4/8 fetch: refactor the population of peer ref OIDsBrandon Williams, Jun 26, 2018
  87. 3/8 upload-pack: test negotiation with changing repositoryBrandon Williams, Jun 26, 2018
  88. Junio C HamanoJun 26, 2018
  89. Brandon WilliamsJun 27, 2018
  90. Jonathan TanJun 27, 2018
  91. 8/8 fetch-pack: implement ref-in-wantBrandon Williams, Jun 26, 2018
  92. Jonathan TanJun 27, 2018
  93. Brandon WilliamsJun 27, 2018
  94. 7/8 fetch-pack: put shallow info in output parameterBrandon Williams, Jun 26, 2018
  95. Junio C HamanoJun 26, 2018
  96. Brandon WilliamsJun 27, 2018
  97. 6/8 fetch: refactor to make function args narrowerBrandon Williams, Jun 26, 2018
  98. Junio C HamanoJun 26, 2018
  99. 0/8 ref-in-wantBrandon Williams, Jun 27, 2018
  100. 1/8 test-pkt-line: add unpack-sideband subcommandBrandon Williams, Jun 27, 2018
  101. 2/8 upload-pack: implement ref-in-wantBrandon Williams, Jun 27, 2018
  102. 3/8 upload-pack: test negotiation with changing repositoryBrandon Williams, Jun 27, 2018
  103. 5/8 fetch: refactor fetch_refs into two functionsBrandon Williams, Jun 27, 2018
  104. 4/8 fetch: refactor the population of peer ref OIDsBrandon Williams, Jun 27, 2018
  105. 7/8 fetch-pack: put shallow info in output parameterBrandon Williams, Jun 27, 2018
  106. 6/8 fetch: refactor to make function args narrowerBrandon Williams, Jun 27, 2018
  107. 8/8 fetch-pack: implement ref-in-wantBrandon Williams, Jun 27, 2018
  108. Duy NguyenJul 22, 2018
  109. Brandon WilliamsJul 23, 2018
  110. Duy NguyenJul 23, 2018
  111. Jonathan NiederJul 23, 2018
  112. fetch-pack: mark die strings for translationBrandon Williams, Jul 23, 2018
  113. Stefan BellerJul 23, 2018
  114. Jonathan NiederJul 23, 2018
  115. Junio C HamanoJul 23, 2018
  116. Junio C HamanoJul 23, 2018
  117. Brandon WilliamsJul 23, 2018
  118. Jonathan TanJun 15, 2018
  119. Brandon WilliamsJun 19, 2018
  120. Jonathan TanJun 19, 2018
  121. Brandon WilliamsJun 19, 2018
  122. Jonathan TanJun 19, 2018

Read the whole thread, see it on lore, or plain text.

$ cat FOOTERMessages come from the public archive at lore.kernel.org/git, fetched every hour. The front page is chosen and written each morning by an AI editor and can be wrong; the threads themselves are the record. About and API. For agents: an MCP server at https://gitlist.dev/mcp, and any thread, story or person page as Markdown by adding .md to its URL (or sending Accept: text/markdown). Details in /llms.txt.