[PATCH v2 0/5] Push negotiation

Thanks for the review, Junio and Stolee.

The first patch in version 1 has been merged to master, I believe, so here are the remaining 5 patches.

As Stolee pointed out, some tests in version 1 fail when GIT_TEST_PROTOCOL_VERSION is set to 0. I have corrected that, but there is one test that still fails in t5601. That seems to also fail in "master". I'll see if I can fix it and send a separate patch for that.

Jonathan Tan (5):
  fetch-pack: refactor process_acks()
  fetch-pack: refactor add_haves()
  fetch-pack: refactor command and capability write
  fetch: teach independent negotiation (no packfile)
  send-pack: support push negotiation

 Documentation/config/push.txt           |   7 +
 Documentation/technical/protocol-v2.txt |   8 +
 builtin/fetch.c                         |  27 ++-
 fetch-pack.c                            | 246 ++++++++++++++++--------
 fetch-pack.h                            |  14 ++
 object.h                                |   2 +-
 send-pack.c                             |  61 +++++-
 t/t5516-fetch-push.sh                   |  35 ++++
 t/t5701-git-serve.sh                    |   2 +-
 t/t5702-protocol-v2.sh                  |  89 +++++++++
 transport-helper.c                      |  10 +
 transport.c                             |  30 ++-
 transport.h                             |   6 +
 upload-pack.c                           |  18 +-
 14 files changed, 455 insertions(+), 100 deletions(-)

Range-diff against v1:
-:  ---------- > 1:  8102570374 fetch-pack: refactor process_acks()
-:  ---------- > 2:  57c3451b2e fetch-pack: refactor add_haves()
-:  ---------- > 3:  6871d0cec6 fetch-pack: refactor command and capability write
1:  3ebe4ada28 ! 4:  1de34a6dce fetch: teach independent negotiation (no packfile)
    @@ Commit message
         There are 2 code paths that do not go through fetch_refs_via_pack() that
         needed to be individually excluded: the bundle transport (excluded
         through requiring smart_options, which the bundle transport doesn't
    -    support) and transport helpers that do not support takeover.
    -    Fortunately, none of these support protocol v2.
    +    support) and transport helpers that do not support takeover. If or when
    +    we support independent negotiation for protocol v0, we will need to
    +    modify these 2 code paths to support it. But for now, report failure if
    +    independent negotiation is requested in these cases.
     
         Signed-off-by: Jonathan Tan <jonathantanmy@google.com>
         Signed-off-by: Junio C Hamano <gitster@pobox.com>
    @@ builtin/fetch.c: int cmd_fetch(int argc, const char **argv, const char *prefix)
     +		const struct object_id *oid;
     +
     +		if (!remote)
    -+			die(_("Must supply remote when using --negotiate-only"));
    ++			die(_("must supply remote when using --negotiate-only"));
     +		gtransport = prepare_transport(remote, 1);
     +		if (gtransport->smart_options) {
     +			gtransport->smart_options->acked_commits = &acked_commits;
    @@ fetch-pack.c
      #include "fsck.h"
      #include "shallow.h"
     +#include "commit-reach.h"
    ++#include "commit-graph.h"
      
      static int transfer_unpack_limit = -1;
      static int fetch_unpack_limit = -1;
    @@ fetch-pack.c: struct ref *fetch_pack(struct fetch_pack_args *args,
     +{
     +	struct object_array *a = data;
     +
    -+	add_object_array(parse_object(the_repository, oid), "", a);
    ++	add_object_array(lookup_object(the_repository, oid), "", a);
     +	return 0;
     +}
     +
    ++static void clear_common_flag(struct oidset *s)
    ++{
    ++	struct oidset_iter iter;
    ++	const struct object_id *oid;
    ++	oidset_iter_init(s, &iter);
    ++
    ++	while ((oid = oidset_iter_next(&iter))) {
    ++		struct object *obj = lookup_object(the_repository, oid);
    ++		obj->flags &= ~COMMON;
    ++	}
    ++}
    ++
     +void negotiate_using_fetch(const struct oid_array *negotiation_tips,
     +			   const struct string_list *server_options,
     +			   int stateless_rpc,
    @@ fetch-pack.c: struct ref *fetch_pack(struct fetch_pack_args *args,
     +	int in_vain = 0;
     +	int seen_ack = 0;
     +	int last_iteration = 0;
    ++	timestamp_t min_generation = GENERATION_NUMBER_INFINITY;
     +
     +	fetch_negotiator_init(the_repository, &negotiator);
     +	mark_tips(&negotiator, negotiation_tips);
    @@ fetch-pack.c: struct ref *fetch_pack(struct fetch_pack_args *args,
     +				   &received_ready)) {
     +			struct commit *commit = lookup_commit(the_repository,
     +							      &common_oid);
    -+			if (commit)
    ++			if (commit) {
    ++				timestamp_t generation;
    ++
    ++				parse_commit_or_die(commit);
     +				commit->object.flags |= COMMON;
    ++				generation = commit_graph_generation(commit);
    ++				if (generation < min_generation)
    ++					min_generation = generation;
    ++			}
     +			in_vain = 0;
     +			seen_ack = 1;
     +			oidset_insert(acked_commits, &common_oid);
    @@ fetch-pack.c: struct ref *fetch_pack(struct fetch_pack_args *args,
     +			do_check_stateless_delimiter(stateless_rpc, &reader);
     +		if (can_all_from_reach_with_flag(&nt_object_array, COMMON,
     +						 REACH_SCRATCH, 0,
    -+						 GENERATION_NUMBER_ZERO))
    ++						 min_generation))
     +			last_iteration = 1;
     +	}
    ++	clear_common_flag(acked_commits);
     +	strbuf_release(&req_buf);
     +}
     +
    @@ fetch-pack.h: struct ref *fetch_pack(struct fetch_pack_args *args,
     +/*
     + * Execute the --negotiate-only mode of "git fetch", adding all known common
     + * commits to acked_commits.
    ++ *
    ++ * In the capability advertisement that has happened prior to invoking this
    ++ * function, the "wait-for-done" capability must be present.
     + */
     +void negotiate_using_fetch(const struct oid_array *negotiation_tips,
     +			   const struct string_list *server_options,
    @@ t/t5702-protocol-v2.sh: test_expect_success 'deepen-relative' '
     +
     +	setup_negotiate_only "$SERVER" "$URI" &&
     +
    -+	git -C client fetch \
    ++	git -c protocol.version=2 -C client fetch \
     +		--no-tags \
     +		--negotiate-only \
     +		--negotiation-tip=$(git -C client rev-parse HEAD) \
    @@ t/t5702-protocol-v2.sh: test_expect_success 'packfile-uri with transfer.fsckobje
     +
     +	setup_negotiate_only "$SERVER" "$URI" &&
     +
    -+	git -C client fetch \
    ++	git -c protocol.version=2 -C client fetch \
     +		--no-tags \
     +		--negotiate-only \
     +		--negotiation-tip=$(git -C client rev-parse HEAD) \
    @@ t/t5702-protocol-v2.sh: test_expect_success 'packfile-uri with transfer.fsckobje
     +	echo "s/ wait-for-done/ xxxx-xxx-xxxx/" \
     +		>"$HTTPD_ROOT_PATH/one-time-perl" &&
     +
    -+	test_must_fail git -C client fetch \
    ++	test_must_fail git -c protocol.version=2 -C client fetch \
     +		--no-tags \
     +		--negotiate-only \
     +		--negotiation-tip=$(git -C client rev-parse HEAD) \
    @@ transport.h: struct git_transport_options {
      	struct oid_array *negotiation_tips;
     +
     +	/*
    -+	 * If set, whenever transport_fetch_refs() is called, add known common
    -+	 * commits to this oidset instead of fetching any packfiles.
    ++	 * If allocated, whenever transport_fetch_refs() is called, add known
    ++	 * common commits to this oidset instead of fetching any packfiles.
     +	 */
     +	struct oidset *acked_commits;
      };
2:  bd55d6ba36 ! 5:  d38a8b7d66 send-pack: support push negotiation
    @@ t/t5516-fetch-push.sh: test_expect_success 'fetch with pushInsteadOf (should not
     +	git push testrepo $the_first_commit:refs/remotes/origin/first_commit &&
     +	git -C testrepo config receive.hideRefs refs/remotes/origin/first_commit &&
     +	echo now pushing without negotiation &&
    -+	GIT_TRACE2_EVENT="$(pwd)/event" git push testrepo refs/heads/main:refs/remotes/origin/main &&
    ++	GIT_TRACE2_EVENT="$(pwd)/event" git -c protocol.version=2 push testrepo refs/heads/main:refs/remotes/origin/main &&
     +	grep_wrote 5 event && # 2 commits, 2 trees, 1 blob
     +
     +	# Same commands, but with negotiation
    @@ t/t5516-fetch-push.sh: test_expect_success 'fetch with pushInsteadOf (should not
     +	mk_empty testrepo &&
     +	git push testrepo $the_first_commit:refs/remotes/origin/first_commit &&
     +	git -C testrepo config receive.hideRefs refs/remotes/origin/first_commit &&
    -+	GIT_TRACE2_EVENT="$(pwd)/event" git -c push.negotiate=1 push testrepo refs/heads/main:refs/remotes/origin/main &&
    ++	GIT_TRACE2_EVENT="$(pwd)/event" git -c protocol.version=2 -c push.negotiate=1 push testrepo refs/heads/main:refs/remotes/origin/main &&
     +	grep_wrote 2 event # 1 commit, 1 tree
     +'
     +
    @@ t/t5516-fetch-push.sh: test_expect_success 'fetch with pushInsteadOf (should not
     +	mk_empty testrepo &&
     +	git push testrepo $the_first_commit:refs/remotes/origin/first_commit &&
     +	git -C testrepo config receive.hideRefs refs/remotes/origin/first_commit &&
    -+	GIT_TRACE_PACKET="$(pwd)/trace" GIT_TEST_PROTOCOL_VERSION=0 GIT_TRACE2_EVENT="$(pwd)/event" \
    ++	GIT_TEST_PROTOCOL_VERSION=0 GIT_TRACE2_EVENT="$(pwd)/event" \
     +		git -c push.negotiate=1 push testrepo refs/heads/main:refs/remotes/origin/main 2>err &&
     +	grep_wrote 5 event && # 2 commits, 2 trees, 1 blob
     +	test_i18ngrep "push negotiation failed" err