Re: [PATCH] fetch: add config to avoid fetching every branch in shallow repo

> From: Harald Nordgren <haraldnordgren@gmail.com>
> 
> In a shallow, sparsely checked out clone of a repository with many
> branches, plain git pull can take minutes or hang outright, even
> though only one branch is actually being worked on.

> Add fetch.shallow, off by default. When enabled, a fetch or pull for
> a shallow repository that isn't already scoped to specific refs
> fetches only the current branch's tracked upstream, instead of every
> branch the remote has. git pull ultimately runs such a fetch under
> the hood, so this fixes pull the same way. It has no effect once the
> repository is no longer shallow, and no effect on a fetch of a remote
> the current branch doesn't track, both fall back to the existing
> behavior.
> 
> This is opt-in rather than automatic because it changes what a plain
> fetch or pull leaves in refs/remotes/<name>/ for anyone who currently
> relies on it syncing every branch of a shallow remote, not just the
> one they are on. Scoping remote.<name>.fetch by hand already covers
> this for a single remote, but that requires knowing the config exists
> and applies it permanently, even to branches that are not currently
> checked out.
> 
> The remote's recorded default branch (remotes/<name>/HEAD) is kept up
> to date the same way it always is, only the other branches are
> skipped.
> 
> Signed-off-by: Harald Nordgren <haraldnordgren@gmail.com>
> ---
>      fetch: add fetch.shallow so pull doesn't fetch every branch on shallow
>      repo
>      
>      Add fetch.shallow config for big shallow repo, so git fetch/pull doesn't
>      hang by fetching every branch.
> 
> Published-As: https://github.com/gitgitgadget/git/releases/tag/pr-git-2412%2FHaraldNordgren%2Ffetch-shallow-narrow-refspec-v1
> Fetch-It-Via: git fetch https://github.com/gitgitgadget/git pr-git-2412/HaraldNordgren/fetch-shallow-narrow-refspec-v1
> Pull-Request: https://github.com/git/git/pull/2412
> 
>   Documentation/config/fetch.adoc |  15 ++++
>   builtin/fetch.c                 |  18 +++-
>   t/t5537-fetch-shallow.sh        | 146 ++++++++++++++++++++++++++++++++
>   3 files changed, 175 insertions(+), 4 deletions(-)
> 
> diff --git a/Documentation/config/fetch.adoc b/Documentation/config/fetch.adoc
> index 00435e9a16..e63e54a463 100644
> --- a/Documentation/config/fetch.adoc
> +++ b/Documentation/config/fetch.adoc
> @@ -145,3 +145,18 @@ remove the value for the `fetch.bundleCreationToken` value before fetching.
>   `never`;;
>   	Never create or modify the `remotes/<name>/HEAD` symbolic-ref.
>   --
> +
> +`fetch.shallow`::
> +	If true, and the repository is a shallow repository (see
> +	linkgit:git-clone[1] `--depth`), a fetch or `git pull` that names no
> +	explicit refspec and would otherwise fall back to the remote's
> +	configured `remote.<name>.fetch` refspec instead fetches only the
> +	current branch's upstream, when that upstream is on the remote being
> +	fetched. This avoids negotiating history for every branch the remote
> +	advertises, which can be slow on a shallow repository that tracks
> +	many disjoint shallow histories. It has no effect on a fetch that
> +	names an explicit remote or refspec, and no effect on a repository
> +	that is not shallow. Defaults to false.
> ++
> +`remotes/<name>/HEAD` is still kept up to date per `fetch.followRemoteHEAD`
> +while this is in effect, only the other branches are skipped.
> diff --git a/builtin/fetch.c b/builtin/fetch.c
> index 533fdfe7d8..b22f7fe5f4 100644
> --- a/builtin/fetch.c
> +++ b/builtin/fetch.c
> @@ -111,6 +111,7 @@ struct fetch_config {
>   	int recurse_submodules;
>   	int parallel;
>   	int submodule_fetch_jobs;
> +	int shallow;
>   };
>   
>   static int git_fetch_config(const char *k, const char *v,
> @@ -175,6 +176,11 @@ static int git_fetch_config(const char *k, const char *v,
>   		return 0;
>   	}
>   
> +	if (!strcmp(k, "fetch.shallow")) {
> +		fetch_config->shallow = git_config_bool(k, v);
> +		return 0;
> +	}
> +
>   	if (!strcmp(k, "fetch.followremotehead")) {
>   		if (!v)
>   			return config_error_nonbool(k);
> @@ -1958,15 +1964,19 @@ static int do_fetch(struct transport *transport,
>   		refspec_ref_prefixes(rs, &transport_ls_refs_options.ref_prefixes);
>   	} else {
>   		struct branch *branch = branch_get(NULL);
> +		int tracks_this_remote = branch && branch_has_merge_config(branch) &&
> +			!strcmp(branch->remote_name, transport->remote->name);
> +		int narrow_to_tracked_ref = config->shallow &&
> +			is_repository_shallow(the_repository) && tracks_this_remote;
>   
>   		if (transport->remote->fetch.nr) {
> -			refspec_ref_prefixes(&transport->remote->fetch,
> -					     &transport_ls_refs_options.ref_prefixes);
> +			if (!narrow_to_tracked_ref)
> +				refspec_ref_prefixes(&transport->remote->fetch,
> +						     &transport_ls_refs_options.ref_prefixes);
>   			if (follow_remote_head != FOLLOW_REMOTE_NEVER)
>   				do_set_head = 1;
>   		}
> -		if (branch && branch_has_merge_config(branch) &&
> -		    !strcmp(branch->remote_name, transport->remote->name)) {
> +		if (tracks_this_remote) {
>   			int i;
>   			for (i = 0; i < branch->merge_nr; i++) {
>   				strvec_push(&transport_ls_refs_options.ref_prefixes,
> diff --git a/t/t5537-fetch-shallow.sh b/t/t5537-fetch-shallow.sh
> index f323ceebd2..8143af9fc3 100755
> --- a/t/t5537-fetch-shallow.sh
> +++ b/t/t5537-fetch-shallow.sh
> @@ -13,6 +13,24 @@ commit() {
>   	git commit -m "$1"
>   }
>   
> +check_upstream_refs () {
> +	git for-each-ref --format="%(refname)" refs/remotes/upstream/ >actual &&
> +	cat >expect &&
> +	test_cmp expect actual
> +}
> +
> +check_upstream_head () {
> +	git symbolic-ref refs/remotes/upstream/HEAD >actual &&
> +	echo "refs/remotes/upstream/$1" >expect &&
> +	test_cmp expect actual
> +}
> +
> +check_same_tip () {
> +	git log --oneline -1 "$1" >expect &&
> +	git -C "$2" log --oneline -1 "$3" >actual &&
> +	test_cmp expect actual
> +}
> +
>   test_expect_success 'setup' '
>   	commit 1 &&
>   	commit 2 &&
> @@ -261,6 +279,134 @@ test_expect_success 'fetch --deepen does not truncate' '
>   	test_cmp expect actual
>   '
>   
> +test_expect_success 'fetch.shallow setup' '
> +	git branch narrow-side &&
> +	git clone --no-local --depth=1 --branch main --single-branch \
> +		.git narrow-default &&
> +	git clone --no-local --depth=1 --branch main --single-branch \
> +		.git narrow-enabled &&
> +	(
> +		cd narrow-default &&
> +		git remote add upstream ../.git &&
> +		git fetch --depth=1 upstream main:refs/remotes/upstream/main &&
> +		git branch --set-upstream-to=upstream/main main
> +	) &&
> +	(
> +		cd narrow-enabled &&
> +		git remote add upstream ../.git &&
> +		git fetch --depth=1 upstream main:refs/remotes/upstream/main &&
> +		git branch --set-upstream-to=upstream/main main &&
> +		git config fetch.shallow true
> +	)
> +'
> +
> +test_expect_success 'a refspec-less fetch expands to the configured refspec by default' '
> +	(
> +		cd narrow-default &&
> +		git fetch upstream &&
> +		check_upstream_refs <<-\EOF
> +		refs/remotes/upstream/HEAD
> +		refs/remotes/upstream/main
> +		refs/remotes/upstream/narrow-side
> +		EOF
> +	)
> +'
> +
> +test_expect_success 'fetch.shallow=true limits a refspec-less fetch to the tracked branch' '
> +	(
> +		cd narrow-enabled &&
> +		git fetch upstream &&
> +		check_upstream_refs <<-\EOF
> +		refs/remotes/upstream/HEAD
> +		refs/remotes/upstream/main
> +		EOF
> +	)
> +'
> +
> +test_expect_success 'fetch.shallow=true still creates refs/remotes/<remote>/HEAD' '
> +	(
> +		cd narrow-enabled &&
> +		git symbolic-ref -d refs/remotes/upstream/HEAD &&
> +		git fetch upstream &&
> +		check_upstream_head main
> +	)
> +'
> +
> +test_expect_success 'fetch.shallow=true with followRemoteHEAD=always corrects a stale HEAD' '
> +	test_when_finished \
> +		"git -C narrow-enabled update-ref -d refs/remotes/upstream/stale-branch" &&
> +	(
> +		cd narrow-enabled &&
> +		git update-ref refs/remotes/upstream/stale-branch refs/remotes/upstream/main &&
> +		git symbolic-ref refs/remotes/upstream/HEAD refs/remotes/upstream/stale-branch &&
> +		git -c fetch.followRemoteHEAD=always fetch upstream &&
> +		check_upstream_head main
> +	)
> +'
> +
> +test_expect_success 'fetch.shallow=true still updates the tracked branch' '
> +	commit 5 &&
> +	git -C narrow-enabled fetch upstream &&
> +	check_same_tip main narrow-enabled refs/remotes/upstream/main
> +'
> +
> +test_expect_success 'fetch.shallow=true keeps git pull narrowed too' '
> +	test_when_finished "git branch -D narrow-side" &&
> +	commit 6 &&
> +	(
> +		cd narrow-enabled &&
> +		git pull &&
> +		check_upstream_refs <<-\EOF
> +		refs/remotes/upstream/HEAD
> +		refs/remotes/upstream/main
> +		EOF
> +	) &&
> +	check_same_tip main narrow-enabled HEAD
> +'
> +
> +test_expect_success 'fetch.shallow=true has no effect on a non-shallow repository' '
> +	git clone --no-local --branch main --single-branch .git narrow-full &&
> +	(
> +		cd narrow-full &&
> +		git rev-parse --is-shallow-repository >actual &&
> +		echo false >expect &&
> +		test_cmp expect actual &&
> +		git remote add upstream ../.git &&
> +		git fetch upstream &&
> +		git branch --set-upstream-to=upstream/main main &&
> +		git config fetch.shallow true
> +	) &&
> +	test_when_finished "git branch -D narrow-full-side" &&
> +	git branch narrow-full-side &&
> +	(
> +		cd narrow-full &&
> +		git fetch upstream &&
> +		check_upstream_refs <<-\EOF
> +		refs/remotes/upstream/HEAD
> +		refs/remotes/upstream/main
> +		refs/remotes/upstream/narrow-full-side
> +		EOF
> +	)
> +'
> +
> +test_expect_success 'fetch.shallow=true only narrows a fetch of the tracked remote' '
> +	test_when_finished "git branch -D other-side" &&
> +	git branch other-side &&
> +	git clone --no-local --depth=1 --branch main --single-branch \
> +		.git narrow-other-remote &&
> +	(
> +		cd narrow-other-remote &&
> +		git remote add upstream ../.git &&
> +		git config fetch.shallow true &&
> +		git fetch upstream &&
> +		check_upstream_refs <<-\EOF
> +		refs/remotes/upstream/HEAD
> +		refs/remotes/upstream/main
> +		refs/remotes/upstream/other-side
> +		EOF
> +	)
> +'
> +
>   . "$TEST_DIRECTORY"/lib-httpd.sh
>   start_httpd
>   
> 
> base-commit: d38352cd43ab9745686d697872408bc3249a153f