patchclone: --dissociate option to mark that reference is only temporary

> While use of the --reference option to borrow objects from an
> existing local repository of the same project is an effective way to
> reduce traffic when cloning a project over the network, it makes the
> resulting "borrowing" repository dependent on the "borrowed"
> repository.  After running
> 
> 	git clone --reference=P $URL Q
> 
> the resulting repository Q will be broken if the borrowed repository
> P disappears.
> 
> The way to allow the borrowed repository to be removed is to repack
> the borrowing repository (i.e. run "git repack -a -d" in Q); while
> power users may know it very well, it is not easily discoverable.
> 
> Teach a new "--dissociate" option to "git clone" to run this
> repacking for the user.

> Signed-off-by: Junio C Hamano <gitster@pobox.com>
> ---
> 
>  * This comes from
>    http://thread.gmane.org/gmane.comp.version-control.git/243918/focus=245397
>    which is one of the low-hanging entries in the leftover-bits list
>    http://git-blame.blogspot.com/p/leftover-bits.html
> 
>    Yes, I must have been really bored to do this ;-)
> 
>  Documentation/git-clone.txt | 11 +++++++++--
>  builtin/clone.c             | 25 +++++++++++++++++++++++++
>  t/t5700-clone-reference.sh  | 17 +++++++++++++++++
>  3 files changed, 51 insertions(+), 2 deletions(-)
> 
> diff --git a/Documentation/git-clone.txt b/Documentation/git-clone.txt
> index 0363d00..f1f2a3f 100644
> --- a/Documentation/git-clone.txt
> +++ b/Documentation/git-clone.txt
> @@ -12,7 +12,7 @@ SYNOPSIS
>  'git clone' [--template=<template_directory>]
>  	  [-l] [-s] [--no-hardlinks] [-q] [-n] [--bare] [--mirror]
>  	  [-o <name>] [-b <name>] [-u <upload-pack>] [--reference <repository>]
> -	  [--separate-git-dir <git dir>]
> +	  [--dissociate] [--separate-git-dir <git dir>]
>  	  [--depth <depth>] [--[no-]single-branch]
>  	  [--recursive | --recurse-submodules] [--] <repository>
>  	  [<directory>]
> @@ -98,7 +98,14 @@ objects from the source repository into a pack in the cloned repository.
>  	require fewer objects to be copied from the repository
>  	being cloned, reducing network and local storage costs.
>  +
> -*NOTE*: see the NOTE for the `--shared` option.
> +*NOTE*: see the NOTE for the `--shared` option, and also the
> +`--dissociate` option.
> +
> +--dissociate::
> +	Borrow the objects from reference repositories specified
> +	with the `--reference` options only to reduce network
> +	transfer and stop borrowing from them after a clone is made
> +	by making necessary local copies of borrowed objects.
>  
>  --quiet::
>  -q::
> diff --git a/builtin/clone.c b/builtin/clone.c
> index bbd169c..780fbd5 100644
> --- a/builtin/clone.c
> +++ b/builtin/clone.c
> @@ -48,6 +48,7 @@ static int option_verbosity;
>  static int option_progress = -1;
>  static struct string_list option_config;
>  static struct string_list option_reference;
> +static int option_dissociate;
>  
>  static int opt_parse_reference(const struct option *opt, const char *arg, int unset)
>  {
> @@ -93,6 +94,8 @@ static struct option builtin_clone_options[] = {
>  		    N_("create a shallow clone of that depth")),
>  	OPT_BOOL(0, "single-branch", &option_single_branch,
>  		    N_("clone only one branch, HEAD or --branch")),
> +	OPT_BOOL(0, "dissociate", &option_dissociate,
> +		 N_("use --reference only while cloning")),
>  	OPT_STRING(0, "separate-git-dir", &real_git_dir, N_("gitdir"),
>  		   N_("separate git dir from working tree")),
>  	OPT_STRING_LIST('c', "config", &option_config, N_("key=value"),
> @@ -736,6 +739,21 @@ static void write_refspec_config(const char* src_ref_prefix,
>  	strbuf_release(&value);
>  }
>  
> +static void dissociate_from_references(void)
> +{
> +	struct child_process cmd;
> +
> +	memset(&cmd, 0, sizeof(cmd));
> +	argv_array_pushl(&cmd.args, "repack", "-a", "-d", NULL);
> +	cmd.git_cmd = 1;
> +	cmd.out = -1;
> +	cmd.no_stdin = 1;
> +	if (run_command(&cmd))
> +		die(_("cannot repack to clean up"));
> +	if (unlink(git_path("objects/info/alternates")) && errno != ENOENT)
> +		die_errno(_("cannot unlink temporary alternates file"));
> +}
> +
>  int cmd_clone(int argc, const char **argv, const char *prefix)
>  {
>  	int is_bundle = 0, is_local;
> @@ -883,6 +901,10 @@ int cmd_clone(int argc, const char **argv, const char *prefix)
>  
>  	if (option_reference.nr)
>  		setup_reference();
> +	else if (option_dissociate) {
> +		warning(_("--dissociate given, but there is no --reference"));
> +		option_dissociate = 0;
> +	}
>  
>  	fetch_pattern = value.buf;
>  	refspec = parse_fetch_refspec(1, &fetch_pattern);
> @@ -996,6 +1018,9 @@ int cmd_clone(int argc, const char **argv, const char *prefix)
>  	transport_unlock_pack(transport);
>  	transport_disconnect(transport);
>  
> +	if (option_dissociate)
> +		dissociate_from_references();
> +
>  	junk_mode = JUNK_LEAVE_REPO;
>  	err = checkout();
>  
> diff --git a/t/t5700-clone-reference.sh b/t/t5700-clone-reference.sh
> index 6537911..3e783fc 100755
> --- a/t/t5700-clone-reference.sh
> +++ b/t/t5700-clone-reference.sh
> @@ -198,4 +198,21 @@ test_expect_success 'clone using repo pointed at by gitfile as reference' '
>  	test_cmp expected "$base_dir/O/.git/objects/info/alternates"
>  '
>  
> +test_expect_success 'clone and dissociate from reference' '
> +	git init P &&
> +	(
> +		cd P &&	test_commit one
> +	) &&
> +	git clone P Q &&
> +	(
> +		cd Q && test_commit two
> +	) &&
> +	git clone --no-local --reference=P Q R &&
> +	git clone --no-local --reference=P --dissociate Q S &&
> +	# removing the reference P would corrupt R but not S
> +	rm -fr P &&
> +	test_must_fail git -C R fsck &&
> +	git -C S fsck
> +'
> +
>  test_done
> 

> I think things would be more understandable if the option was "--dissociate
> <repository>" and was an explicit alternative to --reference:
> 	[[--reference | --dissociate] <repository>]
>
> I'm still not liking the name "--dissociate" though.  The original suggestion
> of "--borrow" is better.  Perhaps "--library" or "--local-cache"?  I dunno...

> Marc Branchaud <marcnarc@xiplink.com> writes:
> 
>> I think things would be more understandable if the option was "--dissociate
>> <repository>" and was an explicit alternative to --reference:
>> 	[[--reference | --dissociate] <repository>]
>>
>> I'm still not liking the name "--dissociate" though.  The original suggestion
>> of "--borrow" is better.  Perhaps "--library" or "--local-cache"?  I dunno...
> 
> I was not thinking when I originally started the topic with
> "--borrow", until I realized that it would not make much sense,
> primarily because we allow multiple references.

> What should this command line do, and how would you implement such a
> behaviour?
> 
>     $ git clone \
>         --reference=/local/pool/linux.git \
>         --borrow=../my/neighbour/linux-hack.git \
>         git://git.kernel.org/...../linux.git
> 
> With "do the usual --reference thing, but then dissociate the result
> from referents" option, there is no ambiguity and that is why I did
> not go with the "--borrow" option suggested in the original thread.

>> So now I'm wondering if the implementation would be more efficient as an
>> extension of the --local operation.  That is, instead of a post-clone repack,
>> do a --local clone first followed by a simple "git fetch" from the source repo.
> 
> The network overhead may be comparable to the "--reference"
> optimization, but if your "clone --local" ends up copying (instead
> of hard-linking), the initial cost to copy locally would be a pure
> extra price over "clone --reference and then --dissociate".  If the
> local clone uses hard-linking, it would be cheaper, but it still
> costs more than dropping an entry into .git/objects/info/alternates,
> I would imagine.  You will pay with your scheme the same cost to run
> "repack -a -d", which is paid by "--dissociate" at the end of clone,
> eventually at the first "gc", so there is no efficiency advantage,
> either.
> 
> The above is my knee-jerk assessment without any measuring, though.

> On 14-10-15 01:29 PM, Junio C Hamano wrote:
>
>>     $ git clone \
>>         --reference=/local/pool/linux.git \
>>         --borrow=../my/neighbour/linux-hack.git \
>>         git://git.kernel.org/...../linux.git
>> 
>> With "do the usual --reference thing, but then dissociate the result
>> from referents" option, there is no ambiguity and that is why I did
>> not go with the "--borrow" option suggested in the original thread.
>
> I had not considered this case.  My limited imagination has a hard time
> coming up with a scenario where more than one --reference (or
> In this example, the --borrow seems
> useless.  How would clone decide that it even needed objects from the
> neighbour repo?  None of the refs on gko need any of the neighbour's unique
> objects.

> Marc Branchaud <marcnarc@xiplink.com> writes:
> 
>> On 14-10-15 01:29 PM, Junio C Hamano wrote:
>>
>>>     $ git clone \
>>>         --reference=/local/pool/linux.git \
>>>         --borrow=../my/neighbour/linux-hack.git \
>>>         git://git.kernel.org/...../linux.git
>>>
>>> With "do the usual --reference thing, but then dissociate the result
>>> from referents" option, there is no ambiguity and that is why I did
>>> not go with the "--borrow" option suggested in the original thread.
>>
>> I had not considered this case.  My limited imagination has a hard time
>> coming up with a scenario where more than one --reference (or
>> In this example, the --borrow seems
>> useless.  How would clone decide that it even needed objects from the
>> neighbour repo?  None of the refs on gko need any of the neighbour's unique
>> objects.
> 
> A probable scenario might go like this.
> 
>     The company-wide pool is designed for everybody's use and will
>     stay, even if it lags behind because it fetches every other day,
>     so it is safe to keep referring to via alternates.  My neighbour
>     is following the linux-next repository and has changes that are
>     meant to land "in the future" to the mainline, but it can
>     disappear without notice so I cannot afford to depend on its
>     presense forever.
> 
> Under that particular scenario, what should happen is fairly clear;
> we want to dissociate from neibour's immediately after clone is
> done, while being still dependent on the shared pool.

> Marc Branchaud <marcnarc@xiplink.com> writes:
> 
>> Yes, but we're cloning gko, not the neighbour.  Doesn't that mean that the
>> clone operation won't know about any of the neighbour's refs?
> 
> No.  --reference (and a natural implementation of --borrow, I would imagine)
> peeks the refs of the repository we borrow from and that is how
> clone can say "I already have objects reachable from these refs, so
> please send me the remainder" to the repository it is cloning from.

> I just don't see what difference the --borrow option makes.  Consider the two
> cases:
>
> With just --reference=/local/pool/linux.git:
> 	1. Set up the alternates file with that path.

> 	2. Copy gko's refs into refs/remotes/origin/.
> 	3. Set up refs/heads/master to refer to gko's HEAD.
> 	4. Checkout refs/heads/master (uses objects from local pool).
>
> With both that --reference and --borrow=../my/neighbour/linux-hack.git:
> 	1. Set up the alternates file with both paths.

> It seems that in order to make something like --borrow useful, "git clone"
> would somehow need to know which of the neighbour's refs you want to *also*
> clone, then copy any unique objects from the neighbour before disassociating
> from it.
>
> 		M.

> Marc Branchaud <marcnarc@xiplink.com> writes:
>
>> I think things would be more understandable if the option was "--dissociate
>> <repository>" and was an explicit alternative to --reference:
>> 	[[--reference | --dissociate] <repository>]
>>
>> I'm still not liking the name "--dissociate" though.  The original suggestion
>> of "--borrow" is better.  Perhaps "--library" or "--local-cache"?  I dunno...
>
> I was not thinking when I originally started the topic with
> "--borrow", until I realized that it would not make much sense,
> primarily because we allow multiple references.
>
> What should this command line do, and how would you implement such a
> behaviour?
>
>     $ git clone \
>         --reference=/local/pool/linux.git \
>         --borrow=../my/neighbour/linux-hack.git \
>         git://git.kernel.org/...../linux.git
>
> With "do the usual --reference thing, but then dissociate the result
> from referents" option, there is no ambiguity and that is why I did
> not go with the "--borrow" option suggested in the original thread.

> +static void dissociate_from_references(void)
> +{
> +	struct child_process cmd;
> +
> +	memset(&cmd, 0, sizeof(cmd));

> +	cmd.no_stdin = 1;
> +	if (run_command(&cmd))
> +		die(_("cannot repack to clean up"));
> +	if (unlink(git_path("objects/info/alternates")) && errno != ENOENT)
> +		die_errno(_("cannot unlink temporary alternates file"));
> +}

>
> diff --git a/builtin/clone.c b/builtin/clone.c
> index 8649090..81efb07 100644
> --- a/builtin/clone.c
> +++ b/builtin/clone.c
> @@ -743,14 +743,9 @@ static void write_refspec_config(const char *src_ref_prefix,
>  
>  static void dissociate_from_references(void)
>  {
> -	struct child_process cmd;
> -
> -	memset(&cmd, 0, sizeof(cmd));
> -	argv_array_pushl(&cmd.args, "repack", "-a", "-d", NULL);
> -	cmd.git_cmd = 1;
> -	cmd.out = -1;
> -	cmd.no_stdin = 1;
> -	if (run_command(&cmd))
> +	static const char* argv[] = { "repack", "-a", "-d", NULL };
> +
> +	if (run_command_v_opt(argv, RUN_GIT_CMD|RUN_COMMAND_NO_STDIN))
>  		die(_("cannot repack to clean up"));
>  	if (unlink(git_path("objects/info/alternates")) && errno != ENOENT)
>  		die_errno(_("cannot unlink temporary alternates file"));