patch, 5 partssubmodule-config: keep submodule groups around

> This passes each group to the `submodule update` invocation and
> additionally configures the groups to be automatically updated.
>
> Signed-off-by: Stefan Beller <sbeller@google.com>
> ---
>   Documentation/git-clone.txt | 11 ++++++++
>   builtin/clone.c             | 33 ++++++++++++++++++++--
>   git-submodule.sh            |  5 ++++
>   t/t7400-submodule-basic.sh  | 69 +++++++++++++++++++++++++++++++++++++++++++++
>   t/t7406-submodule-update.sh | 32 +++++++++++++++++++++
>   5 files changed, 147 insertions(+), 3 deletions(-)
>
> diff --git a/Documentation/git-clone.txt b/Documentation/git-clone.txt
> index 59d8c67..fbf68ab 100644
> --- a/Documentation/git-clone.txt
> +++ b/Documentation/git-clone.txt
> @@ -209,6 +209,17 @@ objects from the source repository into a pack in the cloned repository.
>   	repository does not have a worktree/checkout (i.e. if any of
>   	`--no-checkout`/`-n`, `--bare`, or `--mirror` is given)
>
> +--group::
> +	After the clone is created, all submodules which are part of the
> +	group are cloned. This option can be given multiple times to specify
> +	different groups.

> +	The group selection will be passed on recursively, i.e. if a submodule
> +	is cloned because of group membership, its submodules will
> +	be cloned according to group membership, too. If a submodule is
> +	not cloned however, its submodules are not evaluated for group
> +	membership.

>   --separate-git-dir=<git dir>::
>   	Instead of placing the cloned repository where it is supposed
>   	to be, place the cloned repository at the specified directory,
> diff --git a/builtin/clone.c b/builtin/clone.c
> index ce578d2..17e9f54 100644
> --- a/builtin/clone.c
> +++ b/builtin/clone.c
> @@ -51,6 +51,7 @@ static struct string_list option_config;
>   static struct string_list option_reference;
>   static int option_dissociate;
>   static int max_jobs = -1;
> +static struct string_list submodule_groups;
>
>   static struct option builtin_clone_options[] = {
>   	OPT__VERBOSITY(&option_verbosity),
> @@ -95,6 +96,8 @@ static struct option builtin_clone_options[] = {
>   		   N_("separate git dir from working tree")),
>   	OPT_STRING_LIST('c', "config", &option_config, N_("key=value"),
>   			N_("set config inside the new repository")),
> +	OPT_STRING_LIST('g', "group", &submodule_groups, N_("group"),
> +			N_("clone specific submodule groups")),
>   	OPT_END()
>   };
>
> @@ -723,9 +726,18 @@ static int checkout(void)
>   	err |= run_hook_le(NULL, "post-checkout", sha1_to_hex(null_sha1),
>   			   sha1_to_hex(sha1), "1", NULL);
>
> -	if (!err && option_recursive) {
> +	if (err)
> +		goto out;
> +
> +	if (option_recursive || submodule_groups.nr > 0) {
>   		struct argv_array args = ARGV_ARRAY_INIT;
> -		argv_array_pushl(&args, "submodule", "update", "--init", "--recursive", NULL);
> +		argv_array_pushl(&args, "submodule", "update", "--init", NULL);
> +
> +		if (option_recursive)
> +			argv_array_pushf(&args, "--recursive");
> +
> +		if (submodule_groups.nr > 0)
> +			argv_array_pushf(&args, "--groups");
>
>   		if (max_jobs != -1)
>   			argv_array_pushf(&args, "--jobs=%d", max_jobs);
> @@ -733,7 +745,7 @@ static int checkout(void)
>   		err = run_command_v_opt(args.argv, RUN_GIT_CMD);
>   		argv_array_clear(&args);
>   	}
> -
> +out:
>   	return err;
>   }
>
> @@ -864,6 +876,21 @@ int cmd_clone(int argc, const char **argv, const char *prefix)
>   		option_no_checkout = 1;
>   	}
>
> +	if (option_recursive && submodule_groups.nr > 0)
> +		die(_("submodule groups and recursive flag are incompatible"));

> +	if (submodule_groups.nr > 0) {
> +		int first_item = 1;
> +		struct string_list_item *item;
> +		struct strbuf sb = STRBUF_INIT;
> +		strbuf_addstr(&sb, "submodule.groups=");
> +		for_each_string_list_item(item, &submodule_groups) {
> +			strbuf_addf(&sb, "%s%s", first_item ? "" : ",", item->string);
> +			first_item = 0;
> +		}
> +		if (submodule_groups.nr > 0)
> +			string_list_append(&option_config, strbuf_detach(&sb, 0));
> +	}
> +
>   	if (!option_origin)
>   		option_origin = "origin";
>
> diff --git a/git-submodule.sh b/git-submodule.sh
> index 4092a48..e3d1667 100755
> --- a/git-submodule.sh
> +++ b/git-submodule.sh
> @@ -611,6 +611,7 @@ cmd_deinit()
>   #
>   cmd_update()
>   {
> +	groups=
>   	# parse $args after "submodule ... update".
>   	while test $# -ne 0
>   	do
> @@ -650,6 +651,9 @@ cmd_update()
>   		--checkout)
>   			update="checkout"
>   			;;
> +		--groups)
> +			groups=1
> +			;;
>   		--depth)
>   			case "$2" in '') usage ;; esac
>   			depth="--depth=$2"
> @@ -691,6 +695,7 @@ cmd_update()
>   		${update:+--update "$update"} \
>   		${reference:+--reference "$reference"} \
>   		${depth:+--depth "$depth"} \
> +		${groups:+--groups} \
>   		${jobs:+$jobs} \
>   		"$@" | {
>   	err=
> diff --git a/t/t7400-submodule-basic.sh b/t/t7400-submodule-basic.sh
> index caed4be..e8654d7 100755
> --- a/t/t7400-submodule-basic.sh
> +++ b/t/t7400-submodule-basic.sh
> @@ -1049,4 +1049,73 @@ test_expect_success 'submodule init --group works' '
>   	)
>   '
>
> +cat <<EOF > expected
> +submodule
> +-submodule1
> +EOF
> +
> +test_expect_success 'submodule update --groups works' '
> +	test_when_finished "rm -rf super super_clone" &&
> +	mkdir super &&
> +	pwd=$(pwd) &&
> +	(
> +		cd super &&
> +		git init &&
> +		git submodule add --group groupA file://"$pwd"/example2 submodule &&
> +		git submodule add file://"$pwd"/example2 submodule1 &&
> +		git commit -a -m "create repository with 2 submodules, one is in a group"
> +	) &&
> +	git clone super super_clone &&
> +	(
> +		cd super_clone &&
> +		git config submodule.groups groupA &&
> +		git submodule init  &&
> +		git submodule update --groups &&
> +		git submodule status |cut -c1,42-52 | tr -d " " >../actual
> +	) &&
> +	test_cmp actual expected
> +'
> +
> +test_expect_success 'submodule update --init --groups works' '
> +	test_when_finished "rm -rf super super_clone" &&
> +	mkdir super &&
> +	pwd=$(pwd) &&
> +	(
> +		cd super &&
> +		git init &&
> +		git submodule add --group groupA file://"$pwd"/example2 submodule &&
> +		git submodule add file://"$pwd"/example2 submodule1 &&
> +		git commit -a -m "create repository with 2 submodules, one is in a group"
> +	) &&
> +	git clone super super_clone &&
> +	(
> +		cd super_clone &&
> +		git config submodule.groups groupA &&
> +		git submodule update --init --groups &&
> +		git submodule status |cut -c1,42-52 | tr -d " " >../actual
> +	) &&
> +	test_cmp actual expected
> +'
> +
> +test_expect_success 'clone --group works' '
> +	test_when_finished "rm -rf super super_clone" &&
> +	mkdir super &&
> +	pwd=$(pwd) &&
> +	(
> +		cd super &&
> +		git init &&
> +		git submodule add --group groupA file://"$pwd"/example2 submodule &&
> +		git submodule add file://"$pwd"/example2 submodule1 &&
> +		git commit -a -m "create repository with 2 submodules, one is in a group"
> +	) &&
> +	git clone --group groupA super super_clone &&
> +	(
> +		cd super_clone &&
> +		test_pause
> +		git submodule status |cut -c1,42-52 | tr -d " " >../actual
> +	) &&
> +	test_cmp actual expected
> +'
> +
> +
>   test_done
> diff --git a/t/t7406-submodule-update.sh b/t/t7406-submodule-update.sh
> index 090891e..7e59846 100755
> --- a/t/t7406-submodule-update.sh
> +++ b/t/t7406-submodule-update.sh
> @@ -801,4 +801,36 @@ test_expect_success 'git clone passes the parallel jobs config on to submodules'
>   	rm -rf super4
>   '
>
> +cat >expect <<-EOF &&
> +-deeper/submodule
> +-merging
> +-moved/sub module
> +-none
> +-rebasing
> +-submodule
> +-submodule1
> +EOF
> +
> +# none, merging rebasing, submodule1, submodule
> +test_expect_success 'git clone works with submodule groups.' '
> +	test_when_finished "rm -rf super5" &&
> +	(
> +		cd super &&
> +		git config -f .gitmodules  submodule.submodule.groups default &&
> +		git config -f .gitmodules  submodule.submodule1.groups "default,testing" &&
> +		git config -f .gitmodules  submodule.none.groups testing &&
> +		git commit -a -m "assigning groups to submodules"
> +	) &&
> +	git clone --group default --group testing super super5 &&
> +	(
> +		cd super5 &&
> +		git submodule status |cut -c1,43- >../actual
> +	) &&
> +	test_cmp actual expect
> +'
> +
> +test_expect_success 'git submodule update --groups' '
> +	true
> +'
> +
>   test_done
>

>> +--group::
>> +       After the clone is created, all submodules which are part of the
>> +       group are cloned. This option can be given multiple times to
>> specify
>> +       different groups.
>
>
> Ah, that answers my question in my response to the cover letter ;-)
>
>> This option will imply automatic submodule
>> +       updates for the groups by setting `submodule.update=groups`.
>
>
> Please don't. The per-submodule update setting configures how a
> submodule has to be updated, adding a global one with a completely
> different meaning (what submodules should be updated?) is confusing.
> Why not "submodule.groups=<groups>"?

>
>> +       The group selection will be passed on recursively, i.e. if a
>> submodule
>> +       is cloned because of group membership, its submodules will
>> +       be cloned according to group membership, too. If a submodule is
>> +       not cloned however, its submodules are not evaluated for group
>> +       membership.
>
>
> What do you mean by the last sentence? Did the clone fail? Then you
> cannot update the submodule anyway ...

>> @@ -864,6 +876,21 @@ int cmd_clone(int argc, const char **argv, const char
>> *prefix)
>>                 option_no_checkout = 1;
>>         }
>>
>> +       if (option_recursive && submodule_groups.nr > 0)
>> +               die(_("submodule groups and recursive flag are
>> incompatible"));
>
>
> Me thinks this contradicts your description of the --group option
> in the man page. I don't see why such a restriction would make
> sense, what incompatibility are you trying to avoid here? Maybe
> we need another submodule-specific setting to tell update what
> groups to use inside that submodule?

> On Wed, Nov 25, 2015 at 9:52 AM, Jens Lehmann <Jens.Lehmann@web.de> wrote:
>>> +--group::
>>> +       After the clone is created, all submodules which are part of the
>>> +       group are cloned. This option can be given multiple times to
>>> specify
>>> +       different groups.
>>
>>
>> Ah, that answers my question in my response to the cover letter ;-)
>>
>>> This option will imply automatic submodule
>>> +       updates for the groups by setting `submodule.update=groups`.
>>
>>
>> Please don't. The per-submodule update setting configures how a
>> submodule has to be updated, adding a global one with a completely
>> different meaning (what submodules should be updated?) is confusing.
>> Why not "submodule.groups=<groups>"?
>
> The documentation is out of date :/ as I was churning through lots of ideas,
> so we do have a config submodule.groups=<groups> by now, but the
> documentation is wrong.

>>
>>> +       The group selection will be passed on recursively, i.e. if a
>>> submodule
>>> +       is cloned because of group membership, its submodules will
>>> +       be cloned according to group membership, too. If a submodule is
>>> +       not cloned however, its submodules are not evaluated for group
>>> +       membership.
>>
>>
>> What do you mean by the last sentence? Did the clone fail? Then you
>> cannot update the submodule anyway ...
>
> Consider nested submodules:
>
>      A: superproject containing
>          B: which contains
>              C.
>
> If you clone A with group <C-but-not-B> you won't get C as we do not traverse
> the submodules of B, as we don't clone B. Maybe it's obvious?

>>> @@ -864,6 +876,21 @@ int cmd_clone(int argc, const char **argv, const char
>>> *prefix)
>>>                  option_no_checkout = 1;
>>>          }
>>>
>>> +       if (option_recursive && submodule_groups.nr > 0)
>>> +               die(_("submodule groups and recursive flag are
>>> incompatible"));
>>
>>
>> Me thinks this contradicts your description of the --group option
>> in the man page. I don't see why such a restriction would make
>> sense, what incompatibility are you trying to avoid here? Maybe
>> we need another submodule-specific setting to tell update what
>> groups to use inside that submodule?
>
> So you want something like
>      "In the top level respect the groups, but recursively get all of them"?

>
>> My thinking is that groups are implying recursive, whereas recursive
>> implies
>> "all groups", so a git clone --group <half-the-submodules> --recursive
>> makes not much sense to me as it begs the question, what does --recursive
>> mean?
>
>
> Groups are only about what submodules to update and have nothing to
> do with recursion. It might make sense to imply recursion, but that's
> just because that should have been the default for submodules from day
> one. Recursion and groups are orthogonal, the first is about what to
> do inside the submodules (carry on or not?) and the latter is about
> what to do in the superproject (shall I update this submodule?).

>
>> Probably recurse into all submodules which are implied by the group
>>
>> <half-the-submodules>.
>
>
> Yep. We also do not recurse into those submodules having set their
> update setting to "none", so we do not do that for submodules not
> in any chosen group either.
>
>> And then get all the nested submodules. But in case
>>
>> you use the grouping feature, you could just mark the nested submodules
>> with
>> groups, too?
>
>
> Not in the top superproject. In a submodule you can specify new groups
> for its sub-submodules, but these will in most cases be different from
> those of the superproject.
>
> Imagine I have this really cool Metaproject which contains the Android
> superproject as a submodule. Those two will define different groups,
> and when recursing into the android submodule I need to choose from the
> Android specific groups. So my Metaproject's .gitmodules could look like
> this:
>
> [submodule "android"]
>         path = android
>         url = git://...
>         groups = default,mobile
>         subgroups = devel
>
> "groups" tells git what superproject groups the android submodule
> belongs to, and "subgroups" tells git what android submodules are
> to be checked out when running recursively into it. If you do not
> configure "subgroups", the whole android submodule is updated when
> one of the groups "default" or "mobile" is chosen in the superproject.

> On Wed, Nov 25, 2015 at 11:50 AM, Jens Lehmann <Jens.Lehmann@web.de> wrote:
>>
>>> My thinking is that groups are implying recursive, whereas recursive
>>> implies
>>> "all groups", so a git clone --group <half-the-submodules> --recursive
>>> makes not much sense to me as it begs the question, what does --recursive
>>> mean?
>>
>>
>> Groups are only about what submodules to update and have nothing to
>> do with recursion. It might make sense to imply recursion, but that's
>> just because that should have been the default for submodules from day
>> one. Recursion and groups are orthogonal, the first is about what to
>> do inside the submodules (carry on or not?) and the latter is about
>> what to do in the superproject (shall I update this submodule?).
>
> I see. So we would not want to mutually exclude recurse and groups,
> but rather have groups implies --recurse, but you are allowed to give
> --no-recurse if you explicitely do not want to recurse into the subsubmodules.

>>> And then get all the nested submodules. But in case
>>>
>>> you use the grouping feature, you could just mark the nested submodules
>>> with
>>> groups, too?
>>
>>
>> Not in the top superproject. In a submodule you can specify new groups
>> for its sub-submodules, but these will in most cases be different from
>> those of the superproject.
>>
>> Imagine I have this really cool Metaproject which contains the Android
>> superproject as a submodule. Those two will define different groups,
>> and when recursing into the android submodule I need to choose from the
>> Android specific groups. So my Metaproject's .gitmodules could look like
>> this:
>>
>> [submodule "android"]
>>          path = android
>>          url = git://...
>>          groups = default,mobile
>>          subgroups = devel
>>
>> "groups" tells git what superproject groups the android submodule
>> belongs to, and "subgroups" tells git what android submodules are
>> to be checked out when running recursively into it. If you do not
>> configure "subgroups", the whole android submodule is updated when
>> one of the groups "default" or "mobile" is chosen in the superproject.
>
> I like the concept of subgroups as it allows to have some control over
> subsubmodules you may want to aggregate from a third party via the
> middleman submodule.

>>
>>
>> I like the concept of subgroups as it allows to have some control over
>> subsubmodules you may want to aggregate from a third party via the
>> middleman submodule.
>
>
> That's the point (though maybe someone might come up with a better
> name than "subgroups" ;-). And each repo configures its own submodule
> groups.
>
>> I'd prefer to delay that feature though by not giving a high priority.
>
>
> No problem, we can start with "check out all subsubmodules" for now.
> But I suspect we'll need subgroups rather sooner than later.

>
>> Also would you go with subsubgroups, too? When does the recursion
>> end?
>
>
> Subsubgroups do not make sense in the superproject, that can only
> configure its direct submodules.

> I think you are talking about the
> groups of the subsubmodules, and these have to be chosen inside the
> first level submodules via the subgroups of its submodules (which
> are the second level submodules of the superproject). Still with
> me? ;-)

>
>> In case we have more than the union of groups, but also prohibitive
>>
>> terms available, could subgroups clash with the submodules groups spec?
>
>
> Not that I'm aware of. Groups decide which submodules to update and
> only for those submodules subgroups tell git what group to use inside
> that submodule. And so on.

> This passes each group to the `submodule update` invocation and
> additionally configures the groups to be automatically updated.
>
> Signed-off-by: Stefan Beller <sbeller@google.com>
> ---
>
> This is a resend of the patch "[PATCH 5/5] builtin/clone: support submodule groups"
> as that's where Jens and I discussed.
>
> * reworded the documentation to match reality of the patch

> This passes each group to the `submodule update` invocation and
> additionally configures the groups to be automatically updated.
> 
> Signed-off-by: Stefan Beller <sbeller@google.com>
> ---
> 
> This is a resend of the patch "[PATCH 5/5] builtin/clone: support submodule groups"
> as that's where Jens and I discussed.

> 
> * reworded the documentation to match reality of the patch
> * --recurse is now implied and can be turned off.
> 
> Thanks for the fast feedback,
> Stefan
> 
> Interdiff to previous version [PATCH 5/5] builtin/clone: support submodule groups
>         --- a/Documentation/git-clone.txt
>         +++ b/Documentation/git-clone.txt
>         @@ -211,14 +211,16 @@ objects from the source repository into a pack in the cloned repository.
>          
>          --group::
>                 After the clone is created, all submodules which are part of the
>         -       group are cloned. This option can be given multiple times to specify
>         -       different groups. This option will imply automatic submodule
>         -       updates for the groups by setting `submodule.update=groups`.
>         -       The group selection will be passed on recursively, i.e. if a submodule
>         -       is cloned because of group membership, its submodules will
>         -       be cloned according to group membership, too. If a submodule is
>         -       not cloned however, its submodules are not evaluated for group
>         -       membership.
>         +       given groups are cloned. To specify multiple groups, you can either
>         +       give the group argument multiple times or comma separate the groups.
>         +       This option will be recorded in the `submodule.groups` config,
>         +       which will affect the behavior of other submodule related commands,
>         +       such as `git submodule update`.
>         +       This option implies recursive submodule checkout. If you don't
>         +       want to recurse into nested submodules, you need to specify
>         +       `--no-recursive`. The group selection will be passed on recursively,
>         +       i.e. if a submodule is cloned because of group membership, its
>         +       submodules will be cloned according to group membership, too.
>          
>          --separate-git-dir=<git dir>::
>                 Instead of placing the cloned repository where it is supposed
>         diff --git a/builtin/clone.c b/builtin/clone.c
>         index 17e9f54..377c031 100644
>         --- a/builtin/clone.c
>         +++ b/builtin/clone.c
>         @@ -39,7 +39,7 @@ static const char * const builtin_clone_usage[] = {
>          };
>          
>          static int option_no_checkout, option_bare, option_mirror, option_single_branch = -1;
>         -static int option_local = -1, option_no_hardlinks, option_shared, option_recursive;
>         +static int option_local = -1, option_no_hardlinks, option_shared, option_recursive = -1;
>          static char *option_template, *option_depth;
>          static char *option_origin = NULL;
>          static char *option_branch = NULL;
>         @@ -875,9 +875,12 @@ int cmd_clone(int argc, const char **argv, const char *prefix)
>                                 die(_("--bare and --separate-git-dir are incompatible."));
>                         option_no_checkout = 1;
>                 }
>         -
>         -       if (option_recursive && submodule_groups.nr > 0)
>         -               die(_("submodule groups and recursive flag are incompatible"));
>         +       if (option_recursive == -1) {
>         +               if (submodule_groups.nr > 0)
>         +                       option_recursive = 1; /* submodule groups implies recursive */
>         +               else
>         +                       option_recursive = 0; /* preserve historical default */
>         +       }
>                 if (submodule_groups.nr > 0) {
>                         int first_item = 1;
>                         struct string_list_item *item;
> 
> Here comes the actual patch:
> 
>  Documentation/git-clone.txt | 13 +++++++++
>  builtin/clone.c             | 38 ++++++++++++++++++++++---
>  git-submodule.sh            |  5 ++++
>  t/t7400-submodule-basic.sh  | 69 +++++++++++++++++++++++++++++++++++++++++++++
>  t/t7406-submodule-update.sh | 32 +++++++++++++++++++++
>  5 files changed, 153 insertions(+), 4 deletions(-)
> 
> diff --git a/Documentation/git-clone.txt b/Documentation/git-clone.txt
> index 59d8c67..2539fea 100644
> --- a/Documentation/git-clone.txt
> +++ b/Documentation/git-clone.txt
> @@ -209,6 +209,19 @@ objects from the source repository into a pack in the cloned repository.
>  	repository does not have a worktree/checkout (i.e. if any of
>  	`--no-checkout`/`-n`, `--bare`, or `--mirror` is given)
>  
> +--group::
> +	After the clone is created, all submodules which are part of the
> +	given groups are cloned. To specify multiple groups, you can either
> +	give the group argument multiple times or comma separate the groups.
> +	This option will be recorded in the `submodule.groups` config,
> +	which will affect the behavior of other submodule related commands,
> +	such as `git submodule update`.
> +	This option implies recursive submodule checkout. If you don't
> +	want to recurse into nested submodules, you need to specify
> +	`--no-recursive`. The group selection will be passed on recursively,
> +	i.e. if a submodule is cloned because of group membership, its
> +	submodules will be cloned according to group membership, too.
> +
>  --separate-git-dir=<git dir>::
>  	Instead of placing the cloned repository where it is supposed
>  	to be, place the cloned repository at the specified directory,
> diff --git a/builtin/clone.c b/builtin/clone.c
> index ce578d2..377c031 100644
> --- a/builtin/clone.c
> +++ b/builtin/clone.c
> @@ -39,7 +39,7 @@ static const char * const builtin_clone_usage[] = {
>  };
>  
>  static int option_no_checkout, option_bare, option_mirror, option_single_branch = -1;
> -static int option_local = -1, option_no_hardlinks, option_shared, option_recursive;
> +static int option_local = -1, option_no_hardlinks, option_shared, option_recursive = -1;
>  static char *option_template, *option_depth;
>  static char *option_origin = NULL;
>  static char *option_branch = NULL;
> @@ -51,6 +51,7 @@ static struct string_list option_config;
>  static struct string_list option_reference;
>  static int option_dissociate;
>  static int max_jobs = -1;
> +static struct string_list submodule_groups;
>  
>  static struct option builtin_clone_options[] = {
>  	OPT__VERBOSITY(&option_verbosity),
> @@ -95,6 +96,8 @@ static struct option builtin_clone_options[] = {
>  		   N_("separate git dir from working tree")),
>  	OPT_STRING_LIST('c', "config", &option_config, N_("key=value"),
>  			N_("set config inside the new repository")),
> +	OPT_STRING_LIST('g', "group", &submodule_groups, N_("group"),
> +			N_("clone specific submodule groups")),
>  	OPT_END()
>  };
>  
> @@ -723,9 +726,18 @@ static int checkout(void)
>  	err |= run_hook_le(NULL, "post-checkout", sha1_to_hex(null_sha1),
>  			   sha1_to_hex(sha1), "1", NULL);
>  
> -	if (!err && option_recursive) {
> +	if (err)
> +		goto out;
> +
> +	if (option_recursive || submodule_groups.nr > 0) {
>  		struct argv_array args = ARGV_ARRAY_INIT;
> -		argv_array_pushl(&args, "submodule", "update", "--init", "--recursive", NULL);
> +		argv_array_pushl(&args, "submodule", "update", "--init", NULL);
> +
> +		if (option_recursive)
> +			argv_array_pushf(&args, "--recursive");
> +
> +		if (submodule_groups.nr > 0)
> +			argv_array_pushf(&args, "--groups");
>  
>  		if (max_jobs != -1)
>  			argv_array_pushf(&args, "--jobs=%d", max_jobs);
> @@ -733,7 +745,7 @@ static int checkout(void)
>  		err = run_command_v_opt(args.argv, RUN_GIT_CMD);
>  		argv_array_clear(&args);
>  	}
> -
> +out:
>  	return err;
>  }
>  
> @@ -863,6 +875,24 @@ int cmd_clone(int argc, const char **argv, const char *prefix)
>  			die(_("--bare and --separate-git-dir are incompatible."));
>  		option_no_checkout = 1;
>  	}
> +	if (option_recursive == -1) {
> +		if (submodule_groups.nr > 0)
> +			option_recursive = 1; /* submodule groups implies recursive */
> +		else
> +			option_recursive = 0; /* preserve historical default */
> +	}
> +	if (submodule_groups.nr > 0) {
> +		int first_item = 1;
> +		struct string_list_item *item;
> +		struct strbuf sb = STRBUF_INIT;
> +		strbuf_addstr(&sb, "submodule.groups=");
> +		for_each_string_list_item(item, &submodule_groups) {
> +			strbuf_addf(&sb, "%s%s", first_item ? "" : ",", item->string);
> +			first_item = 0;
> +		}
> +		if (submodule_groups.nr > 0)
> +			string_list_append(&option_config, strbuf_detach(&sb, 0));
> +	}
>  
>  	if (!option_origin)
>  		option_origin = "origin";
> diff --git a/git-submodule.sh b/git-submodule.sh
> index 4092a48..e3d1667 100755
> --- a/git-submodule.sh
> +++ b/git-submodule.sh
> @@ -611,6 +611,7 @@ cmd_deinit()
>  #
>  cmd_update()
>  {
> +	groups=
>  	# parse $args after "submodule ... update".
>  	while test $# -ne 0
>  	do
> @@ -650,6 +651,9 @@ cmd_update()
>  		--checkout)
>  			update="checkout"
>  			;;
> +		--groups)
> +			groups=1
> +			;;
>  		--depth)
>  			case "$2" in '') usage ;; esac
>  			depth="--depth=$2"
> @@ -691,6 +695,7 @@ cmd_update()
>  		${update:+--update "$update"} \
>  		${reference:+--reference "$reference"} \
>  		${depth:+--depth "$depth"} \
> +		${groups:+--groups} \
>  		${jobs:+$jobs} \
>  		"$@" | {
>  	err=
> diff --git a/t/t7400-submodule-basic.sh b/t/t7400-submodule-basic.sh
> index caed4be..e8654d7 100755
> --- a/t/t7400-submodule-basic.sh
> +++ b/t/t7400-submodule-basic.sh
> @@ -1049,4 +1049,73 @@ test_expect_success 'submodule init --group works' '
>  	)
>  '
>  
> +cat <<EOF > expected
> +submodule
> +-submodule1
> +EOF
> +
> +test_expect_success 'submodule update --groups works' '
> +	test_when_finished "rm -rf super super_clone" &&
> +	mkdir super &&
> +	pwd=$(pwd) &&
> +	(
> +		cd super &&
> +		git init &&
> +		git submodule add --group groupA file://"$pwd"/example2 submodule &&
> +		git submodule add file://"$pwd"/example2 submodule1 &&
> +		git commit -a -m "create repository with 2 submodules, one is in a group"
> +	) &&
> +	git clone super super_clone &&
> +	(
> +		cd super_clone &&
> +		git config submodule.groups groupA &&
> +		git submodule init  &&
> +		git submodule update --groups &&
> +		git submodule status |cut -c1,42-52 | tr -d " " >../actual
> +	) &&
> +	test_cmp actual expected
> +'
> +
> +test_expect_success 'submodule update --init --groups works' '
> +	test_when_finished "rm -rf super super_clone" &&
> +	mkdir super &&
> +	pwd=$(pwd) &&
> +	(
> +		cd super &&
> +		git init &&
> +		git submodule add --group groupA file://"$pwd"/example2 submodule &&
> +		git submodule add file://"$pwd"/example2 submodule1 &&
> +		git commit -a -m "create repository with 2 submodules, one is in a group"
> +	) &&
> +	git clone super super_clone &&
> +	(
> +		cd super_clone &&
> +		git config submodule.groups groupA &&
> +		git submodule update --init --groups &&
> +		git submodule status |cut -c1,42-52 | tr -d " " >../actual
> +	) &&
> +	test_cmp actual expected
> +'
> +
> +test_expect_success 'clone --group works' '
> +	test_when_finished "rm -rf super super_clone" &&
> +	mkdir super &&
> +	pwd=$(pwd) &&
> +	(
> +		cd super &&
> +		git init &&
> +		git submodule add --group groupA file://"$pwd"/example2 submodule &&
> +		git submodule add file://"$pwd"/example2 submodule1 &&
> +		git commit -a -m "create repository with 2 submodules, one is in a group"
> +	) &&
> +	git clone --group groupA super super_clone &&
> +	(
> +		cd super_clone &&
> +		test_pause
> +		git submodule status |cut -c1,42-52 | tr -d " " >../actual
> +	) &&
> +	test_cmp actual expected
> +'
> +
> +
>  test_done
> diff --git a/t/t7406-submodule-update.sh b/t/t7406-submodule-update.sh
> index 090891e..7e59846 100755
> --- a/t/t7406-submodule-update.sh
> +++ b/t/t7406-submodule-update.sh
> @@ -801,4 +801,36 @@ test_expect_success 'git clone passes the parallel jobs config on to submodules'
>  	rm -rf super4
>  '
>  
> +cat >expect <<-EOF &&
> +-deeper/submodule
> +-merging
> +-moved/sub module
> +-none
> +-rebasing
> +-submodule
> +-submodule1
> +EOF
> +
> +# none, merging rebasing, submodule1, submodule
> +test_expect_success 'git clone works with submodule groups.' '
> +	test_when_finished "rm -rf super5" &&
> +	(
> +		cd super &&
> +		git config -f .gitmodules  submodule.submodule.groups default &&
> +		git config -f .gitmodules  submodule.submodule1.groups "default,testing" &&
> +		git config -f .gitmodules  submodule.none.groups testing &&
> +		git commit -a -m "assigning groups to submodules"
> +	) &&
> +	git clone --group default --group testing super super5 &&
> +	(
> +		cd super5 &&
> +		git submodule status |cut -c1,43- >../actual
> +	) &&
> +	test_cmp actual expect
> +'
> +
> +test_expect_success 'git submodule update --groups' '
> +	true
> +'
> +
>  test_done
> -- 
> 2.6.1.261.g0d9c4c1
> 
> --
> To unsubscribe from this list: send the line "unsubscribe git" in
> the body of a message to majordomo@vger.kernel.org
> More majordomo info at  http://vger.kernel.org/majordomo-info.html

> Seeing the recent sparse checkout discussion I realized it might be
> useful to have a similar sort of feature for sparse checkouts.  So say I
> had a mobile and desktop client in the same repo and wanted to be able to
> checkout the commoncode and one client without having to explicitly list
> all the paths I care about. It seems like UI wise you might want to use
> --group there too, or at least explaining the difference to users might
> be interesting, but maybe that's worrying way too much abouta possible
> future feature.

> + cc Duy, Michael, who discussed the sparse checkout recently
> 
> On Wed, Nov 25, 2015 at 9:00 PM, Trevor Saunders <tbsaunde@tbsaunde.org> wrote:
>> Seeing the recent sparse checkout discussion I realized it might be
>> useful to have a similar sort of feature for sparse checkouts.  So say I
>> had a mobile and desktop client in the same repo and wanted to be able to
>> checkout the commoncode and one client without having to explicitly list
>> all the paths I care about. It seems like UI wise you might want to use
>> --group there too, or at least explaining the difference to users might
>> be interesting, but maybe that's worrying way too much abouta possible
>> future feature.
> 
> For reference what I was proposing (coverletter [RFC PATCH 0/5]
> Submodule Groups):
> -->8--
> This is also available at
> https://github.com/stefanbeller/git/tree/submodule-groups
> It applies on top of the submodule-parallel-patch series I sent a few
> minutes ago.
> 
> Consider having a real large software project in Git with each component
> in a submodule (such as an operating system, Android, Debian, Fedora,
> no toy OS such as https://github.com/gittup/gittup as that doesn't quite
> demonstrate the scale of the problem).
> 
> If you have lots of submodules, you probably don't need all of them at once,
> but you have functional units. Some submodules are absolutely required,
> some are optional and only for very specific purposes.
> 
> This patch series adds meaning to a "groups" field in the .gitmodules file.
> 
> So you could have a .gitmodules file such as:
> 
> [submodule "gcc"]
>         path = gcc
>         url = git://...
>         groups = default,devel
> [submodule "linux"]
>         path = linux
>         url = git://...
>         groups = default
> [submodule "nethack"]
>         path = nethack
>         url = git://...
>         groups = optional,games
> 
> and by this series you can work on an arbitrary subgroup of these
> submodules such
> using these commands:
> 
>     git clone --group default --group devel git://...
>     # will clone the superproject and recursively
>     # checkout any submodule being in at least one of the groups.
> 
>     git submodule add --group default --group devel git://... ..
>     # will add a submodule, adding 2 submodule
>     # groups to its entry in .gitmodule
> 
>     # as support for clone we want to have:
>     git config submodule.groups default
>     git submodule init --groups
>     # will init all submodules from the default group
> 
>     # as support for clone we want to have:
>     git config submodule.groups default
>     git submodule update --groups
>     # will update all submodules from the default group
> 
> Any feedback welcome, specially on the design level!
> (Do we want to have it stored in the .gitmodules file? Do we want to have
> the groups configured in .git/config as "submodule.groups", any other way
> to make it future proof and extend the groups syntax?)
> -->8--
> 
> I think the biggest advantage with the groups is to have it not depending on the
> path. Consider your one repository containing both mobile and desktop code,
> where you have a sparse checkout for mobile.
> 
> Now what happens if files are renamed, i.e. the leading directory?
> As this change comes in from your dear coworker, who has no idea how
> your .git/info/sparse-checkout looks like, it may happen that more files appear
> in your worktree as your patterns did not cover the renamed case.
> Or some file contents go missing as they are in one of the ignored paths.
> 
> This groups feature would solve that as the groups are not dependent on
> the paths or the data itself. However the groups in this proposal are only
> meant to be applied on a submodule level, not in a repository itself.
> 
> How would you do that?
> I could imagine a file like .gitgroups (It should be part of the repository,
> such that everybody talks about the same groups), with a similar syntax like
> .gitattributes where some file patterns are assigned one or more groups.
> And in either .git/config (as it is for the submodules here) or in a file
> .git/info/groups (as the sparse checkouts do it in that directory)
> you'd configure the groups you are interested in.
> 
> It would be cool to have the same mechanism for both sparse-group-checkout
> and submodules, so I'd propose to use a config option in .git/config,
> such as checkout-group which covers both submodules as well as the
> pattern groups as specified by .gitgroups.
> 
> ---
> Beware, this is just a first shot spinning around some ideas.
> 
> Thanks,
> Stefan

> I think we have to solve more basic issues for sparse checkouts first.
> I'm using them with extra worktrees now and everything seems to be
> working fine. But we need to get the UI right for the simple case (no
> submodules, maybe not even extra worktrees) first: setting up patterns
> before checkout etc. Having submodules in mind doesn't hurt, tough.

>
> I still consider sparse checkouts a local "cludge" (not technically
> cludgy) in the sense that it helps you cater to some specific local
> needs; not something whose config you'd want to transport as part of the
> object store.

> This is also available at https://github.com/stefanbeller/git/tree/submodule-groups
> It applies on top of the submodule-parallel-patch series I sent a few minutes ago.
>
> Consider having a real large software project in Git with each component
> in a submodule (such as an operating system, Android, Debian, Fedora,
> no toy OS such as https://github.com/gittup/gittup as that doesn't quite
> demonstrate the scale of the problem).
>
> If you have lots of submodules, you probably don't need all of them at once,
> but you have functional units. Some submodules are absolutely required,
> some are optional and only for very specific purposes.
>
> This patch series adds meaning to a "groups" field in the .gitmodules file.
>
> So you could have a .gitmodules file such as:
>
> [submodule "gcc"]
>          path = gcc
>          url = git://...
>          groups = default,devel
> [submodule "linux"]
>          path = linux
>          url = git://...
>          groups = default
> [submodule "nethack"]
>          path = nethack
>          url = git://...
>          groups = optional,games

> and by this series you can work on an arbitrary subgroup of these submodules such
> using these commands:
>
>      git clone --group default --group devel git://...
>      # will clone the superproject and recursively
>      # checkout any submodule being in at least one of the groups.

>      # will init all submodules from the default group
>
>      # as support for clone we want to have:
>      git config submodule.groups default
>      git submodule update --groups
>
>      # will update all submodules from the default group
>
> Any feedback welcome, specially on the design level!
> (Do we want to have it stored in the .gitmodules file? Do we want to have
> the groups configured in .git/config as "submodule.groups", any other way
> to make it future proof and extend the groups syntax?)

>> [submodule "gcc"]
>>          path = gcc
>>          url = git://...
>>          groups = default,devel
>> [submodule "linux"]
>>          path = linux
>>          url = git://...
>>          groups = default
>> [submodule "nethack"]
>>          path = nethack
>>          url = git://...
>>          groups = optional,games
>
>
> Yup. Do you want the user to select only a single group or do you
> plan to support selecting multiple groups at the same time too?

>
>> and by this series you can work on an arbitrary subgroup of these
>> submodules such
>> using these commands:
>>
>>      git clone --group default --group devel git://...
>>      # will clone the superproject and recursively
>>      # checkout any submodule being in at least one of the groups.
>
>
> Does this automatically configure the given group in .git/config, so
> that all future submodule related commands know about this choice?
> Me thinks that would make sense ...

>
>>      git submodule add --group default --group devel git://... ..
>>      # will add a submodule, adding 2 submodule
>>      # groups to its entry in .gitmodule
>
>
> Maybe '--groups default,devel' is easier to grok? Dunno.

>
>>      # as support for clone we want to have:
>>      git config submodule.groups default
>>      git submodule init --groups
>
>
> Hmm, I doubt it makes much sense to add the --group option to "git
> submodule init". I'd rather init all submodules and do the group
> handling only in the "git submodule update" command. That way
> upstream can change grouping later without having the user to
> fiddle with her configuration to make that work.

>
>>      # will init all submodules from the default group
>>
>>      # as support for clone we want to have:
>>      git config submodule.groups default
>>      git submodule update --groups
>>
>>      # will update all submodules from the default group
>>
>> Any feedback welcome, specially on the design level!
>> (Do we want to have it stored in the .gitmodules file? Do we want to have
>> the groups configured in .git/config as "submodule.groups", any other way
>> to make it future proof and extend the groups syntax?)
>
>
> Not sure what exactly you mean by "it" here ;-)
>
> Talking about what groups a submodule belongs to, an entry in the
> .gitmodules file makes the most sense to me. That way upstream can
> change submodule grouping or add new submodules with group assignments
> from commit to commit, and "git submodule update" will do the right
> thing for the superproject commit checked out.
>
> And I believe that the choice which group(s?) the user is interested
> should be recorded in .git/config, as that is his personal setting
> that shouldn't be influenced by upstream changes.

> --cc Johannes Sixt
>
> On Wed, Nov 25, 2015 at 9:35 AM, Jens Lehmann <Jens.Lehmann@web.de> wrote:
>>> [submodule "gcc"]
>>>           path = gcc
>>>           url = git://...
>>>           groups = default,devel
>>> [submodule "linux"]
>>>           path = linux
>>>           url = git://...
>>>           groups = default
>>> [submodule "nethack"]
>>>           path = nethack
>>>           url = git://...
>>>           groups = optional,games
>>
>>
>> Yup. Do you want the user to select only a single group or do you
>> plan to support selecting multiple groups at the same time too?
>
> Yes you should be able to select multiple groups, such as
> default+devel or alternatively default+games.
>
> The logical OR is supported in this patch series (all submodules which are
> in at least one of the specified groups,i.e. A OR B OR C ...)

>>> and by this series you can work on an arbitrary subgroup of these
>>> submodules such
>>> using these commands:
>>>
>>>       git clone --group default --group devel git://...
>>>       # will clone the superproject and recursively
>>>       # checkout any submodule being in at least one of the groups.
>>
>>
>> Does this automatically configure the given group in .git/config, so
>> that all future submodule related commands know about this choice?
>> Me thinks that would make sense ...
>
> It does. Internally it does
>
>      git config submodule.groups A,B
>      git submodule update --init --groups
>
> whereas submodule update checks if the submodule.groups
> value is set and if so operates on the groups only.

>>
>>>       # as support for clone we want to have:
>>>       git config submodule.groups default
>>>       git submodule init --groups
>>
>>
>> Hmm, I doubt it makes much sense to add the --group option to "git
>> submodule init". I'd rather init all submodules and do the group
>> handling only in the "git submodule update" command. That way
>> upstream can change grouping later without having the user to
>> fiddle with her configuration to make that work.
>
> Well if upstream changes grouping later, you could just run
>
>      git submodule update --init --groups
>
> and get what you want?

>>>       # will init all submodules from the default group
>>>
>>>       # as support for clone we want to have:
>>>       git config submodule.groups default
>>>       git submodule update --groups
>>>
>>>       # will update all submodules from the default group
>>>
>>> Any feedback welcome, specially on the design level!
>>> (Do we want to have it stored in the .gitmodules file? Do we want to have
>>> the groups configured in .git/config as "submodule.groups", any other way
>>> to make it future proof and extend the groups syntax?)
>>
>>
>> Not sure what exactly you mean by "it" here ;-)
>>
>> Talking about what groups a submodule belongs to, an entry in the
>> .gitmodules file makes the most sense to me. That way upstream can
>> change submodule grouping or add new submodules with group assignments
>> from commit to commit, and "git submodule update" will do the right
>> thing for the superproject commit checked out.
>>
>> And I believe that the choice which group(s?) the user is interested
>> should be recorded in .git/config, as that is his personal setting
>> that shouldn't be influenced by upstream changes.
>
> Right. I once discussed with Jonathan Nieder, who dreamed of a more
> logical approach to the groups/sets of submodules. So more like set theory,
> i.e. have a more complicated grammar: Get all submodules which are
> in either A or B or (D AND E), but which are never in F.
> So I'd imagine the groups are more like bit tags, and you can describe
> a patterns you want.

>>>
>>> Hmm, I doubt it makes much sense to add the --group option to "git
>>> submodule init". I'd rather init all submodules and do the group
>>> handling only in the "git submodule update" command. That way
>>> upstream can change grouping later without having the user to
>>> fiddle with her configuration to make that work.
>>

>>
>> Well if upstream changes grouping later, you could just run
>>
>>      git submodule update --init --groups
>>
>> and get what you want?
>
>
> And make life harder than necessary for our users without having
> a reason for that?

> Except for the URL copying submodule settings
> on init is wrong, as it sets in stone what happened to be in the
> .gitmodules file when you ran init and doesn't allow upstream to
> easily change defaults later. We still do that with the update
> setting for historical reasons, but I avoided making the same
> mistake with all the options I added later. You can override
> these settings if you want or need to, but that shouldn't be
> necessary by default to make life easier for our users.
>

> On Wed, Nov 25, 2015 at 11:18 AM, Jens Lehmann <Jens.Lehmann@web.de> wrote:
>>>>
>>>> Hmm, I doubt it makes much sense to add the --group option to "git
>>>> submodule init". I'd rather init all submodules and do the group
>>>> handling only in the "git submodule update" command. That way
>>>> upstream can change grouping later without having the user to
>>>> fiddle with her configuration to make that work.
>>>
>
> Mind to elaborate a bit more here?
> The way I understand you now is to pass not --groups to init,
> but init initializes all submodules. But that is worse IMHO

> (In the naive way of dealing with groups in the first patch series)
> as then we open up two possibilities:
>   * a submodule which happened to be part of the repository
>     when cloning is added to a new group, which a user has
>     configured, on pulling, this is no problem, we just checkout
>     the desired version of the submodule.

> So when a change of the set of submodules as defined by groups
> occurs, that is the point in time, when we want to init/fetch/checkout
> these submodules, no?
>
>>>
>>> Well if upstream changes grouping later, you could just run
>>>
>>>       git submodule update --init --groups
>>>
>>> and get what you want?
>>
>>
>> And make life harder than necessary for our users without having
>> a reason for that?
>
> So if upstream changes groups, ideally we want to follow without much
> hassle for the user. So a plain git pull should /just work/. (I am repeating
> myself here I'd guess), we would need to react to that. if we drop the
> --groups call to init, we'd still tell the user to run
>
>       git submodule update

>> Except for the URL copying submodule settings
>> on init is wrong, as it sets in stone what happened to be in the
>> .gitmodules file when you ran init and doesn't allow upstream to
>> easily change defaults later. We still do that with the update
>> setting for historical reasons, but I avoided making the same
>> mistake with all the options I added later. You can override
>> these settings if you want or need to, but that shouldn't be
>> necessary by default to make life easier for our users.
>>
>

> This is also available at https://github.com/stefanbeller/git/tree/submodule-groups
> It applies on top of the submodule-parallel-patch series I sent a few minutes ago.
>
> Consider having a real large software project in Git with each component
> in a submodule (such as an operating system, Android, Debian, Fedora,
> no toy OS such as https://github.com/gittup/gittup as that doesn't quite
> demonstrate the scale of the problem).
>
> If you have lots of submodules, you probably don't need all of them at once,
> but you have functional units. Some submodules are absolutely required,
> some are optional and only for very specific purposes.
>
> This patch series adds meaning to a "groups" field in the .gitmodules file.
>
> So you could have a .gitmodules file such as:
>
> [submodule "gcc"]
>          path = gcc
>          url = git://...
>          groups = default,devel
> [submodule "linux"]
>          path = linux
>          url = git://...
>          groups = default
> [submodule "nethack"]
>          path = nethack
>          url = git://...
>          groups = optional,games

> and by this series you can work on an arbitrary subgroup of these submodules such
> using these commands:
>
>      git clone --group default --group devel git://...
>      # will clone the superproject and recursively
>      # checkout any submodule being in at least one of the groups.

>      # will init all submodules from the default group
>
>      # as support for clone we want to have:
>      git config submodule.groups default
>      git submodule update --groups
>
>      # will update all submodules from the default group
>
> Any feedback welcome, specially on the design level!
> (Do we want to have it stored in the .gitmodules file? Do we want to have
> the groups configured in .git/config as "submodule.groups", any other way
> to make it future proof and extend the groups syntax?)