threads / patch / 32695

patchall: new command used for multi-repo operations

Subject: [PATCH] all: new command used for multi-repo operations

## tl;dr

11 messages between Jan 22, 2013 and Jan 23, 2013. Diffs are folded; open one to read it.

replies: 10people: 4as markdown or json

Lars Hjemli· Jan 22, 2013, 21:10 UTC · lore

When working with multiple, unrelated (or loosly related) git repos, there is often a need to locate all repos with uncommitted work and perform some action on them (say, commit and push). Before this patch, such tasks would require manually visiting all repositories, running `git status` within each one and then decide what to do next.

This mundane task can now be automated by e.g. `git all --dirty status`, which will find all git repositories below the current directory (even nested ones), check if they are dirty (as defined by `git diff --quiet && git diff --cached --quiet`), and for each dirty repo print the path to the repo and then execute `git status` within the repo.

The command also honours the option '--clean' which restricts the set of repos to those which '--dirty' would skip.

Finally, the command to execute within each repo is optional. If none is given, git-all will just print the path to each repo found.

Signed-off-by: Lars Hjemli <hjemli@gmail.com>
---
 Documentation/git-all.txt |  37 ++++++++++++++++
 Makefile                  |   1 +
 builtin.h                 |   1 +
 builtin/all.c             | 105 ++++++++++++++++++++++++++++++++++++++++++++++
 command-list.txt          |   1 +
 git.c                     |   1 +
 t/t0064-all.sh            |  42 +++++++++++++++++++
 7 files changed, 188 insertions(+)
 create mode 100644 Documentation/git-all.txt
 create mode 100644 builtin/all.c
 create mode 100755 t/t0064-all.sh
Show changes to 7 files +188 −0

Documentation/git-all.txt, Makefile, builtin.h, builtin/all.c, command-list.txt, git.c, t/t0064-all.sh

diff --git a/Documentation/git-all.txt b/Documentation/git-all.txt
new file mode 100644
index 0000000..b25f23c
--- /dev/null
+++ b/Documentation/git-all.txt
@@ -0,0 +1,37 @@
+git-all(1)
+==========
+
+NAME
+----
+git-all - Execute a git command in multiple repositories
+
+SYNOPSIS
+--------
+[verse]
+'git all' [--dirty|--clean] [command]
+
+DESCRIPTION
+-----------
+The git-all command is used to locate all git repositoris within the
+current directory tree, and optionally execute a git command in each
+of the found repos.
+
+OPTIONS
+-------
+-c::
+--clean::
+	Only include repositories with a clean worktree.
+
+-d::
+--dirty::
+	Only include repositories with a dirty worktree.
+
+NOTES
+-----
+
+For the purpose of `git-all`, a dirty worktree is defined as a worktree
+with uncommitted changes.
+
+GIT
+---
+Part of the linkgit:git[1] suite
diff --git a/Makefile b/Makefile
index 1b30d7b..8bf0583 100644
--- a/Makefile
+++ b/Makefile
@@ -840,6 +840,7 @@ LIB_OBJS += xdiff-interface.o
 LIB_OBJS += zlib.o
 
 BUILTIN_OBJS += builtin/add.o
+BUILTIN_OBJS += builtin/all.o
 BUILTIN_OBJS += builtin/annotate.o
 BUILTIN_OBJS += builtin/apply.o
 BUILTIN_OBJS += builtin/archive.o
diff --git a/builtin.h b/builtin.h
index 7e7bbd6..438c265 100644
--- a/builtin.h
+++ b/builtin.h
@@ -41,6 +41,7 @@ void finish_copy_notes_for_rewrite(struct notes_rewrite_cfg *c);
 extern int textconv_object(const char *path, unsigned mode, const unsigned char *sha1, int sha1_valid, char **buf, unsigned long *buf_size);
 
 extern int cmd_add(int argc, const char **argv, const char *prefix);
+extern int cmd_all(int argc, const char **argv, const char *prefix);
 extern int cmd_annotate(int argc, const char **argv, const char *prefix);
 extern int cmd_apply(int argc, const char **argv, const char *prefix);
 extern int cmd_archive(int argc, const char **argv, const char *prefix);
diff --git a/builtin/all.c b/builtin/all.c
new file mode 100644
index 0000000..ee9270d
--- /dev/null
+++ b/builtin/all.c
@@ -0,0 +1,105 @@
+/*
+ * "git all" builtin command.
+ *
+ * Copyright (c) 2013 Lars Hjemli <hjemli@gmail.com>
+ */
+#include "cache.h"
+#include "color.h"
+#include "builtin.h"
+#include "run-command.h"
+#include "parse-options.h"
+
+static int only_dirty;
+static int only_clean;
+char root[PATH_MAX];
+
+static const char * const builtin_all_usage[] = {
+	N_("git all [options] [cmd]"),
+	NULL
+};
+
+static struct option builtin_all_options[] = {
+	OPT_BOOLEAN('c', "clean", &only_clean, N_("only show clean repositories")),
+	OPT_BOOLEAN('d', "dirty", &only_dirty, N_("only show dirty repositories")),
+	OPT_END(),
+};
+
+static int is_dirty()
+{
+	const char *diffidx[] = {"diff", "--quiet", "--cached", NULL};
+	const char *diffwd[] = {"diff", "--quiet", NULL};
+
+	if (run_command_v_opt(diffidx, RUN_GIT_CMD) != 0)
+		return 1;
+	if (run_command_v_opt(diffwd, RUN_GIT_CMD) != 0)
+		return 1;
+	return 0;
+}
+
+static void handle_repo(char *path, const char **argv)
+{
+	int dirty;
+
+	if (path[0] == '.' && path[1] == '/')
+		path += 2;
+	if (only_dirty || only_clean) {
+		dirty = is_dirty();
+		if ((dirty && only_clean) ||
+		     (!dirty && only_dirty))
+			return;
+	}
+	if (*argv) {
+		color_fprintf_ln(stdout, GIT_COLOR_YELLOW, "[%s]", path);
+		run_command_v_opt(argv, RUN_GIT_CMD);
+	} else
+		printf("%s\n", path);
+}
+
+static int walk(struct strbuf *path, int argc, const char **argv)
+{
+	DIR *dir;
+	struct dirent *ent;
+	size_t len;
+
+	dir = opendir(path->buf);
+	if (!dir)
+		return errno;
+	strbuf_addstr(path, "/");
+	len = path->len;
+	while ((ent = readdir(dir))) {
+		if (!strcmp(ent->d_name, ".") || !strcmp(ent->d_name, ".."))
+			continue;
+		if (!strcmp(ent->d_name, ".git")) {
+			strbuf_setlen(path, len - 1);
+			chdir(path->buf);
+			handle_repo(path->buf, argv);
+			chdir(root);
+			strbuf_addstr(path, "/");
+			continue;
+		}
+		if (ent->d_type != DT_DIR)
+			continue;
+		strbuf_setlen(path, len);
+		strbuf_addstr(path, ent->d_name);
+		walk(path, argc, argv);
+	}
+	closedir(dir);
+	return 0;
+}
+
+int cmd_all(int argc, const char **argv, const char *prefix)
+{
+	struct strbuf path = STRBUF_INIT;
+
+	if (!getcwd(root, sizeof(root)))
+		return 1;
+
+	argc = parse_options(argc, argv, prefix, builtin_all_options,
+			     builtin_all_usage, PARSE_OPT_STOP_AT_NON_OPTION);
+
+	unsetenv(GIT_DIR_ENVIRONMENT);
+	unsetenv(GIT_WORK_TREE_ENVIRONMENT);
+
+	strbuf_addstr(&path, ".");
+	return walk(&path, argc, argv);
+}
diff --git a/command-list.txt b/command-list.txt
index 7e8cfec..f955895 100644
--- a/command-list.txt
+++ b/command-list.txt
@@ -1,6 +1,7 @@
 # List of known git commands.
 # command name				category [deprecated] [common]
 git-add                                 mainporcelain common
+git-all                                 mainporcelain
 git-am                                  mainporcelain
 git-annotate                            ancillaryinterrogators
 git-apply                               plumbingmanipulators
diff --git a/git.c b/git.c
index ed66c66..53fd963 100644
--- a/git.c
+++ b/git.c
@@ -304,6 +304,7 @@ static void handle_internal_command(int argc, const char **argv)
 	const char *cmd = argv[0];
 	static struct cmd_struct commands[] = {
 		{ "add", cmd_add, RUN_SETUP | NEED_WORK_TREE },
+		{ "all", cmd_all },
 		{ "annotate", cmd_annotate, RUN_SETUP },
 		{ "apply", cmd_apply, RUN_SETUP_GENTLY },
 		{ "archive", cmd_archive },
diff --git a/t/t0064-all.sh b/t/t0064-all.sh
new file mode 100755
index 0000000..932e374
--- /dev/null
+++ b/t/t0064-all.sh
@@ -0,0 +1,42 @@
+#!/bin/sh
+#
+# Copyright (c) 2013 Lars Hjemli
+#
+
+test_description='Test the git-all command'
+
+. ./test-lib.sh
+
+test_expect_success "setup" '
+	test_create_repo clean &&
+	(cd clean && test_commit foo) &&
+	test_create_repo dirty-wt &&
+	(cd dirty-wt && test_commit foo && rm foo.t) &&
+	test_create_repo dirty-idx &&
+	(cd dirty-idx && test_commit foo && git rm foo.t)
+'
+
+test_expect_success "without flags, all repos are included" '
+	echo "." >expect &&
+	echo "clean" >>expect &&
+	echo "dirty-idx" >>expect &&
+	echo "dirty-wt" >>expect &&
+	git all | sort >actual &&
+	test_cmp expect actual
+'
+
+test_expect_success "--dirty only includes dirty repos" '
+	echo "dirty-idx" >expect &&
+	echo "dirty-wt" >>expect &&
+	git all --dirty | sort >actual &&
+	test_cmp expect actual
+'
+
+test_expect_success "--clean only includes clean repos" '
+	echo "." >expect &&
+	echo "clean" >>expect &&
+	git all --clean | sort >actual &&
+	test_cmp expect actual
+'
+
+test_done
-- 
1.8.1.1.296.g725455c
Junio C Hamano· Jan 22, 2013, 22:01 UTC · re: Lars Hjemli · lore

Re: [PATCH] all: new command used for multi-repo operations

Lars Hjemli <hjemli@gmail.com> writes:
Show 31 quoted lines
> diff --git a/builtin/all.c b/builtin/all.c
> new file mode 100644
> index 0000000..ee9270d
> --- /dev/null
> +++ b/builtin/all.c
> @@ -0,0 +1,105 @@
> +/*
> + * "git all" builtin command.
> + *
> + * Copyright (c) 2013 Lars Hjemli <hjemli@gmail.com>
> + */
> +#include "cache.h"
> +#include "color.h"
> +#include "builtin.h"
> +#include "run-command.h"
> +#include "parse-options.h"
> +
> +static int only_dirty;
> +static int only_clean;
> +char root[PATH_MAX];
> +
> +static const char * const builtin_all_usage[] = {
> +	N_("git all [options] [cmd]"),
> +	NULL
> +};
> +
> +static struct option builtin_all_options[] = {
> +	OPT_BOOLEAN('c', "clean", &only_clean, N_("only show clean repositories")),
> +	OPT_BOOLEAN('d', "dirty", &only_dirty, N_("only show dirty repositories")),
> +	OPT_END(),
> +};

Shouldn't this be more like OPT_SET_INT() on a same variable that is initialized to "all"? Alternatively you could validate the input and die when both are given.

Show 12 quoted lines
> +int cmd_all(int argc, const char **argv, const char *prefix)
> +{
> +	struct strbuf path = STRBUF_INIT;
> +
> +	if (!getcwd(root, sizeof(root)))
> +		return 1;
> +
> +	argc = parse_options(argc, argv, prefix, builtin_all_options,
> +			     builtin_all_usage, PARSE_OPT_STOP_AT_NON_OPTION);
> +
> +	unsetenv(GIT_DIR_ENVIRONMENT);
> +	unsetenv(GIT_WORK_TREE_ENVIRONMENT);

Don't you need to clear other variables whose uses are closely tied to a single repository, like GIT_INDEX_FILE, etc.?

I suspect that explicitly exporting GIT_DIR and GIT_WORK_TREE (and nothing else) in handle_repo() to the location you discovered before you run the per-repository command via run_command_v_opt(), might be a better alternative. The user could be sharing objects in all repositories by permanently setting GIT_OBJECT_DIRECTORY to a single place.

Show 12 quoted lines
> diff --git a/command-list.txt b/command-list.txt
> index 7e8cfec..f955895 100644
> --- a/command-list.txt
> +++ b/command-list.txt
> @@ -1,6 +1,7 @@
>  # List of known git commands.
>  # command name				category [deprecated] [common]
>  git-add                                 mainporcelain common
> +git-all                                 mainporcelain
>  git-am                                  mainporcelain
>  git-annotate                            ancillaryinterrogators
>  git-apply                               plumbingmanipulators

I am not very interested in this topic in the first place, but this does not (at least not yet) sound like a main Porcelain to me.

"all" may be a word other people may want to use to call collections of things other than "Git repositories", and that use may turn out to be more useful in general. A name that makes it clear that this is about "repositories", i.e. along the lines of "git for-each-repo" or something, would be a better name that does not squat on such a short and sweet name.

Lars Hjemli· Jan 22, 2013, 23:35 UTC · re: Junio C Hamano · lore

Re: [PATCH] all: new command used for multi-repo operations

On Tue, Jan 22, 2013 at 11:01 PM, Junio C Hamano <gitster@pobox.com> wrote:
Show 11 quoted lines
> Lars Hjemli <hjemli@gmail.com> writes:
>
>> +static struct option builtin_all_options[] = {
>> +     OPT_BOOLEAN('c', "clean", &only_clean, N_("only show clean repositories")),
>> +     OPT_BOOLEAN('d', "dirty", &only_dirty, N_("only show dirty repositories")),
>> +     OPT_END(),
>> +};
>
> Shouldn't this be more like OPT_SET_INT() on a same variable that is
> initialized to "all"?  Alternatively you could validate the input
> and die when both are given.
OPT_SET_INT() seems appropriate, will fix.
Show 24 quoted lines
>
>> +int cmd_all(int argc, const char **argv, const char *prefix)
>> +{
>> +     struct strbuf path = STRBUF_INIT;
>> +
>> +     if (!getcwd(root, sizeof(root)))
>> +             return 1;
>> +
>> +     argc = parse_options(argc, argv, prefix, builtin_all_options,
>> +                          builtin_all_usage, PARSE_OPT_STOP_AT_NON_OPTION);
>> +
>> +     unsetenv(GIT_DIR_ENVIRONMENT);
>> +     unsetenv(GIT_WORK_TREE_ENVIRONMENT);
>
> Don't you need to clear other variables whose uses are closely tied
> to a single repository, like GIT_INDEX_FILE, etc.?
>
> I suspect that explicitly exporting GIT_DIR and GIT_WORK_TREE (and
> nothing else) in handle_repo() to the location you discovered before
> you run the per-repository command via run_command_v_opt(), might be
> a better alternative.  The user could be sharing objects in all
> repositories by permanently setting GIT_OBJECT_DIRECTORY to a single
> place.
>
This sounds like a nice plan, I'll test it and send an updated patch.
Show 15 quoted lines
>> diff --git a/command-list.txt b/command-list.txt
>> index 7e8cfec..f955895 100644
>> --- a/command-list.txt
>> +++ b/command-list.txt
>> @@ -1,6 +1,7 @@
>>  # List of known git commands.
>>  # command name                               category [deprecated] [common]
>>  git-add                                 mainporcelain common
>> +git-all                                 mainporcelain
>>  git-am                                  mainporcelain
>>  git-annotate                            ancillaryinterrogators
>>  git-apply                               plumbingmanipulators
>
> I am not very interested in this topic in the first place, but this
> does not (at least not yet) sound like a main Porcelain to me.
There doesn't seem to be a better category, but I'm open for suggestions.
Show 8 quoted lines
>
> "all" may be a word other people may want to use to call collections
> of things other than "Git repositories", and that use may turn out
> to be more useful in general.  A name that makes it clear that this
> is about "repositories", i.e. along the lines of "git for-each-repo"
> or something, would be a better name that does not squat on such a
> short and sweet name.
>

In principle I agree with your reasoning on this, but in practice I fail to see what other kind of things `git all` could naturally refer to. Also, having a short and sweet way to perform the tasks implemented by this patch was my main motivation for writing it [1], hence `git for-each-repo` isn't as compelling (too much typing). There's always aliases, but I'd prefer it if future git supported `git all` by default.

-- 
larsh

[1] Originally as a shell script, used at $WORK for ~2 years
Junio C Hamano· Jan 23, 2013, 00:13 UTC · re: Lars Hjemli · lore

Re: [PATCH] all: new command used for multi-repo operations

Lars Hjemli <hjemli@gmail.com> writes:
> In principle I agree with your reasoning on this, but in practice I
> fail to see what other kind of things `git all` could naturally refer
> to.

For example, the designers of 'for-each-ref' could have called it "git all", as it is to iterate over all refs.

David Aguilar· Jan 23, 2013, 00:43 UTC · re: Lars Hjemli · lore

Re: [PATCH] all: new command used for multi-repo operations

On Tue, Jan 22, 2013 at 1:10 PM, Lars Hjemli <hjemli@gmail.com> wrote:
Show 184 quoted lines
> When working with multiple, unrelated (or loosly related) git repos,
> there is often a need to locate all repos with uncommitted work and
> perform some action on them (say, commit and push). Before this patch,
> such tasks would require manually visiting all repositories, running
> `git status` within each one and then decide what to do next.
>
> This mundane task can now be automated by e.g. `git all --dirty status`,
> which will find all git repositories below the current directory (even
> nested ones), check if they are dirty (as defined by `git diff --quiet &&
> git diff --cached --quiet`), and for each dirty repo print the path to the
> repo and then execute `git status` within the repo.
>
> The command also honours the option '--clean' which restricts the set of
> repos to those which '--dirty' would skip.
>
> Finally, the command to execute within each repo is optional. If none is
> given, git-all will just print the path to each repo found.
>
> Signed-off-by: Lars Hjemli <hjemli@gmail.com>
> ---
>  Documentation/git-all.txt |  37 ++++++++++++++++
>  Makefile                  |   1 +
>  builtin.h                 |   1 +
>  builtin/all.c             | 105 ++++++++++++++++++++++++++++++++++++++++++++++
>  command-list.txt          |   1 +
>  git.c                     |   1 +
>  t/t0064-all.sh            |  42 +++++++++++++++++++
>  7 files changed, 188 insertions(+)
>  create mode 100644 Documentation/git-all.txt
>  create mode 100644 builtin/all.c
>  create mode 100755 t/t0064-all.sh
>
> diff --git a/Documentation/git-all.txt b/Documentation/git-all.txt
> new file mode 100644
> index 0000000..b25f23c
> --- /dev/null
> +++ b/Documentation/git-all.txt
> @@ -0,0 +1,37 @@
> +git-all(1)
> +==========
> +
> +NAME
> +----
> +git-all - Execute a git command in multiple repositories
> +
> +SYNOPSIS
> +--------
> +[verse]
> +'git all' [--dirty|--clean] [command]
> +
> +DESCRIPTION
> +-----------
> +The git-all command is used to locate all git repositoris within the
> +current directory tree, and optionally execute a git command in each
> +of the found repos.
> +
> +OPTIONS
> +-------
> +-c::
> +--clean::
> +       Only include repositories with a clean worktree.
> +
> +-d::
> +--dirty::
> +       Only include repositories with a dirty worktree.
> +
> +NOTES
> +-----
> +
> +For the purpose of `git-all`, a dirty worktree is defined as a worktree
> +with uncommitted changes.
> +
> +GIT
> +---
> +Part of the linkgit:git[1] suite
> diff --git a/Makefile b/Makefile
> index 1b30d7b..8bf0583 100644
> --- a/Makefile
> +++ b/Makefile
> @@ -840,6 +840,7 @@ LIB_OBJS += xdiff-interface.o
>  LIB_OBJS += zlib.o
>
>  BUILTIN_OBJS += builtin/add.o
> +BUILTIN_OBJS += builtin/all.o
>  BUILTIN_OBJS += builtin/annotate.o
>  BUILTIN_OBJS += builtin/apply.o
>  BUILTIN_OBJS += builtin/archive.o
> diff --git a/builtin.h b/builtin.h
> index 7e7bbd6..438c265 100644
> --- a/builtin.h
> +++ b/builtin.h
> @@ -41,6 +41,7 @@ void finish_copy_notes_for_rewrite(struct notes_rewrite_cfg *c);
>  extern int textconv_object(const char *path, unsigned mode, const unsigned char *sha1, int sha1_valid, char **buf, unsigned long *buf_size);
>
>  extern int cmd_add(int argc, const char **argv, const char *prefix);
> +extern int cmd_all(int argc, const char **argv, const char *prefix);
>  extern int cmd_annotate(int argc, const char **argv, const char *prefix);
>  extern int cmd_apply(int argc, const char **argv, const char *prefix);
>  extern int cmd_archive(int argc, const char **argv, const char *prefix);
> diff --git a/builtin/all.c b/builtin/all.c
> new file mode 100644
> index 0000000..ee9270d
> --- /dev/null
> +++ b/builtin/all.c
> @@ -0,0 +1,105 @@
> +/*
> + * "git all" builtin command.
> + *
> + * Copyright (c) 2013 Lars Hjemli <hjemli@gmail.com>
> + */
> +#include "cache.h"
> +#include "color.h"
> +#include "builtin.h"
> +#include "run-command.h"
> +#include "parse-options.h"
> +
> +static int only_dirty;
> +static int only_clean;
> +char root[PATH_MAX];
> +
> +static const char * const builtin_all_usage[] = {
> +       N_("git all [options] [cmd]"),
> +       NULL
> +};
> +
> +static struct option builtin_all_options[] = {
> +       OPT_BOOLEAN('c', "clean", &only_clean, N_("only show clean repositories")),
> +       OPT_BOOLEAN('d', "dirty", &only_dirty, N_("only show dirty repositories")),
> +       OPT_END(),
> +};
> +
> +static int is_dirty()
> +{
> +       const char *diffidx[] = {"diff", "--quiet", "--cached", NULL};
> +       const char *diffwd[] = {"diff", "--quiet", NULL};
> +
> +       if (run_command_v_opt(diffidx, RUN_GIT_CMD) != 0)
> +               return 1;
> +       if (run_command_v_opt(diffwd, RUN_GIT_CMD) != 0)
> +               return 1;
> +       return 0;
> +}
> +
> +static void handle_repo(char *path, const char **argv)
> +{
> +       int dirty;
> +
> +       if (path[0] == '.' && path[1] == '/')
> +               path += 2;
> +       if (only_dirty || only_clean) {
> +               dirty = is_dirty();
> +               if ((dirty && only_clean) ||
> +                    (!dirty && only_dirty))
> +                       return;
> +       }
> +       if (*argv) {
> +               color_fprintf_ln(stdout, GIT_COLOR_YELLOW, "[%s]", path);
> +               run_command_v_opt(argv, RUN_GIT_CMD);
> +       } else
> +               printf("%s\n", path);
> +}
> +
> +static int walk(struct strbuf *path, int argc, const char **argv)
> +{
> +       DIR *dir;
> +       struct dirent *ent;
> +       size_t len;
> +
> +       dir = opendir(path->buf);
> +       if (!dir)
> +               return errno;
> +       strbuf_addstr(path, "/");
> +       len = path->len;
> +       while ((ent = readdir(dir))) {
> +               if (!strcmp(ent->d_name, ".") || !strcmp(ent->d_name, ".."))
> +                       continue;
> +               if (!strcmp(ent->d_name, ".git")) {
> +                       strbuf_setlen(path, len - 1);
> +                       chdir(path->buf);
> +                       handle_repo(path->buf, argv);
> +                       chdir(root);
> +                       strbuf_addstr(path, "/");
> +                       continue;
> +               }

Does this section above properly handle .git files (where .git is a file, not a directory)? I wonder whether the check should be tighter, e.g. something closer to what's done in setup.c:is_git_repository().

The name of this function leads me to believe that this walks everything below the current directory looking for Git repos. How deep does it walk? Does this handle nested repositories, e.g. foo/bar/ and foo/bar/baz/ when you are inside foo/ ? After re-reading the documentation I am led to believe that it only walks one level deep.

I did not notice a test for nested repos which is what sparked my curiosity. ;-) If we do not expect to handle them in the first version then we should have a test to ensure the expected behavior. It would also be nice to see a test with a .git file.

I do wonder what the end user experience is with this command when used alongside other Git aggregate commands such as "repo" or "git submodule". This command is basically "git submodule foreach ..." without needing to buy into the whole submodule thing. This is an argument for naming it something like "git foreach-repo" since it would be named more closely to the "foreach" submodule command.

While "all" is less to type than "foreach-repo", most of the extra work can be eliminated by installing the wonderful git completion scripts for bash/zsh.

All that said, I have very real use cases for this command. Thanks for writing it.

Show 105 quoted lines
> +               if (ent->d_type != DT_DIR)
> +                       continue;
> +               strbuf_setlen(path, len);
> +               strbuf_addstr(path, ent->d_name);
> +               walk(path, argc, argv);
> +       }
> +       closedir(dir);
> +       return 0;
> +}
> +
> +int cmd_all(int argc, const char **argv, const char *prefix)
> +{
> +       struct strbuf path = STRBUF_INIT;
> +
> +       if (!getcwd(root, sizeof(root)))
> +               return 1;
> +
> +       argc = parse_options(argc, argv, prefix, builtin_all_options,
> +                            builtin_all_usage, PARSE_OPT_STOP_AT_NON_OPTION);
> +
> +       unsetenv(GIT_DIR_ENVIRONMENT);
> +       unsetenv(GIT_WORK_TREE_ENVIRONMENT);
> +
> +       strbuf_addstr(&path, ".");
> +       return walk(&path, argc, argv);
> +}
> diff --git a/command-list.txt b/command-list.txt
> index 7e8cfec..f955895 100644
> --- a/command-list.txt
> +++ b/command-list.txt
> @@ -1,6 +1,7 @@
>  # List of known git commands.
>  # command name                         category [deprecated] [common]
>  git-add                                 mainporcelain common
> +git-all                                 mainporcelain
>  git-am                                  mainporcelain
>  git-annotate                            ancillaryinterrogators
>  git-apply                               plumbingmanipulators
> diff --git a/git.c b/git.c
> index ed66c66..53fd963 100644
> --- a/git.c
> +++ b/git.c
> @@ -304,6 +304,7 @@ static void handle_internal_command(int argc, const char **argv)
>         const char *cmd = argv[0];
>         static struct cmd_struct commands[] = {
>                 { "add", cmd_add, RUN_SETUP | NEED_WORK_TREE },
> +               { "all", cmd_all },
>                 { "annotate", cmd_annotate, RUN_SETUP },
>                 { "apply", cmd_apply, RUN_SETUP_GENTLY },
>                 { "archive", cmd_archive },
> diff --git a/t/t0064-all.sh b/t/t0064-all.sh
> new file mode 100755
> index 0000000..932e374
> --- /dev/null
> +++ b/t/t0064-all.sh
> @@ -0,0 +1,42 @@
> +#!/bin/sh
> +#
> +# Copyright (c) 2013 Lars Hjemli
> +#
> +
> +test_description='Test the git-all command'
> +
> +. ./test-lib.sh
> +
> +test_expect_success "setup" '
> +       test_create_repo clean &&
> +       (cd clean && test_commit foo) &&
> +       test_create_repo dirty-wt &&
> +       (cd dirty-wt && test_commit foo && rm foo.t) &&
> +       test_create_repo dirty-idx &&
> +       (cd dirty-idx && test_commit foo && git rm foo.t)
> +'
> +
> +test_expect_success "without flags, all repos are included" '
> +       echo "." >expect &&
> +       echo "clean" >>expect &&
> +       echo "dirty-idx" >>expect &&
> +       echo "dirty-wt" >>expect &&
> +       git all | sort >actual &&
> +       test_cmp expect actual
> +'
> +
> +test_expect_success "--dirty only includes dirty repos" '
> +       echo "dirty-idx" >expect &&
> +       echo "dirty-wt" >>expect &&
> +       git all --dirty | sort >actual &&
> +       test_cmp expect actual
> +'
> +
> +test_expect_success "--clean only includes clean repos" '
> +       echo "." >expect &&
> +       echo "clean" >>expect &&
> +       git all --clean | sort >actual &&
> +       test_cmp expect actual
> +'
> +
> +test_done
> --
> 1.8.1.1.296.g725455c
>
> --
> To unsubscribe from this list: send the line "unsubscribe git" in
> the body of a message to majordomo@vger.kernel.org
> More majordomo info at  http://vger.kernel.org/majordomo-info.html
-- 
David
Junio C Hamano· Jan 23, 2013, 06:44 UTC · re: David Aguilar · lore

Re: [PATCH] all: new command used for multi-repo operations

David Aguilar <davvid@gmail.com> writes:
Show 25 quoted lines
>> +static int walk(struct strbuf *path, int argc, const char **argv)
>> +{
>> +       DIR *dir;
>> +       struct dirent *ent;
>> +       size_t len;
>> +
>> +       dir = opendir(path->buf);
>> +       if (!dir)
>> +               return errno;
>> +       strbuf_addstr(path, "/");
>> +       len = path->len;
>> +       while ((ent = readdir(dir))) {
>> +               if (!strcmp(ent->d_name, ".") || !strcmp(ent->d_name, ".."))
>> +                       continue;
>> +               if (!strcmp(ent->d_name, ".git")) {
>> +                       strbuf_setlen(path, len - 1);
>> +                       chdir(path->buf);
>> +                       handle_repo(path->buf, argv);
>> +                       chdir(root);
>> +                       strbuf_addstr(path, "/");
>> +                       continue;
>> +               }
>
> Does this section above properly handle .git files (where .git is a
> file, not a directory)?

This scans a directory $D to ask "is there '.git' in you?" and if the answer is "yes", then hands $D (not "$D/.git") to handle_repo(). That logic will not miss a gitfile that points at the real $GIT_DIR elsewhere.

There is a recursive call to walk() later in the same loop when the found entry ent turns out to be a directory, and "$D/" + ent->d_name is given to this function.

But I do not think the loop structure of this function is right. If $D has ".git" in it, should it even try to feed other subdirectories of $D (say "$D/a") to itself in recursion to see if $D/a/.git exists?

I think it should be more like
	walk(struct strbuf *path)
        {
		size_t dirlen = path->len;
		int has_git;
		strbuf_addstr(path, "/.git");
		has_git = !lstat(path->buf);
		strbuf_setlen(path, dirlen);
                if (has_git) {
                        handle_repo(path->buf);
			return;
		}
                dir = opendir(path->buf);
		while ((ent = readdir(dir))) {
			... skip . and .. ...
			strbuf_addstr(path, ent->d_name);
			walk(path);
			strbuf_setlen(path, dirlen);
                }
	}

The determination of has_git can be a bit fancier than a simple !lstat() as you mentioned.

Lars Hjemli· Jan 23, 2013, 07:39 UTC · re: Junio C Hamano · lore

Re: [PATCH] all: new command used for multi-repo operations

On Wed, Jan 23, 2013 at 7:44 AM, Junio C Hamano <gitster@pobox.com> wrote:
> But I do not think the loop structure of this function is right.  If
> $D has ".git" in it, should it even try to feed other subdirectories
> of $D (say "$D/a") to itself in recursion to see if $D/a/.git exists?

Yes, this is needed to meet one of the goals for git-all: to be an alternative to git-submodule when submodules doesn't fit (the other goal is to manage a collection of unrelated (not nested) repos).

-- 
larsh
Junio C Hamano· Jan 23, 2013, 06:52 UTC · re: Lars Hjemli · lore

Re: [PATCH] all: new command used for multi-repo operations

Lars Hjemli <hjemli@gmail.com> writes:
Show 5 quoted lines
> +static struct option builtin_all_options[] = {
> +	OPT_BOOLEAN('c', "clean", &only_clean, N_("only show clean repositories")),
> +	OPT_BOOLEAN('d', "dirty", &only_dirty, N_("only show dirty repositories")),
> +	OPT_END(),
> +};

If you were to go in the OPT_SET_INT route, that would give users the usual "last one wins" semantics, e.g.

	$ git for-each-repo --clean --dirty

will look for only dirty repositories. For completeness, we would probably want "all" to defeat either of them, i.e.

	$ git for-each-repo --clean --all
Show 15 quoted lines
> +static int walk(struct strbuf *path, int argc, const char **argv)
> +{
> +	DIR *dir;
> +	struct dirent *ent;
> +	size_t len;
> +
> +	dir = opendir(path->buf);
> +	if (!dir)
> +		return errno;
> +	strbuf_addstr(path, "/");
> +	len = path->len;
> +	while ((ent = readdir(dir))) {
> +		if (!strcmp(ent->d_name, ".") || !strcmp(ent->d_name, ".."))
> +			continue;
> +		if (!strcmp(ent->d_name, ".git")) {

This only looks for the top of working tree. Have you considered if this "iterate over directories and list git repositories in them" may be useful for collection of bare repositories, and if it is, how to go about implementing the discovery process?

> +		if (ent->d_type != DT_DIR)
> +			continue;
I think this is wrong.

On platforms that need a NO_D_TYPE_IN_DIRENT build, your compilation may fail here (you would need to lstat() it yourself). See how dir.c does this without ugly #ifdef's in the code, especially around the use of get_dtype() and DTYPE() macro.

Duy Nguyen· Jan 23, 2013, 08:39 UTC · re: Lars Hjemli · lore

Re: [PATCH] all: new command used for multi-repo operations

On Wed, Jan 23, 2013 at 4:10 AM, Lars Hjemli <hjemli@gmail.com> wrote:
Show 11 quoted lines
> When working with multiple, unrelated (or loosly related) git repos,
> there is often a need to locate all repos with uncommitted work and
> perform some action on them (say, commit and push). Before this patch,
> such tasks would require manually visiting all repositories, running
> `git status` within each one and then decide what to do next.
>
> This mundane task can now be automated by e.g. `git all --dirty status`,
> which will find all git repositories below the current directory (even
> nested ones), check if they are dirty (as defined by `git diff --quiet &&
> git diff --cached --quiet`), and for each dirty repo print the path to the
> repo and then execute `git status` within the repo.

I think it should leave out the execute part. The command, say ls-repo, lists repositories in specified state. The execute part could be easily done by

xargs -I{} git --git-dir={} status blah

I haven't thought it through. I know xargs does not support chdir'ing into a repo, so maybe a new git-xargs could be introduced for that. But there's still a problem with git-xargs (or git-all), printed paths are relative to the subrepos, not where the git-all/xargs command is executed. This could be confusing. Personally I'd like to do it without chdir. That is

xargs -I{} git --git-dir={}/.git --work-tree={} status blah

should print paths relative to current working directory even if cwd is outside the repo.

-- 
Duy
Lars Hjemli· Jan 23, 2013, 08:46 UTC · re: Duy Nguyen · lore

Re: [PATCH] all: new command used for multi-repo operations

On Wed, Jan 23, 2013 at 9:39 AM, Duy Nguyen <pclouds@gmail.com> wrote:
Show 18 quoted lines
> On Wed, Jan 23, 2013 at 4:10 AM, Lars Hjemli <hjemli@gmail.com> wrote:
>> When working with multiple, unrelated (or loosly related) git repos,
>> there is often a need to locate all repos with uncommitted work and
>> perform some action on them (say, commit and push). Before this patch,
>> such tasks would require manually visiting all repositories, running
>> `git status` within each one and then decide what to do next.
>>
>> This mundane task can now be automated by e.g. `git all --dirty status`,
>> which will find all git repositories below the current directory (even
>> nested ones), check if they are dirty (as defined by `git diff --quiet &&
>> git diff --cached --quiet`), and for each dirty repo print the path to the
>> repo and then execute `git status` within the repo.
>
> I think it should leave out the execute part. The command, say
> ls-repo, lists repositories in specified state. The execute part could
> be easily done by
>
> xargs -I{} git --git-dir={} status blah
Not so easily on windows, which I need to use at $WORK :(

-- larsh

Lars Hjemli· Jan 23, 2013, 08:52 UTC · lore
[*git@vger.kernel.org accidentally dropped from cc *]
On Wed, Jan 23, 2013 at 7:52 AM, Junio C Hamano <gitster@pobox.com> wrote:
Show 22 quoted lines
> Lars Hjemli <hjemli@gmail.com> writes:
>
>> +static int walk(struct strbuf *path, int argc, const char **argv)
>> +{
>> +     DIR *dir;
>> +     struct dirent *ent;
>> +     size_t len;
>> +
>> +     dir = opendir(path->buf);
>> +     if (!dir)
>> +             return errno;
>> +     strbuf_addstr(path, "/");
>> +     len = path->len;
>> +     while ((ent = readdir(dir))) {
>> +             if (!strcmp(ent->d_name, ".") || !strcmp(ent->d_name, ".."))
>> +                     continue;
>> +             if (!strcmp(ent->d_name, ".git")) {
>
> This only looks for the top of working tree.  Have you considered if
> this "iterate over directories and list git repositories in them"
> may be useful for collection of bare repositories, and if it is, how
> to go about implementing the discovery process?

Yes, occasionally I've needed this, but implementing it in my original shell script was cumbersome. Doing it in C should be as simple as invoking is_git_directory() on each subdir.

Show 11 quoted lines
>
>> +             if (ent->d_type != DT_DIR)
>> +                     continue;
>
> I think this is wrong.
>
> On platforms that need a NO_D_TYPE_IN_DIRENT build, your compilation
> may fail here (you would need to lstat() it yourself).  See how
> dir.c does this without ugly #ifdef's in the code, especially around
> the use of get_dtype() and DTYPE() macro.
>
Thanks for the pointer, will fix.

-- larsh

← back to recent threads