Re: "git grep" parallelism question

From

Junio C Hamano <gitster@pobox.com>

Date

Apr 26, 2013, 19:19 UTC

Message-ID

<7vip39w14d.fsf@alter.siamese.dyndns.org>

In-Reply-To

<CA+55aFw+6pL5DoEPsPZpJCAbqEGaWYYKcdjZzbsHVzSSMrQmww@mail.gmail.com>

Linus Torvalds <torvalds@linux-foundation.org> writes:

> Yes. Also, I'm not sure if the 15% possible improvement on my SSD case
> is even worth it for something that in the end isn't necessarily the
> common case.

> Are there people out there who use "git grep" over NFS and have been
> unhappy with performance? If are willing to recompile git with a
> different THREAD value in builtin/grep.c,...

 builtin/grep.c | 30 ++++++++++++++++++++++--------
 1 file changed, 22 insertions(+), 8 deletions(-)

diff --git a/builtin/grep.c b/builtin/grep.c
index 159e65d..f635cd5 100644
--- a/builtin/grep.c
+++ b/builtin/grep.c
@@ -26,12 +26,14 @@ static char const * const grep_usage[] = {
 static int use_threads = 1;
 
 #ifndef NO_PTHREADS
-#define THREADS 8
-static pthread_t threads[THREADS];
+#define MAX_THREADS 100
+static int num_threads = 8;
+static pthread_t threads[MAX_THREADS];
 
-/* We use one producer thread and THREADS consumer
- * threads. The producer adds struct work_items to 'todo' and the
- * consumers pick work items from the same array.
+/*
+ * We use one producer thread and THREADS consumer threads. The
+ * producer adds struct work_items to 'todo' and the consumers pick
+ * work items from the same array.
  */
 struct work_item {
 	struct grep_source source;
@@ -205,7 +207,7 @@ static void start_threads(struct grep_opt *opt)
 		strbuf_init(&todo[i].out, 0);
 	}
 
-	for (i = 0; i < ARRAY_SIZE(threads); i++) {
+	for (i = 0; i < num_threads; i++) {
 		int err;
 		struct grep_opt *o = grep_opt_dup(opt);
 		o->output = strbuf_out;
@@ -237,7 +239,7 @@ static int wait_all(void)
 	pthread_cond_broadcast(&cond_add);
 	grep_unlock();
 
-	for (i = 0; i < ARRAY_SIZE(threads); i++) {
+	for (i = 0; i < num_threads; i++) {
 		void *h;
 		pthread_join(threads[i], &h);
 		hit |= (int) (intptr_t) h;
@@ -636,6 +638,7 @@ int cmd_grep(int argc, const char **argv, const char *prefix)
 	int i;
 	int dummy;
 	int use_index = 1;
+	int num_threads_explicit = -1;
 	int pattern_type_arg = GREP_PATTERN_TYPE_UNSPECIFIED;
 
 	struct option options[] = {
@@ -743,6 +746,8 @@ int cmd_grep(int argc, const char **argv, const char *prefix)
 			    N_("allow calling of grep(1) (ignored by this build)")),
 		{ OPTION_CALLBACK, 0, "help-all", &options, NULL, N_("show usage"),
 		  PARSE_OPT_HIDDEN | PARSE_OPT_NOARG, help_callback },
+		OPT_INTEGER(0, "threads", &num_threads_explicit,
+			    N_("use threads when searching")),
 		OPT_END()
 	};
 
@@ -773,6 +778,15 @@ int cmd_grep(int argc, const char **argv, const char *prefix)
 			     PARSE_OPT_NO_INTERNAL_HELP);
 	grep_commit_pattern_type(pattern_type_arg, &opt);
 
+	if (MAX_THREADS <= num_threads_explicit) {
+		warning("limiting --threads to %d", MAX_THREADS);
+		num_threads = MAX_THREADS;
+	} else if (num_threads_explicit < 0) {
+		; /* keep num_threads to compiled-in default */
+	} else {
+		num_threads = num_threads_explicit;
+	}
+
 	if (use_index && !startup_info->have_repository)
 		/* die the same way as if we did it at the beginning */
 		setup_git_directory();
@@ -834,7 +848,7 @@ int cmd_grep(int argc, const char **argv, const char *prefix)
 	}
 
 #ifndef NO_PTHREADS
-	if (list.nr || cached || online_cpus() == 1)
+	if ((list.nr || cached || online_cpus() == 1) && num_threads_explicit < 0)
 		use_threads = 0;
 #else
 	use_threads = 0;