git/list[1] front-page[2] threads[3] people[4] search[5] about
 

Re: [PATCH v2 3/3] grep: disable threading in all but worktree case

From
René Scharfe <rene.scharfe@lsrfire.ath.cx>
Date
Dec 6, 2011, 22:48 UTC
Message-ID
<4EDE9BBA.2010409@lsrfire.ath.cx>
In-Reply-To
<4ED8F9AE.8030605@lsrfire.ath.cx>
Am 02.12.2011 17:15, schrieb René Scharfe:
> How about adding a parameter to control the number of threads 
> (--threads?) instead that defaults to eight (or five) for the worktree 
> and one for the rest? That would also make benchmarking easier.
Like this:
-- >8 --
Subject: grep: add parameter --threads

Allow the number of threads to be specified by the user. This makes benchmarking the performance impact of different numbers of threads much easier.

Move the code for thread handling after argument parsing. This allows to change the default number of threads based on the kind of search (worktree etc.) later on.

Signed-off-by: Rene Scharfe <rene.scharfe@lsrfire.ath.cx>
---
Applies on top of your second patch.
 Documentation/git-grep.txt |    4 ++
 builtin/grep.c             |   75 +++++++++++++++++++++++--------------------
 2 files changed, 44 insertions(+), 35 deletions(-)
diff --git a/Documentation/git-grep.txt b/Documentation/git-grep.txt
index 15d6711..47ac188 100644
--- a/Documentation/git-grep.txt
+++ b/Documentation/git-grep.txt
@@ -227,6 +227,10 @@ OPTIONS
 	Do not output matched lines; instead, exit with status 0 when
 	there is a match and with non-zero status when there isn't.
 
+--threads <n>::
+	Run <n> search threads in parallel.  Default is 8.  This option
+	is ignored if git was built without support for POSIX threads.
+
 <tree>...::
 	Instead of searching tracked files in the working tree, search
 	blobs in the given trees.
diff --git a/builtin/grep.c b/builtin/grep.c
index 65b1ffe..0bda900 100644
--- a/builtin/grep.c
+++ b/builtin/grep.c
@@ -24,11 +24,10 @@ static char const * const grep_usage[] = {
 	NULL
 };
 
-static int use_threads = 1;
-
 #ifndef NO_PTHREADS
 #define THREADS 8
-static pthread_t threads[THREADS];
+static pthread_t *threads;
+static int nr_threads = -1;
 
 static void *load_sha1(const unsigned char *sha1, unsigned long *size,
 		       const char *name);
@@ -76,13 +75,13 @@ static pthread_mutex_t grep_mutex;
 
 static inline void grep_lock(void)
 {
-	if (use_threads)
+	if (nr_threads > 0)
 		pthread_mutex_lock(&grep_mutex);
 }
 
 static inline void grep_unlock(void)
 {
-	if (use_threads)
+	if (nr_threads > 0)
 		pthread_mutex_unlock(&grep_mutex);
 }
 
@@ -91,13 +90,13 @@ static pthread_mutex_t read_sha1_mutex;
 
 static inline void read_sha1_lock(void)
 {
-	if (use_threads)
+	if (nr_threads > 0)
 		pthread_mutex_lock(&read_sha1_mutex);
 }
 
 static inline void read_sha1_unlock(void)
 {
-	if (use_threads)
+	if (nr_threads > 0)
 		pthread_mutex_unlock(&read_sha1_mutex);
 }
 
@@ -254,6 +253,8 @@ static void start_threads(struct grep_opt *opt)
 {
 	int i;
 
+	threads = xcalloc(nr_threads, sizeof(pthread_t));
+
 	pthread_mutex_init(&grep_mutex, NULL);
 	pthread_mutex_init(&read_sha1_mutex, NULL);
 	pthread_mutex_init(&grep_attr_mutex, NULL);
@@ -265,7 +266,7 @@ static void start_threads(struct grep_opt *opt)
 		strbuf_init(&todo[i].out, 0);
 	}
 
-	for (i = 0; i < ARRAY_SIZE(threads); i++) {
+	for (i = 0; i < nr_threads; i++) {
 		int err;
 		struct grep_opt *o = grep_opt_dup(opt);
 		o->output = strbuf_out;
@@ -296,7 +297,7 @@ static int wait_all(void)
 	pthread_cond_broadcast(&cond_add);
 	grep_unlock();
 
-	for (i = 0; i < ARRAY_SIZE(threads); i++) {
+	for (i = 0; i < nr_threads; i++) {
 		void *h;
 		pthread_join(threads[i], &h);
 		hit |= (int) (intptr_t) h;
@@ -309,6 +310,8 @@ static int wait_all(void)
 	pthread_cond_destroy(&cond_write);
 	pthread_cond_destroy(&cond_result);
 
+	free(threads);
+
 	return hit;
 }
 #else /* !NO_PTHREADS */
@@ -410,7 +413,7 @@ static int grep_sha1(struct grep_opt *opt, const unsigned char *sha1,
 	name = strbuf_detach(&pathbuf, NULL);
 
 #ifndef NO_PTHREADS
-	if (use_threads) {
+	if (nr_threads > 0) {
 		grep_sha1_async(opt, name, sha1);
 		return 0;
 	} else
@@ -472,7 +475,7 @@ static int grep_file(struct grep_opt *opt, const char *filename)
 	name = strbuf_detach(&buf, NULL);
 
 #ifndef NO_PTHREADS
-	if (use_threads) {
+	if (nr_threads > 0) {
 		grep_file_async(opt, name, filename);
 		return 0;
 	} else
@@ -895,6 +898,13 @@ int cmd_grep(int argc, const char **argv, const char *prefix)
 			PARSE_OPT_OPTARG, NULL, (intptr_t)default_pager },
 		OPT_BOOLEAN(0, "ext-grep", &external_grep_allowed__ignored,
 			    "allow calling of grep(1) (ignored by this build)"),
+#ifdef NO_PTHREADS
+		OPT_INTEGER(0, "threads", &nr_threads,
+			"handle <n> files in parallel (ignored by this build)"),
+#else
+		OPT_INTEGER(0, "threads", &nr_threads,
+			"handle <n> files in parallel"),
+#endif
 		{ OPTION_CALLBACK, 0, "help-all", &options, NULL, "show usage",
 		  PARSE_OPT_HIDDEN | PARSE_OPT_NOARG, help_callback },
 		OPT_END()
@@ -995,7 +1005,7 @@ int cmd_grep(int argc, const char **argv, const char *prefix)
 		opt.output_priv = &path_list;
 		opt.output = append_path;
 		string_list_append(&path_list, show_in_pager);
-		use_threads = 0;
+		nr_threads = 0;
 	}
 
 	if (!opt.pattern_list)
@@ -1003,28 +1013,6 @@ int cmd_grep(int argc, const char **argv, const char *prefix)
 	if (!opt.fixed && opt.ignore_case)
 		opt.regflags |= REG_ICASE;
 
-#ifndef NO_PTHREADS
-	if (online_cpus() == 1)
-		use_threads = 0;
-#else
-	use_threads = 0;
-#endif
-
-	opt.use_threads = use_threads;
-
-#ifndef NO_PTHREADS
-	if (use_threads) {
-		if (opt.pre_context || opt.post_context || opt.file_break ||
-		    opt.funcbody)
-			skip_first_line = 1;
-		start_threads(&opt);
-	}
-#else
-	use_threads = 0;
-#endif
-
-	compile_grep_patterns(&opt);
-
 	/* Check revs and then paths */
 	for (i = 0; i < argc; i++) {
 		const char *arg = argv[i];
@@ -1056,6 +1044,23 @@ int cmd_grep(int argc, const char **argv, const char *prefix)
 	pathspec.max_depth = opt.max_depth;
 	pathspec.recursive = 1;
 
+#ifdef NO_PTHREADS
+	nr_threads = 0;
+#else
+	if (nr_threads == -1)
+		nr_threads = (online_cpus() > 1) ? THREADS : 0;
+
+	if (nr_threads > 0) {
+		opt.use_threads = 1;
+		if (opt.pre_context || opt.post_context || opt.file_break ||
+		    opt.funcbody)
+			skip_first_line = 1;
+		start_threads(&opt);
+	}
+#endif
+
+	compile_grep_patterns(&opt);
+
 	if (show_in_pager && (cached || list.nr))
 		die(_("--open-files-in-pager only works on the worktree"));
 
@@ -1100,7 +1105,7 @@ int cmd_grep(int argc, const char **argv, const char *prefix)
 		hit = grep_objects(&opt, &pathspec, &list);
 	}
 
-	if (use_threads)
+	if (nr_threads > 0)
 		hit |= wait_all();
 	if (hit && show_in_pager)
 		run_pager(&opt, prefix);

-- 
1.7.8
Previous: Thomas RastNext: René Scharfe
Message 7 of 35 in “grep multithreading and scaling”
  1. 0/3 grep multithreading and scalingThomas Rast, Dec 2, 2011
  2. 1/3 grep: load funcname patterns for -WThomas Rast, Dec 2, 2011
  3. 2/3 grep: enable threading with -p and -W using lazy attribute lookupThomas Rast, Dec 2, 2011
  4. 3/3 grep: disable threading in all but worktree caseThomas Rast, Dec 2, 2011
  5. René ScharfeDec 2, 2011
  6. Thomas RastDec 5, 2011
  7. René ScharfeDec 6, 2011
  8. 4/2 grep: turn off threading for non-worktreeRené Scharfe, Dec 6, 2011
  9. Jeff KingDec 7, 2011
  10. René ScharfeDec 7, 2011
  11. Jeff KingDec 7, 2011
  12. J. Bruce FieldsDec 7, 2011
  13. Jeff KingDec 7, 2011
  14. Thomas RastDec 7, 2011
  15. René ScharfeDec 7, 2011
  16. Pete WyckoffDec 10, 2011
  17. René ScharfeDec 12, 2011
  18. Jeff KingDec 7, 2011
  19. René ScharfeDec 7, 2011
  20. Jeff KingDec 7, 2011
  21. Thomas RastDec 7, 2011
  22. René ScharfeDec 7, 2011
  23. Ævar Arnfjörð BjarmasonDec 23, 2011
  24. Thomas RastDec 23, 2011
  25. Ævar Arnfjörð BjarmasonDec 24, 2011
  26. Jeff KingDec 24, 2011
  27. Nguyen Thai Ngoc DuyDec 24, 2011
  28. Nguyen Thai Ngoc DuyDec 24, 2011
  29. Jeff KingDec 24, 2011
  30. Nguyen Thai Ngoc DuyDec 25, 2011
  31. Jeff KingDec 2, 2011
  32. Thomas RastDec 5, 2011
  33. Thomas RastDec 5, 2011
  34. Jeff KingDec 6, 2011
  35. Eric HermanDec 2, 2011

Read the whole thread, see it on lore, or plain text.

$ cat FOOTERMessages come from the public archive at lore.kernel.org/git, fetched every hour. The front page is chosen and written each morning by an AI editor and can be wrong; the threads themselves are the record. About and API. For agents: an MCP server at https://gitlist.dev/mcp, and any thread, story or person page as Markdown by adding .md to its URL (or sending Accept: text/markdown). Details in /llms.txt.