git/list[1] front-page[2] threads[3] people[4] search[5] about
 

[PATCH 1/4] run-command: add pipe_output to run_processes_parallel

From
Calvin Wan <calvinwan@google.com>
Date
Sep 22, 2022, 23:29 UTC
Message-ID
<20220922232947.631309-2-calvinwan@google.com>
In-Reply-To
<20220922232947.631309-1-calvinwan@google.com>

run_processes_parallel periodically collects output from its child processes, prints it, and then resets the buffers for each child. Add run_processes_parallel_pipe_output variable so output can be collected and fed to task_finished. When set, the function referenced by task_finished should parse the output of each child process.

Signed-off-by: Calvin Wan <calvinwan@google.com>
---
 run-command.c               |  6 +++++-
 run-command.h               |  9 +++++++++
 t/helper/test-run-command.c | 31 ++++++++++++++++++++++++++++---
 t/t0061-run-command.sh      | 26 ++++++++++++++++++++++++++
 4 files changed, 68 insertions(+), 4 deletions(-)
diff --git a/run-command.c b/run-command.c
index 14f17830f5..893bc1d294 100644
--- a/run-command.c
+++ b/run-command.c
@@ -1474,6 +1474,7 @@ enum child_state {
 };
 
 int run_processes_parallel_ungroup;
+int run_processes_parallel_pipe_output;
 struct parallel_processes {
 	void *data;
 
@@ -1770,10 +1771,12 @@ int run_processes_parallel(int n,
 	int output_timeout = 100;
 	int spawn_cap = 4;
 	int ungroup = run_processes_parallel_ungroup;
+	int pipe_output = run_processes_parallel_pipe_output;
 	struct parallel_processes pp;
 
 	/* unset for the next API user */
 	run_processes_parallel_ungroup = 0;
+	run_processes_parallel_pipe_output = 0;
 
 	pp_init(&pp, n, get_next_task, start_failure, task_finished, pp_cb,
 		ungroup);
@@ -1800,7 +1803,8 @@ int run_processes_parallel(int n,
 				pp.children[i].state = GIT_CP_WAIT_CLEANUP;
 		} else {
 			pp_buffer_stderr(&pp, output_timeout);
-			pp_output(&pp);
+			if (!pipe_output)
+				pp_output(&pp);
 		}
 		code = pp_collect_finished(&pp);
 		if (code) {
diff --git a/run-command.h b/run-command.h
index 0e85e5846a..a5b1d63f49 100644
--- a/run-command.h
+++ b/run-command.h
@@ -483,8 +483,17 @@ typedef int (*task_finished_fn)(int result,
  * "run_processes_parallel_ungroup" to "1" before invoking
  * run_processes_parallel(), it will be set back to "0" as soon as the
  * API reads that setting.
+ * 
+ * If the "pipe_output" option is specified, the output will be piped
+ * to task_finished_fn in the "struct strbuf *out" variable. The output
+ * will still be printed unless the callback resets the strbuf. The
+ * "pipe_output" option can be enabled by setting the global
+ * "run_processes_parallel_pipe_output" to "1" before invoking
+ * run_processes_parallel(), it will be set back to "0" as soon as the
+ * API reads that setting.
  */
 extern int run_processes_parallel_ungroup;
+extern int run_processes_parallel_pipe_output;
 int run_processes_parallel(int n,
 			   get_next_task_fn,
 			   start_failure_fn,
diff --git a/t/helper/test-run-command.c b/t/helper/test-run-command.c
index c9283b47af..030e533c6b 100644
--- a/t/helper/test-run-command.c
+++ b/t/helper/test-run-command.c
@@ -20,6 +20,7 @@
 #include "wildmatch.h"
 #include "gettext.h"
 
+static int pipe_output = 0;
 static int number_callbacks;
 static int parallel_next(struct child_process *cp,
 			 struct strbuf *err,
@@ -52,15 +53,32 @@ static int no_job(struct child_process *cp,
 	return 0;
 }
 
+static int task_finished_pipe_output(int result,
+			 struct strbuf *err,
+			 void *pp_cb,
+			 void *pp_task_cb)
+{
+	if (err && pipe_output) {
+		fprintf(stderr, "%s", err->buf);
+		strbuf_reset(err);
+	}
+	return 0;
+}
+
 static int task_finished(int result,
 			 struct strbuf *err,
 			 void *pp_cb,
 			 void *pp_task_cb)
 {
-	if (err)
+	if (err) {
 		strbuf_addstr(err, "asking for a quick stop\n");
-	else
+		if (pipe_output) {
+			fprintf(stderr, "%s", err->buf);
+			strbuf_reset(err);
+		}
+	} else {
 		fprintf(stderr, "asking for a quick stop\n");
+	}
 	return 1;
 }
 
@@ -423,13 +441,20 @@ int cmd__run_command(int argc, const char **argv)
 		run_processes_parallel_ungroup = 1;
 	}
 
+	if (!strcmp(argv[1], "--pipe-output")) {
+		argv += 1;
+		argc -= 1;
+		run_processes_parallel_pipe_output = 1;
+		pipe_output = 1;
+	}
+
 	jobs = atoi(argv[2]);
 	strvec_clear(&proc.args);
 	strvec_pushv(&proc.args, (const char **)argv + 3);
 
 	if (!strcmp(argv[1], "run-command-parallel"))
 		exit(run_processes_parallel(jobs, parallel_next,
-					    NULL, NULL, &proc));
+					    NULL, task_finished_pipe_output, &proc));
 
 	if (!strcmp(argv[1], "run-command-abort"))
 		exit(run_processes_parallel(jobs, parallel_next,
diff --git a/t/t0061-run-command.sh b/t/t0061-run-command.sh
index 7b5423eebd..97ca942a74 100755
--- a/t/t0061-run-command.sh
+++ b/t/t0061-run-command.sh
@@ -140,6 +140,11 @@ test_expect_success 'run_command runs ungrouped in parallel with more jobs avail
 	test_line_count = 4 err
 '
 
+test_expect_success 'run_command runs pipe_output in parallel with more jobs available than tasks' '
+	test-tool run-command --pipe-output run-command-parallel 5 sh -c "printf \"%s\n%s\n\" Hello World" 2>actual &&
+	test_cmp expect actual
+'
+
 test_expect_success 'run_command runs in parallel with as many jobs as tasks' '
 	test-tool run-command run-command-parallel 4 sh -c "printf \"%s\n%s\n\" Hello World" 2>actual &&
 	test_cmp expect actual
@@ -151,6 +156,11 @@ test_expect_success 'run_command runs ungrouped in parallel with as many jobs as
 	test_line_count = 4 err
 '
 
+test_expect_success 'run_command runs pipe_output in parallel with as many jobs as tasks' '
+	test-tool run-command --pipe-output run-command-parallel 4 sh -c "printf \"%s\n%s\n\" Hello World" 2>actual &&
+	test_cmp expect actual
+'
+
 test_expect_success 'run_command runs in parallel with more tasks than jobs available' '
 	test-tool run-command run-command-parallel 3 sh -c "printf \"%s\n%s\n\" Hello World" 2>actual &&
 	test_cmp expect actual
@@ -162,6 +172,12 @@ test_expect_success 'run_command runs ungrouped in parallel with more tasks than
 	test_line_count = 4 err
 '
 
+test_expect_success 'run_command runs pipe_output in parallel with more tasks than jobs available' '
+	test-tool run-command --pipe-output run-command-parallel 3 sh -c "printf \"%s\n%s\n\" Hello World" 2>actual &&
+	test_cmp expect actual
+'
+
+
 cat >expect <<-EOF
 preloaded output of a child
 asking for a quick stop
@@ -182,6 +198,11 @@ test_expect_success 'run_command is asked to abort gracefully (ungroup)' '
 	test_line_count = 6 err
 '
 
+test_expect_success 'run_command is asked to abort gracefully (pipe_output)' '
+	test-tool run-command --pipe-output run-command-abort 3 false 2>actual &&
+	test_cmp expect actual
+'
+
 cat >expect <<-EOF
 no further jobs available
 EOF
@@ -197,6 +218,11 @@ test_expect_success 'run_command outputs (ungroup) ' '
 	test_cmp expect err
 '
 
+test_expect_success 'run_command outputs (pipe_output) ' '
+	test-tool run-command --pipe-output run-command-no-jobs 3 sh -c "printf \"%s\n%s\n\" Hello World" 2>actual &&
+	test_cmp expect actual
+'
+
 test_trace () {
 	expect="$1"
 	shift

-- 
2.37.3.998.g577e59143f-goog
Previous: Calvin WanNext: Ævar Arnfjörð Bjarmason
Message 2 of 32 in “submodule: parallelize status”
  1. 0/4 submodule: parallelize statusCalvin Wan, Sep 22, 2022
  2. 1/4 run-command: add pipe_output to run_processes_parallelCalvin Wan, Sep 22, 2022
  3. Ævar Arnfjörð BjarmasonSep 23, 2022
  4. Calvin WanSep 26, 2022
  5. Ævar Arnfjörð BjarmasonSep 27, 2022
  6. Junio C HamanoSep 23, 2022
  7. Calvin WanSep 26, 2022
  8. Junio C HamanoSep 27, 2022
  9. Calvin WanSep 27, 2022
  10. Junio C HamanoSep 27, 2022
  11. Ævar Arnfjörð BjarmasonSep 27, 2022
  12. Calvin WanSep 27, 2022
  13. Ævar Arnfjörð BjarmasonSep 27, 2022
  14. Calvin WanSep 27, 2022
  15. Ævar Arnfjörð BjarmasonSep 28, 2022
  16. Calvin WanSep 29, 2022
  17. 2/4 submodule: move status parsing into functionCalvin Wan, Sep 22, 2022
  18. 3/4 diff-lib: refactor functionsCalvin Wan, Sep 22, 2022
  19. Junio C HamanoSep 23, 2022
  20. Calvin WanSep 26, 2022
  21. 4/4 diff-lib: parallelize run_diff_files for submodulesCalvin Wan, Sep 22, 2022
  22. Ævar Arnfjörð BjarmasonSep 23, 2022
  23. Junio C HamanoSep 24, 2022
  24. Calvin WanSep 26, 2022
  25. Junio C HamanoSep 23, 2022
  26. Calvin WanSep 26, 2022
  27. Phillip WoodSep 25, 2022
  28. Junio C HamanoSep 26, 2022
  29. Calvin WanSep 26, 2022
  30. Emily ShafferSep 27, 2022
  31. Junio C HamanoSep 23, 2022
  32. Calvin WanSep 26, 2022

Read the whole thread, see it on lore, or plain text.

$ cat FOOTERMessages come from the public archive at lore.kernel.org/git, fetched every hour. The front page is chosen and written each morning by an AI editor and can be wrong; the threads themselves are the record. About and API. For agents: an MCP server at https://gitlist.dev/mcp, and any thread, story or person page as Markdown by adding .md to its URL (or sending Accept: text/markdown). Details in /llms.txt.