[PATCH v2 1/4] run-command: add pipe_output_fn to run_processes_parallel_opts

[Calvin Wan <calvinwan@google.com>]

Add pipe_output_fn as an optionally set function in
run_process_parallel_opts. If set, output from each child process is
piped to the callback function to allow for separate parsing.

Signed-off-by: Calvin Wan <calvinwan@google.com>
---
 run-command.c               | 12 +++++--
 run-command.h               | 22 +++++++++++++
 t/helper/test-run-command.c | 18 ++++++++++
 t/t0061-run-command.sh      | 65 +++++++++++++++++++++++--------------
 4 files changed, 90 insertions(+), 27 deletions(-)

diff --git a/run-command.c b/run-command.c
index da02631933..c6090e4cb8 100644
--- a/run-command.c
+++ b/run-command.c
@@ -1689,12 +1689,16 @@ static void pp_buffer_stderr(struct parallel_processes *pp, int output_timeout)
 	}
 }
 
-static void pp_output(struct parallel_processes *pp)
+static void pp_output(struct parallel_processes *pp,
+			const struct run_process_parallel_opts *opts)
 {
 	int i = pp->output_owner;
 
 	if (pp->children[i].state == GIT_CP_WORKING &&
 	    pp->children[i].err.len) {
+		if (opts->pipe_output)
+			opts->pipe_output(&pp->children[i].err, pp->data,
+						pp->children[i].data);
 		strbuf_write(&pp->children[i].err, stderr);
 		strbuf_reset(&pp->children[i].err);
 	}
@@ -1716,6 +1720,10 @@ static int pp_collect_finished(struct parallel_processes *pp,
 
 		code = finish_command(&pp->children[i].process);
 
+		if (opts->pipe_output)
+			opts->pipe_output(&pp->children[i].err, pp->data,
+						pp->children[i].data);
+
 		if (opts->task_finished)
 			code = opts->task_finished(code, opts->ungroup ? NULL :
 						   &pp->children[i].err, pp->data,
@@ -1803,7 +1811,7 @@ void run_processes_parallel(const struct run_process_parallel_opts *opts)
 				pp.children[i].state = GIT_CP_WAIT_CLEANUP;
 		} else {
 			pp_buffer_stderr(&pp, output_timeout);
-			pp_output(&pp);
+			pp_output(&pp, opts);
 		}
 		code = pp_collect_finished(&pp, opts);
 		if (code) {
diff --git a/run-command.h b/run-command.h
index 075bd9b9de..cb51c56ea6 100644
--- a/run-command.h
+++ b/run-command.h
@@ -440,6 +440,22 @@ typedef int (*start_failure_fn)(struct strbuf *out,
 				void *pp_cb,
 				void *pp_task_cb);
 
+/**
+ * This callback is periodically called while child processes are
+ * running and also when a child process finishes.
+ *
+ * "struct strbuf *out" contains the output collected from pp_task_cb
+ * since the last call of this function.
+ *
+ * pp_cb is the callback cookie as passed into run_processes_parallel,
+ * pp_task_cb is the callback cookie as passed into get_next_task_fn.
+ *
+ * This function is incompatible with "ungroup"
+ */
+typedef void (*pipe_output_fn)(struct strbuf *out,
+				void *pp_cb,
+				void *pp_task_cb);
+
 /**
  * This callback is called on every child process that finished processing.
  *
@@ -493,6 +509,12 @@ struct run_process_parallel_opts
 	 */
 	start_failure_fn start_failure;
 
+	/**
+	 * pipe_output: See pipe_output_fn() above. This should be
+	 * NULL unless process specific output is needed
+	 */
+	pipe_output_fn pipe_output;
+
 	/**
 	 * task_finished: See task_finished_fn() above. This can be
 	 * NULL to omit any special handling.
diff --git a/t/helper/test-run-command.c b/t/helper/test-run-command.c
index 46bac2bb70..d3c3df7960 100644
--- a/t/helper/test-run-command.c
+++ b/t/helper/test-run-command.c
@@ -52,6 +52,18 @@ static int no_job(struct child_process *cp,
 	return 0;
 }
 
+static void pipe_output(struct strbuf *out,
+				void *pp_cb,
+				void *pp_task_cb)
+{
+	fprintf(stderr, "%s", out->buf);
+	/*
+	 * Resetting output to show that piped output would print the
+	 * same as other tests without the pipe_output() function set
+	 */
+	strbuf_reset(out);
+}
+
 static int task_finished(int result,
 			 struct strbuf *err,
 			 void *pp_cb,
@@ -439,6 +451,12 @@ int cmd__run_command(int argc, const char **argv)
 		opts.ungroup = 1;
 	}
 
+	if (!strcmp(argv[1], "--pipe-output")) {
+		argv += 1;
+		argc -= 1;
+		opts.pipe_output = pipe_output;
+	}
+
 	jobs = atoi(argv[2]);
 	strvec_clear(&proc.args);
 	strvec_pushv(&proc.args, (const char **)argv + 3);
diff --git a/t/t0061-run-command.sh b/t/t0061-run-command.sh
index 19af082750..feabb3717b 100755
--- a/t/t0061-run-command.sh
+++ b/t/t0061-run-command.sh
@@ -129,11 +129,14 @@ Hello
 World
 EOF
 
-test_expect_success 'run_command runs in parallel with more jobs available than tasks' '
-	test-tool run-command run-command-parallel 5 sh -c "printf \"%s\n%s\n\" Hello World" >out 2>err &&
-	test_must_be_empty out &&
-	test_cmp expect err
-'
+for opt in '' '--pipe-output'
+do
+	test_expect_success "run_command runs in parallel with more jobs available than tasks $opt" '
+		test-tool run-command run-command-parallel 5 sh -c "printf \"%s\n%s\n\" Hello World" >out 2>err &&
+		test_must_be_empty out &&
+		test_cmp expect err
+	'
+done
 
 test_expect_success 'run_command runs ungrouped in parallel with more jobs available than tasks' '
 	test-tool run-command --ungroup run-command-parallel 5 sh -c "printf \"%s\n%s\n\" Hello World" >out 2>err &&
@@ -141,11 +144,14 @@ test_expect_success 'run_command runs ungrouped in parallel with more jobs avail
 	test_line_count = 4 err
 '
 
-test_expect_success 'run_command runs in parallel with as many jobs as tasks' '
-	test-tool run-command run-command-parallel 4 sh -c "printf \"%s\n%s\n\" Hello World" >out 2>err &&
-	test_must_be_empty out &&
-	test_cmp expect err
-'
+for opt in '' '--pipe-output'
+do
+	test_expect_success "run_command runs in parallel with as many jobs as tasks $opt" '
+		test-tool run-command run-command-parallel 4 sh -c "printf \"%s\n%s\n\" Hello World" >out 2>err &&
+		test_must_be_empty out &&
+		test_cmp expect err
+	'
+done
 
 test_expect_success 'run_command runs ungrouped in parallel with as many jobs as tasks' '
 	test-tool run-command --ungroup run-command-parallel 4 sh -c "printf \"%s\n%s\n\" Hello World" >out 2>err &&
@@ -153,11 +159,14 @@ test_expect_success 'run_command runs ungrouped in parallel with as many jobs as
 	test_line_count = 4 err
 '
 
-test_expect_success 'run_command runs in parallel with more tasks than jobs available' '
-	test-tool run-command run-command-parallel 3 sh -c "printf \"%s\n%s\n\" Hello World" >out 2>err &&
-	test_must_be_empty out &&
-	test_cmp expect err
-'
+for opt in '' '--pipe-output'
+do
+	test_expect_success "run_command runs in parallel with more tasks than jobs available $opt" '
+		test-tool run-command run-command-parallel 3 sh -c "printf \"%s\n%s\n\" Hello World" >out 2>err &&
+		test_must_be_empty out &&
+		test_cmp expect err
+	'
+done
 
 test_expect_success 'run_command runs ungrouped in parallel with more tasks than jobs available' '
 	test-tool run-command --ungroup run-command-parallel 3 sh -c "printf \"%s\n%s\n\" Hello World" >out 2>err &&
@@ -174,11 +183,14 @@ preloaded output of a child
 asking for a quick stop
 EOF
 
-test_expect_success 'run_command is asked to abort gracefully' '
-	test-tool run-command run-command-abort 3 false >out 2>err &&
-	test_must_be_empty out &&
-	test_cmp expect err
-'
+for opt in '' '--pipe-output'
+do
+	test_expect_success "run_command is asked to abort gracefully $opt" '
+		test-tool run-command run-command-abort 3 false >out 2>err &&
+		test_must_be_empty out &&
+		test_cmp expect err
+	'
+done
 
 test_expect_success 'run_command is asked to abort gracefully (ungroup)' '
 	test-tool run-command --ungroup run-command-abort 3 false >out 2>err &&
@@ -190,11 +202,14 @@ cat >expect <<-EOF
 no further jobs available
 EOF
 
-test_expect_success 'run_command outputs ' '
-	test-tool run-command run-command-no-jobs 3 sh -c "printf \"%s\n%s\n\" Hello World" >out 2>err &&
-	test_must_be_empty out &&
-	test_cmp expect err
-'
+for opt in '' '--pipe-output'
+do
+	test_expect_success "run_command outputs $opt" '
+		test-tool run-command run-command-no-jobs 3 sh -c "printf \"%s\n%s\n\" Hello World" >out 2>err &&
+		test_must_be_empty out &&
+		test_cmp expect err
+	'
+done
 
 test_expect_success 'run_command outputs (ungroup) ' '
 	test-tool run-command --ungroup run-command-no-jobs 3 sh -c "printf \"%s\n%s\n\" Hello World" >out 2>err &&
-- 
2.38.0.rc1.362.ged0d419d3c-goog