[RFC PATCH 4/4] submodule: add infrastructure to fetch submodules in parallel

[Stefan Beller <sbeller@google.com>]

This makes use of the new workdispatcher to fetch a number
of submodules at the same time.

Still todo: sort the output of the fetch commands. I am unsure
if this should be hooked into the workdispatcher as the problem
of sorted output will appear likely again, so a general solution
would not hurt.

Signed-off-by: Stefan Beller <sbeller@google.com>
---
 builtin/fetch.c |  3 ++-
 submodule.c     | 74 ++++++++++++++++++++++++++++++++++++++++++++-------------
 submodule.h     |  2 +-
 3 files changed, 60 insertions(+), 19 deletions(-)

diff --git a/builtin/fetch.c b/builtin/fetch.c
index 8d5b2db..9053e8b 100644
--- a/builtin/fetch.c
+++ b/builtin/fetch.c
@@ -1207,7 +1207,8 @@ int cmd_fetch(int argc, const char **argv, const char *prefix)
 		result = fetch_populated_submodules(&options,
 						    submodule_prefix,
 						    recurse_submodules,
-						    verbosity < 0);
+						    verbosity < 0,
+						    1);
 		argv_array_clear(&options);
 	}
 
diff --git a/submodule.c b/submodule.c
index 872967f..0b2842b 100644
--- a/submodule.c
+++ b/submodule.c
@@ -11,6 +11,7 @@
 #include "sha1-array.h"
 #include "argv-array.h"
 #include "blob.h"
+#include "workdispatcher.h"
 
 static struct string_list config_name_for_path;
 static struct string_list config_fetch_recurse_submodules_for_name;
@@ -696,13 +697,49 @@ const char* submodule_name_for_path(const char* path)
 		return NULL;
 }
 
+struct submodule_parallel_fetch {
+	struct child_process cp;
+	struct argv_array argv;
+	struct strbuf sb;
+	int quiet;
+};
+
+void submodule_parallel_fetch_init(struct submodule_parallel_fetch *spf)
+{
+	child_process_init(&spf->cp);
+	argv_array_init(&spf->argv);
+	strbuf_init(&spf->sb, 0);
+	spf->quiet = 0;
+}
+
+void *run_command_and_cleanup(void *arg)
+{
+	struct submodule_parallel_fetch *spf = arg;
+	void *ret = NULL;
+
+	if (!spf->quiet)
+		puts(spf->sb.buf);
+
+	spf->cp.argv = spf->argv.argv;
+
+	if (run_command(&spf->cp))
+		ret = (void *)1;
+
+	strbuf_release(&spf->cp);
+	argv_array_clear(spf->argv);
+	free(spf);
+	return ret;
+}
+
 int fetch_populated_submodules(const struct argv_array *options,
 			       const char *prefix, int command_line_option,
-			       int quiet)
+			       int quiet, int max_parallel_jobs)
 {
 	int i, result = 0;
-	struct child_process cp = CHILD_PROCESS_INIT;
+	struct workdispatcher *wd;
+	struct return_values *rv;
 	struct argv_array argv = ARGV_ARRAY_INIT;
+	struct submodule_parallel_fetch *spf;
 	const char *name_for_path;
 	const char *work_tree = get_git_work_tree();
 	if (!work_tree)
@@ -717,12 +754,9 @@ int fetch_populated_submodules(const struct argv_array *options,
 	argv_array_push(&argv, "--recurse-submodules-default");
 	/* default value, "--submodule-prefix" and its value are added later */
 
-	cp.env = local_repo_env;
-	cp.git_cmd = 1;
-	cp.no_stdin = 1;
-
 	calculate_changed_submodule_paths();
 
+	wd = create_workdispatcher(&run_command_and_cleanup, max_parallel_jobs);
 	for (i = 0; i < active_nr; i++) {
 		struct strbuf submodule_path = STRBUF_INIT;
 		struct strbuf submodule_git_dir = STRBUF_INIT;
@@ -771,24 +805,30 @@ int fetch_populated_submodules(const struct argv_array *options,
 		if (!git_dir)
 			git_dir = submodule_git_dir.buf;
 		if (is_directory(git_dir)) {
+			spf = xmalloc(sizeof(*spf));
+			submodule_parallel_fetch_init(spf);
+			spf->cp.env = local_repo_env;
+			spf->cp.git_cmd = 1;
+			spf->cp.no_stdin = 1;
+			spf->cp.dir = strbuf_detach(&submodule_path, NULL);
+			spf->quiet = quiet;
 			if (!quiet)
-				printf("Fetching submodule %s%s\n", prefix, ce->name);
-			cp.dir = submodule_path.buf;
-			argv_array_push(&argv, default_argv);
-			argv_array_push(&argv, "--submodule-prefix");
-			argv_array_push(&argv, submodule_prefix.buf);
-			cp.argv = argv.argv;
-			if (run_command(&cp))
-				result = 1;
-			argv_array_pop(&argv);
-			argv_array_pop(&argv);
-			argv_array_pop(&argv);
+				strbuf_addf(&spf->sb, "Fetching submodule %s%s", prefix, ce->name);
+			argv_array_copy(&argv, &spf->argv);
+			argv_array_push(&spf->argv, default_argv);
+			argv_array_push(&spf->argv, "--submodule-prefix");
+			argv_array_push(&spf->argv, submodule_prefix.buf);
+			add_task(wd, spf);
 		}
 		strbuf_release(&submodule_path);
 		strbuf_release(&submodule_git_dir);
 		strbuf_release(&submodule_prefix);
 	}
 	argv_array_clear(&argv);
+	rv = wait_workdispatcher(wd);
+	for (i = 0; i < rv->count; i++)
+		if (rv->ret[i])
+			result = 1;
 out:
 	string_list_clear(&changed_submodule_paths, 1);
 	return result;
diff --git a/submodule.h b/submodule.h
index e3dd854..51195ea 100644
--- a/submodule.h
+++ b/submodule.h
@@ -31,7 +31,7 @@ void set_config_fetch_recurse_submodules(int value);
 void check_for_new_submodule_commits(unsigned char new_sha1[20]);
 int fetch_populated_submodules(const struct argv_array *options,
 			       const char *prefix, int command_line_option,
-			       int quiet);
+			       int quiet, int max_parallel_jobs);
 unsigned is_submodule_modified(const char *path, int ignore_untracked);
 int submodule_uses_gitfile(const char *path);
 int ok_to_remove_submodule(const char *path);
-- 
2.5.0.239.g9728e1d.dirty