Re: [PATCHv3 08/11] fetching submodules: respect `submodule.jobs` config option

[Jens Lehmann <Jens.Lehmann@web.de>]

Am 04.11.2015 um 01:37 schrieb Stefan Beller:
> This allows to configure fetching and updating in parallel
> without having the command line option.
>
> This moved the responsibility to determine how many parallel processes
> to start from builtin/fetch to submodule.c as we need a way to communicate
> "The user did not specify the number of parallel processes in the command
> line options" in the builtin fetch. The submodule code takes care of
> the precedence (CLI > config > default)
>
> Signed-off-by: Stefan Beller <sbeller@google.com>
> ---
>   Documentation/config.txt    |  7 +++++++
>   builtin/fetch.c             |  2 +-
>   submodule-config.c          | 15 +++++++++++++++
>   submodule-config.h          |  2 ++
>   submodule.c                 |  5 +++++
>   t/t5526-fetch-submodules.sh | 14 ++++++++++++++
>   6 files changed, 44 insertions(+), 1 deletion(-)
>
> diff --git a/Documentation/config.txt b/Documentation/config.txt
> index 391a0c3..70e1b88 100644
> --- a/Documentation/config.txt
> +++ b/Documentation/config.txt
> @@ -2643,6 +2643,13 @@ submodule.<name>.ignore::
>   	"--ignore-submodules" option. The 'git submodule' commands are not
>   	affected by this setting.
>
> +submodule.jobs::
> +	This is used to determine how many submodules can be operated on in
> +	parallel. Specifying a positive integer allows up to that number
> +	of submodules being fetched in parallel. This is used in fetch
> +	and clone operations only. A value of 0 will give some reasonable
> +	configuration. It defaults to 1.
> +

Just curious (and sorry if this has already been discussed and I missed
it, but the volume of your output is too much for my current git time
budget ;-): While this config is for fetching only, do I recall correctly
that you have plans to do submodule work tree updates in parallel too?
If so, would it make sense to have different settings for fetching and
updating?

>   tag.sort::
>   	This variable controls the sort ordering of tags when displayed by
>   	linkgit:git-tag[1]. Without the "--sort=<value>" option provided, the
> diff --git a/builtin/fetch.c b/builtin/fetch.c
> index 9cc1c9d..60e6797 100644
> --- a/builtin/fetch.c
> +++ b/builtin/fetch.c
> @@ -37,7 +37,7 @@ static int prune = -1; /* unspecified */
>   static int all, append, dry_run, force, keep, multiple, update_head_ok, verbosity;
>   static int progress = -1, recurse_submodules = RECURSE_SUBMODULES_DEFAULT;
>   static int tags = TAGS_DEFAULT, unshallow, update_shallow;
> -static int max_children = 1;
> +static int max_children = -1;
>   static const char *depth;
>   static const char *upload_pack;
>   static struct strbuf default_rla = STRBUF_INIT;
> diff --git a/submodule-config.c b/submodule-config.c
> index 29e21b2..475551a 100644
> --- a/submodule-config.c
> +++ b/submodule-config.c
> @@ -32,6 +32,7 @@ enum lookup_type {
>
>   static struct submodule_cache cache;
>   static int is_cache_init;
> +static int parallel_jobs = -1;
>
>   static int config_path_cmp(const struct submodule_entry *a,
>   			   const struct submodule_entry *b,
> @@ -239,6 +240,15 @@ static int parse_generic_submodule_config(const char *key,
>   					  const char *value,
>   					  struct parse_config_parameter *me)
>   {
> +	if (!strcmp(key, "jobs")) {
> +		parallel_jobs = strtol(value, NULL, 10);
> +		if (parallel_jobs < 0) {
> +			warning("submodule.jobs not allowed to be negative.");
> +			parallel_jobs = 1;
> +			return 1;
> +		}
> +	}
> +
>   	return 0;
>   }
>
> @@ -482,3 +492,8 @@ void submodule_free(void)
>   	cache_free(&cache);
>   	is_cache_init = 0;
>   }
> +
> +int config_parallel_submodules(void)
> +{
> +	return parallel_jobs;
> +}
> diff --git a/submodule-config.h b/submodule-config.h
> index f9e2a29..d9bbf9a 100644
> --- a/submodule-config.h
> +++ b/submodule-config.h
> @@ -27,4 +27,6 @@ const struct submodule *submodule_from_path(const unsigned char *commit_sha1,
>   		const char *path);
>   void submodule_free(void);
>
> +int config_parallel_submodules(void);
> +
>   #endif /* SUBMODULE_CONFIG_H */
> diff --git a/submodule.c b/submodule.c
> index 0257ea3..188ba02 100644
> --- a/submodule.c
> +++ b/submodule.c
> @@ -752,6 +752,11 @@ int fetch_populated_submodules(const struct argv_array *options,
>   	argv_array_push(&spf.args, "--recurse-submodules-default");
>   	/* default value, "--submodule-prefix" and its value are added later */
>
> +	if (max_parallel_jobs < 0)
> +		max_parallel_jobs = config_parallel_submodules();
> +	if (max_parallel_jobs < 0)
> +		max_parallel_jobs = 1;
> +
>   	calculate_changed_submodule_paths();
>   	run_processes_parallel(max_parallel_jobs,
>   			       get_next_submodule,
> diff --git a/t/t5526-fetch-submodules.sh b/t/t5526-fetch-submodules.sh
> index 1b4ce69..5c3579c 100755
> --- a/t/t5526-fetch-submodules.sh
> +++ b/t/t5526-fetch-submodules.sh
> @@ -470,4 +470,18 @@ test_expect_success "don't fetch submodule when newly recorded commits are alrea
>   	test_i18ncmp expect.err actual.err
>   '
>
> +test_expect_success 'fetching submodules respects parallel settings' '
> +	git config fetch.recurseSubmodules true &&
> +	(
> +		cd downstream &&
> +		GIT_TRACE=$(pwd)/trace.out git fetch --jobs 7 &&
> +		grep "7 children" trace.out &&
> +		git config submodule.jobs 8 &&
> +		GIT_TRACE=$(pwd)/trace.out git fetch &&
> +		grep "8 children" trace.out &&
> +		GIT_TRACE=$(pwd)/trace.out git fetch --jobs 9 &&
> +		grep "9 children" trace.out
> +	)
> +'
> +
>   test_done
>