Multi-threaded npm build

[michal.frynas@sigma.se]

[-- Attachment #1: Type: text/plain, Size: 2643 bytes --]

Task do_configure delivered by npm.bbclass takes a lot of time, the longer shrinkwrap dependency list the longer time needed for do_configure to execute.
Following improvement allows to execute do_configure on any available CPI threads.

> 
> diff --git a/bitbake/lib/bb/fetch2/npmsw.py
> b/bitbake/lib/bb/fetch2/npmsw.py
> index 0c3511d8ab..0e6075859c 100644
> --- a/bitbake/lib/bb/fetch2/npmsw.py
> +++ b/bitbake/lib/bb/fetch2/npmsw.py
> @@ -29,8 +29,9 @@ from bb.fetch2.npm import npm_integrity
> from bb.fetch2.npm import npm_localfile
> from bb.fetch2.npm import npm_unpack
> from bb.utils import is_semver
> +from concurrent.futures import ThreadPoolExecutor
> 
> -def foreach_dependencies(shrinkwrap, callback=None, dev=False):
> +def foreach_dependencies(shrinkwrap, callback=None, dev=False,
> executor=None):
> """
> Run a callback for each dependencies of a shrinkwrap file.
> The callback is using the format:
> @@ -49,7 +50,10 @@ def foreach_dependencies(shrinkwrap, callback=None,
> dev=False):
> continue
> elif deps[name].get("bundled", False):
> continue
> -                callback(name, deps[name], subtree)
> +                if executor:
> +                    executor.submit(callback, name=name,
> params=deps[name], deptree=subtree)
> +                else:
> +                    callback(name, deps[name], subtree)
> 
> _walk_deps(shrinkwrap.get("dependencies", {}), [])
> 
> diff --git a/meta/classes/npm.bbclass b/meta/classes/npm.bbclass
> index 068032a1e5..1869b63cdb 100644
> --- a/meta/classes/npm.bbclass
> +++ b/meta/classes/npm.bbclass
> @@ -80,6 +80,7 @@ python npm_do_configure() {
> from bb.fetch2.npm import npm_unpack
> from bb.fetch2.npmsw import foreach_dependencies
> from bb.progress import OutOfProgressHandler
> +    from concurrent.futures import ThreadPoolExecutor
> 
> bb.utils.remove(d.getVar("NPM_CACHE"), recurse=True)
> bb.utils.remove(d.getVar("NPM_PACKAGE"), recurse=True)
> @@ -164,9 +165,11 @@ python npm_do_configure() {
> progress_done += 1
> progress.write("%d/%d" % (progress_done, progress_total))
> 
> +    executor = ThreadPoolExecutor(max_workers = os.cpu_count())
> dev = bb.utils.to_boolean(d.getVar("NPM_INSTALL_DEV"), False)
> foreach_dependencies(orig_shrinkwrap, _count_dependency, dev)
> -    foreach_dependencies(orig_shrinkwrap, _cache_dependency, dev)
> +    foreach_dependencies(orig_shrinkwrap, _cache_dependency, dev,
> executor)
> +    executor.shutdown(True)
> 
> # Configure the main package
> with tempfile.TemporaryDirectory() as tmpdir:
> 
>

[-- Attachment #2: Type: text/html, Size: 15362 bytes --]