* Re: [OE-core] Multi-threaded npm build
2020-12-02 16:25 Multi-threaded npm build michal.frynas
@ 2020-12-02 17:54 ` Randy MacLeod
2020-12-02 19:01 ` Marek Belisko
2020-12-02 19:08 ` Jean-Marie Lemetayer
2 siblings, 0 replies; 4+ messages in thread
From: Randy MacLeod @ 2020-12-02 17:54 UTC (permalink / raw)
To: michal.frynas, openembedded-core
[-- Attachment #1: Type: text/plain, Size: 3194 bytes --]
On 2020-12-02 11:25 a.m., michal.frynas@sigma.se wrote:
> Task do_configure delivered by npm.bbclass takes a lot of time, the
> longer shrinkwrap dependency list the longer time needed for
> do_configure to execute.
> Following improvement allows to execute do_configure on any available
> CPI threads.
Thanks for the patch Michal.
Can you resend to:
bitbake-devel@lists.openembedded.org
using git send-email as per:
https://www.openembedded.org/wiki/How_to_submit_a_patch_to_OpenEmbedded
Thanks,
../Randy
> diff --git a/bitbake/lib/bb/fetch2/npmsw.py
> b/bitbake/lib/bb/fetch2/npmsw.py
> index 0c3511d8ab..0e6075859c 100644
> --- a/bitbake/lib/bb/fetch2/npmsw.py
> +++ b/bitbake/lib/bb/fetch2/npmsw.py
> @@ -29,8 +29,9 @@ from bb.fetch2.npm import npm_integrity
> from bb.fetch2.npm import npm_localfile
> from bb.fetch2.npm import npm_unpack
> from bb.utils import is_semver
> +from concurrent.futures import ThreadPoolExecutor
> -def foreach_dependencies(shrinkwrap, callback=None, dev=False):
> +def foreach_dependencies(shrinkwrap, callback=None, dev=False,
> executor=None):
> """
> Run a callback for each dependencies of a shrinkwrap file.
> The callback is using the format:
> @@ -49,7 +50,10 @@ def foreach_dependencies(shrinkwrap,
> callback=None, dev=False):
> continue
> elif deps[name].get("bundled", False):
> continue
> - callback(name, deps[name], subtree)
> + if executor:
> + executor.submit(callback, name=name,
> params=deps[name], deptree=subtree)
> + else:
> + callback(name, deps[name], subtree)
> _walk_deps(shrinkwrap.get("dependencies", {}), [])
> diff --git a/meta/classes/npm.bbclass b/meta/classes/npm.bbclass
> index 068032a1e5..1869b63cdb 100644
> --- a/meta/classes/npm.bbclass
> +++ b/meta/classes/npm.bbclass
> @@ -80,6 +80,7 @@ python npm_do_configure() {
> from bb.fetch2.npm import npm_unpack
> from bb.fetch2.npmsw import foreach_dependencies
> from bb.progress import OutOfProgressHandler
> + from concurrent.futures import ThreadPoolExecutor
> bb.utils.remove(d.getVar("NPM_CACHE"), recurse=True)
> bb.utils.remove(d.getVar("NPM_PACKAGE"), recurse=True)
> @@ -164,9 +165,11 @@ python npm_do_configure() {
> progress_done += 1
> progress.write("%d/%d" % (progress_done, progress_total))
> + executor = ThreadPoolExecutor(max_workers = os.cpu_count())
> dev = bb.utils.to_boolean(d.getVar("NPM_INSTALL_DEV"), False)
> foreach_dependencies(orig_shrinkwrap, _count_dependency, dev)
> - foreach_dependencies(orig_shrinkwrap, _cache_dependency, dev)
> + foreach_dependencies(orig_shrinkwrap, _cache_dependency, dev,
> executor)
> + executor.shutdown(True)
> # Configure the main package
> with tempfile.TemporaryDirectory() as tmpdir:
>
>
>
>
--
# Randy MacLeod
# Wind River Linux
[-- Attachment #2: Type: text/html, Size: 20526 bytes --]
^ permalink raw reply [flat|nested] 4+ messages in thread* Re: [OE-core] Multi-threaded npm build
2020-12-02 16:25 Multi-threaded npm build michal.frynas
2020-12-02 17:54 ` [OE-core] " Randy MacLeod
@ 2020-12-02 19:01 ` Marek Belisko
2020-12-02 19:08 ` Jean-Marie Lemetayer
2 siblings, 0 replies; 4+ messages in thread
From: Marek Belisko @ 2020-12-02 19:01 UTC (permalink / raw)
To: michal.frynas; +Cc: Patches and discussions about the oe-core layer
Hi Michal,
On Wed, Dec 2, 2020 at 5:25 PM <michal.frynas@sigma.se> wrote:
>
> Task do_configure delivered by npm.bbclass takes a lot of time, the longer shrinkwrap dependency list the longer time needed for do_configure to execute.
> Following improvement allows to execute do_configure on any available CPI threads.
Do you have some number how fast it is using with and without your
patch? I'm currently hitting the same issue with the slow npm
configure phase.
>
> diff --git a/bitbake/lib/bb/fetch2/npmsw.py b/bitbake/lib/bb/fetch2/npmsw.py
> index 0c3511d8ab..0e6075859c 100644
> --- a/bitbake/lib/bb/fetch2/npmsw.py
> +++ b/bitbake/lib/bb/fetch2/npmsw.py
> @@ -29,8 +29,9 @@ from bb.fetch2.npm import npm_integrity
> from bb.fetch2.npm import npm_localfile
> from bb.fetch2.npm import npm_unpack
> from bb.utils import is_semver
> +from concurrent.futures import ThreadPoolExecutor
>
> -def foreach_dependencies(shrinkwrap, callback=None, dev=False):
> +def foreach_dependencies(shrinkwrap, callback=None, dev=False, executor=None):
> """
> Run a callback for each dependencies of a shrinkwrap file.
> The callback is using the format:
> @@ -49,7 +50,10 @@ def foreach_dependencies(shrinkwrap, callback=None, dev=False):
> continue
> elif deps[name].get("bundled", False):
> continue
> - callback(name, deps[name], subtree)
> + if executor:
> + executor.submit(callback, name=name, params=deps[name], deptree=subtree)
> + else:
> + callback(name, deps[name], subtree)
>
> _walk_deps(shrinkwrap.get("dependencies", {}), [])
>
> diff --git a/meta/classes/npm.bbclass b/meta/classes/npm.bbclass
> index 068032a1e5..1869b63cdb 100644
> --- a/meta/classes/npm.bbclass
> +++ b/meta/classes/npm.bbclass
> @@ -80,6 +80,7 @@ python npm_do_configure() {
> from bb.fetch2.npm import npm_unpack
> from bb.fetch2.npmsw import foreach_dependencies
> from bb.progress import OutOfProgressHandler
> + from concurrent.futures import ThreadPoolExecutor
>
> bb.utils.remove(d.getVar("NPM_CACHE"), recurse=True)
> bb.utils.remove(d.getVar("NPM_PACKAGE"), recurse=True)
> @@ -164,9 +165,11 @@ python npm_do_configure() {
> progress_done += 1
> progress.write("%d/%d" % (progress_done, progress_total))
>
> + executor = ThreadPoolExecutor(max_workers = os.cpu_count())
> dev = bb.utils.to_boolean(d.getVar("NPM_INSTALL_DEV"), False)
> foreach_dependencies(orig_shrinkwrap, _count_dependency, dev)
> - foreach_dependencies(orig_shrinkwrap, _cache_dependency, dev)
> + foreach_dependencies(orig_shrinkwrap, _cache_dependency, dev, executor)
> + executor.shutdown(True)
>
> # Configure the main package
> with tempfile.TemporaryDirectory() as tmpdir:
>
>
>
>
>
>
BR,
marek
^ permalink raw reply [flat|nested] 4+ messages in thread* Re: [OE-core] Multi-threaded npm build
2020-12-02 16:25 Multi-threaded npm build michal.frynas
2020-12-02 17:54 ` [OE-core] " Randy MacLeod
2020-12-02 19:01 ` Marek Belisko
@ 2020-12-02 19:08 ` Jean-Marie Lemetayer
2 siblings, 0 replies; 4+ messages in thread
From: Jean-Marie Lemetayer @ 2020-12-02 19:08 UTC (permalink / raw)
To: michal.frynas; +Cc: openembedded-core
[-- Attachment #1: Type: text/plain, Size: 3237 bytes --]
Hi Michal,
Thanks, this is definitely helpful.
Have you checked if your improvement is tested. I think it should be tested
by the test_devtool_add_npm under meta/lib/oeqa/selftest/cases/devtool.py
See:
- https://wiki.yoctoproject.org/wiki/Oe-selftest
-
https://git.yoctoproject.org/cgit/cgit.cgi/poky/commit/?id=22dd46cc34629d0750177fddff2e1c178c854340
Best Regards,
Jean-Marie
On Wed, Dec 2, 2020 at 5:26 PM <michal.frynas@sigma.se> wrote:
> Task do_configure delivered by npm.bbclass takes a lot of time, the longer
> shrinkwrap dependency list the longer time needed for do_configure to
> execute.
> Following improvement allows to execute do_configure on any available CPI
> threads.
>
> diff --git a/bitbake/lib/bb/fetch2/npmsw.py
> b/bitbake/lib/bb/fetch2/npmsw.py
> index 0c3511d8ab..0e6075859c 100644
> --- a/bitbake/lib/bb/fetch2/npmsw.py
> +++ b/bitbake/lib/bb/fetch2/npmsw.py
> @@ -29,8 +29,9 @@ from bb.fetch2.npm import npm_integrity
> from bb.fetch2.npm import npm_localfile
> from bb.fetch2.npm import npm_unpack
> from bb.utils import is_semver
> +from concurrent.futures import ThreadPoolExecutor
>
> -def foreach_dependencies(shrinkwrap, callback=None, dev=False):
> +def foreach_dependencies(shrinkwrap, callback=None, dev=False,
> executor=None):
> """
> Run a callback for each dependencies of a shrinkwrap file.
> The callback is using the format:
> @@ -49,7 +50,10 @@ def foreach_dependencies(shrinkwrap, callback=None,
> dev=False):
> continue
> elif deps[name].get("bundled", False):
> continue
> - callback(name, deps[name], subtree)
> + if executor:
> + executor.submit(callback, name=name,
> params=deps[name], deptree=subtree)
> + else:
> + callback(name, deps[name], subtree)
>
> _walk_deps(shrinkwrap.get("dependencies", {}), [])
>
> diff --git a/meta/classes/npm.bbclass b/meta/classes/npm.bbclass
> index 068032a1e5..1869b63cdb 100644
> --- a/meta/classes/npm.bbclass
> +++ b/meta/classes/npm.bbclass
> @@ -80,6 +80,7 @@ python npm_do_configure() {
> from bb.fetch2.npm import npm_unpack
> from bb.fetch2.npmsw import foreach_dependencies
> from bb.progress import OutOfProgressHandler
> + from concurrent.futures import ThreadPoolExecutor
>
> bb.utils.remove(d.getVar("NPM_CACHE"), recurse=True)
> bb.utils.remove(d.getVar("NPM_PACKAGE"), recurse=True)
> @@ -164,9 +165,11 @@ python npm_do_configure() {
> progress_done += 1
> progress.write("%d/%d" % (progress_done, progress_total))
>
> + executor = ThreadPoolExecutor(max_workers = os.cpu_count())
> dev = bb.utils.to_boolean(d.getVar("NPM_INSTALL_DEV"), False)
> foreach_dependencies(orig_shrinkwrap, _count_dependency, dev)
> - foreach_dependencies(orig_shrinkwrap, _cache_dependency, dev)
> + foreach_dependencies(orig_shrinkwrap, _cache_dependency, dev,
> executor)
> + executor.shutdown(True)
>
> # Configure the main package
> with tempfile.TemporaryDirectory() as tmpdir:
>
>
>
>
>
>
>
[-- Attachment #2: Type: text/html, Size: 14689 bytes --]
^ permalink raw reply [flat|nested] 4+ messages in thread