From mboxrd@z Thu Jan 1 00:00:00 1970 Subject: Multi-threaded npm build To: openembedded-core@lists.openembedded.org From: michal.frynas@sigma.se X-Originating-Location: Wrocław, Lower Silesia, PL (78.8.146.236) X-Originating-Platform: Linux Chrome 87 User-Agent: GROUPS.IO Web Poster MIME-Version: 1.0 Date: Wed, 02 Dec 2020 08:25:56 -0800 Message-ID: Content-Type: multipart/alternative; boundary="an0uumEmsZBP97ZybBqF" --an0uumEmsZBP97ZybBqF Content-Type: text/plain; charset="utf-8" Content-Transfer-Encoding: quoted-printable Task do_configure delivered by npm.bbclass takes a lot of time, the longer = shrinkwrap dependency list the longer time needed for do_configure to execu= te. Following improvement allows to execute do_configure on any available CPI = threads. >=20 > diff --git a/bitbake/lib/bb/fetch2/npmsw.py > b/bitbake/lib/bb/fetch2/npmsw.py > index 0c3511d8ab..0e6075859c 100644 > --- a/bitbake/lib/bb/fetch2/npmsw.py > +++ b/bitbake/lib/bb/fetch2/npmsw.py > @@ -29,8 +29,9 @@ from bb.fetch2.npm import npm_integrity > from bb.fetch2.npm import npm_localfile > from bb.fetch2.npm import npm_unpack > from bb.utils import is_semver > +from concurrent.futures import ThreadPoolExecutor >=20 > -def foreach_dependencies(shrinkwrap, callback=3DNone, dev=3DFalse): > +def foreach_dependencies(shrinkwrap, callback=3DNone, dev=3DFalse, > executor=3DNone): > """ > Run a callback for each dependencies of a shrinkwrap file. > The callback is using the format: > @@ -49,7 +50,10 @@ def foreach_dependencies(shrinkwrap, callback=3DNone, > dev=3DFalse): > continue > elif deps[name].get("bundled", False): > continue > -=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 callback(name, = deps[name], subtree) > +=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 if executor: > +=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 e= xecutor.submit(callback, name=3Dname, > params=3Ddeps[name], deptree=3Dsubtree) > +=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 else: > +=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 c= allback(name, deps[name], subtree) >=20 > _walk_deps(shrinkwrap.get("dependencies", {}), []) >=20 > diff --git a/meta/classes/npm.bbclass b/meta/classes/npm.bbclass > index 068032a1e5..1869b63cdb 100644 > --- a/meta/classes/npm.bbclass > +++ b/meta/classes/npm.bbclass > @@ -80,6 +80,7 @@ python npm_do_configure() { > from bb.fetch2.npm import npm_unpack > from bb.fetch2.npmsw import foreach_dependencies > from bb.progress import OutOfProgressHandler > +=C2=A0 =C2=A0 from concurrent.futures import ThreadPoolExecutor >=20 > bb.utils.remove(d.getVar("NPM_CACHE"), recurse=3DTrue) > bb.utils.remove(d.getVar("NPM_PACKAGE"), recurse=3DTrue) > @@ -164,9 +165,11 @@ python npm_do_configure() { > progress_done +=3D 1 > progress.write("%d/%d" % (progress_done, progress_total)) >=20 > +=C2=A0 =C2=A0 executor =3D ThreadPoolExecutor(max_workers =3D os.cpu_co= unt()) > dev =3D bb.utils.to_boolean(d.getVar("NPM_INSTALL_DEV"), False) > foreach_dependencies(orig_shrinkwrap, _count_dependency, dev) > -=C2=A0 =C2=A0 foreach_dependencies(orig_shrinkwrap, _cache_dependency, = dev) > +=C2=A0 =C2=A0 foreach_dependencies(orig_shrinkwrap, _cache_dependency, = dev, > executor) > +=C2=A0 =C2=A0 executor.shutdown(True) >=20 > # Configure the main package > with tempfile.TemporaryDirectory() as tmpdir: >=20 > --an0uumEmsZBP97ZybBqF Content-Type: text/html; charset="utf-8" Content-Transfer-Encoding: quoted-printable
Task do_configure deliv= ered by npm.bbclass takes a lot of time, the longer shrinkwrap dependency l= ist the longer time needed for do_configure to execute.
Following improvement = allows to execute do_configure on any available CPI threads.
diff --git a/bi= tbake/lib/bb/fetch2/npmsw.py b/bitbake/lib/bb/fetch2/npmsw.py
index 0c3511d8a= b..0e6075859c 100644
--- a/bitbake/l= ib/bb/fetch2/npmsw.py
+++ b/bitbake/l= ib/bb/fetch2/npmsw.py
@@ -29,8 +29,9 = @@ from bb.fetch2.npm import npm_integrity
 from bb.f= etch2.npm import npm_localfile
 from bb.f= etch2.npm import npm_unpack
 from bb.u= tils import is_semver
+from concurren= t.futures import ThreadPoolExecutor
 
-def foreach_de= pendencies(shrinkwrap, callback=3DNone, dev=3DFalse):
+def foreach_de= pendencies(shrinkwrap, callback=3DNone, dev=3DFalse, executor=3DNone):
    &= nbsp;"""
    &= nbsp;    Run a callback for each dependencies of a shrinkwrap fil= e.
    &= nbsp;    The callback is using the format:
@@ -49,7 +50,10= @@ def foreach_dependencies(shrinkwrap, callback=3DNone, dev=3DFalse):
    &= nbsp;                continue
    &= nbsp;            elif deps[name].get("bundled= ", False):
    &= nbsp;                continue
-    =             callback(name, deps[name], subtre= e)
+    =             if executor:
+    =                 executor.submit(cal= lback, name=3Dname, params=3Ddeps[name], deptree=3Dsubtree)
+    =             else:
+    =                 callback(name, deps= [name], subtree)
 
    &= nbsp;_walk_deps(shrinkwrap.get("dependencies", {}), [])
 
diff --git a/me= ta/classes/npm.bbclass b/meta/classes/npm.bbclass
index 068032a1e= 5..1869b63cdb 100644
--- a/meta/clas= ses/npm.bbclass
+++ b/meta/clas= ses/npm.bbclass
@@ -80,6 +80,7 = @@ python npm_do_configure() {
    &= nbsp;from bb.fetch2.npm import npm_unpack
    &= nbsp;from bb.fetch2.npmsw import foreach_dependencies
    &= nbsp;from bb.progress import OutOfProgressHandler
+    = from concurrent.futures import ThreadPoolExecutor
 
    &= nbsp;bb.utils.remove(d.getVar("NPM_CACHE"), recurse=3DTrue)
    &= nbsp;bb.utils.remove(d.getVar("NPM_PACKAGE"), recurse=3DTrue)
@@ -164,9 +165,= 11 @@ python npm_do_configure() {
    &= nbsp;        progress_done +=3D 1
    &= nbsp;        progress.write("%d/%d" % (progress_done, p= rogress_total))
 
+    = executor =3D ThreadPoolExecutor(max_workers =3D os.cpu_count())
    &= nbsp;dev =3D bb.utils.to_boolean(d.getVar("NPM_INSTALL_DEV"), False)
    &= nbsp;foreach_dependencies(orig_shrinkwrap, _count_dependency, dev)
-    = foreach_dependencies(orig_shrinkwrap, _cache_dependency, dev)
+    = foreach_dependencies(orig_shrinkwrap, _cache_dependency, dev, executor)
+    = executor.shutdown(True)
 
    &= nbsp;# Configure the main package
    &= nbsp;with tempfile.TemporaryDirectory() as tmpdir:
 
 
--an0uumEmsZBP97ZybBqF--