Re: [PATCH] utils.py: add parallel make helpers

[Joshua Watt <jpewhacker@gmail.com>]

[-- Attachment #1: Type: text/plain, Size: 7945 bytes --]

On Fri, 2018-02-16 at 21:14 +0100, Martin Jansa wrote:
> +            v = max(limit, v)
> 
> 
> this seems to be doing the opposite of what the limit was used for
> before:
> 
> 
> -                v = min(64, int(v))
> 

Hmm, you're right


On Mon, Feb 12, 2018 at 8:39 PM, Joshua Watt <jpewhacker@gmail.com> wrote:
The code to extract the integer number of parallel build threads and

construct a new argument from them has started to be copied in multiple

locations, so create two new helper utilities to aid recipes.


The first helper (parallel_make()) extracts the integer number of

parallel build threads from PARALLEL_MAKE. The second

(parallel_make_argument()) does the same and then puts the result back

into a format string, optionally clamping it to some maximum value.


Additionally, rework the oe-core recipes that were manually doing this

to use the new helper utilities.


Signed-off-by: Joshua Watt <JPEWhacker@gmail.com>

---

 meta/classes/go.bbclass              | 19 +---------------

 meta/classes/waf.bbclass             | 24 +-------------------

 meta/lib/oe/utils.py                 | 43 ++++++++++++++++++++++++++++++++++++

 meta/recipes-core/ovmf/ovmf_git.bb   |  2 +-

 meta/recipes-support/boost/boost.inc | 24 +-------------------

 5 files changed, 47 insertions(+), 65 deletions(-)


diff --git a/meta/classes/go.bbclass b/meta/classes/go.bbclass

index 09b01a84c37..7ecd8c92541 100644

--- a/meta/classes/go.bbclass

+++ b/meta/classes/go.bbclass

@@ -1,23 +1,6 @@

 inherit goarch ptest


-def get_go_parallel_make(d):

-    pm = (d.getVar('PARALLEL_MAKE') or '').split()

-    # look for '-j' and throw other options (e.g. '-l') away

-    # because they might have a different meaning in golang

-    while pm:

-        opt = pm.pop(0)

-        if opt == '-j':

-            v = pm.pop(0)

-        elif opt.startswith('-j'):

-            v = opt[2:].strip()

-        else:

-            continue

-

-        return '-p %d' % int(v)

-

-    return ""

-

-GO_PARALLEL_BUILD ?= "${@get_go_parallel_make(d)}"

+GO_PARALLEL_BUILD ?= "${@oe.utils.parallel_make_argument(d, '-p %d')}"


 GOROOT_class-native = "${STAGING_LIBDIR_NATIVE}/go"

 GOROOT_class-nativesdk = "${STAGING_DIR_TARGET}${libdir}/go"

diff --git a/meta/classes/waf.bbclass b/meta/classes/waf.bbclass

index bdbdc56767c..f9a851d0759 100644

--- a/meta/classes/waf.bbclass

+++ b/meta/classes/waf.bbclass

@@ -3,28 +3,6 @@ DISABLE_STATIC = ""


 EXTRA_OECONF_append = " ${PACKAGECONFIG_CONFARGS}"


-def get_waf_parallel_make(d):

-    pm = d.getVar('PARALLEL_MAKE')

-    if pm:

-        # look for '-j' and throw other options (e.g. '-l') away

-        # because they might have different meaning in bjam

-        pm = pm.split()

-        while pm:

-            v = None

-            opt = pm.pop(0)

-            if opt == '-j':

-                v = pm.pop(0)

-            elif opt.startswith('-j'):

-                v = opt[2:].strip()

-            else:

-                v = None

-

-            if v:

-                v = min(64, int(v))

-                return '-j' + str(v)

-

-    return ""

-

 python waf_preconfigure() {

     import subprocess

     from distutils.version import StrictVersion

@@ -47,7 +25,7 @@ waf_do_configure() {


 do_compile[progress] = "outof:^\[\s*(\d+)/\s*(\d+)\]\s+"

 waf_do_compile()  {

-       ${S}/waf build ${@get_waf_parallel_make(d)}

+       ${S}/waf build ${@oe.utils.parallel_make_argument(d, '-j%d', limit=64)}

 }


 waf_do_install() {

diff --git a/meta/lib/oe/utils.py b/meta/lib/oe/utils.py

index 7a79d752b69..ec91927233a 100644

--- a/meta/lib/oe/utils.py

+++ b/meta/lib/oe/utils.py

@@ -156,6 +156,49 @@ def any_distro_features(d, features, truevalue="1", falsevalue=""):

     """

     return bb.utils.contains_any("DISTRO_FEATURES", features, truevalue, falsevalue, d)


+def parallel_make(d):

+    """

+    Return the integer value for the number of parallel threads to use when

+    building, scraped out of PARALLEL_MAKE. If no parallelization option is

+    found, returns None

+

+    e.g. if PARALLEL_MAKE = "-j 10", this will return 10 as an integer.

+    """

+    pm = (d.getVar('PARALLEL_MAKE') or '').split()

+    # look for '-j' and throw other options (e.g. '-l') away

+    while pm:

+        opt = pm.pop(0)

+        if opt == '-j':

+            v = pm.pop(0)

+        elif opt.startswith('-j'):

+            v = opt[2:].strip()

+        else:

+            continue

+

+        return int(v)

+

+    return None

+

+def parallel_make_argument(d, fmt, limit=None):

+    """

+    Helper utility to construct a parallel make argument from the number of

+    parallel threads specified in PARALLEL_MAKE.

+

+    Returns the input format string `fmt` where a single '%d' will be expanded

+    with the number of parallel threads to use. If `limit` is specified, the

+    number of parallel threads will be no larger than it. If no parallelization

+    option is found in PARALLEL_MAKE, returns an empty string

+

+    e.g. if PARALLEL_MAKE = "-j 10", parallel_make_argument(d, "-n %d") will return

+    "-n 10"

+    """

+    v = parallel_make(d)

+    if v:

+        if limit:

+            v = max(limit, v)

+        return fmt % v

+    return ''

+

 def packages_filter_out_system(d):

     """

     Return a list of packages from PACKAGES with the "system" packages such as

diff --git a/meta/recipes-core/ovmf/ovmf_git.bb b/meta/recipes-core/ovmf/ovmf_git.bb

index fa0d66291d1..8750b3c528d 100644

--- a/meta/recipes-core/ovmf/ovmf_git.bb

+++ b/meta/recipes-core/ovmf/ovmf_git.bb

@@ -151,7 +151,7 @@ do_compile_class-native() {


 do_compile_class-target() {

     export LFLAGS="${LDFLAGS}"

-    PARALLEL_JOBS="${@ '${PARALLEL_MAKE}'.replace('-j', '-n ')}"

+    PARALLEL_JOBS="${@oe.utils.parallel_make_argument(d, '-n %d')}"

     OVMF_ARCH="X64"

     if [ "${TARGET_ARCH}" != "x86_64" ] ; then

         OVMF_ARCH="IA32"

diff --git a/meta/recipes-support/boost/boost.inc b/meta/recipes-support/boost/boost.inc

index 41fc90fb211..0461ec6fcf1 100644

--- a/meta/recipes-support/boost/boost.inc

+++ b/meta/recipes-support/boost/boost.inc

@@ -135,29 +135,7 @@ BJAM_TOOLS   = "--ignore-site-config \


 # use PARALLEL_MAKE to speed up the build, but limit it by -j 64, greater parallelism causes bjam to segfault or to ignore -j

 # https://svn.boost.org/trac/boost/ticket/7634

-def get_boost_parallel_make(d):

-    pm = d.getVar('PARALLEL_MAKE')

-    if pm:

-        # look for '-j' and throw other options (e.g. '-l') away

-        # because they might have different meaning in bjam

-        pm = pm.split()

-        while pm:

-            v = None

-            opt = pm.pop(0)

-            if opt == '-j':

-                v = pm.pop(0)

-            elif opt.startswith('-j'):

-                v = opt[2:].strip()

-            else:

-                v = None

-

-            if v:

-                v = min(64, int(v))

-                return '-j' + str(v)

-

-    return ""

-

-BOOST_PARALLEL_MAKE = "${@get_boost_parallel_make(d)}"

+BOOST_PARALLEL_MAKE = "${@oe.utils.parallel_make_argument(d, '-j%d', limit=64)}"

 BJAM_OPTS    = '${BOOST_PARALLEL_MAKE} -d+2 -q \

                ${BJAM_TOOLS} \

                -sBOOST_BUILD_USER_CONFIG=${WORKDIR}/user-config.jam \
--

2.14.3


--

_______________________________________________

Openembedded-core mailing list
Openembedded-core@lists.openembedded.org
http://lists.openembedded.org/mailman/listinfo/openembedded-core

[-- Attachment #2: Type: text/html, Size: 14616 bytes --]