* [PATCH] utils.py: add parallel make helpers
@ 2018-02-12 19:39 Joshua Watt
2018-02-16 20:14 ` Martin Jansa
0 siblings, 1 reply; 6+ messages in thread
From: Joshua Watt @ 2018-02-12 19:39 UTC (permalink / raw)
To: openembedded-core
The code to extract the integer number of parallel build threads and
construct a new argument from them has started to be copied in multiple
locations, so create two new helper utilities to aid recipes.
The first helper (parallel_make()) extracts the integer number of
parallel build threads from PARALLEL_MAKE. The second
(parallel_make_argument()) does the same and then puts the result back
into a format string, optionally clamping it to some maximum value.
Additionally, rework the oe-core recipes that were manually doing this
to use the new helper utilities.
Signed-off-by: Joshua Watt <JPEWhacker@gmail.com>
---
meta/classes/go.bbclass | 19 +---------------
meta/classes/waf.bbclass | 24 +-------------------
meta/lib/oe/utils.py | 43 ++++++++++++++++++++++++++++++++++++
meta/recipes-core/ovmf/ovmf_git.bb | 2 +-
meta/recipes-support/boost/boost.inc | 24 +-------------------
5 files changed, 47 insertions(+), 65 deletions(-)
diff --git a/meta/classes/go.bbclass b/meta/classes/go.bbclass
index 09b01a84c37..7ecd8c92541 100644
--- a/meta/classes/go.bbclass
+++ b/meta/classes/go.bbclass
@@ -1,23 +1,6 @@
inherit goarch ptest
-def get_go_parallel_make(d):
- pm = (d.getVar('PARALLEL_MAKE') or '').split()
- # look for '-j' and throw other options (e.g. '-l') away
- # because they might have a different meaning in golang
- while pm:
- opt = pm.pop(0)
- if opt == '-j':
- v = pm.pop(0)
- elif opt.startswith('-j'):
- v = opt[2:].strip()
- else:
- continue
-
- return '-p %d' % int(v)
-
- return ""
-
-GO_PARALLEL_BUILD ?= "${@get_go_parallel_make(d)}"
+GO_PARALLEL_BUILD ?= "${@oe.utils.parallel_make_argument(d, '-p %d')}"
GOROOT_class-native = "${STAGING_LIBDIR_NATIVE}/go"
GOROOT_class-nativesdk = "${STAGING_DIR_TARGET}${libdir}/go"
diff --git a/meta/classes/waf.bbclass b/meta/classes/waf.bbclass
index bdbdc56767c..f9a851d0759 100644
--- a/meta/classes/waf.bbclass
+++ b/meta/classes/waf.bbclass
@@ -3,28 +3,6 @@ DISABLE_STATIC = ""
EXTRA_OECONF_append = " ${PACKAGECONFIG_CONFARGS}"
-def get_waf_parallel_make(d):
- pm = d.getVar('PARALLEL_MAKE')
- if pm:
- # look for '-j' and throw other options (e.g. '-l') away
- # because they might have different meaning in bjam
- pm = pm.split()
- while pm:
- v = None
- opt = pm.pop(0)
- if opt == '-j':
- v = pm.pop(0)
- elif opt.startswith('-j'):
- v = opt[2:].strip()
- else:
- v = None
-
- if v:
- v = min(64, int(v))
- return '-j' + str(v)
-
- return ""
-
python waf_preconfigure() {
import subprocess
from distutils.version import StrictVersion
@@ -47,7 +25,7 @@ waf_do_configure() {
do_compile[progress] = "outof:^\[\s*(\d+)/\s*(\d+)\]\s+"
waf_do_compile() {
- ${S}/waf build ${@get_waf_parallel_make(d)}
+ ${S}/waf build ${@oe.utils.parallel_make_argument(d, '-j%d', limit=64)}
}
waf_do_install() {
diff --git a/meta/lib/oe/utils.py b/meta/lib/oe/utils.py
index 7a79d752b69..ec91927233a 100644
--- a/meta/lib/oe/utils.py
+++ b/meta/lib/oe/utils.py
@@ -156,6 +156,49 @@ def any_distro_features(d, features, truevalue="1", falsevalue=""):
"""
return bb.utils.contains_any("DISTRO_FEATURES", features, truevalue, falsevalue, d)
+def parallel_make(d):
+ """
+ Return the integer value for the number of parallel threads to use when
+ building, scraped out of PARALLEL_MAKE. If no parallelization option is
+ found, returns None
+
+ e.g. if PARALLEL_MAKE = "-j 10", this will return 10 as an integer.
+ """
+ pm = (d.getVar('PARALLEL_MAKE') or '').split()
+ # look for '-j' and throw other options (e.g. '-l') away
+ while pm:
+ opt = pm.pop(0)
+ if opt == '-j':
+ v = pm.pop(0)
+ elif opt.startswith('-j'):
+ v = opt[2:].strip()
+ else:
+ continue
+
+ return int(v)
+
+ return None
+
+def parallel_make_argument(d, fmt, limit=None):
+ """
+ Helper utility to construct a parallel make argument from the number of
+ parallel threads specified in PARALLEL_MAKE.
+
+ Returns the input format string `fmt` where a single '%d' will be expanded
+ with the number of parallel threads to use. If `limit` is specified, the
+ number of parallel threads will be no larger than it. If no parallelization
+ option is found in PARALLEL_MAKE, returns an empty string
+
+ e.g. if PARALLEL_MAKE = "-j 10", parallel_make_argument(d, "-n %d") will return
+ "-n 10"
+ """
+ v = parallel_make(d)
+ if v:
+ if limit:
+ v = max(limit, v)
+ return fmt % v
+ return ''
+
def packages_filter_out_system(d):
"""
Return a list of packages from PACKAGES with the "system" packages such as
diff --git a/meta/recipes-core/ovmf/ovmf_git.bb b/meta/recipes-core/ovmf/ovmf_git.bb
index fa0d66291d1..8750b3c528d 100644
--- a/meta/recipes-core/ovmf/ovmf_git.bb
+++ b/meta/recipes-core/ovmf/ovmf_git.bb
@@ -151,7 +151,7 @@ do_compile_class-native() {
do_compile_class-target() {
export LFLAGS="${LDFLAGS}"
- PARALLEL_JOBS="${@ '${PARALLEL_MAKE}'.replace('-j', '-n ')}"
+ PARALLEL_JOBS="${@oe.utils.parallel_make_argument(d, '-n %d')}"
OVMF_ARCH="X64"
if [ "${TARGET_ARCH}" != "x86_64" ] ; then
OVMF_ARCH="IA32"
diff --git a/meta/recipes-support/boost/boost.inc b/meta/recipes-support/boost/boost.inc
index 41fc90fb211..0461ec6fcf1 100644
--- a/meta/recipes-support/boost/boost.inc
+++ b/meta/recipes-support/boost/boost.inc
@@ -135,29 +135,7 @@ BJAM_TOOLS = "--ignore-site-config \
# use PARALLEL_MAKE to speed up the build, but limit it by -j 64, greater parallelism causes bjam to segfault or to ignore -j
# https://svn.boost.org/trac/boost/ticket/7634
-def get_boost_parallel_make(d):
- pm = d.getVar('PARALLEL_MAKE')
- if pm:
- # look for '-j' and throw other options (e.g. '-l') away
- # because they might have different meaning in bjam
- pm = pm.split()
- while pm:
- v = None
- opt = pm.pop(0)
- if opt == '-j':
- v = pm.pop(0)
- elif opt.startswith('-j'):
- v = opt[2:].strip()
- else:
- v = None
-
- if v:
- v = min(64, int(v))
- return '-j' + str(v)
-
- return ""
-
-BOOST_PARALLEL_MAKE = "${@get_boost_parallel_make(d)}"
+BOOST_PARALLEL_MAKE = "${@oe.utils.parallel_make_argument(d, '-j%d', limit=64)}"
BJAM_OPTS = '${BOOST_PARALLEL_MAKE} -d+2 -q \
${BJAM_TOOLS} \
-sBOOST_BUILD_USER_CONFIG=${WORKDIR}/user-config.jam \
--
2.14.3
^ permalink raw reply related [flat|nested] 6+ messages in thread
* Re: [PATCH] utils.py: add parallel make helpers
2018-02-12 19:39 [PATCH] utils.py: add parallel make helpers Joshua Watt
@ 2018-02-16 20:14 ` Martin Jansa
2018-02-16 20:23 ` Joshua Watt
2018-02-17 23:57 ` Khem Raj
0 siblings, 2 replies; 6+ messages in thread
From: Martin Jansa @ 2018-02-16 20:14 UTC (permalink / raw)
To: Joshua Watt; +Cc: Patches and discussions about the oe-core layer
[-- Attachment #1: Type: text/plain, Size: 7918 bytes --]
+ v = max(limit, v)
this seems to be doing the opposite of what the limit was used for before:
- v = min(64, int(v))
On Mon, Feb 12, 2018 at 8:39 PM, Joshua Watt <jpewhacker@gmail.com> wrote:
> The code to extract the integer number of parallel build threads and
> construct a new argument from them has started to be copied in multiple
> locations, so create two new helper utilities to aid recipes.
>
> The first helper (parallel_make()) extracts the integer number of
> parallel build threads from PARALLEL_MAKE. The second
> (parallel_make_argument()) does the same and then puts the result back
> into a format string, optionally clamping it to some maximum value.
>
> Additionally, rework the oe-core recipes that were manually doing this
> to use the new helper utilities.
>
> Signed-off-by: Joshua Watt <JPEWhacker@gmail.com>
> ---
> meta/classes/go.bbclass | 19 +---------------
> meta/classes/waf.bbclass | 24 +-------------------
> meta/lib/oe/utils.py | 43 ++++++++++++++++++++++++++++++
> ++++++
> meta/recipes-core/ovmf/ovmf_git.bb | 2 +-
> meta/recipes-support/boost/boost.inc | 24 +-------------------
> 5 files changed, 47 insertions(+), 65 deletions(-)
>
> diff --git a/meta/classes/go.bbclass b/meta/classes/go.bbclass
> index 09b01a84c37..7ecd8c92541 100644
> --- a/meta/classes/go.bbclass
> +++ b/meta/classes/go.bbclass
> @@ -1,23 +1,6 @@
> inherit goarch ptest
>
> -def get_go_parallel_make(d):
> - pm = (d.getVar('PARALLEL_MAKE') or '').split()
> - # look for '-j' and throw other options (e.g. '-l') away
> - # because they might have a different meaning in golang
> - while pm:
> - opt = pm.pop(0)
> - if opt == '-j':
> - v = pm.pop(0)
> - elif opt.startswith('-j'):
> - v = opt[2:].strip()
> - else:
> - continue
> -
> - return '-p %d' % int(v)
> -
> - return ""
> -
> -GO_PARALLEL_BUILD ?= "${@get_go_parallel_make(d)}"
> +GO_PARALLEL_BUILD ?= "${@oe.utils.parallel_make_argument(d, '-p %d')}"
>
> GOROOT_class-native = "${STAGING_LIBDIR_NATIVE}/go"
> GOROOT_class-nativesdk = "${STAGING_DIR_TARGET}${libdir}/go"
> diff --git a/meta/classes/waf.bbclass b/meta/classes/waf.bbclass
> index bdbdc56767c..f9a851d0759 100644
> --- a/meta/classes/waf.bbclass
> +++ b/meta/classes/waf.bbclass
> @@ -3,28 +3,6 @@ DISABLE_STATIC = ""
>
> EXTRA_OECONF_append = " ${PACKAGECONFIG_CONFARGS}"
>
> -def get_waf_parallel_make(d):
> - pm = d.getVar('PARALLEL_MAKE')
> - if pm:
> - # look for '-j' and throw other options (e.g. '-l') away
> - # because they might have different meaning in bjam
> - pm = pm.split()
> - while pm:
> - v = None
> - opt = pm.pop(0)
> - if opt == '-j':
> - v = pm.pop(0)
> - elif opt.startswith('-j'):
> - v = opt[2:].strip()
> - else:
> - v = None
> -
> - if v:
> - v = min(64, int(v))
> - return '-j' + str(v)
> -
> - return ""
> -
> python waf_preconfigure() {
> import subprocess
> from distutils.version import StrictVersion
> @@ -47,7 +25,7 @@ waf_do_configure() {
>
> do_compile[progress] = "outof:^\[\s*(\d+)/\s*(\d+)\]\s+"
> waf_do_compile() {
> - ${S}/waf build ${@get_waf_parallel_make(d)}
> + ${S}/waf build ${@oe.utils.parallel_make_argument(d, '-j%d',
> limit=64)}
> }
>
> waf_do_install() {
> diff --git a/meta/lib/oe/utils.py b/meta/lib/oe/utils.py
> index 7a79d752b69..ec91927233a 100644
> --- a/meta/lib/oe/utils.py
> +++ b/meta/lib/oe/utils.py
> @@ -156,6 +156,49 @@ def any_distro_features(d, features, truevalue="1",
> falsevalue=""):
> """
> return bb.utils.contains_any("DISTRO_FEATURES", features, truevalue,
> falsevalue, d)
>
> +def parallel_make(d):
> + """
> + Return the integer value for the number of parallel threads to use
> when
> + building, scraped out of PARALLEL_MAKE. If no parallelization option
> is
> + found, returns None
> +
> + e.g. if PARALLEL_MAKE = "-j 10", this will return 10 as an integer.
> + """
> + pm = (d.getVar('PARALLEL_MAKE') or '').split()
> + # look for '-j' and throw other options (e.g. '-l') away
> + while pm:
> + opt = pm.pop(0)
> + if opt == '-j':
> + v = pm.pop(0)
> + elif opt.startswith('-j'):
> + v = opt[2:].strip()
> + else:
> + continue
> +
> + return int(v)
> +
> + return None
> +
> +def parallel_make_argument(d, fmt, limit=None):
> + """
> + Helper utility to construct a parallel make argument from the number
> of
> + parallel threads specified in PARALLEL_MAKE.
> +
> + Returns the input format string `fmt` where a single '%d' will be
> expanded
> + with the number of parallel threads to use. If `limit` is specified,
> the
> + number of parallel threads will be no larger than it. If no
> parallelization
> + option is found in PARALLEL_MAKE, returns an empty string
> +
> + e.g. if PARALLEL_MAKE = "-j 10", parallel_make_argument(d, "-n %d")
> will return
> + "-n 10"
> + """
> + v = parallel_make(d)
> + if v:
> + if limit:
> + v = max(limit, v)
> + return fmt % v
> + return ''
> +
> def packages_filter_out_system(d):
> """
> Return a list of packages from PACKAGES with the "system" packages
> such as
> diff --git a/meta/recipes-core/ovmf/ovmf_git.bb b/meta/recipes-core/ovmf/
> ovmf_git.bb
> index fa0d66291d1..8750b3c528d 100644
> --- a/meta/recipes-core/ovmf/ovmf_git.bb
> +++ b/meta/recipes-core/ovmf/ovmf_git.bb
> @@ -151,7 +151,7 @@ do_compile_class-native() {
>
> do_compile_class-target() {
> export LFLAGS="${LDFLAGS}"
> - PARALLEL_JOBS="${@ '${PARALLEL_MAKE}'.replace('-j', '-n ')}"
> + PARALLEL_JOBS="${@oe.utils.parallel_make_argument(d, '-n %d')}"
> OVMF_ARCH="X64"
> if [ "${TARGET_ARCH}" != "x86_64" ] ; then
> OVMF_ARCH="IA32"
> diff --git a/meta/recipes-support/boost/boost.inc
> b/meta/recipes-support/boost/boost.inc
> index 41fc90fb211..0461ec6fcf1 100644
> --- a/meta/recipes-support/boost/boost.inc
> +++ b/meta/recipes-support/boost/boost.inc
> @@ -135,29 +135,7 @@ BJAM_TOOLS = "--ignore-site-config \
>
> # use PARALLEL_MAKE to speed up the build, but limit it by -j 64, greater
> parallelism causes bjam to segfault or to ignore -j
> # https://svn.boost.org/trac/boost/ticket/7634
> -def get_boost_parallel_make(d):
> - pm = d.getVar('PARALLEL_MAKE')
> - if pm:
> - # look for '-j' and throw other options (e.g. '-l') away
> - # because they might have different meaning in bjam
> - pm = pm.split()
> - while pm:
> - v = None
> - opt = pm.pop(0)
> - if opt == '-j':
> - v = pm.pop(0)
> - elif opt.startswith('-j'):
> - v = opt[2:].strip()
> - else:
> - v = None
> -
> - if v:
> - v = min(64, int(v))
> - return '-j' + str(v)
> -
> - return ""
> -
> -BOOST_PARALLEL_MAKE = "${@get_boost_parallel_make(d)}"
> +BOOST_PARALLEL_MAKE = "${@oe.utils.parallel_make_argument(d, '-j%d',
> limit=64)}"
> BJAM_OPTS = '${BOOST_PARALLEL_MAKE} -d+2 -q \
> ${BJAM_TOOLS} \
> -sBOOST_BUILD_USER_CONFIG=${WORKDIR}/user-config.jam \
> --
> 2.14.3
>
> --
> _______________________________________________
> Openembedded-core mailing list
> Openembedded-core@lists.openembedded.org
> http://lists.openembedded.org/mailman/listinfo/openembedded-core
>
[-- Attachment #2: Type: text/html, Size: 13007 bytes --]
^ permalink raw reply [flat|nested] 6+ messages in thread
* Re: [PATCH] utils.py: add parallel make helpers
2018-02-16 20:14 ` Martin Jansa
@ 2018-02-16 20:23 ` Joshua Watt
2018-02-17 23:57 ` Khem Raj
1 sibling, 0 replies; 6+ messages in thread
From: Joshua Watt @ 2018-02-16 20:23 UTC (permalink / raw)
To: Martin Jansa; +Cc: Patches and discussions about the oe-core layer
[-- Attachment #1: Type: text/plain, Size: 7945 bytes --]
On Fri, 2018-02-16 at 21:14 +0100, Martin Jansa wrote:
> + v = max(limit, v)
>
>
> this seems to be doing the opposite of what the limit was used for
> before:
>
>
> - v = min(64, int(v))
>
Hmm, you're right
On Mon, Feb 12, 2018 at 8:39 PM, Joshua Watt <jpewhacker@gmail.com> wrote:
The code to extract the integer number of parallel build threads and
construct a new argument from them has started to be copied in multiple
locations, so create two new helper utilities to aid recipes.
The first helper (parallel_make()) extracts the integer number of
parallel build threads from PARALLEL_MAKE. The second
(parallel_make_argument()) does the same and then puts the result back
into a format string, optionally clamping it to some maximum value.
Additionally, rework the oe-core recipes that were manually doing this
to use the new helper utilities.
Signed-off-by: Joshua Watt <JPEWhacker@gmail.com>
---
meta/classes/go.bbclass | 19 +---------------
meta/classes/waf.bbclass | 24 +-------------------
meta/lib/oe/utils.py | 43 ++++++++++++++++++++++++++++++++++++
meta/recipes-core/ovmf/ovmf_git.bb | 2 +-
meta/recipes-support/boost/boost.inc | 24 +-------------------
5 files changed, 47 insertions(+), 65 deletions(-)
diff --git a/meta/classes/go.bbclass b/meta/classes/go.bbclass
index 09b01a84c37..7ecd8c92541 100644
--- a/meta/classes/go.bbclass
+++ b/meta/classes/go.bbclass
@@ -1,23 +1,6 @@
inherit goarch ptest
-def get_go_parallel_make(d):
- pm = (d.getVar('PARALLEL_MAKE') or '').split()
- # look for '-j' and throw other options (e.g. '-l') away
- # because they might have a different meaning in golang
- while pm:
- opt = pm.pop(0)
- if opt == '-j':
- v = pm.pop(0)
- elif opt.startswith('-j'):
- v = opt[2:].strip()
- else:
- continue
-
- return '-p %d' % int(v)
-
- return ""
-
-GO_PARALLEL_BUILD ?= "${@get_go_parallel_make(d)}"
+GO_PARALLEL_BUILD ?= "${@oe.utils.parallel_make_argument(d, '-p %d')}"
GOROOT_class-native = "${STAGING_LIBDIR_NATIVE}/go"
GOROOT_class-nativesdk = "${STAGING_DIR_TARGET}${libdir}/go"
diff --git a/meta/classes/waf.bbclass b/meta/classes/waf.bbclass
index bdbdc56767c..f9a851d0759 100644
--- a/meta/classes/waf.bbclass
+++ b/meta/classes/waf.bbclass
@@ -3,28 +3,6 @@ DISABLE_STATIC = ""
EXTRA_OECONF_append = " ${PACKAGECONFIG_CONFARGS}"
-def get_waf_parallel_make(d):
- pm = d.getVar('PARALLEL_MAKE')
- if pm:
- # look for '-j' and throw other options (e.g. '-l') away
- # because they might have different meaning in bjam
- pm = pm.split()
- while pm:
- v = None
- opt = pm.pop(0)
- if opt == '-j':
- v = pm.pop(0)
- elif opt.startswith('-j'):
- v = opt[2:].strip()
- else:
- v = None
-
- if v:
- v = min(64, int(v))
- return '-j' + str(v)
-
- return ""
-
python waf_preconfigure() {
import subprocess
from distutils.version import StrictVersion
@@ -47,7 +25,7 @@ waf_do_configure() {
do_compile[progress] = "outof:^\[\s*(\d+)/\s*(\d+)\]\s+"
waf_do_compile() {
- ${S}/waf build ${@get_waf_parallel_make(d)}
+ ${S}/waf build ${@oe.utils.parallel_make_argument(d, '-j%d', limit=64)}
}
waf_do_install() {
diff --git a/meta/lib/oe/utils.py b/meta/lib/oe/utils.py
index 7a79d752b69..ec91927233a 100644
--- a/meta/lib/oe/utils.py
+++ b/meta/lib/oe/utils.py
@@ -156,6 +156,49 @@ def any_distro_features(d, features, truevalue="1", falsevalue=""):
"""
return bb.utils.contains_any("DISTRO_FEATURES", features, truevalue, falsevalue, d)
+def parallel_make(d):
+ """
+ Return the integer value for the number of parallel threads to use when
+ building, scraped out of PARALLEL_MAKE. If no parallelization option is
+ found, returns None
+
+ e.g. if PARALLEL_MAKE = "-j 10", this will return 10 as an integer.
+ """
+ pm = (d.getVar('PARALLEL_MAKE') or '').split()
+ # look for '-j' and throw other options (e.g. '-l') away
+ while pm:
+ opt = pm.pop(0)
+ if opt == '-j':
+ v = pm.pop(0)
+ elif opt.startswith('-j'):
+ v = opt[2:].strip()
+ else:
+ continue
+
+ return int(v)
+
+ return None
+
+def parallel_make_argument(d, fmt, limit=None):
+ """
+ Helper utility to construct a parallel make argument from the number of
+ parallel threads specified in PARALLEL_MAKE.
+
+ Returns the input format string `fmt` where a single '%d' will be expanded
+ with the number of parallel threads to use. If `limit` is specified, the
+ number of parallel threads will be no larger than it. If no parallelization
+ option is found in PARALLEL_MAKE, returns an empty string
+
+ e.g. if PARALLEL_MAKE = "-j 10", parallel_make_argument(d, "-n %d") will return
+ "-n 10"
+ """
+ v = parallel_make(d)
+ if v:
+ if limit:
+ v = max(limit, v)
+ return fmt % v
+ return ''
+
def packages_filter_out_system(d):
"""
Return a list of packages from PACKAGES with the "system" packages such as
diff --git a/meta/recipes-core/ovmf/ovmf_git.bb b/meta/recipes-core/ovmf/ovmf_git.bb
index fa0d66291d1..8750b3c528d 100644
--- a/meta/recipes-core/ovmf/ovmf_git.bb
+++ b/meta/recipes-core/ovmf/ovmf_git.bb
@@ -151,7 +151,7 @@ do_compile_class-native() {
do_compile_class-target() {
export LFLAGS="${LDFLAGS}"
- PARALLEL_JOBS="${@ '${PARALLEL_MAKE}'.replace('-j', '-n ')}"
+ PARALLEL_JOBS="${@oe.utils.parallel_make_argument(d, '-n %d')}"
OVMF_ARCH="X64"
if [ "${TARGET_ARCH}" != "x86_64" ] ; then
OVMF_ARCH="IA32"
diff --git a/meta/recipes-support/boost/boost.inc b/meta/recipes-support/boost/boost.inc
index 41fc90fb211..0461ec6fcf1 100644
--- a/meta/recipes-support/boost/boost.inc
+++ b/meta/recipes-support/boost/boost.inc
@@ -135,29 +135,7 @@ BJAM_TOOLS = "--ignore-site-config \
# use PARALLEL_MAKE to speed up the build, but limit it by -j 64, greater parallelism causes bjam to segfault or to ignore -j
# https://svn.boost.org/trac/boost/ticket/7634
-def get_boost_parallel_make(d):
- pm = d.getVar('PARALLEL_MAKE')
- if pm:
- # look for '-j' and throw other options (e.g. '-l') away
- # because they might have different meaning in bjam
- pm = pm.split()
- while pm:
- v = None
- opt = pm.pop(0)
- if opt == '-j':
- v = pm.pop(0)
- elif opt.startswith('-j'):
- v = opt[2:].strip()
- else:
- v = None
-
- if v:
- v = min(64, int(v))
- return '-j' + str(v)
-
- return ""
-
-BOOST_PARALLEL_MAKE = "${@get_boost_parallel_make(d)}"
+BOOST_PARALLEL_MAKE = "${@oe.utils.parallel_make_argument(d, '-j%d', limit=64)}"
BJAM_OPTS = '${BOOST_PARALLEL_MAKE} -d+2 -q \
${BJAM_TOOLS} \
-sBOOST_BUILD_USER_CONFIG=${WORKDIR}/user-config.jam \
--
2.14.3
--
_______________________________________________
Openembedded-core mailing list
Openembedded-core@lists.openembedded.org
http://lists.openembedded.org/mailman/listinfo/openembedded-core
[-- Attachment #2: Type: text/html, Size: 14616 bytes --]
^ permalink raw reply [flat|nested] 6+ messages in thread
* Re: [PATCH] utils.py: add parallel make helpers
2018-02-16 20:14 ` Martin Jansa
2018-02-16 20:23 ` Joshua Watt
@ 2018-02-17 23:57 ` Khem Raj
2018-02-18 0:48 ` Joshua Watt
1 sibling, 1 reply; 6+ messages in thread
From: Khem Raj @ 2018-02-17 23:57 UTC (permalink / raw)
To: openembedded-core
On 2/16/18 12:14 PM, Martin Jansa wrote:
> + v = max(limit, v)
>
> this seems to be doing the opposite of what the limit was used for before:
>
> - v = min(64, int(v))
>
moreover, I think the limit should be based on ncpu() instead of hard
coding it.
> On Mon, Feb 12, 2018 at 8:39 PM, Joshua Watt <jpewhacker@gmail.com
> <mailto:jpewhacker@gmail.com>> wrote:
>
> The code to extract the integer number of parallel build threads and
> construct a new argument from them has started to be copied in multiple
> locations, so create two new helper utilities to aid recipes.
>
> The first helper (parallel_make()) extracts the integer number of
> parallel build threads from PARALLEL_MAKE. The second
> (parallel_make_argument()) does the same and then puts the result back
> into a format string, optionally clamping it to some maximum value.
>
> Additionally, rework the oe-core recipes that were manually doing this
> to use the new helper utilities.
>
> Signed-off-by: Joshua Watt <JPEWhacker@gmail.com
> <mailto:JPEWhacker@gmail.com>>
> ---
> meta/classes/go.bbclass | 19 +---------------
> meta/classes/waf.bbclass | 24 +-------------------
> meta/lib/oe/utils.py | 43
> ++++++++++++++++++++++++++++++++++++
> meta/recipes-core/ovmf/ovmf_git.bb <http://ovmf_git.bb> | 2 +-
> meta/recipes-support/boost/boost.inc | 24 +-------------------
> 5 files changed, 47 insertions(+), 65 deletions(-)
>
> diff --git a/meta/classes/go.bbclass b/meta/classes/go.bbclass
> index 09b01a84c37..7ecd8c92541 100644
> --- a/meta/classes/go.bbclass
> +++ b/meta/classes/go.bbclass
> @@ -1,23 +1,6 @@
> inherit goarch ptest
>
> -def get_go_parallel_make(d):
> - pm = (d.getVar('PARALLEL_MAKE') or '').split()
> - # look for '-j' and throw other options (e.g. '-l') away
> - # because they might have a different meaning in golang
> - while pm:
> - opt = pm.pop(0)
> - if opt == '-j':
> - v = pm.pop(0)
> - elif opt.startswith('-j'):
> - v = opt[2:].strip()
> - else:
> - continue
> -
> - return '-p %d' % int(v)
> -
> - return ""
> -
> -GO_PARALLEL_BUILD ?= "${@get_go_parallel_make(d)}"
> +GO_PARALLEL_BUILD ?= "${@oe.utils.parallel_make_argument(d, '-p %d')}"
>
> GOROOT_class-native = "${STAGING_LIBDIR_NATIVE}/go"
> GOROOT_class-nativesdk = "${STAGING_DIR_TARGET}${libdir}/go"
> diff --git a/meta/classes/waf.bbclass b/meta/classes/waf.bbclass
> index bdbdc56767c..f9a851d0759 100644
> --- a/meta/classes/waf.bbclass
> +++ b/meta/classes/waf.bbclass
> @@ -3,28 +3,6 @@ DISABLE_STATIC = ""
>
> EXTRA_OECONF_append = " ${PACKAGECONFIG_CONFARGS}"
>
> -def get_waf_parallel_make(d):
> - pm = d.getVar('PARALLEL_MAKE')
> - if pm:
> - # look for '-j' and throw other options (e.g. '-l') away
> - # because they might have different meaning in bjam
> - pm = pm.split()
> - while pm:
> - v = None
> - opt = pm.pop(0)
> - if opt == '-j':
> - v = pm.pop(0)
> - elif opt.startswith('-j'):
> - v = opt[2:].strip()
> - else:
> - v = None
> -
> - if v:
> - v = min(64, int(v))
> - return '-j' + str(v)
> -
> - return ""
> -
> python waf_preconfigure() {
> import subprocess
> from distutils.version import StrictVersion
> @@ -47,7 +25,7 @@ waf_do_configure() {
>
> do_compile[progress] = "outof:^\[\s*(\d+)/\s*(\d+)\]\s+"
> waf_do_compile() {
> - ${S}/waf build ${@get_waf_parallel_make(d)}
> + ${S}/waf build ${@oe.utils.parallel_make_argument(d, '-j%d',
> limit=64)}
> }
>
> waf_do_install() {
> diff --git a/meta/lib/oe/utils.py b/meta/lib/oe/utils.py
> index 7a79d752b69..ec91927233a 100644
> --- a/meta/lib/oe/utils.py
> +++ b/meta/lib/oe/utils.py
> @@ -156,6 +156,49 @@ def any_distro_features(d, features,
> truevalue="1", falsevalue=""):
> """
> return bb.utils.contains_any("DISTRO_FEATURES", features,
> truevalue, falsevalue, d)
>
> +def parallel_make(d):
> + """
> + Return the integer value for the number of parallel threads to
> use when
> + building, scraped out of PARALLEL_MAKE. If no parallelization
> option is
> + found, returns None
> +
> + e.g. if PARALLEL_MAKE = "-j 10", this will return 10 as an integer.
> + """
> + pm = (d.getVar('PARALLEL_MAKE') or '').split()
> + # look for '-j' and throw other options (e.g. '-l') away
> + while pm:
> + opt = pm.pop(0)
> + if opt == '-j':
> + v = pm.pop(0)
> + elif opt.startswith('-j'):
> + v = opt[2:].strip()
> + else:
> + continue
> +
> + return int(v)
> +
> + return None
> +
> +def parallel_make_argument(d, fmt, limit=None):
> + """
> + Helper utility to construct a parallel make argument from the
> number of
> + parallel threads specified in PARALLEL_MAKE.
> +
> + Returns the input format string `fmt` where a single '%d' will
> be expanded
> + with the number of parallel threads to use. If `limit` is
> specified, the
> + number of parallel threads will be no larger than it. If no
> parallelization
> + option is found in PARALLEL_MAKE, returns an empty string
> +
> + e.g. if PARALLEL_MAKE = "-j 10", parallel_make_argument(d, "-n
> %d") will return
> + "-n 10"
> + """
> + v = parallel_make(d)
> + if v:
> + if limit:
> + v = max(limit, v)
> + return fmt % v
> + return ''
> +
> def packages_filter_out_system(d):
> """
> Return a list of packages from PACKAGES with the "system"
> packages such as
> diff --git a/meta/recipes-core/ovmf/ovmf_git.bb <http://ovmf_git.bb>
> b/meta/recipes-core/ovmf/ovmf_git.bb <http://ovmf_git.bb>
> index fa0d66291d1..8750b3c528d 100644
> --- a/meta/recipes-core/ovmf/ovmf_git.bb <http://ovmf_git.bb>
> +++ b/meta/recipes-core/ovmf/ovmf_git.bb <http://ovmf_git.bb>
> @@ -151,7 +151,7 @@ do_compile_class-native() {
>
> do_compile_class-target() {
> export LFLAGS="${LDFLAGS}"
> - PARALLEL_JOBS="${@ '${PARALLEL_MAKE}'.replace('-j', '-n ')}"
> + PARALLEL_JOBS="${@oe.utils.parallel_make_argument(d, '-n %d')}"
> OVMF_ARCH="X64"
> if [ "${TARGET_ARCH}" != "x86_64" ] ; then
> OVMF_ARCH="IA32"
> diff --git a/meta/recipes-support/boost/boost.inc
> b/meta/recipes-support/boost/boost.inc
> index 41fc90fb211..0461ec6fcf1 100644
> --- a/meta/recipes-support/boost/boost.inc
> +++ b/meta/recipes-support/boost/boost.inc
> @@ -135,29 +135,7 @@ BJAM_TOOLS = "--ignore-site-config \
>
> # use PARALLEL_MAKE to speed up the build, but limit it by -j 64,
> greater parallelism causes bjam to segfault or to ignore -j
> # https://svn.boost.org/trac/boost/ticket/7634
> <https://svn.boost.org/trac/boost/ticket/7634>
> -def get_boost_parallel_make(d):
> - pm = d.getVar('PARALLEL_MAKE')
> - if pm:
> - # look for '-j' and throw other options (e.g. '-l') away
> - # because they might have different meaning in bjam
> - pm = pm.split()
> - while pm:
> - v = None
> - opt = pm.pop(0)
> - if opt == '-j':
> - v = pm.pop(0)
> - elif opt.startswith('-j'):
> - v = opt[2:].strip()
> - else:
> - v = None
> -
> - if v:
> - v = min(64, int(v))
> - return '-j' + str(v)
> -
> - return ""
> -
> -BOOST_PARALLEL_MAKE = "${@get_boost_parallel_make(d)}"
> +BOOST_PARALLEL_MAKE = "${@oe.utils.parallel_make_argument(d,
> '-j%d', limit=64)}"
> BJAM_OPTS = '${BOOST_PARALLEL_MAKE} -d+2 -q \
> ${BJAM_TOOLS} \
> -sBOOST_BUILD_USER_CONFIG=${WORKDIR}/user-config.jam \
> --
> 2.14.3
>
> --
> _______________________________________________
> Openembedded-core mailing list
> Openembedded-core@lists.openembedded.org
> <mailto:Openembedded-core@lists.openembedded.org>
> http://lists.openembedded.org/mailman/listinfo/openembedded-core
> <http://lists.openembedded.org/mailman/listinfo/openembedded-core>
>
>
>
^ permalink raw reply [flat|nested] 6+ messages in thread
* Re: [PATCH] utils.py: add parallel make helpers
2018-02-17 23:57 ` Khem Raj
@ 2018-02-18 0:48 ` Joshua Watt
2018-02-18 1:01 ` Khem Raj
0 siblings, 1 reply; 6+ messages in thread
From: Joshua Watt @ 2018-02-18 0:48 UTC (permalink / raw)
To: Khem Raj; +Cc: OE-core
[-- Attachment #1: Type: text/plain, Size: 10010 bytes --]
On Feb 17, 2018 18:25, "Khem Raj" <raj.khem@gmail.com> wrote:
On 2/16/18 12:14 PM, Martin Jansa wrote:
> + v = max(limit, v)
>
> this seems to be doing the opposite of what the limit was used for before:
>
> - v = min(64, int(v))
>
>
moreover, I think the limit should be based on ncpu() instead of hard
coding it.
I don't know about that. Doesn't PARALLEL_MAKE already default to something
based on ncpu()? My understanding of the limit was that some recipes
(boost? waf?) have a limit on the amount of parallism they can reasonably
stand, and you don't want those recipes to break if someone sets
PARALLEL_MAKE="-j
1000".
As long as that is true I think this is correct. Whether those recipes
still need to limit their parallism is another discussion.
On Mon, Feb 12, 2018 at 8:39 PM, Joshua Watt <jpewhacker@gmail.com <mailto:
> jpewhacker@gmail.com>> wrote:
>
> The code to extract the integer number of parallel build threads and
> construct a new argument from them has started to be copied in multiple
> locations, so create two new helper utilities to aid recipes.
>
> The first helper (parallel_make()) extracts the integer number of
> parallel build threads from PARALLEL_MAKE. The second
> (parallel_make_argument()) does the same and then puts the result back
> into a format string, optionally clamping it to some maximum value.
>
> Additionally, rework the oe-core recipes that were manually doing this
> to use the new helper utilities.
>
> Signed-off-by: Joshua Watt <JPEWhacker@gmail.com
> <mailto:JPEWhacker@gmail.com>>
>
> ---
> meta/classes/go.bbclass | 19 +---------------
> meta/classes/waf.bbclass | 24 +-------------------
> meta/lib/oe/utils.py | 43
> ++++++++++++++++++++++++++++++++++++
> meta/recipes-core/ovmf/ovmf_git.bb <http://ovmf_git.bb> | 2 +-
>
> meta/recipes-support/boost/boost.inc | 24 +-------------------
> 5 files changed, 47 insertions(+), 65 deletions(-)
>
> diff --git a/meta/classes/go.bbclass b/meta/classes/go.bbclass
> index 09b01a84c37..7ecd8c92541 100644
> --- a/meta/classes/go.bbclass
> +++ b/meta/classes/go.bbclass
> @@ -1,23 +1,6 @@
> inherit goarch ptest
>
> -def get_go_parallel_make(d):
> - pm = (d.getVar('PARALLEL_MAKE') or '').split()
> - # look for '-j' and throw other options (e.g. '-l') away
> - # because they might have a different meaning in golang
> - while pm:
> - opt = pm.pop(0)
> - if opt == '-j':
> - v = pm.pop(0)
> - elif opt.startswith('-j'):
> - v = opt[2:].strip()
> - else:
> - continue
> -
> - return '-p %d' % int(v)
> -
> - return ""
> -
> -GO_PARALLEL_BUILD ?= "${@get_go_parallel_make(d)}"
> +GO_PARALLEL_BUILD ?= "${@oe.utils.parallel_make_argument(d, '-p
> %d')}"
>
> GOROOT_class-native = "${STAGING_LIBDIR_NATIVE}/go"
> GOROOT_class-nativesdk = "${STAGING_DIR_TARGET}${libdir}/go"
> diff --git a/meta/classes/waf.bbclass b/meta/classes/waf.bbclass
> index bdbdc56767c..f9a851d0759 100644
> --- a/meta/classes/waf.bbclass
> +++ b/meta/classes/waf.bbclass
> @@ -3,28 +3,6 @@ DISABLE_STATIC = ""
>
> EXTRA_OECONF_append = " ${PACKAGECONFIG_CONFARGS}"
>
> -def get_waf_parallel_make(d):
> - pm = d.getVar('PARALLEL_MAKE')
> - if pm:
> - # look for '-j' and throw other options (e.g. '-l') away
> - # because they might have different meaning in bjam
> - pm = pm.split()
> - while pm:
> - v = None
> - opt = pm.pop(0)
> - if opt == '-j':
> - v = pm.pop(0)
> - elif opt.startswith('-j'):
> - v = opt[2:].strip()
> - else:
> - v = None
> -
> - if v:
> - v = min(64, int(v))
> - return '-j' + str(v)
> -
> - return ""
> -
> python waf_preconfigure() {
> import subprocess
> from distutils.version import StrictVersion
> @@ -47,7 +25,7 @@ waf_do_configure() {
>
> do_compile[progress] = "outof:^\[\s*(\d+)/\s*(\d+)\]\s+"
> waf_do_compile() {
> - ${S}/waf build ${@get_waf_parallel_make(d)}
> + ${S}/waf build ${@oe.utils.parallel_make_argument(d, '-j%d',
> limit=64)}
> }
>
> waf_do_install() {
> diff --git a/meta/lib/oe/utils.py b/meta/lib/oe/utils.py
> index 7a79d752b69..ec91927233a 100644
> --- a/meta/lib/oe/utils.py
> +++ b/meta/lib/oe/utils.py
> @@ -156,6 +156,49 @@ def any_distro_features(d, features,
> truevalue="1", falsevalue=""):
> """
> return bb.utils.contains_any("DISTRO_FEATURES", features,
> truevalue, falsevalue, d)
>
> +def parallel_make(d):
> + """
> + Return the integer value for the number of parallel threads to
> use when
> + building, scraped out of PARALLEL_MAKE. If no parallelization
> option is
> + found, returns None
> +
> + e.g. if PARALLEL_MAKE = "-j 10", this will return 10 as an
> integer.
> + """
> + pm = (d.getVar('PARALLEL_MAKE') or '').split()
> + # look for '-j' and throw other options (e.g. '-l') away
> + while pm:
> + opt = pm.pop(0)
> + if opt == '-j':
> + v = pm.pop(0)
> + elif opt.startswith('-j'):
> + v = opt[2:].strip()
> + else:
> + continue
> +
> + return int(v)
> +
> + return None
> +
> +def parallel_make_argument(d, fmt, limit=None):
> + """
> + Helper utility to construct a parallel make argument from the
> number of
> + parallel threads specified in PARALLEL_MAKE.
> +
> + Returns the input format string `fmt` where a single '%d' will
> be expanded
> + with the number of parallel threads to use. If `limit` is
> specified, the
> + number of parallel threads will be no larger than it. If no
> parallelization
> + option is found in PARALLEL_MAKE, returns an empty string
> +
> + e.g. if PARALLEL_MAKE = "-j 10", parallel_make_argument(d, "-n
> %d") will return
> + "-n 10"
> + """
> + v = parallel_make(d)
> + if v:
> + if limit:
> + v = max(limit, v)
> + return fmt % v
> + return ''
> +
> def packages_filter_out_system(d):
> """
> Return a list of packages from PACKAGES with the "system"
> packages such as
> diff --git a/meta/recipes-core/ovmf/ovmf_git.bb <http://ovmf_git.bb>
> b/meta/recipes-core/ovmf/ovmf_git.bb <http://ovmf_git.bb>
> index fa0d66291d1..8750b3c528d 100644
> --- a/meta/recipes-core/ovmf/ovmf_git.bb <http://ovmf_git.bb>
> +++ b/meta/recipes-core/ovmf/ovmf_git.bb <http://ovmf_git.bb>
>
> @@ -151,7 +151,7 @@ do_compile_class-native() {
>
> do_compile_class-target() {
> export LFLAGS="${LDFLAGS}"
> - PARALLEL_JOBS="${@ '${PARALLEL_MAKE}'.replace('-j', '-n ')}"
> + PARALLEL_JOBS="${@oe.utils.parallel_make_argument(d, '-n %d')}"
> OVMF_ARCH="X64"
> if [ "${TARGET_ARCH}" != "x86_64" ] ; then
> OVMF_ARCH="IA32"
> diff --git a/meta/recipes-support/boost/boost.inc
> b/meta/recipes-support/boost/boost.inc
> index 41fc90fb211..0461ec6fcf1 100644
> --- a/meta/recipes-support/boost/boost.inc
> +++ b/meta/recipes-support/boost/boost.inc
> @@ -135,29 +135,7 @@ BJAM_TOOLS = "--ignore-site-config \
>
> # use PARALLEL_MAKE to speed up the build, but limit it by -j 64,
> greater parallelism causes bjam to segfault or to ignore -j
> # https://svn.boost.org/trac/boost/ticket/7634
> <https://svn.boost.org/trac/boost/ticket/7634>
> -def get_boost_parallel_make(d):
> - pm = d.getVar('PARALLEL_MAKE')
> - if pm:
> - # look for '-j' and throw other options (e.g. '-l') away
> - # because they might have different meaning in bjam
> - pm = pm.split()
> - while pm:
> - v = None
> - opt = pm.pop(0)
> - if opt == '-j':
> - v = pm.pop(0)
> - elif opt.startswith('-j'):
> - v = opt[2:].strip()
> - else:
> - v = None
> -
> - if v:
> - v = min(64, int(v))
> - return '-j' + str(v)
> -
> - return ""
> -
> -BOOST_PARALLEL_MAKE = "${@get_boost_parallel_make(d)}"
> +BOOST_PARALLEL_MAKE = "${@oe.utils.parallel_make_argument(d,
> '-j%d', limit=64)}"
> BJAM_OPTS = '${BOOST_PARALLEL_MAKE} -d+2 -q \
> ${BJAM_TOOLS} \
> -sBOOST_BUILD_USER_CONFIG=${WORKDIR}/user-config.jam
> \
> --
> 2.14.3
>
> --
> _______________________________________________
> Openembedded-core mailing list
> Openembedded-core@lists.openembedded.org
> <mailto:Openembedded-core@lists.openembedded.org>
> http://lists.openembedded.org/mailman/listinfo/openembedded-core
> <http://lists.openembedded.org/mailman/listinfo/openembedded-core>
>
>
>
> --
_______________________________________________
Openembedded-core mailing list
Openembedded-core@lists.openembedded.org
http://lists.openembedded.org/mailman/listinfo/openembedded-core
[-- Attachment #2: Type: text/html, Size: 15085 bytes --]
^ permalink raw reply [flat|nested] 6+ messages in thread
* Re: [PATCH] utils.py: add parallel make helpers
2018-02-18 0:48 ` Joshua Watt
@ 2018-02-18 1:01 ` Khem Raj
0 siblings, 0 replies; 6+ messages in thread
From: Khem Raj @ 2018-02-18 1:01 UTC (permalink / raw)
To: Joshua Watt; +Cc: OE-core
[-- Attachment #1: Type: text/plain, Size: 10426 bytes --]
On Sat, Feb 17, 2018 at 4:48 PM Joshua Watt <jpewhacker@gmail.com> wrote:
>
>
> On Feb 17, 2018 18:25, "Khem Raj" <raj.khem@gmail.com> wrote:
>
>
>
> On 2/16/18 12:14 PM, Martin Jansa wrote:
>
>> + v = max(limit, v)
>>
>> this seems to be doing the opposite of what the limit was used for before:
>>
>> - v = min(64, int(v))
>>
>>
> moreover, I think the limit should be based on ncpu() instead of hard
> coding it.
>
>
> I don't know about that. Doesn't PARALLEL_MAKE already default to
> something based on ncpu()? My understanding of the limit was that some
> recipes (boost? waf?) have a limit on the amount of parallism they can
> reasonably stand, and you don't want those recipes to break if someone sets PARALLEL_MAKE="-j
> 1000".
>
Ah yes that’s a fair point
>
> As long as that is true I think this is correct. Whether those recipes
> still need to limit their parallism is another discussion.
>
>
> On Mon, Feb 12, 2018 at 8:39 PM, Joshua Watt <jpewhacker@gmail.com
>> <mailto:jpewhacker@gmail.com>> wrote:
>>
>> The code to extract the integer number of parallel build threads and
>> construct a new argument from them has started to be copied in
>> multiple
>> locations, so create two new helper utilities to aid recipes.
>>
>> The first helper (parallel_make()) extracts the integer number of
>> parallel build threads from PARALLEL_MAKE. The second
>> (parallel_make_argument()) does the same and then puts the result back
>> into a format string, optionally clamping it to some maximum value.
>>
>> Additionally, rework the oe-core recipes that were manually doing this
>> to use the new helper utilities.
>>
>> Signed-off-by: Joshua Watt <JPEWhacker@gmail.com
>> <mailto:JPEWhacker@gmail.com>>
>>
>> ---
>> meta/classes/go.bbclass | 19 +---------------
>> meta/classes/waf.bbclass | 24 +-------------------
>> meta/lib/oe/utils.py | 43
>> ++++++++++++++++++++++++++++++++++++
>> meta/recipes-core/ovmf/ovmf_git.bb <http://ovmf_git.bb> | 2 +-
>>
>> meta/recipes-support/boost/boost.inc | 24 +-------------------
>> 5 files changed, 47 insertions(+), 65 deletions(-)
>>
>> diff --git a/meta/classes/go.bbclass b/meta/classes/go.bbclass
>> index 09b01a84c37..7ecd8c92541 100644
>> --- a/meta/classes/go.bbclass
>> +++ b/meta/classes/go.bbclass
>> @@ -1,23 +1,6 @@
>> inherit goarch ptest
>>
>> -def get_go_parallel_make(d):
>> - pm = (d.getVar('PARALLEL_MAKE') or '').split()
>> - # look for '-j' and throw other options (e.g. '-l') away
>> - # because they might have a different meaning in golang
>> - while pm:
>> - opt = pm.pop(0)
>> - if opt == '-j':
>> - v = pm.pop(0)
>> - elif opt.startswith('-j'):
>> - v = opt[2:].strip()
>> - else:
>> - continue
>> -
>> - return '-p %d' % int(v)
>> -
>> - return ""
>> -
>> -GO_PARALLEL_BUILD ?= "${@get_go_parallel_make(d)}"
>> +GO_PARALLEL_BUILD ?= "${@oe.utils.parallel_make_argument(d, '-p
>> %d')}"
>>
>> GOROOT_class-native = "${STAGING_LIBDIR_NATIVE}/go"
>> GOROOT_class-nativesdk = "${STAGING_DIR_TARGET}${libdir}/go"
>> diff --git a/meta/classes/waf.bbclass b/meta/classes/waf.bbclass
>> index bdbdc56767c..f9a851d0759 100644
>> --- a/meta/classes/waf.bbclass
>> +++ b/meta/classes/waf.bbclass
>> @@ -3,28 +3,6 @@ DISABLE_STATIC = ""
>>
>> EXTRA_OECONF_append = " ${PACKAGECONFIG_CONFARGS}"
>>
>> -def get_waf_parallel_make(d):
>> - pm = d.getVar('PARALLEL_MAKE')
>> - if pm:
>> - # look for '-j' and throw other options (e.g. '-l') away
>> - # because they might have different meaning in bjam
>> - pm = pm.split()
>> - while pm:
>> - v = None
>> - opt = pm.pop(0)
>> - if opt == '-j':
>> - v = pm.pop(0)
>> - elif opt.startswith('-j'):
>> - v = opt[2:].strip()
>> - else:
>> - v = None
>> -
>> - if v:
>> - v = min(64, int(v))
>> - return '-j' + str(v)
>> -
>> - return ""
>> -
>> python waf_preconfigure() {
>> import subprocess
>> from distutils.version import StrictVersion
>> @@ -47,7 +25,7 @@ waf_do_configure() {
>>
>> do_compile[progress] = "outof:^\[\s*(\d+)/\s*(\d+)\]\s+"
>> waf_do_compile() {
>> - ${S}/waf build ${@get_waf_parallel_make(d)}
>> + ${S}/waf build ${@oe.utils.parallel_make_argument(d, '-j%d',
>> limit=64)}
>> }
>>
>> waf_do_install() {
>> diff --git a/meta/lib/oe/utils.py b/meta/lib/oe/utils.py
>> index 7a79d752b69..ec91927233a 100644
>> --- a/meta/lib/oe/utils.py
>> +++ b/meta/lib/oe/utils.py
>> @@ -156,6 +156,49 @@ def any_distro_features(d, features,
>> truevalue="1", falsevalue=""):
>> """
>> return bb.utils.contains_any("DISTRO_FEATURES", features,
>> truevalue, falsevalue, d)
>>
>> +def parallel_make(d):
>> + """
>> + Return the integer value for the number of parallel threads to
>> use when
>> + building, scraped out of PARALLEL_MAKE. If no parallelization
>> option is
>> + found, returns None
>> +
>> + e.g. if PARALLEL_MAKE = "-j 10", this will return 10 as an
>> integer.
>> + """
>> + pm = (d.getVar('PARALLEL_MAKE') or '').split()
>> + # look for '-j' and throw other options (e.g. '-l') away
>> + while pm:
>> + opt = pm.pop(0)
>> + if opt == '-j':
>> + v = pm.pop(0)
>> + elif opt.startswith('-j'):
>> + v = opt[2:].strip()
>> + else:
>> + continue
>> +
>> + return int(v)
>> +
>> + return None
>> +
>> +def parallel_make_argument(d, fmt, limit=None):
>> + """
>> + Helper utility to construct a parallel make argument from the
>> number of
>> + parallel threads specified in PARALLEL_MAKE.
>> +
>> + Returns the input format string `fmt` where a single '%d' will
>> be expanded
>> + with the number of parallel threads to use. If `limit` is
>> specified, the
>> + number of parallel threads will be no larger than it. If no
>> parallelization
>> + option is found in PARALLEL_MAKE, returns an empty string
>> +
>> + e.g. if PARALLEL_MAKE = "-j 10", parallel_make_argument(d, "-n
>> %d") will return
>> + "-n 10"
>> + """
>> + v = parallel_make(d)
>> + if v:
>> + if limit:
>> + v = max(limit, v)
>> + return fmt % v
>> + return ''
>> +
>> def packages_filter_out_system(d):
>> """
>> Return a list of packages from PACKAGES with the "system"
>> packages such as
>> diff --git a/meta/recipes-core/ovmf/ovmf_git.bb <http://ovmf_git.bb>
>> b/meta/recipes-core/ovmf/ovmf_git.bb <http://ovmf_git.bb>
>> index fa0d66291d1..8750b3c528d 100644
>> --- a/meta/recipes-core/ovmf/ovmf_git.bb <http://ovmf_git.bb>
>> +++ b/meta/recipes-core/ovmf/ovmf_git.bb <http://ovmf_git.bb>
>>
>> @@ -151,7 +151,7 @@ do_compile_class-native() {
>>
>> do_compile_class-target() {
>> export LFLAGS="${LDFLAGS}"
>> - PARALLEL_JOBS="${@ '${PARALLEL_MAKE}'.replace('-j', '-n ')}"
>> + PARALLEL_JOBS="${@oe.utils.parallel_make_argument(d, '-n %d')}"
>> OVMF_ARCH="X64"
>> if [ "${TARGET_ARCH}" != "x86_64" ] ; then
>> OVMF_ARCH="IA32"
>> diff --git a/meta/recipes-support/boost/boost.inc
>> b/meta/recipes-support/boost/boost.inc
>> index 41fc90fb211..0461ec6fcf1 100644
>> --- a/meta/recipes-support/boost/boost.inc
>> +++ b/meta/recipes-support/boost/boost.inc
>> @@ -135,29 +135,7 @@ BJAM_TOOLS = "--ignore-site-config \
>>
>> # use PARALLEL_MAKE to speed up the build, but limit it by -j 64,
>> greater parallelism causes bjam to segfault or to ignore -j
>> # https://svn.boost.org/trac/boost/ticket/7634
>> <https://svn.boost.org/trac/boost/ticket/7634>
>> -def get_boost_parallel_make(d):
>> - pm = d.getVar('PARALLEL_MAKE')
>> - if pm:
>> - # look for '-j' and throw other options (e.g. '-l') away
>> - # because they might have different meaning in bjam
>> - pm = pm.split()
>> - while pm:
>> - v = None
>> - opt = pm.pop(0)
>> - if opt == '-j':
>> - v = pm.pop(0)
>> - elif opt.startswith('-j'):
>> - v = opt[2:].strip()
>> - else:
>> - v = None
>> -
>> - if v:
>> - v = min(64, int(v))
>> - return '-j' + str(v)
>> -
>> - return ""
>> -
>> -BOOST_PARALLEL_MAKE = "${@get_boost_parallel_make(d)}"
>> +BOOST_PARALLEL_MAKE = "${@oe.utils.parallel_make_argument(d,
>> '-j%d', limit=64)}"
>> BJAM_OPTS = '${BOOST_PARALLEL_MAKE} -d+2 -q \
>> ${BJAM_TOOLS} \
>> -sBOOST_BUILD_USER_CONFIG=${WORKDIR}/user-config.jam
>> \
>> --
>> 2.14.3
>>
>> --
>> _______________________________________________
>> Openembedded-core mailing list
>> Openembedded-core@lists.openembedded.org
>> <mailto:Openembedded-core@lists.openembedded.org>
>> http://lists.openembedded.org/mailman/listinfo/openembedded-core
>> <http://lists.openembedded.org/mailman/listinfo/openembedded-core>
>>
>>
>>
>> --
> _______________________________________________
> Openembedded-core mailing list
> Openembedded-core@lists.openembedded.org
>
> http://lists.openembedded.org/mailman/listinfo/openembedded-core
>
>
>
[-- Attachment #2: Type: text/html, Size: 16472 bytes --]
^ permalink raw reply [flat|nested] 6+ messages in thread
end of thread, other threads:[~2018-02-18 1:01 UTC | newest]
Thread overview: 6+ messages (download: mbox.gz follow: Atom feed
-- links below jump to the message on this page --
2018-02-12 19:39 [PATCH] utils.py: add parallel make helpers Joshua Watt
2018-02-16 20:14 ` Martin Jansa
2018-02-16 20:23 ` Joshua Watt
2018-02-17 23:57 ` Khem Raj
2018-02-18 0:48 ` Joshua Watt
2018-02-18 1:01 ` Khem Raj
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox