[Buildroot] [PATCH 0/7] automatically determine job concurrency

[Buildroot] [PATCH 0/7] automatically determine job concurrency

[Nathan Lynch]

This series implements the option of "autodetecting" the number of
jobs to run concurrently (make -jN), using the number of reported CPUs
when BR2_JLEVEL is set to 0.

Nathan Lynch (7):
  introduce PARALLEL_JOBS build system variable
  boost: change uses of BR2_JLEVEL to PARALLEL_JOBS
  cmake: change use of BR2_JLEVEL to PARALLEL_JOBS
  midori: change use of BR2_JLEVEL to PARALLEL_JOBS
  qt: change use of BR2_JLEVEL to PARALLEL_JOBS
  toolchain/crosstool-ng: change use of BR2_JLEVEL to PARALLEL_JOBS
  automatically set PARALLEL_JOBS when BR2_JLEVEL is 0

 Config.in                                        |    8 +++++---
 package/Makefile.in                              |    8 +++++++-
 package/boost/boost.mk                           |    4 ++--
 package/cmake/cmake.mk                           |    2 +-
 package/midori/midori.mk                         |    2 +-
 package/qt/qt.mk                                 |    2 +-
 toolchain/toolchain-crosstool-ng/crosstool-ng.mk |    2 +-
 7 files changed, 18 insertions(+), 10 deletions(-)

-- 
1.7.6.5

[Buildroot] [PATCH 1/7] introduce PARALLEL_JOBS build system variable

[Nathan Lynch]

BR2_JLEVEL currently is expanded directly in $(MAKE), and used in
invocations of other build software (e.g. ct-ng).  However, we are
going to allow "0" to be a meaningful value for BR2_JLEVEL, which
won't work for these uses.  Given that it is not permissible to modify
BR2_-prefixed variables in Makefiles, we need an intermediate
variable.

Define PARALLEL_JOBS to $(BR2_JLEVEL), and use the former in MAKE's
definition.  Uses of BR2_JLEVEL throughout the rest of the tree to be
adjusted similarly in follow-up patches.

Signed-off-by: Nathan Lynch <ntl@pobox.com>
---
 package/Makefile.in |    4 +++-
 1 files changed, 3 insertions(+), 1 deletions(-)

diff --git a/package/Makefile.in b/package/Makefile.in
index df7042c..9177a1b 100644
--- a/package/Makefile.in
+++ b/package/Makefile.in
@@ -6,8 +6,10 @@ HOSTMAKE=$(MAKE)
 endif
 HOSTMAKE :=$(shell which $(HOSTMAKE) || type -p $(HOSTMAKE) || echo make)
 
+PARALLEL_JOBS:=$(BR2_JLEVEL)
+
 MAKE1:=$(HOSTMAKE) -j1
-MAKE:=$(HOSTMAKE) -j$(BR2_JLEVEL)
+MAKE:=$(HOSTMAKE) -j$(PARALLEL_JOBS)
 
 # Compute GNU_TARGET_NAME and REAL_GNU_TARGET_NAME
 GNU_TARGET_NAME=$(ARCH)-linux
-- 
1.7.6.5

[Buildroot] [PATCH 2/7] boost: change uses of BR2_JLEVEL to PARALLEL_JOBS

[Nathan Lynch]

Signed-off-by: Nathan Lynch <ntl@pobox.com>
---
 package/boost/boost.mk |    4 ++--
 1 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/package/boost/boost.mk b/package/boost/boost.mk
index 3667964..abdf3a2 100644
--- a/package/boost/boost.mk
+++ b/package/boost/boost.mk
@@ -70,7 +70,7 @@ define BOOST_CONFIGURE_CMDS
 endef
 
 define BOOST_INSTALL_TARGET_CMDS
-	(cd $(@D) && ./b2 -j$(BR2_JLEVEL) -q -d+2 \
+	(cd $(@D) && ./b2 -j$(PARALLEL_JOBS) -q -d+2 \
 	--user-config=$(@D)/user-config.jam \
 	$(BOOST_OPT) \
 	--prefix=$(TARGET_DIR)/usr \
@@ -78,7 +78,7 @@ define BOOST_INSTALL_TARGET_CMDS
 endef
 
 define BOOST_INSTALL_STAGING_CMDS
-	(cd $(@D) && ./bjam -j$(BR2_JLEVEL) -d+2 \
+	(cd $(@D) && ./bjam -j$(PARALLEL_JOBS) -d+2 \
 	--user-config=$(@D)/user-config.jam \
 	$(BOOST_OPT) \
 	--prefix=$(STAGING_DIR)/usr \
-- 
1.7.6.5

[Buildroot] [PATCH 3/7] cmake: change use of BR2_JLEVEL to PARALLEL_JOBS

[Nathan Lynch]

Signed-off-by: Nathan Lynch <ntl@pobox.com>
---
 package/cmake/cmake.mk |    2 +-
 1 files changed, 1 insertions(+), 1 deletions(-)

diff --git a/package/cmake/cmake.mk b/package/cmake/cmake.mk
index af88eea..af6c53c 100644
--- a/package/cmake/cmake.mk
+++ b/package/cmake/cmake.mk
@@ -5,7 +5,7 @@ define HOST_CMAKE_CONFIGURE_CMDS
  (cd $(@D); \
 	LDFLAGS="$(HOST_LDFLAGS)" \
 	CFLAGS="$(HOST_CFLAGS)" \
-	./bootstrap --prefix=$(HOST_DIR)/usr --parallel=$(BR2_JLEVEL) \
+	./bootstrap --prefix=$(HOST_DIR)/usr --parallel=$(PARALLEL_JOBS) \
  )
 endef
 
-- 
1.7.6.5

[Buildroot] [PATCH 4/7] midori: change use of BR2_JLEVEL to PARALLEL_JOBS

[Nathan Lynch]

Signed-off-by: Nathan Lynch <ntl@pobox.com>
---
 package/midori/midori.mk |    2 +-
 1 files changed, 1 insertions(+), 1 deletions(-)

diff --git a/package/midori/midori.mk b/package/midori/midori.mk
index 0b3439c..aa376a2 100644
--- a/package/midori/midori.mk
+++ b/package/midori/midori.mk
@@ -34,7 +34,7 @@ define MIDORI_CONFIGURE_CMDS
 endef
 
 define MIDORI_BUILD_CMDS
-       (cd $(@D); ./waf build -j $(BR2_JLEVEL))
+       (cd $(@D); ./waf build -j $(PARALLEL_JOBS))
 endef
 
 define MIDORI_INSTALL_TARGET_CMDS
-- 
1.7.6.5

[Buildroot] [PATCH 5/7] qt: change use of BR2_JLEVEL to PARALLEL_JOBS

[Nathan Lynch]

Signed-off-by: Nathan Lynch <ntl@pobox.com>
---
 package/qt/qt.mk |    2 +-
 1 files changed, 1 insertions(+), 1 deletions(-)

diff --git a/package/qt/qt.mk b/package/qt/qt.mk
index b751696..ff9c1bf 100644
--- a/package/qt/qt.mk
+++ b/package/qt/qt.mk
@@ -488,7 +488,7 @@ define QT_CONFIGURE_CMDS
 		PKG_CONFIG_SYSROOT_DIR="$(STAGING_DIR)" \
 		PKG_CONFIG="$(PKG_CONFIG_HOST_BINARY)" \
 		PKG_CONFIG_PATH="$(STAGING_DIR)/usr/lib/pkgconfig:$(PKG_CONFIG_PATH)" \
-		MAKEFLAGS="$(MAKEFLAGS) -j$(BR2_JLEVEL)" ./configure \
+		MAKEFLAGS="$(MAKEFLAGS) -j$(PARALLEL_JOBS)" ./configure \
 		$(if $(VERBOSE),-verbose,-silent) \
 		-force-pkg-config \
 		$(QT_CONFIGURE_OPTS) \
-- 
1.7.6.5

[Buildroot] [PATCH 6/7] toolchain/crosstool-ng: change use of BR2_JLEVEL to PARALLEL_JOBS

[Nathan Lynch]

Signed-off-by: Nathan Lynch <ntl@pobox.com>
---
 toolchain/toolchain-crosstool-ng/crosstool-ng.mk |    2 +-
 1 files changed, 1 insertions(+), 1 deletions(-)

diff --git a/toolchain/toolchain-crosstool-ng/crosstool-ng.mk b/toolchain/toolchain-crosstool-ng/crosstool-ng.mk
index 25d2a16..c3a8a01 100644
--- a/toolchain/toolchain-crosstool-ng/crosstool-ng.mk
+++ b/toolchain/toolchain-crosstool-ng/crosstool-ng.mk
@@ -81,7 +81,7 @@ $(STAMP_DIR)/ct-ng-toolchain-installed: $(STAMP_DIR)/ct-ng-toolchain-built
 #       depending on the selected C library. Those deps are added later
 
 $(STAMP_DIR)/ct-ng-toolchain-built: $(CTNG_DIR)/.config
-	$(Q)$(call ctng,build.$(BR2_JLEVEL))
+	$(Q)$(call ctng,build.$(PARALLEL_JOBS))
 	$(Q)printf "\n"
 	$(Q)touch $@
 
-- 
1.7.6.5

[Buildroot] [PATCH 7/7] automatically set PARALLEL_JOBS when BR2_JLEVEL is 0

[Nathan Lynch]

When BR2_JLEVEL is 0, set PARALLEL_JOBS to the number of CPUs
detected.  This allows one to more or less fully utilize the host
system without manually tuning the configuration.

Also make 0 the default value for BR2_JLEVEL.

Signed-off-by: Nathan Lynch <ntl@pobox.com>
---
 Config.in           |    8 +++++---
 package/Makefile.in |    4 ++++
 2 files changed, 9 insertions(+), 3 deletions(-)

diff --git a/Config.in b/Config.in
index 95c2e8c..3745255 100644
--- a/Config.in
+++ b/Config.in
@@ -178,10 +178,12 @@ config BR2_DEBIAN_MIRROR
 endmenu
 
 config BR2_JLEVEL
-	int "Number of jobs to run simultaneously"
-	default "2"
+	int "Number of jobs to run simultaneously (0 for auto)"
+	default "0"
 	help
-	  Number of jobs to run simultaneously
+	  Number of jobs to run simultaneously.  If 0, determine
+	  automatically according to number of CPUs on the host
+	  system.
 
 config BR2_CCACHE
 	bool "Enable compiler cache"
diff --git a/package/Makefile.in b/package/Makefile.in
index 9177a1b..a9d2771 100644
--- a/package/Makefile.in
+++ b/package/Makefile.in
@@ -6,7 +6,11 @@ HOSTMAKE=$(MAKE)
 endif
 HOSTMAKE :=$(shell which $(HOSTMAKE) || type -p $(HOSTMAKE) || echo make)
 
+ifeq ($(BR2_JLEVEL),0)
+PARALLEL_JOBS:=$(shell getconf _NPROCESSORS_ONLN 2>/dev/null || echo 1)
+else
 PARALLEL_JOBS:=$(BR2_JLEVEL)
+endif
 
 MAKE1:=$(HOSTMAKE) -j1
 MAKE:=$(HOSTMAKE) -j$(PARALLEL_JOBS)
-- 
1.7.6.5

[Buildroot] [PATCH 7/7] automatically set PARALLEL_JOBS when BR2_JLEVEL is 0

[Arnout Vandecappelle]

On 06/12/12 04:09, Nathan Lynch wrote:
> When BR2_JLEVEL is 0, set PARALLEL_JOBS to the number of CPUs
> detected.  This allows one to more or less fully utilize the host
> system without manually tuning the configuration.

  In my experience, you need 2 times the number of CPUs to fully utilize
the processor because most compilations have long iowait latencies.

  Otherwise, looks good.

  Regards,
  Arnout

-- 
Arnout Vandecappelle                               arnout at mind be
Senior Embedded Software Architect                 +32-16-286540
Essensium/Mind                                     http://www.mind.be
G.Geenslaan 9, 3001 Leuven, Belgium                BE 872 984 063 RPR Leuven
LinkedIn profile: http://www.linkedin.com/in/arnoutvandecappelle
GPG fingerprint:  7CB5 E4CC 6C2E EFD4 6E3D A754 F963 ECAB 2450 2F1F

[Buildroot] [PATCH 7/7] automatically set PARALLEL_JOBS when BR2_JLEVEL is 0

[Nathan Lynch]

On Thu, 2012-06-14 at 01:27 +0200, Arnout Vandecappelle wrote:
> On 06/12/12 04:09, Nathan Lynch wrote:
> > When BR2_JLEVEL is 0, set PARALLEL_JOBS to the number of CPUs
> > detected.  This allows one to more or less fully utilize the host
> > system without manually tuning the configuration.
> 
>   In my experience, you need 2 times the number of CPUs to fully utilize
> the processor because most compilations have long iowait latencies.

Yeah.  I'm happy to make this change, assuming others agree?

Thanks for having a look!

[Buildroot] [PATCH 7/7] automatically set PARALLEL_JOBS when BR2_JLEVEL is 0

[Peter Korsgaard]

>>>>> "Nathan" == Nathan Lynch <ntl@pobox.com> writes:

 >> In my experience, you need 2 times the number of CPUs to fully utilize
 >> the processor because most compilations have long iowait latencies.

 Nathan> Yeah.  I'm happy to make this change, assuming others agree?

I was going to say something like that as well, so yes please.

-- 
Bye, Peter Korsgaard

[Buildroot] [PATCH 7/7] automatically set PARALLEL_JOBS when BR2_JLEVEL is 0

[Nathan Lynch]

On Thu, 2012-06-14 at 08:07 +0200, Peter Korsgaard wrote:
> >>>>> "Nathan" == Nathan Lynch <ntl@pobox.com> writes:
> 
>  >> In my experience, you need 2 times the number of CPUs to fully utilize
>  >> the processor because most compilations have long iowait latencies.
> 
>  Nathan> Yeah.  I'm happy to make this change, assuming others agree?
> 
> I was going to say something like that as well, so yes please.

Certainly.  Is an update of patch #7 acceptable or shall I re-send the
series?

[Buildroot] [PATCH 7/7] automatically set PARALLEL_JOBS when BR2_JLEVEL is 0

[Richard Braun]

On Thu, Jun 14, 2012 at 01:27:05AM +0200, Arnout Vandecappelle wrote:
>  In my experience, you need 2 times the number of CPUs to fully utilize
> the processor because most compilations have long iowait latencies.

Unless you have little memory, this is no longer true, and as some
compilations can eat a lot of memory, doubling their number reduces the
amount of available page cache, potentially increasing the amount of
I/O. Using the number of available processors should be just fine.

-- 
Richard Braun

[Buildroot] [PATCH 7/7] automatically set PARALLEL_JOBS when BR2_JLEVEL is 0

[Peter Korsgaard]

>>>>> "Nathan" == Nathan Lynch <ntl@pobox.com> writes:

 Nathan> On Thu, 2012-06-14 at 08:07 +0200, Peter Korsgaard wrote:
 >> >>>>> "Nathan" == Nathan Lynch <ntl@pobox.com> writes:
 >> 
 >> >> In my experience, you need 2 times the number of CPUs to fully utilize
 >> >> the processor because most compilations have long iowait latencies.
 >> 
 Nathan> Yeah.  I'm happy to make this change, assuming others agree?
 >> 
 >> I was going to say something like that as well, so yes please.

 Nathan> Certainly.  Is an update of patch #7 acceptable or shall I
 Nathan> re-send the series?

Just #7 (with a clearly marked [PATCHv2] in subject) is fine if that's
the only change since last time.

-- 
Bye, Peter Korsgaard

[Buildroot] [PATCHv2 7/7] automatically set PARALLEL_JOBS when BR2_JLEVEL is 0

[Nathan Lynch]

When BR2_JLEVEL is 0, set PARALLEL_JOBS to double the number of CPUs
detected.  This allows one to more or less fully utilize the host
system without manually tuning the configuration.

Also make 0 the default value for BR2_JLEVEL.

Signed-off-by: Nathan Lynch <ntl@pobox.com>
---

v2: Updated to use double number of detected CPUs per Arnout and Peter.

 Config.in           |    8 +++++---
 package/Makefile.in |    5 +++++
 2 files changed, 10 insertions(+), 3 deletions(-)

diff --git a/Config.in b/Config.in
index 95c2e8c..3745255 100644
--- a/Config.in
+++ b/Config.in
@@ -178,10 +178,12 @@ config BR2_DEBIAN_MIRROR
 endmenu
 
 config BR2_JLEVEL
-	int "Number of jobs to run simultaneously"
-	default "2"
+	int "Number of jobs to run simultaneously (0 for auto)"
+	default "0"
 	help
-	  Number of jobs to run simultaneously
+	  Number of jobs to run simultaneously.  If 0, determine
+	  automatically according to number of CPUs on the host
+	  system.
 
 config BR2_CCACHE
 	bool "Enable compiler cache"
diff --git a/package/Makefile.in b/package/Makefile.in
index 9177a1b..95e0ae6 100644
--- a/package/Makefile.in
+++ b/package/Makefile.in
@@ -6,7 +6,12 @@ HOSTMAKE=$(MAKE)
 endif
 HOSTMAKE :=$(shell which $(HOSTMAKE) || type -p $(HOSTMAKE) || echo make)
 
+ifeq ($(BR2_JLEVEL),0)
+PARALLEL_JOBS:=$(shell echo \
+	$$((2 * `getconf _NPROCESSORS_ONLN 2>/dev/null || echo 1`)))
+else
 PARALLEL_JOBS:=$(BR2_JLEVEL)
+endif
 
 MAKE1:=$(HOSTMAKE) -j1
 MAKE:=$(HOSTMAKE) -j$(PARALLEL_JOBS)
-- 
1.7.10.2

[Buildroot] [PATCHv2 7/7] automatically set PARALLEL_JOBS when BR2_JLEVEL is 0

[Baruch Siach]

Hi Nathan,

On Thu, Jun 14, 2012 at 10:02:45AM -0500, Nathan Lynch wrote:
> When BR2_JLEVEL is 0, set PARALLEL_JOBS to double the number of CPUs
> detected.  This allows one to more or less fully utilize the host
> system without manually tuning the configuration.
> 
> Also make 0 the default value for BR2_JLEVEL.
> 
> Signed-off-by: Nathan Lynch <ntl@pobox.com>
> ---

[snip]

> +ifeq ($(BR2_JLEVEL),0)
> +PARALLEL_JOBS:=$(shell echo \
> +	$$((2 * `getconf _NPROCESSORS_ONLN 2>/dev/null || echo 1`)))
> +else
>  PARALLEL_JOBS:=$(BR2_JLEVEL)
> +endif

IMO, there should be a comment here explaining the magic (2*CPUs) number.

baruch

-- 
     http://baruch.siach.name/blog/                  ~. .~   Tk Open Systems
=}------------------------------------------------ooO--U--Ooo------------{=
   - baruch at tkos.co.il - tel: +972.2.679.5364, http://www.tkos.co.il -

[Buildroot] [PATCHv3 7/7] automatically set PARALLEL_JOBS when BR2_JLEVEL is 0

[Nathan Lynch]

When BR2_JLEVEL is 0, set PARALLEL_JOBS to double the number of CPUs
detected.  This allows one to more or less fully utilize the host
system without manually tuning the configuration.

Also make 0 the default value for BR2_JLEVEL.

Signed-off-by: Nathan Lynch <ntl@pobox.com>
---

v3: Added comment to PARALLEL_JOBS calculation per Baruch.

v2: Updated to use double number of detected CPUs per Arnout and Peter.

 Config.in           |    8 +++++---
 package/Makefile.in |    8 ++++++++
 2 files changed, 13 insertions(+), 3 deletions(-)

diff --git a/Config.in b/Config.in
index 95c2e8c..3745255 100644
--- a/Config.in
+++ b/Config.in
@@ -178,10 +178,12 @@ config BR2_DEBIAN_MIRROR
 endmenu
 
 config BR2_JLEVEL
-	int "Number of jobs to run simultaneously"
-	default "2"
+	int "Number of jobs to run simultaneously (0 for auto)"
+	default "0"
 	help
-	  Number of jobs to run simultaneously
+	  Number of jobs to run simultaneously.  If 0, determine
+	  automatically according to number of CPUs on the host
+	  system.
 
 config BR2_CCACHE
 	bool "Enable compiler cache"
diff --git a/package/Makefile.in b/package/Makefile.in
index 9177a1b..ffcb9b6 100644
--- a/package/Makefile.in
+++ b/package/Makefile.in
@@ -6,7 +6,15 @@ HOSTMAKE=$(MAKE)
 endif
 HOSTMAKE :=$(shell which $(HOSTMAKE) || type -p $(HOSTMAKE) || echo make)
 
+# If BR2_LEVEL is 0, scale the maximum concurrency with the number of
+# CPUs.  A coefficient of 2 is used in order to keep processors busy
+# while waiting on I/O.
+ifeq ($(BR2_JLEVEL),0)
+PARALLEL_JOBS:=$(shell echo \
+	$$((2 * `getconf _NPROCESSORS_ONLN 2>/dev/null || echo 1`)))
+else
 PARALLEL_JOBS:=$(BR2_JLEVEL)
+endif
 
 MAKE1:=$(HOSTMAKE) -j1
 MAKE:=$(HOSTMAKE) -j$(PARALLEL_JOBS)
-- 
1.7.10.2

[Buildroot] [PATCH 0/7] automatically determine job concurrency

[Peter Korsgaard]

>>>>> "Nathan" == Nathan Lynch <ntl@pobox.com> writes:

 Nathan> This series implements the option of "autodetecting" the number of
 Nathan> jobs to run concurrently (make -jN), using the number of reported CPUs
 Nathan> when BR2_JLEVEL is set to 0.

Committed series, thanks!

-- 
Bye, Peter Korsgaard