[Buildroot] [PATCH 0 of 5 v6] dependencies: some improvements

[Buildroot] [PATCH 0 of 5 v6] dependencies: some improvements

[Thomas De Schampheleire]

As far as I'm concerned, this should be the last version of this patch
series. I think it is clean and complete and has received the necessary review.

Signed-off-by: Thomas De Schampheleire <thomas.de.schampheleire@gmail.com>

---
v4: add Acks by ThomasP and update after feedback
v5: add extra patch to fix download problem because dl dir is absent
v6: use $(@D) and line up with latest git

[Buildroot] [PATCH 1 of 5 v6] dependencies: move from toolchain/ to support/

[Thomas De Schampheleire]

As suggested by Arnout Vandecappelle, move toolchain/dependencies to
support/dependencies, as it really is not toolchain-specific anymore.

Signed-off-by: Thomas De Schampheleire <thomas.de.schampheleire@gmail.com>
Acked-by: Thomas Petazzoni <thomas.petazzoni@free-electrons.com>
Acked-by: Arnout Vandecappelle (Essensium/Mind) <arnout@mind.be>

---
 Makefile                               |    2 ++
 toolchain/dependencies/dependencies.mk |    2 +-
 toolchain/dependencies/dependencies.sh |  Bin
 toolchain/toolchain-buildroot.mk       |    1 -
 toolchain/toolchain-crosstool-ng.mk    |    1 -
 toolchain/toolchain-external.mk        |    1 -
 6 files changed, 3 insertions(+), 4 deletions(-)

diff --git a/Makefile b/Makefile
--- a/Makefile
+++ b/Makefile
@@ -297,6 +297,8 @@ include package/Makefile.in
 
 all: world
 
+include support/dependencies/dependencies.mk
+
 # We also need the various per-package makefiles, which also add
 # each selected package to TARGETS if that package was selected
 # in the .config file.
diff --git a/toolchain/dependencies/dependencies.mk b/support/dependencies/dependencies.mk
rename from toolchain/dependencies/dependencies.mk
rename to support/dependencies/dependencies.mk
--- a/toolchain/dependencies/dependencies.mk
+++ b/support/dependencies/dependencies.mk
@@ -14,7 +14,7 @@ core-dependencies:
 	@HOSTCC="$(firstword $(HOSTCC))" MAKE="$(MAKE)" \
 		CONFIG_FILE="$(CONFIG_DIR)/.config" \
 		DL_TOOLS="$(sort $(DL_TOOLS_DEPENDENCIES))" \
-		$(TOPDIR)/toolchain/dependencies/dependencies.sh
+		$(TOPDIR)/support/dependencies/dependencies.sh
 
 dependencies: core-dependencies $(DEPENDENCIES_HOST_PREREQ)
 
diff --git a/toolchain/dependencies/dependencies.sh b/support/dependencies/dependencies.sh
rename from toolchain/dependencies/dependencies.sh
rename to support/dependencies/dependencies.sh
diff --git a/toolchain/toolchain-buildroot.mk b/toolchain/toolchain-buildroot.mk
--- a/toolchain/toolchain-buildroot.mk
+++ b/toolchain/toolchain-buildroot.mk
@@ -1,6 +1,5 @@
 # Include files required for the internal toolchain backend
 
-include toolchain/dependencies/dependencies.mk
 include toolchain/elf2flt/elf2flt.mk
 include toolchain/gcc/gcc-uclibc-4.x.mk
 include toolchain/gdb/gdb.mk
diff --git a/toolchain/toolchain-crosstool-ng.mk b/toolchain/toolchain-crosstool-ng.mk
--- a/toolchain/toolchain-crosstool-ng.mk
+++ b/toolchain/toolchain-crosstool-ng.mk
@@ -2,7 +2,6 @@
 
 # Explicit ordering:
 include toolchain/helpers.mk
-include toolchain/dependencies/dependencies.mk
 include toolchain/elf2flt/elf2flt.mk
 include toolchain/gcc/gcc-uclibc-4.x.mk
 include toolchain/gdb/gdb.mk
diff --git a/toolchain/toolchain-external.mk b/toolchain/toolchain-external.mk
--- a/toolchain/toolchain-external.mk
+++ b/toolchain/toolchain-external.mk
@@ -1,7 +1,6 @@
 # Required includes for the external toolchain backend
 
 include toolchain/helpers.mk
-include toolchain/dependencies/dependencies.mk
 include toolchain/elf2flt/elf2flt.mk
 include toolchain/gcc/gcc-uclibc-4.x.mk
 include toolchain/gdb/gdb.mk

[Buildroot] [PATCH 1 of 5 v6] dependencies: move from toolchain/ to support/

[Peter Korsgaard]

>>>>> "Thomas" == Thomas De Schampheleire <patrickdepinguin+buildroot@gmail.com> writes:

 Thomas> As suggested by Arnout Vandecappelle, move toolchain/dependencies to
 Thomas> support/dependencies, as it really is not toolchain-specific anymore.

Committed, thanks.

-- 
Bye, Peter Korsgaard

[Buildroot] [PATCH 2 of 5 v6] dependencies: check minimal make version early on

[Thomas De Schampheleire]

Although support/dependencies/dependencies.sh checks for the version of make,
this script doesn't get a chance to run if make encounters a syntax error as a
result of being too old.

For example, the following syntax is only supported from make 3.81 onwards:

ifeq ($(BR2_TOOLCHAIN_BUILDROOT),y)
include toolchain/toolchain-buildroot.mk
else ifeq ($(BR2_TOOLCHAIN_EXTERNAL),y)
include toolchain/toolchain-external.mk
else ifeq ($(BR2_TOOLCHAIN_CTNG),y)
include toolchain/toolchain-crosstool-ng.mk
endif

This patch adds a check for the version of make very early in the Makefile, so
that old make versions are handled gracefully.

Signed-off-by: Thomas De Schampheleire <thomas.de.schampheleire@gmail.com>
Acked-by: Thomas Petazzoni <thomas.petazzoni@free-electrons.com>

---
 Makefile |  6 ++++++
 1 files changed, 6 insertions(+), 0 deletions(-)

diff --git a/Makefile b/Makefile
--- a/Makefile
+++ b/Makefile
@@ -26,6 +26,12 @@
 # Set and export the version string
 export BR2_VERSION:=2012.02-git
 
+# Check for minimal make version (note: this check will break at make 10.x)
+MIN_MAKE_VERSION=3.81
+ifeq (,$(filter $(MIN_MAKE_VERSION),$(firstword $(sort $(MAKE_VERSION) $(MIN_MAKE_VERSION)))))
+$(error You have make '$(MAKE_VERSION)' installed. GNU make >= $(MIN_MAKE_VERSION) is required)
+endif
+
 # This top-level Makefile can *not* be executed in parallel
 .NOTPARALLEL:
 

[Buildroot] [PATCH 2 of 5 v6] dependencies: check minimal make version early on

[Peter Korsgaard]

>>>>> "Thomas" == Thomas De Schampheleire <patrickdepinguin+buildroot@gmail.com> writes:

 Thomas> Although support/dependencies/dependencies.sh checks for the
 Thomas> version of make, this script doesn't get a chance to run if
 Thomas> make encounters a syntax error as a result of being too old.

 Thomas>  export BR2_VERSION:=2012.02-git
 
 Thomas> +# Check for minimal make version (note: this check will break
 Thomas> at make 10.x)

 Thomas> +MIN_MAKE_VERSION=3.81
 Thomas> +ifeq (,$(filter $(MIN_MAKE_VERSION),$(firstword $(sort $(MAKE_VERSION) $(MIN_MAKE_VERSION)))))

It seems simpler to just compare with MAKE_VERSION, so I changed it to
be:

ifeq ($(firstword $(sort $(MAKE_VERSION) $(MIN_MAKE_VERSION))),$(MAKE_VERSION))

Committed, thanks.

-- 
Bye, Peter Korsgaard

[Buildroot] [PATCH 2 of 5 v6] dependencies: check minimal make version early on

[Peter Korsgaard]

>>>>> "Peter" == Peter Korsgaard <jacmet@uclibc.org> writes:

 Peter> It seems simpler to just compare with MAKE_VERSION, so I changed it to
 Peter> be:

 Peter> ifeq ($(firstword $(sort $(MAKE_VERSION) $(MIN_MAKE_VERSION))),$(MAKE_VERSION))

Ehh, swap that:

ifneq ($(firstword $(sort $(MAKE_VERSION) $(MIN_MAKE_VERSION))),$(MIN_MAKE_VERSION))

Otherwise it would fail for 3.81.

-- 
Bye, Peter Korsgaard

[Buildroot] [PATCH 3 of 5 v6] dependencies: add function suitable-host-package

[Thomas De Schampheleire]

Sometimes, buildroot needs a certain host tool to do its job, e.g. tar. In
many cases, we expect this tool to be present on the host system, but this is
not always the case. Or maybe, the version on the host system is not
suitable, and we need a more recent one.

In some of these cases, instead of bailing out, buildroot could build the
package first (but only if the existing system package is not suitable).

To aid in detecting if a host package is suitable or not, this patch adds a
function suitable-host-package. When called with parameter foo, it will
execute check-host-foo.sh. This script should return either the path to the
suitable host package, or the empty string if no suitable package can be found.
The rules to determine whether something is suitable or not is left to
check-host-foo.sh and depends on foo.

An example usage of suitable-host-package is:
DEPENDENCIES_HOST_PREREQ += $(if $(call suitable-host-package,foo),,host-foo)

To avoid cluttering the existing dependencies.mk file, it includes any
check-host-foo.mk file. These files can be used to hold appropriate
dependency-related actions for foo.

Signed-off-by: Thomas De Schampheleire <thomas.de.schampheleire@gmail.com>
v1 Reviewed-by: Arnout Vandecappelle (Essensium/Mind) <arnout@mind.be>
Acked-by: Thomas Petazzoni <thomas.petazzoni@free-electrons.com>

---
v3: Add optional second parameter to suitable-host-package, to specify a
    candidate tool.

 support/dependencies/dependencies.mk |  10 ++++++++++
 1 files changed, 10 insertions(+), 0 deletions(-)

diff --git a/support/dependencies/dependencies.mk b/support/dependencies/dependencies.mk
--- a/support/dependencies/dependencies.mk
+++ b/support/dependencies/dependencies.mk
@@ -6,6 +6,16 @@
 ######################################################################
 
 DEPENDENCIES_HOST_PREREQ:=
+
+# suitable-host-pkg: calls check-host-$(1).sh shell script. Parameter (2)
+# can be the candidate to be checked. If not present, the check-host-$(1).sh
+# script should use 'which' to find a candidate. The script should return
+# the path to the suitable host tool, or nothing if no suitable tool was found.
+define suitable-host-package
+$(shell support/dependencies/check-host-$(1).sh $(2))
+endef
+-include support/dependencies/check-host-*.mk
+
 ifeq ($(BR2_STRIP_sstrip),y)
 DEPENDENCIES_HOST_PREREQ+=host-sstrip
 endif

[Buildroot] [PATCH 3 of 5 v6] dependencies: add function suitable-host-package

[Peter Korsgaard]

>>>>> "Thomas" == Thomas De Schampheleire <patrickdepinguin+buildroot@gmail.com> writes:

 Thomas> Sometimes, buildroot needs a certain host tool to do its job,
 Thomas> e.g. tar. In many cases, we expect this tool to be present on
 Thomas> the host system, but this is

Committed, thanks.

-- 
Bye, Peter Korsgaard

[Buildroot] [PATCH 4 of 5 v6] Makefile: change order of dirs and dependencies

[Thomas De Schampheleire]

If during the dependencies step, a package needs to be downloaded, the
download directory already has to be present. If not, the file will be
downloaded under the name 'dl' instead of in the directory 'dl'.

This patch changes the order of dirs and dependencies in the world target to
fix this.

Signed-off-by: Thomas De Schampheleire <thomas.de.schampheleire@gmail.com>

---
 Makefile |  2 +-
 1 files changed, 1 insertions(+), 1 deletions(-)

diff --git a/Makefile b/Makefile
--- a/Makefile
+++ b/Makefile
@@ -375,7 +375,7 @@ dirs: $(DL_DIR) $(TOOLCHAIN_DIR) $(BUILD
 
 prepare: $(BUILD_DIR)/buildroot-config/auto.conf
 
-world: prepare dependencies dirs $(BASE_TARGETS) $(TARGETS_ALL)
+world: prepare dirs dependencies $(BASE_TARGETS) $(TARGETS_ALL)
 
 $(O)/toolchainfile.cmake:
 	@echo -en "\

[Buildroot] [PATCH 4 of 5 v6] Makefile: change order of dirs and dependencies

[Peter Korsgaard]

>>>>> "Thomas" == Thomas De Schampheleire <patrickdepinguin+buildroot@gmail.com> writes:

 Thomas> If during the dependencies step, a package needs to be downloaded, the
 Thomas> download directory already has to be present. If not, the file will be
 Thomas> downloaded under the name 'dl' instead of in the directory 'dl'.

 Thomas> This patch changes the order of dirs and dependencies in the
 Thomas> world target to fix this.

Committed, thanks.

-- 
Bye, Peter Korsgaard

[Buildroot] [PATCH 5 of 5 v6] dependencies: build a host-tar if no suitable tar can be found

[Thomas De Schampheleire]

Some toolchains, like the one built with buildroot itself, use hardlinks (for
example to link between the c++ and g++ binary). Unpacking such a toolchain
with the --strip-components options does not work correctly if the system tar
is too old (<1.17). Even recent releases of RedHat/CentOS still ship with
tar 1.15.

This patch checks for a suitable tar version (tar 1.17+) on the host system,
and adds host-tar to the host dependencies if none can be found.

host-tar is download and extracted as cpio.gz instead of tar.gz, to prevent
chicken-egg problem.

Signed-off-by: Thomas De Schampheleire <thomas.de.schampheleire@gmail.com>
v4 Acked-by: Arnout Vandecappelle (Essensium/Mind) <arnout@mind.be>

---
v3: remove chicken-egg problem, causing great simplification; update check-
    host-tar.sh with Arnout's comments.
v4: use 'define' for HOST_TAR_EXTRACT_CMDS after suggestion by ThomasP
v6: use $(@D) instead of HOST_TAR_DIR, suggestion from Arnout. Additionally,
    I needed to set HOST_TAR_DEPENDENCIES to nothing, after the automatic
    derivation of HOST_ dependencies.

 package/Makefile.package.in            |  12 ------------
 package/tar/tar.mk                     |  13 +++++++++++++
 support/dependencies/check-host-tar.mk |   9 +++++++++
 support/dependencies/check-host-tar.sh |  35 +++++++++++++++++++++++++++++++++++
 4 files changed, 57 insertions(+), 12 deletions(-)

diff --git a/package/Makefile.package.in b/package/Makefile.package.in
--- a/package/Makefile.package.in
+++ b/package/Makefile.package.in
@@ -299,18 +299,6 @@ define DOWNLOAD
 	exit 1
 endef
 
-# Utility programs used to build packages
-TAR ?= tar
-
-# Automatically detect tar --strip-path/components option
-TAR_STRIP_COMPONENTS := \
-  $(shell $(TAR) --help | grep strip-path > /dev/null ; \
-  if test $$? = 0 ; then \
-   echo '--strip-path' ; \
-  else \
-   echo '--strip-components' ; \
-  fi)
-
 # Needed for the foreach loops to loop over the list of hooks, so that
 # each hook call is properly separated by a newline.
 define sep
diff --git a/package/tar/tar.mk b/package/tar/tar.mk
--- a/package/tar/tar.mk
+++ b/package/tar/tar.mk
@@ -10,6 +10,19 @@ TAR_SITE = $(BR2_GNU_MIRROR)/tar
 # Prefer full-blown tar over buybox's version
 ifeq ($(BR2_PACKAGE_BUSYBOX),y)
 TAR_DEPENDENCIES += busybox
+HOST_TAR_DEPENDENCIES =
 endif
 
 $(eval $(call AUTOTARGETS))
+
+# host-tar: use cpio.gz instead of tar.gz to prevent chicken-egg problem
+# of needing tar to build tar.
+HOST_TAR_SOURCE = tar-$(TAR_VERSION).cpio.gz
+define HOST_TAR_EXTRACT_CMDS
+	mkdir -p $(@D)
+	cd $(@D) && \
+		$(INFLATE.gz) $(DL_DIR)/$(HOST_TAR_SOURCE) | cpio -i
+	mv $(@D)/tar-$(TAR_VERSION)/* $(@D)
+	rmdir $(@D)/tar-$(TAR_VERSION)
+endef
+$(eval $(call AUTOTARGETS,host))
diff --git a/support/dependencies/check-host-tar.mk b/support/dependencies/check-host-tar.mk
new file mode 100644
--- /dev/null
+++ b/support/dependencies/check-host-tar.mk
@@ -0,0 +1,9 @@
+TAR ?= tar
+
+ifeq (,$(call suitable-host-package,tar,$(TAR)))
+  DEPENDENCIES_HOST_PREREQ += host-tar
+  TAR = $(HOST_DIR)/usr/bin/tar
+endif
+
+# Since TAR is at least 1.17, it will certainly support --strip-components
+TAR_STRIP_COMPONENTS = --strip-components
diff --git a/support/dependencies/check-host-tar.sh b/support/dependencies/check-host-tar.sh
new file mode 100755
--- /dev/null
+++ b/support/dependencies/check-host-tar.sh
@@ -0,0 +1,35 @@
+#!/bin/sh
+
+candidate="$1"
+
+tar=`which $candidate`
+if [ ! -x "$tar" ]; then
+	tar=`which tar`
+	if [ ! -x "$tar" ]; then
+		# echo nothing: no suitable tar found
+		exit 1
+	fi
+fi
+
+# Output of 'tar --version' examples:
+# tar (GNU tar) 1.15.1
+# tar (GNU tar) 1.25
+version=`$tar --version | head -n 1 | sed 's/^.*\s\([0-9]\+\.\S\+\).*$/\1/'`
+major=`echo "$version" | cut -d. -f1`
+minor=`echo "$version" | cut -d. -f2`
+bugfix=`echo "$version" | cut -d. -f3`
+
+# Minimal version = 1.17 (previous versions do not correctly unpack archives
+# containing hard-links if the --strip-components option is used).
+major_min=1
+minor_min=17
+if [ $major -gt $major_min ]; then
+	echo $tar
+else
+	if [ $major -eq $major_min -a $minor -ge $minor_min ]; then
+		echo $tar
+	else
+		# echo nothing: no suitable tar found
+		exit 1
+	fi
+fi

[Buildroot] [PATCH 5 of 5 v6] dependencies: build a host-tar if no suitable tar can be found

[Peter Korsgaard]

>>>>> "Thomas" == Thomas De Schampheleire <patrickdepinguin+buildroot@gmail.com> writes:

 Thomas> Some toolchains, like the one built with buildroot itself, use
 Thomas> hardlinks (for example to link between the c++ and g++
 Thomas> binary). Unpacking such a toolchain with the --strip-components
 Thomas> options does not work correctly if the system tar is too old
 Thomas> (<1.17). Even recent releases of RedHat/CentOS still ship with
 Thomas> tar 1.15.

Committed, thanks.

-- 
Bye, Peter Korsgaard