[Buildroot] [PATCH 1/3] toolchain-external: fix the SYSROOT_DIR mangling logic

[Buildroot] [PATCH 1/3] toolchain-external: fix the SYSROOT_DIR mangling logic

[Thomas Petazzoni]

In a1d94aaa3a21911 ('toolchain-external: add support for musl C
library'), we made the following change to the SYSROOT_DIR mangling
logic:

-       SYSROOT_DIR=`echo $${LIBC_A_LOCATION} | sed -r -e 's:usr/lib(32|64)?/(.*/)?libc\.a::'` ; \
+       SYSROOT_DIR=`echo $${LIBC_A_LOCATION} | sed -r -e 's:(usr/)?lib(32|64)?/(.*/)?libc\.a::'` ; \

This was needed to accomodate for musl based toolchains that don't
have libc.a in usr/lib/..., but directory in lib/... Basically, the
change makes the usr/ at the beginning optional.

However, with the very permissive (.*) matching in the middle of the
path, the change above had an unexpected consequence: any path contain
'/lib' would be truncated before this lib. As an example, Peter
reported that his builds, running from /var/lib/buildbot/ were no
longer working because the SYSROOT_DIR was decided to be /var instead
of something like
/var/lib/buildbot/buildroot/output/host/opt/ext-toolchain/arm-linux-gnueabihf/libc/.

So, this commit changes (again!) this regexp by changing (.*) to
([^/]*), the idea being that it will match only *one* path
component. Note that this intermediate (.*) directory was added in
e6e60becb008 ('external-toolchain: add support for Linaro 2012.01') to
accomodate for Linaro toolchains that have a subdirectory in their
sysroot named after the target tuple:

$ ./output/host/opt/ext-toolchain/bin/arm-linux-gnueabihf-gcc -print-file-name=libc.a
/home/thomas/projets/buildroot/output/host/opt/ext-toolchain/bin/../arm-linux-gnueabihf/libc/usr/lib/arm-linux-gnueabihf/libc.a

In addition to this, this commit also makes sure that the change
making usr/ optional is properly reported on all the instances of this
regular expression.

Signed-off-by: Thomas Petazzoni <thomas.petazzoni@free-electrons.com>
---
 toolchain/toolchain-external/toolchain-external.mk | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/toolchain/toolchain-external/toolchain-external.mk b/toolchain/toolchain-external/toolchain-external.mk
index 2722600..e669d21 100644
--- a/toolchain/toolchain-external/toolchain-external.mk
+++ b/toolchain/toolchain-external/toolchain-external.mk
@@ -381,7 +381,7 @@ define TOOLCHAIN_EXTERNAL_CONFIGURE_CMDS
 	$(Q)$(call check_cross_compiler_exists,$(TOOLCHAIN_EXTERNAL_CC))
 	$(Q)$(call check_unusable_toolchain,$(TOOLCHAIN_EXTERNAL_CC))
 	$(Q)LIBC_A_LOCATION=`readlink -f $$(LANG=C $(TOOLCHAIN_EXTERNAL_CC) -print-file-name=libc.a)` ; \
-	SYSROOT_DIR=`echo $${LIBC_A_LOCATION} | sed -r -e 's:(usr/)?lib(32|64)?/(.*/)?libc\.a::'` ; \
+	SYSROOT_DIR=`echo $${LIBC_A_LOCATION} | sed -r -e 's:(usr/)?lib(32|64)?/([^/]*/)?libc\.a::'` ; \
 	if test -z "$${SYSROOT_DIR}" ; then \
 		@echo "External toolchain doesn't support --sysroot. Cannot use." ; \
 		exit 1 ; \
@@ -463,14 +463,14 @@ endif
 
 define TOOLCHAIN_EXTERNAL_INSTALL_CORE
 	$(Q)LIBC_A_LOCATION=`readlink -f $$(LANG=C $(TOOLCHAIN_EXTERNAL_CC) -print-file-name=libc.a)` ; \
-	SYSROOT_DIR=`echo $${LIBC_A_LOCATION} | sed -r -e 's:usr/lib(32|64)?/(.*/)?libc\.a::'` ; \
+	SYSROOT_DIR=`echo $${LIBC_A_LOCATION} | sed -r -e 's:(usr/)?lib(32|64)?/([^/]*/)?libc\.a::'` ; \
 	if test -z "$${SYSROOT_DIR}" ; then \
 		@echo "External toolchain doesn't support --sysroot. Cannot use." ; \
 		exit 1 ; \
 	fi ; \
 	ARCH_LIBC_A_LOCATION=`readlink -f $$(LANG=C $(TOOLCHAIN_EXTERNAL_CC) $(TOOLCHAIN_EXTERNAL_CFLAGS) -print-file-name=libc.a)` ; \
-	ARCH_SYSROOT_DIR=`echo $${ARCH_LIBC_A_LOCATION} | sed -r -e 's:usr/lib(32|64)?/(.*/)?libc\.a::'` ; \
-	ARCH_LIB_DIR=`echo $${ARCH_LIBC_A_LOCATION} | sed -r -e 's:.*/usr/(lib(32|64)?)/(.*/)?libc.a:\1:'` ; \
+	ARCH_SYSROOT_DIR=`echo $${ARCH_LIBC_A_LOCATION} | sed -r -e 's:(usr/)?lib(32|64)?/([^/]*/)?libc\.a::'` ; \
+	ARCH_LIB_DIR=`echo $${ARCH_LIBC_A_LOCATION} | sed -r -e 's:.*/(usr/)?(lib(32|64)?)/([^/]*/)?libc.a:\2:'` ; \
 	SUPPORT_LIB_DIR="" ; \
 	if test `find $${ARCH_SYSROOT_DIR} -name 'libstdc++.a' | wc -l` -eq 0 ; then \
 		LIBSTDCPP_A_LOCATION=$$(LANG=C $(TOOLCHAIN_EXTERNAL_CC) $(TOOLCHAIN_EXTERNAL_CFLAGS) -print-file-name=libstdc++.a) ; \
@@ -516,8 +516,8 @@ define TOOLCHAIN_EXTERNAL_INSTALL_BFIN_FDPIC
 	$(Q)$(call MESSAGE,"Install external toolchain FDPIC libraries to target...") ; \
 	FDPIC_EXTERNAL_CC=$(dir $(TOOLCHAIN_EXTERNAL_CC))/../../bfin-linux-uclibc/bin/bfin-linux-uclibc-gcc ; \
 	FDPIC_LIBC_A_LOCATION=`readlink -f $$(LANG=C $${FDPIC_EXTERNAL_CC} $(TOOLCHAIN_EXTERNAL_CFLAGS) -print-file-name=libc.a)` ; \
-	FDPIC_SYSROOT_DIR=`echo $${FDPIC_LIBC_A_LOCATION} | sed -r -e 's:usr/lib(32|64)?/(.*/)?libc\.a::'` ; \
-	FDPIC_LIB_DIR=`echo $${FDPIC_LIBC_A_LOCATION} | sed -r -e 's:.*/usr/(lib(32|64)?)/(.*/)?libc.a:\1:'` ; \
+	FDPIC_SYSROOT_DIR=`echo $${FDPIC_LIBC_A_LOCATION} | sed -r -e 's:(usr/)?lib(32|64)?/([^/]*/)?libc\.a::'` ; \
+	FDPIC_LIB_DIR=`echo $${FDPIC_LIBC_A_LOCATION} | sed -r -e 's:.*/(usr/)?(lib(32|64)?)/([^/]*/)?libc.a:\2:'` ; \
 	FDPIC_SUPPORT_LIB_DIR="" ; \
 	if test `find $${FDPIC_SYSROOT_DIR} -name 'libstdc++.a' | wc -l` -eq 0 ; then \
 	        FDPIC_LIBSTDCPP_A_LOCATION=$$(LANG=C $${FDPIC_EXTERNAL_CC} $(TOOLCHAIN_EXTERNAL_CFLAGS) -print-file-name=libstdc++.a) ; \
-- 
1.8.1.2

[Buildroot] [PATCH 2/3] toolchain-external: update comments

[Thomas Petazzoni]

Signed-off-by: Thomas Petazzoni <thomas.petazzoni@free-electrons.com>
---
 toolchain/toolchain-external/toolchain-external.mk | 42 ++++++++++++----------
 1 file changed, 24 insertions(+), 18 deletions(-)

diff --git a/toolchain/toolchain-external/toolchain-external.mk b/toolchain/toolchain-external/toolchain-external.mk
index e669d21..a423f75 100644
--- a/toolchain/toolchain-external/toolchain-external.mk
+++ b/toolchain/toolchain-external/toolchain-external.mk
@@ -13,14 +13,21 @@
 #
 #  * Toolchains generated by Crosstool-NG
 #  * Toolchains generated by Buildroot
-#  * ARM, MIPS and PowerPC toolchains made available by
-#    Codesourcery. For the MIPS toolchain, the -muclibc variant isn't
-#    supported yet, only the default glibc-based variant is.
+#  * Toolchains provided by Linaro for the ARM and AArch64
+#    architectures
+#  * Sourcery CodeBench toolchains (from Mentor Graphics) for the ARM,
+#    MIPS, PowerPC, x86, x86_64 and NIOS 2 architectures. For the MIPS
+#    toolchain, the -muclibc variant isn't # supported yet, only the
+#    default glibc-based variant is.
+#  * Analog Devices toolchains for the Blackfin architecture
+#  * Xilinx toolchains for the Microblaze architecture
 #
 # The basic principle is the following
 #
 #  1. If the toolchain is not pre-installed, download and extract it
-#  in $(TOOLCHAIN_EXTERNAL_INSTALL_DIR).
+#  in $(TOOLCHAIN_EXTERNAL_INSTALL_DIR). Otherwise,
+#  $(TOOLCHAIN_EXTERNAL_INSTALL_DIR) points to were the toolchain has
+#  already been installed by the user.
 #
 #  2. For all external toolchains, perform some checks on the
 #  conformity between the toolchain configuration described in the
@@ -85,22 +92,21 @@ LIB_EXTERNAL_LIBS+=$(call qstrip,$(BR2_TOOLCHAIN_EXTRA_EXTERNAL_LIBS))
 
 # Details about sysroot directory selection.
 #
-# To find the sysroot directory:
+# To find the sysroot directory, we use the trick of looking for the
+# 'libc.a' file with the -print-file-name gcc option, and then
+# mangling the path to find the base directory of the sysroot.
 #
-#  * We first try the -print-sysroot option, available in gcc 4.4.x
-#    and in some Codesourcery toolchains.
+# Note that we do not use the -print-sysroot option, because it is
+# only available since gcc 4.4.x, and we still support 4.2.x (for
+# AVR32) and 4.3.x.
 #
-#  * If this option is not available, we fallback to the value of
-#    --with-sysroot as visible in CROSS-gcc -v.
-#
-# When doing those tests, we don't pass any option to gcc that could
-# select a multilib variant (such as -march) as we want the "main"
-# sysroot, which contains all variants of the C library in the case of
-# multilib toolchains. We use the TARGET_CC_NO_SYSROOT variable, which
-# is the path of the cross-compiler, without the
-# --sysroot=$(STAGING_DIR), since what we want to find is the location
-# of the original toolchain sysroot. This "main" sysroot directory is
-# stored in SYSROOT_DIR.
+# When doing this, we don't pass any option to gcc that could select a
+# multilib variant (such as -march) as we want the "main" sysroot,
+# which contains all variants of the C library in the case of multilib
+# toolchains. We use the TARGET_CC_NO_SYSROOT variable, which is the
+# path of the cross-compiler, without the --sysroot=$(STAGING_DIR),
+# since what we want to find is the location of the original toolchain
+# sysroot. This "main" sysroot directory is stored in SYSROOT_DIR.
 #
 # Then, multilib toolchains are a little bit more complicated, since
 # they in fact have multiple sysroots, one for each variant supported
-- 
1.8.1.2

[Buildroot] [PATCH 3/3] toolchain-external: factorize regular expressions

[Thomas Petazzoni]

Based on a suggestion from Peter, this commit factorizes the logic and
regular expressions that are used to find the sysroot and libdir for a
given compiler. It reduces a bit the duplication of code, and
centralizes the most bizarre part of this logic in one place.

Signed-off-by: Thomas Petazzoni <thomas.petazzoni@free-electrons.com>
---
 toolchain/toolchain-external/toolchain-external.mk | 32 +++++++++++++++-------
 1 file changed, 22 insertions(+), 10 deletions(-)

diff --git a/toolchain/toolchain-external/toolchain-external.mk b/toolchain/toolchain-external/toolchain-external.mk
index a423f75..aaa4380 100644
--- a/toolchain/toolchain-external/toolchain-external.mk
+++ b/toolchain/toolchain-external/toolchain-external.mk
@@ -379,6 +379,22 @@ define TOOLCHAIN_EXTERNAL_EXTRACT_CMDS
 endef
 endif
 
+# Returns the location of the libc.a file for the given compiler + flags
+define toolchain_find_libc_a
+$$(readlink -f $$(LANG=C $(1) -print-file-name=libc.a))
+endef
+
+# Returns the sysroot location for the given compiler + flags
+define toolchain_find_sysroot
+$$(echo -n $(call toolchain_find_libc_a,$(1)) | sed -r -e 's:(usr/)?lib(32|64)?/([^/]*/)?libc\.a::')
+endef
+
+# Returns the lib subdirectory for the given compiler + flags (i.e
+# typically lib32 or lib64 for some toolchains)
+define toolchain_find_libdir
+$$(echo -n $(call toolchain_find_libc_a,$(1)) | sed -r -e 's:.*/(usr/)?(lib(32|64)?)/([^/]*/)?libc.a:\2:')
+endef
+
 # Checks for an already installed toolchain: check the toolchain
 # location, check that it supports sysroot, and then verify that it
 # matches the configuration provided in Buildroot: ABI, C++ support,
@@ -386,8 +402,7 @@ endif
 define TOOLCHAIN_EXTERNAL_CONFIGURE_CMDS
 	$(Q)$(call check_cross_compiler_exists,$(TOOLCHAIN_EXTERNAL_CC))
 	$(Q)$(call check_unusable_toolchain,$(TOOLCHAIN_EXTERNAL_CC))
-	$(Q)LIBC_A_LOCATION=`readlink -f $$(LANG=C $(TOOLCHAIN_EXTERNAL_CC) -print-file-name=libc.a)` ; \
-	SYSROOT_DIR=`echo $${LIBC_A_LOCATION} | sed -r -e 's:(usr/)?lib(32|64)?/([^/]*/)?libc\.a::'` ; \
+	$(Q)SYSROOT_DIR="$(call toolchain_find_sysroot,$(TOOLCHAIN_EXTERNAL_CC))" ; \
 	if test -z "$${SYSROOT_DIR}" ; then \
 		@echo "External toolchain doesn't support --sysroot. Cannot use." ; \
 		exit 1 ; \
@@ -468,15 +483,13 @@ endif
 #                       to the target filesystem.
 
 define TOOLCHAIN_EXTERNAL_INSTALL_CORE
-	$(Q)LIBC_A_LOCATION=`readlink -f $$(LANG=C $(TOOLCHAIN_EXTERNAL_CC) -print-file-name=libc.a)` ; \
-	SYSROOT_DIR=`echo $${LIBC_A_LOCATION} | sed -r -e 's:(usr/)?lib(32|64)?/([^/]*/)?libc\.a::'` ; \
+	$(Q)SYSROOT_DIR="$(call toolchain_find_sysroot,$(TOOLCHAIN_EXTERNAL_CC))" ; \
 	if test -z "$${SYSROOT_DIR}" ; then \
 		@echo "External toolchain doesn't support --sysroot. Cannot use." ; \
 		exit 1 ; \
 	fi ; \
-	ARCH_LIBC_A_LOCATION=`readlink -f $$(LANG=C $(TOOLCHAIN_EXTERNAL_CC) $(TOOLCHAIN_EXTERNAL_CFLAGS) -print-file-name=libc.a)` ; \
-	ARCH_SYSROOT_DIR=`echo $${ARCH_LIBC_A_LOCATION} | sed -r -e 's:(usr/)?lib(32|64)?/([^/]*/)?libc\.a::'` ; \
-	ARCH_LIB_DIR=`echo $${ARCH_LIBC_A_LOCATION} | sed -r -e 's:.*/(usr/)?(lib(32|64)?)/([^/]*/)?libc.a:\2:'` ; \
+	ARCH_SYSROOT_DIR="$(call toolchain_find_sysroot,$(TOOLCHAIN_EXTERNAL_CC) $(TOOLCHAIN_EXTERNAL_CFLAGS))" ; \
+	ARCH_LIB_DIR="$(call toolchain_find_libdir,$(TOOLCHAIN_EXTERNAL_CC) $(TOOLCHAIN_EXTERNAL_CFLAGS))" ; \
 	SUPPORT_LIB_DIR="" ; \
 	if test `find $${ARCH_SYSROOT_DIR} -name 'libstdc++.a' | wc -l` -eq 0 ; then \
 		LIBSTDCPP_A_LOCATION=$$(LANG=C $(TOOLCHAIN_EXTERNAL_CC) $(TOOLCHAIN_EXTERNAL_CFLAGS) -print-file-name=libstdc++.a) ; \
@@ -521,9 +534,8 @@ ifeq ($(BR2_BFIN_INSTALL_FDPIC_SHARED),y)
 define TOOLCHAIN_EXTERNAL_INSTALL_BFIN_FDPIC
 	$(Q)$(call MESSAGE,"Install external toolchain FDPIC libraries to target...") ; \
 	FDPIC_EXTERNAL_CC=$(dir $(TOOLCHAIN_EXTERNAL_CC))/../../bfin-linux-uclibc/bin/bfin-linux-uclibc-gcc ; \
-	FDPIC_LIBC_A_LOCATION=`readlink -f $$(LANG=C $${FDPIC_EXTERNAL_CC} $(TOOLCHAIN_EXTERNAL_CFLAGS) -print-file-name=libc.a)` ; \
-	FDPIC_SYSROOT_DIR=`echo $${FDPIC_LIBC_A_LOCATION} | sed -r -e 's:(usr/)?lib(32|64)?/([^/]*/)?libc\.a::'` ; \
-	FDPIC_LIB_DIR=`echo $${FDPIC_LIBC_A_LOCATION} | sed -r -e 's:.*/(usr/)?(lib(32|64)?)/([^/]*/)?libc.a:\2:'` ; \
+	FDPIC_SYSROOT_DIR="$(call toolchain_find_sysroot,$${FDPIC_EXTERNAL_CC} $(TOOLCHAIN_EXTERNAL_CFLAGS))" ; \
+	FDPIC_LIB_DIR="$(call toolchain_find_libdir,$${FDPIC_EXTERNAL_CC} $(TOOLCHAIN_EXTERNAL_CFLAGS))" ; \
 	FDPIC_SUPPORT_LIB_DIR="" ; \
 	if test `find $${FDPIC_SYSROOT_DIR} -name 'libstdc++.a' | wc -l` -eq 0 ; then \
 	        FDPIC_LIBSTDCPP_A_LOCATION=$$(LANG=C $${FDPIC_EXTERNAL_CC} $(TOOLCHAIN_EXTERNAL_CFLAGS) -print-file-name=libstdc++.a) ; \
-- 
1.8.1.2

[Buildroot] [PATCH 2/3] toolchain-external: update comments

[Baruch Siach]

Hi Thomas,

On Thu, Nov 07, 2013 at 12:08:02AM +0100, Thomas Petazzoni wrote:
> Signed-off-by: Thomas Petazzoni <thomas.petazzoni@free-electrons.com>
> ---
>  toolchain/toolchain-external/toolchain-external.mk | 42 ++++++++++++----------
>  1 file changed, 24 insertions(+), 18 deletions(-)
> 
> diff --git a/toolchain/toolchain-external/toolchain-external.mk b/toolchain/toolchain-external/toolchain-external.mk
> index e669d21..a423f75 100644
> --- a/toolchain/toolchain-external/toolchain-external.mk
> +++ b/toolchain/toolchain-external/toolchain-external.mk
> @@ -13,14 +13,21 @@
>  #
>  #  * Toolchains generated by Crosstool-NG
>  #  * Toolchains generated by Buildroot
> -#  * ARM, MIPS and PowerPC toolchains made available by
> -#    Codesourcery. For the MIPS toolchain, the -muclibc variant isn't
> -#    supported yet, only the default glibc-based variant is.
> +#  * Toolchains provided by Linaro for the ARM and AArch64
> +#    architectures
> +#  * Sourcery CodeBench toolchains (from Mentor Graphics) for the ARM,
> +#    MIPS, PowerPC, x86, x86_64 and NIOS 2 architectures. For the MIPS
> +#    toolchain, the -muclibc variant isn't # supported yet, only the

A stray '#' in the line above.

baruch

> +#    default glibc-based variant is.
> +#  * Analog Devices toolchains for the Blackfin architecture
> +#  * Xilinx toolchains for the Microblaze architecture

-- 
     http://baruch.siach.name/blog/                  ~. .~   Tk Open Systems
=}------------------------------------------------ooO--U--Ooo------------{=
   - baruch at tkos.co.il - tel: +972.2.679.5364, http://www.tkos.co.il -

[Buildroot] [PATCH 1/3] toolchain-external: fix the SYSROOT_DIR mangling logic

[Peter Korsgaard]

>>>>> "Thomas" == Thomas Petazzoni <thomas.petazzoni@free-electrons.com> writes:

> In a1d94aaa3a21911 ('toolchain-external: add support for musl C
> library'), we made the following change to the SYSROOT_DIR mangling
> logic:

> -       SYSROOT_DIR=`echo $${LIBC_A_LOCATION} | sed -r -e 's:usr/lib(32|64)?/(.*/)?libc\.a::'` ; \
> +       SYSROOT_DIR=`echo $${LIBC_A_LOCATION} | sed -r -e 's:(usr/)?lib(32|64)?/(.*/)?libc\.a::'` ; \

> This was needed to accomodate for musl based toolchains that don't
> have libc.a in usr/lib/..., but directory in lib/... Basically, the
> change makes the usr/ at the beginning optional.

> However, with the very permissive (.*) matching in the middle of the
> path, the change above had an unexpected consequence: any path contain
> '/lib' would be truncated before this lib. As an example, Peter
> reported that his builds, running from /var/lib/buildbot/ were no
> longer working because the SYSROOT_DIR was decided to be /var instead
> of something like
> /var/lib/buildbot/buildroot/output/host/opt/ext-toolchain/arm-linux-gnueabihf/libc/.

> So, this commit changes (again!) this regexp by changing (.*) to
> ([^/]*), the idea being that it will match only *one* path
> component. Note that this intermediate (.*) directory was added in
> e6e60becb008 ('external-toolchain: add support for Linaro 2012.01') to
> accomodate for Linaro toolchains that have a subdirectory in their
> sysroot named after the target tuple:

> $ ./output/host/opt/ext-toolchain/bin/arm-linux-gnueabihf-gcc -print-file-name=libc.a
> /home/thomas/projets/buildroot/output/host/opt/ext-toolchain/bin/../arm-linux-gnueabihf/libc/usr/lib/arm-linux-gnueabihf/libc.a

> In addition to this, this commit also makes sure that the change
> making usr/ optional is properly reported on all the instances of this
> regular expression.

Committed series, thanks.

-- 
Bye, Peter Korsgaard