[Buildroot] [PATCH 1/5] support/scripts/bl-toolchain-gen: add new script to support Bootlin toolchains

[Thomas Petazzoni <thomas.petazzoni@bootlin.com>]

https://toolchains.bootlin.com/ has been providing for a few years a
number of ready-to-use pre-built toolchains, for a wide range of
architectures (which it turns out, are all built using Buildroot).

While toolchains.bootlin.com provides Buildroot config fragments to
easily use those toolchains with Buildroot (see [0] for example), this
is not visible anywhere. So instead, we would like to add support for
these toolchains in Buildroot just like we have existing support for
Linaro, ARM, Synopsys, etc. toolchains.

[0] https://toolchains.bootlin.com/downloads/releases/toolchains/aarch64/fragments/aarch64--glibc--bleeding-edge-2020.02-2.frag

However, the number of toolchains provided by toolchains.bootlin.com
is really large, and they are regularly updated. Maintaining that
manually would be time consuming and error-prone. So instead, this
commit introduces a script that automatically generates:

 - toolchain/toolchain-external/toolchain-external-bootlin/Config.in.options
 - toolchain/toolchain-external/toolchain-external-bootlin/toolchain-external-bootlin.mk
 - toolchain/toolchain-external/toolchain-external-bootlin/toolchain-external-bootlin.hash
 - support/testing/tests/toolchain/test_external_bootlin.py

We create a single external toolchain package, with a Kconfig "choice"
as a sub-option to select the toolchain variant to be used. The script
contains a Python dict that provides the mapping between the
toolchains provided by toolchains.bootlin.com, and the architecture
options/variants they are applicable to.

The test cases allow to verify that the toolchain configuration is
correct, and that it is able to build a Busybox based system. It
doesn't do any runtime testing as such testing is already done by
toolchains.bootlin.com: the test cases here are only meant to verify
that the toolchain-external-bootlin package works as expected.

Signed-off-by: Thomas Petazzoni <thomas.petazzoni@bootlin.com>
---
 support/scripts/bl-toolchains-gen | 472 ++++++++++++++++++++++++++++++
 1 file changed, 472 insertions(+)
 create mode 100755 support/scripts/bl-toolchains-gen

diff --git a/support/scripts/bl-toolchains-gen b/support/scripts/bl-toolchains-gen
new file mode 100755
index 0000000000..178f531304
--- /dev/null
+++ b/support/scripts/bl-toolchains-gen
@@ -0,0 +1,472 @@
+#!/usr/bin/env python3
+
+import htmllistparse
+import os.path
+import re
+import requests
+import textwrap
+import sys
+
+BASE_URL = "https://toolchains.bootlin.com/downloads/releases/toolchains"
+
+# In the below dict:
+
+# - 'conditions' indicate the cumulative conditions under which the
+#   toolchain will be made available. In several situations, a given
+#   toolchain is usable on several architectures variants (for
+#   example, an ARMv6 toolchain can be used on ARMv7)
+# - 'test_options' indicate one specific configuration where the
+#   toolchain can be used. It is used to create the runtime test
+#   cases. If 'test_options' does not exist, the code assumes it can
+#   be made equal to 'conditions'
+# - 'prefix' is the prefix of the cross-compilation toolchain tools
+
+arches = {
+    'aarch64': {
+        'conditions': ['BR2_aarch64'],
+        'prefix': 'aarch64',
+    },
+    'aarch64be': {
+        'conditions': ['BR2_aarch64_be'],
+        'prefix': 'aarch64_be',
+    },
+    'arcle-750d': {
+        'conditions': ['BR2_arcle', 'BR2_arc750d'],
+        'prefix': 'arc',
+    },
+    'arcle-hs38': {
+        'conditions': ['BR2_arcle', 'BR2_archs38'],
+        'prefix': 'arc',
+    },
+    'armv5-eabi': {
+        'conditions': ['BR2_ARM_CPU_ARMV5', 'BR2_ARM_EABI'],
+        'test_options': ['BR2_arm', 'BR2_arm926t', 'BR2_ARM_EABI'],
+        'prefix': 'arm',
+    },
+    'armv6-eabihf': {
+        'conditions': ['BR2_ARM_CPU_ARMV6', 'BR2_ARM_EABIHF'],
+        'test_options': ['BR2_arm', 'BR2_arm1176jzf_s', 'BR2_ARM_EABIHF'],
+        'prefix': 'arm',
+    },
+    'armv7-eabihf': {
+        'conditions': ['BR2_ARM_CPU_ARMV7A', 'BR2_ARM_EABIHF'],
+        'test_options': ['BR2_arm', 'BR2_cortex_a8', 'BR2_ARM_EABIHF'],
+        'prefix': 'arm',
+    },
+    'armv7m': {
+        'conditions': ['BR2_ARM_CPU_ARMV7M'],
+        'test_options': ['BR2_arm', 'BR2_cortex_m4'],
+        'prefix': 'arm',
+    },
+    'm68k-68xxx': {
+        'conditions': ['BR2_m68k_m68k'],
+        'test_options': ['BR2_m68k', 'BR2_m68k_68040'],
+        'prefix': 'm68k',
+    },
+    'm68k-coldfire': {
+        'conditions': ['BR2_m68k_cf'],
+        'test_options': ['BR2_m68k', 'BR2_m68k_cf5208'],
+        'prefix': 'm68k',
+    },
+    'microblazebe': {
+        'conditions': ['BR2_microblazebe'],
+        'prefix': 'microblaze',
+    },
+    'microblazeel': {
+        'conditions': ['BR2_microblazeel'],
+        'prefix': 'microblazeel',
+    },
+    'mips32': {
+        # Not sure it could be used by other mips32 variants?
+        'conditions': ['BR2_mips', 'BR2_mips_32', '!BR2_MIPS_SOFT_FLOAT'],
+        'prefix': 'mips',
+    },
+    'mips32el': {
+        # Not sure it could be used by other mips32el variants?
+        'conditions': ['BR2_mipsel', 'BR2_mips_32', '!BR2_MIPS_SOFT_FLOAT'],
+        'prefix': 'mipsel',
+    },
+    'mips32r5el': {
+        'conditions': ['BR2_mipsel', 'BR2_mips_32r5', '!BR2_MIPS_SOFT_FLOAT'],
+        'prefix': 'mipsel',
+    },
+    'mips32r6el': {
+        'conditions': ['BR2_mipsel', 'BR2_mips_32r6', '!BR2_MIPS_SOFT_FLOAT'],
+        'prefix': 'mipsel',
+    },
+    'mips64': {
+        # Not sure it could be used by other mips64 variants?
+        'conditions': ['BR2_mips64', 'BR2_mips_64', '!BR2_MIPS_SOFT_FLOAT'],
+        'prefix': 'mips64',
+    },
+    'mips64-n32': {
+        # Not sure it could be used by other mips64 variants?
+        'conditions': ['BR2_mips64', 'BR2_mips_64', 'BR2_MIPS_NABI32', '!BR2_MIPS_SOFT_FLOAT'],
+        'prefix': 'mips64',
+    },
+    'mips64el-n32': {
+        # Not sure it could be used by other mips64el variants?
+        'conditions': ['BR2_mips64el', 'BR2_mips_64', 'BR2_MIPS_NABI32', '!BR2_MIPS_SOFT_FLOAT'],
+        'prefix': 'mips64el',
+    },
+    'mips64r6el-n32': {
+        'conditions': ['BR2_mips64el', 'BR2_mips_64r6', 'BR2_MIPS_NABI32', '!BR2_MIPS_SOFT_FLOAT'],
+        'prefix': 'mips64el',
+    },
+    'nios2': {
+        'conditions': ['BR2_nios2'],
+        'prefix': 'nios2',
+    },
+    'openrisc': {
+        'conditions': ['BR2_or1k'],
+        'prefix': 'or1k',
+    },
+    'powerpc-e500mc': {
+        # Not sure it could be used by other powerpc variants?
+        'conditions': ['BR2_powerpc', 'BR2_powerpc_e500mc'],
+        'prefix': 'powerpc',
+    },
+    'powerpc64-e5500': {
+        'conditions': ['BR2_powerpc64', 'BR2_powerpc_e5500'],
+        'prefix': 'powerpc64',
+    },
+    'powerpc64-power8': {
+        'conditions': ['BR2_powerpc64', 'BR2_powerpc_power8'],
+        'prefix': 'powerpc64',
+    },
+    'powerpc64le-power8': {
+        'conditions': ['BR2_powerpc64le', 'BR2_powerpc_power8'],
+        'prefix': 'powerpc64le',
+    },
+    'riscv32-ilp32d': {
+        'conditions': ['BR2_riscv', 'BR2_riscv_g', 'BR2_RISCV_32', 'BR2_RISCV_ABI_ILP32D'],
+        'prefix': 'riscv32',
+    },
+    'riscv64': {
+        'conditions': ['BR2_riscv', 'BR2_riscv_g', 'BR2_RISCV_64', 'BR2_RISCV_ABI_LP64'],
+        'prefix': 'riscv64',
+    },
+    'sh-sh4': {
+        'conditions': ['BR2_sh', 'BR2_sh4'],
+        'prefix': 'sh4',
+    },
+    'sh-sh4aeb': {
+        'conditions': ['BR2_sh', 'BR2_sh4aeb'],
+        'prefix': 'sh4aeb',
+    },
+    'sparc64': {
+        'conditions': ['BR2_sparc64', 'BR2_sparc_v9'],
+        'prefix': 'sparc64',
+    },
+    'sparcv8': {
+        'conditions': ['BR2_sparc', 'BR2_sparc_v8'],
+        'prefix': 'sparc',
+    },
+    'x86-64-core-i7': {
+        'conditions': ['BR2_x86_64',
+                       'BR2_X86_CPU_HAS_MMX',
+                       'BR2_X86_CPU_HAS_SSE',
+                       'BR2_X86_CPU_HAS_SSE2',
+                       'BR2_X86_CPU_HAS_SSE3',
+                       'BR2_X86_CPU_HAS_SSSE3',
+                       'BR2_X86_CPU_HAS_SSE4',
+                       'BR2_X86_CPU_HAS_SSE42'],
+        'test_options': ['BR2_x86_64', 'BR2_x86_corei7'],
+        'prefix': 'x86_64',
+    },
+    'x86-core2': {
+        'conditions': ['BR2_i386',
+                       'BR2_X86_CPU_HAS_MMX',
+                       'BR2_X86_CPU_HAS_SSE',
+                       'BR2_X86_CPU_HAS_SSE2',
+                       'BR2_X86_CPU_HAS_SSE3',
+                       'BR2_X86_CPU_HAS_SSSE3'],
+        'test_options': ['BR2_i386', 'BR2_x86_core2'],
+        'prefix': 'i686',
+    },
+    'x86-i686': {
+        'conditions': ['BR2_i386',
+                       '!BR2_x86_i486',
+                       '!BR2_x86_i586',
+                       '!BR2_x86_x1000'],
+        'test_options': ['BR2_i386',
+                         'BR2_x86_i686'],
+        'prefix': 'i686',
+    },
+    'xtensa-lx60': {
+        'conditions': ['BR2_xtensa', 'BR2_xtensa_fsf'],
+        'prefix': 'xtensa',
+    },
+}
+
+
+class Toolchain:
+    def __init__(self, arch, libc, variant, version):
+        self.arch = arch
+        self.libc = libc
+        self.variant = variant
+        self.version = version
+        self.fname_prefix = "%s--%s--%s-%s" % (self.arch, self.libc, self.variant, self.version)
+        self.option_name = "BR2_TOOLCHAIN_EXTERNAL_BOOTLIN_%s_%s_%s" % \
+            (self.arch.replace("-", "_").upper(), self.libc.upper(), self.variant.replace("-", "_").upper())
+        self.fragment = requests.get(self.fragment_url()).text.split("\n")
+        self.sha256 = requests.get(self.hash_url()).text.split(" ")[0]
+
+    def tarball_url(self):
+        return os.path.join(BASE_URL, self.arch, "tarballs",
+                            self.fname_prefix + ".tar.bz2")
+
+    def hash_url(self):
+        return os.path.join(BASE_URL, self.arch, "tarballs",
+                            self.fname_prefix + ".sha256")
+
+    def fragment_url(self):
+        return os.path.join(BASE_URL, self.arch, "fragments",
+                            self.fname_prefix + ".frag")
+
+    def gen_config_in_options(self, f):
+        f.write("config %s\n" % self.option_name)
+        f.write("\tbool \"%s %s %s %s\"\n" %
+                (self.arch, self.libc, self.variant, self.version))
+        for c in arches[self.arch]['conditions']:
+            f.write("\tdepends on %s\n" % c)
+        selects = []
+        for frag in self.fragment:
+            # libc type
+            if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_CUSTOM_UCLIBC"):
+                selects.append("BR2_TOOLCHAIN_EXTERNAL_UCLIBC")
+            elif frag.startswith("BR2_TOOLCHAIN_EXTERNAL_CUSTOM_GLIBC"):
+                selects.append("BR2_TOOLCHAIN_EXTERNAL_GLIBC")
+                # all glibc toolchains have RPC support
+                selects.append("BR2_TOOLCHAIN_HAS_NATIVE_RPC")
+            elif frag.startswith("BR2_TOOLCHAIN_EXTERNAL_CUSTOM_MUSL"):
+                selects.append("BR2_TOOLCHAIN_EXTERNAL_MUSL")
+
+            # gcc version
+            if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_GCC_"):
+                m = re.match("^BR2_TOOLCHAIN_EXTERNAL_GCC_([0-9_]*)=y$", frag)
+                if m is None:
+                    print("ERROR: cannot get gcc version for toolchain %s" % self.fname_prefix)
+                    sys.exit(1)
+                selects.append("BR2_TOOLCHAIN_GCC_AT_LEAST_%s" % m[1])
+
+            # kernel headers version
+            if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_HEADERS_"):
+                m = re.match("^BR2_TOOLCHAIN_EXTERNAL_HEADERS_([0-9_]*)=y$", frag)
+                if m is None:
+                    print("ERROR: cannot get kernel headers version for toolchain %s" % self.fname_prefix)
+                    sys.exit(1)
+                selects.append("BR2_TOOLCHAIN_HEADERS_AT_LEAST_%s" % m[1])
+
+            # C++
+            if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_CXX"):
+                selects.append("BR2_INSTALL_LIBSTDCPP")
+
+            # SSP
+            if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_HAS_SSP"):
+                selects.append("BR2_TOOLCHAIN_HAS_SSP")
+
+            # wchar
+            if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_WCHAR"):
+                selects.append("BR2_USE_WCHAR")
+
+            # locale
+            if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_LOCALE"):
+                # locale implies the availability of wchar
+                selects.append("BR2_USE_WCHAR")
+                selects.append("BR2_ENABLE_LOCALE")
+
+            # thread support
+            if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_HAS_THREADS"):
+                selects.append("BR2_TOOLCHAIN_HAS_THREADS")
+
+            if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_HAS_THREADS_DEBUG"):
+                selects.append("BR2_TOOLCHAIN_HAS_THREADS_DEBUG")
+
+            if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_HAS_THREADS_NPTL"):
+                selects.append("BR2_TOOLCHAIN_HAS_THREADS_NPTL")
+
+            # RPC
+            if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_INET_RPC"):
+                selects.append("BR2_TOOLCHAIN_HAS_NATIVE_RPC")
+
+            # D language
+            if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_DLANG"):
+                selects.append("BR2_TOOLCHAIN_HAS_DLANG")
+
+            # fortran
+            if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_FORTRAN"):
+                selects.append("BR2_TOOLCHAIN_HAS_FORTRAN")
+
+            # OpenMP
+            if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_OPENMP"):
+                selects.append("BR2_TOOLCHAIN_HAS_OPENMP")
+
+        for select in selects:
+            f.write("\tselect %s\n" % select)
+
+        f.write("\thelp\n")
+
+        desc = "Bootlin toolchain for the %s architecture, using the %s C library" % \
+            (self.arch, self.libc)
+
+        f.write(textwrap.fill(desc, width=62, initial_indent="\t  ", subsequent_indent="\t  ") + "\n")
+        f.write("\n")
+        f.write("\t  https://toolchains.bootlin.com/\n")
+
+        f.write("\n")
+
+    def gen_mk(self, f):
+        f.write("ifeq ($(%s),y)\n" % self.option_name)
+        f.write("TOOLCHAIN_EXTERNAL_BOOTLIN_VERSION = %s\n" % self.version)
+        f.write("TOOLCHAIN_EXTERNAL_BOOTLIN_SOURCE = %s--%s--%s-$(TOOLCHAIN_EXTERNAL_BOOTLIN_VERSION).tar.bz2\n" %
+                (self.arch, self.libc, self.variant))
+        f.write("TOOLCHAIN_EXTERNAL_BOOTLIN_SITE = %s\n" %
+                os.path.join(BASE_URL, self.arch, "tarballs"))
+        f.write("endif\n\n")
+        pass
+
+    def gen_hash(self, f):
+        f.write("# From %s\n" % self.hash_url())
+        f.write("sha256  %s  %s\n" % (self.sha256, os.path.basename(self.tarball_url())))
+
+    def gen_test(self, f):
+        if self.variant == "stable":
+            variant = "Stable"
+        else:
+            variant = "BleedingEdge"
+        testname = "TestExternalToolchainBootlin" + \
+            self.arch.replace("-", "").capitalize() + \
+            self.libc.capitalize() + variant
+        f.write("\n\n")
+        f.write("class %s(TestExternalToolchain):\n" % testname)
+        f.write("    config = \"\"\"\n")
+        if 'test_options' in arches[self.arch]:
+            test_options = arches[self.arch]['test_options']
+        else:
+            test_options = arches[self.arch]['conditions']
+        for opt in test_options:
+            if opt.startswith("!"):
+                f.write("        # %s is not set\n" % opt[1:])
+            else:
+                f.write("        %s=y\n" % opt)
+        f.write("        BR2_TOOLCHAIN_EXTERNAL=y\n")
+        f.write("        BR2_TOOLCHAIN_EXTERNAL_BOOTLIN=y\n")
+        f.write("        %s=y\n" % self.option_name)
+        f.write("        # BR2_TARGET_ROOTFS_TAR is not set\n")
+        f.write("        \"\"\"\n")
+        f.write("    toolchain_prefix = \"%s-linux\"\n" % arches[self.arch]['prefix'])
+        f.write("\n")
+        f.write("    def test_run(self):\n")
+        f.write("        TestExternalToolchain.common_check(self)\n")
+
+    def __repr__(self):
+        return "Toolchain(arch=%s libc=%s variant=%s version=%s, option=%s)" % \
+            (self.arch, self.libc, self.variant, self.version, self.option_name)
+
+
+def get_toolchains():
+    toolchains = list()
+    for arch, details in arches.items():
+        print(arch)
+        url = os.path.join(BASE_URL, arch, "available_toolchains")
+        cwd, listing = htmllistparse.fetch_listing(url)
+        fnames = [f.name for f in listing]
+        # Sorting the list so we have the most recent version last
+        fnames.sort()
+        # This dict will allow us to keep only the latest version for
+        # each toolchain.
+        tmp = dict()
+        for fname in fnames:
+            parts = fname.split('--')
+            if parts[0] != arch:
+                print("FATAL: arch does not match: %s vs. %s" % (parts[0], arch))
+                sys.exit(1)
+            libc = parts[1]
+            if parts[2].startswith("stable-"):
+                variant = "stable"
+                version = parts[2][len("stable-"):]
+            elif parts[2].startswith("bleeding-edge-"):
+                variant = "bleeding-edge"
+                version = parts[2][len("bleeding-edge-"):]
+            tmp[(arch, libc, variant)] = version
+
+        for k, v in tmp.items():
+            t = Toolchain(k[0], k[1], k[2], v)
+            toolchains.append(t)
+
+    return toolchains
+
+
+def gen_config_in_options(toolchains, fpath):
+    with open(fpath, "w") as f:
+        f.write("# This file is auto-generated by support/scripts/bl-toolchains-gen\n")
+        f.write("# Do not edit\n\n")
+
+        f.write("config BR2_TOOLCHAIN_EXTERNAL_BOOTLIN_ARCH_SUPPORTS\n")
+        f.write("\tbool\n")
+        for arch, details in arches.items():
+            f.write("\tdefault y if %s\n" % " && ".join(details['conditions']))
+        f.write("\n")
+
+        f.write("if BR2_TOOLCHAIN_EXTERNAL_BOOTLIN\n\n")
+
+        f.write("config BR2_TOOLCHAIN_EXTERNAL_PREFIX\n")
+        f.write("\tdefault \"$(ARCH)-linux\"\n")
+
+        f.write("\n")
+
+        f.write("config BR2_PACKAGE_PROVIDES_TOOLCHAIN_EXTERNAL\n")
+        f.write("\tdefault \"toolchain-external-bootlin\"\n")
+
+        f.write("\n")
+
+        f.write("choice\n")
+        f.write("\tprompt \"Bootlin toolchain variant\"\n")
+
+        for toolchain in toolchains:
+            toolchain.gen_config_in_options(f)
+
+        f.write("endchoice\n")
+        f.write("endif\n")
+
+
+def gen_mk(toolchains, fpath):
+    with open(fpath, "w") as f:
+        f.write("#" * 80 + "\n")
+        f.write("#\n")
+        f.write("# toolchain-external-bootlin\n")
+        f.write("#\n")
+        f.write("#" * 80 + "\n")
+        f.write("\n")
+        f.write("# This file is auto-generated by support/scripts/bl-toolchains-gen\n")
+        f.write("# Do not edit\n\n")
+        for toolchain in toolchains:
+            toolchain.gen_mk(f)
+        f.write("$(eval $(toolchain-external-package))\n")
+
+
+def gen_hash(toolchains, fpath):
+    with open(fpath, "w") as f:
+        for toolchain in toolchains:
+            toolchain.gen_hash(f)
+
+
+def gen_runtime_test(toolchains, fpath):
+    with open(fpath, "w") as f:
+        f.write("from tests.toolchain.test_external import TestExternalToolchain\n")
+        for toolchain in toolchains:
+            toolchain.gen_test(f)
+
+
+def gen_toolchains(toolchains):
+    maindir = "toolchain/toolchain-external/toolchain-external-bootlin"
+    gen_config_in_options(toolchains, os.path.join(maindir, "Config.in.options"))
+    gen_mk(toolchains, os.path.join(maindir, "toolchain-external-bootlin.mk"))
+    gen_hash(toolchains, os.path.join(maindir, "toolchain-external-bootlin.hash"))
+    gen_runtime_test(toolchains,
+                     os.path.join("support", "testing", "tests", "toolchain", "test_external_bootlin.py"))
+
+
+toolchains = get_toolchains()
+gen_toolchains(toolchains)
-- 
2.26.2