From mboxrd@z Thu Jan 1 00:00:00 1970 From: Yann E. MORIN Date: Fri, 14 Aug 2020 00:06:01 +0200 Subject: [Buildroot] [PATCH v2 1/5] support/scripts/gen-bootlin-toolchains: add new script to support Bootlin toolchains In-Reply-To: <20200809193818.1139805-2-thomas.petazzoni@bootlin.com> References: <20200809193818.1139805-1-thomas.petazzoni@bootlin.com> <20200809193818.1139805-2-thomas.petazzoni@bootlin.com> Message-ID: <20200813220601.GN13263@scaer> List-Id: MIME-Version: 1.0 Content-Type: text/plain; charset="us-ascii" Content-Transfer-Encoding: 7bit To: buildroot@busybox.net Thomas, All, On 2020-08-09 21:38 +0200, Thomas Petazzoni spake thusly: > https://toolchains.bootlin.com/ has been providing for a few years a > number of ready-to-use pre-built toolchains, for a wide range of > architectures (which it turns out, are all built using Buildroot). [--SNIP--] > Signed-off-by: Thomas Petazzoni [--SNIP--] > diff --git a/support/scripts/gen-bootlin-toolchains b/support/scripts/gen-bootlin-toolchains > new file mode 100755 > index 0000000000..a991797903 > --- /dev/null > +++ b/support/scripts/gen-bootlin-toolchains > @@ -0,0 +1,475 @@ > +#!/usr/bin/env python3 > + > +import htmllistparse Traceback (most recent call last): File "support/scripts/gen-bootlin-toolchains", line 3, in import htmllistparse ModuleNotFoundError: No module named 'htmllistparse' htmllistparse is not available on my distro (Ubuntu 19.10), and it is not even available in the latest LTS either (https://packages.ubuntu.com/htmllistparse) Sometimes, it is not possible to install from PyPi (enterprise-grade machines barred from accessing Pypi for example). Regards, Yann E. MORIN. > +import os.path > +import re > +import requests > +import textwrap > + > +BASE_URL = "https://toolchains.bootlin.com/downloads/releases/toolchains" > + > +AUTOGENERATED_COMMENT = """# This file was auto-generated by support/scripts/gen-bootlin-toolchains > +# Do not edit > +""" > + > +# In the below dict: > + > +# - 'conditions' indicate the cumulative conditions under which the > +# toolchain will be made available. In several situations, a given > +# toolchain is usable on several architectures variants (for > +# example, an ARMv6 toolchain can be used on ARMv7) > +# - 'test_options' indicate one specific configuration where the > +# toolchain can be used. It is used to create the runtime test > +# cases. If 'test_options' does not exist, the code assumes it can > +# be made equal to 'conditions' > +# - 'prefix' is the prefix of the cross-compilation toolchain tools > + > +arches = { > + 'aarch64': { > + 'conditions': ['BR2_aarch64'], > + 'prefix': 'aarch64', > + }, > + 'aarch64be': { > + 'conditions': ['BR2_aarch64_be'], > + 'prefix': 'aarch64_be', > + }, > + 'arcle-750d': { > + 'conditions': ['BR2_arcle', 'BR2_arc750d'], > + 'prefix': 'arc', > + }, > + 'arcle-hs38': { > + 'conditions': ['BR2_arcle', 'BR2_archs38'], > + 'prefix': 'arc', > + }, > + 'armv5-eabi': { > + 'conditions': ['BR2_ARM_CPU_ARMV5', 'BR2_ARM_EABI'], > + 'test_options': ['BR2_arm', 'BR2_arm926t', 'BR2_ARM_EABI'], > + 'prefix': 'arm', > + }, > + 'armv6-eabihf': { > + 'conditions': ['BR2_ARM_CPU_ARMV6', 'BR2_ARM_EABIHF'], > + 'test_options': ['BR2_arm', 'BR2_arm1176jzf_s', 'BR2_ARM_EABIHF'], > + 'prefix': 'arm', > + }, > + 'armv7-eabihf': { > + 'conditions': ['BR2_ARM_CPU_ARMV7A', 'BR2_ARM_EABIHF'], > + 'test_options': ['BR2_arm', 'BR2_cortex_a8', 'BR2_ARM_EABIHF'], > + 'prefix': 'arm', > + }, > + 'armv7m': { > + 'conditions': ['BR2_ARM_CPU_ARMV7M'], > + 'test_options': ['BR2_arm', 'BR2_cortex_m4'], > + 'prefix': 'arm', > + }, > + 'm68k-68xxx': { > + 'conditions': ['BR2_m68k_m68k'], > + 'test_options': ['BR2_m68k', 'BR2_m68k_68040'], > + 'prefix': 'm68k', > + }, > + 'm68k-coldfire': { > + 'conditions': ['BR2_m68k_cf'], > + 'test_options': ['BR2_m68k', 'BR2_m68k_cf5208'], > + 'prefix': 'm68k', > + }, > + 'microblazebe': { > + 'conditions': ['BR2_microblazebe'], > + 'prefix': 'microblaze', > + }, > + 'microblazeel': { > + 'conditions': ['BR2_microblazeel'], > + 'prefix': 'microblazeel', > + }, > + 'mips32': { > + # Not sure it could be used by other mips32 variants? > + 'conditions': ['BR2_mips', 'BR2_mips_32', '!BR2_MIPS_SOFT_FLOAT'], > + 'prefix': 'mips', > + }, > + 'mips32el': { > + # Not sure it could be used by other mips32el variants? > + 'conditions': ['BR2_mipsel', 'BR2_mips_32', '!BR2_MIPS_SOFT_FLOAT'], > + 'prefix': 'mipsel', > + }, > + 'mips32r5el': { > + 'conditions': ['BR2_mipsel', 'BR2_mips_32r5', '!BR2_MIPS_SOFT_FLOAT'], > + 'prefix': 'mipsel', > + }, > + 'mips32r6el': { > + 'conditions': ['BR2_mipsel', 'BR2_mips_32r6', '!BR2_MIPS_SOFT_FLOAT'], > + 'prefix': 'mipsel', > + }, > + 'mips64': { > + # Not sure it could be used by other mips64 variants? > + 'conditions': ['BR2_mips64', 'BR2_mips_64', '!BR2_MIPS_SOFT_FLOAT'], > + 'prefix': 'mips64', > + }, > + 'mips64-n32': { > + # Not sure it could be used by other mips64 variants? > + 'conditions': ['BR2_mips64', 'BR2_mips_64', 'BR2_MIPS_NABI32', '!BR2_MIPS_SOFT_FLOAT'], > + 'prefix': 'mips64', > + }, > + 'mips64el-n32': { > + # Not sure it could be used by other mips64el variants? > + 'conditions': ['BR2_mips64el', 'BR2_mips_64', 'BR2_MIPS_NABI32', '!BR2_MIPS_SOFT_FLOAT'], > + 'prefix': 'mips64el', > + }, > + 'mips64r6el-n32': { > + 'conditions': ['BR2_mips64el', 'BR2_mips_64r6', 'BR2_MIPS_NABI32', '!BR2_MIPS_SOFT_FLOAT'], > + 'prefix': 'mips64el', > + }, > + 'nios2': { > + 'conditions': ['BR2_nios2'], > + 'prefix': 'nios2', > + }, > + 'openrisc': { > + 'conditions': ['BR2_or1k'], > + 'prefix': 'or1k', > + }, > + 'powerpc-e500mc': { > + # Not sure it could be used by other powerpc variants? > + 'conditions': ['BR2_powerpc', 'BR2_powerpc_e500mc'], > + 'prefix': 'powerpc', > + }, > + 'powerpc64-e5500': { > + 'conditions': ['BR2_powerpc64', 'BR2_powerpc_e5500'], > + 'prefix': 'powerpc64', > + }, > + 'powerpc64-power8': { > + 'conditions': ['BR2_powerpc64', 'BR2_powerpc_power8'], > + 'prefix': 'powerpc64', > + }, > + 'powerpc64le-power8': { > + 'conditions': ['BR2_powerpc64le', 'BR2_powerpc_power8'], > + 'prefix': 'powerpc64le', > + }, > + 'riscv32-ilp32d': { > + 'conditions': ['BR2_riscv', 'BR2_riscv_g', 'BR2_RISCV_32', 'BR2_RISCV_ABI_ILP32D'], > + 'prefix': 'riscv32', > + }, > + 'riscv64': { > + 'conditions': ['BR2_riscv', 'BR2_riscv_g', 'BR2_RISCV_64', 'BR2_RISCV_ABI_LP64'], > + 'prefix': 'riscv64', > + }, > + 'sh-sh4': { > + 'conditions': ['BR2_sh', 'BR2_sh4'], > + 'prefix': 'sh4', > + }, > + 'sh-sh4aeb': { > + 'conditions': ['BR2_sh', 'BR2_sh4aeb'], > + 'prefix': 'sh4aeb', > + }, > + 'sparc64': { > + 'conditions': ['BR2_sparc64', 'BR2_sparc_v9'], > + 'prefix': 'sparc64', > + }, > + 'sparcv8': { > + 'conditions': ['BR2_sparc', 'BR2_sparc_v8'], > + 'prefix': 'sparc', > + }, > + 'x86-64-core-i7': { > + 'conditions': ['BR2_x86_64', > + 'BR2_X86_CPU_HAS_MMX', > + 'BR2_X86_CPU_HAS_SSE', > + 'BR2_X86_CPU_HAS_SSE2', > + 'BR2_X86_CPU_HAS_SSE3', > + 'BR2_X86_CPU_HAS_SSSE3', > + 'BR2_X86_CPU_HAS_SSE4', > + 'BR2_X86_CPU_HAS_SSE42'], > + 'test_options': ['BR2_x86_64', 'BR2_x86_corei7'], > + 'prefix': 'x86_64', > + }, > + 'x86-core2': { > + 'conditions': ['BR2_i386', > + 'BR2_X86_CPU_HAS_MMX', > + 'BR2_X86_CPU_HAS_SSE', > + 'BR2_X86_CPU_HAS_SSE2', > + 'BR2_X86_CPU_HAS_SSE3', > + 'BR2_X86_CPU_HAS_SSSE3'], > + 'test_options': ['BR2_i386', 'BR2_x86_core2'], > + 'prefix': 'i686', > + }, > + 'x86-i686': { > + 'conditions': ['BR2_i386', > + '!BR2_x86_i486', > + '!BR2_x86_i586', > + '!BR2_x86_x1000'], > + 'test_options': ['BR2_i386', > + 'BR2_x86_i686'], > + 'prefix': 'i686', > + }, > + 'xtensa-lx60': { > + 'conditions': ['BR2_xtensa', 'BR2_xtensa_fsf'], > + 'prefix': 'xtensa', > + }, > +} > + > + > +class Toolchain: > + def __init__(self, arch, libc, variant, version): > + self.arch = arch > + self.libc = libc > + self.variant = variant > + self.version = version > + self.fname_prefix = "%s--%s--%s-%s" % (self.arch, self.libc, self.variant, self.version) > + self.option_name = "BR2_TOOLCHAIN_EXTERNAL_BOOTLIN_%s_%s_%s" % \ > + (self.arch.replace("-", "_").upper(), self.libc.upper(), self.variant.replace("-", "_").upper()) > + self.fragment = requests.get(self.fragment_url).text.split("\n") > + self.sha256 = requests.get(self.hash_url).text.split(" ")[0] > + > + @property > + def tarball_url(self): > + return os.path.join(BASE_URL, self.arch, "tarballs", > + self.fname_prefix + ".tar.bz2") > + > + @property > + def hash_url(self): > + return os.path.join(BASE_URL, self.arch, "tarballs", > + self.fname_prefix + ".sha256") > + > + @property > + def fragment_url(self): > + return os.path.join(BASE_URL, self.arch, "fragments", > + self.fname_prefix + ".frag") > + > + def gen_config_in_options(self, f): > + f.write("config %s\n" % self.option_name) > + f.write("\tbool \"%s %s %s %s\"\n" % > + (self.arch, self.libc, self.variant, self.version)) > + for c in arches[self.arch]['conditions']: > + f.write("\tdepends on %s\n" % c) > + selects = [] > + for frag in self.fragment: > + # libc type > + if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_CUSTOM_UCLIBC"): > + selects.append("BR2_TOOLCHAIN_EXTERNAL_UCLIBC") > + elif frag.startswith("BR2_TOOLCHAIN_EXTERNAL_CUSTOM_GLIBC"): > + selects.append("BR2_TOOLCHAIN_EXTERNAL_GLIBC") > + # all glibc toolchains have RPC support > + selects.append("BR2_TOOLCHAIN_HAS_NATIVE_RPC") > + elif frag.startswith("BR2_TOOLCHAIN_EXTERNAL_CUSTOM_MUSL"): > + selects.append("BR2_TOOLCHAIN_EXTERNAL_MUSL") > + > + # gcc version > + if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_GCC_"): > + m = re.match("^BR2_TOOLCHAIN_EXTERNAL_GCC_([0-9_]*)=y$", frag) > + assert m, "Cannot get gcc version for toolchain %s" % self.fname_prefix > + selects.append("BR2_TOOLCHAIN_GCC_AT_LEAST_%s" % m[1]) > + > + # kernel headers version > + if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_HEADERS_"): > + m = re.match("^BR2_TOOLCHAIN_EXTERNAL_HEADERS_([0-9_]*)=y$", frag) > + assert m, "Cannot get kernel headers version for toolchain %s" % self.fname_prefix > + selects.append("BR2_TOOLCHAIN_HEADERS_AT_LEAST_%s" % m[1]) > + > + # C++ > + if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_CXX"): > + selects.append("BR2_INSTALL_LIBSTDCPP") > + > + # SSP > + if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_HAS_SSP"): > + selects.append("BR2_TOOLCHAIN_HAS_SSP") > + > + # wchar > + if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_WCHAR"): > + selects.append("BR2_USE_WCHAR") > + > + # locale > + if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_LOCALE"): > + # locale implies the availability of wchar > + selects.append("BR2_USE_WCHAR") > + selects.append("BR2_ENABLE_LOCALE") > + > + # thread support > + if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_HAS_THREADS"): > + selects.append("BR2_TOOLCHAIN_HAS_THREADS") > + > + if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_HAS_THREADS_DEBUG"): > + selects.append("BR2_TOOLCHAIN_HAS_THREADS_DEBUG") > + > + if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_HAS_THREADS_NPTL"): > + selects.append("BR2_TOOLCHAIN_HAS_THREADS_NPTL") > + > + # RPC > + if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_INET_RPC"): > + selects.append("BR2_TOOLCHAIN_HAS_NATIVE_RPC") > + > + # D language > + if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_DLANG"): > + selects.append("BR2_TOOLCHAIN_HAS_DLANG") > + > + # fortran > + if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_FORTRAN"): > + selects.append("BR2_TOOLCHAIN_HAS_FORTRAN") > + > + # OpenMP > + if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_OPENMP"): > + selects.append("BR2_TOOLCHAIN_HAS_OPENMP") > + > + for select in selects: > + f.write("\tselect %s\n" % select) > + > + f.write("\thelp\n") > + > + desc = "Bootlin toolchain for the %s architecture, using the %s C library. " % \ > + (self.arch, self.libc) > + > + if self.variant == "stable": > + desc += "This is a stable version, which means it is using stable and proven versions of gcc, gdb and binutils." > + else: > + desc += "This is a bleeding-edge version, which means it is using the latest versions of gcc, gdb and binutils." > + > + f.write(textwrap.fill(desc, width=62, initial_indent="\t ", subsequent_indent="\t ") + "\n") > + f.write("\n") > + f.write("\t https://toolchains.bootlin.com/\n") > + > + f.write("\n") > + > + def gen_mk(self, f): > + f.write("ifeq ($(%s),y)\n" % self.option_name) > + f.write("TOOLCHAIN_EXTERNAL_BOOTLIN_VERSION = %s\n" % self.version) > + f.write("TOOLCHAIN_EXTERNAL_BOOTLIN_SOURCE = %s--%s--%s-$(TOOLCHAIN_EXTERNAL_BOOTLIN_VERSION).tar.bz2\n" % > + (self.arch, self.libc, self.variant)) > + f.write("TOOLCHAIN_EXTERNAL_BOOTLIN_SITE = %s\n" % > + os.path.join(BASE_URL, self.arch, "tarballs")) > + f.write("endif\n\n") > + pass > + > + def gen_hash(self, f): > + f.write("# From %s\n" % self.hash_url) > + f.write("sha256 %s %s\n" % (self.sha256, os.path.basename(self.tarball_url))) > + > + def gen_test(self, f): > + if self.variant == "stable": > + variant = "Stable" > + else: > + variant = "BleedingEdge" > + testname = "TestExternalToolchainBootlin" + \ > + self.arch.replace("-", "").capitalize() + \ > + self.libc.capitalize() + variant > + f.write("\n\n") > + f.write("class %s(TestExternalToolchain):\n" % testname) > + f.write(" config = \"\"\"\n") > + if 'test_options' in arches[self.arch]: > + test_options = arches[self.arch]['test_options'] > + else: > + test_options = arches[self.arch]['conditions'] > + for opt in test_options: > + if opt.startswith("!"): > + f.write(" # %s is not set\n" % opt[1:]) > + else: > + f.write(" %s=y\n" % opt) > + f.write(" BR2_TOOLCHAIN_EXTERNAL=y\n") > + f.write(" BR2_TOOLCHAIN_EXTERNAL_BOOTLIN=y\n") > + f.write(" %s=y\n" % self.option_name) > + f.write(" # BR2_TARGET_ROOTFS_TAR is not set\n") > + f.write(" \"\"\"\n") > + f.write(" toolchain_prefix = \"%s-linux\"\n" % arches[self.arch]['prefix']) > + f.write("\n") > + f.write(" def test_run(self):\n") > + f.write(" TestExternalToolchain.common_check(self)\n") > + > + def __repr__(self): > + return "Toolchain(arch=%s libc=%s variant=%s version=%s, option=%s)" % \ > + (self.arch, self.libc, self.variant, self.version, self.option_name) > + > + > +def get_toolchains(): > + toolchains = list() > + for arch, details in arches.items(): > + print(arch) > + url = os.path.join(BASE_URL, arch, "available_toolchains") > + cwd, listing = htmllistparse.fetch_listing(url) > + fnames = [f.name for f in listing] > + # Sorting the list so we have the most recent version last > + fnames.sort() > + # This dict will allow us to keep only the latest version for > + # each toolchain. > + tmp = dict() > + for fname in fnames: > + parts = fname.split('--') > + assert parts[0] == arch, "Arch does not match: %s vs. %s" % (parts[0], arch) > + libc = parts[1] > + if parts[2].startswith("stable-"): > + variant = "stable" > + version = parts[2][len("stable-"):] > + elif parts[2].startswith("bleeding-edge-"): > + variant = "bleeding-edge" > + version = parts[2][len("bleeding-edge-"):] > + tmp[(arch, libc, variant)] = version > + > + toolchains += [Toolchain(k[0], k[1], k[2], v) for k, v in tmp.items()] > + > + return toolchains > + > + > +def gen_config_in_options(toolchains, fpath): > + with open(fpath, "w") as f: > + f.write(AUTOGENERATED_COMMENT) > + > + f.write("config BR2_TOOLCHAIN_EXTERNAL_BOOTLIN_ARCH_SUPPORTS\n") > + f.write("\tbool\n") > + for arch, details in arches.items(): > + f.write("\tdefault y if %s\n" % " && ".join(details['conditions'])) > + f.write("\n") > + > + f.write("if BR2_TOOLCHAIN_EXTERNAL_BOOTLIN\n\n") > + > + f.write("config BR2_TOOLCHAIN_EXTERNAL_PREFIX\n") > + f.write("\tdefault \"$(ARCH)-linux\"\n") > + > + f.write("\n") > + > + f.write("config BR2_PACKAGE_PROVIDES_TOOLCHAIN_EXTERNAL\n") > + f.write("\tdefault \"toolchain-external-bootlin\"\n") > + > + f.write("\n") > + > + f.write("choice\n") > + f.write("\tprompt \"Bootlin toolchain variant\"\n") > + > + for toolchain in toolchains: > + toolchain.gen_config_in_options(f) > + > + f.write("endchoice\n") > + f.write("endif\n") > + > + > +def gen_mk(toolchains, fpath): > + with open(fpath, "w") as f: > + f.write("#" * 80 + "\n") > + f.write("#\n") > + f.write("# toolchain-external-bootlin\n") > + f.write("#\n") > + f.write("#" * 80 + "\n") > + f.write("\n") > + f.write(AUTOGENERATED_COMMENT) > + for toolchain in toolchains: > + toolchain.gen_mk(f) > + f.write("$(eval $(toolchain-external-package))\n") > + > + > +def gen_hash(toolchains, fpath): > + with open(fpath, "w") as f: > + f.write(AUTOGENERATED_COMMENT) > + for toolchain in toolchains: > + toolchain.gen_hash(f) > + > + > +def gen_runtime_test(toolchains, fpath): > + with open(fpath, "w") as f: > + f.write(AUTOGENERATED_COMMENT) > + f.write("from tests.toolchain.test_external import TestExternalToolchain\n") > + for toolchain in toolchains: > + toolchain.gen_test(f) > + > + > +def gen_toolchains(toolchains): > + maindir = "toolchain/toolchain-external/toolchain-external-bootlin" > + gen_config_in_options(toolchains, os.path.join(maindir, "Config.in.options")) > + gen_mk(toolchains, os.path.join(maindir, "toolchain-external-bootlin.mk")) > + gen_hash(toolchains, os.path.join(maindir, "toolchain-external-bootlin.hash")) > + gen_runtime_test(toolchains, > + os.path.join("support", "testing", "tests", "toolchain", "test_external_bootlin.py")) > + > + > +toolchains = get_toolchains() > +gen_toolchains(toolchains) > -- > 2.26.2 > -- .-----------------.--------------------.------------------.--------------------. | Yann E. MORIN | Real-Time Embedded | /"\ ASCII RIBBON | Erics' conspiracy: | | +33 662 376 056 | Software Designer | \ / CAMPAIGN | ___ | | +33 561 099 427 `------------.-------: X AGAINST | \e/ There is no | | http://ymorin.is-a-geek.org/ | _/*\_ | / \ HTML MAIL | v conspiracy. | '------------------------------^-------^------------------^--------------------'