new file mode 100755
@@ -0,0 +1,472 @@
+#!/usr/bin/env python3
+
+import htmllistparse
+import os.path
+import re
+import requests
+import textwrap
+import sys
+
+BASE_URL = "https://toolchains.bootlin.com/downloads/releases/toolchains"
+
+# In the below dict:
+
+# - 'conditions' indicate the cumulative conditions under which the
+# toolchain will be made available. In several situations, a given
+# toolchain is usable on several architectures variants (for
+# example, an ARMv6 toolchain can be used on ARMv7)
+# - 'test_options' indicate one specific configuration where the
+# toolchain can be used. It is used to create the runtime test
+# cases. If 'test_options' does not exist, the code assumes it can
+# be made equal to 'conditions'
+# - 'prefix' is the prefix of the cross-compilation toolchain tools
+
+arches = {
+ 'aarch64': {
+ 'conditions': ['BR2_aarch64'],
+ 'prefix': 'aarch64',
+ },
+ 'aarch64be': {
+ 'conditions': ['BR2_aarch64_be'],
+ 'prefix': 'aarch64_be',
+ },
+ 'arcle-750d': {
+ 'conditions': ['BR2_arcle', 'BR2_arc750d'],
+ 'prefix': 'arc',
+ },
+ 'arcle-hs38': {
+ 'conditions': ['BR2_arcle', 'BR2_archs38'],
+ 'prefix': 'arc',
+ },
+ 'armv5-eabi': {
+ 'conditions': ['BR2_ARM_CPU_ARMV5', 'BR2_ARM_EABI'],
+ 'test_options': ['BR2_arm', 'BR2_arm926t', 'BR2_ARM_EABI'],
+ 'prefix': 'arm',
+ },
+ 'armv6-eabihf': {
+ 'conditions': ['BR2_ARM_CPU_ARMV6', 'BR2_ARM_EABIHF'],
+ 'test_options': ['BR2_arm', 'BR2_arm1176jzf_s', 'BR2_ARM_EABIHF'],
+ 'prefix': 'arm',
+ },
+ 'armv7-eabihf': {
+ 'conditions': ['BR2_ARM_CPU_ARMV7A', 'BR2_ARM_EABIHF'],
+ 'test_options': ['BR2_arm', 'BR2_cortex_a8', 'BR2_ARM_EABIHF'],
+ 'prefix': 'arm',
+ },
+ 'armv7m': {
+ 'conditions': ['BR2_ARM_CPU_ARMV7M'],
+ 'test_options': ['BR2_arm', 'BR2_cortex_m4'],
+ 'prefix': 'arm',
+ },
+ 'm68k-68xxx': {
+ 'conditions': ['BR2_m68k_m68k'],
+ 'test_options': ['BR2_m68k', 'BR2_m68k_68040'],
+ 'prefix': 'm68k',
+ },
+ 'm68k-coldfire': {
+ 'conditions': ['BR2_m68k_cf'],
+ 'test_options': ['BR2_m68k', 'BR2_m68k_cf5208'],
+ 'prefix': 'm68k',
+ },
+ 'microblazebe': {
+ 'conditions': ['BR2_microblazebe'],
+ 'prefix': 'microblaze',
+ },
+ 'microblazeel': {
+ 'conditions': ['BR2_microblazeel'],
+ 'prefix': 'microblazeel',
+ },
+ 'mips32': {
+ # Not sure it could be used by other mips32 variants?
+ 'conditions': ['BR2_mips', 'BR2_mips_32', '!BR2_MIPS_SOFT_FLOAT'],
+ 'prefix': 'mips',
+ },
+ 'mips32el': {
+ # Not sure it could be used by other mips32el variants?
+ 'conditions': ['BR2_mipsel', 'BR2_mips_32', '!BR2_MIPS_SOFT_FLOAT'],
+ 'prefix': 'mipsel',
+ },
+ 'mips32r5el': {
+ 'conditions': ['BR2_mipsel', 'BR2_mips_32r5', '!BR2_MIPS_SOFT_FLOAT'],
+ 'prefix': 'mipsel',
+ },
+ 'mips32r6el': {
+ 'conditions': ['BR2_mipsel', 'BR2_mips_32r6', '!BR2_MIPS_SOFT_FLOAT'],
+ 'prefix': 'mipsel',
+ },
+ 'mips64': {
+ # Not sure it could be used by other mips64 variants?
+ 'conditions': ['BR2_mips64', 'BR2_mips_64', '!BR2_MIPS_SOFT_FLOAT'],
+ 'prefix': 'mips64',
+ },
+ 'mips64-n32': {
+ # Not sure it could be used by other mips64 variants?
+ 'conditions': ['BR2_mips64', 'BR2_mips_64', 'BR2_MIPS_NABI32', '!BR2_MIPS_SOFT_FLOAT'],
+ 'prefix': 'mips64',
+ },
+ 'mips64el-n32': {
+ # Not sure it could be used by other mips64el variants?
+ 'conditions': ['BR2_mips64el', 'BR2_mips_64', 'BR2_MIPS_NABI32', '!BR2_MIPS_SOFT_FLOAT'],
+ 'prefix': 'mips64el',
+ },
+ 'mips64r6el-n32': {
+ 'conditions': ['BR2_mips64el', 'BR2_mips_64r6', 'BR2_MIPS_NABI32', '!BR2_MIPS_SOFT_FLOAT'],
+ 'prefix': 'mips64el',
+ },
+ 'nios2': {
+ 'conditions': ['BR2_nios2'],
+ 'prefix': 'nios2',
+ },
+ 'openrisc': {
+ 'conditions': ['BR2_or1k'],
+ 'prefix': 'or1k',
+ },
+ 'powerpc-e500mc': {
+ # Not sure it could be used by other powerpc variants?
+ 'conditions': ['BR2_powerpc', 'BR2_powerpc_e500mc'],
+ 'prefix': 'powerpc',
+ },
+ 'powerpc64-e5500': {
+ 'conditions': ['BR2_powerpc64', 'BR2_powerpc_e5500'],
+ 'prefix': 'powerpc64',
+ },
+ 'powerpc64-power8': {
+ 'conditions': ['BR2_powerpc64', 'BR2_powerpc_power8'],
+ 'prefix': 'powerpc64',
+ },
+ 'powerpc64le-power8': {
+ 'conditions': ['BR2_powerpc64le', 'BR2_powerpc_power8'],
+ 'prefix': 'powerpc64le',
+ },
+ 'riscv32-ilp32d': {
+ 'conditions': ['BR2_riscv', 'BR2_riscv_g', 'BR2_RISCV_32', 'BR2_RISCV_ABI_ILP32D'],
+ 'prefix': 'riscv32',
+ },
+ 'riscv64': {
+ 'conditions': ['BR2_riscv', 'BR2_riscv_g', 'BR2_RISCV_64', 'BR2_RISCV_ABI_LP64'],
+ 'prefix': 'riscv64',
+ },
+ 'sh-sh4': {
+ 'conditions': ['BR2_sh', 'BR2_sh4'],
+ 'prefix': 'sh4',
+ },
+ 'sh-sh4aeb': {
+ 'conditions': ['BR2_sh', 'BR2_sh4aeb'],
+ 'prefix': 'sh4aeb',
+ },
+ 'sparc64': {
+ 'conditions': ['BR2_sparc64', 'BR2_sparc_v9'],
+ 'prefix': 'sparc64',
+ },
+ 'sparcv8': {
+ 'conditions': ['BR2_sparc', 'BR2_sparc_v8'],
+ 'prefix': 'sparc',
+ },
+ 'x86-64-core-i7': {
+ 'conditions': ['BR2_x86_64',
+ 'BR2_X86_CPU_HAS_MMX',
+ 'BR2_X86_CPU_HAS_SSE',
+ 'BR2_X86_CPU_HAS_SSE2',
+ 'BR2_X86_CPU_HAS_SSE3',
+ 'BR2_X86_CPU_HAS_SSSE3',
+ 'BR2_X86_CPU_HAS_SSE4',
+ 'BR2_X86_CPU_HAS_SSE42'],
+ 'test_options': ['BR2_x86_64', 'BR2_x86_corei7'],
+ 'prefix': 'x86_64',
+ },
+ 'x86-core2': {
+ 'conditions': ['BR2_i386',
+ 'BR2_X86_CPU_HAS_MMX',
+ 'BR2_X86_CPU_HAS_SSE',
+ 'BR2_X86_CPU_HAS_SSE2',
+ 'BR2_X86_CPU_HAS_SSE3',
+ 'BR2_X86_CPU_HAS_SSSE3'],
+ 'test_options': ['BR2_i386', 'BR2_x86_core2'],
+ 'prefix': 'i686',
+ },
+ 'x86-i686': {
+ 'conditions': ['BR2_i386',
+ '!BR2_x86_i486',
+ '!BR2_x86_i586',
+ '!BR2_x86_x1000'],
+ 'test_options': ['BR2_i386',
+ 'BR2_x86_i686'],
+ 'prefix': 'i686',
+ },
+ 'xtensa-lx60': {
+ 'conditions': ['BR2_xtensa', 'BR2_xtensa_fsf'],
+ 'prefix': 'xtensa',
+ },
+}
+
+
+class Toolchain:
+ def __init__(self, arch, libc, variant, version):
+ self.arch = arch
+ self.libc = libc
+ self.variant = variant
+ self.version = version
+ self.fname_prefix = "%s--%s--%s-%s" % (self.arch, self.libc, self.variant, self.version)
+ self.option_name = "BR2_TOOLCHAIN_EXTERNAL_BOOTLIN_%s_%s_%s" % \
+ (self.arch.replace("-", "_").upper(), self.libc.upper(), self.variant.replace("-", "_").upper())
+ self.fragment = requests.get(self.fragment_url()).text.split("\n")
+ self.sha256 = requests.get(self.hash_url()).text.split(" ")[0]
+
+ def tarball_url(self):
+ return os.path.join(BASE_URL, self.arch, "tarballs",
+ self.fname_prefix + ".tar.bz2")
+
+ def hash_url(self):
+ return os.path.join(BASE_URL, self.arch, "tarballs",
+ self.fname_prefix + ".sha256")
+
+ def fragment_url(self):
+ return os.path.join(BASE_URL, self.arch, "fragments",
+ self.fname_prefix + ".frag")
+
+ def gen_config_in_options(self, f):
+ f.write("config %s\n" % self.option_name)
+ f.write("\tbool \"%s %s %s %s\"\n" %
+ (self.arch, self.libc, self.variant, self.version))
+ for c in arches[self.arch]['conditions']:
+ f.write("\tdepends on %s\n" % c)
+ selects = []
+ for frag in self.fragment:
+ # libc type
+ if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_CUSTOM_UCLIBC"):
+ selects.append("BR2_TOOLCHAIN_EXTERNAL_UCLIBC")
+ elif frag.startswith("BR2_TOOLCHAIN_EXTERNAL_CUSTOM_GLIBC"):
+ selects.append("BR2_TOOLCHAIN_EXTERNAL_GLIBC")
+ # all glibc toolchains have RPC support
+ selects.append("BR2_TOOLCHAIN_HAS_NATIVE_RPC")
+ elif frag.startswith("BR2_TOOLCHAIN_EXTERNAL_CUSTOM_MUSL"):
+ selects.append("BR2_TOOLCHAIN_EXTERNAL_MUSL")
+
+ # gcc version
+ if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_GCC_"):
+ m = re.match("^BR2_TOOLCHAIN_EXTERNAL_GCC_([0-9_]*)=y$", frag)
+ if m is None:
+ print("ERROR: cannot get gcc version for toolchain %s" % self.fname_prefix)
+ sys.exit(1)
+ selects.append("BR2_TOOLCHAIN_GCC_AT_LEAST_%s" % m[1])
+
+ # kernel headers version
+ if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_HEADERS_"):
+ m = re.match("^BR2_TOOLCHAIN_EXTERNAL_HEADERS_([0-9_]*)=y$", frag)
+ if m is None:
+ print("ERROR: cannot get kernel headers version for toolchain %s" % self.fname_prefix)
+ sys.exit(1)
+ selects.append("BR2_TOOLCHAIN_HEADERS_AT_LEAST_%s" % m[1])
+
+ # C++
+ if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_CXX"):
+ selects.append("BR2_INSTALL_LIBSTDCPP")
+
+ # SSP
+ if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_HAS_SSP"):
+ selects.append("BR2_TOOLCHAIN_HAS_SSP")
+
+ # wchar
+ if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_WCHAR"):
+ selects.append("BR2_USE_WCHAR")
+
+ # locale
+ if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_LOCALE"):
+ # locale implies the availability of wchar
+ selects.append("BR2_USE_WCHAR")
+ selects.append("BR2_ENABLE_LOCALE")
+
+ # thread support
+ if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_HAS_THREADS"):
+ selects.append("BR2_TOOLCHAIN_HAS_THREADS")
+
+ if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_HAS_THREADS_DEBUG"):
+ selects.append("BR2_TOOLCHAIN_HAS_THREADS_DEBUG")
+
+ if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_HAS_THREADS_NPTL"):
+ selects.append("BR2_TOOLCHAIN_HAS_THREADS_NPTL")
+
+ # RPC
+ if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_INET_RPC"):
+ selects.append("BR2_TOOLCHAIN_HAS_NATIVE_RPC")
+
+ # D language
+ if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_DLANG"):
+ selects.append("BR2_TOOLCHAIN_HAS_DLANG")
+
+ # fortran
+ if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_FORTRAN"):
+ selects.append("BR2_TOOLCHAIN_HAS_FORTRAN")
+
+ # OpenMP
+ if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_OPENMP"):
+ selects.append("BR2_TOOLCHAIN_HAS_OPENMP")
+
+ for select in selects:
+ f.write("\tselect %s\n" % select)
+
+ f.write("\thelp\n")
+
+ desc = "Bootlin toolchain for the %s architecture, using the %s C library" % \
+ (self.arch, self.libc)
+
+ f.write(textwrap.fill(desc, width=62, initial_indent="\t ", subsequent_indent="\t ") + "\n")
+ f.write("\n")
+ f.write("\t https://toolchains.bootlin.com/\n")
+
+ f.write("\n")
+
+ def gen_mk(self, f):
+ f.write("ifeq ($(%s),y)\n" % self.option_name)
+ f.write("TOOLCHAIN_EXTERNAL_BOOTLIN_VERSION = %s\n" % self.version)
+ f.write("TOOLCHAIN_EXTERNAL_BOOTLIN_SOURCE = %s--%s--%s-$(TOOLCHAIN_EXTERNAL_BOOTLIN_VERSION).tar.bz2\n" %
+ (self.arch, self.libc, self.variant))
+ f.write("TOOLCHAIN_EXTERNAL_BOOTLIN_SITE = %s\n" %
+ os.path.join(BASE_URL, self.arch, "tarballs"))
+ f.write("endif\n\n")
+ pass
+
+ def gen_hash(self, f):
+ f.write("# From %s\n" % self.hash_url())
+ f.write("sha256 %s %s\n" % (self.sha256, os.path.basename(self.tarball_url())))
+
+ def gen_test(self, f):
+ if self.variant == "stable":
+ variant = "Stable"
+ else:
+ variant = "BleedingEdge"
+ testname = "TestExternalToolchainBootlin" + \
+ self.arch.replace("-", "").capitalize() + \
+ self.libc.capitalize() + variant
+ f.write("\n\n")
+ f.write("class %s(TestExternalToolchain):\n" % testname)
+ f.write(" config = \"\"\"\n")
+ if 'test_options' in arches[self.arch]:
+ test_options = arches[self.arch]['test_options']
+ else:
+ test_options = arches[self.arch]['conditions']
+ for opt in test_options:
+ if opt.startswith("!"):
+ f.write(" # %s is not set\n" % opt[1:])
+ else:
+ f.write(" %s=y\n" % opt)
+ f.write(" BR2_TOOLCHAIN_EXTERNAL=y\n")
+ f.write(" BR2_TOOLCHAIN_EXTERNAL_BOOTLIN=y\n")
+ f.write(" %s=y\n" % self.option_name)
+ f.write(" # BR2_TARGET_ROOTFS_TAR is not set\n")
+ f.write(" \"\"\"\n")
+ f.write(" toolchain_prefix = \"%s-linux\"\n" % arches[self.arch]['prefix'])
+ f.write("\n")
+ f.write(" def test_run(self):\n")
+ f.write(" TestExternalToolchain.common_check(self)\n")
+
+ def __repr__(self):
+ return "Toolchain(arch=%s libc=%s variant=%s version=%s, option=%s)" % \
+ (self.arch, self.libc, self.variant, self.version, self.option_name)
+
+
+def get_toolchains():
+ toolchains = list()
+ for arch, details in arches.items():
+ print(arch)
+ url = os.path.join(BASE_URL, arch, "available_toolchains")
+ cwd, listing = htmllistparse.fetch_listing(url)
+ fnames = [f.name for f in listing]
+ # Sorting the list so we have the most recent version last
+ fnames.sort()
+ # This dict will allow us to keep only the latest version for
+ # each toolchain.
+ tmp = dict()
+ for fname in fnames:
+ parts = fname.split('--')
+ if parts[0] != arch:
+ print("FATAL: arch does not match: %s vs. %s" % (parts[0], arch))
+ sys.exit(1)
+ libc = parts[1]
+ if parts[2].startswith("stable-"):
+ variant = "stable"
+ version = parts[2][len("stable-"):]
+ elif parts[2].startswith("bleeding-edge-"):
+ variant = "bleeding-edge"
+ version = parts[2][len("bleeding-edge-"):]
+ tmp[(arch, libc, variant)] = version
+
+ for k, v in tmp.items():
+ t = Toolchain(k[0], k[1], k[2], v)
+ toolchains.append(t)
+
+ return toolchains
+
+
+def gen_config_in_options(toolchains, fpath):
+ with open(fpath, "w") as f:
+ f.write("# This file is auto-generated by support/scripts/bl-toolchains-gen\n")
+ f.write("# Do not edit\n\n")
+
+ f.write("config BR2_TOOLCHAIN_EXTERNAL_BOOTLIN_ARCH_SUPPORTS\n")
+ f.write("\tbool\n")
+ for arch, details in arches.items():
+ f.write("\tdefault y if %s\n" % " && ".join(details['conditions']))
+ f.write("\n")
+
+ f.write("if BR2_TOOLCHAIN_EXTERNAL_BOOTLIN\n\n")
+
+ f.write("config BR2_TOOLCHAIN_EXTERNAL_PREFIX\n")
+ f.write("\tdefault \"$(ARCH)-linux\"\n")
+
+ f.write("\n")
+
+ f.write("config BR2_PACKAGE_PROVIDES_TOOLCHAIN_EXTERNAL\n")
+ f.write("\tdefault \"toolchain-external-bootlin\"\n")
+
+ f.write("\n")
+
+ f.write("choice\n")
+ f.write("\tprompt \"Bootlin toolchain variant\"\n")
+
+ for toolchain in toolchains:
+ toolchain.gen_config_in_options(f)
+
+ f.write("endchoice\n")
+ f.write("endif\n")
+
+
+def gen_mk(toolchains, fpath):
+ with open(fpath, "w") as f:
+ f.write("#" * 80 + "\n")
+ f.write("#\n")
+ f.write("# toolchain-external-bootlin\n")
+ f.write("#\n")
+ f.write("#" * 80 + "\n")
+ f.write("\n")
+ f.write("# This file is auto-generated by support/scripts/bl-toolchains-gen\n")
+ f.write("# Do not edit\n\n")
+ for toolchain in toolchains:
+ toolchain.gen_mk(f)
+ f.write("$(eval $(toolchain-external-package))\n")
+
+
+def gen_hash(toolchains, fpath):
+ with open(fpath, "w") as f:
+ for toolchain in toolchains:
+ toolchain.gen_hash(f)
+
+
+def gen_runtime_test(toolchains, fpath):
+ with open(fpath, "w") as f:
+ f.write("from tests.toolchain.test_external import TestExternalToolchain\n")
+ for toolchain in toolchains:
+ toolchain.gen_test(f)
+
+
+def gen_toolchains(toolchains):
+ maindir = "toolchain/toolchain-external/toolchain-external-bootlin"
+ gen_config_in_options(toolchains, os.path.join(maindir, "Config.in.options"))
+ gen_mk(toolchains, os.path.join(maindir, "toolchain-external-bootlin.mk"))
+ gen_hash(toolchains, os.path.join(maindir, "toolchain-external-bootlin.hash"))
+ gen_runtime_test(toolchains,
+ os.path.join("support", "testing", "tests", "toolchain", "test_external_bootlin.py"))
+
+
+toolchains = get_toolchains()
+gen_toolchains(toolchains)
https://toolchains.bootlin.com/ has been providing for a few years a number of ready-to-use pre-built toolchains, for a wide range of architectures (which it turns out, are all built using Buildroot). While toolchains.bootlin.com provides Buildroot config fragments to easily use those toolchains with Buildroot (see [0] for example), this is not visible anywhere. So instead, we would like to add support for these toolchains in Buildroot just like we have existing support for Linaro, ARM, Synopsys, etc. toolchains. [0] https://toolchains.bootlin.com/downloads/releases/toolchains/aarch64/fragments/aarch64--glibc--bleeding-edge-2020.02-2.frag However, the number of toolchains provided by toolchains.bootlin.com is really large, and they are regularly updated. Maintaining that manually would be time consuming and error-prone. So instead, this commit introduces a script that automatically generates: - toolchain/toolchain-external/toolchain-external-bootlin/Config.in.options - toolchain/toolchain-external/toolchain-external-bootlin/toolchain-external-bootlin.mk - toolchain/toolchain-external/toolchain-external-bootlin/toolchain-external-bootlin.hash - support/testing/tests/toolchain/test_external_bootlin.py We create a single external toolchain package, with a Kconfig "choice" as a sub-option to select the toolchain variant to be used. The script contains a Python dict that provides the mapping between the toolchains provided by toolchains.bootlin.com, and the architecture options/variants they are applicable to. The test cases allow to verify that the toolchain configuration is correct, and that it is able to build a Busybox based system. It doesn't do any runtime testing as such testing is already done by toolchains.bootlin.com: the test cases here are only meant to verify that the toolchain-external-bootlin package works as expected. Signed-off-by: Thomas Petazzoni <thomas.petazzoni@bootlin.com> --- support/scripts/bl-toolchains-gen | 472 ++++++++++++++++++++++++++++++ 1 file changed, 472 insertions(+) create mode 100755 support/scripts/bl-toolchains-gen