@@ -18,24 +18,14 @@
# This script generates a random configuration for testing Buildroot.
-import contextlib
+import asyncio
import csv
import os
from random import randint
-import subprocess
import sys
from distutils.version import StrictVersion
import platform
-if sys.hexversion >= 0x3000000:
- import urllib.request as _urllib
-else:
- import urllib2 as _urllib
-
-
-def urlopen_closing(uri):
- return contextlib.closing(_urllib.urlopen(uri))
-
class SystemInfo:
DEFAULT_NEEDED_PROGS = ["make", "git", "gcc", "timeout"]
@@ -63,7 +53,7 @@ class SystemInfo:
# --
return None
- def has(self, prog):
+ async def has(self, prog):
"""Checks whether a program is available.
Lazily evaluates missing entries.
@@ -78,11 +68,13 @@ class SystemInfo:
have_it = self.find_prog(prog)
# java[c] needs special care
if have_it and prog in ('java', 'javac'):
- with open(os.devnull, "w") as devnull:
- if subprocess.call("%s -version | grep gcj" % prog,
- shell=True,
- stdout=devnull, stderr=devnull) != 1:
- have_it = False
+ proc = await asyncio.create_subprocess_shell(
+ "%s -version | grep gcj" % prog,
+ stdout=asyncio.subprocess.DEVNULL,
+ stderr=asyncio.subprocess.DEVNULL)
+ ret = await proc.wait()
+ if ret != 1:
+ have_it = False
# --
self.progs[prog] = have_it
return have_it
@@ -159,7 +151,7 @@ def get_toolchain_configs(toolchains_csv, buildrootdir):
return configs
-def is_toolchain_usable(configfile, config):
+async def is_toolchain_usable(configfile, config):
"""Check if the toolchain is actually usable."""
with open(configfile) as configf:
@@ -179,8 +171,12 @@ def is_toolchain_usable(configfile, config):
'BR2_TOOLCHAIN_EXTERNAL_LINARO_AARCH64=y\n' in configlines or \
'BR2_TOOLCHAIN_EXTERNAL_LINARO_AARCH64_BE=y\n' in configlines or \
'BR2_TOOLCHAIN_EXTERNAL_LINARO_ARMEB=y\n' in configlines:
- ldd_version_output = subprocess.check_output(['ldd', '--version'])
- glibc_version = ldd_version_output.splitlines()[0].split()[-1]
+ proc = await asyncio.create_subprocess_exec(
+ 'ldd', '--version', stdout=asyncio.subprocess.PIPE)
+ ldd_version_output, _ = await proc.communicate()
+ if proc.returncode:
+ return False
+ glibc_version = ldd_version_output.splitlines()[0].split()[-1].decode('utf-8')
if StrictVersion('2.14') > StrictVersion(glibc_version):
print("WARN: ignoring the Linaro ARM toolchains because too old host glibc", file=sys.stderr)
return False
@@ -188,7 +184,7 @@ def is_toolchain_usable(configfile, config):
return True
-def fixup_config(sysinfo, configfile):
+async def fixup_config(sysinfo, configfile):
"""Finalize the configuration and reject any problematic combinations
This function returns 'True' when the configuration has been
@@ -202,7 +198,7 @@ def fixup_config(sysinfo, configfile):
BR2_TOOLCHAIN_EXTERNAL_URL = 'BR2_TOOLCHAIN_EXTERNAL_URL="http://autobuild.buildroot.org/toolchains/tarballs/'
- if "BR2_NEEDS_HOST_JAVA=y\n" in configlines and not sysinfo.has("java"):
+ if "BR2_NEEDS_HOST_JAVA=y\n" in configlines and not await sysinfo.has("java"):
return False
# The ctng toolchain is affected by PR58854
if 'BR2_PACKAGE_LTTNG_TOOLS=y\n' in configlines and \
@@ -507,7 +503,7 @@ def fixup_config(sysinfo, configfile):
return True
-def gen_config(args):
+async def gen_config(args):
"""Generate a new random configuration
This function generates the configuration, by choosing a random
@@ -565,7 +561,7 @@ def gen_config(args):
# Randomly enable BR2_REPRODUCIBLE 10% of times
# also enable tar filesystem images for testing
- if sysinfo.has("diffoscope") and randint(0, 10) == 0:
+ if await sysinfo.has("diffoscope") and randint(0, 10) == 0:
configlines.append("BR2_REPRODUCIBLE=y\n")
configlines.append("BR2_TARGET_ROOTFS_TAR=y\n")
@@ -579,10 +575,13 @@ def gen_config(args):
with open(configfile, "w+") as configf:
configf.writelines(configlines)
- subprocess.check_call(["make", "O=%s" % args.outputdir, "-C", args.buildrootdir,
- "olddefconfig"])
+ proc = await asyncio.create_subprocess_exec(
+ "make", "O=%s" % args.outputdir, "-C", args.buildrootdir, "olddefconfig")
+ ret = await proc.wait()
+ if ret:
+ return ret
- if not is_toolchain_usable(configfile, toolchainconfig):
+ if not await is_toolchain_usable(configfile, toolchainconfig):
return 2
# Now, generate the random selection of packages, and fixup
@@ -596,21 +595,32 @@ def gen_config(args):
file=sys.stderr)
return 1
bounded_loop -= 1
- subprocess.check_call(["make", "O=%s" % args.outputdir, "-C", args.buildrootdir,
- "KCONFIG_PROBABILITY=%d" % randint(1, 20),
- "randpackageconfig" if args.toolchains_csv else "randconfig"])
-
- if fixup_config(sysinfo, configfile):
+ proc = await asyncio.create_subprocess_exec(
+ "make", "O=%s" % args.outputdir, "-C", args.buildrootdir,
+ "KCONFIG_PROBABILITY=%d" % randint(1, 20),
+ "randpackageconfig" if args.toolchains_csv else "randconfig")
+ ret = await proc.wait()
+ if ret:
+ return ret
+
+ if await fixup_config(sysinfo, configfile):
break
- subprocess.check_call(["make", "O=%s" % args.outputdir, "-C", args.buildrootdir,
- "olddefconfig"])
+ proc = await asyncio.create_subprocess_exec(
+ "make", "O=%s" % args.outputdir, "-C", args.buildrootdir, "olddefconfig")
+ ret = await proc.wait()
+ if ret:
+ return ret
- subprocess.check_call(["make", "O=%s" % args.outputdir, "-C", args.buildrootdir,
- "savedefconfig"])
+ proc = await asyncio.create_subprocess_exec(
+ "make", "O=%s" % args.outputdir, "-C", args.buildrootdir, "savedefconfig")
+ ret = await proc.wait()
+ if ret:
+ return ret
- return subprocess.call(["make", "O=%s" % args.outputdir, "-C", args.buildrootdir,
- "dependencies"])
+ proc = await asyncio.create_subprocess_exec(
+ "make", "O=%s" % args.outputdir, "-C", args.buildrootdir, "dependencies")
+ return await proc.wait()
if __name__ == '__main__':
@@ -642,7 +652,7 @@ if __name__ == '__main__':
args.outputdir = os.path.abspath(args.outputdir)
try:
- ret = gen_config(args)
+ ret = asyncio.run(gen_config(args))
except Exception as e:
print(str(e), file=sys.stderr)
parser.exit(1)
Since genrandconfig no longer appears to support python2 we can migrate the subprocess calls to use asyncio variants. This has the advantage of allowing for runners like autobuild-run to integrate directly into genrandconfig by calling the asyncio gen_config using importlib instead of having to run genrandconfig as a subprocess. Using asyncio is advantageous here as it eliminates the requirement for the runner to deal with blocking subprocess calls(by having to use threading for example). Also cleanup some unused functions/python2 compatibility shims. Signed-off-by: James Hilliard <james.hilliard1@gmail.com> --- utils/genrandconfig | 88 +++++++++++++++++++++++++-------------------- 1 file changed, 49 insertions(+), 39 deletions(-)