| Message ID | 1426693843-28792-9-git-send-email-dywi@mailerd.de |
|---|---|
| State | Superseded |
| Headers | show |
Hi André, Thomas, all, On Wed, Mar 18, 2015 at 4:50 PM, André Erdmann <dywi@mailerd.de> wrote: > Preparation step for passing LANG to worker (sub-)processes, > allows to redirect stdin/stdout/stderr, which all default to devnull now > unless specified otherwise. > This makes the "yes"-pipe in "make oldconfig" redundant. > > Signed-off-by: André Erdmann <dywi@mailerd.de> [...] > > return 0 > @@ -459,6 +480,7 @@ def gen_config(**kwargs): > > idir = "instance-%d" % kwargs['instance'] > log = kwargs['log'] > + sysinfo = kwargs['sysinfo'] > > # We need the absolute path to use with O=, because the relative > # path to the output directory here is not relative to the > @@ -493,10 +515,7 @@ def gen_config(**kwargs): > with open(os.path.join(outputdir, ".config"), "w+") as configf: > configf.writelines(configlines) > > - devnull = open(os.devnull, "w") > - > - ret = subprocess.call(["yes '' 2>/dev/null| make O=%s -C %s oldconfig" % \ > - (outputdir, srcdir)], shell=True, stdout=devnull, stderr=devnull) > + ret = sysinfo.run_cmd(["make", "O=%s" % outputdir, "-C", srcdir, "oldconfig"]) Any reason to drop the yes pipe in the command? > if ret != 0: > log_write(log, "ERROR: cannot oldconfig") > return -1 > @@ -504,23 +523,20 @@ def gen_config(**kwargs): > # Now, generate the random selection of packages, and fixup > # things if needed. > while True: > - ret = subprocess.call(["make", "O=%s" % outputdir, "-C", srcdir, > - "KCONFIG_PROBABILITY=%d" % randint(1,30), "randpackageconfig"], > - stdout=devnull, stderr=devnull) > + ret = sysinfo.run_cmd(["make", "O=%s" % outputdir, "-C", srcdir, > + "KCONFIG_PROBABILITY=%d" % randint(1,30), "randpackageconfig"]) > if ret != 0: > log_write(log, "ERROR: cannot generate random configuration") > return -1 > if fixup_config(**kwargs): > break > > - ret = subprocess.call(["yes '' 2>/dev/null| make O=%s -C %s oldconfig" % \ > - (outputdir, srcdir)], shell=True, stdout=devnull, stderr=devnull) > + ret = sysinfo.run_cmd(["make", "O=%s" % outputdir, "-C", srcdir, "oldconfig"]) ditto > if ret != 0: > log_write(log, "ERROR: cannot oldconfig") > return -1 > > - ret = subprocess.call(["make", "O=%s" % outputdir, "-C", srcdir, "savedefconfig"], > - stdout=devnull, stderr=devnull) > + ret = sysinfo.run_cmd(["make", "O=%s" % outputdir, "-C", srcdir, "savedefconfig"]) > if ret != 0: > log_write(log, "ERROR: cannot savedefconfig") > return -1 [...] > @@ -595,15 +613,14 @@ def send_results(result, **kwargs): > shutil.copyfile(os.path.join(outputdir, "legal-info", "manifest.csv"), > os.path.join(resultdir, "licenses-manifest.csv")) > > - subprocess.call(["git log master -n 1 --pretty=format:%%H > %s" % \ > + sysinfo.run_cmd(["git log master -n 1 --pretty=format:%%H > %s" % \ > os.path.join(resultdir, "gitid")], > - shell=True, cwd=srcdir) > + shell=True, cwd=srcdir, stderr=None) IIRC, stderr=None dumps the error output in the process error output, so only available for the autobuilder owner. So, how about log stderr in the log file using sysinfo.run_cmd_get_output (stdout would be anyway redirected to gitid, and stderr would be part of the log)? > > def get_failure_reason(): > # Output is a tuple (package, version), or None. > - lastlines = decode_bytes(subprocess.Popen( > - ["tail", "-n", "3", os.path.join(outputdir, "logfile")], > - stdout=subprocess.PIPE).communicate()[0]).splitlines() > + lastlines = sysinfo.run_cmd_get_stdout( > + ["tail", "-n", "3", os.path.join(outputdir, "logfile")])[1].splitlines() > > regexp = re.compile(r'make: \*\*\* .*/(?:build|toolchain)/([^/]*)/') > for line in lastlines: > @@ -618,9 +635,9 @@ def send_results(result, **kwargs): > """Save the last part of the build log, starting from the failed package""" > > def extract_last_500_lines(): > - subprocess.call(["tail -500 %s > %s" % \ > + sysinfo.run_cmd(["tail -500 %s > %s" % \ > (os.path.join(outputdir, "logfile"), resultfile)], > - shell=True) > + shell=True, stderr=None) ditto > > reason = get_failure_reason() > if not reason: > @@ -677,8 +694,8 @@ def send_results(result, **kwargs): > > # Yes, shutil.make_archive() would be nice, but it doesn't exist > # in Python 2.6. > - ret = subprocess.call(["tar", "cjf", "results.tar.bz2", "results"], > - cwd=outputdir, stdout=log, stderr=log) > + ret = sysinfo.run_cmd_write_to( > + log, ["tar", "cjf", "results.tar.bz2", "results"], cwd=outputdir) > if ret != 0: > log_write(log, "ERROR: could not make results tarball") > sys.exit(1) > @@ -687,13 +704,14 @@ def send_results(result, **kwargs): > # Submit results. Yes, Python has some HTTP libraries, but > # none of the ones that are part of the standard library can > # upload a file without writing dozens of lines of code. > - ret = subprocess.call(["curl", "-u", > - "%s:%s" % (kwargs['http_login'], kwargs['http_password']), > - "-H", "Expect:", > - "-F", "uploadedfile=@%s" % os.path.join(outputdir, "results.tar.bz2"), > - "-F", "uploadsubmit=1", > - "http://autobuild.buildroot.org/submit/"], > - stdout=log, stderr=log) > + ret = sysinfo.run_cmd_write_to( > + log, > + ["curl", "-u", > + "%s:%s" % (kwargs['http_login'], kwargs['http_password']), > + "-H", "Expect:", > + "-F", "uploadedfile=@%s" % os.path.join(outputdir, "results.tar.bz2"), > + "-F", "uploadsubmit=1", > + "http://autobuild.buildroot.org/submit/"]) > if ret != 0: > log_write(log, "INFO: results could not be submitted, %d" % ret) > else: > -- > 2.3.2 > > _______________________________________________ > buildroot mailing list > buildroot@busybox.net > http://lists.busybox.net/mailman/listinfo/buildroot Regards,
Hi, 2015-04-12 9:51 GMT+02:00 Samuel Martin <s.martin49@gmail.com>: > Hi André, Thomas, all, > > On Wed, Mar 18, 2015 at 4:50 PM, André Erdmann <dywi@mailerd.de> wrote: >> Preparation step for passing LANG to worker (sub-)processes, >> allows to redirect stdin/stdout/stderr, which all default to devnull now >> unless specified otherwise. >> This makes the "yes"-pipe in "make oldconfig" redundant. >> >> Signed-off-by: André Erdmann <dywi@mailerd.de> > [...] >> >> return 0 >> @@ -459,6 +480,7 @@ def gen_config(**kwargs): >> >> idir = "instance-%d" % kwargs['instance'] >> log = kwargs['log'] >> + sysinfo = kwargs['sysinfo'] >> >> # We need the absolute path to use with O=, because the relative >> # path to the output directory here is not relative to the >> @@ -493,10 +515,7 @@ def gen_config(**kwargs): >> with open(os.path.join(outputdir, ".config"), "w+") as configf: >> configf.writelines(configlines) >> >> - devnull = open(os.devnull, "w") >> - >> - ret = subprocess.call(["yes '' 2>/dev/null| make O=%s -C %s oldconfig" % \ >> - (outputdir, srcdir)], shell=True, stdout=devnull, stderr=devnull) >> + ret = sysinfo.run_cmd(["make", "O=%s" % outputdir, "-C", srcdir, "oldconfig"]) > Any reason to drop the yes pipe in the command? > It's not needed anymore: SystemInfo::run_cmd() redirects stdin to /dev/null, so this command reads as "make ... oldconfig < /dev/null", which behaves identical to "yes '' | make ... oldconfig". >> if ret != 0: >> log_write(log, "ERROR: cannot oldconfig") >> return -1 >> @@ -504,23 +523,20 @@ def gen_config(**kwargs): >> # Now, generate the random selection of packages, and fixup >> # things if needed. >> while True: >> - ret = subprocess.call(["make", "O=%s" % outputdir, "-C", srcdir, >> - "KCONFIG_PROBABILITY=%d" % randint(1,30), "randpackageconfig"], >> - stdout=devnull, stderr=devnull) >> + ret = sysinfo.run_cmd(["make", "O=%s" % outputdir, "-C", srcdir, >> + "KCONFIG_PROBABILITY=%d" % randint(1,30), "randpackageconfig"]) >> if ret != 0: >> log_write(log, "ERROR: cannot generate random configuration") >> return -1 >> if fixup_config(**kwargs): >> break >> >> - ret = subprocess.call(["yes '' 2>/dev/null| make O=%s -C %s oldconfig" % \ >> - (outputdir, srcdir)], shell=True, stdout=devnull, stderr=devnull) >> + ret = sysinfo.run_cmd(["make", "O=%s" % outputdir, "-C", srcdir, "oldconfig"]) > ditto > >> if ret != 0: >> log_write(log, "ERROR: cannot oldconfig") >> return -1 >> >> - ret = subprocess.call(["make", "O=%s" % outputdir, "-C", srcdir, "savedefconfig"], >> - stdout=devnull, stderr=devnull) >> + ret = sysinfo.run_cmd(["make", "O=%s" % outputdir, "-C", srcdir, "savedefconfig"]) >> if ret != 0: >> log_write(log, "ERROR: cannot savedefconfig") >> return -1 > [...] >> @@ -595,15 +613,14 @@ def send_results(result, **kwargs): >> shutil.copyfile(os.path.join(outputdir, "legal-info", "manifest.csv"), >> os.path.join(resultdir, "licenses-manifest.csv")) >> >> - subprocess.call(["git log master -n 1 --pretty=format:%%H > %s" % \ >> + sysinfo.run_cmd(["git log master -n 1 --pretty=format:%%H > %s" % \ >> os.path.join(resultdir, "gitid")], >> - shell=True, cwd=srcdir) >> + shell=True, cwd=srcdir, stderr=None) > IIRC, stderr=None dumps the error output in the process error output, > so only available for the autobuilder owner. > So, how about log stderr in the log file using > sysinfo.run_cmd_get_output (stdout would be anyway redirected to > gitid, and stderr would be part of the log)? > Sure! But I'd rather do that in a follow-up patch since this one tries to keep the functional changes at a minimum, except for the stdin-/dev/null redirection. (The "git log" command used to write to console before this series, same applies to the "tar" command below.) >> >> def get_failure_reason(): >> # Output is a tuple (package, version), or None. >> - lastlines = decode_bytes(subprocess.Popen( >> - ["tail", "-n", "3", os.path.join(outputdir, "logfile")], >> - stdout=subprocess.PIPE).communicate()[0]).splitlines() >> + lastlines = sysinfo.run_cmd_get_stdout( >> + ["tail", "-n", "3", os.path.join(outputdir, "logfile")])[1].splitlines() >> >> regexp = re.compile(r'make: \*\*\* .*/(?:build|toolchain)/([^/]*)/') >> for line in lastlines: >> @@ -618,9 +635,9 @@ def send_results(result, **kwargs): >> """Save the last part of the build log, starting from the failed package""" >> >> def extract_last_500_lines(): >> - subprocess.call(["tail -500 %s > %s" % \ >> + sysinfo.run_cmd(["tail -500 %s > %s" % \ >> (os.path.join(outputdir, "logfile"), resultfile)], >> - shell=True) >> + shell=True, stderr=None) > ditto > >> >> reason = get_failure_reason() >> if not reason: >> @@ -677,8 +694,8 @@ def send_results(result, **kwargs): >> >> # Yes, shutil.make_archive() would be nice, but it doesn't exist >> # in Python 2.6. >> - ret = subprocess.call(["tar", "cjf", "results.tar.bz2", "results"], >> - cwd=outputdir, stdout=log, stderr=log) >> + ret = sysinfo.run_cmd_write_to( >> + log, ["tar", "cjf", "results.tar.bz2", "results"], cwd=outputdir) >> if ret != 0: >> log_write(log, "ERROR: could not make results tarball") >> sys.exit(1) >> @@ -687,13 +704,14 @@ def send_results(result, **kwargs): >> # Submit results. Yes, Python has some HTTP libraries, but >> # none of the ones that are part of the standard library can >> # upload a file without writing dozens of lines of code. >> - ret = subprocess.call(["curl", "-u", >> - "%s:%s" % (kwargs['http_login'], kwargs['http_password']), >> - "-H", "Expect:", >> - "-F", "uploadedfile=@%s" % os.path.join(outputdir, "results.tar.bz2"), >> - "-F", "uploadsubmit=1", >> - "http://autobuild.buildroot.org/submit/"], >> - stdout=log, stderr=log) >> + ret = sysinfo.run_cmd_write_to( >> + log, >> + ["curl", "-u", >> + "%s:%s" % (kwargs['http_login'], kwargs['http_password']), >> + "-H", "Expect:", >> + "-F", "uploadedfile=@%s" % os.path.join(outputdir, "results.tar.bz2"), >> + "-F", "uploadsubmit=1", >> + "http://autobuild.buildroot.org/submit/"]) >> if ret != 0: >> log_write(log, "INFO: results could not be submitted, %d" % ret) >> else: >> -- >> 2.3.2 >>
diff --git a/scripts/autobuild-run b/scripts/autobuild-run index 409570d..0ee6d57 100755 --- a/scripts/autobuild-run +++ b/scripts/autobuild-run @@ -172,6 +172,7 @@ class SystemInfo: self.needed_progs = list(self.__class__.DEFAULT_NEEDED_PROGS) self.optional_progs = list(self.__class__.DEFAULT_OPTIONAL_PROGS) self.progs = {} + self.devnull = open(os.devnull, "w") def find_prog(self, name, flags=os.X_OK, env=os.environ): if not name or name[0] == os.sep: raise ValueError(name) @@ -203,10 +204,8 @@ class SystemInfo: have_it = self.find_prog(prog) # java[c] needs special care if have_it and prog in ('java', 'javac'): - with open(os.devnull, "w") as devnull: - if subprocess.call("%s -version | grep gcj" % prog, shell=True, - stdout=devnull, stderr=devnull) != 1: - have_it = False + if self.run_cmd("%s -version | grep gcj" % prog, shell=True) != 1: + have_it = False # -- self.progs[prog] = have_it return have_it @@ -231,6 +230,27 @@ class SystemInfo: return not missing_requirements + def popen(self, cmdv, **kwargs): + kwargs.setdefault('stdin', self.devnull) + kwargs.setdefault('stdout', self.devnull) + kwargs.setdefault('stderr', self.devnull) + return subprocess.Popen(cmdv, **kwargs) + + def run_cmd(self, cmdv, **kwargs): + proc = self.popen(cmdv, **kwargs) + proc.communicate() + return proc.poll() + + def run_cmd_write_to(self, outstream, cmdv, **kwargs): + kwargs.update(stdout=outstream, stderr=outstream) + return self.run_cmd(cmdv, **kwargs) + + def run_cmd_get_stdout(self, cmdv, **kwargs): + proc = self.popen(cmdv, stdout=subprocess.PIPE, **kwargs) + stdout_data, _ = proc.communicate() + return (proc.poll(), decode_bytes(stdout_data) if stdout_data else None) + + def get_toolchain_configs(): """Fetch and return the possible toolchain configurations @@ -274,6 +294,7 @@ def prepare_build(**kwargs): idir = "instance-%d" % kwargs['instance'] log = kwargs['log'] + sysinfo = kwargs['sysinfo'] log_write(log, "INFO: preparing a new build") @@ -298,15 +319,15 @@ def prepare_build(**kwargs): # didn't exist already. srcdir = os.path.join(idir, "buildroot") if not os.path.exists(srcdir): - ret = subprocess.call(["git", "clone", "git://git.busybox.net/buildroot", srcdir], - stdout=log, stderr=log) + ret = sysinfo.run_cmd_write_to( + log, ["git", "clone", "git://git.busybox.net/buildroot", srcdir]) if ret != 0: log_write(log, "ERROR: could not clone Buildroot sources") return -1 # Update the Buildroot sources. abssrcdir = os.path.abspath(srcdir) - ret = subprocess.call(["git", "pull"], cwd=abssrcdir, stdout=log, stderr=log) + ret = sysinfo.run_cmd_write_to(log, ["git", "pull"], cwd=abssrcdir) if ret != 0: log_write(log, "ERROR: could not pull Buildroot sources") return -1 @@ -315,7 +336,7 @@ def prepare_build(**kwargs): outputdir = os.path.join(idir, "output") if os.path.exists(outputdir): # shutil.rmtree doesn't remove write-protected files - subprocess.call(["rm", "-rf", outputdir]) + sysinfo.run_cmd(["rm", "-rf", outputdir]) os.mkdir(outputdir) return 0 @@ -459,6 +480,7 @@ def gen_config(**kwargs): idir = "instance-%d" % kwargs['instance'] log = kwargs['log'] + sysinfo = kwargs['sysinfo'] # We need the absolute path to use with O=, because the relative # path to the output directory here is not relative to the @@ -493,10 +515,7 @@ def gen_config(**kwargs): with open(os.path.join(outputdir, ".config"), "w+") as configf: configf.writelines(configlines) - devnull = open(os.devnull, "w") - - ret = subprocess.call(["yes '' 2>/dev/null| make O=%s -C %s oldconfig" % \ - (outputdir, srcdir)], shell=True, stdout=devnull, stderr=devnull) + ret = sysinfo.run_cmd(["make", "O=%s" % outputdir, "-C", srcdir, "oldconfig"]) if ret != 0: log_write(log, "ERROR: cannot oldconfig") return -1 @@ -504,23 +523,20 @@ def gen_config(**kwargs): # Now, generate the random selection of packages, and fixup # things if needed. while True: - ret = subprocess.call(["make", "O=%s" % outputdir, "-C", srcdir, - "KCONFIG_PROBABILITY=%d" % randint(1,30), "randpackageconfig"], - stdout=devnull, stderr=devnull) + ret = sysinfo.run_cmd(["make", "O=%s" % outputdir, "-C", srcdir, + "KCONFIG_PROBABILITY=%d" % randint(1,30), "randpackageconfig"]) if ret != 0: log_write(log, "ERROR: cannot generate random configuration") return -1 if fixup_config(**kwargs): break - ret = subprocess.call(["yes '' 2>/dev/null| make O=%s -C %s oldconfig" % \ - (outputdir, srcdir)], shell=True, stdout=devnull, stderr=devnull) + ret = sysinfo.run_cmd(["make", "O=%s" % outputdir, "-C", srcdir, "oldconfig"]) if ret != 0: log_write(log, "ERROR: cannot oldconfig") return -1 - ret = subprocess.call(["make", "O=%s" % outputdir, "-C", srcdir, "savedefconfig"], - stdout=devnull, stderr=devnull) + ret = sysinfo.run_cmd(["make", "O=%s" % outputdir, "-C", srcdir, "savedefconfig"]) if ret != 0: log_write(log, "ERROR: cannot savedefconfig") return -1 @@ -532,6 +548,7 @@ def do_build(**kwargs): idir = "instance-%d" % kwargs['instance'] log = kwargs['log'] + sysinfo = kwargs['sysinfo'] # We need the absolute path to use with O=, because the relative # path to the output directory here is not relative to the @@ -547,7 +564,7 @@ def do_build(**kwargs): "-C", srcdir, "BR2_DL_DIR=%s" % dldir, "BR2_JLEVEL=%s" % kwargs['njobs']] \ + kwargs['make_opts'].split() - sub = subprocess.Popen(cmd, stdout=f, stderr=f) + sub = sysinfo.popen(cmd, stdout=f, stderr=f) kwargs['buildpid'][kwargs['instance']] = sub.pid ret = sub.wait() kwargs['buildpid'][kwargs['instance']] = 0 @@ -560,7 +577,7 @@ def do_build(**kwargs): if ret != 0: log_write(log, "INFO: build failed") return -1 - ret = subprocess.call(["make", "O=%s" % outputdir, "-C", srcdir], stdout=f, stderr=f) + ret = sysinfo.run_cmd_write_to(f, ["make", "O=%s" % outputdir, "-C", srcdir]) if ret != 0: log_write(log, "INFO: build failed during legal-info") return -1 @@ -577,6 +594,7 @@ def send_results(result, **kwargs): idir = "instance-%d" % kwargs['instance'] log = kwargs['log'] + sysinfo = kwargs['sysinfo'] outputdir = os.path.abspath(os.path.join(idir, "output")) srcdir = os.path.join(idir, "buildroot") @@ -595,15 +613,14 @@ def send_results(result, **kwargs): shutil.copyfile(os.path.join(outputdir, "legal-info", "manifest.csv"), os.path.join(resultdir, "licenses-manifest.csv")) - subprocess.call(["git log master -n 1 --pretty=format:%%H > %s" % \ + sysinfo.run_cmd(["git log master -n 1 --pretty=format:%%H > %s" % \ os.path.join(resultdir, "gitid")], - shell=True, cwd=srcdir) + shell=True, cwd=srcdir, stderr=None) def get_failure_reason(): # Output is a tuple (package, version), or None. - lastlines = decode_bytes(subprocess.Popen( - ["tail", "-n", "3", os.path.join(outputdir, "logfile")], - stdout=subprocess.PIPE).communicate()[0]).splitlines() + lastlines = sysinfo.run_cmd_get_stdout( + ["tail", "-n", "3", os.path.join(outputdir, "logfile")])[1].splitlines() regexp = re.compile(r'make: \*\*\* .*/(?:build|toolchain)/([^/]*)/') for line in lastlines: @@ -618,9 +635,9 @@ def send_results(result, **kwargs): """Save the last part of the build log, starting from the failed package""" def extract_last_500_lines(): - subprocess.call(["tail -500 %s > %s" % \ + sysinfo.run_cmd(["tail -500 %s > %s" % \ (os.path.join(outputdir, "logfile"), resultfile)], - shell=True) + shell=True, stderr=None) reason = get_failure_reason() if not reason: @@ -677,8 +694,8 @@ def send_results(result, **kwargs): # Yes, shutil.make_archive() would be nice, but it doesn't exist # in Python 2.6. - ret = subprocess.call(["tar", "cjf", "results.tar.bz2", "results"], - cwd=outputdir, stdout=log, stderr=log) + ret = sysinfo.run_cmd_write_to( + log, ["tar", "cjf", "results.tar.bz2", "results"], cwd=outputdir) if ret != 0: log_write(log, "ERROR: could not make results tarball") sys.exit(1) @@ -687,13 +704,14 @@ def send_results(result, **kwargs): # Submit results. Yes, Python has some HTTP libraries, but # none of the ones that are part of the standard library can # upload a file without writing dozens of lines of code. - ret = subprocess.call(["curl", "-u", - "%s:%s" % (kwargs['http_login'], kwargs['http_password']), - "-H", "Expect:", - "-F", "uploadedfile=@%s" % os.path.join(outputdir, "results.tar.bz2"), - "-F", "uploadsubmit=1", - "http://autobuild.buildroot.org/submit/"], - stdout=log, stderr=log) + ret = sysinfo.run_cmd_write_to( + log, + ["curl", "-u", + "%s:%s" % (kwargs['http_login'], kwargs['http_password']), + "-H", "Expect:", + "-F", "uploadedfile=@%s" % os.path.join(outputdir, "results.tar.bz2"), + "-F", "uploadsubmit=1", + "http://autobuild.buildroot.org/submit/"]) if ret != 0: log_write(log, "INFO: results could not be submitted, %d" % ret) else:
Preparation step for passing LANG to worker (sub-)processes, allows to redirect stdin/stdout/stderr, which all default to devnull now unless specified otherwise. This makes the "yes"-pipe in "make oldconfig" redundant. Signed-off-by: André Erdmann <dywi@mailerd.de> --- scripts/autobuild-run | 92 ++++++++++++++++++++++++++++++--------------------- 1 file changed, 55 insertions(+), 37 deletions(-)