From patchwork Mon Jul 8 08:14:03 2019 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Victor Huesca X-Patchwork-Id: 1128915 Return-Path: X-Original-To: incoming-buildroot@patchwork.ozlabs.org Delivered-To: patchwork-incoming-buildroot@bilbo.ozlabs.org Authentication-Results: ozlabs.org; spf=pass (mailfrom) smtp.mailfrom=busybox.net (client-ip=140.211.166.138; helo=whitealder.osuosl.org; envelope-from=buildroot-bounces@busybox.net; receiver=) Authentication-Results: ozlabs.org; dmarc=none (p=none dis=none) header.from=bootlin.com Received: from whitealder.osuosl.org (smtp1.osuosl.org [140.211.166.138]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by ozlabs.org (Postfix) with ESMTPS id 45hysx1Z3Wz9sNF for ; Mon, 8 Jul 2019 18:14:45 +1000 (AEST) Received: from localhost (localhost [127.0.0.1]) by whitealder.osuosl.org (Postfix) with ESMTP id 67F8A86196; Mon, 8 Jul 2019 08:14:43 +0000 (UTC) X-Virus-Scanned: amavisd-new at osuosl.org Received: from whitealder.osuosl.org ([127.0.0.1]) by localhost (.osuosl.org [127.0.0.1]) (amavisd-new, port 10024) with ESMTP id Wd3sX+E4g+CK; Mon, 8 Jul 2019 08:14:32 +0000 (UTC) Received: from ash.osuosl.org (ash.osuosl.org [140.211.166.34]) by whitealder.osuosl.org (Postfix) with ESMTP id 031A386055; Mon, 8 Jul 2019 08:14:32 +0000 (UTC) X-Original-To: buildroot@lists.busybox.net Delivered-To: buildroot@osuosl.org Received: from silver.osuosl.org (smtp3.osuosl.org [140.211.166.136]) by ash.osuosl.org (Postfix) with ESMTP id BBF3F1BF476 for ; Mon, 8 Jul 2019 08:14:27 +0000 (UTC) Received: from localhost (localhost [127.0.0.1]) by silver.osuosl.org (Postfix) with ESMTP id B96F12044C for ; Mon, 8 Jul 2019 08:14:27 +0000 (UTC) X-Virus-Scanned: amavisd-new at osuosl.org Received: from silver.osuosl.org ([127.0.0.1]) by localhost (.osuosl.org [127.0.0.1]) (amavisd-new, port 10024) with ESMTP id 8oW7HcPgt5yt for ; Mon, 8 Jul 2019 08:14:24 +0000 (UTC) X-Greylist: domain auto-whitelisted by SQLgrey-1.7.6 Received: from relay8-d.mail.gandi.net (relay8-d.mail.gandi.net [217.70.183.201]) by silver.osuosl.org (Postfix) with ESMTPS id 1D4E9204E2 for ; Mon, 8 Jul 2019 08:14:23 +0000 (UTC) X-Originating-IP: 86.250.200.211 Received: from localhost.localdomain (lfbn-1-17395-211.w86-250.abo.wanadoo.fr [86.250.200.211]) (Authenticated sender: victor.huesca@bootlin.com) by relay8-d.mail.gandi.net (Postfix) with ESMTPSA id 68E121BF218; Mon, 8 Jul 2019 08:14:15 +0000 (UTC) From: Victor Huesca To: buildroot@buildroot.org Date: Mon, 8 Jul 2019 10:14:03 +0200 Message-Id: <20190708081406.28215-2-victor.huesca@bootlin.com> X-Mailer: git-send-email 2.21.0 In-Reply-To: <20190708081406.28215-1-victor.huesca@bootlin.com> References: <20190708081406.28215-1-victor.huesca@bootlin.com> MIME-Version: 1.0 Subject: [Buildroot] [PATCH buildroot-test 1/4] utils/daily-mail: add information about version that may need an update X-BeenThere: buildroot@busybox.net X-Mailman-Version: 2.1.29 Precedence: list List-Id: Discussion and development of buildroot List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Cc: Victor Huesca , thomas@bootlin.com, antoine@bootlin.com Errors-To: buildroot-bounces@busybox.net Sender: "buildroot" This patch use the output of buildroot pkg-stats to inform maintainers about packages they maintain that may need an update. Signed-off-by: Victor Huesca --- utils/daily-mail | 96 +++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 83 insertions(+), 13 deletions(-) diff --git a/utils/daily-mail b/utils/daily-mail index cf600c5..4ab7710 100755 --- a/utils/daily-mail +++ b/utils/daily-mail @@ -11,10 +11,15 @@ from datetime import date, timedelta import localconfig import csv from collections import defaultdict +import json +from packaging import version +import re sys.path.append(os.path.join(localconfig.brbase, "utils")) import getdeveloperlib +RE_HASH_40 = re.compile(r'.*[a-fA-F0-9]{40}.*') + baseurl = "autobuild.buildroot.net" http_baseurl = "http://" + baseurl @@ -59,6 +64,7 @@ class Notification: def __init__(self): self.arch_notifications = defaultdict(list) self.package_notifications = defaultdict(list) + self.package_version_notification = [] # Calculate the list of .mk files in the Buildroot source tree, will # be used to guess the name of the packages that caused build @@ -139,6 +145,19 @@ def add_package_notification(branch, notifications, build_result): n.package_notifications[branch].append(build_result) build_result['orphan'] = orphan +def add_outdated_pkg_notification(notifications, outdated_pkg): + for pkg in outdated_pkg: + orphan = True + for dev in developers: + if pkg['name'] in dev.packages: + orphan = False + notif = get_notification_for_dev(notifications, dev) + notif.package_version_notification.append(pkg) + if orphan: + pkg['orphan'] = True + notif = get_notification_for_dev(notifications, get_orphan_developer()) + notif.package_version_notification.append(pkg) + def show_results(results, show_status, show_orphan=False): contents = "" for r in results: @@ -157,33 +176,55 @@ def show_results(results, show_status, show_orphan=False): orphan_str = "" url = http_baseurl + "/results/" + r['identifier'] if show_status: - contents += "%12s | %30s | %3s | %40s" % (arch, reason, status_str, url) + contents += "%12s | %30s | %3s | %79s" % (arch, reason, status_str, url) else: - contents += "%12s | %30s | %40s" % (arch, reason, url) + contents += "%12s | %30s | %79s" % (arch, reason, url) if show_orphan: contents += " | %4s\n" % (orphan_str) else: contents += "\n" return contents +def show_outdated(packages, show_orphan=False, show_header=False): + contents = '' + if show_header: + contents += '{:^30} | {:^8} | {:^44} | {:^12} | {:^12}'.format('name', 'found by', + 'link to release-moinotring.org', + 'version', 'upstream') + if show_orphan: + contents += ' | {:^5}'.format('orph?') + contents += '\n{0:-^30}-+-{0:-^8}-+-{0:-^44}-+-{0:-^12}-+-{0:-^12}-'.format('') + if show_orphan: + contents += '+-{:-^5}-'.format('') + contents += '\n' + + for pkg in packages: + pkg_str = { k: (v[:9] + '...' if k in ('version', 'upstream') and len(v) > 12 else v) for k, v in pkg.items() } + orphan_str = 'ORHP' if 'orphan' in pkg and pkg['orphan'] else '' + contents += '{name:>30} | {from:^8} | https://release-monitoring.org/project/{id:0>5} | {version:12} | {upstream:12}'.format(**pkg_str) + if show_orphan: + contents += ' | {:4}'.format(orphan_str) + contents += '\n' + return contents + # Send the e-mails to the individual developers def developers_email(smtp, branches, notifications, datestr, dry_run): - for k, v in notifications.iteritems(): - to = k.name + for dev, notif in notifications.iteritems(): + to = dev.name email_from = localconfig.fromaddr subject = "[%s] Your build results for %s" % (baseurl, datestr) contents = "Hello,\n\n" contents += textwrap.fill("This is the list of Buildroot build failures that occured on %s, and for which you are a registered architecture developer or package developer. Please help us improving the quality of Buildroot by investigating those build failures and sending patches to fix them. Thanks!" % datestr) contents += "\n\n" - show_orphan = k.name == ORPHAN_DEVELOPER + show_orphan = dev.name == ORPHAN_DEVELOPER for branch in branches: - if v.arch_notifications.has_key(branch): - archs = v.arch_notifications[branch] + if notif.arch_notifications.has_key(branch): + archs = notif.arch_notifications[branch] else: archs = [] - if v.package_notifications.has_key(branch): - packages = v.package_notifications[branch] + if notif.package_notifications.has_key(branch): + packages = notif.package_notifications[branch] else: packages = [] @@ -203,6 +244,12 @@ def developers_email(smtp, branches, notifications, datestr, dry_run): contents += "\n" + outdated = notif.package_version_notification + if len(outdated) != 0: + contents += "Packages you maintain that have a new upstream version:\n\n" + contents += show_outdated(outdated, show_orphan=show_orphan) + contents += '\n' + contents += "-- \n" contents += http_baseurl if dry_run: @@ -220,7 +267,7 @@ def developers_email(smtp, branches, notifications, datestr, dry_run): msg['From'] = email_from msg['Date'] = formatdate() smtp.sendmail(email_from, to, msg.as_string()) - print "To: %s" % k.name + print "To: %s" % dev.name def global_email_branch_result(results, results_by_reason, branch): contents = "Results for branch '%s'\n" % branch @@ -241,7 +288,7 @@ def global_email_branch_result(results, results_by_reason, branch): return contents # Send the global e-mail to the mailing list -def global_email(smtp, results, results_by_reason, datestr, overall, dry_run): +def global_email(smtp, results, results_by_reason, datestr, overall, dry_run, outdated=None): to = "buildroot@buildroot.org" email_from = localconfig.fromaddr subject = "[%s] Build results for %s" % (baseurl, datestr) @@ -260,6 +307,11 @@ def global_email(smtp, results, results_by_reason, datestr, overall, dry_run): if len(results[branch]) == 0: continue contents += global_email_branch_result(results[branch], results_by_reason[branch], branch) + if outdated: + contents += '\n\n' + contents += "Packages having a newer version\n" + contents += "===============================\n\n" + contents += show_outdated(outdated, show_orphan=True) contents += "\n" contents += "-- \n" contents += http_baseurl @@ -304,7 +356,7 @@ def get_build_results_grouped_by_reason(db, datestr, branches): # Prepare the notifications{} dict for the notifications to individual # developers, based on architecture developers and package # developers -def calculate_notifications(results): +def calculate_notifications(results, outdated_pkg): notifications = {} for branch in results.keys(): for result in results[branch]: @@ -313,8 +365,25 @@ def calculate_notifications(results): continue add_arch_notification(branch, notifications, result) add_package_notification(branch, notifications, result) + add_outdated_pkg_notification(notifications, outdated_pkg) return notifications +def get_outdated_pkg(path): + with open(path, 'r') as f: + stats = json.load(f) + s = [] + for name, pkg in stats['packages'].items(): + status, latest_ver, p_id = pkg['latest_version'] + cur_ver = pkg['current_version'] + if status in (2, 3) and cur_ver and latest_ver and not RE_HASH_40.match(cur_ver) \ + and version.parse(str(cur_ver)) < version.parse(str(latest_ver)): + s.append( { 'name': str(name), + 'id': p_id, + 'version': str(cur_ver), + 'upstream': str(latest_ver), + 'from': 'MAPPING' if status == 2 else 'NAME' } ) + return sorted(s, key=lambda pkg: pkg['name']) + def __main__(): yesterday = date.today() - timedelta(1) yesterday_str = yesterday.strftime('%Y-%m-%d') @@ -327,7 +396,8 @@ def __main__(): overall_stats = get_overall_stats(db, yesterday_str, branches) results = get_build_results(db, yesterday_str, branches) results_by_reason = get_build_results_grouped_by_reason(db, yesterday_str, branches) - notifications = calculate_notifications(results) + outdated_pkg = get_outdated_pkg(localconfig.pkg_stats) + notifications = calculate_notifications(results, outdated_pkg) dry_run = False if len(sys.argv) == 2 and sys.argv[1] == "--dry-run": dry_run = True From patchwork Mon Jul 8 08:14:04 2019 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Victor Huesca X-Patchwork-Id: 1128911 Return-Path: X-Original-To: incoming-buildroot@patchwork.ozlabs.org Delivered-To: patchwork-incoming-buildroot@bilbo.ozlabs.org Authentication-Results: ozlabs.org; spf=pass (mailfrom) smtp.mailfrom=busybox.net (client-ip=140.211.166.133; helo=hemlock.osuosl.org; envelope-from=buildroot-bounces@busybox.net; receiver=) Authentication-Results: ozlabs.org; dmarc=none (p=none dis=none) header.from=bootlin.com Received: from hemlock.osuosl.org (smtp2.osuosl.org [140.211.166.133]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by ozlabs.org (Postfix) with ESMTPS id 45hysj73wbz9sNF for ; Mon, 8 Jul 2019 18:14:33 +1000 (AEST) Received: from localhost (localhost [127.0.0.1]) by hemlock.osuosl.org (Postfix) with ESMTP id 18E3D8734E; Mon, 8 Jul 2019 08:14:28 +0000 (UTC) X-Virus-Scanned: amavisd-new at osuosl.org Received: from hemlock.osuosl.org ([127.0.0.1]) by localhost (.osuosl.org [127.0.0.1]) (amavisd-new, port 10024) with ESMTP id Mx9Z9rJGEcHW; Mon, 8 Jul 2019 08:14:26 +0000 (UTC) Received: from ash.osuosl.org (ash.osuosl.org [140.211.166.34]) by hemlock.osuosl.org (Postfix) with ESMTP id 4A6CE8733D; Mon, 8 Jul 2019 08:14:26 +0000 (UTC) X-Original-To: buildroot@lists.busybox.net Delivered-To: buildroot@osuosl.org Received: from silver.osuosl.org (smtp3.osuosl.org [140.211.166.136]) by ash.osuosl.org (Postfix) with ESMTP id A4BFA1BF476 for ; Mon, 8 Jul 2019 08:14:25 +0000 (UTC) Received: from localhost (localhost [127.0.0.1]) by silver.osuosl.org (Postfix) with ESMTP id 97A92204EA for ; Mon, 8 Jul 2019 08:14:25 +0000 (UTC) X-Virus-Scanned: amavisd-new at osuosl.org Received: from silver.osuosl.org ([127.0.0.1]) by localhost (.osuosl.org [127.0.0.1]) (amavisd-new, port 10024) with ESMTP id MWhCKF7eeaJP for ; Mon, 8 Jul 2019 08:14:24 +0000 (UTC) X-Greylist: domain auto-whitelisted by SQLgrey-1.7.6 Received: from relay8-d.mail.gandi.net (relay8-d.mail.gandi.net [217.70.183.201]) by silver.osuosl.org (Postfix) with ESMTPS id 094912044C for ; Mon, 8 Jul 2019 08:14:23 +0000 (UTC) X-Originating-IP: 86.250.200.211 Received: from localhost.localdomain (lfbn-1-17395-211.w86-250.abo.wanadoo.fr [86.250.200.211]) (Authenticated sender: victor.huesca@bootlin.com) by relay8-d.mail.gandi.net (Postfix) with ESMTPSA id E48F01BF20A; Mon, 8 Jul 2019 08:14:15 +0000 (UTC) From: Victor Huesca To: buildroot@buildroot.org Date: Mon, 8 Jul 2019 10:14:04 +0200 Message-Id: <20190708081406.28215-3-victor.huesca@bootlin.com> X-Mailer: git-send-email 2.21.0 In-Reply-To: <20190708081406.28215-1-victor.huesca@bootlin.com> References: <20190708081406.28215-1-victor.huesca@bootlin.com> MIME-Version: 1.0 Subject: [Buildroot] [PATCH buildroot-test 2/4] utils/daily-mail: allow to print a header for results section X-BeenThere: buildroot@busybox.net X-Mailman-Version: 2.1.29 Precedence: list List-Id: Discussion and development of buildroot List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Cc: Victor Huesca , thomas@bootlin.com, antoine@bootlin.com Errors-To: buildroot-bounces@busybox.net Sender: "buildroot" This table quite clear without header but as I had a template for the outdated table I also added the possibility to include one for results too. Note that this commit only add the option to print the header but does not enable it, so the existing behavior is preserved. Signed-off-by: Victor Huesca --- utils/daily-mail | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/utils/daily-mail b/utils/daily-mail index 4ab7710..b4d0a56 100755 --- a/utils/daily-mail +++ b/utils/daily-mail @@ -158,8 +158,25 @@ def add_outdated_pkg_notification(notifications, outdated_pkg): notif = get_notification_for_dev(notifications, get_orphan_developer()) notif.package_version_notification.append(pkg) -def show_results(results, show_status, show_orphan=False): + +def show_results(results, show_status, show_orphan=False, show_header=False): contents = "" + if show_header: + if show_status: + contents += '{:^12} | {:^30} | {:^3} | {:^79}'.format('arch', 'reason', 'OK?', 'url') + else: + contents += '{:^12} | {:^30} | {:^79}'.format('arch', 'reason', 'url') + if show_orphan: + contents += ' | {:^5}'.format('orph?') + + if show_status: + contents += '\n{0:-^12}-+-{0:-^30}-+-{0:-^3}-+-{0:-^79}-'.format('') + else: + contents += '\n{0:-^12}-+-{0:-^30}-+-{0:-^79}-'.format('') + if show_orphan: + contents += '+-{:-^5}-'.format('') + contents += '\n' + for r in results: arch = r['arch'] reason = r['reason'] From patchwork Mon Jul 8 08:14:05 2019 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Victor Huesca X-Patchwork-Id: 1128914 Return-Path: X-Original-To: incoming-buildroot@patchwork.ozlabs.org Delivered-To: patchwork-incoming-buildroot@bilbo.ozlabs.org Authentication-Results: ozlabs.org; spf=pass (mailfrom) smtp.mailfrom=busybox.net (client-ip=140.211.166.138; helo=whitealder.osuosl.org; envelope-from=buildroot-bounces@busybox.net; receiver=) Authentication-Results: ozlabs.org; dmarc=none (p=none dis=none) header.from=bootlin.com Received: from whitealder.osuosl.org (smtp1.osuosl.org [140.211.166.138]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by ozlabs.org (Postfix) with ESMTPS id 45hyst1GMGz9sNF for ; Mon, 8 Jul 2019 18:14:42 +1000 (AEST) Received: from localhost (localhost [127.0.0.1]) by whitealder.osuosl.org (Postfix) with ESMTP id 99B5C860BE; Mon, 8 Jul 2019 08:14:40 +0000 (UTC) X-Virus-Scanned: amavisd-new at osuosl.org Received: from whitealder.osuosl.org ([127.0.0.1]) by localhost (.osuosl.org [127.0.0.1]) (amavisd-new, port 10024) with ESMTP id 8VyZ-cEJeden; Mon, 8 Jul 2019 08:14:34 +0000 (UTC) Received: from ash.osuosl.org (ash.osuosl.org [140.211.166.34]) by whitealder.osuosl.org (Postfix) with ESMTP id C947E8610B; Mon, 8 Jul 2019 08:14:33 +0000 (UTC) X-Original-To: buildroot@lists.busybox.net Delivered-To: buildroot@osuosl.org Received: from whitealder.osuosl.org (smtp1.osuosl.org [140.211.166.138]) by ash.osuosl.org (Postfix) with ESMTP id 27B8B1BF476 for ; Mon, 8 Jul 2019 08:14:28 +0000 (UTC) Received: from localhost (localhost [127.0.0.1]) by whitealder.osuosl.org (Postfix) with ESMTP id 204DD85D57 for ; Mon, 8 Jul 2019 08:14:28 +0000 (UTC) X-Virus-Scanned: amavisd-new at osuosl.org Received: from whitealder.osuosl.org ([127.0.0.1]) by localhost (.osuosl.org [127.0.0.1]) (amavisd-new, port 10024) with ESMTP id XdBmnjjOpXvC for ; Mon, 8 Jul 2019 08:14:25 +0000 (UTC) X-Greylist: domain auto-whitelisted by SQLgrey-1.7.6 Received: from relay8-d.mail.gandi.net (relay8-d.mail.gandi.net [217.70.183.201]) by whitealder.osuosl.org (Postfix) with ESMTPS id A9E4285C0B for ; Mon, 8 Jul 2019 08:14:24 +0000 (UTC) X-Originating-IP: 86.250.200.211 Received: from localhost.localdomain (lfbn-1-17395-211.w86-250.abo.wanadoo.fr [86.250.200.211]) (Authenticated sender: victor.huesca@bootlin.com) by relay8-d.mail.gandi.net (Postfix) with ESMTPSA id 538091BF21A; Mon, 8 Jul 2019 08:14:16 +0000 (UTC) From: Victor Huesca To: buildroot@buildroot.org Date: Mon, 8 Jul 2019 10:14:05 +0200 Message-Id: <20190708081406.28215-4-victor.huesca@bootlin.com> X-Mailer: git-send-email 2.21.0 In-Reply-To: <20190708081406.28215-1-victor.huesca@bootlin.com> References: <20190708081406.28215-1-victor.huesca@bootlin.com> MIME-Version: 1.0 Subject: [Buildroot] [PATCH buildroot-test 3/4] utils/daily-mail: add some debug information for the Notification class X-BeenThere: buildroot@busybox.net X-Mailman-Version: 2.1.29 Precedence: list List-Id: Discussion and development of buildroot List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Cc: Victor Huesca , thomas@bootlin.com, antoine@bootlin.com Errors-To: buildroot-bounces@busybox.net Sender: "buildroot" It was useful to me when working on adding support for new notification so it may help someone else. Signed-off-by: Victor Huesca --- utils/daily-mail | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/utils/daily-mail b/utils/daily-mail index b4d0a56..85a1c4e 100755 --- a/utils/daily-mail +++ b/utils/daily-mail @@ -65,6 +65,18 @@ class Notification: self.arch_notifications = defaultdict(list) self.package_notifications = defaultdict(list) self.package_version_notification = [] + def __repr__(self): + things = [] + if len(self.arch_notifications): + things.append( '{} archs'.format(len(self.arch_notifications)) ) + if len(self.package_notifications): + things.append( '{} pkgs'.format(len(self.package_notifications)) ) + if len(self.package_version_notification): + things.append( '{} outdated pkgs'.format(len(self.package_version_notification)) ) + if things: + return 'Notification <' + ', '.join(things) + '>' + else: + return None # Calculate the list of .mk files in the Buildroot source tree, will # be used to guess the name of the packages that caused build From patchwork Mon Jul 8 08:14:06 2019 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Victor Huesca X-Patchwork-Id: 1128912 Return-Path: X-Original-To: incoming-buildroot@patchwork.ozlabs.org Delivered-To: patchwork-incoming-buildroot@bilbo.ozlabs.org Authentication-Results: ozlabs.org; spf=pass (mailfrom) smtp.mailfrom=busybox.net (client-ip=140.211.166.137; helo=fraxinus.osuosl.org; envelope-from=buildroot-bounces@busybox.net; receiver=) Authentication-Results: ozlabs.org; dmarc=none (p=none dis=none) header.from=bootlin.com Received: from fraxinus.osuosl.org (smtp4.osuosl.org [140.211.166.137]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by ozlabs.org (Postfix) with ESMTPS id 45hysm1rYMz9sNF for ; Mon, 8 Jul 2019 18:14:35 +1000 (AEST) Received: from localhost (localhost [127.0.0.1]) by fraxinus.osuosl.org (Postfix) with ESMTP id 407CC85631; Mon, 8 Jul 2019 08:14:32 +0000 (UTC) X-Virus-Scanned: amavisd-new at osuosl.org Received: from fraxinus.osuosl.org ([127.0.0.1]) by localhost (.osuosl.org [127.0.0.1]) (amavisd-new, port 10024) with ESMTP id vOv0gtWKy_DK; Mon, 8 Jul 2019 08:14:30 +0000 (UTC) Received: from ash.osuosl.org (ash.osuosl.org [140.211.166.34]) by fraxinus.osuosl.org (Postfix) with ESMTP id 2BDD8855CB; Mon, 8 Jul 2019 08:14:30 +0000 (UTC) X-Original-To: buildroot@lists.busybox.net Delivered-To: buildroot@osuosl.org Received: from silver.osuosl.org (smtp3.osuosl.org [140.211.166.136]) by ash.osuosl.org (Postfix) with ESMTP id 4EBAB1BF476 for ; Mon, 8 Jul 2019 08:14:26 +0000 (UTC) Received: from localhost (localhost [127.0.0.1]) by silver.osuosl.org (Postfix) with ESMTP id 4C08B2044C for ; Mon, 8 Jul 2019 08:14:26 +0000 (UTC) X-Virus-Scanned: amavisd-new at osuosl.org Received: from silver.osuosl.org ([127.0.0.1]) by localhost (.osuosl.org [127.0.0.1]) (amavisd-new, port 10024) with ESMTP id T5htaeNfz19R for ; Mon, 8 Jul 2019 08:14:24 +0000 (UTC) X-Greylist: domain auto-whitelisted by SQLgrey-1.7.6 Received: from relay8-d.mail.gandi.net (relay8-d.mail.gandi.net [217.70.183.201]) by silver.osuosl.org (Postfix) with ESMTPS id 20AE0204E8 for ; Mon, 8 Jul 2019 08:14:23 +0000 (UTC) X-Originating-IP: 86.250.200.211 Received: from localhost.localdomain (lfbn-1-17395-211.w86-250.abo.wanadoo.fr [86.250.200.211]) (Authenticated sender: victor.huesca@bootlin.com) by relay8-d.mail.gandi.net (Postfix) with ESMTPSA id B02771BF206; Mon, 8 Jul 2019 08:14:16 +0000 (UTC) From: Victor Huesca To: buildroot@buildroot.org Date: Mon, 8 Jul 2019 10:14:06 +0200 Message-Id: <20190708081406.28215-5-victor.huesca@bootlin.com> X-Mailer: git-send-email 2.21.0 In-Reply-To: <20190708081406.28215-1-victor.huesca@bootlin.com> References: <20190708081406.28215-1-victor.huesca@bootlin.com> MIME-Version: 1.0 Subject: [Buildroot] [PATCH buildroot-test 4/4] utils/daily-mail: add multiple argument to ease modularity of the script X-BeenThere: buildroot@busybox.net X-Mailman-Version: 2.1.29 Precedence: list List-Id: Discussion and development of buildroot List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Cc: Victor Huesca , thomas@bootlin.com, antoine@bootlin.com Errors-To: buildroot-bounces@busybox.net Sender: "buildroot" Options added: - Include out-to-date packages - Specify a custom date instead of today - Enable/disable a header for tables displayed - Specify branches to include instead of reading the 'branches' file - Allow to send emails to developers only / to the maillist only Signed-off-by: Victor Huesca --- utils/daily-mail | 66 +++++++++++++++++++++++++++++++++--------------- 1 file changed, 45 insertions(+), 21 deletions(-) diff --git a/utils/daily-mail b/utils/daily-mail index 85a1c4e..9c64645 100755 --- a/utils/daily-mail +++ b/utils/daily-mail @@ -14,11 +14,13 @@ from collections import defaultdict import json from packaging import version import re +import argparse sys.path.append(os.path.join(localconfig.brbase, "utils")) import getdeveloperlib RE_HASH_40 = re.compile(r'.*[a-fA-F0-9]{40}.*') +RE_DATE = re.compile(r'^\d\d\d\d-\d\d-\d\d$') baseurl = "autobuild.buildroot.net" http_baseurl = "http://" + baseurl @@ -237,7 +239,7 @@ def show_outdated(packages, show_orphan=False, show_header=False): return contents # Send the e-mails to the individual developers -def developers_email(smtp, branches, notifications, datestr, dry_run): +def developers_email(smtp, branches, notifications, datestr, dry_run, show_header=False): for dev, notif in notifications.iteritems(): to = dev.name email_from = localconfig.fromaddr @@ -265,18 +267,18 @@ def developers_email(smtp, branches, notifications, datestr, dry_run): if len(archs) != 0: contents += "Build failures related to your architectures:\n\n" - contents += show_results(archs, show_status=False, show_orphan=False) + contents += show_results(archs, show_status=False, show_orphan=False, show_header=show_header) contents += "\n\n" if len(packages) != 0: contents += "Build failures related to your packages:\n\n" - contents += show_results(packages, show_status=False, show_orphan=show_orphan) + contents += show_results(packages, show_status=False, show_orphan=show_orphan, show_header=show_header) contents += "\n" outdated = notif.package_version_notification if len(outdated) != 0: contents += "Packages you maintain that have a new upstream version:\n\n" - contents += show_outdated(outdated, show_orphan=show_orphan) + contents += show_outdated(outdated, show_orphan=show_orphan, show_header=show_header) contents += '\n' contents += "-- \n" @@ -298,11 +300,14 @@ def developers_email(smtp, branches, notifications, datestr, dry_run): smtp.sendmail(email_from, to, msg.as_string()) print "To: %s" % dev.name -def global_email_branch_result(results, results_by_reason, branch): +def global_email_branch_result(results, results_by_reason, branch, show_header=False): contents = "Results for branch '%s'\n" % branch contents += "=====================" + "=" * len(branch) + "\n\n" contents += "Classification of failures by reason\n" contents += "------------------------------------\n\n" + if show_header: + contents += "{:>30} | {:<2}\n".format('reason', 'n') + contents += "{0:-^30}-+-{0:-^2}\n".format('') for r in results_by_reason: reason = r['reason'] if len(reason) > 30: @@ -312,35 +317,37 @@ def global_email_branch_result(results, results_by_reason, branch): contents += "\n\n" contents += "Detail of failures\n" contents += "------------------\n\n" - contents += show_results(results, show_status=True, show_orphan=True) + contents += show_results(results, show_status=True, show_orphan=True, show_header=show_header) contents += "\n" return contents # Send the global e-mail to the mailing list -def global_email(smtp, results, results_by_reason, datestr, overall, dry_run, outdated=None): +def global_email(smtp, results, results_by_reason, datestr, overall, outdated, dry_run, show_header=False): to = "buildroot@buildroot.org" email_from = localconfig.fromaddr subject = "[%s] Build results for %s" % (baseurl, datestr) contents = "Hello,\n\n" contents += "Build statistics for %s\n" % datestr contents += "===============================\n\n" - contents += " branch | OK | NOK | TIM | TOT |\n" + contents += " branch | OK | NOK | TIM | TOT\n" + if show_header: + contents += "-------------+-----+-----+-----+-----\n" for branch in sorted(overall.iterkeys()): stats = overall[branch] if stats[3] == 0: continue - contents += " %10s | %3d | %3d | %3d | %3d |\n" % \ + contents += " %10s | %3d | %3d | %3d | %3d\n" % \ (branch, stats[0], stats[1], stats[2], stats[3]) contents += "\n" for branch in results.keys(): if len(results[branch]) == 0: continue - contents += global_email_branch_result(results[branch], results_by_reason[branch], branch) + contents += global_email_branch_result(results[branch], results_by_reason[branch], branch, show_header=show_header) if outdated: contents += '\n\n' contents += "Packages having a newer version\n" contents += "===============================\n\n" - contents += show_outdated(outdated, show_orphan=True) + contents += show_outdated(outdated, show_orphan=True, show_header=show_header) contents += "\n" contents += "-- \n" contents += http_baseurl @@ -413,10 +420,29 @@ def get_outdated_pkg(path): 'from': 'MAPPING' if status == 2 else 'NAME' } ) return sorted(s, key=lambda pkg: pkg['name']) +def parse_args(): + parser = argparse.ArgumentParser() + parser.add_argument('--dry-run', action='store_true', help='Do not send email, only show output') + parser.add_argument('--outdated', action='store_true', help='Include outdated packages in emails') + parser.add_argument('--date', action='store', help='Use this date instead of today') + parser.add_argument('--no-header', dest='header', action='store_false', help='Dont not print header for the different tables') + parser.add_argument('--branches', dest='branches', action='store', help='List of branches (comma separated)') + group = parser.add_mutually_exclusive_group() + group.add_argument('--global-only', dest='dev_email', action='store_false', help='Only send global email') + group.add_argument('--dev-only', dest='global_email', action='store_false', help='Only send developers email') + args = parser.parse_args() + if args.date and not RE_DATE.match(args.date): + parser.error('date must be in format YYYY-MM-DD') + return args + def __main__(): - yesterday = date.today() - timedelta(1) - yesterday_str = yesterday.strftime('%Y-%m-%d') - branches = get_branches() + args = parse_args() + if not args.date: + yesterday = date.today() - timedelta(1) + yesterday_str = yesterday.strftime('%Y-%m-%d') + else: + yesterday_str = args.date + branches = get_branches() if not args.branches else args.branches.split(',') db = _mysql.connect(host=localconfig.host, user=localconfig.user, @@ -425,17 +451,15 @@ def __main__(): overall_stats = get_overall_stats(db, yesterday_str, branches) results = get_build_results(db, yesterday_str, branches) results_by_reason = get_build_results_grouped_by_reason(db, yesterday_str, branches) - outdated_pkg = get_outdated_pkg(localconfig.pkg_stats) + outdated_pkg = get_outdated_pkg(localconfig.pkg_stats) if args.outdated else [] notifications = calculate_notifications(results, outdated_pkg) - dry_run = False - if len(sys.argv) == 2 and sys.argv[1] == "--dry-run": - dry_run = True smtp = smtplib.SMTP(localconfig.smtphost, localconfig.smtpport) smtp.starttls() smtp.login(localconfig.smtpuser, localconfig.smtppass) - developers_email(smtp, branches, notifications, yesterday_str, dry_run) - global_email(smtp, results, results_by_reason, yesterday_str, - overall_stats, dry_run) + if args.dev_email: + developers_email(smtp, branches, notifications, yesterday_str, args.dry_run, args.header) + if args.global_email: + global_email(smtp, results, results_by_reason, yesterday_str, overall_stats, outdated_pkg, args.dry_run, args.header) smtp.quit() __main__()