[{"id":1797929,"web_url":"http://patchwork.ozlabs.org/comment/1797929/","msgid":"","list_archive_url":null,"date":"2017-11-02T13:58:04","subject":"Re: [kteam-tools PATCH] duplicate-bugs: find duplicates given two\n\tcycles","submitter":{"id":71419,"url":"http://patchwork.ozlabs.org/api/people/71419/","name":"Kleber Sacilotto de Souza","email":"kleber.souza@canonical.com"},"content":"On 11/02/17 12:34, Thadeu Lima de Souza Cascardo wrote:\n> Given two cycles and a series, find bugs that match them, and mark the\n> bugs from the old cycle as duplicate of those from the new cycle.\n> \n> Signed-off-by: Thadeu Lima de Souza Cascardo \n> ---\n> stable/duplicate-bugs | 206 ++++++++++++++++++++++++++++++++++++++++++++++++++\n> 1 file changed, 206 insertions(+)\n> create mode 100755 stable/duplicate-bugs\n> \n> diff --git a/stable/duplicate-bugs b/stable/duplicate-bugs\n> new file mode 100755\n> index 00000000..fc15e1ef\n> --- /dev/null\n> +++ b/stable/duplicate-bugs\n> @@ -0,0 +1,206 @@\n> +#!/usr/bin/env python3\n> +#\n> +import sys\n> +import os\n> +\n> +sys.path.append(os.path.realpath(os.path.join(os.path.dirname(__file__), '..', 'py3')))\n> +\n> +\n> +from datetime import datetime, timedelta\n> +from argparse import ArgumentParser, RawDescriptionHelpFormatter\n> +from logging import basicConfig, DEBUG, INFO, WARNING\n> +from ktl.log import center, cleave, cdebug, cinfo, Clog\n> +from ktl.launchpad import Launchpad\n> +from ktl.ubuntu import Ubuntu\n> +\n> +# AppError\n> +#\n> +# A general exception that can be raised when an error is encountered in the app.\n> +#\n> +class AppError(Exception):\n> + # __init__\n> + #\n> + def __init__(self, error=''):\n> + self.msg = error\n> +\n> +# Tracking\n> +#\n> +class Tracking():\n> + '''\n> + '''\n> + # __init__\n> + #\n> + def __init__(s, args):\n> + s.args = args\n> + s.launchpad = Launchpad('start-sru-cycle').service\n> + s.project_tracked = 'kernel-sru-workflow'\n> + ubuntu = Ubuntu()\n> + series = ubuntu.supported_series_version\n> + series.append(ubuntu.development_series_version)\n> + s.series = []\n> + for ss in sorted(series):\n> + s.series.append(ubuntu.db[ss]['name'])\n> +\n> + def get_master(s, project, cycle, series):\n> + '''\n> + Return the bug id of the master bug\n> + '''\n> + center(s.__class__.__name__ + '.get_master')\n> +\n> + retval = None\n> +\n> + cdebug('project: %s' % project)\n> + cycle = 'kernel-sru-cycle-' + cycle\n> + search_tags = [cycle, series, 'kernel-sru-master-kernel']\n> + search_tags_combinator = \"All\"\n> + # A list of the bug statuses that we care about\n> + #\n> + search_status = [\"New\", \"In Progress\", \"Incomplete\", \"Fix Committed\", \"Invalid\"]\n> + # The tracking bugs that we are interested in should have been created recently (days).\n> + #\n> + search_since = datetime.utcnow() - timedelta(days=30)\n> + lp_project = s.launchpad.projects[project]\n> + tasks = lp_project.searchTasks(status=search_status, tags=search_tags, tags_combinator=search_tags_combinator, modified_since=search_since, omit_duplicates=False)\n> +\n> + if len(tasks) == 1:\n> + retval = tasks[0].bug.id\n> +\n> + cleave(s.__class__.__name__ + '.get_master')\n> + return retval\n> +\n> + def get_derivatives(s, project, bugid):\n> + '''\n> + Return the list of bug ids that are derivatives or backports of bugid\n> + '''\n> + center(s.__class__.__name__ + '.get_master')\n> +\n> + retval = []\n> +\n> + cdebug('project: %s' % project)\n> + search_tags = [\"kernel-sru-derivative-of-\" + str(bugid), \"kernel-sru-backport-of-\" + str(bugid)]\n> + search_tags_combinator = \"Any\"\n> + # A list of the bug statuses that we care about\n> + #\n> + search_status = [\"New\", \"In Progress\", \"Incomplete\", \"Fix Committed\", \"Invalid\"]\n> + # The tracking bugs that we are interested in should have been created recently (days).\n> + #\n> + search_since = datetime.utcnow() - timedelta(days=30)\n> + lp_project = s.launchpad.projects[project]\n> + tasks = lp_project.searchTasks(status=search_status, tags=search_tags, tags_combinator=search_tags_combinator, modified_since=search_since, omit_duplicates=False)\n> +\n> + for task in tasks:\n> + retval.append(task.bug.id)\n> +\n> + cleave(s.__class__.__name__ + '.get_master')\n> + return retval\n> +\n> + def get_bugs(s, project, cycle, series):\n> + master = s.get_master(project, cycle, series)\n> + if not master:\n> + return []\n> + bugs = s.get_derivatives(s.project_tracked, master)\n> + bugs.append(master)\n> + return bugs\n> +\n> + def get_series(s, lpbug):\n> + '''\n> + Get series for a given bug\n> + '''\n> + for series in s.series:\n> + if series in lpbug.tags:\n> + return series\n> + return None\n> +\n> + # main\n> + #\n> + def main(s):\n> + retval = 1\n> + try:\n> + previous_cycle = s.get_bugs(s.project_tracked, s.args.sru_cycle, s.args.series)\n> + next_cycle = s.get_bugs(s.project_tracked, s.args.next_cycle, s.args.series)\n> +\n> + previous_bugs = {}\n> + next_bugs = {}\n> + for bug in previous_cycle:\n> + lpbug = s.launchpad.bugs[bug]\n> + package = lpbug.title.split(\":\")[0]\n> + series = s.get_series(lpbug)\n> + key = package + \":\" + series\n> + if previous_bugs.get(key):\n> + raise AppError(\"duplicate package in previous cycle: %s\" % (key))\n> + previous_bugs[key] = lpbug\n> + for bug in next_cycle:\n> + lpbug = s.launchpad.bugs[bug]\n> + package = lpbug.title.split(\":\")[0]\n> + series = s.get_series(lpbug)\n> + key = package + \":\" + series\n> + if next_bugs.get(key):\n> + raise AppError(\"duplicate package in next cycle: %s\" % (key))\n> + next_bugs[key] = lpbug\n> +\n> + for package in next_bugs:\n> + if not previous_bugs.get(package):\n> + raise AppError(\"could not find package %s in previous cycle\" % (package))\n> + for package in previous_bugs:\n> + if not next_bugs.get(package):\n> + raise AppError(\"could not find package %s in next cycle\" % (package))\n\nWould it make sense to just show a warning message here instead of\naborting the script? We should have a 1x1 relationship between the old\nand the new tracking bugs, but if for some reason that's the not the\ncase we can still use the script to duplicate automatically most of the\nbugs.\n\n> +\n> + for pkg in previous_bugs:\n> + bug = next_bugs[pkg]\n> + prev = previous_bugs[pkg]\n> + prev.duplicate_of = bug\n> + prev.lp_save()\n> + print(\"Marked #{} as duplicate of #{}: {}\".format(prev.id, bug.id, bug.title))\n> +\n> + retval = 0\n> +\n> + except AppError as e:\n> + print(\"ERROR: \" + str(e), file=sys.stderr)\n> +\n> + # Handle the user presses .\n> + #\n> + except KeyboardInterrupt:\n> + print(\"Aborting ...\")\n> +\n> + if retval > 0:\n> + print(\"\")\n> + print(\"Due to the above error(s), this script is unable to continue and is terminating.\")\n> + print(\"\")\n> +\n> + return retval\n> +\n> +if __name__ == '__main__':\n> + app_description = '''\n> + '''\n> +\n> + app_epilog = '''\n> + '''\n> +\n> + parser = ArgumentParser(description=app_description, epilog=app_epilog, formatter_class=RawDescriptionHelpFormatter)\n> + parser.add_argument('--info', action='store_true', default=False, help='')\n> + parser.add_argument('--debug', action='store_true', default=False, help='')\n> + parser.add_argument('--dry-run', action='store_true', default=False, help='')\n> + parser.add_argument('--re-run', action='store_true', default=False, help='')\n\nThe two options above seems to be not used anywhere. '--dry-run' would\nbe easy to implement though, and quite useful :-).\n\n> + parser.add_argument('--sru-cycle', action='store', required=True, help='')\n> + parser.add_argument('--next-cycle', action='store', required=True, help='')\n> + parser.add_argument('--series', action='store', required=True, help=\"\")\n> + args = parser.parse_args()\n> +\n> + # If logging parameters were set on the command line, handle them\n> + # here.\n> + #\n> + Clog.color = True\n> + if args.debug:\n> + log_format = \"%(levelname)s - %(message)s\"\n> + basicConfig(level=DEBUG, format=log_format)\n> + Clog.dbg = True\n> + elif args.info:\n> + log_format = \"%(message)s\"\n> + basicConfig(level=INFO, format=log_format)\n> + else:\n> + log_format = \"%(message)s\"\n> + basicConfig(level=WARNING, format=log_format)\n> +\n> + exit(Tracking(args).main())\n> +\n> +# vi:set ts=4 sw=4 expandtab syntax=python:\n>","headers":{"Return-Path":"","X-Original-To":"incoming@patchwork.ozlabs.org","Delivered-To":"patchwork-incoming@bilbo.ozlabs.org","Authentication-Results":"ozlabs.org;\n\tspf=none (mailfrom) smtp.mailfrom=lists.ubuntu.com\n\t(client-ip=91.189.94.19; helo=huckleberry.canonical.com;\n\tenvelope-from=kernel-team-bounces@lists.ubuntu.com;\n\treceiver=)","Received":["from huckleberry.canonical.com (huckleberry.canonical.com\n\t[91.189.94.19])\n\tby ozlabs.org (Postfix) with ESMTP id 3ySRWC0mDvz9t2f;\n\tFri, 3 Nov 2017 00:58:15 +1100 (AEDT)","from localhost ([127.0.0.1] helo=huckleberry.canonical.com)\n\tby huckleberry.canonical.com with esmtp (Exim 4.86_2)\n\t(envelope-from )\n\tid 1eAG0Y-0005gK-68; Thu, 02 Nov 2017 13:58:10 +0000","from youngberry.canonical.com ([91.189.89.112])\n\tby huckleberry.canonical.com with esmtps\n\t(TLS1.0:DHE_RSA_AES_128_CBC_SHA1:128)\n\t(Exim 4.86_2) (envelope-from )\n\tid 1eAG0W-0005fF-Gf\n\tfor kernel-team@lists.ubuntu.com; Thu, 02 Nov 2017 13:58:08 +0000","from mail-wr0-f198.google.com ([209.85.128.198])\n\tby youngberry.canonical.com with esmtps\n\t(TLS1.0:RSA_AES_128_CBC_SHA1:16)\n\t(Exim 4.76) (envelope-from )\n\tid 1eAG0W-0004bh-9O\n\tfor kernel-team@lists.ubuntu.com; Thu, 02 Nov 2017 13:58:08 +0000","by mail-wr0-f198.google.com with SMTP id k15so3079810wrc.1\n\tfor ;\n\tThu, 02 Nov 2017 06:58:08 -0700 (PDT)","from ?IPv6:2a02:8109:a540:7e8:8926:2094:ede9:dddc?\n\t([2a02:8109:a540:7e8:8926:2094:ede9:dddc])\n\tby smtp.gmail.com with ESMTPSA id\n\t143sm5711177wmj.35.2017.11.02.06.58.05\n\t(version=TLS1_2 cipher=ECDHE-RSA-AES128-GCM-SHA256 bits=128/128);\n\tThu, 02 Nov 2017 06:58:06 -0700 (PDT)"],"X-Google-DKIM-Signature":"v=1; a=rsa-sha256; c=relaxed/relaxed;\n\td=1e100.net; s=20161025;\n\th=x-gm-message-state:subject:to:references:from:message-id:date\n\t:user-agent:mime-version:in-reply-to:content-language\n\t:content-transfer-encoding;\n\tbh=EyINpNHaY1oCcRT8GarpQlGMxM+cyFmdWwALMZYOY0g=;\n\tb=gg/H0vcOS456Kn+/b988xJhJZ5eGNK2poPMp49LaEx5tS/Rf12O+hhP1/ClpJakkDz\n\tBRYkEHEKOtEze6IevkJHz2rn57eFHzQyhElCddhCAT0Yq/ac9yg8lLW9mvxPFBnzhD6G\n\tpmSPGCqhgHaJfvUm4rQjqJusnLAM+tcyBsyaf9KASmBQ6wIr8xRb+u9h4WwIj5Fm0tWo\n\tr+kXlzLcMIXCIitN0Ze+R0Bq9dcU1abmUXy13A8qhp3xiAiQA4CispIdx33SE3c5ZBvQ\n\tQsIqv8NNxdAswpT2TPJC6eghRUK8Y5TAnGwDEuA48B1+8TASp5BIyzn3m3SMmfPFOx5v\n\tJnXw==","X-Gm-Message-State":"AMCzsaXup6igJsnZOQXnAjH1WEcqYH5Ky9fBcBHyRIDpOGllGo1Bnl9k\n\tQh3CeKbCNfb6o5uFwPSTsHssncvhUQz0QhVUV5o3gLFvwKZTg5PRG5kQLrwlPMomJdVAJX1PGPD\n\tLE4bA2/REdz7W9JICvmdefFHvPy6oKgWnDPYZuTfS/A==","X-Received":["by 10.223.130.196 with SMTP id 62mr3334336wrc.60.1509631087378; \n\tThu, 02 Nov 2017 06:58:07 -0700 (PDT)","by 10.223.130.196 with SMTP id 62mr3334317wrc.60.1509631087007; \n\tThu, 02 Nov 2017 06:58:07 -0700 (PDT)"],"X-Google-Smtp-Source":"ABhQp+SuOTNK3M44DrnsLQcDz48JoSg9SxG0HpsVRgT5iiih6FMmyuugjBOyfwYPq1vYobRzaITA/Q==","Subject":"Re: [kteam-tools PATCH] duplicate-bugs: find duplicates given two\n\tcycles","To":"Thadeu Lima de Souza Cascardo ,\n\tkernel-team@lists.ubuntu.com","References":"<20171102113435.26653-1-cascardo@canonical.com>","From":"Kleber Souza ","Message-ID":"","Date":"Thu, 2 Nov 2017 14:58:04 +0100","User-Agent":"Mozilla/5.0 (X11; Linux x86_64; rv:52.0) Gecko/20100101\n\tThunderbird/52.4.0","MIME-Version":"1.0","In-Reply-To":"<20171102113435.26653-1-cascardo@canonical.com>","Content-Language":"en-US","X-BeenThere":"kernel-team@lists.ubuntu.com","X-Mailman-Version":"2.1.20","Precedence":"list","List-Id":"Kernel team discussions ","List-Unsubscribe":",\n\t","List-Archive":"","List-Post":"","List-Help":"","List-Subscribe":",\n\t","Content-Type":"text/plain; charset=\"utf-8\"","Content-Transfer-Encoding":"base64","Errors-To":"kernel-team-bounces@lists.ubuntu.com","Sender":"\"kernel-team\" "}},{"id":1797945,"web_url":"http://patchwork.ozlabs.org/comment/1797945/","msgid":"<20171102143310.7x7xclvqcalvu6es@calabresa>","list_archive_url":null,"date":"2017-11-02T14:33:11","subject":"Re: [kteam-tools PATCH] duplicate-bugs: find duplicates given two\n\tcycles","submitter":{"id":70574,"url":"http://patchwork.ozlabs.org/api/people/70574/","name":"Thadeu Lima de Souza Cascardo","email":"cascardo@canonical.com"},"content":"On Thu, Nov 02, 2017 at 02:58:04PM +0100, Kleber Souza wrote:\n> On 11/02/17 12:34, Thadeu Lima de Souza Cascardo wrote:\n> > Given two cycles and a series, find bugs that match them, and mark the\n> > bugs from the old cycle as duplicate of those from the new cycle.\n> > \n> > Signed-off-by: Thadeu Lima de Souza Cascardo \n> > ---\n> > stable/duplicate-bugs | 206 ++++++++++++++++++++++++++++++++++++++++++++++++++\n> > 1 file changed, 206 insertions(+)\n> > create mode 100755 stable/duplicate-bugs\n> > \n> > diff --git a/stable/duplicate-bugs b/stable/duplicate-bugs\n> > new file mode 100755\n> > index 00000000..fc15e1ef\n> > --- /dev/null\n> > +++ b/stable/duplicate-bugs\n[...]\n> > + for package in next_bugs:\n> > + if not previous_bugs.get(package):\n> > + raise AppError(\"could not find package %s in previous cycle\" % (package))\n> > + for package in previous_bugs:\n> > + if not next_bugs.get(package):\n> > + raise AppError(\"could not find package %s in next cycle\" % (package))\n> \n> Would it make sense to just show a warning message here instead of\n> aborting the script? We should have a 1x1 relationship between the old\n> and the new tracking bugs, but if for some reason that's the not the\n> case we can still use the script to duplicate automatically most of the\n> bugs.\n> \n\nWe could use a force option for that. That, together with a dry-run\noption to show what would happen before running again with the force\noption would be a nice addition.\n\n\n[...]\n> > + parser = ArgumentParser(description=app_description, epilog=app_epilog, formatter_class=RawDescriptionHelpFormatter)\n> > + parser.add_argument('--info', action='store_true', default=False, help='')\n> > + parser.add_argument('--debug', action='store_true', default=False, help='')\n> > + parser.add_argument('--dry-run', action='store_true', default=False, help='')\n> > + parser.add_argument('--re-run', action='store_true', default=False, help='')\n> \n> The two options above seems to be not used anywhere. '--dry-run' would\n> be easy to implement though, and quite useful :-).\n> \n\nYeah, I started with link-to-tracker, so those are leftovers. I will\nresend with the force and dry-run options.\n\nThanks for the feedback.\nCascardo.\n\n> > + parser.add_argument('--sru-cycle', action='store', required=True, help='')\n> > + parser.add_argument('--next-cycle', action='store', required=True, help='')\n> > + parser.add_argument('--series', action='store', required=True, help=\"\")\n> > + args = parser.parse_args()\n[...]","headers":{"Return-Path":"","X-Original-To":"incoming@patchwork.ozlabs.org","Delivered-To":"patchwork-incoming@bilbo.ozlabs.org","Authentication-Results":"ozlabs.org;\n\tspf=none (mailfrom) smtp.mailfrom=lists.ubuntu.com\n\t(client-ip=91.189.94.19; helo=huckleberry.canonical.com;\n\tenvelope-from=kernel-team-bounces@lists.ubuntu.com;\n\treceiver=)","Received":["from huckleberry.canonical.com (huckleberry.canonical.com\n\t[91.189.94.19])\n\tby ozlabs.org (Postfix) with ESMTP id 3ySSHl4vJwz9sNw;\n\tFri, 3 Nov 2017 01:33:23 +1100 (AEDT)","from localhost ([127.0.0.1] helo=huckleberry.canonical.com)\n\tby huckleberry.canonical.com with esmtp (Exim 4.86_2)\n\t(envelope-from )\n\tid 1eAGYX-0005Ir-Fu; Thu, 02 Nov 2017 14:33:17 +0000","from youngberry.canonical.com ([91.189.89.112])\n\tby huckleberry.canonical.com with esmtps\n\t(TLS1.0:DHE_RSA_AES_128_CBC_SHA1:128)\n\t(Exim 4.86_2) (envelope-from )\n\tid 1eAGYW-0005IZ-5h\n\tfor kernel-team@lists.ubuntu.com; Thu, 02 Nov 2017 14:33:16 +0000","from 1.general.cascardo.us.vpn ([10.172.70.58] helo=calabresa)\n\tby youngberry.canonical.com with esmtpsa\n\t(TLS1.0:RSA_AES_256_CBC_SHA1:32)\n\t(Exim 4.76) (envelope-from )\n\tid 1eAGYV-0006OC-En; Thu, 02 Nov 2017 14:33:15 +0000"],"Date":"Thu, 2 Nov 2017 12:33:11 -0200","From":"Thadeu Lima de Souza Cascardo ","To":"Kleber Souza ","Subject":"Re: [kteam-tools PATCH] duplicate-bugs: find duplicates given two\n\tcycles","Message-ID":"<20171102143310.7x7xclvqcalvu6es@calabresa>","References":"<20171102113435.26653-1-cascardo@canonical.com>\n\t","MIME-Version":"1.0","Content-Disposition":"inline","In-Reply-To":"","User-Agent":"NeoMutt/20170113 (1.7.2)","X-BeenThere":"kernel-team@lists.ubuntu.com","X-Mailman-Version":"2.1.20","Precedence":"list","List-Id":"Kernel team discussions ","List-Unsubscribe":",\n\t","List-Archive":"","List-Post":"","List-Help":"","List-Subscribe":",\n\t","Cc":"kernel-team@lists.ubuntu.com","Content-Type":"text/plain; charset=\"utf-8\"","Content-Transfer-Encoding":"base64","Errors-To":"kernel-team-bounces@lists.ubuntu.com","Sender":"\"kernel-team\" "}}]