ceph/src/script/backport-create-issue

   1 #!/usr/bin/env python3
   2 #
   3 # backport-create-issue
   4 #
   5 # Standalone version of the "backport-create-issue" subcommand of
   6 # "ceph-workbench" by Loic Dachary.
   7 #
   8 # This script scans Redmine (tracker.ceph.com) for issues in "Pending Backport"
   9 # status and creates backport issues for them, based on the contents of the
  10 # "Backport" field while trying to avoid creating duplicate backport issues.
  11 #
  12 # Copyright (C) 2015 <contact@redhat.com>
  13 # Copyright (C) 2018, SUSE LLC
  14 #
  15 # Author: Loic Dachary <loic@dachary.org>
  16 # Author: Nathan Cutler <ncutler@suse.com>
  17 #
  18 # This program is free software: you can redistribute it and/or modify
  19 # it under the terms of the GNU Affero General Public License as
  20 # published by the Free Software Foundation, either version 3 of the
  21 # License, or (at your option) any later version.
  22 #
  23 # This program is distributed in the hope that it will be useful,
  24 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  25 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  26 # GNU Affero General Public License for more details.
  27 #
  28 # You should have received a copy of the GNU Affero General Public License
  29 # along with this program.  If not, see http://www.gnu.org/licenses/>
  30 #
  31 import argparse
  32 import logging
  33 import re
  34 import time
  35 from redminelib import Redmine  # https://pypi.org/project/python-redmine/
  36
  37 redmine_endpoint = "http://tracker.ceph.com"
  38 project_name = "Ceph"
  39 release_id = 16
  40 delay_seconds = 5
  41 #
  42 # NOTE: release_id is hard-coded because
  43 # http://www.redmine.org/projects/redmine/wiki/Rest_CustomFields
  44 # requires administrative permissions. If and when
  45 # https://www.redmine.org/issues/18875
  46 # is resolved, it could maybe be replaced by the following code:
  47 #
  48 # for field in redmine.custom_field.all():
  49 #     if field.name == 'Release':
  50 #         release_id = field.id
  51 #
  52 status2status_id = {}
  53 project_id2project = {}
  54 tracker2tracker_id = {}
  55 version2version_id = {}
  56
  57 def usage():
  58     logging.error("Command-line arguments must include either a Redmine key (--key) "
  59                   "or a Redmine username and password (via --user and --password). "
  60                   "Optionally, one or more issue numbers can be given via positional "
  61                   "argument(s). In the absence of positional arguments, the script "
  62                   "will loop through all issues in Pending Backport status.")
  63     exit(-1)
  64
  65 def parse_arguments():
  66     parser = argparse.ArgumentParser()
  67     parser.add_argument("issue_numbers", nargs='*', help="Issue number")
  68     parser.add_argument("--key", help="Redmine user key")
  69     parser.add_argument("--user", help="Redmine user")
  70     parser.add_argument("--password", help="Redmine password")
  71     parser.add_argument("--debug", help="Show debug-level messages",
  72                         action="store_true")
  73     parser.add_argument("--dry-run", help="Do not write anything to Redmine",
  74                         action="store_true")
  75     return parser.parse_args()
  76
  77 def set_logging_level(a):
  78     if a.debug:
  79         logging.basicConfig(level=logging.DEBUG)
  80     else:
  81         logging.basicConfig(level=logging.INFO)
  82     return None
  83
  84 def report_dry_run(a):
  85     if a.dry_run:
  86          logging.info("Dry run: nothing will be written to Redmine")
  87     else:
  88          logging.warning("Missing issues will be created in Backport tracker "
  89                          "of the relevant Redmine project")
  90
  91 def connect_to_redmine(a):
  92     if a.key:
  93         logging.info("Redmine key was provided; using it")
  94         return Redmine(redmine_endpoint, key=a.key)
  95     elif a.user and a.password:
  96         logging.info("Redmine username and password were provided; using them")
  97         return Redmine(redmine_endpoint, username=a.user, password=a.password)
  98     else:
  99         usage()
 100
 101 def releases():
 102     return ('argonaut', 'bobtail', 'cuttlefish', 'dumpling', 'emperor',
 103             'firefly', 'giant', 'hammer', 'infernalis', 'jewel', 'kraken',
 104             'luminous', 'mimic')
 105
 106 def populate_status_dict(r):
 107     for status in r.issue_status.all():
 108         status2status_id[status.name] = status.id
 109     logging.debug("Statuses {}".format(status2status_id))
 110     return None
 111
 112 # not used currently, but might be useful
 113 def populate_version_dict(r, p_id):
 114     versions = r.version.filter(project_id=p_id)
 115     for version in versions:
 116         version2version_id[version.name] = version.id
 117     #logging.debug("Versions {}".format(version2version_id))
 118     return None
 119
 120 def populate_tracker_dict(r):
 121     for tracker in r.tracker.all():
 122         tracker2tracker_id[tracker.name] = tracker.id
 123     logging.debug("Trackers {}".format(tracker2tracker_id))
 124     return None
 125
 126 def has_tracker(r, p_id, tracker_name):
 127     for tracker in get_project(r, p_id).trackers:
 128         if tracker['name'] == tracker_name:
 129             return True
 130     return False
 131
 132 def get_project(r, p_id):
 133     if p_id not in project_id2project:
 134         p_obj = r.project.get(p_id, include='trackers')
 135         project_id2project[p_id] = p_obj
 136     return project_id2project[p_id]
 137
 138 def url(issue):
 139     return redmine_endpoint + "/issues/" + str(issue['id'])
 140
 141 def set_backport(issue):
 142     for field in issue['custom_fields']:
 143         if field['name'] == 'Backport' and field['value'] != 0:
 144             issue['backports'] = set(re.findall('\w+', field['value']))
 145             logging.debug("backports for " + str(issue['id']) +
 146                           " is " + str(field['value']) + " " +
 147                           str(issue['backports']))
 148             return True
 149     return False
 150
 151 def get_release(issue):
 152     for field in issue.custom_fields:
 153         if field['name'] == 'Release':
 154             return field['value']
 155
 156 def update_relations(r, issue, dry_run):
 157     relations = r.issue_relation.filter(issue_id=issue['id'])
 158     existing_backports = set()
 159     for relation in relations:
 160         other = r.issue.get(relation['issue_to_id'])
 161         if other['tracker']['name'] != 'Backport':
 162             logging.debug(url(issue) + " ignore relation to " +
 163                 url(other) + " because it is not in the Backport " +
 164                 "tracker")
 165             continue
 166         if relation['relation_type'] != 'copied_to':
 167             logging.error(url(issue) + " unexpected relation '" +
 168                 relation['relation_type'] + "' to " + url(other))
 169             continue
 170         release = get_release(other)
 171         if release in existing_backports:
 172             logging.error(url(issue) + " duplicate " + release +
 173                           " backport issue detected")
 174             continue
 175         existing_backports.add(release)
 176         logging.debug(url(issue) + " backport to " + release + " is " +
 177             redmine_endpoint + "/issues/" + str(relation['issue_to_id']))
 178     if existing_backports == issue['backports']:
 179         logging.debug(url(issue) + " has all the required backport issues")
 180         return None
 181     if existing_backports.issuperset(issue['backports']):
 182         logging.error(url(issue) + " has more backport issues (" +
 183             ",".join(sorted(existing_backports)) + ") than expected (" +
 184             ",".join(sorted(issue['backports'])) + ")")
 185         return None
 186     backport_tracker_id = tracker2tracker_id['Backport']
 187     for release in issue['backports'] - existing_backports:
 188         if release not in releases():
 189             logging.error(url(issue) + " requires backport to " +
 190                 "unknown release " + release)
 191             break
 192         subject = release + ": " + issue['subject']
 193         if dry_run:
 194             logging.info(url(issue) + " add backport to " + release)
 195             continue
 196         other = r.issue.create(project_id=issue['project']['id'],
 197                                tracker_id=backport_tracker_id,
 198                                subject=subject,
 199                                priority='Normal',
 200                                target_version=None,
 201                                custom_fields=[{
 202                                    "id": release_id,
 203                                    "value": release,
 204                                }])
 205         logging.debug("Rate-limiting to avoid seeming like a spammer")
 206         time.sleep(delay_seconds)
 207         r.issue_relation.create(issue_id=issue['id'],
 208                                 issue_to_id=other['id'],
 209                                 relation_type='copied_to')
 210         logging.info(url(issue) + " added backport to " +
 211                      release + " " + url(other))
 212     return None
 213
 214 def iterate_over_backports(r, issues, dry_run):
 215     counter = 0
 216     for issue in issues:
 217         counter += 1
 218         logging.debug("{} ({}) {}".format(issue.id, issue.project,
 219             issue.subject))
 220         print('{}\r'.format(issue.id), end='', flush=True)
 221         if not has_tracker(r, issue['project']['id'], 'Backport'):
 222             logging.info("{} skipped because the project {} does not "
 223                 "have a Backport tracker".format(url(issue),
 224                 issue['project']['name']))
 225             continue
 226         if not set_backport(issue):
 227             logging.error(url(issue) + " no backport field")
 228             continue
 229         if len(issue['backports']) == 0:
 230             logging.error(url(issue) + " the backport field is empty")
 231         update_relations(r, issue, dry_run)
 232     logging.info("Processed {} issues with status Pending Backport"
 233                  .format(counter))
 234     return None
 235
 236
 237 if __name__ == '__main__':
 238     args = parse_arguments()
 239     set_logging_level(args)
 240     report_dry_run(args)
 241     redmine = connect_to_redmine(args)
 242     project = redmine.project.get(project_name)
 243     ceph_project_id = project.id
 244     logging.debug("Project {} has ID {}".format(project_name, ceph_project_id))
 245     populate_status_dict(redmine)
 246     pending_backport_status_id = status2status_id["Pending Backport"]
 247     logging.debug("Pending Backport status has ID {}"
 248         .format(pending_backport_status_id))
 249     populate_tracker_dict(redmine)
 250     if args.issue_numbers:
 251         issue_list = ','.join(args.issue_numbers)
 252         logging.info("Processing issue list ->{}<-".format(issue_list))
 253         issues = redmine.issue.filter(project_id=ceph_project_id,
 254                                       issue_id=issue_list,
 255                                       status_id=pending_backport_status_id)
 256     else:
 257         issues = redmine.issue.filter(project_id=ceph_project_id,
 258                                       status_id=pending_backport_status_id)
 259     iterate_over_backports(redmine, issues, args.dry_run)