ceph/src/script/backport-create-issue

   1 #!/usr/bin/env python3
   2 #
   3 # backport-create-issue
   4 #
   5 # Standalone version of the "backport-create-issue" subcommand of
   6 # "ceph-workbench" by Loic Dachary.
   7 #
   8 # This script scans Redmine (tracker.ceph.com) for issues in "Pending Backport"
   9 # status and creates backport issues for them, based on the contents of the
  10 # "Backport" field while trying to avoid creating duplicate backport issues.
  11 #
  12 # Copyright (C) 2015 <contact@redhat.com>
  13 # Copyright (C) 2018, SUSE LLC
  14 #
  15 # Author: Loic Dachary <loic@dachary.org>
  16 # Author: Nathan Cutler <ncutler@suse.com>
  17 #
  18 # This program is free software: you can redistribute it and/or modify
  19 # it under the terms of the GNU Affero General Public License as
  20 # published by the Free Software Foundation, either version 3 of the
  21 # License, or (at your option) any later version.
  22 #
  23 # This program is distributed in the hope that it will be useful,
  24 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  25 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  26 # GNU Affero General Public License for more details.
  27 #
  28 # You should have received a copy of the GNU Affero General Public License
  29 # along with this program.  If not, see http://www.gnu.org/licenses/>
  30 #
  31 import argparse
  32 import logging
  33 import os
  34 import re
  35 import time
  36 from redminelib import Redmine  # https://pypi.org/project/python-redmine/
  37
  38 redmine_endpoint = "https://tracker.ceph.com"
  39 project_name = "Ceph"
  40 release_id = 16
  41 delay_seconds = 5
  42 redmine_key_file="~/.redmine_key"
  43 #
  44 # NOTE: release_id is hard-coded because
  45 # http://www.redmine.org/projects/redmine/wiki/Rest_CustomFields
  46 # requires administrative permissions. If and when
  47 # https://www.redmine.org/issues/18875
  48 # is resolved, it could maybe be replaced by the following code:
  49 #
  50 # for field in redmine.custom_field.all():
  51 #     if field.name == 'Release':
  52 #         release_id = field.id
  53 #
  54 status2status_id = {}
  55 project_id2project = {}
  56 tracker2tracker_id = {}
  57 version2version_id = {}
  58 resolve_parent = None
  59
  60 def usage():
  61     logging.error("Redmine credentials are required to perform this operation. "
  62                   "Please provide either a Redmine key (via %s) "
  63                   "or a Redmine username and password (via --user and --password). "
  64                   "Optionally, one or more issue numbers can be given via positional "
  65                   "argument(s). In the absence of positional arguments, the script "
  66                   "will loop through all issues in Pending Backport status." % redmine_key_file)
  67     exit(-1)
  68
  69 def parse_arguments():
  70     parser = argparse.ArgumentParser()
  71     parser.add_argument("issue_numbers", nargs='*', help="Issue number")
  72     parser.add_argument("--user", help="Redmine user")
  73     parser.add_argument("--password", help="Redmine password")
  74     parser.add_argument("--resolve-parent", help="Resolve parent issue if all backports resolved/rejected",
  75                         action="store_true")
  76     parser.add_argument("--debug", help="Show debug-level messages",
  77                         action="store_true")
  78     parser.add_argument("--dry-run", help="Do not write anything to Redmine",
  79                         action="store_true")
  80     parser.add_argument("--force", help="Create backport issues even if status not Pending Backport",
  81                         action="store_true")
  82     return parser.parse_args()
  83
  84 def set_logging_level(a):
  85     if a.debug:
  86         logging.basicConfig(level=logging.DEBUG)
  87     else:
  88         logging.basicConfig(level=logging.INFO)
  89     return None
  90
  91 def report_dry_run(a):
  92     if a.dry_run:
  93         logging.info("Dry run: nothing will be written to Redmine")
  94     else:
  95         logging.warning("Missing issues will be created in Backport tracker "
  96                          "of the relevant Redmine project")
  97
  98 def process_resolve_parent_option(a):
  99     global resolve_parent
 100     resolve_parent = a.resolve_parent
 101     if a.resolve_parent:
 102         logging.warning("Parent issues with all backports resolved/rejected will be marked Resolved")
 103
 104 def connect_to_redmine(a):
 105     full_path=os.path.expanduser(redmine_key_file)
 106     redmine_key=''
 107     try:
 108         with open(full_path, "r") as f:
 109             redmine_key = f.read().strip()
 110     except FileNotFoundError:
 111         pass
 112
 113     if a.user and a.password:
 114         logging.info("Redmine username and password were provided; using them")
 115         return Redmine(redmine_endpoint, username=a.user, password=a.password)
 116     elif redmine_key:
 117         logging.info("Redmine key was read from '%s'; using it" % redmine_key_file)
 118         return Redmine(redmine_endpoint, key=redmine_key)
 119     else:
 120         usage()
 121
 122 def releases():
 123     return ('argonaut', 'bobtail', 'cuttlefish', 'dumpling', 'emperor',
 124             'firefly', 'giant', 'hammer', 'infernalis', 'jewel', 'kraken',
 125             'luminous', 'mimic', 'nautilus')
 126
 127 def populate_status_dict(r):
 128     for status in r.issue_status.all():
 129         status2status_id[status.name] = status.id
 130     logging.debug("Statuses {}".format(status2status_id))
 131     return None
 132
 133 # not used currently, but might be useful
 134 def populate_version_dict(r, p_id):
 135     versions = r.version.filter(project_id=p_id)
 136     for version in versions:
 137         version2version_id[version.name] = version.id
 138     #logging.debug("Versions {}".format(version2version_id))
 139     return None
 140
 141 def populate_tracker_dict(r):
 142     for tracker in r.tracker.all():
 143         tracker2tracker_id[tracker.name] = tracker.id
 144     logging.debug("Trackers {}".format(tracker2tracker_id))
 145     return None
 146
 147 def has_tracker(r, p_id, tracker_name):
 148     for tracker in get_project(r, p_id).trackers:
 149         if tracker['name'] == tracker_name:
 150             return True
 151     return False
 152
 153 def get_project(r, p_id):
 154     if p_id not in project_id2project:
 155         p_obj = r.project.get(p_id, include='trackers')
 156         project_id2project[p_id] = p_obj
 157     return project_id2project[p_id]
 158
 159 def url(issue):
 160     return redmine_endpoint + "/issues/" + str(issue['id'])
 161
 162 def set_backport(issue):
 163     for field in issue['custom_fields']:
 164         if field['name'] == 'Backport' and field['value'] != 0:
 165             issue['backports'] = set(re.findall('\w+', field['value']))
 166             logging.debug("backports for " + str(issue['id']) +
 167                           " is " + str(field['value']) + " " +
 168                           str(issue['backports']))
 169             return True
 170     return False
 171
 172 def get_release(issue):
 173     for field in issue.custom_fields:
 174         if field['name'] == 'Release':
 175             return field['value']
 176
 177 def update_relations(r, issue, dry_run):
 178     global resolve_parent
 179     relations = r.issue_relation.filter(issue_id=issue['id'])
 180     existing_backports = set()
 181     existing_backports_dict = {}
 182     for relation in relations:
 183         other = r.issue.get(relation['issue_to_id'])
 184         if other['tracker']['name'] != 'Backport':
 185             logging.debug(url(issue) + " ignore relation to " +
 186                 url(other) + " because it is not in the Backport " +
 187                 "tracker")
 188             continue
 189         if relation['relation_type'] != 'copied_to':
 190             logging.error(url(issue) + " unexpected relation '" +
 191                 relation['relation_type'] + "' to " + url(other))
 192             continue
 193         release = get_release(other)
 194         if release in existing_backports:
 195             logging.error(url(issue) + " duplicate " + release +
 196                           " backport issue detected")
 197             continue
 198         existing_backports.add(release)
 199         existing_backports_dict[release] = relation['issue_to_id']
 200         logging.debug(url(issue) + " backport to " + release + " is " +
 201             redmine_endpoint + "/issues/" + str(relation['issue_to_id']))
 202     if existing_backports == issue['backports']:
 203         logging.debug(url(issue) + " has all the required backport issues")
 204         if resolve_parent:
 205             maybe_resolve(issue, existing_backports_dict, dry_run)
 206         return None
 207     if existing_backports.issuperset(issue['backports']):
 208         logging.error(url(issue) + " has more backport issues (" +
 209             ",".join(sorted(existing_backports)) + ") than expected (" +
 210             ",".join(sorted(issue['backports'])) + ")")
 211         return None
 212     backport_tracker_id = tracker2tracker_id['Backport']
 213     for release in issue['backports'] - existing_backports:
 214         if release not in releases():
 215             logging.error(url(issue) + " requires backport to " +
 216                 "unknown release " + release)
 217             break
 218         subject = (release + ": " + issue['subject'])[:255]
 219         if dry_run:
 220             logging.info(url(issue) + " add backport to " + release)
 221             continue
 222         other = r.issue.create(project_id=issue['project']['id'],
 223                                tracker_id=backport_tracker_id,
 224                                subject=subject,
 225                                priority='Normal',
 226                                target_version=None,
 227                                custom_fields=[{
 228                                    "id": release_id,
 229                                    "value": release,
 230                                }])
 231         logging.debug("Rate-limiting to avoid seeming like a spammer")
 232         time.sleep(delay_seconds)
 233         r.issue_relation.create(issue_id=issue['id'],
 234                                 issue_to_id=other['id'],
 235                                 relation_type='copied_to')
 236         logging.info(url(issue) + " added backport to " +
 237                      release + " " + url(other))
 238     return None
 239
 240 def maybe_resolve(issue, backports, dry_run):
 241     '''
 242     issue is a parent issue in Pending Backports status, and backports is a dict
 243     like, e.g., { "luminous": 25345, "mimic": 32134 }.
 244     If all the backport issues are Resolved/Rejected, set the parent issue to Resolved, too.
 245     '''
 246     global delay_seconds
 247     global redmine
 248     global status2status_id
 249     if not backports:
 250         return None
 251     pending_backport_status_id = status2status_id["Pending Backport"]
 252     resolved_status_id = status2status_id["Resolved"]
 253     rejected_status_id = status2status_id["Rejected"]
 254     logging.debug("entering maybe_resolve with parent issue ->{}<- backports ->{}<-"
 255                   .format(issue.id, backports))
 256     assert issue.status.id == pending_backport_status_id, \
 257         "Parent Redmine issue ->{}<- has status ->{}<- (expected Pending Backport)".format(issue.id, issue.status)
 258     all_resolved = True
 259     resolved_equiv_statuses = [resolved_status_id, rejected_status_id]
 260     for backport in backports.keys():
 261         tracker_issue_id = backports[backport]
 262         backport_issue = redmine.issue.get(tracker_issue_id)
 263         logging.debug("{} backport is in status {}".format(backport, backport_issue.status.name))
 264         if backport_issue.status.id not in resolved_equiv_statuses:
 265             all_resolved = False
 266             break
 267     if all_resolved:
 268         logging.debug("Parent ->{}<- all backport issues in status Resolved".format(url(issue)))
 269         note = ("While running with --resolve-parent, the script \"backport-create-issue\" "
 270                 "noticed that all backports of this issue are in status \"Resolved\" or \"Rejected\".")
 271         if dry_run:
 272             logging.info("Set status of parent ->{}<- to Resolved".format(url(issue)))
 273         else:
 274             redmine.issue.update(issue.id, status_id=resolved_status_id, notes=note)
 275             logging.info("Parent ->{}<- status changed from Pending Backport to Resolved".format(url(issue)))
 276             logging.debug("Rate-limiting to avoid seeming like a spammer")
 277             time.sleep(delay_seconds)
 278     else:
 279         logging.debug("Some backport issues are still unresolved: leaving parent issue open")
 280
 281 def iterate_over_backports(r, issues, dry_run=False):
 282     counter = 0
 283     for issue in issues:
 284         counter += 1
 285         logging.debug("{} ({}) {}".format(issue.id, issue.project,
 286             issue.subject))
 287         print('Examining issue#{} ({}/{})\r'.format(issue.id, counter, len(issues)), end='', flush=True)
 288         if not has_tracker(r, issue['project']['id'], 'Backport'):
 289             logging.info("{} skipped because the project {} does not "
 290                 "have a Backport tracker".format(url(issue),
 291                 issue['project']['name']))
 292             continue
 293         if not set_backport(issue):
 294             logging.error(url(issue) + " no backport field")
 295             continue
 296         if len(issue['backports']) == 0:
 297             logging.error(url(issue) + " the backport field is empty")
 298         update_relations(r, issue, dry_run)
 299     print('                                     \r', end='', flush=True)
 300     logging.info("Processed {} issues".format(counter))
 301     return None
 302
 303
 304 if __name__ == '__main__':
 305     args = parse_arguments()
 306     set_logging_level(args)
 307     process_resolve_parent_option(args)
 308     report_dry_run(args)
 309     redmine = connect_to_redmine(args)
 310     project = redmine.project.get(project_name)
 311     ceph_project_id = project.id
 312     logging.debug("Project {} has ID {}".format(project_name, ceph_project_id))
 313     populate_status_dict(redmine)
 314     pending_backport_status_id = status2status_id["Pending Backport"]
 315     logging.debug("Pending Backport status has ID {}"
 316         .format(pending_backport_status_id))
 317     populate_tracker_dict(redmine)
 318     force_create = False
 319     if args.issue_numbers:
 320         issue_list = ','.join(args.issue_numbers)
 321         logging.info("Processing issue list ->{}<-".format(issue_list))
 322         if args.force:
 323             force_create = True
 324             logging.warn("--force option was given: ignoring issue status!")
 325             issues = redmine.issue.filter(project_id=ceph_project_id,
 326                                           issue_id=issue_list)
 327
 328         else:
 329             issues = redmine.issue.filter(project_id=ceph_project_id,
 330                                           issue_id=issue_list,
 331                                           status_id=pending_backport_status_id)
 332     else:
 333         if args.force:
 334             logging.warn("ignoring --force option, which can only be used with an explicit issue list")
 335         issues = redmine.issue.filter(project_id=ceph_project_id,
 336                                       status_id=pending_backport_status_id)
 337     if force_create:
 338         logging.info("Processing {} issues regardless of status"
 339                      .format(len(issues)))
 340     else:
 341         logging.info("Processing {} issues with status Pending Backport"
 342                      .format(len(issues)))
 343     iterate_over_backports(redmine, issues, dry_run=args.dry_run)