ceph/src/script/ptl-tool.py

   1 #!/usr/bin/python3
   2
   3 # README:
   4 #
   5 # This tool's purpose is to make it easier to merge PRs into test branches and
   6 # into master. Make sure you generate a Personal access token in GitHub and
   7 # add it your ~/.github.key.
   8 #
   9 # Because developers often have custom names for the ceph upstream remote
  10 # (https://github.com/ceph/ceph.git), You will probably want to export the
  11 # PTL_TOOL_BASE_PATH environment variable in your shell rc files before using
  12 # this script:
  13 #
  14 #     export PTL_TOOL_BASE_PATH=refs/remotes/<remotename>/
  15 #
  16 # and PTL_TOOL_BASE_REMOTE as the name of your Ceph upstream remote (default: "upstream"):
  17 #
  18 #     export PTL_TOOL_BASE_REMOTE=<remotename>
  19 #
  20 #
  21 # ** Here are some basic exmples to get started: **
  22 #
  23 # Merging all PRs labeled 'wip-pdonnell-testing' into a new test branch:
  24 #
  25 # $ src/script/ptl-tool.py --pr-label wip-pdonnell-testing
  26 # Adding labeled PR #18805 to PR list
  27 # Adding labeled PR #18774 to PR list
  28 # Adding labeled PR #18600 to PR list
  29 # Will merge PRs: [18805, 18774, 18600]
  30 # Detaching HEAD onto base: master
  31 # Merging PR #18805
  32 # Merging PR #18774
  33 # Merging PR #18600
  34 # Checked out new branch wip-pdonnell-testing-20171108.054517
  35 # Created tag testing/wip-pdonnell-testing-20171108.054517
  36 #
  37 #
  38 # Merging all PRs labeled 'wip-pdonnell-testing' into master:
  39 #
  40 # $ src/script/ptl-tool.py --pr-label wip-pdonnell-testing --branch master
  41 # Adding labeled PR #18805 to PR list
  42 # Adding labeled PR #18774 to PR list
  43 # Adding labeled PR #18600 to PR list
  44 # Will merge PRs: [18805, 18774, 18600]
  45 # Detaching HEAD onto base: master
  46 # Merging PR #18805
  47 # Merging PR #18774
  48 # Merging PR #18600
  49 # Checked out branch master
  50 #
  51 # Now push to master:
  52 # $ git push upstream master
  53 # ...
  54 #
  55 #
  56 # Merging PR #1234567 and #2345678 into a new test branch with a testing label added to the PR:
  57 #
  58 # $ src/script/ptl-tool.py 1234567 2345678 --label wip-pdonnell-testing
  59 # Detaching HEAD onto base: master
  60 # Merging PR #1234567
  61 # Labeled PR #1234567 wip-pdonnell-testing
  62 # Merging PR #2345678
  63 # Labeled PR #2345678 wip-pdonnell-testing
  64 # Deleted old test branch wip-pdonnell-testing-20170928
  65 # Created branch wip-pdonnell-testing-20170928
  66 # Created tag testing/wip-pdonnell-testing-20170928_03
  67 #
  68 #
  69 # Merging PR #1234567 into master leaving a detached HEAD (i.e. do not update your repo's master branch) and do not label:
  70 #
  71 # $ src/script/ptl-tool.py --branch HEAD --merge-branch-name master 1234567
  72 # Detaching HEAD onto base: master
  73 # Merging PR #1234567
  74 # Leaving HEAD detached; no branch anchors your commits
  75 #
  76 # Now push to master:
  77 # $ git push upstream HEAD:master
  78 #
  79 #
  80 # Merging PR #12345678 into luminous leaving a detached HEAD (i.e. do not update your repo's master branch) and do not label:
  81 #
  82 # $ src/script/ptl-tool.py --base luminous --branch HEAD --merge-branch-name luminous 12345678
  83 # Detaching HEAD onto base: luminous
  84 # Merging PR #12345678
  85 # Leaving HEAD detached; no branch anchors your commits
  86 #
  87 # Now push to luminous:
  88 # $ git push upstream HEAD:luminous
  89 #
  90 #
  91 # Merging all PRs labelled 'wip-pdonnell-testing' into master leaving a detached HEAD:
  92 #
  93 # $ src/script/ptl-tool.py --base master --branch HEAD --merge-branch-name master --pr-label wip-pdonnell-testing
  94 # Adding labeled PR #18192 to PR list
  95 # Will merge PRs: [18192]
  96 # Detaching HEAD onto base: master
  97 # Merging PR #18192
  98 # Leaving HEAD detached; no branch anchors your commit
  99
 100
 101 # TODO
 102 # Look for check failures?
 103 # redmine issue update: http://www.redmine.org/projects/redmine/wiki/Rest_Issues
 104
 105 import argparse
 106 import codecs
 107 import datetime
 108 import getpass
 109 import git
 110 import itertools
 111 import json
 112 import logging
 113 import os
 114 import re
 115 import requests
 116 import sys
 117
 118 from os.path import expanduser
 119
 120 log = logging.getLogger(__name__)
 121 log.addHandler(logging.StreamHandler())
 122 log.setLevel(logging.INFO)
 123
 124 BASE_PROJECT = os.getenv("PTL_TOOL_BASE_PROJECT", "ceph")
 125 BASE_REPO = os.getenv("PTL_TOOL_BASE_REPO", "ceph")
 126 BASE_REMOTE = os.getenv("PTL_TOOL_BASE_REMOTE", "upstream")
 127 BASE_PATH = os.getenv("PTL_TOOL_BASE_PATH", "refs/remotes/upstream/")
 128 GITDIR = os.getenv("PTL_TOOL_GITDIR", ".")
 129 USER = os.getenv("PTL_TOOL_USER", getpass.getuser())
 130 with open(expanduser("~/.github.key")) as f:
 131     PASSWORD = f.read().strip()
 132 TEST_BRANCH = os.getenv("PTL_TOOL_TEST_BRANCH", "wip-{user}-testing-%Y%m%d.%H%M%S")
 133
 134 SPECIAL_BRANCHES = ('master', 'luminous', 'jewel', 'HEAD')
 135
 136 INDICATIONS = [
 137     re.compile("(Reviewed-by: .+ <[\w@.-]+>)", re.IGNORECASE),
 138     re.compile("(Acked-by: .+ <[\w@.-]+>)", re.IGNORECASE),
 139     re.compile("(Tested-by: .+ <[\w@.-]+>)", re.IGNORECASE),
 140 ]
 141
 142 # find containing git dir
 143 git_dir = GITDIR
 144 max_levels = 6
 145 while not os.path.exists(git_dir + '/.git'):
 146     git_dir += '/..'
 147     max_levels -= 1
 148     if max_levels < 0:
 149         break
 150
 151 CONTRIBUTORS = {}
 152 NEW_CONTRIBUTORS = {}
 153 with codecs.open(git_dir + "/.githubmap", encoding='utf-8') as f:
 154     comment = re.compile("\s*#")
 155     patt = re.compile("([\w-]+)\s+(.*)")
 156     for line in f:
 157         if comment.match(line):
 158             continue
 159         m = patt.match(line)
 160         CONTRIBUTORS[m.group(1)] = m.group(2)
 161
 162 BZ_MATCH = re.compile("(.*https?://bugzilla.redhat.com/.*)")
 163 TRACKER_MATCH = re.compile("(.*https?://tracker.ceph.com/.*)")
 164
 165 def get(session, url, params=None, paging=True):
 166     if params is None:
 167         params = {}
 168     params['per_page'] = 100
 169
 170     log.debug(f"Fetching {url}")
 171     response = session.get(url, auth=(USER, PASSWORD), params=params)
 172     log.debug(f"Response = {response}; links = {response.headers.get('link', '')}")
 173     if response.status_code != 200:
 174         log.error(f"Failed to fetch {url}: {response}")
 175         sys.exit(1)
 176     j = response.json()
 177     yield j
 178     if paging:
 179         link = response.headers.get('link', None)
 180         page = 2
 181         while link is not None and 'next' in link:
 182             log.debug(f"Fetching {url}")
 183             new_params = dict(params)
 184             new_params.update({'page': page})
 185             response = session.get(url, auth=(USER, PASSWORD), params=new_params)
 186             log.debug(f"Response = {response}; links = {response.headers.get('link', '')}")
 187             if response.status_code != 200:
 188                 log.error(f"Failed to fetch {url}: {response}")
 189                 sys.exit(1)
 190             yield response.json()
 191             link = response.headers.get('link', None)
 192             page += 1
 193
 194 def get_credits(session, pr, pr_req):
 195     comments = [pr_req]
 196
 197     log.debug(f"Getting comments for #{pr}")
 198     endpoint = f"https://api.github.com/repos/{BASE_PROJECT}/{BASE_REPO}/issues/{pr}/comments"
 199     for c in get(session, endpoint):
 200         comments.extend(c)
 201
 202     log.debug(f"Getting reviews for #{pr}")
 203     endpoint = f"https://api.github.com/repos/{BASE_PROJECT}/{BASE_REPO}/pulls/{pr}/reviews"
 204     reviews = []
 205     for c in get(session, endpoint):
 206         comments.extend(c)
 207         reviews.extend(c)
 208
 209     log.debug(f"Getting review comments for #{pr}")
 210     endpoint = f"https://api.github.com/repos/{BASE_PROJECT}/{BASE_REPO}/pulls/{pr}/comments"
 211     for c in get(session, endpoint):
 212         comments.extend(c)
 213
 214     credits = set()
 215     for comment in comments:
 216         body = comment["body"]
 217         if body:
 218             url = comment["html_url"]
 219             for m in BZ_MATCH.finditer(body):
 220                 log.info("[ {url} ] BZ cited: {cite}".format(url=url, cite=m.group(1)))
 221             for m in TRACKER_MATCH.finditer(body):
 222                 log.info("[ {url} ] Ceph tracker cited: {cite}".format(url=url, cite=m.group(1)))
 223             for indication in INDICATIONS:
 224                 for cap in indication.findall(comment["body"]):
 225                     credits.add(cap)
 226
 227     new_new_contributors = {}
 228     for review in reviews:
 229         if review["state"] == "APPROVED":
 230             user = review["user"]["login"]
 231             try:
 232                 credits.add("Reviewed-by: "+CONTRIBUTORS[user])
 233             except KeyError as e:
 234                 try:
 235                     credits.add("Reviewed-by: "+NEW_CONTRIBUTORS[user])
 236                 except KeyError as e:
 237                     try:
 238                         name = input("Need name for contributor \"%s\" (use ^D to skip); Reviewed-by: " % user)
 239                         name = name.strip()
 240                         if len(name) == 0:
 241                             continue
 242                         NEW_CONTRIBUTORS[user] = name
 243                         new_new_contributors[user] = name
 244                         credits.add("Reviewed-by: "+name)
 245                     except EOFError as e:
 246                         continue
 247
 248     return "\n".join(credits), new_new_contributors
 249
 250 def build_branch(args):
 251     base = args.base
 252     branch = datetime.datetime.utcnow().strftime(args.branch).format(user=USER)
 253     label = args.label
 254     merge_branch_name = args.merge_branch_name
 255     if merge_branch_name is False:
 256         merge_branch_name = branch
 257
 258     session = requests.Session()
 259
 260     if label:
 261         # Check the label format
 262         if re.search(r'\bwip-(.*?)-testing\b', label) is None:
 263             log.error("Unknown Label '{lblname}'. Label Format: wip-<name>-testing".format(lblname=label))
 264             sys.exit(1)
 265
 266         # Check if the Label exist in the repo
 267         endpoint = f"https://api.github.com/repos/{BASE_PROJECT}/{BASE_REPO}/labels/{label}"
 268         get(session, endpoint, paging=False)
 269
 270     G = git.Repo(args.git)
 271
 272     # First get the latest base branch and PRs from BASE_REMOTE
 273     remote = getattr(G.remotes, BASE_REMOTE)
 274     remote.fetch()
 275
 276     prs = args.prs
 277     if args.pr_label is not None:
 278         if args.pr_label == '' or args.pr_label.isspace():
 279             log.error("--pr-label must have a non-space value")
 280             sys.exit(1)
 281         payload = {'labels': args.pr_label, 'sort': 'created', 'direction': 'desc'}
 282         endpoint = f"https://api.github.com/repos/{BASE_PROJECT}/{BASE_REPO}/issues"
 283         labeled_prs = []
 284         for l in get(session, endpoint, params=payload):
 285             labeled_prs.extend(l)
 286         if len(labeled_prs) == 0:
 287             log.error("Search for PRs matching label '{}' returned no results!".format(args.pr_label))
 288             sys.exit(1)
 289         for pr in labeled_prs:
 290             if pr['pull_request']:
 291                 n = pr['number']
 292                 log.info("Adding labeled PR #{} to PR list".format(n))
 293                 prs.append(n)
 294     log.info("Will merge PRs: {}".format(prs))
 295
 296     if base == 'HEAD':
 297         log.info("Branch base is HEAD; not checking out!")
 298     else:
 299         log.info("Detaching HEAD onto base: {}".format(base))
 300         try:
 301             base_path = args.base_path + base
 302             base = next(ref for ref in G.refs if ref.path == base_path)
 303         except StopIteration:
 304             log.error("Branch " + base + " does not exist!")
 305             sys.exit(1)
 306
 307         # So we know that we're not on an old test branch, detach HEAD onto ref:
 308         base.checkout()
 309
 310     for pr in prs:
 311         pr = int(pr)
 312         log.info("Merging PR #{pr}".format(pr=pr))
 313
 314         remote_ref = "refs/pull/{pr}/head".format(pr=pr)
 315         fi = remote.fetch(remote_ref)
 316         if len(fi) != 1:
 317             log.error("PR {pr} does not exist?".format(pr=pr))
 318             sys.exit(1)
 319         tip = fi[0].ref.commit
 320
 321         endpoint = f"https://api.github.com/repos/{BASE_PROJECT}/{BASE_REPO}/pulls/{pr}"
 322         response = next(get(session, endpoint, paging=False))
 323
 324         message = "Merge PR #%d into %s\n\n* %s:\n" % (pr, merge_branch_name, remote_ref)
 325
 326         for commit in G.iter_commits(rev="HEAD.."+str(tip)):
 327             message = message + ("\t%s\n" % commit.message.split('\n', 1)[0])
 328             # Get tracker issues / bzs cited so the PTL can do updates
 329             short = commit.hexsha[:8]
 330             for m in BZ_MATCH.finditer(commit.message):
 331                 log.info("[ {sha1} ] BZ cited: {cite}".format(sha1=short, cite=m.group(1)))
 332             for m in TRACKER_MATCH.finditer(commit.message):
 333                 log.info("[ {sha1} ] Ceph tracker cited: {cite}".format(sha1=short, cite=m.group(1)))
 334
 335         message = message + "\n"
 336         if args.credits:
 337             (addendum, new_contributors) = get_credits(session, pr, response)
 338             message += addendum
 339         else:
 340             new_contributors = []
 341
 342         G.git.merge(tip.hexsha, '--no-ff', m=message)
 343
 344         if new_contributors:
 345             # Check out the PR, add a commit adding to .githubmap
 346             log.info("adding new contributors to githubmap in merge commit")
 347             with open(git_dir + "/.githubmap", "a") as f:
 348                 for c in new_contributors:
 349                     f.write("%s %s\n" % (c, new_contributors[c]))
 350             G.index.add([".githubmap"])
 351             G.git.commit("--amend", "--no-edit")
 352
 353         if label:
 354             req = session.post("https://api.github.com/repos/{project}/{repo}/issues/{pr}/labels".format(pr=pr, project=BASE_PROJECT, repo=BASE_REPO), data=json.dumps([label]), auth=(USER, PASSWORD))
 355             if req.status_code != 200:
 356                 log.error("PR #%d could not be labeled %s: %s" % (pr, label, req))
 357                 sys.exit(1)
 358             log.info("Labeled PR #{pr} {label}".format(pr=pr, label=label))
 359
 360     # If the branch is 'HEAD', leave HEAD detached (but use "master" for commit message)
 361     if branch == 'HEAD':
 362         log.info("Leaving HEAD detached; no branch anchors your commits")
 363     else:
 364         created_branch = False
 365         try:
 366             G.head.reference = G.create_head(branch)
 367             log.info("Checked out new branch {branch}".format(branch=branch))
 368             created_branch = True
 369         except:
 370             G.head.reference = G.create_head(branch, force=True)
 371             log.info("Checked out branch {branch}".format(branch=branch))
 372
 373         if created_branch:
 374             # tag it for future reference.
 375             tag = "testing/%s" % branch
 376             git.refs.tag.Tag.create(G, tag)
 377             log.info("Created tag %s" % tag)
 378
 379 def main():
 380     parser = argparse.ArgumentParser(description="Ceph PTL tool")
 381     default_base = 'master'
 382     default_branch = TEST_BRANCH
 383     default_label = ''
 384     if len(sys.argv) > 1 and sys.argv[1] in SPECIAL_BRANCHES:
 385         argv = sys.argv[2:]
 386         default_branch = 'HEAD' # Leave HEAD detached
 387         default_base = default_branch
 388         default_label = False
 389     else:
 390         argv = sys.argv[1:]
 391     parser.add_argument('--branch', dest='branch', action='store', default=default_branch, help='branch to create ("HEAD" leaves HEAD detached; i.e. no branch is made)')
 392     parser.add_argument('--merge-branch-name', dest='merge_branch_name', action='store', default=False, help='name of the branch for merge messages')
 393     parser.add_argument('--base', dest='base', action='store', default=default_base, help='base for branch')
 394     parser.add_argument('--base-path', dest='base_path', action='store', default=BASE_PATH, help='base for branch')
 395     parser.add_argument('--git-dir', dest='git', action='store', default=git_dir, help='git directory')
 396     parser.add_argument('--label', dest='label', action='store', default=default_label, help='label PRs for testing')
 397     parser.add_argument('--pr-label', dest='pr_label', action='store', help='label PRs for testing')
 398     parser.add_argument('--no-credits', dest='credits', action='store_false', help='skip indication search (Reviewed-by, etc.)')
 399     parser.add_argument('prs', metavar="PR", type=int, nargs='*', help='Pull Requests to merge')
 400     args = parser.parse_args(argv)
 401     return build_branch(args)
 402
 403 if __name__ == "__main__":
 404     main()