ceph/src/script/ptl-tool.py

   1 #!/usr/bin/env python2
   2
   3 # README:
   4 #
   5 # This tool's purpose is to make it easier to merge PRs into test branches and
   6 # into master. Make sure you generate a Personal access token in GitHub and
   7 # add it your ~/.github.key.
   8 #
   9 # Because developers often have custom names for the ceph upstream remote
  10 # (https://github.com/ceph/ceph.git), You will probably want to export the
  11 # PTL_TOOL_BASE_PATH environment variable in your shell rc files before using
  12 # this script:
  13 #
  14 #     export PTL_TOOL_BASE_PATH=refs/remotes/<remotename>/
  15 #
  16 # and PTL_TOOL_BASE_REMOTE as the name of your Ceph upstream remote (default: "upstream"):
  17 #
  18 #     export PTL_TOOL_BASE_REMOTE=<remotename>
  19 #
  20 #
  21 # ** Here are some basic exmples to get started: **
  22 #
  23 # Merging all PRs labeled 'wip-pdonnell-testing' into a new test branch:
  24 #
  25 # $ src/script/ptl-tool.py --pr-label wip-pdonnell-testing
  26 # Adding labeled PR #18805 to PR list
  27 # Adding labeled PR #18774 to PR list
  28 # Adding labeled PR #18600 to PR list
  29 # Will merge PRs: [18805, 18774, 18600]
  30 # Detaching HEAD onto base: master
  31 # Merging PR #18805
  32 # Merging PR #18774
  33 # Merging PR #18600
  34 # Checked out new branch wip-pdonnell-testing-20171108.054517
  35 # Created tag testing/wip-pdonnell-testing-20171108.054517
  36 #
  37 #
  38 # Merging all PRs labeled 'wip-pdonnell-testing' into master:
  39 #
  40 # $ src/script/ptl-tool.py --pr-label wip-pdonnell-testing --branch master
  41 # Adding labeled PR #18805 to PR list
  42 # Adding labeled PR #18774 to PR list
  43 # Adding labeled PR #18600 to PR list
  44 # Will merge PRs: [18805, 18774, 18600]
  45 # Detaching HEAD onto base: master
  46 # Merging PR #18805
  47 # Merging PR #18774
  48 # Merging PR #18600
  49 # Checked out branch master
  50 #
  51 # Now push to master:
  52 # $ git push upstream master
  53 # ...
  54 #
  55 #
  56 # Merging PR #1234567 and #2345678 into a new test branch with a testing label added to the PR:
  57 #
  58 # $ src/script/ptl-tool.py 1234567 2345678 --label wip-pdonnell-testing
  59 # Detaching HEAD onto base: master
  60 # Merging PR #1234567
  61 # Labeled PR #1234567 wip-pdonnell-testing
  62 # Merging PR #2345678
  63 # Labeled PR #2345678 wip-pdonnell-testing
  64 # Deleted old test branch wip-pdonnell-testing-20170928
  65 # Created branch wip-pdonnell-testing-20170928
  66 # Created tag testing/wip-pdonnell-testing-20170928_03
  67 #
  68 #
  69 # Merging PR #1234567 into master leaving a detached HEAD (i.e. do not update your repo's master branch) and do not label:
  70 #
  71 # $ src/script/ptl-tool.py --branch HEAD --merge-branch-name master 1234567
  72 # Detaching HEAD onto base: master
  73 # Merging PR #1234567
  74 # Leaving HEAD detached; no branch anchors your commits
  75 #
  76 # Now push to master:
  77 # $ git push upstream HEAD:master
  78 #
  79 #
  80 # Merging PR #12345678 into luminous leaving a detached HEAD (i.e. do not update your repo's master branch) and do not label:
  81 #
  82 # $ src/script/ptl-tool.py --base luminous --branch HEAD --merge-branch-name luminous 12345678
  83 # Detaching HEAD onto base: luminous
  84 # Merging PR #12345678
  85 # Leaving HEAD detached; no branch anchors your commits
  86 #
  87 # Now push to luminous:
  88 # $ git push upstream HEAD:luminous
  89 #
  90 #
  91 # Merging all PRs labelled 'wip-pdonnell-testing' into master leaving a detached HEAD:
  92 #
  93 # $ src/script/ptl-tool.py --base master --branch HEAD --merge-branch-name master --pr-label wip-pdonnell-testing
  94 # Adding labeled PR #18192 to PR list
  95 # Will merge PRs: [18192]
  96 # Detaching HEAD onto base: master
  97 # Merging PR #18192
  98 # Leaving HEAD detached; no branch anchors your commit
  99
 100
 101 # TODO
 102 # Look for check failures?
 103 # redmine issue update: http://www.redmine.org/projects/redmine/wiki/Rest_Issues
 104
 105 import argparse
 106 import codecs
 107 import datetime
 108 import getpass
 109 import git
 110 import itertools
 111 import json
 112 import logging
 113 import os
 114 import re
 115 import requests
 116 import sys
 117
 118 from os.path import expanduser
 119
 120 log = logging.getLogger(__name__)
 121 log.addHandler(logging.StreamHandler())
 122 log.setLevel(logging.INFO)
 123
 124 BASE_PROJECT = os.getenv("PTL_TOOL_BASE_PROJECT", "ceph")
 125 BASE_REPO = os.getenv("PTL_TOOL_BASE_REPO", "ceph")
 126 BASE_REMOTE = os.getenv("PTL_TOOL_BASE_REMOTE", "upstream")
 127 BASE_PATH = os.getenv("PTL_TOOL_BASE_PATH", "refs/remotes/upstream/")
 128 GITDIR = os.getenv("PTL_TOOL_GITDIR", ".")
 129 USER = os.getenv("PTL_TOOL_USER", getpass.getuser())
 130 with open(expanduser("~/.github.key")) as f:
 131     PASSWORD = f.read().strip()
 132 TEST_BRANCH = os.getenv("PTL_TOOL_TEST_BRANCH", "wip-{user}-testing-%Y%m%d.%H%M%S")
 133
 134 SPECIAL_BRANCHES = ('master', 'luminous', 'jewel', 'HEAD')
 135
 136 INDICATIONS = [
 137     re.compile("(Reviewed-by: .+ <[\w@.-]+>)", re.IGNORECASE),
 138     re.compile("(Acked-by: .+ <[\w@.-]+>)", re.IGNORECASE),
 139     re.compile("(Tested-by: .+ <[\w@.-]+>)", re.IGNORECASE),
 140 ]
 141
 142 # find containing git dir
 143 git_dir = GITDIR
 144 max_levels = 6
 145 while not os.path.exists(git_dir + '/.git'):
 146     git_dir += '/..'
 147     max_levels -= 1
 148     if max_levels < 0:
 149         break
 150
 151 CONTRIBUTORS = {}
 152 NEW_CONTRIBUTORS = {}
 153 with codecs.open(git_dir + "/.githubmap", encoding='utf-8') as f:
 154     comment = re.compile("\s*#")
 155     patt = re.compile("([\w-]+)\s+(.*)")
 156     for line in f:
 157         if comment.match(line):
 158             continue
 159         m = patt.match(line)
 160         CONTRIBUTORS[m.group(1)] = m.group(2)
 161
 162 BZ_MATCH = re.compile("(.*https?://bugzilla.redhat.com/.*)")
 163 TRACKER_MATCH = re.compile("(.*https?://tracker.ceph.com/.*)")
 164
 165 def build_branch(args):
 166     base = args.base
 167     branch = datetime.datetime.utcnow().strftime(args.branch).format(user=USER)
 168     label = args.label
 169     merge_branch_name = args.merge_branch_name
 170     if merge_branch_name is False:
 171         merge_branch_name = branch
 172
 173     if label:
 174         #Check the label format
 175         if re.search(r'\bwip-(.*?)-testing\b', label) is None:
 176             log.error("Unknown Label '{lblname}'. Label Format: wip-<name>-testing".format(lblname=label))
 177             sys.exit(1)
 178
 179         #Check if the Label exist in the repo
 180         res = requests.get("https://api.github.com/repos/{project}/{repo}/labels/{lblname}".format(lblname=label, project=BASE_PROJECT, repo=BASE_REPO), auth=(USER, PASSWORD))
 181         if res.status_code != 200:
 182             log.error("Label '{lblname}' not found in the repo".format(lblname=label))
 183             sys.exit(1)
 184
 185     G = git.Repo(args.git)
 186
 187     # First get the latest base branch and PRs from BASE_REMOTE
 188     remote = getattr(G.remotes, BASE_REMOTE)
 189     remote.fetch()
 190
 191     prs = args.prs
 192     if args.pr_label is not None:
 193         if args.pr_label == '' or args.pr_label.isspace():
 194             log.error("--pr-label must have a non-space value")
 195             sys.exit(1)
 196         payload = {'labels': args.pr_label, 'sort': 'created', 'direction': 'desc'}
 197         labeled_prs = requests.get("https://api.github.com/repos/{project}/{repo}/issues".format(project=BASE_PROJECT, repo=BASE_REPO), auth=(USER, PASSWORD), params=payload)
 198         if labeled_prs.status_code != 200:
 199             log.error("Failed to load labeled PRs: {}".format(labeled_prs))
 200             sys.exit(1)
 201         labeled_prs = labeled_prs.json()
 202         if len(labeled_prs) == 0:
 203             log.error("Search for PRs matching label '{}' returned no results!".format(args.pr_label))
 204             sys.exit(1)
 205         for pr in labeled_prs:
 206             if pr['pull_request']:
 207                 n = pr['number']
 208                 log.info("Adding labeled PR #{} to PR list".format(n))
 209                 prs.append(n)
 210     log.info("Will merge PRs: {}".format(prs))
 211
 212     if base == 'HEAD':
 213         log.info("Branch base is HEAD; not checking out!")
 214     else:
 215         log.info("Detaching HEAD onto base: {}".format(base))
 216         try:
 217             base_path = args.base_path + base
 218             base = filter(lambda r: r.path == base_path, G.refs)[0]
 219         except IndexError:
 220             log.error("Branch " + base + " does not exist!")
 221             sys.exit(1)
 222
 223         # So we know that we're not on an old test branch, detach HEAD onto ref:
 224         base.checkout()
 225
 226     for pr in prs:
 227         pr = int(pr)
 228         log.info("Merging PR #{pr}".format(pr=pr))
 229
 230         remote_ref = "refs/pull/{pr}/head".format(pr=pr)
 231         fi = remote.fetch(remote_ref)
 232         if len(fi) != 1:
 233             log.error("PR {pr} does not exist?".format(pr=pr))
 234             sys.exit(1)
 235         tip = fi[0].ref.commit
 236
 237         pr_req = requests.get("https://api.github.com/repos/ceph/ceph/pulls/{pr}".format(pr=pr), auth=(USER, PASSWORD))
 238         if pr_req.status_code != 200:
 239             log.error("PR '{pr}' not found: {c}".format(pr=pr,c=pr_req))
 240             sys.exit(1)
 241
 242         message = "Merge PR #%d into %s\n\n* %s:\n" % (pr, merge_branch_name, remote_ref)
 243
 244         for commit in G.iter_commits(rev="HEAD.."+str(tip)):
 245             message = message + ("\t%s\n" % commit.message.split('\n', 1)[0])
 246             # Get tracker issues / bzs cited so the PTL can do updates
 247             short = commit.hexsha[:8]
 248             for m in BZ_MATCH.finditer(commit.message):
 249                 log.info("[ {sha1} ] BZ cited: {cite}".format(sha1=short, cite=m.group(1)))
 250             for m in TRACKER_MATCH.finditer(commit.message):
 251                 log.info("[ {sha1} ] Ceph tracker cited: {cite}".format(sha1=short, cite=m.group(1)))
 252
 253         message = message + "\n"
 254
 255         comments = requests.get("https://api.github.com/repos/{project}/{repo}/issues/{pr}/comments".format(pr=pr, project=BASE_PROJECT, repo=BASE_REPO), auth=(USER, PASSWORD))
 256         if comments.status_code != 200:
 257             log.error("PR '{pr}' not found: {c}".format(pr=pr,c=comments))
 258             sys.exit(1)
 259
 260         reviews = requests.get("https://api.github.com/repos/{project}/{repo}/pulls/{pr}/reviews".format(pr=pr, project=BASE_PROJECT, repo=BASE_REPO), auth=(USER, PASSWORD))
 261         if reviews.status_code != 200:
 262             log.error("PR '{pr}' not found: {c}".format(pr=pr,c=comments))
 263             sys.exit(1)
 264
 265         review_comments = requests.get("https://api.github.com/repos/{project}/{repo}/pulls/{pr}/comments".format(pr=pr, project=BASE_PROJECT, repo=BASE_REPO), auth=(USER, PASSWORD))
 266         if review_comments.status_code != 200:
 267             log.error("PR '{pr}' not found: {c}".format(pr=pr,c=comments))
 268             sys.exit(1)
 269
 270         indications = set()
 271         for comment in [pr_req.json()]+comments.json()+reviews.json()+review_comments.json():
 272             body = comment["body"]
 273             if body:
 274                 url = comment["html_url"]
 275                 for m in BZ_MATCH.finditer(body):
 276                     log.info("[ {url} ] BZ cited: {cite}".format(url=url, cite=m.group(1)))
 277                 for m in TRACKER_MATCH.finditer(body):
 278                     log.info("[ {url} ] Ceph tracker cited: {cite}".format(url=url, cite=m.group(1)))
 279                 for indication in INDICATIONS:
 280                     for cap in indication.findall(comment["body"]):
 281                         indications.add(cap)
 282
 283         new_new_contributors = {}
 284         for review in reviews.json():
 285             if review["state"] == "APPROVED":
 286                 user = review["user"]["login"]
 287                 try:
 288                     indications.add("Reviewed-by: "+CONTRIBUTORS[user])
 289                 except KeyError as e:
 290                     try:
 291                         indications.add("Reviewed-by: "+NEW_CONTRIBUTORS[user])
 292                     except KeyError as e:
 293                         try:
 294                             name = raw_input("Need name for contributor \"%s\" (use ^D to skip); Reviewed-by: " % user)
 295                             name = name.strip()
 296                             if len(name) == 0:
 297                                 continue
 298                             NEW_CONTRIBUTORS[user] = name
 299                             new_new_contributors[user] = name
 300                             indications.add("Reviewed-by: "+name)
 301                         except EOFError as e:
 302                             continue
 303
 304         for indication in indications:
 305             message = message + indication + "\n"
 306
 307         G.git.merge(tip.hexsha, '--no-ff', m=message)
 308
 309         if new_new_contributors:
 310             # Check out the PR, add a commit adding to .githubmap
 311             log.info("adding new contributors to githubmap in merge commit")
 312             with open(git_dir + "/.githubmap", "a") as f:
 313                 for c in new_new_contributors:
 314                     f.write("%s %s\n" % (c, new_new_contributors[c]))
 315             G.index.add([".githubmap"])
 316             G.git.commit("--amend", "--no-edit")
 317
 318         if label:
 319             req = requests.post("https://api.github.com/repos/{project}/{repo}/issues/{pr}/labels".format(pr=pr, project=BASE_PROJECT, repo=BASE_REPO), data=json.dumps([label]), auth=(USER, PASSWORD))
 320             if req.status_code != 200:
 321                 log.error("PR #%d could not be labeled %s: %s" % (pr, label, req))
 322                 sys.exit(1)
 323             log.info("Labeled PR #{pr} {label}".format(pr=pr, label=label))
 324
 325     # If the branch is 'HEAD', leave HEAD detached (but use "master" for commit message)
 326     if branch == 'HEAD':
 327         log.info("Leaving HEAD detached; no branch anchors your commits")
 328     else:
 329         created_branch = False
 330         try:
 331             G.head.reference = G.create_head(branch)
 332             log.info("Checked out new branch {branch}".format(branch=branch))
 333             created_branch = True
 334         except:
 335             G.head.reference = G.create_head(branch, force=True)
 336             log.info("Checked out branch {branch}".format(branch=branch))
 337
 338         if created_branch:
 339             # tag it for future reference.
 340             tag = "testing/%s" % branch
 341             git.refs.tag.Tag.create(G, tag)
 342             log.info("Created tag %s" % tag)
 343
 344 def main():
 345     parser = argparse.ArgumentParser(description="Ceph PTL tool")
 346     default_base = 'master'
 347     default_branch = TEST_BRANCH
 348     default_label = ''
 349     if len(sys.argv) > 1 and sys.argv[1] in SPECIAL_BRANCHES:
 350         argv = sys.argv[2:]
 351         default_branch = 'HEAD' # Leave HEAD detached
 352         default_base = default_branch
 353         default_label = False
 354     else:
 355         argv = sys.argv[1:]
 356     parser.add_argument('--branch', dest='branch', action='store', default=default_branch, help='branch to create ("HEAD" leaves HEAD detached; i.e. no branch is made)')
 357     parser.add_argument('--merge-branch-name', dest='merge_branch_name', action='store', default=False, help='name of the branch for merge messages')
 358     parser.add_argument('--base', dest='base', action='store', default=default_base, help='base for branch')
 359     parser.add_argument('--base-path', dest='base_path', action='store', default=BASE_PATH, help='base for branch')
 360     parser.add_argument('--git-dir', dest='git', action='store', default=git_dir, help='git directory')
 361     parser.add_argument('--label', dest='label', action='store', default=default_label, help='label PRs for testing')
 362     parser.add_argument('--pr-label', dest='pr_label', action='store', help='label PRs for testing')
 363     parser.add_argument('prs', metavar="PR", type=int, nargs='*', help='Pull Requests to merge')
 364     args = parser.parse_args(argv)
 365     return build_branch(args)
 366
 367 if __name__ == "__main__":
 368     main()