ceph/src/script/ceph-release-notes

   1 #!/usr/bin/env python
   2 # Originally modified from A. Israel's script seen at
   3 # https://gist.github.com/aisrael/b2b78d9dfdd176a232b9
   4 """To run this script first install the dependencies
   5
   6
   7   virtualenv v
   8   source v/bin/activate
   9   pip install githubpy GitPython requests
  10
  11 Generate a github access token; this is needed as the anonymous access
  12 to Github's API will easily hit the limit even with a single invocation.
  13 For details see:
  14 https://help.github.com/articles/creating-an-access-token-for-command-line-use/
  15
  16 Next either set the github token as an env variable
  17 `GITHUB_ACCESS_TOKEN` or alternatively invoke the script with
  18 `--token` switch.
  19
  20 Example:
  21
  22   ceph-release-notes -r tags/v0.87..origin/giant \
  23       $(git rev-parse --show-toplevel)
  24
  25 """
  26
  27 from __future__ import print_function
  28 import argparse
  29 import github
  30 import os
  31 import re
  32 import sys
  33 import requests
  34
  35 from git import Repo
  36
  37
  38 fixes_re = re.compile(r"Fixes\:? #(\d+)")
  39 # labels is the list of relevant labels defined for github.com/ceph/ceph
  40 labels = ['bluestore', 'build/ops', 'cephfs', 'common', 'core', 'mgr',
  41           'mon', 'performance', 'pybind', 'rdma', 'rgw', 'rbd', 'tests',
  42           'tools']
  43 merge_re = re.compile("Merge pull request #(\d+).*")
  44 # prefixes is the list of commit description prefixes we recognize
  45 prefixes = ['bluestore', 'build/ops', 'cephfs', 'cephx', 'cli', 'cmake',
  46             'common', 'core', 'crush', 'doc', 'fs', 'librados', 'librbd',
  47             'log', 'mds', 'mgr', 'mon', 'msg', 'objecter', 'osd', 'pybind',
  48             'rbd', 'rbd-mirror', 'rbd-nbd', 'rgw', 'tests', 'tools']
  49 signed_off_re = re.compile("Signed-off-by: (.+) <")
  50 tracker_re = re.compile("http://tracker.ceph.com/issues/(\d+)")
  51 tracker_uri = "http://tracker.ceph.com/issues/{0}.json"
  52
  53
  54 def get_original_issue(issue, verbose):
  55     r = requests.get(tracker_uri.format(issue),
  56                      params={"include": "relations"}).json()
  57
  58     # looking up for the original issue only makes sense
  59     # when dealing with an issue in the Backport tracker
  60     if r["issue"]["tracker"]["name"] != "Backport":
  61         if verbose:
  62             print ("http://tracker.ceph.com/issues/" + issue +
  63                    " is from the tracker " + r["issue"]["tracker"]["name"] +
  64                    ", do not look for the original issue")
  65         return issue
  66
  67     # if a Backport issue does not have a relation, keep it
  68     if "relations" not in r["issue"]:
  69         if verbose:
  70             print ("http://tracker.ceph.com/issues/" + issue +
  71                    " has no relations, do not look for the original issue")
  72         return issue
  73
  74     copied_to = [
  75                     str(i['issue_id']) for i in r["issue"]["relations"]
  76                     if i["relation_type"] == "copied_to"
  77                 ]
  78     if copied_to:
  79         if len(copied_to) > 1:
  80             if verbose:
  81                 print ("ERROR: http://tracker.ceph.com/issues/" + issue +
  82                        " has more than one Copied To relation")
  83             return issue
  84         if verbose:
  85             print ("http://tracker.ceph.com/issues/" + issue +
  86                    " is the backport of http://tracker.ceph.com/issues/" +
  87                    copied_to[0])
  88         return copied_to[0]
  89     else:
  90         if verbose:
  91             print ("http://tracker.ceph.com/issues/" + issue +
  92                    " has no copied_to relations; do not look for the" +
  93                    " original issue")
  94         return issue
  95
  96
  97 def split_component(title, gh, number):
  98     title_re = '(' + '|'.join(prefixes) + ')(:.*)'
  99     match = re.match(title_re, title)
 100     if match:
 101         return match.group(1)+match.group(2)
 102     else:
 103         issue = gh.repos("ceph")("ceph").issues(number).get()
 104         issue_labels = {it['name'] for it in issue['labels']}
 105         if 'documentation' in issue_labels:
 106             return 'doc: ' + title
 107         item = labels.intersection(issue_labels)
 108         if item:
 109             return ",".join(item) + ': ' + title
 110         else:
 111             return 'UNKNOWN: ' + title
 112
 113
 114 def make_release_notes(gh, repo, ref, plaintext, verbose, strict, use_tags):
 115
 116     issue2prs = {}
 117     pr2issues = {}
 118     pr2info = {}
 119
 120     for commit in repo.iter_commits(ref, merges=True):
 121         merge = merge_re.match(commit.summary)
 122         if merge:
 123             number = merge.group(1)
 124             print ("Considering PR#" + number)
 125             # do not pick up ceph/ceph-qa-suite.git PRs
 126             if int(number) < 1311:
 127                 print ("Ignoring low-numbered PR, probably picked up from"
 128                        " ceph/ceph-qa-suite.git")
 129                 continue
 130             pr = gh.repos("ceph")("ceph").pulls(number).get()
 131             title = pr['title']
 132             message = None
 133             message_lines = commit.message.split('\n')
 134             if not strict and len(message_lines) > 1:
 135                 lines = []
 136                 for line in message_lines[1:]:
 137                     if 'Reviewed-by' in line:
 138                         continue
 139                     line = line.strip()
 140                     if line:
 141                         lines.append(line)
 142                 if len(lines) == 0:
 143                     continue
 144                 duplicates_pr_title = lines[0] == pr['title'].strip()
 145                 if duplicates_pr_title:
 146                     continue
 147                 assert len(lines) > 0, "missing message content"
 148                 if len(lines) == 1:
 149                     # assume that a single line means the intention is to
 150                     # re-write the PR title
 151                     title = lines[0]
 152                     message = None
 153                 else:
 154                     message = "    " + "\n    ".join(lines)
 155             issues = []
 156             if pr['body']:
 157                 issues = fixes_re.findall(pr['body']) + tracker_re.findall(
 158                     pr['body']
 159                 )
 160
 161             authors = {}
 162             for c in repo.iter_commits(
 163                          "{sha1}^1..{sha1}^2".format(sha1=commit.hexsha)
 164                      ):
 165                 for author in re.findall(
 166                                   "Signed-off-by:\s*(.*?)\s*<", c.message
 167                               ):
 168                     authors[author] = 1
 169                 issues.extend(fixes_re.findall(c.message) +
 170                               tracker_re.findall(c.message))
 171             if authors:
 172                 author = ", ".join(authors.keys())
 173             else:
 174                 author = commit.parents[-1].author.name
 175
 176             if strict and not issues:
 177                 print ("ERROR: https://github.com/ceph/ceph/pull/" +
 178                        str(number) + " has no associated issue")
 179                 continue
 180
 181             if strict:
 182                 title_re = (
 183                     '^(?:hammer|infernalis|jewel|kraken):\s+(' +
 184                     '|'.join(prefixes) +
 185                     ')(:.*)'
 186                 )
 187                 match = re.match(title_re, title)
 188                 if not match:
 189                     print ("ERROR: https://github.com/ceph/ceph/pull/" +
 190                            str(number) + " title " + title.encode("utf-8") +
 191                            " does not match " + title_re)
 192                 else:
 193                     title = match.group(1) + match.group(2)
 194             if use_tags:
 195                 title = split_component(title, gh, number)
 196
 197             title = title.strip(' \t\n\r\f\v\.\,\;\:\-\=')
 198             # escape asterisks, which is used by reStructuredTextrst for inline
 199             # emphasis
 200             title = title.replace('*', '\*')
 201             pr2info[number] = (author, title, message)
 202
 203             for issue in set(issues):
 204                 if strict:
 205                     issue = get_original_issue(issue, verbose)
 206                 issue2prs.setdefault(issue, set([])).add(number)
 207                 pr2issues.setdefault(number, set([])).add(issue)
 208             sys.stdout.write('.')
 209
 210     print (" done collecting merges.")
 211
 212     if strict:
 213         for (issue, prs) in issue2prs.iteritems():
 214             if len(prs) > 1:
 215                 print (">>>>>>> " + str(len(prs)) + " pr for issue " +
 216                        issue + " " + str(prs))
 217
 218     for (pr, (author, title, message)) in sorted(
 219         pr2info.iteritems(), key=lambda (k, v): (v[2], v[1])
 220     ):
 221         if pr in pr2issues:
 222             if plaintext:
 223                 issues = map(lambda issue: '#' + str(issue), pr2issues[pr])
 224             else:
 225                 issues = map(lambda issue: (
 226                     '`issue#{issue} <http://tracker.ceph.com/issues/{issue}>`_'
 227                     ).format(issue=issue), pr2issues[pr]
 228                 )
 229             issues = ", ".join(issues) + ", "
 230         else:
 231             issues = ''
 232         if plaintext:
 233             print ("* {title} ({issues}{author})".format(
 234                     title=title.encode("utf-8"),
 235                     issues=issues,
 236                     author=author.encode("utf-8")
 237                 )
 238             )
 239         else:
 240             print (
 241                 (
 242                     "* {title} ({issues}`pr#{pr} <"
 243                     "https://github.com/ceph/ceph/pull/{pr}"
 244                     ">`_, {author})"
 245                 ).format(
 246                     title=title.encode("utf-8"),
 247                     issues=issues,
 248                     author=author.encode("utf-8"), pr=pr
 249                 )
 250             )
 251         if message:
 252             print (message)
 253
 254
 255 if __name__ == "__main__":
 256     desc = '''
 257     Make ceph release notes for a given revision. Eg usage:
 258
 259     $ ceph-release-notes -r tags/v0.87..origin/giant \
 260         $(git rev-parse --show-toplevel)
 261
 262     It is recommended to set the github env. token in order to avoid
 263     hitting the api rate limits.
 264     '''
 265
 266     parser = argparse.ArgumentParser(
 267         description=desc,
 268         formatter_class=argparse.RawTextHelpFormatter
 269     )
 270
 271     parser.add_argument("--rev", "-r",
 272                         help="git revision range for creating release notes")
 273     parser.add_argument("--text", "-t",
 274                         action='store_true', default=None,
 275                         help="output plain text only, no links")
 276     parser.add_argument("--verbose", "-v",
 277                         action='store_true', default=None,
 278                         help="verbose")
 279     parser.add_argument("--strict",
 280                         action='store_true', default=None,
 281                         help="strict, recommended only for backport releases")
 282     parser.add_argument("repo", metavar="repo",
 283                         help="path to ceph git repo")
 284     parser.add_argument(
 285         "--token",
 286         default=os.getenv("GITHUB_ACCESS_TOKEN"),
 287         help="Github Access Token ($GITHUB_ACCESS_TOKEN otherwise)",
 288     )
 289     parser.add_argument("--use-tags", default=False,
 290                         help="Use github tags to guess the component")
 291
 292     args = parser.parse_args()
 293     gh = github.GitHub(
 294         access_token=args.token)
 295
 296     make_release_notes(
 297         gh,
 298         Repo(args.repo),
 299         args.rev,
 300         args.text,
 301         args.verbose,
 302         args.strict,
 303         args.use_tags
 304     )