ceph/src/script/ceph-release-notes

   1 #!/usr/bin/env python
   2 # Originally modified from A. Israel's script seen at
   3 # https://gist.github.com/aisrael/b2b78d9dfdd176a232b9
   4 """To run this script first install the dependencies
   5
   6
   7   python3 -m venv v
   8   source v/bin/activate
   9   pip install githubpy GitPython requests
  10
  11 Generate a github access token; this is needed as the anonymous access
  12 to Github's API will easily hit the limit even with a single invocation.
  13 For details see:
  14 https://help.github.com/articles/creating-an-access-token-for-command-line-use/
  15
  16 Next either set the github token as an env variable
  17 `GITHUB_ACCESS_TOKEN` or alternatively invoke the script with
  18 `--token` switch.
  19
  20 Example:
  21
  22   ceph-release-notes -r tags/v0.87..origin/giant \
  23       $(git rev-parse --show-toplevel)
  24
  25 """
  26
  27 from __future__ import print_function
  28 import argparse
  29 import github
  30 import os
  31 import re
  32 import sys
  33 import requests
  34 import time
  35
  36 from git import Repo
  37
  38
  39 fixes_re = re.compile(r"Fixes\:? #(\d+)")
  40 reviewed_by_re = re.compile(r"Rev(.*)By", re.IGNORECASE)
  41 # labels is the list of relevant labels defined for github.com/ceph/ceph
  42 labels = {'bluestore', 'build/ops', 'cephfs', 'common', 'core', 'mgr',
  43           'mon', 'performance', 'pybind', 'rdma', 'rgw', 'rbd', 'tests',
  44           'tools'}
  45 merge_re = re.compile("Merge (pull request|PR) #(\d+).*")
  46 # prefixes is the list of commit description prefixes we recognize
  47 prefixes = ['bluestore', 'build/ops', 'cephfs', 'cephx', 'cli', 'cmake',
  48             'common', 'core', 'crush', 'doc', 'fs', 'librados', 'librbd',
  49             'log', 'mds', 'mgr', 'mon', 'msg', 'objecter', 'osd', 'pybind',
  50             'rbd', 'rbd-mirror', 'rbd-nbd', 'rgw', 'tests', 'tools']
  51 signed_off_re = re.compile("Signed-off-by: (.+) <")
  52 tracker_re = re.compile("http://tracker.ceph.com/issues/(\d+)")
  53 rst_link_re = re.compile(r"([a-zA-Z0-9])_(\W)")
  54 release_re = re.compile(r"^(nautilus|octopus|pacific|quincy):\s*")
  55
  56 tracker_uri = "http://tracker.ceph.com/issues/{0}.json"
  57
  58
  59 def get_original_issue(issue, verbose):
  60     r = requests.get(tracker_uri.format(issue),
  61                      params={"include": "relations"}).json()
  62
  63     # looking up for the original issue only makes sense
  64     # when dealing with an issue in the Backport tracker
  65     if r["issue"]["tracker"]["name"] != "Backport":
  66         if verbose:
  67             print ("http://tracker.ceph.com/issues/" + issue +
  68                    " is from the tracker " + r["issue"]["tracker"]["name"] +
  69                    ", do not look for the original issue")
  70         return issue
  71
  72     # if a Backport issue does not have a relation, keep it
  73     if "relations" not in r["issue"]:
  74         if verbose:
  75             print ("http://tracker.ceph.com/issues/" + issue +
  76                    " has no relations, do not look for the original issue")
  77         return issue
  78
  79     copied_to = [
  80                     str(i['issue_id']) for i in r["issue"]["relations"]
  81                     if i["relation_type"] == "copied_to"
  82                 ]
  83     if copied_to:
  84         if len(copied_to) > 1:
  85             if verbose:
  86                 print ("ERROR: http://tracker.ceph.com/issues/" + issue +
  87                        " has more than one Copied To relation")
  88             return issue
  89         if verbose:
  90             print ("http://tracker.ceph.com/issues/" + issue +
  91                    " is the backport of http://tracker.ceph.com/issues/" +
  92                    copied_to[0])
  93         return copied_to[0]
  94     else:
  95         if verbose:
  96             print ("http://tracker.ceph.com/issues/" + issue +
  97                    " has no copied_to relations; do not look for the" +
  98                    " original issue")
  99         return issue
 100
 101
 102 def split_component(title, gh, number):
 103     title_re = '(' + '|'.join(prefixes) + ')(:.*)'
 104     match = re.match(title_re, title)
 105     if match:
 106         return match.group(1)+match.group(2)
 107     else:
 108         issue = gh.repos("ceph")("ceph").issues(number).get()
 109         issue_labels = {it['name'] for it in issue['labels']}
 110         if 'documentation' in issue_labels:
 111             return 'doc: ' + title
 112         item = set(prefixes).intersection(issue_labels)
 113         if item:
 114             return ",".join(sorted(item)) + ': ' + title
 115         else:
 116             return 'UNKNOWN: ' + title
 117
 118 def _title_message(commit, pr, strict):
 119     title = pr['title']
 120     message_lines = commit.message.split('\n')
 121     if strict or len(message_lines) < 1:
 122         return (title, None)
 123     lines = []
 124     for line in message_lines[1:]:
 125         if reviewed_by_re.match(line):
 126             continue
 127         line = line.strip()
 128         if line:
 129             lines.append(line)
 130     if len(lines) == 0:
 131         return (title, None)
 132     duplicates_pr_title = lines[0] == pr['title'].strip()
 133     if duplicates_pr_title:
 134         return (title, None)
 135     assert len(lines) > 0, "missing message content"
 136     if len(lines) == 1:
 137         # assume that a single line means the intention is to
 138         # re-write the PR title
 139         return (lines[0], None)
 140     elif len(lines) < 3 and 'refs/pull' in lines[0]:
 141         # assume the intent was rewriting the title and something like
 142         # ptl-tool was used to generate the merge message
 143         return (lines[1], None)
 144     message = "    " + "\n    ".join(lines)
 145     return (title, message)
 146
 147 def make_release_notes(gh, repo, ref, plaintext, html, markdown, verbose, strict, use_tags, include_pr_messages):
 148
 149     issue2prs = {}
 150     pr2issues = {}
 151     pr2info = {}
 152
 153     for commit in repo.iter_commits(ref, merges=True):
 154         merge = merge_re.match(commit.summary)
 155         if not merge:
 156             continue
 157         number = merge.group(2)
 158         print ("Considering PR#" + number)
 159         # do not pick up ceph/ceph-qa-suite.git PRs
 160         if int(number) < 1311:
 161             print ("Ignoring low-numbered PR, probably picked up from"
 162                    " ceph/ceph-qa-suite.git")
 163             continue
 164
 165         attempts = 0
 166         retries = 30
 167         while attempts < retries:
 168             try:
 169                 pr = gh.repos("ceph")("ceph").pulls(number).get()
 170                 break
 171             except Exception:
 172                 if attempts < retries:
 173                     attempts += 1
 174                     sleep_time = 2 * attempts
 175                     print(f"Failed to fetch PR {number}, sleeping for {sleep_time} seconds")
 176                     time.sleep(sleep_time)
 177                 else:
 178                     print(f"Could not fetch PR {number} in {retries} tries.")
 179                     raise
 180         (title, message) = _title_message(commit, pr, strict)
 181         issues = []
 182         if pr['body']:
 183             issues = fixes_re.findall(pr['body']) + tracker_re.findall(
 184                 pr['body']
 185             )
 186
 187         authors = {}
 188         for c in repo.iter_commits(
 189                      "{sha1}^1..{sha1}^2".format(sha1=commit.hexsha)
 190                  ):
 191             for author in re.findall(
 192                               "Signed-off-by:\s*(.*?)\s*<", c.message
 193                           ):
 194                 authors[author] = 1
 195             issues.extend(fixes_re.findall(c.message) +
 196                           tracker_re.findall(c.message))
 197         if authors:
 198             author = ", ".join(authors.keys())
 199         else:
 200             author = commit.parents[-1].author.name
 201
 202         if strict and not issues:
 203             print ("ERROR: https://github.com/ceph/ceph/pull/" +
 204                    str(number) + " has no associated issue")
 205             continue
 206
 207         if strict:
 208             title_re = (
 209                 '^(?:nautilus|octopus|pacific|quincy):\s+(' +
 210                 '|'.join(prefixes) +
 211                 ')(:.*)'
 212             )
 213             match = re.match(title_re, title)
 214             if not match:
 215                 print ("ERROR: https://github.com/ceph/ceph/pull/" +
 216                        str(number) + " title " + title +
 217                        " does not match " + title_re)
 218             else:
 219                 title = match.group(1) + match.group(2)
 220         if use_tags:
 221             title = split_component(title, gh, number)
 222
 223         title = title.strip(' \t\n\r\f\v\.\,\;\:\-\=')
 224         # escape asterisks, which is used by reStructuredTextrst for inline
 225         # emphasis
 226         title = title.replace('*', '\*')
 227         # and escape the underscores for noting a link
 228         title = rst_link_re.sub(r'\1\_\2', title)
 229         # remove release prefix for backports
 230         title = release_re.sub('', title)
 231         pr2info[number] = (author, title, message)
 232
 233         for issue in set(issues):
 234             if strict:
 235                 issue = get_original_issue(issue, verbose)
 236             issue2prs.setdefault(issue, set([])).add(number)
 237             pr2issues.setdefault(number, set([])).add(issue)
 238         sys.stdout.write('.')
 239
 240     print (" done collecting merges.")
 241
 242     if strict:
 243         for (issue, prs) in issue2prs.items():
 244             if len(prs) > 1:
 245                 print (">>>>>>> " + str(len(prs)) + " pr for issue " +
 246                        issue + " " + str(prs))
 247
 248     for (pr, (author, title, message)) in sorted(
 249         pr2info.items(), key=lambda title: title[1][1].lower()
 250     ):
 251         if pr in pr2issues:
 252             if plaintext:
 253                 issues = map(lambda issue: '#' + str(issue), pr2issues[pr])
 254             elif html:
 255                 issues = map(lambda issue: (
 256                     '<a href="http://tracker.ceph.com/issues/{issue}">issue#{issue}</a>'
 257                     ).format(issue=issue), pr2issues[pr]
 258                 )
 259             elif markdown:
 260                 issues = map(lambda issue: (
 261                     '[issue#{issue}](http://tracker.ceph.com/issues/{issue})'
 262                     ).format(issue=issue), pr2issues[pr]
 263                 )
 264             else:
 265                 issues = map(lambda issue: (
 266                     '`issue#{issue} <http://tracker.ceph.com/issues/{issue}>`_'
 267                     ).format(issue=issue), pr2issues[pr]
 268                 )
 269             issues = ", ".join(issues) + ", "
 270         else:
 271             issues = ''
 272         if plaintext:
 273             print ("* {title} ({issues}{author})".format(
 274                     title=title,
 275                     issues=issues,
 276                     author=author
 277                 )
 278             )
 279         elif html:
 280             print (
 281                 (
 282                     "<li><p>{title} ({issues}<a href=\""
 283                     "https://github.com/ceph/ceph/pull/{pr}\""
 284                     ">pr#{pr}</a>, {author})</p></li>"
 285                 ).format(
 286                     title=title,
 287                     issues=issues,
 288                     author=author, pr=pr
 289                 )
 290             )
 291         elif markdown:
 292             markdown_title = title.replace('_', '\_').replace('.', '<span></span>.')
 293             print ("- {title} ({issues}[pr#{pr}](https://github.com/ceph/ceph/pull/{pr}), {author})\n".format(
 294                     title=markdown_title,
 295                     issues=issues,
 296                     author=author, pr=pr
 297                 )
 298             )
 299         else:
 300             print (
 301                 (
 302                     "* {title} ({issues}`pr#{pr} <"
 303                     "https://github.com/ceph/ceph/pull/{pr}"
 304                     ">`_, {author})"
 305                 ).format(
 306                     title=title,
 307                     issues=issues,
 308                     author=author, pr=pr
 309                 )
 310             )
 311         if include_pr_messages and message:
 312             print (message)
 313
 314
 315 if __name__ == "__main__":
 316     desc = '''
 317     Make ceph release notes for a given revision. Eg usage:
 318
 319     $ ceph-release-notes -r tags/v0.87..origin/giant \
 320         $(git rev-parse --show-toplevel)
 321
 322     It is recommended to set the github env. token in order to avoid
 323     hitting the api rate limits.
 324     '''
 325
 326     parser = argparse.ArgumentParser(
 327         description=desc,
 328         formatter_class=argparse.RawTextHelpFormatter
 329     )
 330
 331     parser.add_argument("--rev", "-r",
 332                         help="git revision range for creating release notes")
 333     parser.add_argument("--text", "-t",
 334                         action='store_true', default=None,
 335                         help="output plain text only, no links")
 336     parser.add_argument("--html",
 337                         action='store_true', default=None,
 338                         help="output html format for (old wordpress) website blog")
 339     parser.add_argument("--markdown",
 340                         action='store_true', default=None,
 341                         help="output markdown format for new ceph.io blog")
 342     parser.add_argument("--verbose", "-v",
 343                         action='store_true', default=None,
 344                         help="verbose")
 345     parser.add_argument("--strict",
 346                         action='store_true', default=None,
 347                         help="strict, recommended only for backport releases")
 348     parser.add_argument("repo", metavar="repo",
 349                         help="path to ceph git repo")
 350     parser.add_argument(
 351         "--token",
 352         default=os.getenv("GITHUB_ACCESS_TOKEN"),
 353         help="Github Access Token ($GITHUB_ACCESS_TOKEN otherwise)",
 354     )
 355     parser.add_argument("--use-tags", default=False,
 356                         help="Use github tags to guess the component")
 357     parser.add_argument("--include-pr-messages", default=False, action='store_true',
 358                         help="Include full PR message in addition to PR title, if available")
 359
 360     args = parser.parse_args()
 361     gh = github.GitHub(
 362         access_token=args.token)
 363
 364     make_release_notes(
 365         gh,
 366         Repo(args.repo),
 367         args.rev,
 368         args.text,
 369         args.html,
 370         args.markdown,
 371         args.verbose,
 372         args.strict,
 373         args.use_tags,
 374         args.include_pr_messages
 375     )