]>
Commit | Line | Data |
---|---|---|
1 | #!/usr/bin/env python | |
2 | # Originally modified from A. Israel's script seen at | |
3 | # https://gist.github.com/aisrael/b2b78d9dfdd176a232b9 | |
4 | """To run this script first install the dependencies | |
5 | ||
6 | ||
7 | virtualenv v | |
8 | source v/bin/activate | |
9 | pip install githubpy GitPython requests | |
10 | ||
11 | Generate a github access token; this is needed as the anonymous access | |
12 | to Github's API will easily hit the limit even with a single invocation. | |
13 | For details see: | |
14 | https://help.github.com/articles/creating-an-access-token-for-command-line-use/ | |
15 | ||
16 | Next either set the github token as an env variable | |
17 | `GITHUB_ACCESS_TOKEN` or alternatively invoke the script with | |
18 | `--token` switch. | |
19 | ||
20 | Example: | |
21 | ||
22 | ceph-release-notes -r tags/v0.87..origin/giant \ | |
23 | $(git rev-parse --show-toplevel) | |
24 | ||
25 | """ | |
26 | ||
27 | from __future__ import print_function | |
28 | import argparse | |
29 | import github | |
30 | import os | |
31 | import re | |
32 | import sys | |
33 | import requests | |
34 | ||
35 | from git import Repo | |
36 | ||
37 | ||
38 | fixes_re = re.compile(r"Fixes\:? #(\d+)") | |
39 | # labels is the list of relevant labels defined for github.com/ceph/ceph | |
40 | labels = ['bluestore', 'build/ops', 'cephfs', 'common', 'core', 'mgr', | |
41 | 'mon', 'performance', 'pybind', 'rdma', 'rgw', 'rbd', 'tests', | |
42 | 'tools'] | |
43 | merge_re = re.compile("Merge pull request #(\d+).*") | |
44 | # prefixes is the list of commit description prefixes we recognize | |
45 | prefixes = ['bluestore', 'build/ops', 'cephfs', 'cephx', 'cli', 'cmake', | |
46 | 'common', 'core', 'crush', 'doc', 'fs', 'librados', 'librbd', | |
47 | 'log', 'mds', 'mgr', 'mon', 'msg', 'objecter', 'osd', 'pybind', | |
48 | 'rbd', 'rbd-mirror', 'rbd-nbd', 'rgw', 'tests', 'tools'] | |
49 | signed_off_re = re.compile("Signed-off-by: (.+) <") | |
50 | tracker_re = re.compile("http://tracker.ceph.com/issues/(\d+)") | |
51 | tracker_uri = "http://tracker.ceph.com/issues/{0}.json" | |
52 | ||
53 | ||
54 | def get_original_issue(issue, verbose): | |
55 | r = requests.get(tracker_uri.format(issue), | |
56 | params={"include": "relations"}).json() | |
57 | ||
58 | # looking up for the original issue only makes sense | |
59 | # when dealing with an issue in the Backport tracker | |
60 | if r["issue"]["tracker"]["name"] != "Backport": | |
61 | if verbose: | |
62 | print ("http://tracker.ceph.com/issues/" + issue + | |
63 | " is from the tracker " + r["issue"]["tracker"]["name"] + | |
64 | ", do not look for the original issue") | |
65 | return issue | |
66 | ||
67 | # if a Backport issue does not have a relation, keep it | |
68 | if "relations" not in r["issue"]: | |
69 | if verbose: | |
70 | print ("http://tracker.ceph.com/issues/" + issue + | |
71 | " has no relations, do not look for the original issue") | |
72 | return issue | |
73 | ||
74 | copied_to = [ | |
75 | str(i['issue_id']) for i in r["issue"]["relations"] | |
76 | if i["relation_type"] == "copied_to" | |
77 | ] | |
78 | if copied_to: | |
79 | if len(copied_to) > 1: | |
80 | if verbose: | |
81 | print ("ERROR: http://tracker.ceph.com/issues/" + issue + | |
82 | " has more than one Copied To relation") | |
83 | return issue | |
84 | if verbose: | |
85 | print ("http://tracker.ceph.com/issues/" + issue + | |
86 | " is the backport of http://tracker.ceph.com/issues/" + | |
87 | copied_to[0]) | |
88 | return copied_to[0] | |
89 | else: | |
90 | if verbose: | |
91 | print ("http://tracker.ceph.com/issues/" + issue + | |
92 | " has no copied_to relations; do not look for the" + | |
93 | " original issue") | |
94 | return issue | |
95 | ||
96 | ||
97 | def split_component(title, gh, number): | |
98 | title_re = '(' + '|'.join(prefixes) + ')(:.*)' | |
99 | match = re.match(title_re, title) | |
100 | if match: | |
101 | return match.group(1)+match.group(2) | |
102 | else: | |
103 | issue = gh.repos("ceph")("ceph").issues(number).get() | |
104 | issue_labels = {it['name'] for it in issue['labels']} | |
105 | if 'documentation' in issue_labels: | |
106 | return 'doc: ' + title | |
107 | item = labels.intersection(issue_labels) | |
108 | if item: | |
109 | return ",".join(item) + ': ' + title | |
110 | else: | |
111 | return 'UNKNOWN: ' + title | |
112 | ||
113 | ||
114 | def make_release_notes(gh, repo, ref, plaintext, verbose, strict, use_tags): | |
115 | ||
116 | issue2prs = {} | |
117 | pr2issues = {} | |
118 | pr2info = {} | |
119 | ||
120 | for commit in repo.iter_commits(ref, merges=True): | |
121 | merge = merge_re.match(commit.summary) | |
122 | if merge: | |
123 | number = merge.group(1) | |
124 | print ("Considering PR#" + number) | |
125 | # do not pick up ceph/ceph-qa-suite.git PRs | |
126 | if int(number) < 1311: | |
127 | print ("Ignoring low-numbered PR, probably picked up from" | |
128 | " ceph/ceph-qa-suite.git") | |
129 | continue | |
130 | pr = gh.repos("ceph")("ceph").pulls(number).get() | |
131 | title = pr['title'] | |
132 | message = None | |
133 | message_lines = commit.message.split('\n') | |
134 | if not strict and len(message_lines) > 1: | |
135 | lines = [] | |
136 | for line in message_lines[1:]: | |
137 | if 'Reviewed-by' in line: | |
138 | continue | |
139 | line = line.strip() | |
140 | if line: | |
141 | lines.append(line) | |
142 | if len(lines) == 0: | |
143 | continue | |
144 | duplicates_pr_title = lines[0] == pr['title'].strip() | |
145 | if duplicates_pr_title: | |
146 | continue | |
147 | assert len(lines) > 0, "missing message content" | |
148 | if len(lines) == 1: | |
149 | # assume that a single line means the intention is to | |
150 | # re-write the PR title | |
151 | title = lines[0] | |
152 | message = None | |
153 | else: | |
154 | message = " " + "\n ".join(lines) | |
155 | issues = [] | |
156 | if pr['body']: | |
157 | issues = fixes_re.findall(pr['body']) + tracker_re.findall( | |
158 | pr['body'] | |
159 | ) | |
160 | ||
161 | authors = {} | |
162 | for c in repo.iter_commits( | |
163 | "{sha1}^1..{sha1}^2".format(sha1=commit.hexsha) | |
164 | ): | |
165 | for author in re.findall( | |
166 | "Signed-off-by:\s*(.*?)\s*<", c.message | |
167 | ): | |
168 | authors[author] = 1 | |
169 | issues.extend(fixes_re.findall(c.message) + | |
170 | tracker_re.findall(c.message)) | |
171 | if authors: | |
172 | author = ", ".join(authors.keys()) | |
173 | else: | |
174 | author = commit.parents[-1].author.name | |
175 | ||
176 | if strict and not issues: | |
177 | print ("ERROR: https://github.com/ceph/ceph/pull/" + | |
178 | str(number) + " has no associated issue") | |
179 | continue | |
180 | ||
181 | if strict: | |
182 | title_re = ( | |
183 | '^(?:hammer|infernalis|jewel|kraken):\s+(' + | |
184 | '|'.join(prefixes) + | |
185 | ')(:.*)' | |
186 | ) | |
187 | match = re.match(title_re, title) | |
188 | if not match: | |
189 | print ("ERROR: https://github.com/ceph/ceph/pull/" + | |
190 | str(number) + " title " + title.encode("utf-8") + | |
191 | " does not match " + title_re) | |
192 | else: | |
193 | title = match.group(1) + match.group(2) | |
194 | if use_tags: | |
195 | title = split_component(title, gh, number) | |
196 | ||
197 | title = title.strip(' \t\n\r\f\v\.\,\;\:\-\=') | |
198 | # escape asterisks, which is used by reStructuredTextrst for inline | |
199 | # emphasis | |
200 | title = title.replace('*', '\*') | |
201 | pr2info[number] = (author, title, message) | |
202 | ||
203 | for issue in set(issues): | |
204 | if strict: | |
205 | issue = get_original_issue(issue, verbose) | |
206 | issue2prs.setdefault(issue, set([])).add(number) | |
207 | pr2issues.setdefault(number, set([])).add(issue) | |
208 | sys.stdout.write('.') | |
209 | ||
210 | print (" done collecting merges.") | |
211 | ||
212 | if strict: | |
213 | for (issue, prs) in issue2prs.iteritems(): | |
214 | if len(prs) > 1: | |
215 | print (">>>>>>> " + str(len(prs)) + " pr for issue " + | |
216 | issue + " " + str(prs)) | |
217 | ||
218 | for (pr, (author, title, message)) in sorted( | |
219 | pr2info.iteritems(), key=lambda (k, v): (v[2], v[1]) | |
220 | ): | |
221 | if pr in pr2issues: | |
222 | if plaintext: | |
223 | issues = map(lambda issue: '#' + str(issue), pr2issues[pr]) | |
224 | else: | |
225 | issues = map(lambda issue: ( | |
226 | '`issue#{issue} <http://tracker.ceph.com/issues/{issue}>`_' | |
227 | ).format(issue=issue), pr2issues[pr] | |
228 | ) | |
229 | issues = ", ".join(issues) + ", " | |
230 | else: | |
231 | issues = '' | |
232 | if plaintext: | |
233 | print ("* {title} ({issues}{author})".format( | |
234 | title=title.encode("utf-8"), | |
235 | issues=issues, | |
236 | author=author.encode("utf-8") | |
237 | ) | |
238 | ) | |
239 | else: | |
240 | print ( | |
241 | ( | |
242 | "* {title} ({issues}`pr#{pr} <" | |
243 | "https://github.com/ceph/ceph/pull/{pr}" | |
244 | ">`_, {author})" | |
245 | ).format( | |
246 | title=title.encode("utf-8"), | |
247 | issues=issues, | |
248 | author=author.encode("utf-8"), pr=pr | |
249 | ) | |
250 | ) | |
251 | if message: | |
252 | print (message) | |
253 | ||
254 | ||
255 | if __name__ == "__main__": | |
256 | desc = ''' | |
257 | Make ceph release notes for a given revision. Eg usage: | |
258 | ||
259 | $ ceph-release-notes -r tags/v0.87..origin/giant \ | |
260 | $(git rev-parse --show-toplevel) | |
261 | ||
262 | It is recommended to set the github env. token in order to avoid | |
263 | hitting the api rate limits. | |
264 | ''' | |
265 | ||
266 | parser = argparse.ArgumentParser( | |
267 | description=desc, | |
268 | formatter_class=argparse.RawTextHelpFormatter | |
269 | ) | |
270 | ||
271 | parser.add_argument("--rev", "-r", | |
272 | help="git revision range for creating release notes") | |
273 | parser.add_argument("--text", "-t", | |
274 | action='store_true', default=None, | |
275 | help="output plain text only, no links") | |
276 | parser.add_argument("--verbose", "-v", | |
277 | action='store_true', default=None, | |
278 | help="verbose") | |
279 | parser.add_argument("--strict", | |
280 | action='store_true', default=None, | |
281 | help="strict, recommended only for backport releases") | |
282 | parser.add_argument("repo", metavar="repo", | |
283 | help="path to ceph git repo") | |
284 | parser.add_argument( | |
285 | "--token", | |
286 | default=os.getenv("GITHUB_ACCESS_TOKEN"), | |
287 | help="Github Access Token ($GITHUB_ACCESS_TOKEN otherwise)", | |
288 | ) | |
289 | parser.add_argument("--use-tags", default=False, | |
290 | help="Use github tags to guess the component") | |
291 | ||
292 | args = parser.parse_args() | |
293 | gh = github.GitHub( | |
294 | access_token=args.token) | |
295 | ||
296 | make_release_notes( | |
297 | gh, | |
298 | Repo(args.repo), | |
299 | args.rev, | |
300 | args.text, | |
301 | args.verbose, | |
302 | args.strict, | |
303 | args.use_tags | |
304 | ) |