]>
Commit | Line | Data |
---|---|---|
11fdf7f2 TL |
1 | #!/usr/bin/env python3 |
2 | # | |
3 | # backport-create-issue | |
4 | # | |
5 | # Standalone version of the "backport-create-issue" subcommand of | |
6 | # "ceph-workbench" by Loic Dachary. | |
7 | # | |
8 | # This script scans Redmine (tracker.ceph.com) for issues in "Pending Backport" | |
9 | # status and creates backport issues for them, based on the contents of the | |
10 | # "Backport" field while trying to avoid creating duplicate backport issues. | |
11 | # | |
12 | # Copyright (C) 2015 <contact@redhat.com> | |
13 | # Copyright (C) 2018, SUSE LLC | |
14 | # | |
15 | # Author: Loic Dachary <loic@dachary.org> | |
16 | # Author: Nathan Cutler <ncutler@suse.com> | |
9f95a23c | 17 | # |
11fdf7f2 TL |
18 | # This program is free software: you can redistribute it and/or modify |
19 | # it under the terms of the GNU Affero General Public License as | |
20 | # published by the Free Software Foundation, either version 3 of the | |
21 | # License, or (at your option) any later version. | |
22 | # | |
23 | # This program is distributed in the hope that it will be useful, | |
24 | # but WITHOUT ANY WARRANTY; without even the implied warranty of | |
25 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
26 | # GNU Affero General Public License for more details. | |
27 | # | |
28 | # You should have received a copy of the GNU Affero General Public License | |
29 | # along with this program. If not, see http://www.gnu.org/licenses/> | |
30 | # | |
31 | import argparse | |
32 | import logging | |
9f95a23c | 33 | import os |
11fdf7f2 TL |
34 | import re |
35 | import time | |
36 | from redminelib import Redmine # https://pypi.org/project/python-redmine/ | |
37 | ||
9f95a23c | 38 | redmine_endpoint = "https://tracker.ceph.com" |
11fdf7f2 TL |
39 | project_name = "Ceph" |
40 | release_id = 16 | |
41 | delay_seconds = 5 | |
9f95a23c | 42 | redmine_key_file="~/.redmine_key" |
11fdf7f2 TL |
43 | # |
44 | # NOTE: release_id is hard-coded because | |
45 | # http://www.redmine.org/projects/redmine/wiki/Rest_CustomFields | |
46 | # requires administrative permissions. If and when | |
47 | # https://www.redmine.org/issues/18875 | |
48 | # is resolved, it could maybe be replaced by the following code: | |
49 | # | |
50 | # for field in redmine.custom_field.all(): | |
51 | # if field.name == 'Release': | |
52 | # release_id = field.id | |
53 | # | |
54 | status2status_id = {} | |
55 | project_id2project = {} | |
56 | tracker2tracker_id = {} | |
57 | version2version_id = {} | |
9f95a23c | 58 | resolve_parent = None |
11fdf7f2 TL |
59 | |
60 | def usage(): | |
9f95a23c TL |
61 | logging.error("Redmine credentials are required to perform this operation. " |
62 | "Please provide either a Redmine key (via %s) " | |
11fdf7f2 TL |
63 | "or a Redmine username and password (via --user and --password). " |
64 | "Optionally, one or more issue numbers can be given via positional " | |
65 | "argument(s). In the absence of positional arguments, the script " | |
9f95a23c | 66 | "will loop through all issues in Pending Backport status." % redmine_key_file) |
11fdf7f2 TL |
67 | exit(-1) |
68 | ||
69 | def parse_arguments(): | |
70 | parser = argparse.ArgumentParser() | |
71 | parser.add_argument("issue_numbers", nargs='*', help="Issue number") | |
11fdf7f2 TL |
72 | parser.add_argument("--user", help="Redmine user") |
73 | parser.add_argument("--password", help="Redmine password") | |
9f95a23c TL |
74 | parser.add_argument("--resolve-parent", help="Resolve parent issue if all backports resolved/rejected", |
75 | action="store_true") | |
11fdf7f2 TL |
76 | parser.add_argument("--debug", help="Show debug-level messages", |
77 | action="store_true") | |
78 | parser.add_argument("--dry-run", help="Do not write anything to Redmine", | |
79 | action="store_true") | |
9f95a23c TL |
80 | parser.add_argument("--force", help="Create backport issues even if status not Pending Backport", |
81 | action="store_true") | |
11fdf7f2 TL |
82 | return parser.parse_args() |
83 | ||
84 | def set_logging_level(a): | |
85 | if a.debug: | |
86 | logging.basicConfig(level=logging.DEBUG) | |
87 | else: | |
88 | logging.basicConfig(level=logging.INFO) | |
89 | return None | |
90 | ||
91 | def report_dry_run(a): | |
92 | if a.dry_run: | |
9f95a23c | 93 | logging.info("Dry run: nothing will be written to Redmine") |
11fdf7f2 | 94 | else: |
9f95a23c | 95 | logging.warning("Missing issues will be created in Backport tracker " |
11fdf7f2 TL |
96 | "of the relevant Redmine project") |
97 | ||
9f95a23c TL |
98 | def process_resolve_parent_option(a): |
99 | global resolve_parent | |
100 | resolve_parent = a.resolve_parent | |
101 | if a.resolve_parent: | |
102 | logging.warning("Parent issues with all backports resolved/rejected will be marked Resolved") | |
103 | ||
11fdf7f2 | 104 | def connect_to_redmine(a): |
9f95a23c TL |
105 | full_path=os.path.expanduser(redmine_key_file) |
106 | redmine_key='' | |
107 | try: | |
108 | with open(full_path, "r") as f: | |
109 | redmine_key = f.read().strip() | |
110 | except FileNotFoundError: | |
111 | pass | |
112 | ||
113 | if a.user and a.password: | |
11fdf7f2 TL |
114 | logging.info("Redmine username and password were provided; using them") |
115 | return Redmine(redmine_endpoint, username=a.user, password=a.password) | |
9f95a23c TL |
116 | elif redmine_key: |
117 | logging.info("Redmine key was read from '%s'; using it" % redmine_key_file) | |
118 | return Redmine(redmine_endpoint, key=redmine_key) | |
11fdf7f2 TL |
119 | else: |
120 | usage() | |
121 | ||
122 | def releases(): | |
123 | return ('argonaut', 'bobtail', 'cuttlefish', 'dumpling', 'emperor', | |
124 | 'firefly', 'giant', 'hammer', 'infernalis', 'jewel', 'kraken', | |
20effc67 | 125 | 'luminous', 'mimic', 'nautilus', 'octopus', 'pacific') |
11fdf7f2 TL |
126 | |
127 | def populate_status_dict(r): | |
128 | for status in r.issue_status.all(): | |
129 | status2status_id[status.name] = status.id | |
130 | logging.debug("Statuses {}".format(status2status_id)) | |
131 | return None | |
132 | ||
133 | # not used currently, but might be useful | |
134 | def populate_version_dict(r, p_id): | |
135 | versions = r.version.filter(project_id=p_id) | |
136 | for version in versions: | |
137 | version2version_id[version.name] = version.id | |
138 | #logging.debug("Versions {}".format(version2version_id)) | |
139 | return None | |
140 | ||
141 | def populate_tracker_dict(r): | |
142 | for tracker in r.tracker.all(): | |
143 | tracker2tracker_id[tracker.name] = tracker.id | |
144 | logging.debug("Trackers {}".format(tracker2tracker_id)) | |
145 | return None | |
146 | ||
147 | def has_tracker(r, p_id, tracker_name): | |
148 | for tracker in get_project(r, p_id).trackers: | |
149 | if tracker['name'] == tracker_name: | |
150 | return True | |
151 | return False | |
152 | ||
153 | def get_project(r, p_id): | |
154 | if p_id not in project_id2project: | |
155 | p_obj = r.project.get(p_id, include='trackers') | |
156 | project_id2project[p_id] = p_obj | |
157 | return project_id2project[p_id] | |
158 | ||
159 | def url(issue): | |
160 | return redmine_endpoint + "/issues/" + str(issue['id']) | |
161 | ||
162 | def set_backport(issue): | |
163 | for field in issue['custom_fields']: | |
164 | if field['name'] == 'Backport' and field['value'] != 0: | |
165 | issue['backports'] = set(re.findall('\w+', field['value'])) | |
166 | logging.debug("backports for " + str(issue['id']) + | |
167 | " is " + str(field['value']) + " " + | |
168 | str(issue['backports'])) | |
169 | return True | |
170 | return False | |
171 | ||
172 | def get_release(issue): | |
173 | for field in issue.custom_fields: | |
174 | if field['name'] == 'Release': | |
175 | return field['value'] | |
176 | ||
177 | def update_relations(r, issue, dry_run): | |
9f95a23c | 178 | global resolve_parent |
11fdf7f2 TL |
179 | relations = r.issue_relation.filter(issue_id=issue['id']) |
180 | existing_backports = set() | |
9f95a23c | 181 | existing_backports_dict = {} |
11fdf7f2 TL |
182 | for relation in relations: |
183 | other = r.issue.get(relation['issue_to_id']) | |
184 | if other['tracker']['name'] != 'Backport': | |
185 | logging.debug(url(issue) + " ignore relation to " + | |
186 | url(other) + " because it is not in the Backport " + | |
187 | "tracker") | |
188 | continue | |
189 | if relation['relation_type'] != 'copied_to': | |
190 | logging.error(url(issue) + " unexpected relation '" + | |
191 | relation['relation_type'] + "' to " + url(other)) | |
192 | continue | |
193 | release = get_release(other) | |
194 | if release in existing_backports: | |
195 | logging.error(url(issue) + " duplicate " + release + | |
196 | " backport issue detected") | |
197 | continue | |
198 | existing_backports.add(release) | |
9f95a23c | 199 | existing_backports_dict[release] = relation['issue_to_id'] |
11fdf7f2 TL |
200 | logging.debug(url(issue) + " backport to " + release + " is " + |
201 | redmine_endpoint + "/issues/" + str(relation['issue_to_id'])) | |
202 | if existing_backports == issue['backports']: | |
203 | logging.debug(url(issue) + " has all the required backport issues") | |
9f95a23c TL |
204 | if resolve_parent: |
205 | maybe_resolve(issue, existing_backports_dict, dry_run) | |
11fdf7f2 TL |
206 | return None |
207 | if existing_backports.issuperset(issue['backports']): | |
208 | logging.error(url(issue) + " has more backport issues (" + | |
209 | ",".join(sorted(existing_backports)) + ") than expected (" + | |
210 | ",".join(sorted(issue['backports'])) + ")") | |
211 | return None | |
212 | backport_tracker_id = tracker2tracker_id['Backport'] | |
213 | for release in issue['backports'] - existing_backports: | |
214 | if release not in releases(): | |
215 | logging.error(url(issue) + " requires backport to " + | |
216 | "unknown release " + release) | |
217 | break | |
9f95a23c | 218 | subject = (release + ": " + issue['subject'])[:255] |
11fdf7f2 TL |
219 | if dry_run: |
220 | logging.info(url(issue) + " add backport to " + release) | |
221 | continue | |
222 | other = r.issue.create(project_id=issue['project']['id'], | |
223 | tracker_id=backport_tracker_id, | |
224 | subject=subject, | |
20effc67 | 225 | priority_id=issue['priority']['id'], |
11fdf7f2 TL |
226 | target_version=None, |
227 | custom_fields=[{ | |
228 | "id": release_id, | |
229 | "value": release, | |
230 | }]) | |
231 | logging.debug("Rate-limiting to avoid seeming like a spammer") | |
232 | time.sleep(delay_seconds) | |
233 | r.issue_relation.create(issue_id=issue['id'], | |
234 | issue_to_id=other['id'], | |
235 | relation_type='copied_to') | |
236 | logging.info(url(issue) + " added backport to " + | |
237 | release + " " + url(other)) | |
238 | return None | |
239 | ||
9f95a23c TL |
240 | def maybe_resolve(issue, backports, dry_run): |
241 | ''' | |
242 | issue is a parent issue in Pending Backports status, and backports is a dict | |
243 | like, e.g., { "luminous": 25345, "mimic": 32134 }. | |
244 | If all the backport issues are Resolved/Rejected, set the parent issue to Resolved, too. | |
245 | ''' | |
246 | global delay_seconds | |
247 | global redmine | |
248 | global status2status_id | |
249 | if not backports: | |
250 | return None | |
251 | pending_backport_status_id = status2status_id["Pending Backport"] | |
252 | resolved_status_id = status2status_id["Resolved"] | |
253 | rejected_status_id = status2status_id["Rejected"] | |
254 | logging.debug("entering maybe_resolve with parent issue ->{}<- backports ->{}<-" | |
255 | .format(issue.id, backports)) | |
256 | assert issue.status.id == pending_backport_status_id, \ | |
257 | "Parent Redmine issue ->{}<- has status ->{}<- (expected Pending Backport)".format(issue.id, issue.status) | |
258 | all_resolved = True | |
259 | resolved_equiv_statuses = [resolved_status_id, rejected_status_id] | |
260 | for backport in backports.keys(): | |
261 | tracker_issue_id = backports[backport] | |
262 | backport_issue = redmine.issue.get(tracker_issue_id) | |
263 | logging.debug("{} backport is in status {}".format(backport, backport_issue.status.name)) | |
264 | if backport_issue.status.id not in resolved_equiv_statuses: | |
265 | all_resolved = False | |
266 | break | |
267 | if all_resolved: | |
268 | logging.debug("Parent ->{}<- all backport issues in status Resolved".format(url(issue))) | |
269 | note = ("While running with --resolve-parent, the script \"backport-create-issue\" " | |
270 | "noticed that all backports of this issue are in status \"Resolved\" or \"Rejected\".") | |
271 | if dry_run: | |
272 | logging.info("Set status of parent ->{}<- to Resolved".format(url(issue))) | |
273 | else: | |
274 | redmine.issue.update(issue.id, status_id=resolved_status_id, notes=note) | |
275 | logging.info("Parent ->{}<- status changed from Pending Backport to Resolved".format(url(issue))) | |
276 | logging.debug("Rate-limiting to avoid seeming like a spammer") | |
277 | time.sleep(delay_seconds) | |
278 | else: | |
279 | logging.debug("Some backport issues are still unresolved: leaving parent issue open") | |
280 | ||
281 | def iterate_over_backports(r, issues, dry_run=False): | |
11fdf7f2 TL |
282 | counter = 0 |
283 | for issue in issues: | |
284 | counter += 1 | |
285 | logging.debug("{} ({}) {}".format(issue.id, issue.project, | |
286 | issue.subject)) | |
9f95a23c | 287 | print('Examining issue#{} ({}/{})\r'.format(issue.id, counter, len(issues)), end='', flush=True) |
11fdf7f2 TL |
288 | if not has_tracker(r, issue['project']['id'], 'Backport'): |
289 | logging.info("{} skipped because the project {} does not " | |
290 | "have a Backport tracker".format(url(issue), | |
291 | issue['project']['name'])) | |
292 | continue | |
293 | if not set_backport(issue): | |
294 | logging.error(url(issue) + " no backport field") | |
295 | continue | |
296 | if len(issue['backports']) == 0: | |
297 | logging.error(url(issue) + " the backport field is empty") | |
298 | update_relations(r, issue, dry_run) | |
9f95a23c TL |
299 | print(' \r', end='', flush=True) |
300 | logging.info("Processed {} issues".format(counter)) | |
11fdf7f2 TL |
301 | return None |
302 | ||
303 | ||
304 | if __name__ == '__main__': | |
305 | args = parse_arguments() | |
306 | set_logging_level(args) | |
9f95a23c | 307 | process_resolve_parent_option(args) |
11fdf7f2 TL |
308 | report_dry_run(args) |
309 | redmine = connect_to_redmine(args) | |
310 | project = redmine.project.get(project_name) | |
311 | ceph_project_id = project.id | |
312 | logging.debug("Project {} has ID {}".format(project_name, ceph_project_id)) | |
313 | populate_status_dict(redmine) | |
314 | pending_backport_status_id = status2status_id["Pending Backport"] | |
315 | logging.debug("Pending Backport status has ID {}" | |
316 | .format(pending_backport_status_id)) | |
317 | populate_tracker_dict(redmine) | |
9f95a23c | 318 | force_create = False |
11fdf7f2 TL |
319 | if args.issue_numbers: |
320 | issue_list = ','.join(args.issue_numbers) | |
321 | logging.info("Processing issue list ->{}<-".format(issue_list)) | |
9f95a23c TL |
322 | if args.force: |
323 | force_create = True | |
324 | logging.warn("--force option was given: ignoring issue status!") | |
325 | issues = redmine.issue.filter(project_id=ceph_project_id, | |
326 | issue_id=issue_list) | |
327 | ||
328 | else: | |
329 | issues = redmine.issue.filter(project_id=ceph_project_id, | |
330 | issue_id=issue_list, | |
331 | status_id=pending_backport_status_id) | |
11fdf7f2 | 332 | else: |
9f95a23c TL |
333 | if args.force: |
334 | logging.warn("ignoring --force option, which can only be used with an explicit issue list") | |
11fdf7f2 TL |
335 | issues = redmine.issue.filter(project_id=ceph_project_id, |
336 | status_id=pending_backport_status_id) | |
9f95a23c TL |
337 | if force_create: |
338 | logging.info("Processing {} issues regardless of status" | |
339 | .format(len(issues))) | |
340 | else: | |
341 | logging.info("Processing {} issues with status Pending Backport" | |
342 | .format(len(issues))) | |
343 | iterate_over_backports(redmine, issues, dry_run=args.dry_run) |