]> git.proxmox.com Git - mirror_edk2.git/blob - BaseTools/Scripts/PatchCheck.py
UefiCpuPkg: Move AsmRelocateApLoopStart from Mpfuncs.nasm to AmdSev.nasm
[mirror_edk2.git] / BaseTools / Scripts / PatchCheck.py
1 ## @file
2 # Check a patch for various format issues
3 #
4 # Copyright (c) 2015 - 2020, Intel Corporation. All rights reserved.<BR>
5 # Copyright (C) 2020, Red Hat, Inc.<BR>
6 #
7 # SPDX-License-Identifier: BSD-2-Clause-Patent
8 #
9
10 from __future__ import print_function
11
12 VersionNumber = '0.1'
13 __copyright__ = "Copyright (c) 2015 - 2016, Intel Corporation All rights reserved."
14
15 import email
16 import argparse
17 import os
18 import re
19 import subprocess
20 import sys
21
22 class Verbose:
23 SILENT, ONELINE, NORMAL = range(3)
24 level = NORMAL
25
26 class EmailAddressCheck:
27 """Checks an email address."""
28
29 def __init__(self, email, description):
30 self.ok = True
31
32 if email is None:
33 self.error('Email address is missing!')
34 return
35 if description is None:
36 self.error('Email description is missing!')
37 return
38
39 self.description = "'" + description + "'"
40 self.check_email_address(email)
41
42 def error(self, *err):
43 if self.ok and Verbose.level > Verbose.ONELINE:
44 print('The ' + self.description + ' email address is not valid:')
45 self.ok = False
46 if Verbose.level < Verbose.NORMAL:
47 return
48 count = 0
49 for line in err:
50 prefix = (' *', ' ')[count > 0]
51 print(prefix, line)
52 count += 1
53
54 email_re1 = re.compile(r'(?:\s*)(.*?)(\s*)<(.+)>\s*$',
55 re.MULTILINE|re.IGNORECASE)
56
57 def check_email_address(self, email):
58 email = email.strip()
59 mo = self.email_re1.match(email)
60 if mo is None:
61 self.error("Email format is invalid: " + email.strip())
62 return
63
64 name = mo.group(1).strip()
65 if name == '':
66 self.error("Name is not provided with email address: " +
67 email)
68 else:
69 quoted = len(name) > 2 and name[0] == '"' and name[-1] == '"'
70 if name.find(',') >= 0 and not quoted:
71 self.error('Add quotes (") around name with a comma: ' +
72 name)
73
74 if mo.group(2) == '':
75 self.error("There should be a space between the name and " +
76 "email address: " + email)
77
78 if mo.group(3).find(' ') >= 0:
79 self.error("The email address cannot contain a space: " +
80 mo.group(3))
81
82 class CommitMessageCheck:
83 """Checks the contents of a git commit message."""
84
85 def __init__(self, subject, message):
86 self.ok = True
87
88 if subject is None and message is None:
89 self.error('Commit message is missing!')
90 return
91
92 self.subject = subject
93 self.msg = message
94
95 print (subject)
96
97 self.check_contributed_under()
98 self.check_signed_off_by()
99 self.check_misc_signatures()
100 self.check_overall_format()
101 self.report_message_result()
102
103 url = 'https://github.com/tianocore/tianocore.github.io/wiki/Commit-Message-Format'
104
105 def report_message_result(self):
106 if Verbose.level < Verbose.NORMAL:
107 return
108 if self.ok:
109 # All checks passed
110 return_code = 0
111 print('The commit message format passed all checks.')
112 else:
113 return_code = 1
114 if not self.ok:
115 print(self.url)
116
117 def error(self, *err):
118 if self.ok and Verbose.level > Verbose.ONELINE:
119 print('The commit message format is not valid:')
120 self.ok = False
121 if Verbose.level < Verbose.NORMAL:
122 return
123 count = 0
124 for line in err:
125 prefix = (' *', ' ')[count > 0]
126 print(prefix, line)
127 count += 1
128
129 # Find 'contributed-under:' at the start of a line ignoring case and
130 # requires ':' to be present. Matches if there is white space before
131 # the tag or between the tag and the ':'.
132 contributed_under_re = \
133 re.compile(r'^\s*contributed-under\s*:', re.MULTILINE|re.IGNORECASE)
134
135 def check_contributed_under(self):
136 match = self.contributed_under_re.search(self.msg)
137 if match is not None:
138 self.error('Contributed-under! (Note: this must be ' +
139 'removed by the code contributor!)')
140
141 @staticmethod
142 def make_signature_re(sig, re_input=False):
143 if re_input:
144 sub_re = sig
145 else:
146 sub_re = sig.replace('-', r'[-\s]+')
147 re_str = (r'^(?P<tag>' + sub_re +
148 r')(\s*):(\s*)(?P<value>\S.*?)(?:\s*)$')
149 try:
150 return re.compile(re_str, re.MULTILINE|re.IGNORECASE)
151 except Exception:
152 print("Tried to compile re:", re_str)
153 raise
154
155 sig_block_re = \
156 re.compile(r'''^
157 (?: (?P<tag>[^:]+) \s* : \s*
158 (?P<value>\S.*?) )
159 |
160 (?: \[ (?P<updater>[^:]+) \s* : \s*
161 (?P<note>.+?) \s* \] )
162 \s* $''',
163 re.VERBOSE | re.MULTILINE)
164
165 def find_signatures(self, sig):
166 if not sig.endswith('-by') and sig != 'Cc':
167 sig += '-by'
168 regex = self.make_signature_re(sig)
169
170 sigs = regex.findall(self.msg)
171
172 bad_case_sigs = filter(lambda m: m[0] != sig, sigs)
173 for s in bad_case_sigs:
174 self.error("'" +s[0] + "' should be '" + sig + "'")
175
176 for s in sigs:
177 if s[1] != '':
178 self.error('There should be no spaces between ' + sig +
179 " and the ':'")
180 if s[2] != ' ':
181 self.error("There should be a space after '" + sig + ":'")
182
183 EmailAddressCheck(s[3], sig)
184
185 return sigs
186
187 def check_signed_off_by(self):
188 sob='Signed-off-by'
189 if self.msg.find(sob) < 0:
190 self.error('Missing Signed-off-by! (Note: this must be ' +
191 'added by the code contributor!)')
192 return
193
194 sobs = self.find_signatures('Signed-off')
195
196 if len(sobs) == 0:
197 self.error('Invalid Signed-off-by format!')
198 return
199
200 sig_types = (
201 'Reviewed',
202 'Reported',
203 'Tested',
204 'Suggested',
205 'Acked',
206 'Cc'
207 )
208
209 def check_misc_signatures(self):
210 for sig in self.sig_types:
211 self.find_signatures(sig)
212
213 cve_re = re.compile('CVE-[0-9]{4}-[0-9]{5}[^0-9]')
214
215 def check_overall_format(self):
216 lines = self.msg.splitlines()
217
218 if len(lines) >= 1 and lines[0].endswith('\r\n'):
219 empty_line = '\r\n'
220 else:
221 empty_line = '\n'
222
223 lines.insert(0, empty_line)
224 lines.insert(0, self.subject + empty_line)
225
226 count = len(lines)
227
228 if count <= 0:
229 self.error('Empty commit message!')
230 return
231
232 if count >= 1 and re.search(self.cve_re, lines[0]):
233 #
234 # If CVE-xxxx-xxxxx is present in subject line, then limit length of
235 # subject line to 92 characters
236 #
237 if len(lines[0].rstrip()) >= 93:
238 self.error(
239 'First line of commit message (subject line) is too long (%d >= 93).' %
240 (len(lines[0].rstrip()))
241 )
242 else:
243 #
244 # If CVE-xxxx-xxxxx is not present in subject line, then limit
245 # length of subject line to 75 characters
246 #
247 if len(lines[0].rstrip()) >= 76:
248 self.error(
249 'First line of commit message (subject line) is too long (%d >= 76).' %
250 (len(lines[0].rstrip()))
251 )
252
253 if count >= 1 and len(lines[0].strip()) == 0:
254 self.error('First line of commit message (subject line) ' +
255 'is empty.')
256
257 if count >= 2 and lines[1].strip() != '':
258 self.error('Second line of commit message should be ' +
259 'empty.')
260
261 for i in range(2, count):
262 if (len(lines[i]) >= 76 and
263 len(lines[i].split()) > 1 and
264 not lines[i].startswith('git-svn-id:')):
265 #
266 # Print a warning if body line is longer than 75 characters
267 #
268 print(
269 'WARNING - Line %d of commit message is too long (%d >= 76).' %
270 (i + 1, len(lines[i]))
271 )
272 print(lines[i])
273
274 last_sig_line = None
275 for i in range(count - 1, 0, -1):
276 line = lines[i]
277 mo = self.sig_block_re.match(line)
278 if mo is None:
279 if line.strip() == '':
280 break
281 elif last_sig_line is not None:
282 err2 = 'Add empty line before "%s"?' % last_sig_line
283 self.error('The line before the signature block ' +
284 'should be empty', err2)
285 else:
286 self.error('The signature block was not found')
287 break
288 last_sig_line = line.strip()
289
290 (START, PRE_PATCH, PATCH) = range(3)
291
292 class GitDiffCheck:
293 """Checks the contents of a git diff."""
294
295 def __init__(self, diff):
296 self.ok = True
297 self.format_ok = True
298 self.lines = diff.splitlines(True)
299 self.count = len(self.lines)
300 self.line_num = 0
301 self.state = START
302 self.new_bin = []
303 while self.line_num < self.count and self.format_ok:
304 line_num = self.line_num
305 self.run()
306 assert(self.line_num > line_num)
307 self.report_message_result()
308
309 def report_message_result(self):
310 if Verbose.level < Verbose.NORMAL:
311 return
312 if self.ok:
313 print('The code passed all checks.')
314 if self.new_bin:
315 print('\nWARNING - The following binary files will be added ' +
316 'into the repository:')
317 for binary in self.new_bin:
318 print(' ' + binary)
319
320 def run(self):
321 line = self.lines[self.line_num]
322
323 if self.state in (PRE_PATCH, PATCH):
324 if line.startswith('diff --git'):
325 self.state = START
326 if self.state == PATCH:
327 if line.startswith('@@ '):
328 self.state = PRE_PATCH
329 elif len(line) >= 1 and line[0] not in ' -+' and \
330 not line.startswith('\r\n') and \
331 not line.startswith(r'\ No newline ') and not self.binary:
332 for line in self.lines[self.line_num + 1:]:
333 if line.startswith('diff --git'):
334 self.format_error('diff found after end of patch')
335 break
336 self.line_num = self.count
337 return
338
339 if self.state == START:
340 if line.startswith('diff --git'):
341 self.state = PRE_PATCH
342 self.filename = line[13:].split(' ', 1)[0]
343 self.is_newfile = False
344 self.force_crlf = True
345 self.force_notabs = True
346 if self.filename.endswith('.sh'):
347 #
348 # Do not enforce CR/LF line endings for linux shell scripts.
349 #
350 self.force_crlf = False
351 if self.filename == '.gitmodules':
352 #
353 # .gitmodules is updated by git and uses tabs and LF line
354 # endings. Do not enforce no tabs and do not enforce
355 # CR/LF line endings.
356 #
357 self.force_crlf = False
358 self.force_notabs = False
359 elif len(line.rstrip()) != 0:
360 self.format_error("didn't find diff command")
361 self.line_num += 1
362 elif self.state == PRE_PATCH:
363 if line.startswith('@@ '):
364 self.state = PATCH
365 self.binary = False
366 elif line.startswith('GIT binary patch') or \
367 line.startswith('Binary files'):
368 self.state = PATCH
369 self.binary = True
370 if self.is_newfile:
371 self.new_bin.append(self.filename)
372 elif line.startswith('new file mode 160000'):
373 #
374 # New submodule. Do not enforce CR/LF line endings
375 #
376 self.force_crlf = False
377 else:
378 ok = False
379 self.is_newfile = self.newfile_prefix_re.match(line)
380 for pfx in self.pre_patch_prefixes:
381 if line.startswith(pfx):
382 ok = True
383 if not ok:
384 self.format_error("didn't find diff hunk marker (@@)")
385 self.line_num += 1
386 elif self.state == PATCH:
387 if self.binary:
388 pass
389 elif line.startswith('-'):
390 pass
391 elif line.startswith('+'):
392 self.check_added_line(line[1:])
393 elif line.startswith('\r\n'):
394 pass
395 elif line.startswith(r'\ No newline '):
396 pass
397 elif not line.startswith(' '):
398 self.format_error("unexpected patch line")
399 self.line_num += 1
400
401 pre_patch_prefixes = (
402 '--- ',
403 '+++ ',
404 'index ',
405 'new file ',
406 'deleted file ',
407 'old mode ',
408 'new mode ',
409 'similarity index ',
410 'copy from ',
411 'copy to ',
412 'rename ',
413 )
414
415 line_endings = ('\r\n', '\n\r', '\n', '\r')
416
417 newfile_prefix_re = \
418 re.compile(r'''^
419 index\ 0+\.\.
420 ''',
421 re.VERBOSE)
422
423 def added_line_error(self, msg, line):
424 lines = [ msg ]
425 if self.filename is not None:
426 lines.append('File: ' + self.filename)
427 lines.append('Line: ' + line)
428
429 self.error(*lines)
430
431 old_debug_re = \
432 re.compile(r'''
433 DEBUG \s* \( \s* \( \s*
434 (?: DEBUG_[A-Z_]+ \s* \| \s*)*
435 EFI_D_ ([A-Z_]+)
436 ''',
437 re.VERBOSE)
438
439 def check_added_line(self, line):
440 eol = ''
441 for an_eol in self.line_endings:
442 if line.endswith(an_eol):
443 eol = an_eol
444 line = line[:-len(eol)]
445
446 stripped = line.rstrip()
447
448 if self.force_crlf and eol != '\r\n':
449 self.added_line_error('Line ending (%s) is not CRLF' % repr(eol),
450 line)
451 if self.force_notabs and '\t' in line:
452 self.added_line_error('Tab character used', line)
453 if len(stripped) < len(line):
454 self.added_line_error('Trailing whitespace found', line)
455
456 mo = self.old_debug_re.search(line)
457 if mo is not None:
458 self.added_line_error('EFI_D_' + mo.group(1) + ' was used, '
459 'but DEBUG_' + mo.group(1) +
460 ' is now recommended', line)
461
462 split_diff_re = re.compile(r'''
463 (?P<cmd>
464 ^ diff \s+ --git \s+ a/.+ \s+ b/.+ $
465 )
466 (?P<index>
467 ^ index \s+ .+ $
468 )
469 ''',
470 re.IGNORECASE | re.VERBOSE | re.MULTILINE)
471
472 def format_error(self, err):
473 self.format_ok = False
474 err = 'Patch format error: ' + err
475 err2 = 'Line: ' + self.lines[self.line_num].rstrip()
476 self.error(err, err2)
477
478 def error(self, *err):
479 if self.ok and Verbose.level > Verbose.ONELINE:
480 print('Code format is not valid:')
481 self.ok = False
482 if Verbose.level < Verbose.NORMAL:
483 return
484 count = 0
485 for line in err:
486 prefix = (' *', ' ')[count > 0]
487 print(prefix, line)
488 count += 1
489
490 class CheckOnePatch:
491 """Checks the contents of a git email formatted patch.
492
493 Various checks are performed on both the commit message and the
494 patch content.
495 """
496
497 def __init__(self, name, patch):
498 self.patch = patch
499 self.find_patch_pieces()
500
501 email_check = EmailAddressCheck(self.author_email, 'Author')
502 email_ok = email_check.ok
503
504 msg_check = CommitMessageCheck(self.commit_subject, self.commit_msg)
505 msg_ok = msg_check.ok
506
507 diff_ok = True
508 if self.diff is not None:
509 diff_check = GitDiffCheck(self.diff)
510 diff_ok = diff_check.ok
511
512 self.ok = email_ok and msg_ok and diff_ok
513
514 if Verbose.level == Verbose.ONELINE:
515 if self.ok:
516 result = 'ok'
517 else:
518 result = list()
519 if not msg_ok:
520 result.append('commit message')
521 if not diff_ok:
522 result.append('diff content')
523 result = 'bad ' + ' and '.join(result)
524 print(name, result)
525
526
527 git_diff_re = re.compile(r'''
528 ^ diff \s+ --git \s+ a/.+ \s+ b/.+ $
529 ''',
530 re.IGNORECASE | re.VERBOSE | re.MULTILINE)
531
532 stat_re = \
533 re.compile(r'''
534 (?P<commit_message> [\s\S\r\n]* )
535 (?P<stat>
536 ^ --- $ [\r\n]+
537 (?: ^ \s+ .+ \s+ \| \s+ \d+ \s+ \+* \-*
538 $ [\r\n]+ )+
539 [\s\S\r\n]+
540 )
541 ''',
542 re.IGNORECASE | re.VERBOSE | re.MULTILINE)
543
544 subject_prefix_re = \
545 re.compile(r'''^
546 \s* (\[
547 [^\[\]]* # Allow all non-brackets
548 \])* \s*
549 ''',
550 re.VERBOSE)
551
552 def find_patch_pieces(self):
553 if sys.version_info < (3, 0):
554 patch = self.patch.encode('ascii', 'ignore')
555 else:
556 patch = self.patch
557
558 self.commit_msg = None
559 self.stat = None
560 self.commit_subject = None
561 self.commit_prefix = None
562 self.diff = None
563
564 if patch.startswith('diff --git'):
565 self.diff = patch
566 return
567
568 pmail = email.message_from_string(patch)
569 parts = list(pmail.walk())
570 assert(len(parts) == 1)
571 assert(parts[0].get_content_type() == 'text/plain')
572 content = parts[0].get_payload(decode=True).decode('utf-8', 'ignore')
573
574 mo = self.git_diff_re.search(content)
575 if mo is not None:
576 self.diff = content[mo.start():]
577 content = content[:mo.start()]
578
579 mo = self.stat_re.search(content)
580 if mo is None:
581 self.commit_msg = content
582 else:
583 self.stat = mo.group('stat')
584 self.commit_msg = mo.group('commit_message')
585 #
586 # Parse subject line from email header. The subject line may be
587 # composed of multiple parts with different encodings. Decode and
588 # combine all the parts to produce a single string with the contents of
589 # the decoded subject line.
590 #
591 parts = email.header.decode_header(pmail.get('subject'))
592 subject = ''
593 for (part, encoding) in parts:
594 if encoding:
595 part = part.decode(encoding)
596 else:
597 try:
598 part = part.decode()
599 except:
600 pass
601 subject = subject + part
602
603 self.commit_subject = subject.replace('\r\n', '')
604 self.commit_subject = self.commit_subject.replace('\n', '')
605 self.commit_subject = self.subject_prefix_re.sub('', self.commit_subject, 1)
606
607 self.author_email = pmail['from']
608
609 class CheckGitCommits:
610 """Reads patches from git based on the specified git revision range.
611
612 The patches are read from git, and then checked.
613 """
614
615 def __init__(self, rev_spec, max_count):
616 commits = self.read_commit_list_from_git(rev_spec, max_count)
617 if len(commits) == 1 and Verbose.level > Verbose.ONELINE:
618 commits = [ rev_spec ]
619 self.ok = True
620 blank_line = False
621 for commit in commits:
622 if Verbose.level > Verbose.ONELINE:
623 if blank_line:
624 print()
625 else:
626 blank_line = True
627 print('Checking git commit:', commit)
628 email = self.read_committer_email_address_from_git(commit)
629 self.ok &= EmailAddressCheck(email, 'Committer').ok
630 patch = self.read_patch_from_git(commit)
631 self.ok &= CheckOnePatch(commit, patch).ok
632 if not commits:
633 print("Couldn't find commit matching: '{}'".format(rev_spec))
634
635 def read_commit_list_from_git(self, rev_spec, max_count):
636 # Run git to get the commit patch
637 cmd = [ 'rev-list', '--abbrev-commit', '--no-walk' ]
638 if max_count is not None:
639 cmd.append('--max-count=' + str(max_count))
640 cmd.append(rev_spec)
641 out = self.run_git(*cmd)
642 return out.split() if out else []
643
644 def read_patch_from_git(self, commit):
645 # Run git to get the commit patch
646 return self.run_git('show', '--pretty=email', '--no-textconv', commit)
647
648 def read_committer_email_address_from_git(self, commit):
649 # Run git to get the committer email
650 return self.run_git('show', '--pretty=%cn <%ce>', '--no-patch', commit)
651
652 def run_git(self, *args):
653 cmd = [ 'git' ]
654 cmd += args
655 p = subprocess.Popen(cmd,
656 stdout=subprocess.PIPE,
657 stderr=subprocess.STDOUT)
658 Result = p.communicate()
659 return Result[0].decode('utf-8', 'ignore') if Result[0] and Result[0].find(b"fatal")!=0 else None
660
661 class CheckOnePatchFile:
662 """Performs a patch check for a single file.
663
664 stdin is used when the filename is '-'.
665 """
666
667 def __init__(self, patch_filename):
668 if patch_filename == '-':
669 patch = sys.stdin.read()
670 patch_filename = 'stdin'
671 else:
672 f = open(patch_filename, 'rb')
673 patch = f.read().decode('utf-8', 'ignore')
674 f.close()
675 if Verbose.level > Verbose.ONELINE:
676 print('Checking patch file:', patch_filename)
677 self.ok = CheckOnePatch(patch_filename, patch).ok
678
679 class CheckOneArg:
680 """Performs a patch check for a single command line argument.
681
682 The argument will be handed off to a file or git-commit based
683 checker.
684 """
685
686 def __init__(self, param, max_count=None):
687 self.ok = True
688 if param == '-' or os.path.exists(param):
689 checker = CheckOnePatchFile(param)
690 else:
691 checker = CheckGitCommits(param, max_count)
692 self.ok = checker.ok
693
694 class PatchCheckApp:
695 """Checks patches based on the command line arguments."""
696
697 def __init__(self):
698 self.parse_options()
699 patches = self.args.patches
700
701 if len(patches) == 0:
702 patches = [ 'HEAD' ]
703
704 self.ok = True
705 self.count = None
706 for patch in patches:
707 self.process_one_arg(patch)
708
709 if self.count is not None:
710 self.process_one_arg('HEAD')
711
712 if self.ok:
713 self.retval = 0
714 else:
715 self.retval = -1
716
717 def process_one_arg(self, arg):
718 if len(arg) >= 2 and arg[0] == '-':
719 try:
720 self.count = int(arg[1:])
721 return
722 except ValueError:
723 pass
724 self.ok &= CheckOneArg(arg, self.count).ok
725 self.count = None
726
727 def parse_options(self):
728 parser = argparse.ArgumentParser(description=__copyright__)
729 parser.add_argument('--version', action='version',
730 version='%(prog)s ' + VersionNumber)
731 parser.add_argument('patches', nargs='*',
732 help='[patch file | git rev list]')
733 group = parser.add_mutually_exclusive_group()
734 group.add_argument("--oneline",
735 action="store_true",
736 help="Print one result per line")
737 group.add_argument("--silent",
738 action="store_true",
739 help="Print nothing")
740 self.args = parser.parse_args()
741 if self.args.oneline:
742 Verbose.level = Verbose.ONELINE
743 if self.args.silent:
744 Verbose.level = Verbose.SILENT
745
746 if __name__ == "__main__":
747 sys.exit(PatchCheckApp().retval)