]> git.proxmox.com Git - mirror_edk2.git/blob - BaseTools/Scripts/PatchCheck.py
MdePkg/Acpi62: Add type 7 NFIT Platform Capabilities Structure support
[mirror_edk2.git] / BaseTools / Scripts / PatchCheck.py
1 ## @file
2 # Check a patch for various format issues
3 #
4 # Copyright (c) 2015 - 2021, Intel Corporation. All rights reserved.<BR>
5 # Copyright (C) 2020, Red Hat, Inc.<BR>
6 # Copyright (c) 2020, ARM Ltd. All rights reserved.<BR>
7 #
8 # SPDX-License-Identifier: BSD-2-Clause-Patent
9 #
10
11 from __future__ import print_function
12
13 VersionNumber = '0.1'
14 __copyright__ = "Copyright (c) 2015 - 2016, Intel Corporation All rights reserved."
15
16 import email
17 import argparse
18 import os
19 import re
20 import subprocess
21 import sys
22
23 import email.header
24
25 class Verbose:
26 SILENT, ONELINE, NORMAL = range(3)
27 level = NORMAL
28
29 class EmailAddressCheck:
30 """Checks an email address."""
31
32 def __init__(self, email, description):
33 self.ok = True
34
35 if email is None:
36 self.error('Email address is missing!')
37 return
38 if description is None:
39 self.error('Email description is missing!')
40 return
41
42 self.description = "'" + description + "'"
43 self.check_email_address(email)
44
45 def error(self, *err):
46 if self.ok and Verbose.level > Verbose.ONELINE:
47 print('The ' + self.description + ' email address is not valid:')
48 self.ok = False
49 if Verbose.level < Verbose.NORMAL:
50 return
51 count = 0
52 for line in err:
53 prefix = (' *', ' ')[count > 0]
54 print(prefix, line)
55 count += 1
56
57 email_re1 = re.compile(r'(?:\s*)(.*?)(\s*)<(.+)>\s*$',
58 re.MULTILINE|re.IGNORECASE)
59
60 def check_email_address(self, email):
61 email = email.strip()
62 mo = self.email_re1.match(email)
63 if mo is None:
64 self.error("Email format is invalid: " + email.strip())
65 return
66
67 name = mo.group(1).strip()
68 if name == '':
69 self.error("Name is not provided with email address: " +
70 email)
71 else:
72 quoted = len(name) > 2 and name[0] == '"' and name[-1] == '"'
73 if name.find(',') >= 0 and not quoted:
74 self.error('Add quotes (") around name with a comma: ' +
75 name)
76
77 if mo.group(2) == '':
78 self.error("There should be a space between the name and " +
79 "email address: " + email)
80
81 if mo.group(3).find(' ') >= 0:
82 self.error("The email address cannot contain a space: " +
83 mo.group(3))
84
85 if ' via Groups.Io' in name and mo.group(3).endswith('@groups.io'):
86 self.error("Email rewritten by lists DMARC / DKIM / SPF: " +
87 email)
88
89 class CommitMessageCheck:
90 """Checks the contents of a git commit message."""
91
92 def __init__(self, subject, message, author_email):
93 self.ok = True
94
95 if subject is None and message is None:
96 self.error('Commit message is missing!')
97 return
98
99 MergifyMerge = False
100 if "mergify[bot]@users.noreply.github.com" in author_email:
101 if "Merge branch" in subject:
102 MergifyMerge = True
103
104 self.subject = subject
105 self.msg = message
106
107 print (subject)
108
109 self.check_contributed_under()
110 if not MergifyMerge:
111 self.check_signed_off_by()
112 self.check_misc_signatures()
113 self.check_overall_format()
114 self.report_message_result()
115
116 url = 'https://github.com/tianocore/tianocore.github.io/wiki/Commit-Message-Format'
117
118 def report_message_result(self):
119 if Verbose.level < Verbose.NORMAL:
120 return
121 if self.ok:
122 # All checks passed
123 return_code = 0
124 print('The commit message format passed all checks.')
125 else:
126 return_code = 1
127 if not self.ok:
128 print(self.url)
129
130 def error(self, *err):
131 if self.ok and Verbose.level > Verbose.ONELINE:
132 print('The commit message format is not valid:')
133 self.ok = False
134 if Verbose.level < Verbose.NORMAL:
135 return
136 count = 0
137 for line in err:
138 prefix = (' *', ' ')[count > 0]
139 print(prefix, line)
140 count += 1
141
142 # Find 'contributed-under:' at the start of a line ignoring case and
143 # requires ':' to be present. Matches if there is white space before
144 # the tag or between the tag and the ':'.
145 contributed_under_re = \
146 re.compile(r'^\s*contributed-under\s*:', re.MULTILINE|re.IGNORECASE)
147
148 def check_contributed_under(self):
149 match = self.contributed_under_re.search(self.msg)
150 if match is not None:
151 self.error('Contributed-under! (Note: this must be ' +
152 'removed by the code contributor!)')
153
154 @staticmethod
155 def make_signature_re(sig, re_input=False):
156 if re_input:
157 sub_re = sig
158 else:
159 sub_re = sig.replace('-', r'[-\s]+')
160 re_str = (r'^(?P<tag>' + sub_re +
161 r')(\s*):(\s*)(?P<value>\S.*?)(?:\s*)$')
162 try:
163 return re.compile(re_str, re.MULTILINE|re.IGNORECASE)
164 except Exception:
165 print("Tried to compile re:", re_str)
166 raise
167
168 sig_block_re = \
169 re.compile(r'''^
170 (?: (?P<tag>[^:]+) \s* : \s*
171 (?P<value>\S.*?) )
172 |
173 (?: \[ (?P<updater>[^:]+) \s* : \s*
174 (?P<note>.+?) \s* \] )
175 \s* $''',
176 re.VERBOSE | re.MULTILINE)
177
178 def find_signatures(self, sig):
179 if not sig.endswith('-by') and sig != 'Cc':
180 sig += '-by'
181 regex = self.make_signature_re(sig)
182
183 sigs = regex.findall(self.msg)
184
185 bad_case_sigs = filter(lambda m: m[0] != sig, sigs)
186 for s in bad_case_sigs:
187 self.error("'" +s[0] + "' should be '" + sig + "'")
188
189 for s in sigs:
190 if s[1] != '':
191 self.error('There should be no spaces between ' + sig +
192 " and the ':'")
193 if s[2] != ' ':
194 self.error("There should be a space after '" + sig + ":'")
195
196 EmailAddressCheck(s[3], sig)
197
198 return sigs
199
200 def check_signed_off_by(self):
201 sob='Signed-off-by'
202 if self.msg.find(sob) < 0:
203 self.error('Missing Signed-off-by! (Note: this must be ' +
204 'added by the code contributor!)')
205 return
206
207 sobs = self.find_signatures('Signed-off')
208
209 if len(sobs) == 0:
210 self.error('Invalid Signed-off-by format!')
211 return
212
213 sig_types = (
214 'Reviewed',
215 'Reported',
216 'Tested',
217 'Suggested',
218 'Acked',
219 'Cc'
220 )
221
222 def check_misc_signatures(self):
223 for sig in self.sig_types:
224 self.find_signatures(sig)
225
226 cve_re = re.compile('CVE-[0-9]{4}-[0-9]{5}[^0-9]')
227
228 def check_overall_format(self):
229 lines = self.msg.splitlines()
230
231 if len(lines) >= 1 and lines[0].endswith('\r\n'):
232 empty_line = '\r\n'
233 else:
234 empty_line = '\n'
235
236 lines.insert(0, empty_line)
237 lines.insert(0, self.subject + empty_line)
238
239 count = len(lines)
240
241 if count <= 0:
242 self.error('Empty commit message!')
243 return
244
245 if count >= 1 and re.search(self.cve_re, lines[0]):
246 #
247 # If CVE-xxxx-xxxxx is present in subject line, then limit length of
248 # subject line to 92 characters
249 #
250 if len(lines[0].rstrip()) >= 93:
251 self.error(
252 'First line of commit message (subject line) is too long (%d >= 93).' %
253 (len(lines[0].rstrip()))
254 )
255 else:
256 #
257 # If CVE-xxxx-xxxxx is not present in subject line, then limit
258 # length of subject line to 75 characters
259 #
260 if len(lines[0].rstrip()) >= 76:
261 self.error(
262 'First line of commit message (subject line) is too long (%d >= 76).' %
263 (len(lines[0].rstrip()))
264 )
265
266 if count >= 1 and len(lines[0].strip()) == 0:
267 self.error('First line of commit message (subject line) ' +
268 'is empty.')
269
270 if count >= 2 and lines[1].strip() != '':
271 self.error('Second line of commit message should be ' +
272 'empty.')
273
274 for i in range(2, count):
275 if (len(lines[i]) >= 76 and
276 len(lines[i].split()) > 1 and
277 not lines[i].startswith('git-svn-id:') and
278 not lines[i].startswith('Reviewed-by') and
279 not lines[i].startswith('Acked-by:') and
280 not lines[i].startswith('Tested-by:') and
281 not lines[i].startswith('Reported-by:') and
282 not lines[i].startswith('Suggested-by:') and
283 not lines[i].startswith('Signed-off-by:') and
284 not lines[i].startswith('Cc:')):
285 #
286 # Print a warning if body line is longer than 75 characters
287 #
288 print(
289 'WARNING - Line %d of commit message is too long (%d >= 76).' %
290 (i + 1, len(lines[i]))
291 )
292 print(lines[i])
293
294 last_sig_line = None
295 for i in range(count - 1, 0, -1):
296 line = lines[i]
297 mo = self.sig_block_re.match(line)
298 if mo is None:
299 if line.strip() == '':
300 break
301 elif last_sig_line is not None:
302 err2 = 'Add empty line before "%s"?' % last_sig_line
303 self.error('The line before the signature block ' +
304 'should be empty', err2)
305 else:
306 self.error('The signature block was not found')
307 break
308 last_sig_line = line.strip()
309
310 (START, PRE_PATCH, PATCH) = range(3)
311
312 class GitDiffCheck:
313 """Checks the contents of a git diff."""
314
315 def __init__(self, diff):
316 self.ok = True
317 self.format_ok = True
318 self.lines = diff.splitlines(True)
319 self.count = len(self.lines)
320 self.line_num = 0
321 self.state = START
322 self.new_bin = []
323 while self.line_num < self.count and self.format_ok:
324 line_num = self.line_num
325 self.run()
326 assert(self.line_num > line_num)
327 self.report_message_result()
328
329 def report_message_result(self):
330 if Verbose.level < Verbose.NORMAL:
331 return
332 if self.ok:
333 print('The code passed all checks.')
334 if self.new_bin:
335 print('\nWARNING - The following binary files will be added ' +
336 'into the repository:')
337 for binary in self.new_bin:
338 print(' ' + binary)
339
340 def run(self):
341 line = self.lines[self.line_num]
342
343 if self.state in (PRE_PATCH, PATCH):
344 if line.startswith('diff --git'):
345 self.state = START
346 if self.state == PATCH:
347 if line.startswith('@@ '):
348 self.state = PRE_PATCH
349 elif len(line) >= 1 and line[0] not in ' -+' and \
350 not line.startswith('\r\n') and \
351 not line.startswith(r'\ No newline ') and not self.binary:
352 for line in self.lines[self.line_num + 1:]:
353 if line.startswith('diff --git'):
354 self.format_error('diff found after end of patch')
355 break
356 self.line_num = self.count
357 return
358
359 if self.state == START:
360 if line.startswith('diff --git'):
361 self.state = PRE_PATCH
362 self.filename = line[13:].split(' ', 1)[0]
363 self.is_newfile = False
364 self.force_crlf = True
365 self.force_notabs = True
366 if self.filename.endswith('.sh') or \
367 self.filename.startswith('BaseTools/BinWrappers/PosixLike/') or \
368 self.filename.startswith('BaseTools/BinPipWrappers/PosixLike/') or \
369 self.filename.startswith('BaseTools/Bin/CYGWIN_NT-5.1-i686/') or \
370 self.filename == 'BaseTools/BuildEnv':
371 #
372 # Do not enforce CR/LF line endings for linux shell scripts.
373 # Some linux shell scripts don't end with the ".sh" extension,
374 # they are identified by their path.
375 #
376 self.force_crlf = False
377 if self.filename == '.gitmodules' or \
378 self.filename == 'BaseTools/Conf/diff.order':
379 #
380 # .gitmodules and diff orderfiles are used internally by git
381 # use tabs and LF line endings. Do not enforce no tabs and
382 # do not enforce CR/LF line endings.
383 #
384 self.force_crlf = False
385 self.force_notabs = False
386 elif len(line.rstrip()) != 0:
387 self.format_error("didn't find diff command")
388 self.line_num += 1
389 elif self.state == PRE_PATCH:
390 if line.startswith('@@ '):
391 self.state = PATCH
392 self.binary = False
393 elif line.startswith('GIT binary patch') or \
394 line.startswith('Binary files'):
395 self.state = PATCH
396 self.binary = True
397 if self.is_newfile:
398 self.new_bin.append(self.filename)
399 elif line.startswith('new file mode 160000'):
400 #
401 # New submodule. Do not enforce CR/LF line endings
402 #
403 self.force_crlf = False
404 else:
405 ok = False
406 self.is_newfile = self.newfile_prefix_re.match(line)
407 for pfx in self.pre_patch_prefixes:
408 if line.startswith(pfx):
409 ok = True
410 if not ok:
411 self.format_error("didn't find diff hunk marker (@@)")
412 self.line_num += 1
413 elif self.state == PATCH:
414 if self.binary:
415 pass
416 elif line.startswith('-'):
417 pass
418 elif line.startswith('+'):
419 self.check_added_line(line[1:])
420 elif line.startswith('\r\n'):
421 pass
422 elif line.startswith(r'\ No newline '):
423 pass
424 elif not line.startswith(' '):
425 self.format_error("unexpected patch line")
426 self.line_num += 1
427
428 pre_patch_prefixes = (
429 '--- ',
430 '+++ ',
431 'index ',
432 'new file ',
433 'deleted file ',
434 'old mode ',
435 'new mode ',
436 'similarity index ',
437 'copy from ',
438 'copy to ',
439 'rename ',
440 )
441
442 line_endings = ('\r\n', '\n\r', '\n', '\r')
443
444 newfile_prefix_re = \
445 re.compile(r'''^
446 index\ 0+\.\.
447 ''',
448 re.VERBOSE)
449
450 def added_line_error(self, msg, line):
451 lines = [ msg ]
452 if self.filename is not None:
453 lines.append('File: ' + self.filename)
454 lines.append('Line: ' + line)
455
456 self.error(*lines)
457
458 old_debug_re = \
459 re.compile(r'''
460 DEBUG \s* \( \s* \( \s*
461 (?: DEBUG_[A-Z_]+ \s* \| \s*)*
462 EFI_D_ ([A-Z_]+)
463 ''',
464 re.VERBOSE)
465
466 def check_added_line(self, line):
467 eol = ''
468 for an_eol in self.line_endings:
469 if line.endswith(an_eol):
470 eol = an_eol
471 line = line[:-len(eol)]
472
473 stripped = line.rstrip()
474
475 if self.force_crlf and eol != '\r\n' and (line.find('Subproject commit') == -1):
476 self.added_line_error('Line ending (%s) is not CRLF' % repr(eol),
477 line)
478 if self.force_notabs and '\t' in line:
479 self.added_line_error('Tab character used', line)
480 if len(stripped) < len(line):
481 self.added_line_error('Trailing whitespace found', line)
482
483 mo = self.old_debug_re.search(line)
484 if mo is not None:
485 self.added_line_error('EFI_D_' + mo.group(1) + ' was used, '
486 'but DEBUG_' + mo.group(1) +
487 ' is now recommended', line)
488
489 split_diff_re = re.compile(r'''
490 (?P<cmd>
491 ^ diff \s+ --git \s+ a/.+ \s+ b/.+ $
492 )
493 (?P<index>
494 ^ index \s+ .+ $
495 )
496 ''',
497 re.IGNORECASE | re.VERBOSE | re.MULTILINE)
498
499 def format_error(self, err):
500 self.format_ok = False
501 err = 'Patch format error: ' + err
502 err2 = 'Line: ' + self.lines[self.line_num].rstrip()
503 self.error(err, err2)
504
505 def error(self, *err):
506 if self.ok and Verbose.level > Verbose.ONELINE:
507 print('Code format is not valid:')
508 self.ok = False
509 if Verbose.level < Verbose.NORMAL:
510 return
511 count = 0
512 for line in err:
513 prefix = (' *', ' ')[count > 0]
514 print(prefix, line)
515 count += 1
516
517 class CheckOnePatch:
518 """Checks the contents of a git email formatted patch.
519
520 Various checks are performed on both the commit message and the
521 patch content.
522 """
523
524 def __init__(self, name, patch):
525 self.patch = patch
526 self.find_patch_pieces()
527
528 email_check = EmailAddressCheck(self.author_email, 'Author')
529 email_ok = email_check.ok
530
531 msg_check = CommitMessageCheck(self.commit_subject, self.commit_msg, self.author_email)
532 msg_ok = msg_check.ok
533
534 diff_ok = True
535 if self.diff is not None:
536 diff_check = GitDiffCheck(self.diff)
537 diff_ok = diff_check.ok
538
539 self.ok = email_ok and msg_ok and diff_ok
540
541 if Verbose.level == Verbose.ONELINE:
542 if self.ok:
543 result = 'ok'
544 else:
545 result = list()
546 if not msg_ok:
547 result.append('commit message')
548 if not diff_ok:
549 result.append('diff content')
550 result = 'bad ' + ' and '.join(result)
551 print(name, result)
552
553
554 git_diff_re = re.compile(r'''
555 ^ diff \s+ --git \s+ a/.+ \s+ b/.+ $
556 ''',
557 re.IGNORECASE | re.VERBOSE | re.MULTILINE)
558
559 stat_re = \
560 re.compile(r'''
561 (?P<commit_message> [\s\S\r\n]* )
562 (?P<stat>
563 ^ --- $ [\r\n]+
564 (?: ^ \s+ .+ \s+ \| \s+ \d+ \s+ \+* \-*
565 $ [\r\n]+ )+
566 [\s\S\r\n]+
567 )
568 ''',
569 re.IGNORECASE | re.VERBOSE | re.MULTILINE)
570
571 subject_prefix_re = \
572 re.compile(r'''^
573 \s* (\[
574 [^\[\]]* # Allow all non-brackets
575 \])* \s*
576 ''',
577 re.VERBOSE)
578
579 def find_patch_pieces(self):
580 if sys.version_info < (3, 0):
581 patch = self.patch.encode('ascii', 'ignore')
582 else:
583 patch = self.patch
584
585 self.commit_msg = None
586 self.stat = None
587 self.commit_subject = None
588 self.commit_prefix = None
589 self.diff = None
590
591 if patch.startswith('diff --git'):
592 self.diff = patch
593 return
594
595 pmail = email.message_from_string(patch)
596 parts = list(pmail.walk())
597 assert(len(parts) == 1)
598 assert(parts[0].get_content_type() == 'text/plain')
599 content = parts[0].get_payload(decode=True).decode('utf-8', 'ignore')
600
601 mo = self.git_diff_re.search(content)
602 if mo is not None:
603 self.diff = content[mo.start():]
604 content = content[:mo.start()]
605
606 mo = self.stat_re.search(content)
607 if mo is None:
608 self.commit_msg = content
609 else:
610 self.stat = mo.group('stat')
611 self.commit_msg = mo.group('commit_message')
612 #
613 # Parse subject line from email header. The subject line may be
614 # composed of multiple parts with different encodings. Decode and
615 # combine all the parts to produce a single string with the contents of
616 # the decoded subject line.
617 #
618 parts = email.header.decode_header(pmail.get('subject'))
619 subject = ''
620 for (part, encoding) in parts:
621 if encoding:
622 part = part.decode(encoding)
623 else:
624 try:
625 part = part.decode()
626 except:
627 pass
628 subject = subject + part
629
630 self.commit_subject = subject.replace('\r\n', '')
631 self.commit_subject = self.commit_subject.replace('\n', '')
632 self.commit_subject = self.subject_prefix_re.sub('', self.commit_subject, 1)
633
634 self.author_email = pmail['from']
635
636 class CheckGitCommits:
637 """Reads patches from git based on the specified git revision range.
638
639 The patches are read from git, and then checked.
640 """
641
642 def __init__(self, rev_spec, max_count):
643 commits = self.read_commit_list_from_git(rev_spec, max_count)
644 if len(commits) == 1 and Verbose.level > Verbose.ONELINE:
645 commits = [ rev_spec ]
646 self.ok = True
647 blank_line = False
648 for commit in commits:
649 if Verbose.level > Verbose.ONELINE:
650 if blank_line:
651 print()
652 else:
653 blank_line = True
654 print('Checking git commit:', commit)
655 email = self.read_committer_email_address_from_git(commit)
656 self.ok &= EmailAddressCheck(email, 'Committer').ok
657 patch = self.read_patch_from_git(commit)
658 self.ok &= CheckOnePatch(commit, patch).ok
659 if not commits:
660 print("Couldn't find commit matching: '{}'".format(rev_spec))
661
662 def read_commit_list_from_git(self, rev_spec, max_count):
663 # Run git to get the commit patch
664 cmd = [ 'rev-list', '--abbrev-commit', '--no-walk' ]
665 if max_count is not None:
666 cmd.append('--max-count=' + str(max_count))
667 cmd.append(rev_spec)
668 out = self.run_git(*cmd)
669 return out.split() if out else []
670
671 def read_patch_from_git(self, commit):
672 # Run git to get the commit patch
673 return self.run_git('show', '--pretty=email', '--no-textconv',
674 '--no-use-mailmap', commit)
675
676 def read_committer_email_address_from_git(self, commit):
677 # Run git to get the committer email
678 return self.run_git('show', '--pretty=%cn <%ce>', '--no-patch',
679 '--no-use-mailmap', commit)
680
681 def run_git(self, *args):
682 cmd = [ 'git' ]
683 cmd += args
684 p = subprocess.Popen(cmd,
685 stdout=subprocess.PIPE,
686 stderr=subprocess.STDOUT)
687 Result = p.communicate()
688 return Result[0].decode('utf-8', 'ignore') if Result[0] and Result[0].find(b"fatal")!=0 else None
689
690 class CheckOnePatchFile:
691 """Performs a patch check for a single file.
692
693 stdin is used when the filename is '-'.
694 """
695
696 def __init__(self, patch_filename):
697 if patch_filename == '-':
698 patch = sys.stdin.read()
699 patch_filename = 'stdin'
700 else:
701 f = open(patch_filename, 'rb')
702 patch = f.read().decode('utf-8', 'ignore')
703 f.close()
704 if Verbose.level > Verbose.ONELINE:
705 print('Checking patch file:', patch_filename)
706 self.ok = CheckOnePatch(patch_filename, patch).ok
707
708 class CheckOneArg:
709 """Performs a patch check for a single command line argument.
710
711 The argument will be handed off to a file or git-commit based
712 checker.
713 """
714
715 def __init__(self, param, max_count=None):
716 self.ok = True
717 if param == '-' or os.path.exists(param):
718 checker = CheckOnePatchFile(param)
719 else:
720 checker = CheckGitCommits(param, max_count)
721 self.ok = checker.ok
722
723 class PatchCheckApp:
724 """Checks patches based on the command line arguments."""
725
726 def __init__(self):
727 self.parse_options()
728 patches = self.args.patches
729
730 if len(patches) == 0:
731 patches = [ 'HEAD' ]
732
733 self.ok = True
734 self.count = None
735 for patch in patches:
736 self.process_one_arg(patch)
737
738 if self.count is not None:
739 self.process_one_arg('HEAD')
740
741 if self.ok:
742 self.retval = 0
743 else:
744 self.retval = -1
745
746 def process_one_arg(self, arg):
747 if len(arg) >= 2 and arg[0] == '-':
748 try:
749 self.count = int(arg[1:])
750 return
751 except ValueError:
752 pass
753 self.ok &= CheckOneArg(arg, self.count).ok
754 self.count = None
755
756 def parse_options(self):
757 parser = argparse.ArgumentParser(description=__copyright__)
758 parser.add_argument('--version', action='version',
759 version='%(prog)s ' + VersionNumber)
760 parser.add_argument('patches', nargs='*',
761 help='[patch file | git rev list]')
762 group = parser.add_mutually_exclusive_group()
763 group.add_argument("--oneline",
764 action="store_true",
765 help="Print one result per line")
766 group.add_argument("--silent",
767 action="store_true",
768 help="Print nothing")
769 self.args = parser.parse_args()
770 if self.args.oneline:
771 Verbose.level = Verbose.ONELINE
772 if self.args.silent:
773 Verbose.level = Verbose.SILENT
774
775 if __name__ == "__main__":
776 sys.exit(PatchCheckApp().retval)