BaseTools/Scripts/PatchCheck.py

   1 ## @file
   2 #  Check a patch for various format issues
   3 #
   4 #  Copyright (c) 2015 - 2019, Intel Corporation. All rights reserved.<BR>
   5 #
   6 #  SPDX-License-Identifier: BSD-2-Clause-Patent
   7 #
   8
   9 from __future__ import print_function
  10
  11 VersionNumber = '0.1'
  12 __copyright__ = "Copyright (c) 2015 - 2016, Intel Corporation  All rights reserved."
  13
  14 import email
  15 import argparse
  16 import os
  17 import re
  18 import subprocess
  19 import sys
  20
  21 class Verbose:
  22     SILENT, ONELINE, NORMAL = range(3)
  23     level = NORMAL
  24
  25 class CommitMessageCheck:
  26     """Checks the contents of a git commit message."""
  27
  28     def __init__(self, subject, message):
  29         self.ok = True
  30
  31         if subject is None and  message is None:
  32             self.error('Commit message is missing!')
  33             return
  34
  35         self.subject = subject
  36         self.msg = message
  37
  38         self.check_contributed_under()
  39         self.check_signed_off_by()
  40         self.check_misc_signatures()
  41         self.check_overall_format()
  42         self.report_message_result()
  43
  44     url = 'https://github.com/tianocore/tianocore.github.io/wiki/Commit-Message-Format'
  45
  46     def report_message_result(self):
  47         if Verbose.level < Verbose.NORMAL:
  48             return
  49         if self.ok:
  50             # All checks passed
  51             return_code = 0
  52             print('The commit message format passed all checks.')
  53         else:
  54             return_code = 1
  55         if not self.ok:
  56             print(self.url)
  57
  58     def error(self, *err):
  59         if self.ok and Verbose.level > Verbose.ONELINE:
  60             print('The commit message format is not valid:')
  61         self.ok = False
  62         if Verbose.level < Verbose.NORMAL:
  63             return
  64         count = 0
  65         for line in err:
  66             prefix = (' *', '  ')[count > 0]
  67             print(prefix, line)
  68             count += 1
  69
  70     # Find 'contributed-under:' at the start of a line ignoring case and
  71     # requires ':' to be present.  Matches if there is white space before
  72     # the tag or between the tag and the ':'.
  73     contributed_under_re = \
  74         re.compile(r'^\s*contributed-under\s*:', re.MULTILINE|re.IGNORECASE)
  75
  76     def check_contributed_under(self):
  77         match = self.contributed_under_re.search(self.msg)
  78         if match is not None:
  79             self.error('Contributed-under! (Note: this must be ' +
  80                        'removed by the code contributor!)')
  81
  82     @staticmethod
  83     def make_signature_re(sig, re_input=False):
  84         if re_input:
  85             sub_re = sig
  86         else:
  87             sub_re = sig.replace('-', r'[-\s]+')
  88         re_str = (r'^(?P<tag>' + sub_re +
  89                   r')(\s*):(\s*)(?P<value>\S.*?)(?:\s*)$')
  90         try:
  91             return re.compile(re_str, re.MULTILINE|re.IGNORECASE)
  92         except Exception:
  93             print("Tried to compile re:", re_str)
  94             raise
  95
  96     sig_block_re = \
  97         re.compile(r'''^
  98                         (?: (?P<tag>[^:]+) \s* : \s*
  99                             (?P<value>\S.*?) )
 100                             |
 101                         (?: \[ (?P<updater>[^:]+) \s* : \s*
 102                                (?P<note>.+?) \s* \] )
 103                     \s* $''',
 104                    re.VERBOSE | re.MULTILINE)
 105
 106     def find_signatures(self, sig):
 107         if not sig.endswith('-by') and sig != 'Cc':
 108             sig += '-by'
 109         regex = self.make_signature_re(sig)
 110
 111         sigs = regex.findall(self.msg)
 112
 113         bad_case_sigs = filter(lambda m: m[0] != sig, sigs)
 114         for s in bad_case_sigs:
 115             self.error("'" +s[0] + "' should be '" + sig + "'")
 116
 117         for s in sigs:
 118             if s[1] != '':
 119                 self.error('There should be no spaces between ' + sig +
 120                            " and the ':'")
 121             if s[2] != ' ':
 122                 self.error("There should be a space after '" + sig + ":'")
 123
 124             self.check_email_address(s[3])
 125
 126         return sigs
 127
 128     email_re1 = re.compile(r'(?:\s*)(.*?)(\s*)<(.+)>\s*$',
 129                            re.MULTILINE|re.IGNORECASE)
 130
 131     def check_email_address(self, email):
 132         email = email.strip()
 133         mo = self.email_re1.match(email)
 134         if mo is None:
 135             self.error("Email format is invalid: " + email.strip())
 136             return
 137
 138         name = mo.group(1).strip()
 139         if name == '':
 140             self.error("Name is not provided with email address: " +
 141                        email)
 142         else:
 143             quoted = len(name) > 2 and name[0] == '"' and name[-1] == '"'
 144             if name.find(',') >= 0 and not quoted:
 145                 self.error('Add quotes (") around name with a comma: ' +
 146                            name)
 147
 148         if mo.group(2) == '':
 149             self.error("There should be a space between the name and " +
 150                        "email address: " + email)
 151
 152         if mo.group(3).find(' ') >= 0:
 153             self.error("The email address cannot contain a space: " +
 154                        mo.group(3))
 155
 156     def check_signed_off_by(self):
 157         sob='Signed-off-by'
 158         if self.msg.find(sob) < 0:
 159             self.error('Missing Signed-off-by! (Note: this must be ' +
 160                        'added by the code contributor!)')
 161             return
 162
 163         sobs = self.find_signatures('Signed-off')
 164
 165         if len(sobs) == 0:
 166             self.error('Invalid Signed-off-by format!')
 167             return
 168
 169     sig_types = (
 170         'Reviewed',
 171         'Reported',
 172         'Tested',
 173         'Suggested',
 174         'Acked',
 175         'Cc'
 176         )
 177
 178     def check_misc_signatures(self):
 179         for sig in self.sig_types:
 180             self.find_signatures(sig)
 181
 182     def check_overall_format(self):
 183         lines = self.msg.splitlines()
 184
 185         if len(lines) >= 1 and lines[0].endswith('\r\n'):
 186             empty_line = '\r\n'
 187         else:
 188             empty_line = '\n'
 189
 190         lines.insert(0, empty_line)
 191         lines.insert(0, self.subject + empty_line)
 192
 193         count = len(lines)
 194
 195         if count <= 0:
 196             self.error('Empty commit message!')
 197             return
 198
 199         if count >= 1 and len(lines[0]) >= 72:
 200             self.error('First line of commit message (subject line) ' +
 201                        'is too long.')
 202
 203         if count >= 1 and len(lines[0].strip()) == 0:
 204             self.error('First line of commit message (subject line) ' +
 205                        'is empty.')
 206
 207         if count >= 2 and lines[1].strip() != '':
 208             self.error('Second line of commit message should be ' +
 209                        'empty.')
 210
 211         for i in range(2, count):
 212             if (len(lines[i]) >= 76 and
 213                 len(lines[i].split()) > 1 and
 214                 not lines[i].startswith('git-svn-id:')):
 215                 self.error('Line %d of commit message is too long.' % (i + 1))
 216
 217         last_sig_line = None
 218         for i in range(count - 1, 0, -1):
 219             line = lines[i]
 220             mo = self.sig_block_re.match(line)
 221             if mo is None:
 222                 if line.strip() == '':
 223                     break
 224                 elif last_sig_line is not None:
 225                     err2 = 'Add empty line before "%s"?' % last_sig_line
 226                     self.error('The line before the signature block ' +
 227                                'should be empty', err2)
 228                 else:
 229                     self.error('The signature block was not found')
 230                 break
 231             last_sig_line = line.strip()
 232
 233 (START, PRE_PATCH, PATCH) = range(3)
 234
 235 class GitDiffCheck:
 236     """Checks the contents of a git diff."""
 237
 238     def __init__(self, diff):
 239         self.ok = True
 240         self.format_ok = True
 241         self.lines = diff.splitlines(True)
 242         self.count = len(self.lines)
 243         self.line_num = 0
 244         self.state = START
 245         self.new_bin = []
 246         while self.line_num < self.count and self.format_ok:
 247             line_num = self.line_num
 248             self.run()
 249             assert(self.line_num > line_num)
 250         self.report_message_result()
 251
 252     def report_message_result(self):
 253         if Verbose.level < Verbose.NORMAL:
 254             return
 255         if self.ok:
 256             print('The code passed all checks.')
 257         if self.new_bin:
 258             print('\nWARNING - The following binary files will be added ' +
 259                   'into the repository:')
 260             for binary in self.new_bin:
 261                 print('  ' + binary)
 262
 263     def run(self):
 264         line = self.lines[self.line_num]
 265
 266         if self.state in (PRE_PATCH, PATCH):
 267             if line.startswith('diff --git'):
 268                 self.state = START
 269         if self.state == PATCH:
 270             if line.startswith('@@ '):
 271                 self.state = PRE_PATCH
 272             elif len(line) >= 1 and line[0] not in ' -+' and \
 273                  not line.startswith('\r\n') and  \
 274                  not line.startswith(r'\ No newline ') and not self.binary:
 275                 for line in self.lines[self.line_num + 1:]:
 276                     if line.startswith('diff --git'):
 277                         self.format_error('diff found after end of patch')
 278                         break
 279                 self.line_num = self.count
 280                 return
 281
 282         if self.state == START:
 283             if line.startswith('diff --git'):
 284                 self.state = PRE_PATCH
 285                 self.filename = line[13:].split(' ', 1)[0]
 286                 self.is_newfile = False
 287                 self.force_crlf = not self.filename.endswith('.sh')
 288             elif len(line.rstrip()) != 0:
 289                 self.format_error("didn't find diff command")
 290             self.line_num += 1
 291         elif self.state == PRE_PATCH:
 292             if line.startswith('@@ '):
 293                 self.state = PATCH
 294                 self.binary = False
 295             elif line.startswith('GIT binary patch') or \
 296                  line.startswith('Binary files'):
 297                 self.state = PATCH
 298                 self.binary = True
 299                 if self.is_newfile:
 300                     self.new_bin.append(self.filename)
 301             else:
 302                 ok = False
 303                 self.is_newfile = self.newfile_prefix_re.match(line)
 304                 for pfx in self.pre_patch_prefixes:
 305                     if line.startswith(pfx):
 306                         ok = True
 307                 if not ok:
 308                     self.format_error("didn't find diff hunk marker (@@)")
 309             self.line_num += 1
 310         elif self.state == PATCH:
 311             if self.binary:
 312                 pass
 313             elif line.startswith('-'):
 314                 pass
 315             elif line.startswith('+'):
 316                 self.check_added_line(line[1:])
 317             elif line.startswith('\r\n'):
 318                 pass
 319             elif line.startswith(r'\ No newline '):
 320                 pass
 321             elif not line.startswith(' '):
 322                 self.format_error("unexpected patch line")
 323             self.line_num += 1
 324
 325     pre_patch_prefixes = (
 326         '--- ',
 327         '+++ ',
 328         'index ',
 329         'new file ',
 330         'deleted file ',
 331         'old mode ',
 332         'new mode ',
 333         'similarity index ',
 334         'rename ',
 335         )
 336
 337     line_endings = ('\r\n', '\n\r', '\n', '\r')
 338
 339     newfile_prefix_re = \
 340         re.compile(r'''^
 341                        index\ 0+\.\.
 342                    ''',
 343                    re.VERBOSE)
 344
 345     def added_line_error(self, msg, line):
 346         lines = [ msg ]
 347         if self.filename is not None:
 348             lines.append('File: ' + self.filename)
 349         lines.append('Line: ' + line)
 350
 351         self.error(*lines)
 352
 353     old_debug_re = \
 354         re.compile(r'''
 355                         DEBUG \s* \( \s* \( \s*
 356                         (?: DEBUG_[A-Z_]+ \s* \| \s*)*
 357                         EFI_D_ ([A-Z_]+)
 358                    ''',
 359                    re.VERBOSE)
 360
 361     def check_added_line(self, line):
 362         eol = ''
 363         for an_eol in self.line_endings:
 364             if line.endswith(an_eol):
 365                 eol = an_eol
 366                 line = line[:-len(eol)]
 367
 368         stripped = line.rstrip()
 369
 370         if self.force_crlf and eol != '\r\n':
 371             self.added_line_error('Line ending (%s) is not CRLF' % repr(eol),
 372                                   line)
 373         if '\t' in line:
 374             self.added_line_error('Tab character used', line)
 375         if len(stripped) < len(line):
 376             self.added_line_error('Trailing whitespace found', line)
 377
 378         mo = self.old_debug_re.search(line)
 379         if mo is not None:
 380             self.added_line_error('EFI_D_' + mo.group(1) + ' was used, '
 381                                   'but DEBUG_' + mo.group(1) +
 382                                   ' is now recommended', line)
 383
 384     split_diff_re = re.compile(r'''
 385                                    (?P<cmd>
 386                                        ^ diff \s+ --git \s+ a/.+ \s+ b/.+ $
 387                                    )
 388                                    (?P<index>
 389                                        ^ index \s+ .+ $
 390                                    )
 391                                ''',
 392                                re.IGNORECASE | re.VERBOSE | re.MULTILINE)
 393
 394     def format_error(self, err):
 395         self.format_ok = False
 396         err = 'Patch format error: ' + err
 397         err2 = 'Line: ' + self.lines[self.line_num].rstrip()
 398         self.error(err, err2)
 399
 400     def error(self, *err):
 401         if self.ok and Verbose.level > Verbose.ONELINE:
 402             print('Code format is not valid:')
 403         self.ok = False
 404         if Verbose.level < Verbose.NORMAL:
 405             return
 406         count = 0
 407         for line in err:
 408             prefix = (' *', '  ')[count > 0]
 409             print(prefix, line)
 410             count += 1
 411
 412 class CheckOnePatch:
 413     """Checks the contents of a git email formatted patch.
 414
 415     Various checks are performed on both the commit message and the
 416     patch content.
 417     """
 418
 419     def __init__(self, name, patch):
 420         self.patch = patch
 421         self.find_patch_pieces()
 422
 423         msg_check = CommitMessageCheck(self.commit_subject, self.commit_msg)
 424         msg_ok = msg_check.ok
 425
 426         diff_ok = True
 427         if self.diff is not None:
 428             diff_check = GitDiffCheck(self.diff)
 429             diff_ok = diff_check.ok
 430
 431         self.ok = msg_ok and diff_ok
 432
 433         if Verbose.level == Verbose.ONELINE:
 434             if self.ok:
 435                 result = 'ok'
 436             else:
 437                 result = list()
 438                 if not msg_ok:
 439                     result.append('commit message')
 440                 if not diff_ok:
 441                     result.append('diff content')
 442                 result = 'bad ' + ' and '.join(result)
 443             print(name, result)
 444
 445
 446     git_diff_re = re.compile(r'''
 447                                  ^ diff \s+ --git \s+ a/.+ \s+ b/.+ $
 448                              ''',
 449                              re.IGNORECASE | re.VERBOSE | re.MULTILINE)
 450
 451     stat_re = \
 452         re.compile(r'''
 453                        (?P<commit_message> [\s\S\r\n]* )
 454                        (?P<stat>
 455                            ^ --- $ [\r\n]+
 456                            (?: ^ \s+ .+ \s+ \| \s+ \d+ \s+ \+* \-*
 457                                $ [\r\n]+ )+
 458                            [\s\S\r\n]+
 459                        )
 460                    ''',
 461                    re.IGNORECASE | re.VERBOSE | re.MULTILINE)
 462
 463     subject_prefix_re = \
 464         re.compile(r'''^
 465                        \s* (\[
 466                         [^\[\]]* # Allow all non-brackets
 467                        \])* \s*
 468                    ''',
 469                    re.VERBOSE)
 470
 471     def find_patch_pieces(self):
 472         if sys.version_info < (3, 0):
 473             patch = self.patch.encode('ascii', 'ignore')
 474         else:
 475             patch = self.patch
 476
 477         self.commit_msg = None
 478         self.stat = None
 479         self.commit_subject = None
 480         self.commit_prefix = None
 481         self.diff = None
 482
 483         if patch.startswith('diff --git'):
 484             self.diff = patch
 485             return
 486
 487         pmail = email.message_from_string(patch)
 488         parts = list(pmail.walk())
 489         assert(len(parts) == 1)
 490         assert(parts[0].get_content_type() == 'text/plain')
 491         content = parts[0].get_payload(decode=True).decode('utf-8', 'ignore')
 492
 493         mo = self.git_diff_re.search(content)
 494         if mo is not None:
 495             self.diff = content[mo.start():]
 496             content = content[:mo.start()]
 497
 498         mo = self.stat_re.search(content)
 499         if mo is None:
 500             self.commit_msg = content
 501         else:
 502             self.stat = mo.group('stat')
 503             self.commit_msg = mo.group('commit_message')
 504
 505         self.commit_subject = pmail['subject'].replace('\r\n', '')
 506         self.commit_subject = self.commit_subject.replace('\n', '')
 507         self.commit_subject = self.subject_prefix_re.sub('', self.commit_subject, 1)
 508
 509 class CheckGitCommits:
 510     """Reads patches from git based on the specified git revision range.
 511
 512     The patches are read from git, and then checked.
 513     """
 514
 515     def __init__(self, rev_spec, max_count):
 516         commits = self.read_commit_list_from_git(rev_spec, max_count)
 517         if len(commits) == 1 and Verbose.level > Verbose.ONELINE:
 518             commits = [ rev_spec ]
 519         self.ok = True
 520         blank_line = False
 521         for commit in commits:
 522             if Verbose.level > Verbose.ONELINE:
 523                 if blank_line:
 524                     print()
 525                 else:
 526                     blank_line = True
 527                 print('Checking git commit:', commit)
 528             patch = self.read_patch_from_git(commit)
 529             self.ok &= CheckOnePatch(commit, patch).ok
 530         if not commits:
 531             print("Couldn't find commit matching: '{}'".format(rev_spec))
 532
 533     def read_commit_list_from_git(self, rev_spec, max_count):
 534         # Run git to get the commit patch
 535         cmd = [ 'rev-list', '--abbrev-commit', '--no-walk' ]
 536         if max_count is not None:
 537             cmd.append('--max-count=' + str(max_count))
 538         cmd.append(rev_spec)
 539         out = self.run_git(*cmd)
 540         return out.split() if out else []
 541
 542     def read_patch_from_git(self, commit):
 543         # Run git to get the commit patch
 544         return self.run_git('show', '--pretty=email', commit)
 545
 546     def run_git(self, *args):
 547         cmd = [ 'git' ]
 548         cmd += args
 549         p = subprocess.Popen(cmd,
 550                      stdout=subprocess.PIPE,
 551                      stderr=subprocess.STDOUT)
 552         Result = p.communicate()
 553         return Result[0].decode('utf-8', 'ignore') if Result[0] and Result[0].find(b"fatal")!=0 else None
 554
 555 class CheckOnePatchFile:
 556     """Performs a patch check for a single file.
 557
 558     stdin is used when the filename is '-'.
 559     """
 560
 561     def __init__(self, patch_filename):
 562         if patch_filename == '-':
 563             patch = sys.stdin.read()
 564             patch_filename = 'stdin'
 565         else:
 566             f = open(patch_filename, 'rb')
 567             patch = f.read().decode('utf-8', 'ignore')
 568             f.close()
 569         if Verbose.level > Verbose.ONELINE:
 570             print('Checking patch file:', patch_filename)
 571         self.ok = CheckOnePatch(patch_filename, patch).ok
 572
 573 class CheckOneArg:
 574     """Performs a patch check for a single command line argument.
 575
 576     The argument will be handed off to a file or git-commit based
 577     checker.
 578     """
 579
 580     def __init__(self, param, max_count=None):
 581         self.ok = True
 582         if param == '-' or os.path.exists(param):
 583             checker = CheckOnePatchFile(param)
 584         else:
 585             checker = CheckGitCommits(param, max_count)
 586         self.ok = checker.ok
 587
 588 class PatchCheckApp:
 589     """Checks patches based on the command line arguments."""
 590
 591     def __init__(self):
 592         self.parse_options()
 593         patches = self.args.patches
 594
 595         if len(patches) == 0:
 596             patches = [ 'HEAD' ]
 597
 598         self.ok = True
 599         self.count = None
 600         for patch in patches:
 601             self.process_one_arg(patch)
 602
 603         if self.count is not None:
 604             self.process_one_arg('HEAD')
 605
 606         if self.ok:
 607             self.retval = 0
 608         else:
 609             self.retval = -1
 610
 611     def process_one_arg(self, arg):
 612         if len(arg) >= 2 and arg[0] == '-':
 613             try:
 614                 self.count = int(arg[1:])
 615                 return
 616             except ValueError:
 617                 pass
 618         self.ok &= CheckOneArg(arg, self.count).ok
 619         self.count = None
 620
 621     def parse_options(self):
 622         parser = argparse.ArgumentParser(description=__copyright__)
 623         parser.add_argument('--version', action='version',
 624                             version='%(prog)s ' + VersionNumber)
 625         parser.add_argument('patches', nargs='*',
 626                             help='[patch file | git rev list]')
 627         group = parser.add_mutually_exclusive_group()
 628         group.add_argument("--oneline",
 629                            action="store_true",
 630                            help="Print one result per line")
 631         group.add_argument("--silent",
 632                            action="store_true",
 633                            help="Print nothing")
 634         self.args = parser.parse_args()
 635         if self.args.oneline:
 636             Verbose.level = Verbose.ONELINE
 637         if self.args.silent:
 638             Verbose.level = Verbose.SILENT
 639
 640 if __name__ == "__main__":
 641     sys.exit(PatchCheckApp().retval)