BaseTools/Scripts/PatchCheck.py

   1 ## @file
   2 #  Check a patch for various format issues
   3 #
   4 #  Copyright (c) 2015 - 2019, Intel Corporation. All rights reserved.<BR>
   5 #
   6 #  SPDX-License-Identifier: BSD-2-Clause-Patent
   7 #
   8
   9 from __future__ import print_function
  10
  11 VersionNumber = '0.1'
  12 __copyright__ = "Copyright (c) 2015 - 2016, Intel Corporation  All rights reserved."
  13
  14 import email
  15 import argparse
  16 import os
  17 import re
  18 import subprocess
  19 import sys
  20
  21 class Verbose:
  22     SILENT, ONELINE, NORMAL = range(3)
  23     level = NORMAL
  24
  25 class CommitMessageCheck:
  26     """Checks the contents of a git commit message."""
  27
  28     def __init__(self, subject, message):
  29         self.ok = True
  30
  31         if subject is None and  message is None:
  32             self.error('Commit message is missing!')
  33             return
  34
  35         self.subject = subject
  36         self.msg = message
  37
  38         self.check_contributed_under()
  39         self.check_signed_off_by()
  40         self.check_misc_signatures()
  41         self.check_overall_format()
  42         self.report_message_result()
  43
  44     url = 'https://github.com/tianocore/tianocore.github.io/wiki/Commit-Message-Format'
  45
  46     def report_message_result(self):
  47         if Verbose.level < Verbose.NORMAL:
  48             return
  49         if self.ok:
  50             # All checks passed
  51             return_code = 0
  52             print('The commit message format passed all checks.')
  53         else:
  54             return_code = 1
  55         if not self.ok:
  56             print(self.url)
  57
  58     def error(self, *err):
  59         if self.ok and Verbose.level > Verbose.ONELINE:
  60             print('The commit message format is not valid:')
  61         self.ok = False
  62         if Verbose.level < Verbose.NORMAL:
  63             return
  64         count = 0
  65         for line in err:
  66             prefix = (' *', '  ')[count > 0]
  67             print(prefix, line)
  68             count += 1
  69
  70     # Find 'contributed-under:' at the start of a line ignoring case and
  71     # requires ':' to be present.  Matches if there is white space before
  72     # the tag or between the tag and the ':'.
  73     contributed_under_re = \
  74         re.compile(r'^\s*contributed-under\s*:', re.MULTILINE|re.IGNORECASE)
  75
  76     def check_contributed_under(self):
  77         match = self.contributed_under_re.search(self.msg)
  78         if match is not None:
  79             self.error('Contributed-under! (Note: this must be ' +
  80                        'removed by the code contributor!)')
  81
  82     @staticmethod
  83     def make_signature_re(sig, re_input=False):
  84         if re_input:
  85             sub_re = sig
  86         else:
  87             sub_re = sig.replace('-', r'[-\s]+')
  88         re_str = (r'^(?P<tag>' + sub_re +
  89                   r')(\s*):(\s*)(?P<value>\S.*?)(?:\s*)$')
  90         try:
  91             return re.compile(re_str, re.MULTILINE|re.IGNORECASE)
  92         except Exception:
  93             print("Tried to compile re:", re_str)
  94             raise
  95
  96     sig_block_re = \
  97         re.compile(r'''^
  98                         (?: (?P<tag>[^:]+) \s* : \s*
  99                             (?P<value>\S.*?) )
 100                             |
 101                         (?: \[ (?P<updater>[^:]+) \s* : \s*
 102                                (?P<note>.+?) \s* \] )
 103                     \s* $''',
 104                    re.VERBOSE | re.MULTILINE)
 105
 106     def find_signatures(self, sig):
 107         if not sig.endswith('-by') and sig != 'Cc':
 108             sig += '-by'
 109         regex = self.make_signature_re(sig)
 110
 111         sigs = regex.findall(self.msg)
 112
 113         bad_case_sigs = filter(lambda m: m[0] != sig, sigs)
 114         for s in bad_case_sigs:
 115             self.error("'" +s[0] + "' should be '" + sig + "'")
 116
 117         for s in sigs:
 118             if s[1] != '':
 119                 self.error('There should be no spaces between ' + sig +
 120                            " and the ':'")
 121             if s[2] != ' ':
 122                 self.error("There should be a space after '" + sig + ":'")
 123
 124             self.check_email_address(s[3])
 125
 126         return sigs
 127
 128     email_re1 = re.compile(r'(?:\s*)(.*?)(\s*)<(.+)>\s*$',
 129                            re.MULTILINE|re.IGNORECASE)
 130
 131     def check_email_address(self, email):
 132         email = email.strip()
 133         mo = self.email_re1.match(email)
 134         if mo is None:
 135             self.error("Email format is invalid: " + email.strip())
 136             return
 137
 138         name = mo.group(1).strip()
 139         if name == '':
 140             self.error("Name is not provided with email address: " +
 141                        email)
 142         else:
 143             quoted = len(name) > 2 and name[0] == '"' and name[-1] == '"'
 144             if name.find(',') >= 0 and not quoted:
 145                 self.error('Add quotes (") around name with a comma: ' +
 146                            name)
 147
 148         if mo.group(2) == '':
 149             self.error("There should be a space between the name and " +
 150                        "email address: " + email)
 151
 152         if mo.group(3).find(' ') >= 0:
 153             self.error("The email address cannot contain a space: " +
 154                        mo.group(3))
 155
 156     def check_signed_off_by(self):
 157         sob='Signed-off-by'
 158         if self.msg.find(sob) < 0:
 159             self.error('Missing Signed-off-by! (Note: this must be ' +
 160                        'added by the code contributor!)')
 161             return
 162
 163         sobs = self.find_signatures('Signed-off')
 164
 165         if len(sobs) == 0:
 166             self.error('Invalid Signed-off-by format!')
 167             return
 168
 169     sig_types = (
 170         'Reviewed',
 171         'Reported',
 172         'Tested',
 173         'Suggested',
 174         'Acked',
 175         'Cc'
 176         )
 177
 178     def check_misc_signatures(self):
 179         for sig in self.sig_types:
 180             self.find_signatures(sig)
 181
 182     def check_overall_format(self):
 183         lines = self.msg.splitlines()
 184
 185         if len(lines) >= 1 and lines[0].endswith('\r\n'):
 186             empty_line = '\r\n'
 187         else:
 188             empty_line = '\n'
 189
 190         lines.insert(0, empty_line)
 191         lines.insert(0, self.subject + empty_line)
 192
 193         count = len(lines)
 194
 195         if count <= 0:
 196             self.error('Empty commit message!')
 197             return
 198
 199         if count >= 1 and len(lines[0]) >= 72:
 200             self.error('First line of commit message (subject line) ' +
 201                        'is too long.')
 202
 203         if count >= 1 and len(lines[0].strip()) == 0:
 204             self.error('First line of commit message (subject line) ' +
 205                        'is empty.')
 206
 207         if count >= 2 and lines[1].strip() != '':
 208             self.error('Second line of commit message should be ' +
 209                        'empty.')
 210
 211         for i in range(2, count):
 212             if (len(lines[i]) >= 76 and
 213                 len(lines[i].split()) > 1 and
 214                 not lines[i].startswith('git-svn-id:')):
 215                 self.error('Line %d of commit message is too long.' % (i + 1))
 216
 217         last_sig_line = None
 218         for i in range(count - 1, 0, -1):
 219             line = lines[i]
 220             mo = self.sig_block_re.match(line)
 221             if mo is None:
 222                 if line.strip() == '':
 223                     break
 224                 elif last_sig_line is not None:
 225                     err2 = 'Add empty line before "%s"?' % last_sig_line
 226                     self.error('The line before the signature block ' +
 227                                'should be empty', err2)
 228                 else:
 229                     self.error('The signature block was not found')
 230                 break
 231             last_sig_line = line.strip()
 232
 233 (START, PRE_PATCH, PATCH) = range(3)
 234
 235 class GitDiffCheck:
 236     """Checks the contents of a git diff."""
 237
 238     def __init__(self, diff):
 239         self.ok = True
 240         self.format_ok = True
 241         self.lines = diff.splitlines(True)
 242         self.count = len(self.lines)
 243         self.line_num = 0
 244         self.state = START
 245         self.new_bin = []
 246         while self.line_num < self.count and self.format_ok:
 247             line_num = self.line_num
 248             self.run()
 249             assert(self.line_num > line_num)
 250         self.report_message_result()
 251
 252     def report_message_result(self):
 253         if Verbose.level < Verbose.NORMAL:
 254             return
 255         if self.ok:
 256             print('The code passed all checks.')
 257         if self.new_bin:
 258             print('\nWARNING - The following binary files will be added ' +
 259                   'into the repository:')
 260             for binary in self.new_bin:
 261                 print('  ' + binary)
 262
 263     def run(self):
 264         line = self.lines[self.line_num]
 265
 266         if self.state in (PRE_PATCH, PATCH):
 267             if line.startswith('diff --git'):
 268                 self.state = START
 269         if self.state == PATCH:
 270             if line.startswith('@@ '):
 271                 self.state = PRE_PATCH
 272             elif len(line) >= 1 and line[0] not in ' -+' and \
 273                  not line.startswith('\r\n') and  \
 274                  not line.startswith(r'\ No newline ') and not self.binary:
 275                 for line in self.lines[self.line_num + 1:]:
 276                     if line.startswith('diff --git'):
 277                         self.format_error('diff found after end of patch')
 278                         break
 279                 self.line_num = self.count
 280                 return
 281
 282         if self.state == START:
 283             if line.startswith('diff --git'):
 284                 self.state = PRE_PATCH
 285                 self.filename = line[13:].split(' ', 1)[0]
 286                 self.is_newfile = False
 287                 self.force_crlf = not self.filename.endswith('.sh')
 288             elif len(line.rstrip()) != 0:
 289                 self.format_error("didn't find diff command")
 290             self.line_num += 1
 291         elif self.state == PRE_PATCH:
 292             if line.startswith('@@ '):
 293                 self.state = PATCH
 294                 self.binary = False
 295             elif line.startswith('GIT binary patch') or \
 296                  line.startswith('Binary files'):
 297                 self.state = PATCH
 298                 self.binary = True
 299                 if self.is_newfile:
 300                     self.new_bin.append(self.filename)
 301             else:
 302                 ok = False
 303                 self.is_newfile = self.newfile_prefix_re.match(line)
 304                 for pfx in self.pre_patch_prefixes:
 305                     if line.startswith(pfx):
 306                         ok = True
 307                 if not ok:
 308                     self.format_error("didn't find diff hunk marker (@@)")
 309             self.line_num += 1
 310         elif self.state == PATCH:
 311             if self.binary:
 312                 pass
 313             elif line.startswith('-'):
 314                 pass
 315             elif line.startswith('+'):
 316                 self.check_added_line(line[1:])
 317             elif line.startswith('\r\n'):
 318                 pass
 319             elif line.startswith(r'\ No newline '):
 320                 pass
 321             elif not line.startswith(' '):
 322                 self.format_error("unexpected patch line")
 323             self.line_num += 1
 324
 325     pre_patch_prefixes = (
 326         '--- ',
 327         '+++ ',
 328         'index ',
 329         'new file ',
 330         'deleted file ',
 331         'old mode ',
 332         'new mode ',
 333         'similarity index ',
 334         'copy from ',
 335         'copy to ',
 336         'rename ',
 337         )
 338
 339     line_endings = ('\r\n', '\n\r', '\n', '\r')
 340
 341     newfile_prefix_re = \
 342         re.compile(r'''^
 343                        index\ 0+\.\.
 344                    ''',
 345                    re.VERBOSE)
 346
 347     def added_line_error(self, msg, line):
 348         lines = [ msg ]
 349         if self.filename is not None:
 350             lines.append('File: ' + self.filename)
 351         lines.append('Line: ' + line)
 352
 353         self.error(*lines)
 354
 355     old_debug_re = \
 356         re.compile(r'''
 357                         DEBUG \s* \( \s* \( \s*
 358                         (?: DEBUG_[A-Z_]+ \s* \| \s*)*
 359                         EFI_D_ ([A-Z_]+)
 360                    ''',
 361                    re.VERBOSE)
 362
 363     def check_added_line(self, line):
 364         eol = ''
 365         for an_eol in self.line_endings:
 366             if line.endswith(an_eol):
 367                 eol = an_eol
 368                 line = line[:-len(eol)]
 369
 370         stripped = line.rstrip()
 371
 372         if self.force_crlf and eol != '\r\n':
 373             self.added_line_error('Line ending (%s) is not CRLF' % repr(eol),
 374                                   line)
 375         if '\t' in line:
 376             self.added_line_error('Tab character used', line)
 377         if len(stripped) < len(line):
 378             self.added_line_error('Trailing whitespace found', line)
 379
 380         mo = self.old_debug_re.search(line)
 381         if mo is not None:
 382             self.added_line_error('EFI_D_' + mo.group(1) + ' was used, '
 383                                   'but DEBUG_' + mo.group(1) +
 384                                   ' is now recommended', line)
 385
 386     split_diff_re = re.compile(r'''
 387                                    (?P<cmd>
 388                                        ^ diff \s+ --git \s+ a/.+ \s+ b/.+ $
 389                                    )
 390                                    (?P<index>
 391                                        ^ index \s+ .+ $
 392                                    )
 393                                ''',
 394                                re.IGNORECASE | re.VERBOSE | re.MULTILINE)
 395
 396     def format_error(self, err):
 397         self.format_ok = False
 398         err = 'Patch format error: ' + err
 399         err2 = 'Line: ' + self.lines[self.line_num].rstrip()
 400         self.error(err, err2)
 401
 402     def error(self, *err):
 403         if self.ok and Verbose.level > Verbose.ONELINE:
 404             print('Code format is not valid:')
 405         self.ok = False
 406         if Verbose.level < Verbose.NORMAL:
 407             return
 408         count = 0
 409         for line in err:
 410             prefix = (' *', '  ')[count > 0]
 411             print(prefix, line)
 412             count += 1
 413
 414 class CheckOnePatch:
 415     """Checks the contents of a git email formatted patch.
 416
 417     Various checks are performed on both the commit message and the
 418     patch content.
 419     """
 420
 421     def __init__(self, name, patch):
 422         self.patch = patch
 423         self.find_patch_pieces()
 424
 425         msg_check = CommitMessageCheck(self.commit_subject, self.commit_msg)
 426         msg_ok = msg_check.ok
 427
 428         diff_ok = True
 429         if self.diff is not None:
 430             diff_check = GitDiffCheck(self.diff)
 431             diff_ok = diff_check.ok
 432
 433         self.ok = msg_ok and diff_ok
 434
 435         if Verbose.level == Verbose.ONELINE:
 436             if self.ok:
 437                 result = 'ok'
 438             else:
 439                 result = list()
 440                 if not msg_ok:
 441                     result.append('commit message')
 442                 if not diff_ok:
 443                     result.append('diff content')
 444                 result = 'bad ' + ' and '.join(result)
 445             print(name, result)
 446
 447
 448     git_diff_re = re.compile(r'''
 449                                  ^ diff \s+ --git \s+ a/.+ \s+ b/.+ $
 450                              ''',
 451                              re.IGNORECASE | re.VERBOSE | re.MULTILINE)
 452
 453     stat_re = \
 454         re.compile(r'''
 455                        (?P<commit_message> [\s\S\r\n]* )
 456                        (?P<stat>
 457                            ^ --- $ [\r\n]+
 458                            (?: ^ \s+ .+ \s+ \| \s+ \d+ \s+ \+* \-*
 459                                $ [\r\n]+ )+
 460                            [\s\S\r\n]+
 461                        )
 462                    ''',
 463                    re.IGNORECASE | re.VERBOSE | re.MULTILINE)
 464
 465     subject_prefix_re = \
 466         re.compile(r'''^
 467                        \s* (\[
 468                         [^\[\]]* # Allow all non-brackets
 469                        \])* \s*
 470                    ''',
 471                    re.VERBOSE)
 472
 473     def find_patch_pieces(self):
 474         if sys.version_info < (3, 0):
 475             patch = self.patch.encode('ascii', 'ignore')
 476         else:
 477             patch = self.patch
 478
 479         self.commit_msg = None
 480         self.stat = None
 481         self.commit_subject = None
 482         self.commit_prefix = None
 483         self.diff = None
 484
 485         if patch.startswith('diff --git'):
 486             self.diff = patch
 487             return
 488
 489         pmail = email.message_from_string(patch)
 490         parts = list(pmail.walk())
 491         assert(len(parts) == 1)
 492         assert(parts[0].get_content_type() == 'text/plain')
 493         content = parts[0].get_payload(decode=True).decode('utf-8', 'ignore')
 494
 495         mo = self.git_diff_re.search(content)
 496         if mo is not None:
 497             self.diff = content[mo.start():]
 498             content = content[:mo.start()]
 499
 500         mo = self.stat_re.search(content)
 501         if mo is None:
 502             self.commit_msg = content
 503         else:
 504             self.stat = mo.group('stat')
 505             self.commit_msg = mo.group('commit_message')
 506
 507         self.commit_subject = pmail['subject'].replace('\r\n', '')
 508         self.commit_subject = self.commit_subject.replace('\n', '')
 509         self.commit_subject = self.subject_prefix_re.sub('', self.commit_subject, 1)
 510
 511 class CheckGitCommits:
 512     """Reads patches from git based on the specified git revision range.
 513
 514     The patches are read from git, and then checked.
 515     """
 516
 517     def __init__(self, rev_spec, max_count):
 518         commits = self.read_commit_list_from_git(rev_spec, max_count)
 519         if len(commits) == 1 and Verbose.level > Verbose.ONELINE:
 520             commits = [ rev_spec ]
 521         self.ok = True
 522         blank_line = False
 523         for commit in commits:
 524             if Verbose.level > Verbose.ONELINE:
 525                 if blank_line:
 526                     print()
 527                 else:
 528                     blank_line = True
 529                 print('Checking git commit:', commit)
 530             patch = self.read_patch_from_git(commit)
 531             self.ok &= CheckOnePatch(commit, patch).ok
 532         if not commits:
 533             print("Couldn't find commit matching: '{}'".format(rev_spec))
 534
 535     def read_commit_list_from_git(self, rev_spec, max_count):
 536         # Run git to get the commit patch
 537         cmd = [ 'rev-list', '--abbrev-commit', '--no-walk' ]
 538         if max_count is not None:
 539             cmd.append('--max-count=' + str(max_count))
 540         cmd.append(rev_spec)
 541         out = self.run_git(*cmd)
 542         return out.split() if out else []
 543
 544     def read_patch_from_git(self, commit):
 545         # Run git to get the commit patch
 546         return self.run_git('show', '--pretty=email', commit)
 547
 548     def run_git(self, *args):
 549         cmd = [ 'git' ]
 550         cmd += args
 551         p = subprocess.Popen(cmd,
 552                      stdout=subprocess.PIPE,
 553                      stderr=subprocess.STDOUT)
 554         Result = p.communicate()
 555         return Result[0].decode('utf-8', 'ignore') if Result[0] and Result[0].find(b"fatal")!=0 else None
 556
 557 class CheckOnePatchFile:
 558     """Performs a patch check for a single file.
 559
 560     stdin is used when the filename is '-'.
 561     """
 562
 563     def __init__(self, patch_filename):
 564         if patch_filename == '-':
 565             patch = sys.stdin.read()
 566             patch_filename = 'stdin'
 567         else:
 568             f = open(patch_filename, 'rb')
 569             patch = f.read().decode('utf-8', 'ignore')
 570             f.close()
 571         if Verbose.level > Verbose.ONELINE:
 572             print('Checking patch file:', patch_filename)
 573         self.ok = CheckOnePatch(patch_filename, patch).ok
 574
 575 class CheckOneArg:
 576     """Performs a patch check for a single command line argument.
 577
 578     The argument will be handed off to a file or git-commit based
 579     checker.
 580     """
 581
 582     def __init__(self, param, max_count=None):
 583         self.ok = True
 584         if param == '-' or os.path.exists(param):
 585             checker = CheckOnePatchFile(param)
 586         else:
 587             checker = CheckGitCommits(param, max_count)
 588         self.ok = checker.ok
 589
 590 class PatchCheckApp:
 591     """Checks patches based on the command line arguments."""
 592
 593     def __init__(self):
 594         self.parse_options()
 595         patches = self.args.patches
 596
 597         if len(patches) == 0:
 598             patches = [ 'HEAD' ]
 599
 600         self.ok = True
 601         self.count = None
 602         for patch in patches:
 603             self.process_one_arg(patch)
 604
 605         if self.count is not None:
 606             self.process_one_arg('HEAD')
 607
 608         if self.ok:
 609             self.retval = 0
 610         else:
 611             self.retval = -1
 612
 613     def process_one_arg(self, arg):
 614         if len(arg) >= 2 and arg[0] == '-':
 615             try:
 616                 self.count = int(arg[1:])
 617                 return
 618             except ValueError:
 619                 pass
 620         self.ok &= CheckOneArg(arg, self.count).ok
 621         self.count = None
 622
 623     def parse_options(self):
 624         parser = argparse.ArgumentParser(description=__copyright__)
 625         parser.add_argument('--version', action='version',
 626                             version='%(prog)s ' + VersionNumber)
 627         parser.add_argument('patches', nargs='*',
 628                             help='[patch file | git rev list]')
 629         group = parser.add_mutually_exclusive_group()
 630         group.add_argument("--oneline",
 631                            action="store_true",
 632                            help="Print one result per line")
 633         group.add_argument("--silent",
 634                            action="store_true",
 635                            help="Print nothing")
 636         self.args = parser.parse_args()
 637         if self.args.oneline:
 638             Verbose.level = Verbose.ONELINE
 639         if self.args.silent:
 640             Verbose.level = Verbose.SILENT
 641
 642 if __name__ == "__main__":
 643     sys.exit(PatchCheckApp().retval)