BaseTools/Scripts/PatchCheck.py

   1 ## @file
   2 #  Check a patch for various format issues
   3 #
   4 #  Copyright (c) 2015 - 2019, Intel Corporation. All rights reserved.<BR>
   5 #
   6 #  SPDX-License-Identifier: BSD-2-Clause-Patent
   7 #
   8
   9 from __future__ import print_function
  10
  11 VersionNumber = '0.1'
  12 __copyright__ = "Copyright (c) 2015 - 2016, Intel Corporation  All rights reserved."
  13
  14 import email
  15 import argparse
  16 import os
  17 import re
  18 import subprocess
  19 import sys
  20
  21 class Verbose:
  22     SILENT, ONELINE, NORMAL = range(3)
  23     level = NORMAL
  24
  25 class CommitMessageCheck:
  26     """Checks the contents of a git commit message."""
  27
  28     def __init__(self, subject, message):
  29         self.ok = True
  30
  31         if subject is None and  message is None:
  32             self.error('Commit message is missing!')
  33             return
  34
  35         self.subject = subject
  36         self.msg = message
  37
  38         self.check_contributed_under()
  39         self.check_signed_off_by()
  40         self.check_misc_signatures()
  41         self.check_overall_format()
  42         self.report_message_result()
  43
  44     url = 'https://github.com/tianocore/tianocore.github.io/wiki/Commit-Message-Format'
  45
  46     def report_message_result(self):
  47         if Verbose.level < Verbose.NORMAL:
  48             return
  49         if self.ok:
  50             # All checks passed
  51             return_code = 0
  52             print('The commit message format passed all checks.')
  53         else:
  54             return_code = 1
  55         if not self.ok:
  56             print(self.url)
  57
  58     def error(self, *err):
  59         if self.ok and Verbose.level > Verbose.ONELINE:
  60             print('The commit message format is not valid:')
  61         self.ok = False
  62         if Verbose.level < Verbose.NORMAL:
  63             return
  64         count = 0
  65         for line in err:
  66             prefix = (' *', '  ')[count > 0]
  67             print(prefix, line)
  68             count += 1
  69
  70     # Find 'contributed-under:' at the start of a line ignoring case and
  71     # requires ':' to be present.  Matches if there is white space before
  72     # the tag or between the tag and the ':'.
  73     contributed_under_re = \
  74         re.compile(r'^\s*contributed-under\s*:', re.MULTILINE|re.IGNORECASE)
  75
  76     def check_contributed_under(self):
  77         match = self.contributed_under_re.search(self.msg)
  78         if match is not None:
  79             self.error('Contributed-under! (Note: this must be ' +
  80                        'removed by the code contributor!)')
  81
  82     @staticmethod
  83     def make_signature_re(sig, re_input=False):
  84         if re_input:
  85             sub_re = sig
  86         else:
  87             sub_re = sig.replace('-', r'[-\s]+')
  88         re_str = (r'^(?P<tag>' + sub_re +
  89                   r')(\s*):(\s*)(?P<value>\S.*?)(?:\s*)$')
  90         try:
  91             return re.compile(re_str, re.MULTILINE|re.IGNORECASE)
  92         except Exception:
  93             print("Tried to compile re:", re_str)
  94             raise
  95
  96     sig_block_re = \
  97         re.compile(r'''^
  98                         (?: (?P<tag>[^:]+) \s* : \s*
  99                             (?P<value>\S.*?) )
 100                             |
 101                         (?: \[ (?P<updater>[^:]+) \s* : \s*
 102                                (?P<note>.+?) \s* \] )
 103                     \s* $''',
 104                    re.VERBOSE | re.MULTILINE)
 105
 106     def find_signatures(self, sig):
 107         if not sig.endswith('-by') and sig != 'Cc':
 108             sig += '-by'
 109         regex = self.make_signature_re(sig)
 110
 111         sigs = regex.findall(self.msg)
 112
 113         bad_case_sigs = filter(lambda m: m[0] != sig, sigs)
 114         for s in bad_case_sigs:
 115             self.error("'" +s[0] + "' should be '" + sig + "'")
 116
 117         for s in sigs:
 118             if s[1] != '':
 119                 self.error('There should be no spaces between ' + sig +
 120                            " and the ':'")
 121             if s[2] != ' ':
 122                 self.error("There should be a space after '" + sig + ":'")
 123
 124             self.check_email_address(s[3])
 125
 126         return sigs
 127
 128     email_re1 = re.compile(r'(?:\s*)(.*?)(\s*)<(.+)>\s*$',
 129                            re.MULTILINE|re.IGNORECASE)
 130
 131     def check_email_address(self, email):
 132         email = email.strip()
 133         mo = self.email_re1.match(email)
 134         if mo is None:
 135             self.error("Email format is invalid: " + email.strip())
 136             return
 137
 138         name = mo.group(1).strip()
 139         if name == '':
 140             self.error("Name is not provided with email address: " +
 141                        email)
 142         else:
 143             quoted = len(name) > 2 and name[0] == '"' and name[-1] == '"'
 144             if name.find(',') >= 0 and not quoted:
 145                 self.error('Add quotes (") around name with a comma: ' +
 146                            name)
 147
 148         if mo.group(2) == '':
 149             self.error("There should be a space between the name and " +
 150                        "email address: " + email)
 151
 152         if mo.group(3).find(' ') >= 0:
 153             self.error("The email address cannot contain a space: " +
 154                        mo.group(3))
 155
 156     def check_signed_off_by(self):
 157         sob='Signed-off-by'
 158         if self.msg.find(sob) < 0:
 159             self.error('Missing Signed-off-by! (Note: this must be ' +
 160                        'added by the code contributor!)')
 161             return
 162
 163         sobs = self.find_signatures('Signed-off')
 164
 165         if len(sobs) == 0:
 166             self.error('Invalid Signed-off-by format!')
 167             return
 168
 169     sig_types = (
 170         'Reviewed',
 171         'Reported',
 172         'Tested',
 173         'Suggested',
 174         'Acked',
 175         'Cc'
 176         )
 177
 178     def check_misc_signatures(self):
 179         for sig in self.sig_types:
 180             self.find_signatures(sig)
 181
 182     def check_overall_format(self):
 183         lines = self.msg.splitlines()
 184
 185         if len(lines) >= 1 and lines[0].endswith('\r\n'):
 186             empty_line = '\r\n'
 187         else:
 188             empty_line = '\n'
 189
 190         lines.insert(0, empty_line)
 191         lines.insert(0, self.subject + empty_line)
 192
 193         count = len(lines)
 194
 195         if count <= 0:
 196             self.error('Empty commit message!')
 197             return
 198
 199         if count >= 1 and len(lines[0]) >= 72:
 200             self.error('First line of commit message (subject line) ' +
 201                        'is too long.')
 202
 203         if count >= 1 and len(lines[0].strip()) == 0:
 204             self.error('First line of commit message (subject line) ' +
 205                        'is empty.')
 206
 207         if count >= 2 and lines[1].strip() != '':
 208             self.error('Second line of commit message should be ' +
 209                        'empty.')
 210
 211         for i in range(2, count):
 212             if (len(lines[i]) >= 76 and
 213                 len(lines[i].split()) > 1 and
 214                 not lines[i].startswith('git-svn-id:')):
 215                 self.error('Line %d of commit message is too long.' % (i + 1))
 216
 217         last_sig_line = None
 218         for i in range(count - 1, 0, -1):
 219             line = lines[i]
 220             mo = self.sig_block_re.match(line)
 221             if mo is None:
 222                 if line.strip() == '':
 223                     break
 224                 elif last_sig_line is not None:
 225                     err2 = 'Add empty line before "%s"?' % last_sig_line
 226                     self.error('The line before the signature block ' +
 227                                'should be empty', err2)
 228                 else:
 229                     self.error('The signature block was not found')
 230                 break
 231             last_sig_line = line.strip()
 232
 233 (START, PRE_PATCH, PATCH) = range(3)
 234
 235 class GitDiffCheck:
 236     """Checks the contents of a git diff."""
 237
 238     def __init__(self, diff):
 239         self.ok = True
 240         self.format_ok = True
 241         self.lines = diff.splitlines(True)
 242         self.count = len(self.lines)
 243         self.line_num = 0
 244         self.state = START
 245         self.new_bin = []
 246         while self.line_num < self.count and self.format_ok:
 247             line_num = self.line_num
 248             self.run()
 249             assert(self.line_num > line_num)
 250         self.report_message_result()
 251
 252     def report_message_result(self):
 253         if Verbose.level < Verbose.NORMAL:
 254             return
 255         if self.ok:
 256             print('The code passed all checks.')
 257         if self.new_bin:
 258             print('\nWARNING - The following binary files will be added ' +
 259                   'into the repository:')
 260             for binary in self.new_bin:
 261                 print('  ' + binary)
 262
 263     def run(self):
 264         line = self.lines[self.line_num]
 265
 266         if self.state in (PRE_PATCH, PATCH):
 267             if line.startswith('diff --git'):
 268                 self.state = START
 269         if self.state == PATCH:
 270             if line.startswith('@@ '):
 271                 self.state = PRE_PATCH
 272             elif len(line) >= 1 and line[0] not in ' -+' and \
 273                  not line.startswith(r'\ No newline ') and not self.binary:
 274                 for line in self.lines[self.line_num + 1:]:
 275                     if line.startswith('diff --git'):
 276                         self.format_error('diff found after end of patch')
 277                         break
 278                 self.line_num = self.count
 279                 return
 280
 281         if self.state == START:
 282             if line.startswith('diff --git'):
 283                 self.state = PRE_PATCH
 284                 self.filename = line[13:].split(' ', 1)[0]
 285                 self.is_newfile = False
 286                 self.force_crlf = not self.filename.endswith('.sh')
 287             elif len(line.rstrip()) != 0:
 288                 self.format_error("didn't find diff command")
 289             self.line_num += 1
 290         elif self.state == PRE_PATCH:
 291             if line.startswith('@@ '):
 292                 self.state = PATCH
 293                 self.binary = False
 294             elif line.startswith('GIT binary patch') or \
 295                  line.startswith('Binary files'):
 296                 self.state = PATCH
 297                 self.binary = True
 298                 if self.is_newfile:
 299                     self.new_bin.append(self.filename)
 300             else:
 301                 ok = False
 302                 self.is_newfile = self.newfile_prefix_re.match(line)
 303                 for pfx in self.pre_patch_prefixes:
 304                     if line.startswith(pfx):
 305                         ok = True
 306                 if not ok:
 307                     self.format_error("didn't find diff hunk marker (@@)")
 308             self.line_num += 1
 309         elif self.state == PATCH:
 310             if self.binary:
 311                 pass
 312             elif line.startswith('-'):
 313                 pass
 314             elif line.startswith('+'):
 315                 self.check_added_line(line[1:])
 316             elif line.startswith(r'\ No newline '):
 317                 pass
 318             elif not line.startswith(' '):
 319                 self.format_error("unexpected patch line")
 320             self.line_num += 1
 321
 322     pre_patch_prefixes = (
 323         '--- ',
 324         '+++ ',
 325         'index ',
 326         'new file ',
 327         'deleted file ',
 328         'old mode ',
 329         'new mode ',
 330         'similarity index ',
 331         'rename ',
 332         )
 333
 334     line_endings = ('\r\n', '\n\r', '\n', '\r')
 335
 336     newfile_prefix_re = \
 337         re.compile(r'''^
 338                        index\ 0+\.\.
 339                    ''',
 340                    re.VERBOSE)
 341
 342     def added_line_error(self, msg, line):
 343         lines = [ msg ]
 344         if self.filename is not None:
 345             lines.append('File: ' + self.filename)
 346         lines.append('Line: ' + line)
 347
 348         self.error(*lines)
 349
 350     old_debug_re = \
 351         re.compile(r'''
 352                         DEBUG \s* \( \s* \( \s*
 353                         (?: DEBUG_[A-Z_]+ \s* \| \s*)*
 354                         EFI_D_ ([A-Z_]+)
 355                    ''',
 356                    re.VERBOSE)
 357
 358     def check_added_line(self, line):
 359         eol = ''
 360         for an_eol in self.line_endings:
 361             if line.endswith(an_eol):
 362                 eol = an_eol
 363                 line = line[:-len(eol)]
 364
 365         stripped = line.rstrip()
 366
 367         if self.force_crlf and eol != '\r\n':
 368             self.added_line_error('Line ending (%s) is not CRLF' % repr(eol),
 369                                   line)
 370         if '\t' in line:
 371             self.added_line_error('Tab character used', line)
 372         if len(stripped) < len(line):
 373             self.added_line_error('Trailing whitespace found', line)
 374
 375         mo = self.old_debug_re.search(line)
 376         if mo is not None:
 377             self.added_line_error('EFI_D_' + mo.group(1) + ' was used, '
 378                                   'but DEBUG_' + mo.group(1) +
 379                                   ' is now recommended', line)
 380
 381     split_diff_re = re.compile(r'''
 382                                    (?P<cmd>
 383                                        ^ diff \s+ --git \s+ a/.+ \s+ b/.+ $
 384                                    )
 385                                    (?P<index>
 386                                        ^ index \s+ .+ $
 387                                    )
 388                                ''',
 389                                re.IGNORECASE | re.VERBOSE | re.MULTILINE)
 390
 391     def format_error(self, err):
 392         self.format_ok = False
 393         err = 'Patch format error: ' + err
 394         err2 = 'Line: ' + self.lines[self.line_num].rstrip()
 395         self.error(err, err2)
 396
 397     def error(self, *err):
 398         if self.ok and Verbose.level > Verbose.ONELINE:
 399             print('Code format is not valid:')
 400         self.ok = False
 401         if Verbose.level < Verbose.NORMAL:
 402             return
 403         count = 0
 404         for line in err:
 405             prefix = (' *', '  ')[count > 0]
 406             print(prefix, line)
 407             count += 1
 408
 409 class CheckOnePatch:
 410     """Checks the contents of a git email formatted patch.
 411
 412     Various checks are performed on both the commit message and the
 413     patch content.
 414     """
 415
 416     def __init__(self, name, patch):
 417         self.patch = patch
 418         self.find_patch_pieces()
 419
 420         msg_check = CommitMessageCheck(self.commit_subject, self.commit_msg)
 421         msg_ok = msg_check.ok
 422
 423         diff_ok = True
 424         if self.diff is not None:
 425             diff_check = GitDiffCheck(self.diff)
 426             diff_ok = diff_check.ok
 427
 428         self.ok = msg_ok and diff_ok
 429
 430         if Verbose.level == Verbose.ONELINE:
 431             if self.ok:
 432                 result = 'ok'
 433             else:
 434                 result = list()
 435                 if not msg_ok:
 436                     result.append('commit message')
 437                 if not diff_ok:
 438                     result.append('diff content')
 439                 result = 'bad ' + ' and '.join(result)
 440             print(name, result)
 441
 442
 443     git_diff_re = re.compile(r'''
 444                                  ^ diff \s+ --git \s+ a/.+ \s+ b/.+ $
 445                              ''',
 446                              re.IGNORECASE | re.VERBOSE | re.MULTILINE)
 447
 448     stat_re = \
 449         re.compile(r'''
 450                        (?P<commit_message> [\s\S\r\n]* )
 451                        (?P<stat>
 452                            ^ --- $ [\r\n]+
 453                            (?: ^ \s+ .+ \s+ \| \s+ \d+ \s+ \+* \-*
 454                                $ [\r\n]+ )+
 455                            [\s\S\r\n]+
 456                        )
 457                    ''',
 458                    re.IGNORECASE | re.VERBOSE | re.MULTILINE)
 459
 460     subject_prefix_re = \
 461         re.compile(r'''^
 462                        \s* (\[
 463                         [^\[\]]* # Allow all non-brackets
 464                        \])* \s*
 465                    ''',
 466                    re.VERBOSE)
 467
 468     def find_patch_pieces(self):
 469         if sys.version_info < (3, 0):
 470             patch = self.patch.encode('ascii', 'ignore')
 471         else:
 472             patch = self.patch
 473
 474         self.commit_msg = None
 475         self.stat = None
 476         self.commit_subject = None
 477         self.commit_prefix = None
 478         self.diff = None
 479
 480         if patch.startswith('diff --git'):
 481             self.diff = patch
 482             return
 483
 484         pmail = email.message_from_string(patch)
 485         parts = list(pmail.walk())
 486         assert(len(parts) == 1)
 487         assert(parts[0].get_content_type() == 'text/plain')
 488         content = parts[0].get_payload(decode=True).decode('utf-8', 'ignore')
 489
 490         mo = self.git_diff_re.search(content)
 491         if mo is not None:
 492             self.diff = content[mo.start():]
 493             content = content[:mo.start()]
 494
 495         mo = self.stat_re.search(content)
 496         if mo is None:
 497             self.commit_msg = content
 498         else:
 499             self.stat = mo.group('stat')
 500             self.commit_msg = mo.group('commit_message')
 501
 502         self.commit_subject = pmail['subject'].replace('\r\n', '')
 503         self.commit_subject = self.commit_subject.replace('\n', '')
 504         self.commit_subject = self.subject_prefix_re.sub('', self.commit_subject, 1)
 505
 506 class CheckGitCommits:
 507     """Reads patches from git based on the specified git revision range.
 508
 509     The patches are read from git, and then checked.
 510     """
 511
 512     def __init__(self, rev_spec, max_count):
 513         commits = self.read_commit_list_from_git(rev_spec, max_count)
 514         if len(commits) == 1 and Verbose.level > Verbose.ONELINE:
 515             commits = [ rev_spec ]
 516         self.ok = True
 517         blank_line = False
 518         for commit in commits:
 519             if Verbose.level > Verbose.ONELINE:
 520                 if blank_line:
 521                     print()
 522                 else:
 523                     blank_line = True
 524                 print('Checking git commit:', commit)
 525             patch = self.read_patch_from_git(commit)
 526             self.ok &= CheckOnePatch(commit, patch).ok
 527         if not commits:
 528             print("Couldn't find commit matching: '{}'".format(rev_spec))
 529
 530     def read_commit_list_from_git(self, rev_spec, max_count):
 531         # Run git to get the commit patch
 532         cmd = [ 'rev-list', '--abbrev-commit', '--no-walk' ]
 533         if max_count is not None:
 534             cmd.append('--max-count=' + str(max_count))
 535         cmd.append(rev_spec)
 536         out = self.run_git(*cmd)
 537         return out.split() if out else []
 538
 539     def read_patch_from_git(self, commit):
 540         # Run git to get the commit patch
 541         return self.run_git('show', '--pretty=email', commit)
 542
 543     def run_git(self, *args):
 544         cmd = [ 'git' ]
 545         cmd += args
 546         p = subprocess.Popen(cmd,
 547                      stdout=subprocess.PIPE,
 548                      stderr=subprocess.STDOUT)
 549         Result = p.communicate()
 550         return Result[0].decode('utf-8', 'ignore') if Result[0] and Result[0].find(b"fatal")!=0 else None
 551
 552 class CheckOnePatchFile:
 553     """Performs a patch check for a single file.
 554
 555     stdin is used when the filename is '-'.
 556     """
 557
 558     def __init__(self, patch_filename):
 559         if patch_filename == '-':
 560             patch = sys.stdin.read()
 561             patch_filename = 'stdin'
 562         else:
 563             f = open(patch_filename, 'rb')
 564             patch = f.read().decode('utf-8', 'ignore')
 565             f.close()
 566         if Verbose.level > Verbose.ONELINE:
 567             print('Checking patch file:', patch_filename)
 568         self.ok = CheckOnePatch(patch_filename, patch).ok
 569
 570 class CheckOneArg:
 571     """Performs a patch check for a single command line argument.
 572
 573     The argument will be handed off to a file or git-commit based
 574     checker.
 575     """
 576
 577     def __init__(self, param, max_count=None):
 578         self.ok = True
 579         if param == '-' or os.path.exists(param):
 580             checker = CheckOnePatchFile(param)
 581         else:
 582             checker = CheckGitCommits(param, max_count)
 583         self.ok = checker.ok
 584
 585 class PatchCheckApp:
 586     """Checks patches based on the command line arguments."""
 587
 588     def __init__(self):
 589         self.parse_options()
 590         patches = self.args.patches
 591
 592         if len(patches) == 0:
 593             patches = [ 'HEAD' ]
 594
 595         self.ok = True
 596         self.count = None
 597         for patch in patches:
 598             self.process_one_arg(patch)
 599
 600         if self.count is not None:
 601             self.process_one_arg('HEAD')
 602
 603         if self.ok:
 604             self.retval = 0
 605         else:
 606             self.retval = -1
 607
 608     def process_one_arg(self, arg):
 609         if len(arg) >= 2 and arg[0] == '-':
 610             try:
 611                 self.count = int(arg[1:])
 612                 return
 613             except ValueError:
 614                 pass
 615         self.ok &= CheckOneArg(arg, self.count).ok
 616         self.count = None
 617
 618     def parse_options(self):
 619         parser = argparse.ArgumentParser(description=__copyright__)
 620         parser.add_argument('--version', action='version',
 621                             version='%(prog)s ' + VersionNumber)
 622         parser.add_argument('patches', nargs='*',
 623                             help='[patch file | git rev list]')
 624         group = parser.add_mutually_exclusive_group()
 625         group.add_argument("--oneline",
 626                            action="store_true",
 627                            help="Print one result per line")
 628         group.add_argument("--silent",
 629                            action="store_true",
 630                            help="Print nothing")
 631         self.args = parser.parse_args()
 632         if self.args.oneline:
 633             Verbose.level = Verbose.ONELINE
 634         if self.args.silent:
 635             Verbose.level = Verbose.SILENT
 636
 637 if __name__ == "__main__":
 638     sys.exit(PatchCheckApp().retval)