BaseTools/Scripts/PatchCheck.py

   1 ## @file
   2 #  Check a patch for various format issues
   3 #
   4 #  Copyright (c) 2015 - 2017, Intel Corporation. All rights reserved.<BR>
   5 #
   6 #  This program and the accompanying materials are licensed and made
   7 #  available under the terms and conditions of the BSD License which
   8 #  accompanies this distribution. The full text of the license may be
   9 #  found at http://opensource.org/licenses/bsd-license.php
  10 #
  11 #  THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS"
  12 #  BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER
  13 #  EXPRESS OR IMPLIED.
  14 #
  15
  16 from __future__ import print_function
  17
  18 VersionNumber = '0.1'
  19 __copyright__ = "Copyright (c) 2015 - 2016, Intel Corporation  All rights reserved."
  20
  21 import email
  22 import argparse
  23 import os
  24 import re
  25 import subprocess
  26 import sys
  27
  28 class Verbose:
  29     SILENT, ONELINE, NORMAL = range(3)
  30     level = NORMAL
  31
  32 class CommitMessageCheck:
  33     """Checks the contents of a git commit message."""
  34
  35     def __init__(self, subject, message):
  36         self.ok = True
  37
  38         if subject is None and  message is None:
  39             self.error('Commit message is missing!')
  40             return
  41
  42         self.subject = subject
  43         self.msg = message
  44
  45         self.check_contributed_under()
  46         self.check_signed_off_by()
  47         self.check_misc_signatures()
  48         self.check_overall_format()
  49         self.report_message_result()
  50
  51     url = 'https://github.com/tianocore/tianocore.github.io/wiki/Commit-Message-Format'
  52
  53     def report_message_result(self):
  54         if Verbose.level < Verbose.NORMAL:
  55             return
  56         if self.ok:
  57             # All checks passed
  58             return_code = 0
  59             print('The commit message format passed all checks.')
  60         else:
  61             return_code = 1
  62         if not self.ok:
  63             print(self.url)
  64
  65     def error(self, *err):
  66         if self.ok and Verbose.level > Verbose.ONELINE:
  67             print('The commit message format is not valid:')
  68         self.ok = False
  69         if Verbose.level < Verbose.NORMAL:
  70             return
  71         count = 0
  72         for line in err:
  73             prefix = (' *', '  ')[count > 0]
  74             print(prefix, line)
  75             count += 1
  76
  77     def check_contributed_under(self):
  78         cu_msg='Contributed-under: TianoCore Contribution Agreement 1.0'
  79         if self.msg.find(cu_msg) < 0:
  80             self.error('Missing Contributed-under! (Note: this must be ' +
  81                        'added by the code contributor!)')
  82
  83     @staticmethod
  84     def make_signature_re(sig, re_input=False):
  85         if re_input:
  86             sub_re = sig
  87         else:
  88             sub_re = sig.replace('-', r'[-\s]+')
  89         re_str = (r'^(?P<tag>' + sub_re +
  90                   r')(\s*):(\s*)(?P<value>\S.*?)(?:\s*)$')
  91         try:
  92             return re.compile(re_str, re.MULTILINE|re.IGNORECASE)
  93         except Exception:
  94             print("Tried to compile re:", re_str)
  95             raise
  96
  97     sig_block_re = \
  98         re.compile(r'''^
  99                         (?: (?P<tag>[^:]+) \s* : \s*
 100                             (?P<value>\S.*?) )
 101                             |
 102                         (?: \[ (?P<updater>[^:]+) \s* : \s*
 103                                (?P<note>.+?) \s* \] )
 104                     \s* $''',
 105                    re.VERBOSE | re.MULTILINE)
 106
 107     def find_signatures(self, sig):
 108         if not sig.endswith('-by') and sig != 'Cc':
 109             sig += '-by'
 110         regex = self.make_signature_re(sig)
 111
 112         sigs = regex.findall(self.msg)
 113
 114         bad_case_sigs = filter(lambda m: m[0] != sig, sigs)
 115         for s in bad_case_sigs:
 116             self.error("'" +s[0] + "' should be '" + sig + "'")
 117
 118         for s in sigs:
 119             if s[1] != '':
 120                 self.error('There should be no spaces between ' + sig +
 121                            " and the ':'")
 122             if s[2] != ' ':
 123                 self.error("There should be a space after '" + sig + ":'")
 124
 125             self.check_email_address(s[3])
 126
 127         return sigs
 128
 129     email_re1 = re.compile(r'(?:\s*)(.*?)(\s*)<(.+)>\s*$',
 130                            re.MULTILINE|re.IGNORECASE)
 131
 132     def check_email_address(self, email):
 133         email = email.strip()
 134         mo = self.email_re1.match(email)
 135         if mo is None:
 136             self.error("Email format is invalid: " + email.strip())
 137             return
 138
 139         name = mo.group(1).strip()
 140         if name == '':
 141             self.error("Name is not provided with email address: " +
 142                        email)
 143         else:
 144             quoted = len(name) > 2 and name[0] == '"' and name[-1] == '"'
 145             if name.find(',') >= 0 and not quoted:
 146                 self.error('Add quotes (") around name with a comma: ' +
 147                            name)
 148
 149         if mo.group(2) == '':
 150             self.error("There should be a space between the name and " +
 151                        "email address: " + email)
 152
 153         if mo.group(3).find(' ') >= 0:
 154             self.error("The email address cannot contain a space: " +
 155                        mo.group(3))
 156
 157     def check_signed_off_by(self):
 158         sob='Signed-off-by'
 159         if self.msg.find(sob) < 0:
 160             self.error('Missing Signed-off-by! (Note: this must be ' +
 161                        'added by the code contributor!)')
 162             return
 163
 164         sobs = self.find_signatures('Signed-off')
 165
 166         if len(sobs) == 0:
 167             self.error('Invalid Signed-off-by format!')
 168             return
 169
 170     sig_types = (
 171         'Reviewed',
 172         'Reported',
 173         'Tested',
 174         'Suggested',
 175         'Acked',
 176         'Cc'
 177         )
 178
 179     def check_misc_signatures(self):
 180         for sig in self.sig_types:
 181             self.find_signatures(sig)
 182
 183     def check_overall_format(self):
 184         lines = self.msg.splitlines()
 185
 186         if len(lines) >= 1 and lines[0].endswith('\r\n'):
 187             empty_line = '\r\n'
 188         else:
 189             empty_line = '\n'
 190
 191         lines.insert(0, empty_line)
 192         lines.insert(0, self.subject + empty_line)
 193
 194         count = len(lines)
 195
 196         if count <= 0:
 197             self.error('Empty commit message!')
 198             return
 199
 200         if count >= 1 and len(lines[0]) >= 72:
 201             self.error('First line of commit message (subject line) ' +
 202                        'is too long.')
 203
 204         if count >= 1 and len(lines[0].strip()) == 0:
 205             self.error('First line of commit message (subject line) ' +
 206                        'is empty.')
 207
 208         if count >= 2 and lines[1].strip() != '':
 209             self.error('Second line of commit message should be ' +
 210                        'empty.')
 211
 212         for i in range(2, count):
 213             if (len(lines[i]) >= 76 and
 214                 len(lines[i].split()) > 1 and
 215                 not lines[i].startswith('git-svn-id:')):
 216                 self.error('Line %d of commit message is too long.' % (i + 1))
 217
 218         last_sig_line = None
 219         for i in range(count - 1, 0, -1):
 220             line = lines[i]
 221             mo = self.sig_block_re.match(line)
 222             if mo is None:
 223                 if line.strip() == '':
 224                     break
 225                 elif last_sig_line is not None:
 226                     err2 = 'Add empty line before "%s"?' % last_sig_line
 227                     self.error('The line before the signature block ' +
 228                                'should be empty', err2)
 229                 else:
 230                     self.error('The signature block was not found')
 231                 break
 232             last_sig_line = line.strip()
 233
 234 (START, PRE_PATCH, PATCH) = range(3)
 235
 236 class GitDiffCheck:
 237     """Checks the contents of a git diff."""
 238
 239     def __init__(self, diff):
 240         self.ok = True
 241         self.format_ok = True
 242         self.lines = diff.splitlines(True)
 243         self.count = len(self.lines)
 244         self.line_num = 0
 245         self.state = START
 246         self.new_bin = []
 247         while self.line_num < self.count and self.format_ok:
 248             line_num = self.line_num
 249             self.run()
 250             assert(self.line_num > line_num)
 251         self.report_message_result()
 252
 253     def report_message_result(self):
 254         if Verbose.level < Verbose.NORMAL:
 255             return
 256         if self.ok:
 257             print('The code passed all checks.')
 258         if self.new_bin:
 259             print('\nWARNING - The following binary files will be added ' +
 260                   'into the repository:')
 261             for binary in self.new_bin:
 262                 print('  ' + binary)
 263
 264     def run(self):
 265         line = self.lines[self.line_num]
 266
 267         if self.state in (PRE_PATCH, PATCH):
 268             if line.startswith('diff --git'):
 269                 self.state = START
 270         if self.state == PATCH:
 271             if line.startswith('@@ '):
 272                 self.state = PRE_PATCH
 273             elif len(line) >= 1 and line[0] not in ' -+' and \
 274                  not line.startswith(r'\ No newline ') and not self.binary:
 275                 for line in self.lines[self.line_num + 1:]:
 276                     if line.startswith('diff --git'):
 277                         self.format_error('diff found after end of patch')
 278                         break
 279                 self.line_num = self.count
 280                 return
 281
 282         if self.state == START:
 283             if line.startswith('diff --git'):
 284                 self.state = PRE_PATCH
 285                 self.filename = line[13:].split(' ',1)[0]
 286                 self.is_newfile = False
 287                 self.force_crlf = not self.filename.endswith('.sh')
 288             elif len(line.rstrip()) != 0:
 289                 self.format_error("didn't find diff command")
 290             self.line_num += 1
 291         elif self.state == PRE_PATCH:
 292             if line.startswith('@@ '):
 293                 self.state = PATCH
 294                 self.binary = False
 295             elif line.startswith('GIT binary patch') or \
 296                  line.startswith('Binary files'):
 297                 self.state = PATCH
 298                 self.binary = True
 299                 if self.is_newfile:
 300                     self.new_bin.append(self.filename)
 301             else:
 302                 ok = False
 303                 self.is_newfile = self.newfile_prefix_re.match(line)
 304                 for pfx in self.pre_patch_prefixes:
 305                     if line.startswith(pfx):
 306                         ok = True
 307                 if not ok:
 308                     self.format_error("didn't find diff hunk marker (@@)")
 309             self.line_num += 1
 310         elif self.state == PATCH:
 311             if self.binary:
 312                 pass
 313             elif line.startswith('-'):
 314                 pass
 315             elif line.startswith('+'):
 316                 self.check_added_line(line[1:])
 317             elif line.startswith(r'\ No newline '):
 318                 pass
 319             elif not line.startswith(' '):
 320                 self.format_error("unexpected patch line")
 321             self.line_num += 1
 322
 323     pre_patch_prefixes = (
 324         '--- ',
 325         '+++ ',
 326         'index ',
 327         'new file ',
 328         'deleted file ',
 329         'old mode ',
 330         'new mode ',
 331         'similarity index ',
 332         'rename ',
 333         )
 334
 335     line_endings = ('\r\n', '\n\r', '\n', '\r')
 336
 337     newfile_prefix_re = \
 338         re.compile(r'''^
 339                        index\ 0+\.\.
 340                    ''',
 341                    re.VERBOSE)
 342
 343     def added_line_error(self, msg, line):
 344         lines = [ msg ]
 345         if self.filename is not None:
 346             lines.append('File: ' + self.filename)
 347         lines.append('Line: ' + line)
 348
 349         self.error(*lines)
 350
 351     old_debug_re = \
 352         re.compile(r'''
 353                         DEBUG \s* \( \s* \( \s*
 354                         (?: DEBUG_[A-Z_]+ \s* \| \s*)*
 355                         EFI_D_ ([A-Z_]+)
 356                    ''',
 357                    re.VERBOSE)
 358
 359     def check_added_line(self, line):
 360         eol = ''
 361         for an_eol in self.line_endings:
 362             if line.endswith(an_eol):
 363                 eol = an_eol
 364                 line = line[:-len(eol)]
 365
 366         stripped = line.rstrip()
 367
 368         if self.force_crlf and eol != '\r\n':
 369             self.added_line_error('Line ending (%s) is not CRLF' % repr(eol),
 370                                   line)
 371         if '\t' in line:
 372             self.added_line_error('Tab character used', line)
 373         if len(stripped) < len(line):
 374             self.added_line_error('Trailing whitespace found', line)
 375
 376         mo = self.old_debug_re.search(line)
 377         if mo is not None:
 378             self.added_line_error('EFI_D_' + mo.group(1) + ' was used, '
 379                                   'but DEBUG_' + mo.group(1) +
 380                                   ' is now recommended', line)
 381
 382     split_diff_re = re.compile(r'''
 383                                    (?P<cmd>
 384                                        ^ diff \s+ --git \s+ a/.+ \s+ b/.+ $
 385                                    )
 386                                    (?P<index>
 387                                        ^ index \s+ .+ $
 388                                    )
 389                                ''',
 390                                re.IGNORECASE | re.VERBOSE | re.MULTILINE)
 391
 392     def format_error(self, err):
 393         self.format_ok = False
 394         err = 'Patch format error: ' + err
 395         err2 = 'Line: ' + self.lines[self.line_num].rstrip()
 396         self.error(err, err2)
 397
 398     def error(self, *err):
 399         if self.ok and Verbose.level > Verbose.ONELINE:
 400             print('Code format is not valid:')
 401         self.ok = False
 402         if Verbose.level < Verbose.NORMAL:
 403             return
 404         count = 0
 405         for line in err:
 406             prefix = (' *', '  ')[count > 0]
 407             print(prefix, line)
 408             count += 1
 409
 410 class CheckOnePatch:
 411     """Checks the contents of a git email formatted patch.
 412
 413     Various checks are performed on both the commit message and the
 414     patch content.
 415     """
 416
 417     def __init__(self, name, patch):
 418         self.patch = patch
 419         self.find_patch_pieces()
 420
 421         msg_check = CommitMessageCheck(self.commit_subject, self.commit_msg)
 422         msg_ok = msg_check.ok
 423
 424         diff_ok = True
 425         if self.diff is not None:
 426             diff_check = GitDiffCheck(self.diff)
 427             diff_ok = diff_check.ok
 428
 429         self.ok = msg_ok and diff_ok
 430
 431         if Verbose.level == Verbose.ONELINE:
 432             if self.ok:
 433                 result = 'ok'
 434             else:
 435                 result = list()
 436                 if not msg_ok:
 437                     result.append('commit message')
 438                 if not diff_ok:
 439                     result.append('diff content')
 440                 result = 'bad ' + ' and '.join(result)
 441             print(name, result)
 442
 443
 444     git_diff_re = re.compile(r'''
 445                                  ^ diff \s+ --git \s+ a/.+ \s+ b/.+ $
 446                              ''',
 447                              re.IGNORECASE | re.VERBOSE | re.MULTILINE)
 448
 449     stat_re = \
 450         re.compile(r'''
 451                        (?P<commit_message> [\s\S\r\n]* )
 452                        (?P<stat>
 453                            ^ --- $ [\r\n]+
 454                            (?: ^ \s+ .+ \s+ \| \s+ \d+ \s+ \+* \-*
 455                                $ [\r\n]+ )+
 456                            [\s\S\r\n]+
 457                        )
 458                    ''',
 459                    re.IGNORECASE | re.VERBOSE | re.MULTILINE)
 460
 461     subject_prefix_re = \
 462         re.compile(r'''^
 463                        \s* (\[
 464                         [^\[\]]* # Allow all non-brackets
 465                        \])* \s*
 466                    ''',
 467                    re.VERBOSE)
 468
 469     def find_patch_pieces(self):
 470         if sys.version_info < (3, 0):
 471             patch = self.patch.encode('ascii', 'ignore')
 472         else:
 473             patch = self.patch
 474
 475         self.commit_msg = None
 476         self.stat = None
 477         self.commit_subject = None
 478         self.commit_prefix = None
 479         self.diff = None
 480
 481         if patch.startswith('diff --git'):
 482             self.diff = patch
 483             return
 484
 485         pmail = email.message_from_string(patch)
 486         parts = list(pmail.walk())
 487         assert(len(parts) == 1)
 488         assert(parts[0].get_content_type() == 'text/plain')
 489         content = parts[0].get_payload(decode=True).decode('utf-8', 'ignore')
 490
 491         mo = self.git_diff_re.search(content)
 492         if mo is not None:
 493             self.diff = content[mo.start():]
 494             content = content[:mo.start()]
 495
 496         mo = self.stat_re.search(content)
 497         if mo is None:
 498             self.commit_msg = content
 499         else:
 500             self.stat = mo.group('stat')
 501             self.commit_msg = mo.group('commit_message')
 502
 503         self.commit_subject = pmail['subject'].replace('\r\n', '')
 504         self.commit_subject = self.commit_subject.replace('\n', '')
 505         self.commit_subject = self.subject_prefix_re.sub('', self.commit_subject, 1)
 506
 507 class CheckGitCommits:
 508     """Reads patches from git based on the specified git revision range.
 509
 510     The patches are read from git, and then checked.
 511     """
 512
 513     def __init__(self, rev_spec, max_count):
 514         commits = self.read_commit_list_from_git(rev_spec, max_count)
 515         if len(commits) == 1 and Verbose.level > Verbose.ONELINE:
 516             commits = [ rev_spec ]
 517         self.ok = True
 518         blank_line = False
 519         for commit in commits:
 520             if Verbose.level > Verbose.ONELINE:
 521                 if blank_line:
 522                     print()
 523                 else:
 524                     blank_line = True
 525                 print('Checking git commit:', commit)
 526             patch = self.read_patch_from_git(commit)
 527             self.ok &= CheckOnePatch(commit, patch).ok
 528
 529     def read_commit_list_from_git(self, rev_spec, max_count):
 530         # Run git to get the commit patch
 531         cmd = [ 'rev-list', '--abbrev-commit', '--no-walk' ]
 532         if max_count is not None:
 533             cmd.append('--max-count=' + str(max_count))
 534         cmd.append(rev_spec)
 535         out = self.run_git(*cmd)
 536         return out.split()
 537
 538     def read_patch_from_git(self, commit):
 539         # Run git to get the commit patch
 540         return self.run_git('show', '--pretty=email', commit)
 541
 542     def run_git(self, *args):
 543         cmd = [ 'git' ]
 544         cmd += args
 545         p = subprocess.Popen(cmd,
 546                      stdout=subprocess.PIPE,
 547                      stderr=subprocess.STDOUT)
 548         return p.communicate()[0].decode('utf-8', 'ignore')
 549
 550 class CheckOnePatchFile:
 551     """Performs a patch check for a single file.
 552
 553     stdin is used when the filename is '-'.
 554     """
 555
 556     def __init__(self, patch_filename):
 557         if patch_filename == '-':
 558             patch = sys.stdin.read()
 559             patch_filename = 'stdin'
 560         else:
 561             f = open(patch_filename, 'rb')
 562             patch = f.read().decode('utf-8', 'ignore')
 563             f.close()
 564         if Verbose.level > Verbose.ONELINE:
 565             print('Checking patch file:', patch_filename)
 566         self.ok = CheckOnePatch(patch_filename, patch).ok
 567
 568 class CheckOneArg:
 569     """Performs a patch check for a single command line argument.
 570
 571     The argument will be handed off to a file or git-commit based
 572     checker.
 573     """
 574
 575     def __init__(self, param, max_count=None):
 576         self.ok = True
 577         if param == '-' or os.path.exists(param):
 578             checker = CheckOnePatchFile(param)
 579         else:
 580             checker = CheckGitCommits(param, max_count)
 581         self.ok = checker.ok
 582
 583 class PatchCheckApp:
 584     """Checks patches based on the command line arguments."""
 585
 586     def __init__(self):
 587         self.parse_options()
 588         patches = self.args.patches
 589
 590         if len(patches) == 0:
 591             patches = [ 'HEAD' ]
 592
 593         self.ok = True
 594         self.count = None
 595         for patch in patches:
 596             self.process_one_arg(patch)
 597
 598         if self.count is not None:
 599             self.process_one_arg('HEAD')
 600
 601         if self.ok:
 602             self.retval = 0
 603         else:
 604             self.retval = -1
 605
 606     def process_one_arg(self, arg):
 607         if len(arg) >= 2 and arg[0] == '-':
 608             try:
 609                 self.count = int(arg[1:])
 610                 return
 611             except ValueError:
 612                 pass
 613         self.ok &= CheckOneArg(arg, self.count).ok
 614         self.count = None
 615
 616     def parse_options(self):
 617         parser = argparse.ArgumentParser(description=__copyright__)
 618         parser.add_argument('--version', action='version',
 619                             version='%(prog)s ' + VersionNumber)
 620         parser.add_argument('patches', nargs='*',
 621                             help='[patch file | git rev list]')
 622         group = parser.add_mutually_exclusive_group()
 623         group.add_argument("--oneline",
 624                            action="store_true",
 625                            help="Print one result per line")
 626         group.add_argument("--silent",
 627                            action="store_true",
 628                            help="Print nothing")
 629         self.args = parser.parse_args()
 630         if self.args.oneline:
 631             Verbose.level = Verbose.ONELINE
 632         if self.args.silent:
 633             Verbose.level = Verbose.SILENT
 634
 635 if __name__ == "__main__":
 636     sys.exit(PatchCheckApp().retval)