BaseTools/Scripts/PatchCheck.py

   1 ## @file
   2 #  Check a patch for various format issues
   3 #
   4 #  Copyright (c) 2015 - 2017, Intel Corporation. All rights reserved.<BR>
   5 #
   6 #  This program and the accompanying materials are licensed and made
   7 #  available under the terms and conditions of the BSD License which
   8 #  accompanies this distribution. The full text of the license may be
   9 #  found at http://opensource.org/licenses/bsd-license.php
  10 #
  11 #  THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS"
  12 #  BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER
  13 #  EXPRESS OR IMPLIED.
  14 #
  15
  16 from __future__ import print_function
  17
  18 VersionNumber = '0.1'
  19 __copyright__ = "Copyright (c) 2015 - 2016, Intel Corporation  All rights reserved."
  20
  21 import email
  22 import argparse
  23 import os
  24 import re
  25 import subprocess
  26 import sys
  27
  28 class Verbose:
  29     SILENT, ONELINE, NORMAL = range(3)
  30     level = NORMAL
  31
  32 class CommitMessageCheck:
  33     """Checks the contents of a git commit message."""
  34
  35     def __init__(self, subject, message):
  36         self.ok = True
  37
  38         if subject is None and  message is None:
  39             self.error('Commit message is missing!')
  40             return
  41
  42         self.subject = subject
  43         self.msg = message
  44
  45         self.check_contributed_under()
  46         self.check_signed_off_by()
  47         self.check_misc_signatures()
  48         self.check_overall_format()
  49         self.report_message_result()
  50
  51     url = 'https://github.com/tianocore/tianocore.github.io/wiki/Commit-Message-Format'
  52
  53     def report_message_result(self):
  54         if Verbose.level < Verbose.NORMAL:
  55             return
  56         if self.ok:
  57             # All checks passed
  58             return_code = 0
  59             print('The commit message format passed all checks.')
  60         else:
  61             return_code = 1
  62         if not self.ok:
  63             print(self.url)
  64
  65     def error(self, *err):
  66         if self.ok and Verbose.level > Verbose.ONELINE:
  67             print('The commit message format is not valid:')
  68         self.ok = False
  69         if Verbose.level < Verbose.NORMAL:
  70             return
  71         count = 0
  72         for line in err:
  73             prefix = (' *', '  ')[count > 0]
  74             print(prefix, line)
  75             count += 1
  76
  77     def check_contributed_under(self):
  78         cu_msg='Contributed-under: TianoCore Contribution Agreement 1.1'
  79         if self.msg.find(cu_msg) < 0:
  80             # Allow 1.0 for now while EDK II community transitions to 1.1
  81             cu_msg='Contributed-under: TianoCore Contribution Agreement 1.0'
  82             if self.msg.find(cu_msg) < 0:
  83                 self.error('Missing Contributed-under! (Note: this must be ' +
  84                            'added by the code contributor!)')
  85
  86     @staticmethod
  87     def make_signature_re(sig, re_input=False):
  88         if re_input:
  89             sub_re = sig
  90         else:
  91             sub_re = sig.replace('-', r'[-\s]+')
  92         re_str = (r'^(?P<tag>' + sub_re +
  93                   r')(\s*):(\s*)(?P<value>\S.*?)(?:\s*)$')
  94         try:
  95             return re.compile(re_str, re.MULTILINE|re.IGNORECASE)
  96         except Exception:
  97             print("Tried to compile re:", re_str)
  98             raise
  99
 100     sig_block_re = \
 101         re.compile(r'''^
 102                         (?: (?P<tag>[^:]+) \s* : \s*
 103                             (?P<value>\S.*?) )
 104                             |
 105                         (?: \[ (?P<updater>[^:]+) \s* : \s*
 106                                (?P<note>.+?) \s* \] )
 107                     \s* $''',
 108                    re.VERBOSE | re.MULTILINE)
 109
 110     def find_signatures(self, sig):
 111         if not sig.endswith('-by') and sig != 'Cc':
 112             sig += '-by'
 113         regex = self.make_signature_re(sig)
 114
 115         sigs = regex.findall(self.msg)
 116
 117         bad_case_sigs = filter(lambda m: m[0] != sig, sigs)
 118         for s in bad_case_sigs:
 119             self.error("'" +s[0] + "' should be '" + sig + "'")
 120
 121         for s in sigs:
 122             if s[1] != '':
 123                 self.error('There should be no spaces between ' + sig +
 124                            " and the ':'")
 125             if s[2] != ' ':
 126                 self.error("There should be a space after '" + sig + ":'")
 127
 128             self.check_email_address(s[3])
 129
 130         return sigs
 131
 132     email_re1 = re.compile(r'(?:\s*)(.*?)(\s*)<(.+)>\s*$',
 133                            re.MULTILINE|re.IGNORECASE)
 134
 135     def check_email_address(self, email):
 136         email = email.strip()
 137         mo = self.email_re1.match(email)
 138         if mo is None:
 139             self.error("Email format is invalid: " + email.strip())
 140             return
 141
 142         name = mo.group(1).strip()
 143         if name == '':
 144             self.error("Name is not provided with email address: " +
 145                        email)
 146         else:
 147             quoted = len(name) > 2 and name[0] == '"' and name[-1] == '"'
 148             if name.find(',') >= 0 and not quoted:
 149                 self.error('Add quotes (") around name with a comma: ' +
 150                            name)
 151
 152         if mo.group(2) == '':
 153             self.error("There should be a space between the name and " +
 154                        "email address: " + email)
 155
 156         if mo.group(3).find(' ') >= 0:
 157             self.error("The email address cannot contain a space: " +
 158                        mo.group(3))
 159
 160     def check_signed_off_by(self):
 161         sob='Signed-off-by'
 162         if self.msg.find(sob) < 0:
 163             self.error('Missing Signed-off-by! (Note: this must be ' +
 164                        'added by the code contributor!)')
 165             return
 166
 167         sobs = self.find_signatures('Signed-off')
 168
 169         if len(sobs) == 0:
 170             self.error('Invalid Signed-off-by format!')
 171             return
 172
 173     sig_types = (
 174         'Reviewed',
 175         'Reported',
 176         'Tested',
 177         'Suggested',
 178         'Acked',
 179         'Cc'
 180         )
 181
 182     def check_misc_signatures(self):
 183         for sig in self.sig_types:
 184             self.find_signatures(sig)
 185
 186     def check_overall_format(self):
 187         lines = self.msg.splitlines()
 188
 189         if len(lines) >= 1 and lines[0].endswith('\r\n'):
 190             empty_line = '\r\n'
 191         else:
 192             empty_line = '\n'
 193
 194         lines.insert(0, empty_line)
 195         lines.insert(0, self.subject + empty_line)
 196
 197         count = len(lines)
 198
 199         if count <= 0:
 200             self.error('Empty commit message!')
 201             return
 202
 203         if count >= 1 and len(lines[0]) >= 72:
 204             self.error('First line of commit message (subject line) ' +
 205                        'is too long.')
 206
 207         if count >= 1 and len(lines[0].strip()) == 0:
 208             self.error('First line of commit message (subject line) ' +
 209                        'is empty.')
 210
 211         if count >= 2 and lines[1].strip() != '':
 212             self.error('Second line of commit message should be ' +
 213                        'empty.')
 214
 215         for i in range(2, count):
 216             if (len(lines[i]) >= 76 and
 217                 len(lines[i].split()) > 1 and
 218                 not lines[i].startswith('git-svn-id:')):
 219                 self.error('Line %d of commit message is too long.' % (i + 1))
 220
 221         last_sig_line = None
 222         for i in range(count - 1, 0, -1):
 223             line = lines[i]
 224             mo = self.sig_block_re.match(line)
 225             if mo is None:
 226                 if line.strip() == '':
 227                     break
 228                 elif last_sig_line is not None:
 229                     err2 = 'Add empty line before "%s"?' % last_sig_line
 230                     self.error('The line before the signature block ' +
 231                                'should be empty', err2)
 232                 else:
 233                     self.error('The signature block was not found')
 234                 break
 235             last_sig_line = line.strip()
 236
 237 (START, PRE_PATCH, PATCH) = range(3)
 238
 239 class GitDiffCheck:
 240     """Checks the contents of a git diff."""
 241
 242     def __init__(self, diff):
 243         self.ok = True
 244         self.format_ok = True
 245         self.lines = diff.splitlines(True)
 246         self.count = len(self.lines)
 247         self.line_num = 0
 248         self.state = START
 249         self.new_bin = []
 250         while self.line_num < self.count and self.format_ok:
 251             line_num = self.line_num
 252             self.run()
 253             assert(self.line_num > line_num)
 254         self.report_message_result()
 255
 256     def report_message_result(self):
 257         if Verbose.level < Verbose.NORMAL:
 258             return
 259         if self.ok:
 260             print('The code passed all checks.')
 261         if self.new_bin:
 262             print('\nWARNING - The following binary files will be added ' +
 263                   'into the repository:')
 264             for binary in self.new_bin:
 265                 print('  ' + binary)
 266
 267     def run(self):
 268         line = self.lines[self.line_num]
 269
 270         if self.state in (PRE_PATCH, PATCH):
 271             if line.startswith('diff --git'):
 272                 self.state = START
 273         if self.state == PATCH:
 274             if line.startswith('@@ '):
 275                 self.state = PRE_PATCH
 276             elif len(line) >= 1 and line[0] not in ' -+' and \
 277                  not line.startswith(r'\ No newline ') and not self.binary:
 278                 for line in self.lines[self.line_num + 1:]:
 279                     if line.startswith('diff --git'):
 280                         self.format_error('diff found after end of patch')
 281                         break
 282                 self.line_num = self.count
 283                 return
 284
 285         if self.state == START:
 286             if line.startswith('diff --git'):
 287                 self.state = PRE_PATCH
 288                 self.filename = line[13:].split(' ', 1)[0]
 289                 self.is_newfile = False
 290                 self.force_crlf = not self.filename.endswith('.sh')
 291             elif len(line.rstrip()) != 0:
 292                 self.format_error("didn't find diff command")
 293             self.line_num += 1
 294         elif self.state == PRE_PATCH:
 295             if line.startswith('@@ '):
 296                 self.state = PATCH
 297                 self.binary = False
 298             elif line.startswith('GIT binary patch') or \
 299                  line.startswith('Binary files'):
 300                 self.state = PATCH
 301                 self.binary = True
 302                 if self.is_newfile:
 303                     self.new_bin.append(self.filename)
 304             else:
 305                 ok = False
 306                 self.is_newfile = self.newfile_prefix_re.match(line)
 307                 for pfx in self.pre_patch_prefixes:
 308                     if line.startswith(pfx):
 309                         ok = True
 310                 if not ok:
 311                     self.format_error("didn't find diff hunk marker (@@)")
 312             self.line_num += 1
 313         elif self.state == PATCH:
 314             if self.binary:
 315                 pass
 316             elif line.startswith('-'):
 317                 pass
 318             elif line.startswith('+'):
 319                 self.check_added_line(line[1:])
 320             elif line.startswith(r'\ No newline '):
 321                 pass
 322             elif not line.startswith(' '):
 323                 self.format_error("unexpected patch line")
 324             self.line_num += 1
 325
 326     pre_patch_prefixes = (
 327         '--- ',
 328         '+++ ',
 329         'index ',
 330         'new file ',
 331         'deleted file ',
 332         'old mode ',
 333         'new mode ',
 334         'similarity index ',
 335         'rename ',
 336         )
 337
 338     line_endings = ('\r\n', '\n\r', '\n', '\r')
 339
 340     newfile_prefix_re = \
 341         re.compile(r'''^
 342                        index\ 0+\.\.
 343                    ''',
 344                    re.VERBOSE)
 345
 346     def added_line_error(self, msg, line):
 347         lines = [ msg ]
 348         if self.filename is not None:
 349             lines.append('File: ' + self.filename)
 350         lines.append('Line: ' + line)
 351
 352         self.error(*lines)
 353
 354     old_debug_re = \
 355         re.compile(r'''
 356                         DEBUG \s* \( \s* \( \s*
 357                         (?: DEBUG_[A-Z_]+ \s* \| \s*)*
 358                         EFI_D_ ([A-Z_]+)
 359                    ''',
 360                    re.VERBOSE)
 361
 362     def check_added_line(self, line):
 363         eol = ''
 364         for an_eol in self.line_endings:
 365             if line.endswith(an_eol):
 366                 eol = an_eol
 367                 line = line[:-len(eol)]
 368
 369         stripped = line.rstrip()
 370
 371         if self.force_crlf and eol != '\r\n':
 372             self.added_line_error('Line ending (%s) is not CRLF' % repr(eol),
 373                                   line)
 374         if '\t' in line:
 375             self.added_line_error('Tab character used', line)
 376         if len(stripped) < len(line):
 377             self.added_line_error('Trailing whitespace found', line)
 378
 379         mo = self.old_debug_re.search(line)
 380         if mo is not None:
 381             self.added_line_error('EFI_D_' + mo.group(1) + ' was used, '
 382                                   'but DEBUG_' + mo.group(1) +
 383                                   ' is now recommended', line)
 384
 385     split_diff_re = re.compile(r'''
 386                                    (?P<cmd>
 387                                        ^ diff \s+ --git \s+ a/.+ \s+ b/.+ $
 388                                    )
 389                                    (?P<index>
 390                                        ^ index \s+ .+ $
 391                                    )
 392                                ''',
 393                                re.IGNORECASE | re.VERBOSE | re.MULTILINE)
 394
 395     def format_error(self, err):
 396         self.format_ok = False
 397         err = 'Patch format error: ' + err
 398         err2 = 'Line: ' + self.lines[self.line_num].rstrip()
 399         self.error(err, err2)
 400
 401     def error(self, *err):
 402         if self.ok and Verbose.level > Verbose.ONELINE:
 403             print('Code format is not valid:')
 404         self.ok = False
 405         if Verbose.level < Verbose.NORMAL:
 406             return
 407         count = 0
 408         for line in err:
 409             prefix = (' *', '  ')[count > 0]
 410             print(prefix, line)
 411             count += 1
 412
 413 class CheckOnePatch:
 414     """Checks the contents of a git email formatted patch.
 415
 416     Various checks are performed on both the commit message and the
 417     patch content.
 418     """
 419
 420     def __init__(self, name, patch):
 421         self.patch = patch
 422         self.find_patch_pieces()
 423
 424         msg_check = CommitMessageCheck(self.commit_subject, self.commit_msg)
 425         msg_ok = msg_check.ok
 426
 427         diff_ok = True
 428         if self.diff is not None:
 429             diff_check = GitDiffCheck(self.diff)
 430             diff_ok = diff_check.ok
 431
 432         self.ok = msg_ok and diff_ok
 433
 434         if Verbose.level == Verbose.ONELINE:
 435             if self.ok:
 436                 result = 'ok'
 437             else:
 438                 result = list()
 439                 if not msg_ok:
 440                     result.append('commit message')
 441                 if not diff_ok:
 442                     result.append('diff content')
 443                 result = 'bad ' + ' and '.join(result)
 444             print(name, result)
 445
 446
 447     git_diff_re = re.compile(r'''
 448                                  ^ diff \s+ --git \s+ a/.+ \s+ b/.+ $
 449                              ''',
 450                              re.IGNORECASE | re.VERBOSE | re.MULTILINE)
 451
 452     stat_re = \
 453         re.compile(r'''
 454                        (?P<commit_message> [\s\S\r\n]* )
 455                        (?P<stat>
 456                            ^ --- $ [\r\n]+
 457                            (?: ^ \s+ .+ \s+ \| \s+ \d+ \s+ \+* \-*
 458                                $ [\r\n]+ )+
 459                            [\s\S\r\n]+
 460                        )
 461                    ''',
 462                    re.IGNORECASE | re.VERBOSE | re.MULTILINE)
 463
 464     subject_prefix_re = \
 465         re.compile(r'''^
 466                        \s* (\[
 467                         [^\[\]]* # Allow all non-brackets
 468                        \])* \s*
 469                    ''',
 470                    re.VERBOSE)
 471
 472     def find_patch_pieces(self):
 473         if sys.version_info < (3, 0):
 474             patch = self.patch.encode('ascii', 'ignore')
 475         else:
 476             patch = self.patch
 477
 478         self.commit_msg = None
 479         self.stat = None
 480         self.commit_subject = None
 481         self.commit_prefix = None
 482         self.diff = None
 483
 484         if patch.startswith('diff --git'):
 485             self.diff = patch
 486             return
 487
 488         pmail = email.message_from_string(patch)
 489         parts = list(pmail.walk())
 490         assert(len(parts) == 1)
 491         assert(parts[0].get_content_type() == 'text/plain')
 492         content = parts[0].get_payload(decode=True).decode('utf-8', 'ignore')
 493
 494         mo = self.git_diff_re.search(content)
 495         if mo is not None:
 496             self.diff = content[mo.start():]
 497             content = content[:mo.start()]
 498
 499         mo = self.stat_re.search(content)
 500         if mo is None:
 501             self.commit_msg = content
 502         else:
 503             self.stat = mo.group('stat')
 504             self.commit_msg = mo.group('commit_message')
 505
 506         self.commit_subject = pmail['subject'].replace('\r\n', '')
 507         self.commit_subject = self.commit_subject.replace('\n', '')
 508         self.commit_subject = self.subject_prefix_re.sub('', self.commit_subject, 1)
 509
 510 class CheckGitCommits:
 511     """Reads patches from git based on the specified git revision range.
 512
 513     The patches are read from git, and then checked.
 514     """
 515
 516     def __init__(self, rev_spec, max_count):
 517         commits = self.read_commit_list_from_git(rev_spec, max_count)
 518         if len(commits) == 1 and Verbose.level > Verbose.ONELINE:
 519             commits = [ rev_spec ]
 520         self.ok = True
 521         blank_line = False
 522         for commit in commits:
 523             if Verbose.level > Verbose.ONELINE:
 524                 if blank_line:
 525                     print()
 526                 else:
 527                     blank_line = True
 528                 print('Checking git commit:', commit)
 529             patch = self.read_patch_from_git(commit)
 530             self.ok &= CheckOnePatch(commit, patch).ok
 531
 532     def read_commit_list_from_git(self, rev_spec, max_count):
 533         # Run git to get the commit patch
 534         cmd = [ 'rev-list', '--abbrev-commit', '--no-walk' ]
 535         if max_count is not None:
 536             cmd.append('--max-count=' + str(max_count))
 537         cmd.append(rev_spec)
 538         out = self.run_git(*cmd)
 539         return out.split()
 540
 541     def read_patch_from_git(self, commit):
 542         # Run git to get the commit patch
 543         return self.run_git('show', '--pretty=email', commit)
 544
 545     def run_git(self, *args):
 546         cmd = [ 'git' ]
 547         cmd += args
 548         p = subprocess.Popen(cmd,
 549                      stdout=subprocess.PIPE,
 550                      stderr=subprocess.STDOUT)
 551         return p.communicate()[0].decode('utf-8', 'ignore')
 552
 553 class CheckOnePatchFile:
 554     """Performs a patch check for a single file.
 555
 556     stdin is used when the filename is '-'.
 557     """
 558
 559     def __init__(self, patch_filename):
 560         if patch_filename == '-':
 561             patch = sys.stdin.read()
 562             patch_filename = 'stdin'
 563         else:
 564             f = open(patch_filename, 'rb')
 565             patch = f.read().decode('utf-8', 'ignore')
 566             f.close()
 567         if Verbose.level > Verbose.ONELINE:
 568             print('Checking patch file:', patch_filename)
 569         self.ok = CheckOnePatch(patch_filename, patch).ok
 570
 571 class CheckOneArg:
 572     """Performs a patch check for a single command line argument.
 573
 574     The argument will be handed off to a file or git-commit based
 575     checker.
 576     """
 577
 578     def __init__(self, param, max_count=None):
 579         self.ok = True
 580         if param == '-' or os.path.exists(param):
 581             checker = CheckOnePatchFile(param)
 582         else:
 583             checker = CheckGitCommits(param, max_count)
 584         self.ok = checker.ok
 585
 586 class PatchCheckApp:
 587     """Checks patches based on the command line arguments."""
 588
 589     def __init__(self):
 590         self.parse_options()
 591         patches = self.args.patches
 592
 593         if len(patches) == 0:
 594             patches = [ 'HEAD' ]
 595
 596         self.ok = True
 597         self.count = None
 598         for patch in patches:
 599             self.process_one_arg(patch)
 600
 601         if self.count is not None:
 602             self.process_one_arg('HEAD')
 603
 604         if self.ok:
 605             self.retval = 0
 606         else:
 607             self.retval = -1
 608
 609     def process_one_arg(self, arg):
 610         if len(arg) >= 2 and arg[0] == '-':
 611             try:
 612                 self.count = int(arg[1:])
 613                 return
 614             except ValueError:
 615                 pass
 616         self.ok &= CheckOneArg(arg, self.count).ok
 617         self.count = None
 618
 619     def parse_options(self):
 620         parser = argparse.ArgumentParser(description=__copyright__)
 621         parser.add_argument('--version', action='version',
 622                             version='%(prog)s ' + VersionNumber)
 623         parser.add_argument('patches', nargs='*',
 624                             help='[patch file | git rev list]')
 625         group = parser.add_mutually_exclusive_group()
 626         group.add_argument("--oneline",
 627                            action="store_true",
 628                            help="Print one result per line")
 629         group.add_argument("--silent",
 630                            action="store_true",
 631                            help="Print nothing")
 632         self.args = parser.parse_args()
 633         if self.args.oneline:
 634             Verbose.level = Verbose.ONELINE
 635         if self.args.silent:
 636             Verbose.level = Verbose.SILENT
 637
 638 if __name__ == "__main__":
 639     sys.exit(PatchCheckApp().retval)