BaseTools/Scripts/PatchCheck.py

   1 ## @file
   2 #  Check a patch for various format issues
   3 #
   4 #  Copyright (c) 2015 - 2018, Intel Corporation. All rights reserved.<BR>
   5 #
   6 #  SPDX-License-Identifier: BSD-2-Clause-Patent
   7 #
   8
   9 from __future__ import print_function
  10
  11 VersionNumber = '0.1'
  12 __copyright__ = "Copyright (c) 2015 - 2016, Intel Corporation  All rights reserved."
  13
  14 import email
  15 import argparse
  16 import os
  17 import re
  18 import subprocess
  19 import sys
  20
  21 class Verbose:
  22     SILENT, ONELINE, NORMAL = range(3)
  23     level = NORMAL
  24
  25 class CommitMessageCheck:
  26     """Checks the contents of a git commit message."""
  27
  28     def __init__(self, subject, message):
  29         self.ok = True
  30
  31         if subject is None and  message is None:
  32             self.error('Commit message is missing!')
  33             return
  34
  35         self.subject = subject
  36         self.msg = message
  37
  38         self.check_contributed_under()
  39         self.check_signed_off_by()
  40         self.check_misc_signatures()
  41         self.check_overall_format()
  42         self.report_message_result()
  43
  44     url = 'https://github.com/tianocore/tianocore.github.io/wiki/Commit-Message-Format'
  45
  46     def report_message_result(self):
  47         if Verbose.level < Verbose.NORMAL:
  48             return
  49         if self.ok:
  50             # All checks passed
  51             return_code = 0
  52             print('The commit message format passed all checks.')
  53         else:
  54             return_code = 1
  55         if not self.ok:
  56             print(self.url)
  57
  58     def error(self, *err):
  59         if self.ok and Verbose.level > Verbose.ONELINE:
  60             print('The commit message format is not valid:')
  61         self.ok = False
  62         if Verbose.level < Verbose.NORMAL:
  63             return
  64         count = 0
  65         for line in err:
  66             prefix = (' *', '  ')[count > 0]
  67             print(prefix, line)
  68             count += 1
  69
  70     def check_contributed_under(self):
  71         cu_msg='Contributed-under: TianoCore Contribution Agreement 1.1'
  72         if self.msg.find(cu_msg) < 0:
  73             # Allow 1.0 for now while EDK II community transitions to 1.1
  74             cu_msg='Contributed-under: TianoCore Contribution Agreement 1.0'
  75             if self.msg.find(cu_msg) < 0:
  76                 self.error('Missing Contributed-under! (Note: this must be ' +
  77                            'added by the code contributor!)')
  78
  79     @staticmethod
  80     def make_signature_re(sig, re_input=False):
  81         if re_input:
  82             sub_re = sig
  83         else:
  84             sub_re = sig.replace('-', r'[-\s]+')
  85         re_str = (r'^(?P<tag>' + sub_re +
  86                   r')(\s*):(\s*)(?P<value>\S.*?)(?:\s*)$')
  87         try:
  88             return re.compile(re_str, re.MULTILINE|re.IGNORECASE)
  89         except Exception:
  90             print("Tried to compile re:", re_str)
  91             raise
  92
  93     sig_block_re = \
  94         re.compile(r'''^
  95                         (?: (?P<tag>[^:]+) \s* : \s*
  96                             (?P<value>\S.*?) )
  97                             |
  98                         (?: \[ (?P<updater>[^:]+) \s* : \s*
  99                                (?P<note>.+?) \s* \] )
 100                     \s* $''',
 101                    re.VERBOSE | re.MULTILINE)
 102
 103     def find_signatures(self, sig):
 104         if not sig.endswith('-by') and sig != 'Cc':
 105             sig += '-by'
 106         regex = self.make_signature_re(sig)
 107
 108         sigs = regex.findall(self.msg)
 109
 110         bad_case_sigs = filter(lambda m: m[0] != sig, sigs)
 111         for s in bad_case_sigs:
 112             self.error("'" +s[0] + "' should be '" + sig + "'")
 113
 114         for s in sigs:
 115             if s[1] != '':
 116                 self.error('There should be no spaces between ' + sig +
 117                            " and the ':'")
 118             if s[2] != ' ':
 119                 self.error("There should be a space after '" + sig + ":'")
 120
 121             self.check_email_address(s[3])
 122
 123         return sigs
 124
 125     email_re1 = re.compile(r'(?:\s*)(.*?)(\s*)<(.+)>\s*$',
 126                            re.MULTILINE|re.IGNORECASE)
 127
 128     def check_email_address(self, email):
 129         email = email.strip()
 130         mo = self.email_re1.match(email)
 131         if mo is None:
 132             self.error("Email format is invalid: " + email.strip())
 133             return
 134
 135         name = mo.group(1).strip()
 136         if name == '':
 137             self.error("Name is not provided with email address: " +
 138                        email)
 139         else:
 140             quoted = len(name) > 2 and name[0] == '"' and name[-1] == '"'
 141             if name.find(',') >= 0 and not quoted:
 142                 self.error('Add quotes (") around name with a comma: ' +
 143                            name)
 144
 145         if mo.group(2) == '':
 146             self.error("There should be a space between the name and " +
 147                        "email address: " + email)
 148
 149         if mo.group(3).find(' ') >= 0:
 150             self.error("The email address cannot contain a space: " +
 151                        mo.group(3))
 152
 153     def check_signed_off_by(self):
 154         sob='Signed-off-by'
 155         if self.msg.find(sob) < 0:
 156             self.error('Missing Signed-off-by! (Note: this must be ' +
 157                        'added by the code contributor!)')
 158             return
 159
 160         sobs = self.find_signatures('Signed-off')
 161
 162         if len(sobs) == 0:
 163             self.error('Invalid Signed-off-by format!')
 164             return
 165
 166     sig_types = (
 167         'Reviewed',
 168         'Reported',
 169         'Tested',
 170         'Suggested',
 171         'Acked',
 172         'Cc'
 173         )
 174
 175     def check_misc_signatures(self):
 176         for sig in self.sig_types:
 177             self.find_signatures(sig)
 178
 179     def check_overall_format(self):
 180         lines = self.msg.splitlines()
 181
 182         if len(lines) >= 1 and lines[0].endswith('\r\n'):
 183             empty_line = '\r\n'
 184         else:
 185             empty_line = '\n'
 186
 187         lines.insert(0, empty_line)
 188         lines.insert(0, self.subject + empty_line)
 189
 190         count = len(lines)
 191
 192         if count <= 0:
 193             self.error('Empty commit message!')
 194             return
 195
 196         if count >= 1 and len(lines[0]) >= 72:
 197             self.error('First line of commit message (subject line) ' +
 198                        'is too long.')
 199
 200         if count >= 1 and len(lines[0].strip()) == 0:
 201             self.error('First line of commit message (subject line) ' +
 202                        'is empty.')
 203
 204         if count >= 2 and lines[1].strip() != '':
 205             self.error('Second line of commit message should be ' +
 206                        'empty.')
 207
 208         for i in range(2, count):
 209             if (len(lines[i]) >= 76 and
 210                 len(lines[i].split()) > 1 and
 211                 not lines[i].startswith('git-svn-id:')):
 212                 self.error('Line %d of commit message is too long.' % (i + 1))
 213
 214         last_sig_line = None
 215         for i in range(count - 1, 0, -1):
 216             line = lines[i]
 217             mo = self.sig_block_re.match(line)
 218             if mo is None:
 219                 if line.strip() == '':
 220                     break
 221                 elif last_sig_line is not None:
 222                     err2 = 'Add empty line before "%s"?' % last_sig_line
 223                     self.error('The line before the signature block ' +
 224                                'should be empty', err2)
 225                 else:
 226                     self.error('The signature block was not found')
 227                 break
 228             last_sig_line = line.strip()
 229
 230 (START, PRE_PATCH, PATCH) = range(3)
 231
 232 class GitDiffCheck:
 233     """Checks the contents of a git diff."""
 234
 235     def __init__(self, diff):
 236         self.ok = True
 237         self.format_ok = True
 238         self.lines = diff.splitlines(True)
 239         self.count = len(self.lines)
 240         self.line_num = 0
 241         self.state = START
 242         self.new_bin = []
 243         while self.line_num < self.count and self.format_ok:
 244             line_num = self.line_num
 245             self.run()
 246             assert(self.line_num > line_num)
 247         self.report_message_result()
 248
 249     def report_message_result(self):
 250         if Verbose.level < Verbose.NORMAL:
 251             return
 252         if self.ok:
 253             print('The code passed all checks.')
 254         if self.new_bin:
 255             print('\nWARNING - The following binary files will be added ' +
 256                   'into the repository:')
 257             for binary in self.new_bin:
 258                 print('  ' + binary)
 259
 260     def run(self):
 261         line = self.lines[self.line_num]
 262
 263         if self.state in (PRE_PATCH, PATCH):
 264             if line.startswith('diff --git'):
 265                 self.state = START
 266         if self.state == PATCH:
 267             if line.startswith('@@ '):
 268                 self.state = PRE_PATCH
 269             elif len(line) >= 1 and line[0] not in ' -+' and \
 270                  not line.startswith(r'\ No newline ') and not self.binary:
 271                 for line in self.lines[self.line_num + 1:]:
 272                     if line.startswith('diff --git'):
 273                         self.format_error('diff found after end of patch')
 274                         break
 275                 self.line_num = self.count
 276                 return
 277
 278         if self.state == START:
 279             if line.startswith('diff --git'):
 280                 self.state = PRE_PATCH
 281                 self.filename = line[13:].split(' ', 1)[0]
 282                 self.is_newfile = False
 283                 self.force_crlf = not self.filename.endswith('.sh')
 284             elif len(line.rstrip()) != 0:
 285                 self.format_error("didn't find diff command")
 286             self.line_num += 1
 287         elif self.state == PRE_PATCH:
 288             if line.startswith('@@ '):
 289                 self.state = PATCH
 290                 self.binary = False
 291             elif line.startswith('GIT binary patch') or \
 292                  line.startswith('Binary files'):
 293                 self.state = PATCH
 294                 self.binary = True
 295                 if self.is_newfile:
 296                     self.new_bin.append(self.filename)
 297             else:
 298                 ok = False
 299                 self.is_newfile = self.newfile_prefix_re.match(line)
 300                 for pfx in self.pre_patch_prefixes:
 301                     if line.startswith(pfx):
 302                         ok = True
 303                 if not ok:
 304                     self.format_error("didn't find diff hunk marker (@@)")
 305             self.line_num += 1
 306         elif self.state == PATCH:
 307             if self.binary:
 308                 pass
 309             elif line.startswith('-'):
 310                 pass
 311             elif line.startswith('+'):
 312                 self.check_added_line(line[1:])
 313             elif line.startswith(r'\ No newline '):
 314                 pass
 315             elif not line.startswith(' '):
 316                 self.format_error("unexpected patch line")
 317             self.line_num += 1
 318
 319     pre_patch_prefixes = (
 320         '--- ',
 321         '+++ ',
 322         'index ',
 323         'new file ',
 324         'deleted file ',
 325         'old mode ',
 326         'new mode ',
 327         'similarity index ',
 328         'rename ',
 329         )
 330
 331     line_endings = ('\r\n', '\n\r', '\n', '\r')
 332
 333     newfile_prefix_re = \
 334         re.compile(r'''^
 335                        index\ 0+\.\.
 336                    ''',
 337                    re.VERBOSE)
 338
 339     def added_line_error(self, msg, line):
 340         lines = [ msg ]
 341         if self.filename is not None:
 342             lines.append('File: ' + self.filename)
 343         lines.append('Line: ' + line)
 344
 345         self.error(*lines)
 346
 347     old_debug_re = \
 348         re.compile(r'''
 349                         DEBUG \s* \( \s* \( \s*
 350                         (?: DEBUG_[A-Z_]+ \s* \| \s*)*
 351                         EFI_D_ ([A-Z_]+)
 352                    ''',
 353                    re.VERBOSE)
 354
 355     def check_added_line(self, line):
 356         eol = ''
 357         for an_eol in self.line_endings:
 358             if line.endswith(an_eol):
 359                 eol = an_eol
 360                 line = line[:-len(eol)]
 361
 362         stripped = line.rstrip()
 363
 364         if self.force_crlf and eol != '\r\n':
 365             self.added_line_error('Line ending (%s) is not CRLF' % repr(eol),
 366                                   line)
 367         if '\t' in line:
 368             self.added_line_error('Tab character used', line)
 369         if len(stripped) < len(line):
 370             self.added_line_error('Trailing whitespace found', line)
 371
 372         mo = self.old_debug_re.search(line)
 373         if mo is not None:
 374             self.added_line_error('EFI_D_' + mo.group(1) + ' was used, '
 375                                   'but DEBUG_' + mo.group(1) +
 376                                   ' is now recommended', line)
 377
 378     split_diff_re = re.compile(r'''
 379                                    (?P<cmd>
 380                                        ^ diff \s+ --git \s+ a/.+ \s+ b/.+ $
 381                                    )
 382                                    (?P<index>
 383                                        ^ index \s+ .+ $
 384                                    )
 385                                ''',
 386                                re.IGNORECASE | re.VERBOSE | re.MULTILINE)
 387
 388     def format_error(self, err):
 389         self.format_ok = False
 390         err = 'Patch format error: ' + err
 391         err2 = 'Line: ' + self.lines[self.line_num].rstrip()
 392         self.error(err, err2)
 393
 394     def error(self, *err):
 395         if self.ok and Verbose.level > Verbose.ONELINE:
 396             print('Code format is not valid:')
 397         self.ok = False
 398         if Verbose.level < Verbose.NORMAL:
 399             return
 400         count = 0
 401         for line in err:
 402             prefix = (' *', '  ')[count > 0]
 403             print(prefix, line)
 404             count += 1
 405
 406 class CheckOnePatch:
 407     """Checks the contents of a git email formatted patch.
 408
 409     Various checks are performed on both the commit message and the
 410     patch content.
 411     """
 412
 413     def __init__(self, name, patch):
 414         self.patch = patch
 415         self.find_patch_pieces()
 416
 417         msg_check = CommitMessageCheck(self.commit_subject, self.commit_msg)
 418         msg_ok = msg_check.ok
 419
 420         diff_ok = True
 421         if self.diff is not None:
 422             diff_check = GitDiffCheck(self.diff)
 423             diff_ok = diff_check.ok
 424
 425         self.ok = msg_ok and diff_ok
 426
 427         if Verbose.level == Verbose.ONELINE:
 428             if self.ok:
 429                 result = 'ok'
 430             else:
 431                 result = list()
 432                 if not msg_ok:
 433                     result.append('commit message')
 434                 if not diff_ok:
 435                     result.append('diff content')
 436                 result = 'bad ' + ' and '.join(result)
 437             print(name, result)
 438
 439
 440     git_diff_re = re.compile(r'''
 441                                  ^ diff \s+ --git \s+ a/.+ \s+ b/.+ $
 442                              ''',
 443                              re.IGNORECASE | re.VERBOSE | re.MULTILINE)
 444
 445     stat_re = \
 446         re.compile(r'''
 447                        (?P<commit_message> [\s\S\r\n]* )
 448                        (?P<stat>
 449                            ^ --- $ [\r\n]+
 450                            (?: ^ \s+ .+ \s+ \| \s+ \d+ \s+ \+* \-*
 451                                $ [\r\n]+ )+
 452                            [\s\S\r\n]+
 453                        )
 454                    ''',
 455                    re.IGNORECASE | re.VERBOSE | re.MULTILINE)
 456
 457     subject_prefix_re = \
 458         re.compile(r'''^
 459                        \s* (\[
 460                         [^\[\]]* # Allow all non-brackets
 461                        \])* \s*
 462                    ''',
 463                    re.VERBOSE)
 464
 465     def find_patch_pieces(self):
 466         if sys.version_info < (3, 0):
 467             patch = self.patch.encode('ascii', 'ignore')
 468         else:
 469             patch = self.patch
 470
 471         self.commit_msg = None
 472         self.stat = None
 473         self.commit_subject = None
 474         self.commit_prefix = None
 475         self.diff = None
 476
 477         if patch.startswith('diff --git'):
 478             self.diff = patch
 479             return
 480
 481         pmail = email.message_from_string(patch)
 482         parts = list(pmail.walk())
 483         assert(len(parts) == 1)
 484         assert(parts[0].get_content_type() == 'text/plain')
 485         content = parts[0].get_payload(decode=True).decode('utf-8', 'ignore')
 486
 487         mo = self.git_diff_re.search(content)
 488         if mo is not None:
 489             self.diff = content[mo.start():]
 490             content = content[:mo.start()]
 491
 492         mo = self.stat_re.search(content)
 493         if mo is None:
 494             self.commit_msg = content
 495         else:
 496             self.stat = mo.group('stat')
 497             self.commit_msg = mo.group('commit_message')
 498
 499         self.commit_subject = pmail['subject'].replace('\r\n', '')
 500         self.commit_subject = self.commit_subject.replace('\n', '')
 501         self.commit_subject = self.subject_prefix_re.sub('', self.commit_subject, 1)
 502
 503 class CheckGitCommits:
 504     """Reads patches from git based on the specified git revision range.
 505
 506     The patches are read from git, and then checked.
 507     """
 508
 509     def __init__(self, rev_spec, max_count):
 510         commits = self.read_commit_list_from_git(rev_spec, max_count)
 511         if len(commits) == 1 and Verbose.level > Verbose.ONELINE:
 512             commits = [ rev_spec ]
 513         self.ok = True
 514         blank_line = False
 515         for commit in commits:
 516             if Verbose.level > Verbose.ONELINE:
 517                 if blank_line:
 518                     print()
 519                 else:
 520                     blank_line = True
 521                 print('Checking git commit:', commit)
 522             patch = self.read_patch_from_git(commit)
 523             self.ok &= CheckOnePatch(commit, patch).ok
 524         if not commits:
 525             print("Couldn't find commit matching: '{}'".format(rev_spec))
 526
 527     def read_commit_list_from_git(self, rev_spec, max_count):
 528         # Run git to get the commit patch
 529         cmd = [ 'rev-list', '--abbrev-commit', '--no-walk' ]
 530         if max_count is not None:
 531             cmd.append('--max-count=' + str(max_count))
 532         cmd.append(rev_spec)
 533         out = self.run_git(*cmd)
 534         return out.split() if out else []
 535
 536     def read_patch_from_git(self, commit):
 537         # Run git to get the commit patch
 538         return self.run_git('show', '--pretty=email', commit)
 539
 540     def run_git(self, *args):
 541         cmd = [ 'git' ]
 542         cmd += args
 543         p = subprocess.Popen(cmd,
 544                      stdout=subprocess.PIPE,
 545                      stderr=subprocess.STDOUT)
 546         Result = p.communicate()
 547         return Result[0].decode('utf-8', 'ignore') if Result[0] and Result[0].find(b"fatal")!=0 else None
 548
 549 class CheckOnePatchFile:
 550     """Performs a patch check for a single file.
 551
 552     stdin is used when the filename is '-'.
 553     """
 554
 555     def __init__(self, patch_filename):
 556         if patch_filename == '-':
 557             patch = sys.stdin.read()
 558             patch_filename = 'stdin'
 559         else:
 560             f = open(patch_filename, 'rb')
 561             patch = f.read().decode('utf-8', 'ignore')
 562             f.close()
 563         if Verbose.level > Verbose.ONELINE:
 564             print('Checking patch file:', patch_filename)
 565         self.ok = CheckOnePatch(patch_filename, patch).ok
 566
 567 class CheckOneArg:
 568     """Performs a patch check for a single command line argument.
 569
 570     The argument will be handed off to a file or git-commit based
 571     checker.
 572     """
 573
 574     def __init__(self, param, max_count=None):
 575         self.ok = True
 576         if param == '-' or os.path.exists(param):
 577             checker = CheckOnePatchFile(param)
 578         else:
 579             checker = CheckGitCommits(param, max_count)
 580         self.ok = checker.ok
 581
 582 class PatchCheckApp:
 583     """Checks patches based on the command line arguments."""
 584
 585     def __init__(self):
 586         self.parse_options()
 587         patches = self.args.patches
 588
 589         if len(patches) == 0:
 590             patches = [ 'HEAD' ]
 591
 592         self.ok = True
 593         self.count = None
 594         for patch in patches:
 595             self.process_one_arg(patch)
 596
 597         if self.count is not None:
 598             self.process_one_arg('HEAD')
 599
 600         if self.ok:
 601             self.retval = 0
 602         else:
 603             self.retval = -1
 604
 605     def process_one_arg(self, arg):
 606         if len(arg) >= 2 and arg[0] == '-':
 607             try:
 608                 self.count = int(arg[1:])
 609                 return
 610             except ValueError:
 611                 pass
 612         self.ok &= CheckOneArg(arg, self.count).ok
 613         self.count = None
 614
 615     def parse_options(self):
 616         parser = argparse.ArgumentParser(description=__copyright__)
 617         parser.add_argument('--version', action='version',
 618                             version='%(prog)s ' + VersionNumber)
 619         parser.add_argument('patches', nargs='*',
 620                             help='[patch file | git rev list]')
 621         group = parser.add_mutually_exclusive_group()
 622         group.add_argument("--oneline",
 623                            action="store_true",
 624                            help="Print one result per line")
 625         group.add_argument("--silent",
 626                            action="store_true",
 627                            help="Print nothing")
 628         self.args = parser.parse_args()
 629         if self.args.oneline:
 630             Verbose.level = Verbose.ONELINE
 631         if self.args.silent:
 632             Verbose.level = Verbose.SILENT
 633
 634 if __name__ == "__main__":
 635     sys.exit(PatchCheckApp().retval)