BaseTools/Scripts/PatchCheck.py

   1 ## @file
   2 #  Check a patch for various format issues
   3 #
   4 #  Copyright (c) 2015 - 2019, Intel Corporation. All rights reserved.<BR>
   5 #  Copyright (C) 2020, Red Hat, Inc.<BR>
   6 #
   7 #  SPDX-License-Identifier: BSD-2-Clause-Patent
   8 #
   9
  10 from __future__ import print_function
  11
  12 VersionNumber = '0.1'
  13 __copyright__ = "Copyright (c) 2015 - 2016, Intel Corporation  All rights reserved."
  14
  15 import email
  16 import argparse
  17 import os
  18 import re
  19 import subprocess
  20 import sys
  21
  22 class Verbose:
  23     SILENT, ONELINE, NORMAL = range(3)
  24     level = NORMAL
  25
  26 class EmailAddressCheck:
  27     """Checks an email address."""
  28
  29     def __init__(self, email):
  30         self.ok = True
  31
  32         if email is None:
  33             self.error('Email address is missing!')
  34             return
  35
  36         self.check_email_address(email)
  37
  38     def error(self, *err):
  39         if self.ok and Verbose.level > Verbose.ONELINE:
  40             print('The email address is not valid:')
  41         self.ok = False
  42         if Verbose.level < Verbose.NORMAL:
  43             return
  44         count = 0
  45         for line in err:
  46             prefix = (' *', '  ')[count > 0]
  47             print(prefix, line)
  48             count += 1
  49
  50     email_re1 = re.compile(r'(?:\s*)(.*?)(\s*)<(.+)>\s*$',
  51                            re.MULTILINE|re.IGNORECASE)
  52
  53     def check_email_address(self, email):
  54         email = email.strip()
  55         mo = self.email_re1.match(email)
  56         if mo is None:
  57             self.error("Email format is invalid: " + email.strip())
  58             return
  59
  60         name = mo.group(1).strip()
  61         if name == '':
  62             self.error("Name is not provided with email address: " +
  63                        email)
  64         else:
  65             quoted = len(name) > 2 and name[0] == '"' and name[-1] == '"'
  66             if name.find(',') >= 0 and not quoted:
  67                 self.error('Add quotes (") around name with a comma: ' +
  68                            name)
  69
  70         if mo.group(2) == '':
  71             self.error("There should be a space between the name and " +
  72                        "email address: " + email)
  73
  74         if mo.group(3).find(' ') >= 0:
  75             self.error("The email address cannot contain a space: " +
  76                        mo.group(3))
  77
  78 class CommitMessageCheck:
  79     """Checks the contents of a git commit message."""
  80
  81     def __init__(self, subject, message):
  82         self.ok = True
  83
  84         if subject is None and  message is None:
  85             self.error('Commit message is missing!')
  86             return
  87
  88         self.subject = subject
  89         self.msg = message
  90
  91         self.check_contributed_under()
  92         self.check_signed_off_by()
  93         self.check_misc_signatures()
  94         self.check_overall_format()
  95         self.report_message_result()
  96
  97     url = 'https://github.com/tianocore/tianocore.github.io/wiki/Commit-Message-Format'
  98
  99     def report_message_result(self):
 100         if Verbose.level < Verbose.NORMAL:
 101             return
 102         if self.ok:
 103             # All checks passed
 104             return_code = 0
 105             print('The commit message format passed all checks.')
 106         else:
 107             return_code = 1
 108         if not self.ok:
 109             print(self.url)
 110
 111     def error(self, *err):
 112         if self.ok and Verbose.level > Verbose.ONELINE:
 113             print('The commit message format is not valid:')
 114         self.ok = False
 115         if Verbose.level < Verbose.NORMAL:
 116             return
 117         count = 0
 118         for line in err:
 119             prefix = (' *', '  ')[count > 0]
 120             print(prefix, line)
 121             count += 1
 122
 123     # Find 'contributed-under:' at the start of a line ignoring case and
 124     # requires ':' to be present.  Matches if there is white space before
 125     # the tag or between the tag and the ':'.
 126     contributed_under_re = \
 127         re.compile(r'^\s*contributed-under\s*:', re.MULTILINE|re.IGNORECASE)
 128
 129     def check_contributed_under(self):
 130         match = self.contributed_under_re.search(self.msg)
 131         if match is not None:
 132             self.error('Contributed-under! (Note: this must be ' +
 133                        'removed by the code contributor!)')
 134
 135     @staticmethod
 136     def make_signature_re(sig, re_input=False):
 137         if re_input:
 138             sub_re = sig
 139         else:
 140             sub_re = sig.replace('-', r'[-\s]+')
 141         re_str = (r'^(?P<tag>' + sub_re +
 142                   r')(\s*):(\s*)(?P<value>\S.*?)(?:\s*)$')
 143         try:
 144             return re.compile(re_str, re.MULTILINE|re.IGNORECASE)
 145         except Exception:
 146             print("Tried to compile re:", re_str)
 147             raise
 148
 149     sig_block_re = \
 150         re.compile(r'''^
 151                         (?: (?P<tag>[^:]+) \s* : \s*
 152                             (?P<value>\S.*?) )
 153                             |
 154                         (?: \[ (?P<updater>[^:]+) \s* : \s*
 155                                (?P<note>.+?) \s* \] )
 156                     \s* $''',
 157                    re.VERBOSE | re.MULTILINE)
 158
 159     def find_signatures(self, sig):
 160         if not sig.endswith('-by') and sig != 'Cc':
 161             sig += '-by'
 162         regex = self.make_signature_re(sig)
 163
 164         sigs = regex.findall(self.msg)
 165
 166         bad_case_sigs = filter(lambda m: m[0] != sig, sigs)
 167         for s in bad_case_sigs:
 168             self.error("'" +s[0] + "' should be '" + sig + "'")
 169
 170         for s in sigs:
 171             if s[1] != '':
 172                 self.error('There should be no spaces between ' + sig +
 173                            " and the ':'")
 174             if s[2] != ' ':
 175                 self.error("There should be a space after '" + sig + ":'")
 176
 177             EmailAddressCheck(s[3])
 178
 179         return sigs
 180
 181     def check_signed_off_by(self):
 182         sob='Signed-off-by'
 183         if self.msg.find(sob) < 0:
 184             self.error('Missing Signed-off-by! (Note: this must be ' +
 185                        'added by the code contributor!)')
 186             return
 187
 188         sobs = self.find_signatures('Signed-off')
 189
 190         if len(sobs) == 0:
 191             self.error('Invalid Signed-off-by format!')
 192             return
 193
 194     sig_types = (
 195         'Reviewed',
 196         'Reported',
 197         'Tested',
 198         'Suggested',
 199         'Acked',
 200         'Cc'
 201         )
 202
 203     def check_misc_signatures(self):
 204         for sig in self.sig_types:
 205             self.find_signatures(sig)
 206
 207     def check_overall_format(self):
 208         lines = self.msg.splitlines()
 209
 210         if len(lines) >= 1 and lines[0].endswith('\r\n'):
 211             empty_line = '\r\n'
 212         else:
 213             empty_line = '\n'
 214
 215         lines.insert(0, empty_line)
 216         lines.insert(0, self.subject + empty_line)
 217
 218         count = len(lines)
 219
 220         if count <= 0:
 221             self.error('Empty commit message!')
 222             return
 223
 224         if count >= 1 and len(lines[0].rstrip()) >= 72:
 225             self.error('First line of commit message (subject line) ' +
 226                        'is too long.')
 227
 228         if count >= 1 and len(lines[0].strip()) == 0:
 229             self.error('First line of commit message (subject line) ' +
 230                        'is empty.')
 231
 232         if count >= 2 and lines[1].strip() != '':
 233             self.error('Second line of commit message should be ' +
 234                        'empty.')
 235
 236         for i in range(2, count):
 237             if (len(lines[i]) >= 76 and
 238                 len(lines[i].split()) > 1 and
 239                 not lines[i].startswith('git-svn-id:')):
 240                 self.error('Line %d of commit message is too long.' % (i + 1))
 241
 242         last_sig_line = None
 243         for i in range(count - 1, 0, -1):
 244             line = lines[i]
 245             mo = self.sig_block_re.match(line)
 246             if mo is None:
 247                 if line.strip() == '':
 248                     break
 249                 elif last_sig_line is not None:
 250                     err2 = 'Add empty line before "%s"?' % last_sig_line
 251                     self.error('The line before the signature block ' +
 252                                'should be empty', err2)
 253                 else:
 254                     self.error('The signature block was not found')
 255                 break
 256             last_sig_line = line.strip()
 257
 258 (START, PRE_PATCH, PATCH) = range(3)
 259
 260 class GitDiffCheck:
 261     """Checks the contents of a git diff."""
 262
 263     def __init__(self, diff):
 264         self.ok = True
 265         self.format_ok = True
 266         self.lines = diff.splitlines(True)
 267         self.count = len(self.lines)
 268         self.line_num = 0
 269         self.state = START
 270         self.new_bin = []
 271         while self.line_num < self.count and self.format_ok:
 272             line_num = self.line_num
 273             self.run()
 274             assert(self.line_num > line_num)
 275         self.report_message_result()
 276
 277     def report_message_result(self):
 278         if Verbose.level < Verbose.NORMAL:
 279             return
 280         if self.ok:
 281             print('The code passed all checks.')
 282         if self.new_bin:
 283             print('\nWARNING - The following binary files will be added ' +
 284                   'into the repository:')
 285             for binary in self.new_bin:
 286                 print('  ' + binary)
 287
 288     def run(self):
 289         line = self.lines[self.line_num]
 290
 291         if self.state in (PRE_PATCH, PATCH):
 292             if line.startswith('diff --git'):
 293                 self.state = START
 294         if self.state == PATCH:
 295             if line.startswith('@@ '):
 296                 self.state = PRE_PATCH
 297             elif len(line) >= 1 and line[0] not in ' -+' and \
 298                  not line.startswith('\r\n') and  \
 299                  not line.startswith(r'\ No newline ') and not self.binary:
 300                 for line in self.lines[self.line_num + 1:]:
 301                     if line.startswith('diff --git'):
 302                         self.format_error('diff found after end of patch')
 303                         break
 304                 self.line_num = self.count
 305                 return
 306
 307         if self.state == START:
 308             if line.startswith('diff --git'):
 309                 self.state = PRE_PATCH
 310                 self.filename = line[13:].split(' ', 1)[0]
 311                 self.is_newfile = False
 312                 self.force_crlf = not self.filename.endswith('.sh')
 313             elif len(line.rstrip()) != 0:
 314                 self.format_error("didn't find diff command")
 315             self.line_num += 1
 316         elif self.state == PRE_PATCH:
 317             if line.startswith('@@ '):
 318                 self.state = PATCH
 319                 self.binary = False
 320             elif line.startswith('GIT binary patch') or \
 321                  line.startswith('Binary files'):
 322                 self.state = PATCH
 323                 self.binary = True
 324                 if self.is_newfile:
 325                     self.new_bin.append(self.filename)
 326             else:
 327                 ok = False
 328                 self.is_newfile = self.newfile_prefix_re.match(line)
 329                 for pfx in self.pre_patch_prefixes:
 330                     if line.startswith(pfx):
 331                         ok = True
 332                 if not ok:
 333                     self.format_error("didn't find diff hunk marker (@@)")
 334             self.line_num += 1
 335         elif self.state == PATCH:
 336             if self.binary:
 337                 pass
 338             elif line.startswith('-'):
 339                 pass
 340             elif line.startswith('+'):
 341                 self.check_added_line(line[1:])
 342             elif line.startswith('\r\n'):
 343                 pass
 344             elif line.startswith(r'\ No newline '):
 345                 pass
 346             elif not line.startswith(' '):
 347                 self.format_error("unexpected patch line")
 348             self.line_num += 1
 349
 350     pre_patch_prefixes = (
 351         '--- ',
 352         '+++ ',
 353         'index ',
 354         'new file ',
 355         'deleted file ',
 356         'old mode ',
 357         'new mode ',
 358         'similarity index ',
 359         'copy from ',
 360         'copy to ',
 361         'rename ',
 362         )
 363
 364     line_endings = ('\r\n', '\n\r', '\n', '\r')
 365
 366     newfile_prefix_re = \
 367         re.compile(r'''^
 368                        index\ 0+\.\.
 369                    ''',
 370                    re.VERBOSE)
 371
 372     def added_line_error(self, msg, line):
 373         lines = [ msg ]
 374         if self.filename is not None:
 375             lines.append('File: ' + self.filename)
 376         lines.append('Line: ' + line)
 377
 378         self.error(*lines)
 379
 380     old_debug_re = \
 381         re.compile(r'''
 382                         DEBUG \s* \( \s* \( \s*
 383                         (?: DEBUG_[A-Z_]+ \s* \| \s*)*
 384                         EFI_D_ ([A-Z_]+)
 385                    ''',
 386                    re.VERBOSE)
 387
 388     def check_added_line(self, line):
 389         eol = ''
 390         for an_eol in self.line_endings:
 391             if line.endswith(an_eol):
 392                 eol = an_eol
 393                 line = line[:-len(eol)]
 394
 395         stripped = line.rstrip()
 396
 397         if self.force_crlf and eol != '\r\n':
 398             self.added_line_error('Line ending (%s) is not CRLF' % repr(eol),
 399                                   line)
 400         if '\t' in line:
 401             self.added_line_error('Tab character used', line)
 402         if len(stripped) < len(line):
 403             self.added_line_error('Trailing whitespace found', line)
 404
 405         mo = self.old_debug_re.search(line)
 406         if mo is not None:
 407             self.added_line_error('EFI_D_' + mo.group(1) + ' was used, '
 408                                   'but DEBUG_' + mo.group(1) +
 409                                   ' is now recommended', line)
 410
 411     split_diff_re = re.compile(r'''
 412                                    (?P<cmd>
 413                                        ^ diff \s+ --git \s+ a/.+ \s+ b/.+ $
 414                                    )
 415                                    (?P<index>
 416                                        ^ index \s+ .+ $
 417                                    )
 418                                ''',
 419                                re.IGNORECASE | re.VERBOSE | re.MULTILINE)
 420
 421     def format_error(self, err):
 422         self.format_ok = False
 423         err = 'Patch format error: ' + err
 424         err2 = 'Line: ' + self.lines[self.line_num].rstrip()
 425         self.error(err, err2)
 426
 427     def error(self, *err):
 428         if self.ok and Verbose.level > Verbose.ONELINE:
 429             print('Code format is not valid:')
 430         self.ok = False
 431         if Verbose.level < Verbose.NORMAL:
 432             return
 433         count = 0
 434         for line in err:
 435             prefix = (' *', '  ')[count > 0]
 436             print(prefix, line)
 437             count += 1
 438
 439 class CheckOnePatch:
 440     """Checks the contents of a git email formatted patch.
 441
 442     Various checks are performed on both the commit message and the
 443     patch content.
 444     """
 445
 446     def __init__(self, name, patch):
 447         self.patch = patch
 448         self.find_patch_pieces()
 449
 450         msg_check = CommitMessageCheck(self.commit_subject, self.commit_msg)
 451         msg_ok = msg_check.ok
 452
 453         diff_ok = True
 454         if self.diff is not None:
 455             diff_check = GitDiffCheck(self.diff)
 456             diff_ok = diff_check.ok
 457
 458         self.ok = msg_ok and diff_ok
 459
 460         if Verbose.level == Verbose.ONELINE:
 461             if self.ok:
 462                 result = 'ok'
 463             else:
 464                 result = list()
 465                 if not msg_ok:
 466                     result.append('commit message')
 467                 if not diff_ok:
 468                     result.append('diff content')
 469                 result = 'bad ' + ' and '.join(result)
 470             print(name, result)
 471
 472
 473     git_diff_re = re.compile(r'''
 474                                  ^ diff \s+ --git \s+ a/.+ \s+ b/.+ $
 475                              ''',
 476                              re.IGNORECASE | re.VERBOSE | re.MULTILINE)
 477
 478     stat_re = \
 479         re.compile(r'''
 480                        (?P<commit_message> [\s\S\r\n]* )
 481                        (?P<stat>
 482                            ^ --- $ [\r\n]+
 483                            (?: ^ \s+ .+ \s+ \| \s+ \d+ \s+ \+* \-*
 484                                $ [\r\n]+ )+
 485                            [\s\S\r\n]+
 486                        )
 487                    ''',
 488                    re.IGNORECASE | re.VERBOSE | re.MULTILINE)
 489
 490     subject_prefix_re = \
 491         re.compile(r'''^
 492                        \s* (\[
 493                         [^\[\]]* # Allow all non-brackets
 494                        \])* \s*
 495                    ''',
 496                    re.VERBOSE)
 497
 498     def find_patch_pieces(self):
 499         if sys.version_info < (3, 0):
 500             patch = self.patch.encode('ascii', 'ignore')
 501         else:
 502             patch = self.patch
 503
 504         self.commit_msg = None
 505         self.stat = None
 506         self.commit_subject = None
 507         self.commit_prefix = None
 508         self.diff = None
 509
 510         if patch.startswith('diff --git'):
 511             self.diff = patch
 512             return
 513
 514         pmail = email.message_from_string(patch)
 515         parts = list(pmail.walk())
 516         assert(len(parts) == 1)
 517         assert(parts[0].get_content_type() == 'text/plain')
 518         content = parts[0].get_payload(decode=True).decode('utf-8', 'ignore')
 519
 520         mo = self.git_diff_re.search(content)
 521         if mo is not None:
 522             self.diff = content[mo.start():]
 523             content = content[:mo.start()]
 524
 525         mo = self.stat_re.search(content)
 526         if mo is None:
 527             self.commit_msg = content
 528         else:
 529             self.stat = mo.group('stat')
 530             self.commit_msg = mo.group('commit_message')
 531
 532         self.commit_subject = pmail['subject'].replace('\r\n', '')
 533         self.commit_subject = self.commit_subject.replace('\n', '')
 534         self.commit_subject = self.subject_prefix_re.sub('', self.commit_subject, 1)
 535
 536 class CheckGitCommits:
 537     """Reads patches from git based on the specified git revision range.
 538
 539     The patches are read from git, and then checked.
 540     """
 541
 542     def __init__(self, rev_spec, max_count):
 543         commits = self.read_commit_list_from_git(rev_spec, max_count)
 544         if len(commits) == 1 and Verbose.level > Verbose.ONELINE:
 545             commits = [ rev_spec ]
 546         self.ok = True
 547         blank_line = False
 548         for commit in commits:
 549             if Verbose.level > Verbose.ONELINE:
 550                 if blank_line:
 551                     print()
 552                 else:
 553                     blank_line = True
 554                 print('Checking git commit:', commit)
 555             patch = self.read_patch_from_git(commit)
 556             self.ok &= CheckOnePatch(commit, patch).ok
 557         if not commits:
 558             print("Couldn't find commit matching: '{}'".format(rev_spec))
 559
 560     def read_commit_list_from_git(self, rev_spec, max_count):
 561         # Run git to get the commit patch
 562         cmd = [ 'rev-list', '--abbrev-commit', '--no-walk' ]
 563         if max_count is not None:
 564             cmd.append('--max-count=' + str(max_count))
 565         cmd.append(rev_spec)
 566         out = self.run_git(*cmd)
 567         return out.split() if out else []
 568
 569     def read_patch_from_git(self, commit):
 570         # Run git to get the commit patch
 571         return self.run_git('show', '--pretty=email', '--no-textconv', commit)
 572
 573     def run_git(self, *args):
 574         cmd = [ 'git' ]
 575         cmd += args
 576         p = subprocess.Popen(cmd,
 577                      stdout=subprocess.PIPE,
 578                      stderr=subprocess.STDOUT)
 579         Result = p.communicate()
 580         return Result[0].decode('utf-8', 'ignore') if Result[0] and Result[0].find(b"fatal")!=0 else None
 581
 582 class CheckOnePatchFile:
 583     """Performs a patch check for a single file.
 584
 585     stdin is used when the filename is '-'.
 586     """
 587
 588     def __init__(self, patch_filename):
 589         if patch_filename == '-':
 590             patch = sys.stdin.read()
 591             patch_filename = 'stdin'
 592         else:
 593             f = open(patch_filename, 'rb')
 594             patch = f.read().decode('utf-8', 'ignore')
 595             f.close()
 596         if Verbose.level > Verbose.ONELINE:
 597             print('Checking patch file:', patch_filename)
 598         self.ok = CheckOnePatch(patch_filename, patch).ok
 599
 600 class CheckOneArg:
 601     """Performs a patch check for a single command line argument.
 602
 603     The argument will be handed off to a file or git-commit based
 604     checker.
 605     """
 606
 607     def __init__(self, param, max_count=None):
 608         self.ok = True
 609         if param == '-' or os.path.exists(param):
 610             checker = CheckOnePatchFile(param)
 611         else:
 612             checker = CheckGitCommits(param, max_count)
 613         self.ok = checker.ok
 614
 615 class PatchCheckApp:
 616     """Checks patches based on the command line arguments."""
 617
 618     def __init__(self):
 619         self.parse_options()
 620         patches = self.args.patches
 621
 622         if len(patches) == 0:
 623             patches = [ 'HEAD' ]
 624
 625         self.ok = True
 626         self.count = None
 627         for patch in patches:
 628             self.process_one_arg(patch)
 629
 630         if self.count is not None:
 631             self.process_one_arg('HEAD')
 632
 633         if self.ok:
 634             self.retval = 0
 635         else:
 636             self.retval = -1
 637
 638     def process_one_arg(self, arg):
 639         if len(arg) >= 2 and arg[0] == '-':
 640             try:
 641                 self.count = int(arg[1:])
 642                 return
 643             except ValueError:
 644                 pass
 645         self.ok &= CheckOneArg(arg, self.count).ok
 646         self.count = None
 647
 648     def parse_options(self):
 649         parser = argparse.ArgumentParser(description=__copyright__)
 650         parser.add_argument('--version', action='version',
 651                             version='%(prog)s ' + VersionNumber)
 652         parser.add_argument('patches', nargs='*',
 653                             help='[patch file | git rev list]')
 654         group = parser.add_mutually_exclusive_group()
 655         group.add_argument("--oneline",
 656                            action="store_true",
 657                            help="Print one result per line")
 658         group.add_argument("--silent",
 659                            action="store_true",
 660                            help="Print nothing")
 661         self.args = parser.parse_args()
 662         if self.args.oneline:
 663             Verbose.level = Verbose.ONELINE
 664         if self.args.silent:
 665             Verbose.level = Verbose.SILENT
 666
 667 if __name__ == "__main__":
 668     sys.exit(PatchCheckApp().retval)