BaseTools/Scripts/PatchCheck.py

   1 ## @file
   2 #  Check a patch for various format issues
   3 #
   4 #  Copyright (c) 2015, Intel Corporation. All rights reserved.<BR>
   5 #
   6 #  This program and the accompanying materials are licensed and made
   7 #  available under the terms and conditions of the BSD License which
   8 #  accompanies this distribution. The full text of the license may be
   9 #  found at http://opensource.org/licenses/bsd-license.php
  10 #
  11 #  THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS"
  12 #  BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER
  13 #  EXPRESS OR IMPLIED.
  14 #
  15
  16 from __future__ import print_function
  17
  18 VersionNumber = '0.1'
  19 __copyright__ = "Copyright (c) 2015, Intel Corporation  All rights reserved."
  20
  21 import email
  22 import argparse
  23 import os
  24 import re
  25 import subprocess
  26 import sys
  27
  28 class Verbose:
  29     SILENT, ONELINE, NORMAL = range(3)
  30     level = NORMAL
  31
  32 class CommitMessageCheck:
  33     """Checks the contents of a git commit message."""
  34
  35     def __init__(self, subject, message):
  36         self.ok = True
  37
  38         if subject is None and  message is None:
  39             self.error('Commit message is missing!')
  40             return
  41
  42         self.subject = subject
  43         self.msg = message
  44
  45         self.check_contributed_under()
  46         self.check_signed_off_by()
  47         self.check_misc_signatures()
  48         self.check_overall_format()
  49         self.report_message_result()
  50
  51     url = 'https://github.com/tianocore/tianocore.github.io/wiki/Commit-Message-Format'
  52
  53     def report_message_result(self):
  54         if Verbose.level < Verbose.NORMAL:
  55             return
  56         if self.ok:
  57             # All checks passed
  58             return_code = 0
  59             print('The commit message format passed all checks.')
  60         else:
  61             return_code = 1
  62         if not self.ok:
  63             print(self.url)
  64
  65     def error(self, *err):
  66         if self.ok and Verbose.level > Verbose.ONELINE:
  67             print('The commit message format is not valid:')
  68         self.ok = False
  69         if Verbose.level < Verbose.NORMAL:
  70             return
  71         count = 0
  72         for line in err:
  73             prefix = (' *', '  ')[count > 0]
  74             print(prefix, line)
  75             count += 1
  76
  77     def check_contributed_under(self):
  78         cu_msg='Contributed-under: TianoCore Contribution Agreement 1.0'
  79         if self.msg.find(cu_msg) < 0:
  80             self.error('Missing Contributed-under! (Note: this must be ' +
  81                        'added by the code contributor!)')
  82
  83     @staticmethod
  84     def make_signature_re(sig, re_input=False):
  85         if re_input:
  86             sub_re = sig
  87         else:
  88             sub_re = sig.replace('-', r'[-\s]+')
  89         re_str = (r'^(?P<tag>' + sub_re +
  90                   r')(\s*):(\s*)(?P<value>\S.*?)(?:\s*)$')
  91         try:
  92             return re.compile(re_str, re.MULTILINE|re.IGNORECASE)
  93         except Exception:
  94             print("Tried to compile re:", re_str)
  95             raise
  96
  97     sig_block_re = \
  98         re.compile(r'''^
  99                         (?: (?P<tag>[^:]+) \s* : \s*
 100                             (?P<value>\S.*?) )
 101                             |
 102                         (?: \[ (?P<updater>[^:]+) \s* : \s*
 103                                (?P<note>.+?) \s* \] )
 104                     \s* $''',
 105                    re.VERBOSE | re.MULTILINE)
 106
 107     def find_signatures(self, sig):
 108         if not sig.endswith('-by') and sig != 'Cc':
 109             sig += '-by'
 110         regex = self.make_signature_re(sig)
 111
 112         sigs = regex.findall(self.msg)
 113
 114         bad_case_sigs = filter(lambda m: m[0] != sig, sigs)
 115         for s in bad_case_sigs:
 116             self.error("'" +s[0] + "' should be '" + sig + "'")
 117
 118         for s in sigs:
 119             if s[1] != '':
 120                 self.error('There should be no spaces between ' + sig +
 121                            " and the ':'")
 122             if s[2] != ' ':
 123                 self.error("There should be a space after '" + sig + ":'")
 124
 125             self.check_email_address(s[3])
 126
 127         return sigs
 128
 129     email_re1 = re.compile(r'(?:\s*)(.*?)(\s*)<(.+)>\s*$',
 130                            re.MULTILINE|re.IGNORECASE)
 131
 132     def check_email_address(self, email):
 133         email = email.strip()
 134         mo = self.email_re1.match(email)
 135         if mo is None:
 136             self.error("Email format is invalid: " + email.strip())
 137             return
 138
 139         name = mo.group(1).strip()
 140         if name == '':
 141             self.error("Name is not provided with email address: " +
 142                        email)
 143         else:
 144             quoted = len(name) > 2 and name[0] == '"' and name[-1] == '"'
 145             if name.find(',') >= 0 and not quoted:
 146                 self.error('Add quotes (") around name with a comma: ' +
 147                            name)
 148
 149         if mo.group(2) == '':
 150             self.error("There should be a space between the name and " +
 151                        "email address: " + email)
 152
 153         if mo.group(3).find(' ') >= 0:
 154             self.error("The email address cannot contain a space: " +
 155                        mo.group(3))
 156
 157     def check_signed_off_by(self):
 158         sob='Signed-off-by'
 159         if self.msg.find(sob) < 0:
 160             self.error('Missing Signed-off-by! (Note: this must be ' +
 161                        'added by the code contributor!)')
 162             return
 163
 164         sobs = self.find_signatures('Signed-off')
 165
 166         if len(sobs) == 0:
 167             self.error('Invalid Signed-off-by format!')
 168             return
 169
 170     sig_types = (
 171         'Reviewed',
 172         'Reported',
 173         'Tested',
 174         'Suggested',
 175         'Acked',
 176         'Cc'
 177         )
 178
 179     def check_misc_signatures(self):
 180         for sig in self.sig_types:
 181             self.find_signatures(sig)
 182
 183     def check_overall_format(self):
 184         lines = self.msg.splitlines()
 185
 186         if len(lines) >= 1 and lines[0].endswith('\r\n'):
 187             empty_line = '\r\n'
 188         else:
 189             empty_line = '\n'
 190
 191         lines.insert(0, empty_line)
 192         lines.insert(0, self.subject + empty_line)
 193
 194         count = len(lines)
 195
 196         if count <= 0:
 197             self.error('Empty commit message!')
 198             return
 199
 200         if count >= 1 and len(lines[0]) > 76:
 201             self.error('First line of commit message (subject line) ' +
 202                        'is too long.')
 203
 204         if count >= 1 and len(lines[0].strip()) == 0:
 205             self.error('First line of commit message (subject line) ' +
 206                        'is empty.')
 207
 208         if count >= 2 and lines[1].strip() != '':
 209             self.error('Second line of commit message should be ' +
 210                        'empty.')
 211
 212         for i in range(2, count):
 213             if (len(lines[i]) > 76 and
 214                 len(lines[i].split()) > 1 and
 215                 not lines[i].startswith('git-svn-id:')):
 216                 self.error('Line %d of commit message is too long.' % (i + 1))
 217
 218         last_sig_line = None
 219         for i in range(count - 1, 0, -1):
 220             line = lines[i]
 221             mo = self.sig_block_re.match(line)
 222             if mo is None:
 223                 if line.strip() == '':
 224                     break
 225                 elif last_sig_line is not None:
 226                     err2 = 'Add empty line before "%s"?' % last_sig_line
 227                     self.error('The line before the signature block ' +
 228                                'should be empty', err2)
 229                 else:
 230                     self.error('The signature block was not found')
 231                 break
 232             last_sig_line = line.strip()
 233
 234 (START, PRE_PATCH, PATCH) = range(3)
 235
 236 class GitDiffCheck:
 237     """Checks the contents of a git diff."""
 238
 239     def __init__(self, diff):
 240         self.ok = True
 241         self.format_ok = True
 242         self.lines = diff.splitlines(True)
 243         self.count = len(self.lines)
 244         self.line_num = 0
 245         self.state = START
 246         while self.line_num < self.count and self.format_ok:
 247             line_num = self.line_num
 248             self.run()
 249             assert(self.line_num > line_num)
 250         self.report_message_result()
 251
 252     def report_message_result(self):
 253         if Verbose.level < Verbose.NORMAL:
 254             return
 255         if self.ok:
 256             print('The code passed all checks.')
 257
 258     def run(self):
 259         line = self.lines[self.line_num]
 260
 261         if self.state in (PRE_PATCH, PATCH):
 262             if line.startswith('diff --git'):
 263                 self.state = START
 264         if self.state == PATCH:
 265             if line.startswith('@@ '):
 266                 self.state = PRE_PATCH
 267             elif len(line) >= 1 and line[0] not in ' -+' and \
 268                  not line.startswith(r'\ No newline '):
 269                 for line in self.lines[self.line_num + 1:]:
 270                     if line.startswith('diff --git'):
 271                         self.format_error('diff found after end of patch')
 272                         break
 273                 self.line_num = self.count
 274                 return
 275
 276         if self.state == START:
 277             if line.startswith('diff --git'):
 278                 self.state = PRE_PATCH
 279                 self.set_filename(None)
 280             elif len(line.rstrip()) != 0:
 281                 self.format_error("didn't find diff command")
 282             self.line_num += 1
 283         elif self.state == PRE_PATCH:
 284             if line.startswith('+++ b/'):
 285                 self.set_filename(line[6:].rstrip())
 286             if line.startswith('@@ '):
 287                 self.state = PATCH
 288             else:
 289                 ok = False
 290                 for pfx in self.pre_patch_prefixes:
 291                     if line.startswith(pfx):
 292                         ok = True
 293                 if not ok:
 294                     self.format_error("didn't find diff hunk marker (@@)")
 295             self.line_num += 1
 296         elif self.state == PATCH:
 297             if line.startswith('-'):
 298                 pass
 299             elif line.startswith('+'):
 300                 self.check_added_line(line[1:])
 301             elif line.startswith(r'\ No newline '):
 302                 pass
 303             elif not line.startswith(' '):
 304                 self.format_error("unexpected patch line")
 305             self.line_num += 1
 306
 307     pre_patch_prefixes = (
 308         '--- ',
 309         '+++ ',
 310         'index ',
 311         'new file ',
 312         'deleted file ',
 313         'old mode ',
 314         'new mode ',
 315         'similarity index ',
 316         'rename ',
 317         'Binary files ',
 318         )
 319
 320     line_endings = ('\r\n', '\n\r', '\n', '\r')
 321
 322     def set_filename(self, filename):
 323         self.hunk_filename = filename
 324         if filename:
 325             self.force_crlf = not filename.endswith('.sh')
 326         else:
 327             self.force_crlf = True
 328
 329     def added_line_error(self, msg, line):
 330         lines = [ msg ]
 331         if self.hunk_filename is not None:
 332             lines.append('File: ' + self.hunk_filename)
 333         lines.append('Line: ' + line)
 334
 335         self.error(*lines)
 336
 337     def check_added_line(self, line):
 338         eol = ''
 339         for an_eol in self.line_endings:
 340             if line.endswith(an_eol):
 341                 eol = an_eol
 342                 line = line[:-len(eol)]
 343
 344         stripped = line.rstrip()
 345
 346         if self.force_crlf and eol != '\r\n':
 347             self.added_line_error('Line ending (%s) is not CRLF' % repr(eol),
 348                                   line)
 349         if '\t' in line:
 350             self.added_line_error('Tab character used', line)
 351         if len(stripped) < len(line):
 352             self.added_line_error('Trailing whitespace found', line)
 353
 354     split_diff_re = re.compile(r'''
 355                                    (?P<cmd>
 356                                        ^ diff \s+ --git \s+ a/.+ \s+ b/.+ $
 357                                    )
 358                                    (?P<index>
 359                                        ^ index \s+ .+ $
 360                                    )
 361                                ''',
 362                                re.IGNORECASE | re.VERBOSE | re.MULTILINE)
 363
 364     def format_error(self, err):
 365         self.format_ok = False
 366         err = 'Patch format error: ' + err
 367         err2 = 'Line: ' + self.lines[self.line_num].rstrip()
 368         self.error(err, err2)
 369
 370     def error(self, *err):
 371         if self.ok and Verbose.level > Verbose.ONELINE:
 372             print('Code format is not valid:')
 373         self.ok = False
 374         if Verbose.level < Verbose.NORMAL:
 375             return
 376         count = 0
 377         for line in err:
 378             prefix = (' *', '  ')[count > 0]
 379             print(prefix, line)
 380             count += 1
 381
 382 class CheckOnePatch:
 383     """Checks the contents of a git email formatted patch.
 384
 385     Various checks are performed on both the commit message and the
 386     patch content.
 387     """
 388
 389     def __init__(self, name, patch):
 390         self.patch = patch
 391         self.find_patch_pieces()
 392
 393         msg_check = CommitMessageCheck(self.commit_subject, self.commit_msg)
 394         msg_ok = msg_check.ok
 395
 396         diff_ok = True
 397         if self.diff is not None:
 398             diff_check = GitDiffCheck(self.diff)
 399             diff_ok = diff_check.ok
 400
 401         self.ok = msg_ok and diff_ok
 402
 403         if Verbose.level == Verbose.ONELINE:
 404             if self.ok:
 405                 result = 'ok'
 406             else:
 407                 result = list()
 408                 if not msg_ok:
 409                     result.append('commit message')
 410                 if not diff_ok:
 411                     result.append('diff content')
 412                 result = 'bad ' + ' and '.join(result)
 413             print(name, result)
 414
 415
 416     git_diff_re = re.compile(r'''
 417                                  ^ diff \s+ --git \s+ a/.+ \s+ b/.+ $
 418                              ''',
 419                              re.IGNORECASE | re.VERBOSE | re.MULTILINE)
 420
 421     stat_re = \
 422         re.compile(r'''
 423                        (?P<commit_message> [\s\S\r\n]* )
 424                        (?P<stat>
 425                            ^ --- $ [\r\n]+
 426                            (?: ^ \s+ .+ \s+ \| \s+ \d+ \s+ \+* \-*
 427                                $ [\r\n]+ )+
 428                            [\s\S\r\n]+
 429                        )
 430                    ''',
 431                    re.IGNORECASE | re.VERBOSE | re.MULTILINE)
 432
 433     def find_patch_pieces(self):
 434         if sys.version_info < (3, 0):
 435             patch = self.patch.encode('ascii', 'ignore')
 436         else:
 437             patch = self.patch
 438
 439         self.commit_msg = None
 440         self.stat = None
 441         self.commit_subject = None
 442         self.commit_prefix = None
 443         self.diff = None
 444
 445         if patch.startswith('diff --git'):
 446             self.diff = patch
 447             return
 448
 449         pmail = email.message_from_string(patch)
 450         parts = list(pmail.walk())
 451         assert(len(parts) == 1)
 452         assert(parts[0].get_content_type() == 'text/plain')
 453         content = parts[0].get_payload(decode=True).decode('utf-8', 'ignore')
 454
 455         mo = self.git_diff_re.search(content)
 456         if mo is not None:
 457             self.diff = content[mo.start():]
 458             content = content[:mo.start()]
 459
 460         mo = self.stat_re.search(content)
 461         if mo is None:
 462             self.commit_msg = content
 463         else:
 464             self.stat = mo.group('stat')
 465             self.commit_msg = mo.group('commit_message')
 466
 467         self.commit_subject = pmail['subject'].replace('\r\n', '')
 468         self.commit_subject = self.commit_subject.replace('\n', '')
 469
 470         pfx_start = self.commit_subject.find('[')
 471         if pfx_start >= 0:
 472             pfx_end = self.commit_subject.find(']')
 473             if pfx_end > pfx_start:
 474                 self.commit_prefix = self.commit_subject[pfx_start + 1 : pfx_end]
 475                 self.commit_subject = self.commit_subject[pfx_end + 1 :].lstrip()
 476
 477
 478 class CheckGitCommits:
 479     """Reads patches from git based on the specified git revision range.
 480
 481     The patches are read from git, and then checked.
 482     """
 483
 484     def __init__(self, rev_spec, max_count):
 485         commits = self.read_commit_list_from_git(rev_spec, max_count)
 486         if len(commits) == 1 and Verbose.level > Verbose.ONELINE:
 487             commits = [ rev_spec ]
 488         self.ok = True
 489         blank_line = False
 490         for commit in commits:
 491             if Verbose.level > Verbose.ONELINE:
 492                 if blank_line:
 493                     print()
 494                 else:
 495                     blank_line = True
 496                 print('Checking git commit:', commit)
 497             patch = self.read_patch_from_git(commit)
 498             self.ok &= CheckOnePatch(commit, patch).ok
 499
 500     def read_commit_list_from_git(self, rev_spec, max_count):
 501         # Run git to get the commit patch
 502         cmd = [ 'rev-list', '--abbrev-commit', '--no-walk' ]
 503         if max_count is not None:
 504             cmd.append('--max-count=' + str(max_count))
 505         cmd.append(rev_spec)
 506         out = self.run_git(*cmd)
 507         return out.split()
 508
 509     def read_patch_from_git(self, commit):
 510         # Run git to get the commit patch
 511         return self.run_git('show', '--pretty=email', commit)
 512
 513     def run_git(self, *args):
 514         cmd = [ 'git' ]
 515         cmd += args
 516         p = subprocess.Popen(cmd,
 517                      stdout=subprocess.PIPE,
 518                      stderr=subprocess.STDOUT)
 519         return p.communicate()[0].decode('utf-8', 'ignore')
 520
 521 class CheckOnePatchFile:
 522     """Performs a patch check for a single file.
 523
 524     stdin is used when the filename is '-'.
 525     """
 526
 527     def __init__(self, patch_filename):
 528         if patch_filename == '-':
 529             patch = sys.stdin.read()
 530             patch_filename = 'stdin'
 531         else:
 532             f = open(patch_filename, 'rb')
 533             patch = f.read().decode('utf-8', 'ignore')
 534             f.close()
 535         if Verbose.level > Verbose.ONELINE:
 536             print('Checking patch file:', patch_filename)
 537         self.ok = CheckOnePatch(patch_filename, patch).ok
 538
 539 class CheckOneArg:
 540     """Performs a patch check for a single command line argument.
 541
 542     The argument will be handed off to a file or git-commit based
 543     checker.
 544     """
 545
 546     def __init__(self, param, max_count=None):
 547         self.ok = True
 548         if param == '-' or os.path.exists(param):
 549             checker = CheckOnePatchFile(param)
 550         else:
 551             checker = CheckGitCommits(param, max_count)
 552         self.ok = checker.ok
 553
 554 class PatchCheckApp:
 555     """Checks patches based on the command line arguments."""
 556
 557     def __init__(self):
 558         self.parse_options()
 559         patches = self.args.patches
 560
 561         if len(patches) == 0:
 562             patches = [ 'HEAD' ]
 563
 564         self.ok = True
 565         self.count = None
 566         for patch in patches:
 567             self.process_one_arg(patch)
 568
 569         if self.count is not None:
 570             self.process_one_arg('HEAD')
 571
 572         if self.ok:
 573             self.retval = 0
 574         else:
 575             self.retval = -1
 576
 577     def process_one_arg(self, arg):
 578         if len(arg) >= 2 and arg[0] == '-':
 579             try:
 580                 self.count = int(arg[1:])
 581                 return
 582             except ValueError:
 583                 pass
 584         self.ok &= CheckOneArg(arg, self.count).ok
 585         self.count = None
 586
 587     def parse_options(self):
 588         parser = argparse.ArgumentParser(description=__copyright__)
 589         parser.add_argument('--version', action='version',
 590                             version='%(prog)s ' + VersionNumber)
 591         parser.add_argument('patches', nargs='*',
 592                             help='[patch file | git rev list]')
 593         group = parser.add_mutually_exclusive_group()
 594         group.add_argument("--oneline",
 595                            action="store_true",
 596                            help="Print one result per line")
 597         group.add_argument("--silent",
 598                            action="store_true",
 599                            help="Print nothing")
 600         self.args = parser.parse_args()
 601         if self.args.oneline:
 602             Verbose.level = Verbose.ONELINE
 603         if self.args.silent:
 604             Verbose.level = Verbose.SILENT
 605
 606 if __name__ == "__main__":
 607     sys.exit(PatchCheckApp().retval)