BaseTools:Fix GenFds issue for BuildOption replace GenFdsOption

[mirror_edk2.git] / BaseTools / Scripts / PatchCheck.py

## @file\r
#  Check a patch for various format issues\r
#\r
#  Copyright (c) 2015 - 2019, Intel Corporation. All rights reserved.<BR>\r
#  Copyright (C) 2020, Red Hat, Inc.<BR>\r
#\r
#  SPDX-License-Identifier: BSD-2-Clause-Patent\r
#\r
\r
from __future__ import print_function\r
\r
VersionNumber = '0.1'\r
__copyright__ = "Copyright (c) 2015 - 2016, Intel Corporation  All rights reserved."\r
\r
import email\r
import argparse\r
import os\r
import re\r
import subprocess\r
import sys\r
\r
class Verbose:\r
    SILENT, ONELINE, NORMAL = range(3)\r
    level = NORMAL\r
\r
class EmailAddressCheck:\r
    """Checks an email address."""\r
\r
    def __init__(self, email, description):\r
        self.ok = True\r
\r
        if email is None:\r
            self.error('Email address is missing!')\r
            return\r
        if description is None:\r
            self.error('Email description is missing!')\r
            return\r
\r
        self.description = "'" + description + "'"\r
        self.check_email_address(email)\r
\r
    def error(self, *err):\r
        if self.ok and Verbose.level > Verbose.ONELINE:\r
            print('The ' + self.description + ' email address is not valid:')\r
        self.ok = False\r
        if Verbose.level < Verbose.NORMAL:\r
            return\r
        count = 0\r
        for line in err:\r
            prefix = (' *', '  ')[count > 0]\r
            print(prefix, line)\r
            count += 1\r
\r
    email_re1 = re.compile(r'(?:\s*)(.*?)(\s*)<(.+)>\s*$',\r
                           re.MULTILINE|re.IGNORECASE)\r
\r
    def check_email_address(self, email):\r
        email = email.strip()\r
        mo = self.email_re1.match(email)\r
        if mo is None:\r
            self.error("Email format is invalid: " + email.strip())\r
            return\r
\r
        name = mo.group(1).strip()\r
        if name == '':\r
            self.error("Name is not provided with email address: " +\r
                       email)\r
        else:\r
            quoted = len(name) > 2 and name[0] == '"' and name[-1] == '"'\r
            if name.find(',') >= 0 and not quoted:\r
                self.error('Add quotes (") around name with a comma: ' +\r
                           name)\r
\r
        if mo.group(2) == '':\r
            self.error("There should be a space between the name and " +\r
                       "email address: " + email)\r
\r
        if mo.group(3).find(' ') >= 0:\r
            self.error("The email address cannot contain a space: " +\r
                       mo.group(3))\r
\r
class CommitMessageCheck:\r
    """Checks the contents of a git commit message."""\r
\r
    def __init__(self, subject, message):\r
        self.ok = True\r
\r
        if subject is None and  message is None:\r
            self.error('Commit message is missing!')\r
            return\r
\r
        self.subject = subject\r
        self.msg = message\r
\r
        self.check_contributed_under()\r
        self.check_signed_off_by()\r
        self.check_misc_signatures()\r
        self.check_overall_format()\r
        self.report_message_result()\r
\r
    url = 'https://github.com/tianocore/tianocore.github.io/wiki/Commit-Message-Format'\r
\r
    def report_message_result(self):\r
        if Verbose.level < Verbose.NORMAL:\r
            return\r
        if self.ok:\r
            # All checks passed\r
            return_code = 0\r
            print('The commit message format passed all checks.')\r
        else:\r
            return_code = 1\r
        if not self.ok:\r
            print(self.url)\r
\r
    def error(self, *err):\r
        if self.ok and Verbose.level > Verbose.ONELINE:\r
            print('The commit message format is not valid:')\r
        self.ok = False\r
        if Verbose.level < Verbose.NORMAL:\r
            return\r
        count = 0\r
        for line in err:\r
            prefix = (' *', '  ')[count > 0]\r
            print(prefix, line)\r
            count += 1\r
\r
    # Find 'contributed-under:' at the start of a line ignoring case and\r
    # requires ':' to be present.  Matches if there is white space before\r
    # the tag or between the tag and the ':'.\r
    contributed_under_re = \\r
        re.compile(r'^\s*contributed-under\s*:', re.MULTILINE|re.IGNORECASE)\r
\r
    def check_contributed_under(self):\r
        match = self.contributed_under_re.search(self.msg)\r
        if match is not None:\r
            self.error('Contributed-under! (Note: this must be ' +\r
                       'removed by the code contributor!)')\r
\r
    @staticmethod\r
    def make_signature_re(sig, re_input=False):\r
        if re_input:\r
            sub_re = sig\r
        else:\r
            sub_re = sig.replace('-', r'[-\s]+')\r
        re_str = (r'^(?P<tag>' + sub_re +\r
                  r')(\s*):(\s*)(?P<value>\S.*?)(?:\s*)$')\r
        try:\r
            return re.compile(re_str, re.MULTILINE|re.IGNORECASE)\r
        except Exception:\r
            print("Tried to compile re:", re_str)\r
            raise\r
\r
    sig_block_re = \\r
        re.compile(r'''^\r
                        (?: (?P<tag>[^:]+) \s* : \s*\r
                            (?P<value>\S.*?) )\r
                            |\r
                        (?: \[ (?P<updater>[^:]+) \s* : \s*\r
                               (?P<note>.+?) \s* \] )\r
                    \s* $''',\r
                   re.VERBOSE | re.MULTILINE)\r
\r
    def find_signatures(self, sig):\r
        if not sig.endswith('-by') and sig != 'Cc':\r
            sig += '-by'\r
        regex = self.make_signature_re(sig)\r
\r
        sigs = regex.findall(self.msg)\r
\r
        bad_case_sigs = filter(lambda m: m[0] != sig, sigs)\r
        for s in bad_case_sigs:\r
            self.error("'" +s[0] + "' should be '" + sig + "'")\r
\r
        for s in sigs:\r
            if s[1] != '':\r
                self.error('There should be no spaces between ' + sig +\r
                           " and the ':'")\r
            if s[2] != ' ':\r
                self.error("There should be a space after '" + sig + ":'")\r
\r
            EmailAddressCheck(s[3], sig)\r
\r
        return sigs\r
\r
    def check_signed_off_by(self):\r
        sob='Signed-off-by'\r
        if self.msg.find(sob) < 0:\r
            self.error('Missing Signed-off-by! (Note: this must be ' +\r
                       'added by the code contributor!)')\r
            return\r
\r
        sobs = self.find_signatures('Signed-off')\r
\r
        if len(sobs) == 0:\r
            self.error('Invalid Signed-off-by format!')\r
            return\r
\r
    sig_types = (\r
        'Reviewed',\r
        'Reported',\r
        'Tested',\r
        'Suggested',\r
        'Acked',\r
        'Cc'\r
        )\r
\r
    def check_misc_signatures(self):\r
        for sig in self.sig_types:\r
            self.find_signatures(sig)\r
\r
    def check_overall_format(self):\r
        lines = self.msg.splitlines()\r
\r
        if len(lines) >= 1 and lines[0].endswith('\r\n'):\r
            empty_line = '\r\n'\r
        else:\r
            empty_line = '\n'\r
\r
        lines.insert(0, empty_line)\r
        lines.insert(0, self.subject + empty_line)\r
\r
        count = len(lines)\r
\r
        if count <= 0:\r
            self.error('Empty commit message!')\r
            return\r
\r
        if count >= 1 and len(lines[0].rstrip()) >= 72:\r
            self.error('First line of commit message (subject line) ' +\r
                       'is too long.')\r
\r
        if count >= 1 and len(lines[0].strip()) == 0:\r
            self.error('First line of commit message (subject line) ' +\r
                       'is empty.')\r
\r
        if count >= 2 and lines[1].strip() != '':\r
            self.error('Second line of commit message should be ' +\r
                       'empty.')\r
\r
        for i in range(2, count):\r
            if (len(lines[i]) >= 76 and\r
                len(lines[i].split()) > 1 and\r
                not lines[i].startswith('git-svn-id:')):\r
                self.error('Line %d of commit message is too long.' % (i + 1))\r
\r
        last_sig_line = None\r
        for i in range(count - 1, 0, -1):\r
            line = lines[i]\r
            mo = self.sig_block_re.match(line)\r
            if mo is None:\r
                if line.strip() == '':\r
                    break\r
                elif last_sig_line is not None:\r
                    err2 = 'Add empty line before "%s"?' % last_sig_line\r
                    self.error('The line before the signature block ' +\r
                               'should be empty', err2)\r
                else:\r
                    self.error('The signature block was not found')\r
                break\r
            last_sig_line = line.strip()\r
\r
(START, PRE_PATCH, PATCH) = range(3)\r
\r
class GitDiffCheck:\r
    """Checks the contents of a git diff."""\r
\r
    def __init__(self, diff):\r
        self.ok = True\r
        self.format_ok = True\r
        self.lines = diff.splitlines(True)\r
        self.count = len(self.lines)\r
        self.line_num = 0\r
        self.state = START\r
        self.new_bin = []\r
        while self.line_num < self.count and self.format_ok:\r
            line_num = self.line_num\r
            self.run()\r
            assert(self.line_num > line_num)\r
        self.report_message_result()\r
\r
    def report_message_result(self):\r
        if Verbose.level < Verbose.NORMAL:\r
            return\r
        if self.ok:\r
            print('The code passed all checks.')\r
        if self.new_bin:\r
            print('\nWARNING - The following binary files will be added ' +\r
                  'into the repository:')\r
            for binary in self.new_bin:\r
                print('  ' + binary)\r
\r
    def run(self):\r
        line = self.lines[self.line_num]\r
\r
        if self.state in (PRE_PATCH, PATCH):\r
            if line.startswith('diff --git'):\r
                self.state = START\r
        if self.state == PATCH:\r
            if line.startswith('@@ '):\r
                self.state = PRE_PATCH\r
            elif len(line) >= 1 and line[0] not in ' -+' and \\r
                 not line.startswith('\r\n') and  \\r
                 not line.startswith(r'\ No newline ') and not self.binary:\r
                for line in self.lines[self.line_num + 1:]:\r
                    if line.startswith('diff --git'):\r
                        self.format_error('diff found after end of patch')\r
                        break\r
                self.line_num = self.count\r
                return\r
\r
        if self.state == START:\r
            if line.startswith('diff --git'):\r
                self.state = PRE_PATCH\r
                self.filename = line[13:].split(' ', 1)[0]\r
                self.is_newfile = False\r
                self.force_crlf = not self.filename.endswith('.sh')\r
            elif len(line.rstrip()) != 0:\r
                self.format_error("didn't find diff command")\r
            self.line_num += 1\r
        elif self.state == PRE_PATCH:\r
            if line.startswith('@@ '):\r
                self.state = PATCH\r
                self.binary = False\r
            elif line.startswith('GIT binary patch') or \\r
                 line.startswith('Binary files'):\r
                self.state = PATCH\r
                self.binary = True\r
                if self.is_newfile:\r
                    self.new_bin.append(self.filename)\r
            else:\r
                ok = False\r
                self.is_newfile = self.newfile_prefix_re.match(line)\r
                for pfx in self.pre_patch_prefixes:\r
                    if line.startswith(pfx):\r
                        ok = True\r
                if not ok:\r
                    self.format_error("didn't find diff hunk marker (@@)")\r
            self.line_num += 1\r
        elif self.state == PATCH:\r
            if self.binary:\r
                pass\r
            elif line.startswith('-'):\r
                pass\r
            elif line.startswith('+'):\r
                self.check_added_line(line[1:])\r
            elif line.startswith('\r\n'):\r
                pass\r
            elif line.startswith(r'\ No newline '):\r
                pass\r
            elif not line.startswith(' '):\r
                self.format_error("unexpected patch line")\r
            self.line_num += 1\r
\r
    pre_patch_prefixes = (\r
        '--- ',\r
        '+++ ',\r
        'index ',\r
        'new file ',\r
        'deleted file ',\r
        'old mode ',\r
        'new mode ',\r
        'similarity index ',\r
        'copy from ',\r
        'copy to ',\r
        'rename ',\r
        )\r
\r
    line_endings = ('\r\n', '\n\r', '\n', '\r')\r
\r
    newfile_prefix_re = \\r
        re.compile(r'''^\r
                       index\ 0+\.\.\r
                   ''',\r
                   re.VERBOSE)\r
\r
    def added_line_error(self, msg, line):\r
        lines = [ msg ]\r
        if self.filename is not None:\r
            lines.append('File: ' + self.filename)\r
        lines.append('Line: ' + line)\r
\r
        self.error(*lines)\r
\r
    old_debug_re = \\r
        re.compile(r'''\r
                        DEBUG \s* \( \s* \( \s*\r
                        (?: DEBUG_[A-Z_]+ \s* \| \s*)*\r
                        EFI_D_ ([A-Z_]+)\r
                   ''',\r
                   re.VERBOSE)\r
\r
    def check_added_line(self, line):\r
        eol = ''\r
        for an_eol in self.line_endings:\r
            if line.endswith(an_eol):\r
                eol = an_eol\r
                line = line[:-len(eol)]\r
\r
        stripped = line.rstrip()\r
\r
        if self.force_crlf and eol != '\r\n':\r
            self.added_line_error('Line ending (%s) is not CRLF' % repr(eol),\r
                                  line)\r
        if '\t' in line:\r
            self.added_line_error('Tab character used', line)\r
        if len(stripped) < len(line):\r
            self.added_line_error('Trailing whitespace found', line)\r
\r
        mo = self.old_debug_re.search(line)\r
        if mo is not None:\r
            self.added_line_error('EFI_D_' + mo.group(1) + ' was used, '\r
                                  'but DEBUG_' + mo.group(1) +\r
                                  ' is now recommended', line)\r
\r
    split_diff_re = re.compile(r'''\r
                                   (?P<cmd>\r
                                       ^ diff \s+ --git \s+ a/.+ \s+ b/.+ $\r
                                   )\r
                                   (?P<index>\r
                                       ^ index \s+ .+ $\r
                                   )\r
                               ''',\r
                               re.IGNORECASE | re.VERBOSE | re.MULTILINE)\r
\r
    def format_error(self, err):\r
        self.format_ok = False\r
        err = 'Patch format error: ' + err\r
        err2 = 'Line: ' + self.lines[self.line_num].rstrip()\r
        self.error(err, err2)\r
\r
    def error(self, *err):\r
        if self.ok and Verbose.level > Verbose.ONELINE:\r
            print('Code format is not valid:')\r
        self.ok = False\r
        if Verbose.level < Verbose.NORMAL:\r
            return\r
        count = 0\r
        for line in err:\r
            prefix = (' *', '  ')[count > 0]\r
            print(prefix, line)\r
            count += 1\r
\r
class CheckOnePatch:\r
    """Checks the contents of a git email formatted patch.\r
\r
    Various checks are performed on both the commit message and the\r
    patch content.\r
    """\r
\r
    def __init__(self, name, patch):\r
        self.patch = patch\r
        self.find_patch_pieces()\r
\r
        email_check = EmailAddressCheck(self.author_email, 'Author')\r
        email_ok = email_check.ok\r
\r
        msg_check = CommitMessageCheck(self.commit_subject, self.commit_msg)\r
        msg_ok = msg_check.ok\r
\r
        diff_ok = True\r
        if self.diff is not None:\r
            diff_check = GitDiffCheck(self.diff)\r
            diff_ok = diff_check.ok\r
\r
        self.ok = email_ok and msg_ok and diff_ok\r
\r
        if Verbose.level == Verbose.ONELINE:\r
            if self.ok:\r
                result = 'ok'\r
            else:\r
                result = list()\r
                if not msg_ok:\r
                    result.append('commit message')\r
                if not diff_ok:\r
                    result.append('diff content')\r
                result = 'bad ' + ' and '.join(result)\r
            print(name, result)\r
\r
\r
    git_diff_re = re.compile(r'''\r
                                 ^ diff \s+ --git \s+ a/.+ \s+ b/.+ $\r
                             ''',\r
                             re.IGNORECASE | re.VERBOSE | re.MULTILINE)\r
\r
    stat_re = \\r
        re.compile(r'''\r
                       (?P<commit_message> [\s\S\r\n]* )\r
                       (?P<stat>\r
                           ^ --- $ [\r\n]+\r
                           (?: ^ \s+ .+ \s+ \| \s+ \d+ \s+ \+* \-*\r
                               $ [\r\n]+ )+\r
                           [\s\S\r\n]+\r
                       )\r
                   ''',\r
                   re.IGNORECASE | re.VERBOSE | re.MULTILINE)\r
\r
    subject_prefix_re = \\r
        re.compile(r'''^\r
                       \s* (\[\r
                        [^\[\]]* # Allow all non-brackets\r
                       \])* \s*\r
                   ''',\r
                   re.VERBOSE)\r
\r
    def find_patch_pieces(self):\r
        if sys.version_info < (3, 0):\r
            patch = self.patch.encode('ascii', 'ignore')\r
        else:\r
            patch = self.patch\r
\r
        self.commit_msg = None\r
        self.stat = None\r
        self.commit_subject = None\r
        self.commit_prefix = None\r
        self.diff = None\r
\r
        if patch.startswith('diff --git'):\r
            self.diff = patch\r
            return\r
\r
        pmail = email.message_from_string(patch)\r
        parts = list(pmail.walk())\r
        assert(len(parts) == 1)\r
        assert(parts[0].get_content_type() == 'text/plain')\r
        content = parts[0].get_payload(decode=True).decode('utf-8', 'ignore')\r
\r
        mo = self.git_diff_re.search(content)\r
        if mo is not None:\r
            self.diff = content[mo.start():]\r
            content = content[:mo.start()]\r
\r
        mo = self.stat_re.search(content)\r
        if mo is None:\r
            self.commit_msg = content\r
        else:\r
            self.stat = mo.group('stat')\r
            self.commit_msg = mo.group('commit_message')\r
\r
        self.commit_subject = pmail['subject'].replace('\r\n', '')\r
        self.commit_subject = self.commit_subject.replace('\n', '')\r
        self.commit_subject = self.subject_prefix_re.sub('', self.commit_subject, 1)\r
\r
        self.author_email = pmail['from']\r
\r
class CheckGitCommits:\r
    """Reads patches from git based on the specified git revision range.\r
\r
    The patches are read from git, and then checked.\r
    """\r
\r
    def __init__(self, rev_spec, max_count):\r
        commits = self.read_commit_list_from_git(rev_spec, max_count)\r
        if len(commits) == 1 and Verbose.level > Verbose.ONELINE:\r
            commits = [ rev_spec ]\r
        self.ok = True\r
        blank_line = False\r
        for commit in commits:\r
            if Verbose.level > Verbose.ONELINE:\r
                if blank_line:\r
                    print()\r
                else:\r
                    blank_line = True\r
                print('Checking git commit:', commit)\r
            email = self.read_committer_email_address_from_git(commit)\r
            self.ok &= EmailAddressCheck(email, 'Committer').ok\r
            patch = self.read_patch_from_git(commit)\r
            self.ok &= CheckOnePatch(commit, patch).ok\r
        if not commits:\r
            print("Couldn't find commit matching: '{}'".format(rev_spec))\r
\r
    def read_commit_list_from_git(self, rev_spec, max_count):\r
        # Run git to get the commit patch\r
        cmd = [ 'rev-list', '--abbrev-commit', '--no-walk' ]\r
        if max_count is not None:\r
            cmd.append('--max-count=' + str(max_count))\r
        cmd.append(rev_spec)\r
        out = self.run_git(*cmd)\r
        return out.split() if out else []\r
\r
    def read_patch_from_git(self, commit):\r
        # Run git to get the commit patch\r
        return self.run_git('show', '--pretty=email', '--no-textconv', commit)\r
\r
    def read_committer_email_address_from_git(self, commit):\r
        # Run git to get the committer email\r
        return self.run_git('show', '--pretty=%cn <%ce>', '--no-patch', commit)\r
\r
    def run_git(self, *args):\r
        cmd = [ 'git' ]\r
        cmd += args\r
        p = subprocess.Popen(cmd,\r
                     stdout=subprocess.PIPE,\r
                     stderr=subprocess.STDOUT)\r
        Result = p.communicate()\r
        return Result[0].decode('utf-8', 'ignore') if Result[0] and Result[0].find(b"fatal")!=0 else None\r
\r
class CheckOnePatchFile:\r
    """Performs a patch check for a single file.\r
\r
    stdin is used when the filename is '-'.\r
    """\r
\r
    def __init__(self, patch_filename):\r
        if patch_filename == '-':\r
            patch = sys.stdin.read()\r
            patch_filename = 'stdin'\r
        else:\r
            f = open(patch_filename, 'rb')\r
            patch = f.read().decode('utf-8', 'ignore')\r
            f.close()\r
        if Verbose.level > Verbose.ONELINE:\r
            print('Checking patch file:', patch_filename)\r
        self.ok = CheckOnePatch(patch_filename, patch).ok\r
\r
class CheckOneArg:\r
    """Performs a patch check for a single command line argument.\r
\r
    The argument will be handed off to a file or git-commit based\r
    checker.\r
    """\r
\r
    def __init__(self, param, max_count=None):\r
        self.ok = True\r
        if param == '-' or os.path.exists(param):\r
            checker = CheckOnePatchFile(param)\r
        else:\r
            checker = CheckGitCommits(param, max_count)\r
        self.ok = checker.ok\r
\r
class PatchCheckApp:\r
    """Checks patches based on the command line arguments."""\r
\r
    def __init__(self):\r
        self.parse_options()\r
        patches = self.args.patches\r
\r
        if len(patches) == 0:\r
            patches = [ 'HEAD' ]\r
\r
        self.ok = True\r
        self.count = None\r
        for patch in patches:\r
            self.process_one_arg(patch)\r
\r
        if self.count is not None:\r
            self.process_one_arg('HEAD')\r
\r
        if self.ok:\r
            self.retval = 0\r
        else:\r
            self.retval = -1\r
\r
    def process_one_arg(self, arg):\r
        if len(arg) >= 2 and arg[0] == '-':\r
            try:\r
                self.count = int(arg[1:])\r
                return\r
            except ValueError:\r
                pass\r
        self.ok &= CheckOneArg(arg, self.count).ok\r
        self.count = None\r
\r
    def parse_options(self):\r
        parser = argparse.ArgumentParser(description=__copyright__)\r
        parser.add_argument('--version', action='version',\r
                            version='%(prog)s ' + VersionNumber)\r
        parser.add_argument('patches', nargs='*',\r
                            help='[patch file | git rev list]')\r
        group = parser.add_mutually_exclusive_group()\r
        group.add_argument("--oneline",\r
                           action="store_true",\r
                           help="Print one result per line")\r
        group.add_argument("--silent",\r
                           action="store_true",\r
                           help="Print nothing")\r
        self.args = parser.parse_args()\r
        if self.args.oneline:\r
            Verbose.level = Verbose.ONELINE\r
        if self.args.silent:\r
            Verbose.level = Verbose.SILENT\r
\r
if __name__ == "__main__":\r
    sys.exit(PatchCheckApp().retval)\r