.pytool/EccCheck: Check ecc_csv exists

[mirror_edk2.git] / .pytool / Plugin / EccCheck / EccCheck.py

# @file EccCheck.py\r
#\r
# Copyright (c) 2021, Arm Limited. All rights reserved.<BR>\r
# Copyright (c) 2020, Intel Corporation. All rights reserved.<BR>\r
# SPDX-License-Identifier: BSD-2-Clause-Patent\r
##\r
\r
import os\r
import shutil\r
import re\r
import csv\r
import xml.dom.minidom\r
from typing import List, Dict, Tuple\r
import logging\r
from io import StringIO\r
from edk2toolext.environment import shell_environment\r
from edk2toolext.environment.plugintypes.ci_build_plugin import ICiBuildPlugin\r
from edk2toolext.environment.var_dict import VarDict\r
from edk2toollib.utility_functions import RunCmd\r
\r
\r
class EccCheck(ICiBuildPlugin):\r
    """\r
    A CiBuildPlugin that finds the Ecc issues of newly added code in pull request.\r
\r
    Configuration options:\r
    "EccCheck": {\r
        "ExceptionList": [],\r
        "IgnoreFiles": []\r
    },\r
    """\r
\r
    ReModifyFile = re.compile(r'[B-Q,S-Z]+[\d]*\t(.*)')\r
    FindModifyFile = re.compile(r'\+\+\+ b\/(.*)')\r
    LineScopePattern = (r'@@ -\d*\,*\d* \+\d*\,*\d* @@.*')\r
    LineNumRange = re.compile(r'@@ -\d*\,*\d* \+(\d*)\,*(\d*) @@.*')\r
\r
    def GetTestName(self, packagename: str, environment: VarDict) -> tuple:\r
        """ Provide the testcase name and classname for use in reporting\r
            testclassname: a descriptive string for the testcase can include whitespace\r
            classname: should be patterned <packagename>.<plugin>.<optionally any unique condition>\r
\r
            Args:\r
              packagename: string containing name of package to build\r
              environment: The VarDict for the test to run in\r
            Returns:\r
                a tuple containing the testcase name and the classname\r
                (testcasename, classname)\r
        """\r
        return ("Check for efi coding style for " + packagename, packagename + ".EccCheck")\r
\r
    ##\r
    # External function of plugin.  This function is used to perform the task of the ci_build_plugin Plugin\r
    #\r
    #   - package is the edk2 path to package.  This means workspace/packagepath relative.\r
    #   - edk2path object configured with workspace and packages path\r
    #   - PkgConfig Object (dict) for the pkg\r
    #   - EnvConfig Object\r
    #   - Plugin Manager Instance\r
    #   - Plugin Helper Obj Instance\r
    #   - Junit Logger\r
    #   - output_stream the StringIO output stream from this plugin via logging\r
    def RunBuildPlugin(self, packagename, Edk2pathObj, pkgconfig, environment, PLM, PLMHelper, tc, output_stream=None):\r
        workspace_path = Edk2pathObj.WorkspacePath\r
        basetools_path = environment.GetValue("EDK_TOOLS_PATH")\r
        python_path = os.path.join(basetools_path, "Source", "Python")\r
        env = shell_environment.GetEnvironment()\r
        env.set_shell_var('PYTHONPATH', python_path)\r
        env.set_shell_var('WORKSPACE', workspace_path)\r
        self.ECC_PASS = True\r
        self.ApplyConfig(pkgconfig, workspace_path, basetools_path, packagename)\r
        modify_dir_list = self.GetModifyDir(packagename)\r
        patch = self.GetDiff(packagename)\r
        ecc_diff_range = self.GetDiffRange(patch, packagename, workspace_path)\r
        self.GenerateEccReport(modify_dir_list, ecc_diff_range, workspace_path, basetools_path)\r
        ecc_log = os.path.join(workspace_path, "Ecc.log")\r
        self.RevertCode()\r
        if self.ECC_PASS:\r
            tc.SetSuccess()\r
            self.RemoveFile(ecc_log)\r
            return 0\r
        else:\r
            with open(ecc_log, encoding='utf8') as output:\r
                ecc_output = output.readlines()\r
                for line in ecc_output:\r
                    logging.error(line.strip())\r
            self.RemoveFile(ecc_log)\r
            tc.SetFailed("EccCheck failed for {0}".format(packagename), "Ecc detected issues")\r
            return 1\r
\r
    def RevertCode(self) -> None:\r
        submoudle_params = "submodule update --init"\r
        RunCmd("git", submoudle_params)\r
        reset_params = "reset HEAD --hard"\r
        RunCmd("git", reset_params)\r
\r
    def GetDiff(self, pkg: str) -> List[str]:\r
        return_buffer = StringIO()\r
        params = "diff --unified=0 origin/master HEAD"\r
        RunCmd("git", params, outstream=return_buffer)\r
        p = return_buffer.getvalue().strip()\r
        patch = p.split("\n")\r
        return_buffer.close()\r
\r
        return patch\r
\r
    def RemoveFile(self, file: str) -> None:\r
        if os.path.exists(file):\r
            os.remove(file)\r
        return\r
\r
    def GetModifyDir(self, pkg: str) -> List[str]:\r
        return_buffer = StringIO()\r
        params = "diff --name-status" + ' HEAD' + ' origin/master'\r
        RunCmd("git", params, outstream=return_buffer)\r
        p1 = return_buffer.getvalue().strip()\r
        dir_list = p1.split("\n")\r
        return_buffer.close()\r
        modify_dir_list = []\r
        for modify_dir in dir_list:\r
            file_path = self.ReModifyFile.findall(modify_dir)\r
            if file_path:\r
                file_dir = os.path.dirname(file_path[0])\r
            else:\r
                continue\r
            if pkg in file_dir and file_dir != pkg:\r
                modify_dir_list.append('%s' % file_dir)\r
            else:\r
                continue\r
\r
        modify_dir_list = list(set(modify_dir_list))\r
        return modify_dir_list\r
\r
    def GetDiffRange(self, patch_diff: List[str], pkg: str, workingdir: str) -> Dict[str, List[Tuple[int, int]]]:\r
        IsDelete = True\r
        StartCheck = False\r
        range_directory: Dict[str, List[Tuple[int, int]]] = {}\r
        for line in patch_diff:\r
            modify_file = self.FindModifyFile.findall(line)\r
            if modify_file and pkg in modify_file[0] and not StartCheck and os.path.isfile(modify_file[0]):\r
                modify_file_comment_dic = self.GetCommentRange(modify_file[0], workingdir)\r
                IsDelete = False\r
                StartCheck = True\r
                modify_file_dic = modify_file[0]\r
                modify_file_dic = modify_file_dic.replace("/", os.sep)\r
                range_directory[modify_file_dic] = []\r
            elif line.startswith('--- '):\r
                StartCheck = False\r
            elif re.match(self.LineScopePattern, line, re.I) and not IsDelete and StartCheck:\r
                start_line = self.LineNumRange.search(line).group(1)\r
                line_range = self.LineNumRange.search(line).group(2)\r
                if not line_range:\r
                    line_range = '1'\r
                range_directory[modify_file_dic].append((int(start_line), int(start_line) + int(line_range) - 1))\r
                for i in modify_file_comment_dic:\r
                    if int(i[0]) <= int(start_line) <= int(i[1]):\r
                        range_directory[modify_file_dic].append(i)\r
        return range_directory\r
\r
    def GetCommentRange(self, modify_file: str, workingdir: str) -> List[Tuple[int, int]]:\r
        modify_file_path = os.path.join(workingdir, modify_file)\r
        with open(modify_file_path) as f:\r
            line_no = 1\r
            comment_range: List[Tuple[int, int]] = []\r
            Start = False\r
            for line in f:\r
                if line.startswith('/**'):\r
                    start_no = line_no\r
                    Start = True\r
                if line.startswith('**/') and Start:\r
                    end_no = line_no\r
                    Start = False\r
                    comment_range.append((int(start_no), int(end_no)))\r
                line_no += 1\r
\r
        if comment_range and comment_range[0][0] == 1:\r
            del comment_range[0]\r
        return comment_range\r
\r
    def GenerateEccReport(self, modify_dir_list: List[str], ecc_diff_range: Dict[str, List[Tuple[int, int]]],\r
                           workspace_path: str, basetools_path: str) -> None:\r
        ecc_need = False\r
        ecc_run = True\r
        config = os.path.join(basetools_path, "Source", "Python", "Ecc", "config.ini")\r
        exception = os.path.join(basetools_path, "Source", "Python", "Ecc", "exception.xml")\r
        report = os.path.join(workspace_path, "Ecc.csv")\r
        for modify_dir in modify_dir_list:\r
            target = os.path.join(workspace_path, modify_dir)\r
            logging.info('Run ECC tool for the commit in %s' % modify_dir)\r
            ecc_need = True\r
            ecc_params = "-c {0} -e {1} -t {2} -r {3}".format(config, exception, target, report)\r
            return_code = RunCmd("Ecc", ecc_params, workingdir=workspace_path)\r
            if return_code != 0:\r
                ecc_run = False\r
                break\r
            if not ecc_run:\r
                logging.error('Fail to run ECC tool')\r
            self.ParseEccReport(ecc_diff_range, workspace_path)\r
\r
        if not ecc_need:\r
            logging.info("Doesn't need run ECC check")\r
\r
        revert_params = "checkout -- {}".format(exception)\r
        RunCmd("git", revert_params)\r
        return\r
\r
    def ParseEccReport(self, ecc_diff_range: Dict[str, List[Tuple[int, int]]], workspace_path: str) -> None:\r
        ecc_log = os.path.join(workspace_path, "Ecc.log")\r
        ecc_csv = os.path.join(workspace_path, "Ecc.csv")\r
        row_lines = []\r
        ignore_error_code = self.GetIgnoreErrorCode()\r
        if os.path.exists(ecc_csv):\r
            with open(ecc_csv) as csv_file:\r
                reader = csv.reader(csv_file)\r
                for row in reader:\r
                    for modify_file in ecc_diff_range:\r
                        if modify_file in row[3]:\r
                            for i in ecc_diff_range[modify_file]:\r
                                line_no = int(row[4])\r
                                if i[0] <= line_no <= i[1] and row[1] not in ignore_error_code:\r
                                    row[0] = '\nEFI coding style error'\r
                                    row[1] = 'Error code: ' + row[1]\r
                                    row[3] = 'file: ' + row[3]\r
                                    row[4] = 'Line number: ' + row[4]\r
                                    row_line = '\n  *'.join(row)\r
                                    row_lines.append(row_line)\r
                                    break\r
                            break\r
        if row_lines:\r
            self.ECC_PASS = False\r
\r
        with open(ecc_log, 'a') as log:\r
            all_line = '\n'.join(row_lines)\r
            all_line = all_line + '\n'\r
            log.writelines(all_line)\r
        return\r
\r
    def ApplyConfig(self, pkgconfig: Dict[str, List[str]], workspace_path: str, basetools_path: str, pkg: str) -> None:\r
        if "IgnoreFiles" in pkgconfig:\r
            for a in pkgconfig["IgnoreFiles"]:\r
                a = os.path.join(workspace_path, pkg, a)\r
                a = a.replace(os.sep, "/")\r
\r
                logging.info("Ignoring Files {0}".format(a))\r
                if os.path.exists(a):\r
                    if os.path.isfile(a):\r
                        self.RemoveFile(a)\r
                    elif os.path.isdir(a):\r
                        shutil.rmtree(a)\r
                else:\r
                    logging.error("EccCheck.IgnoreInf -> {0} not found in filesystem.  Invalid ignore files".format(a))\r
\r
        if "ExceptionList" in pkgconfig:\r
            exception_list = pkgconfig["ExceptionList"]\r
            exception_xml = os.path.join(basetools_path, "Source", "Python", "Ecc", "exception.xml")\r
            try:\r
                logging.info("Appending exceptions")\r
                self.AppendException(exception_list, exception_xml)\r
            except Exception as e:\r
                logging.error("Fail to apply exceptions")\r
                raise e\r
        return\r
\r
    def AppendException(self, exception_list: List[str], exception_xml: str) -> None:\r
        error_code_list = exception_list[::2]\r
        keyword_list = exception_list[1::2]\r
        dom_tree = xml.dom.minidom.parse(exception_xml)\r
        root_node = dom_tree.documentElement\r
        for error_code, keyword in zip(error_code_list, keyword_list):\r
            customer_node = dom_tree.createElement("Exception")\r
            keyword_node = dom_tree.createElement("KeyWord")\r
            keyword_node_text_value = dom_tree.createTextNode(keyword)\r
            keyword_node.appendChild(keyword_node_text_value)\r
            customer_node.appendChild(keyword_node)\r
            error_code_node = dom_tree.createElement("ErrorID")\r
            error_code_text_value = dom_tree.createTextNode(error_code)\r
            error_code_node.appendChild(error_code_text_value)\r
            customer_node.appendChild(error_code_node)\r
            root_node.appendChild(customer_node)\r
        with open(exception_xml, 'w') as f:\r
            dom_tree.writexml(f, indent='', addindent='', newl='\n', encoding='UTF-8')\r
        return\r
\r
    def GetIgnoreErrorCode(self) -> set:\r
        """\r
        Below are kinds of error code that are accurate in ecc scanning of edk2 level.\r
        But EccCheck plugin is partial scanning so they are always false positive issues.\r
        The mapping relationship of error code and error message is listed BaseTools/Sourc/Python/Ecc/EccToolError.py\r
        """\r
        ignore_error_code = {\r
                             "10000",\r
                             "10001",\r
                             "10002",\r
                             "10003",\r
                             "10004",\r
                             "10005",\r
                             "10006",\r
                             "10007",\r
                             "10008",\r
                             "10009",\r
                             "10010",\r
                             "10011",\r
                             "10012",\r
                             "10013",\r
                             "10015",\r
                             "10016",\r
                             "10017",\r
                             "10022",\r
                            }\r
        return ignore_error_code\r