.pytool/Plugin/EccCheck/EccCheck.py

   1 # @file EccCheck.py
   2 #
   3 # Copyright (c) 2021, Arm Limited. All rights reserved.<BR>
   4 # Copyright (c) 2020, Intel Corporation. All rights reserved.<BR>
   5 # SPDX-License-Identifier: BSD-2-Clause-Patent
   6 ##
   7
   8 import os
   9 import shutil
  10 import re
  11 import csv
  12 import xml.dom.minidom
  13 from typing import List, Dict, Tuple
  14 import logging
  15 from io import StringIO
  16 from edk2toolext.environment import shell_environment
  17 from edk2toolext.environment.plugintypes.ci_build_plugin import ICiBuildPlugin
  18 from edk2toolext.environment.var_dict import VarDict
  19 from edk2toollib.utility_functions import RunCmd
  20
  21
  22 class EccCheck(ICiBuildPlugin):
  23     """
  24     A CiBuildPlugin that finds the Ecc issues of newly added code in pull request.
  25
  26     Configuration options:
  27     "EccCheck": {
  28         "ExceptionList": [],
  29         "IgnoreFiles": []
  30     },
  31     """
  32
  33     ReModifyFile = re.compile(r'[B-Q,S-Z]+[\d]*\t(.*)')
  34     FindModifyFile = re.compile(r'\+\+\+ b\/(.*)')
  35     LineScopePattern = (r'@@ -\d*\,*\d* \+\d*\,*\d* @@.*')
  36     LineNumRange = re.compile(r'@@ -\d*\,*\d* \+(\d*)\,*(\d*) @@.*')
  37
  38     def GetTestName(self, packagename: str, environment: VarDict) -> tuple:
  39         """ Provide the testcase name and classname for use in reporting
  40             testclassname: a descriptive string for the testcase can include whitespace
  41             classname: should be patterned <packagename>.<plugin>.<optionally any unique condition>
  42
  43             Args:
  44               packagename: string containing name of package to build
  45               environment: The VarDict for the test to run in
  46             Returns:
  47                 a tuple containing the testcase name and the classname
  48                 (testcasename, classname)
  49         """
  50         return ("Check for efi coding style for " + packagename, packagename + ".EccCheck")
  51
  52     ##
  53     # External function of plugin.  This function is used to perform the task of the ci_build_plugin Plugin
  54     #
  55     #   - package is the edk2 path to package.  This means workspace/packagepath relative.
  56     #   - edk2path object configured with workspace and packages path
  57     #   - PkgConfig Object (dict) for the pkg
  58     #   - EnvConfig Object
  59     #   - Plugin Manager Instance
  60     #   - Plugin Helper Obj Instance
  61     #   - Junit Logger
  62     #   - output_stream the StringIO output stream from this plugin via logging
  63     def RunBuildPlugin(self, packagename, Edk2pathObj, pkgconfig, environment, PLM, PLMHelper, tc, output_stream=None):
  64         workspace_path = Edk2pathObj.WorkspacePath
  65         basetools_path = environment.GetValue("EDK_TOOLS_PATH")
  66         python_path = os.path.join(basetools_path, "Source", "Python")
  67         env = shell_environment.GetEnvironment()
  68         env.set_shell_var('PYTHONPATH', python_path)
  69         env.set_shell_var('WORKSPACE', workspace_path)
  70         self.ECC_PASS = True
  71         self.ApplyConfig(pkgconfig, workspace_path, basetools_path, packagename)
  72         modify_dir_list = self.GetModifyDir(packagename)
  73         patch = self.GetDiff(packagename)
  74         ecc_diff_range = self.GetDiffRange(patch, packagename, workspace_path)
  75         self.GenerateEccReport(modify_dir_list, ecc_diff_range, workspace_path, basetools_path)
  76         ecc_log = os.path.join(workspace_path, "Ecc.log")
  77         self.RevertCode()
  78         if self.ECC_PASS:
  79             tc.SetSuccess()
  80             self.RemoveFile(ecc_log)
  81             return 0
  82         else:
  83             with open(ecc_log, encoding='utf8') as output:
  84                 ecc_output = output.readlines()
  85                 for line in ecc_output:
  86                     logging.error(line.strip())
  87             self.RemoveFile(ecc_log)
  88             tc.SetFailed("EccCheck failed for {0}".format(packagename), "Ecc detected issues")
  89             return 1
  90
  91     def RevertCode(self) -> None:
  92         submoudle_params = "submodule update --init"
  93         RunCmd("git", submoudle_params)
  94         reset_params = "reset HEAD --hard"
  95         RunCmd("git", reset_params)
  96
  97     def GetDiff(self, pkg: str) -> List[str]:
  98         return_buffer = StringIO()
  99         params = "diff --unified=0 origin/master HEAD"
 100         RunCmd("git", params, outstream=return_buffer)
 101         p = return_buffer.getvalue().strip()
 102         patch = p.split("\n")
 103         return_buffer.close()
 104
 105         return patch
 106
 107     def RemoveFile(self, file: str) -> None:
 108         if os.path.exists(file):
 109             os.remove(file)
 110         return
 111
 112     def GetModifyDir(self, pkg: str) -> List[str]:
 113         return_buffer = StringIO()
 114         params = "diff --name-status" + ' HEAD' + ' origin/master'
 115         RunCmd("git", params, outstream=return_buffer)
 116         p1 = return_buffer.getvalue().strip()
 117         dir_list = p1.split("\n")
 118         return_buffer.close()
 119         modify_dir_list = []
 120         for modify_dir in dir_list:
 121             file_path = self.ReModifyFile.findall(modify_dir)
 122             if file_path:
 123                 file_dir = os.path.dirname(file_path[0])
 124             else:
 125                 continue
 126             if pkg in file_dir and file_dir != pkg:
 127                 modify_dir_list.append('%s' % file_dir)
 128             else:
 129                 continue
 130
 131         modify_dir_list = list(set(modify_dir_list))
 132         return modify_dir_list
 133
 134     def GetDiffRange(self, patch_diff: List[str], pkg: str, workingdir: str) -> Dict[str, List[Tuple[int, int]]]:
 135         IsDelete = True
 136         StartCheck = False
 137         range_directory: Dict[str, List[Tuple[int, int]]] = {}
 138         for line in patch_diff:
 139             modify_file = self.FindModifyFile.findall(line)
 140             if modify_file and pkg in modify_file[0] and not StartCheck and os.path.isfile(modify_file[0]):
 141                 modify_file_comment_dic = self.GetCommentRange(modify_file[0], workingdir)
 142                 IsDelete = False
 143                 StartCheck = True
 144                 modify_file_dic = modify_file[0]
 145                 modify_file_dic = modify_file_dic.replace("/", os.sep)
 146                 range_directory[modify_file_dic] = []
 147             elif line.startswith('--- '):
 148                 StartCheck = False
 149             elif re.match(self.LineScopePattern, line, re.I) and not IsDelete and StartCheck:
 150                 start_line = self.LineNumRange.search(line).group(1)
 151                 line_range = self.LineNumRange.search(line).group(2)
 152                 if not line_range:
 153                     line_range = '1'
 154                 range_directory[modify_file_dic].append((int(start_line), int(start_line) + int(line_range) - 1))
 155                 for i in modify_file_comment_dic:
 156                     if int(i[0]) <= int(start_line) <= int(i[1]):
 157                         range_directory[modify_file_dic].append(i)
 158         return range_directory
 159
 160     def GetCommentRange(self, modify_file: str, workingdir: str) -> List[Tuple[int, int]]:
 161         modify_file_path = os.path.join(workingdir, modify_file)
 162         with open(modify_file_path) as f:
 163             line_no = 1
 164             comment_range: List[Tuple[int, int]] = []
 165             Start = False
 166             for line in f:
 167                 if line.startswith('/**'):
 168                     start_no = line_no
 169                     Start = True
 170                 if line.startswith('**/') and Start:
 171                     end_no = line_no
 172                     Start = False
 173                     comment_range.append((int(start_no), int(end_no)))
 174                 line_no += 1
 175
 176         if comment_range and comment_range[0][0] == 1:
 177             del comment_range[0]
 178         return comment_range
 179
 180     def GenerateEccReport(self, modify_dir_list: List[str], ecc_diff_range: Dict[str, List[Tuple[int, int]]],
 181                            workspace_path: str, basetools_path: str) -> None:
 182         ecc_need = False
 183         ecc_run = True
 184         config = os.path.join(basetools_path, "Source", "Python", "Ecc", "config.ini")
 185         exception = os.path.join(basetools_path, "Source", "Python", "Ecc", "exception.xml")
 186         report = os.path.join(workspace_path, "Ecc.csv")
 187         for modify_dir in modify_dir_list:
 188             target = os.path.join(workspace_path, modify_dir)
 189             logging.info('Run ECC tool for the commit in %s' % modify_dir)
 190             ecc_need = True
 191             ecc_params = "-c {0} -e {1} -t {2} -r {3}".format(config, exception, target, report)
 192             return_code = RunCmd("Ecc", ecc_params, workingdir=workspace_path)
 193             if return_code != 0:
 194                 ecc_run = False
 195                 break
 196             if not ecc_run:
 197                 logging.error('Fail to run ECC tool')
 198             self.ParseEccReport(ecc_diff_range, workspace_path)
 199
 200         if not ecc_need:
 201             logging.info("Doesn't need run ECC check")
 202
 203         revert_params = "checkout -- {}".format(exception)
 204         RunCmd("git", revert_params)
 205         return
 206
 207     def ParseEccReport(self, ecc_diff_range: Dict[str, List[Tuple[int, int]]], workspace_path: str) -> None:
 208         ecc_log = os.path.join(workspace_path, "Ecc.log")
 209         ecc_csv = os.path.join(workspace_path, "Ecc.csv")
 210         row_lines = []
 211         ignore_error_code = self.GetIgnoreErrorCode()
 212         if os.path.exists(ecc_csv):
 213             with open(ecc_csv) as csv_file:
 214                 reader = csv.reader(csv_file)
 215                 for row in reader:
 216                     for modify_file in ecc_diff_range:
 217                         if modify_file in row[3]:
 218                             for i in ecc_diff_range[modify_file]:
 219                                 line_no = int(row[4])
 220                                 if i[0] <= line_no <= i[1] and row[1] not in ignore_error_code:
 221                                     row[0] = '\nEFI coding style error'
 222                                     row[1] = 'Error code: ' + row[1]
 223                                     row[3] = 'file: ' + row[3]
 224                                     row[4] = 'Line number: ' + row[4]
 225                                     row_line = '\n  *'.join(row)
 226                                     row_lines.append(row_line)
 227                                     break
 228                             break
 229         if row_lines:
 230             self.ECC_PASS = False
 231
 232         with open(ecc_log, 'a') as log:
 233             all_line = '\n'.join(row_lines)
 234             all_line = all_line + '\n'
 235             log.writelines(all_line)
 236         return
 237
 238     def ApplyConfig(self, pkgconfig: Dict[str, List[str]], workspace_path: str, basetools_path: str, pkg: str) -> None:
 239         if "IgnoreFiles" in pkgconfig:
 240             for a in pkgconfig["IgnoreFiles"]:
 241                 a = os.path.join(workspace_path, pkg, a)
 242                 a = a.replace(os.sep, "/")
 243
 244                 logging.info("Ignoring Files {0}".format(a))
 245                 if os.path.exists(a):
 246                     if os.path.isfile(a):
 247                         self.RemoveFile(a)
 248                     elif os.path.isdir(a):
 249                         shutil.rmtree(a)
 250                 else:
 251                     logging.error("EccCheck.IgnoreInf -> {0} not found in filesystem.  Invalid ignore files".format(a))
 252
 253         if "ExceptionList" in pkgconfig:
 254             exception_list = pkgconfig["ExceptionList"]
 255             exception_xml = os.path.join(basetools_path, "Source", "Python", "Ecc", "exception.xml")
 256             try:
 257                 logging.info("Appending exceptions")
 258                 self.AppendException(exception_list, exception_xml)
 259             except Exception as e:
 260                 logging.error("Fail to apply exceptions")
 261                 raise e
 262         return
 263
 264     def AppendException(self, exception_list: List[str], exception_xml: str) -> None:
 265         error_code_list = exception_list[::2]
 266         keyword_list = exception_list[1::2]
 267         dom_tree = xml.dom.minidom.parse(exception_xml)
 268         root_node = dom_tree.documentElement
 269         for error_code, keyword in zip(error_code_list, keyword_list):
 270             customer_node = dom_tree.createElement("Exception")
 271             keyword_node = dom_tree.createElement("KeyWord")
 272             keyword_node_text_value = dom_tree.createTextNode(keyword)
 273             keyword_node.appendChild(keyword_node_text_value)
 274             customer_node.appendChild(keyword_node)
 275             error_code_node = dom_tree.createElement("ErrorID")
 276             error_code_text_value = dom_tree.createTextNode(error_code)
 277             error_code_node.appendChild(error_code_text_value)
 278             customer_node.appendChild(error_code_node)
 279             root_node.appendChild(customer_node)
 280         with open(exception_xml, 'w') as f:
 281             dom_tree.writexml(f, indent='', addindent='', newl='\n', encoding='UTF-8')
 282         return
 283
 284     def GetIgnoreErrorCode(self) -> set:
 285         """
 286         Below are kinds of error code that are accurate in ecc scanning of edk2 level.
 287         But EccCheck plugin is partial scanning so they are always false positive issues.
 288         The mapping relationship of error code and error message is listed BaseTools/Sourc/Python/Ecc/EccToolError.py
 289         """
 290         ignore_error_code = {
 291                              "10000",
 292                              "10001",
 293                              "10002",
 294                              "10003",
 295                              "10004",
 296                              "10005",
 297                              "10006",
 298                              "10007",
 299                              "10008",
 300                              "10009",
 301                              "10010",
 302                              "10011",
 303                              "10012",
 304                              "10013",
 305                              "10015",
 306                              "10016",
 307                              "10017",
 308                              "10022",
 309                             }
 310         return ignore_error_code