BaseTools/Source/Python/Ecc/EccMain.py

   1 ## @file
   2 # This file is used to be the main entrance of ECC tool
   3 #
   4 # Copyright (c) 2009 - 2018, Intel Corporation. All rights reserved.<BR>
   5 # Copyright (c) Microsoft Corporation.<BR>
   6 # SPDX-License-Identifier: BSD-2-Clause-Patent
   7 #
   8
   9 ##
  10 # Import Modules
  11 #
  12 from __future__ import absolute_import
  13 import Common.LongFilePathOs as os, time, glob, sys
  14 import Common.EdkLogger as EdkLogger
  15 from Ecc import Database
  16 from Ecc import EccGlobalData
  17 from Ecc.MetaDataParser import *
  18 from optparse import OptionParser
  19 from Ecc.Configuration import Configuration
  20 from Ecc.Check import Check
  21 import Common.GlobalData as GlobalData
  22
  23 from Common.StringUtils import NormPath
  24 from Common.BuildVersion import gBUILD_VERSION
  25 from Common import BuildToolError
  26 from Common.Misc import PathClass
  27 from Common.Misc import DirCache
  28 from Ecc.MetaFileWorkspace.MetaFileParser import DscParser
  29 from Ecc.MetaFileWorkspace.MetaFileParser import DecParser
  30 from Ecc.MetaFileWorkspace.MetaFileParser import InfParser
  31 from Ecc.MetaFileWorkspace.MetaFileParser import Fdf
  32 from Ecc.MetaFileWorkspace.MetaFileTable import MetaFileStorage
  33 from Ecc import c
  34 import re, string
  35 from Ecc.Exception import *
  36 from Common.LongFilePathSupport import OpenLongFilePath as open
  37 from Common.MultipleWorkspace import MultipleWorkspace as mws
  38
  39 ## Ecc
  40 #
  41 # This class is used to define Ecc main entrance
  42 #
  43 # @param object:          Inherited from object class
  44 #
  45 class Ecc(object):
  46     def __init__(self):
  47         # Version and Copyright
  48         self.VersionNumber = ("1.0" + " Build " + gBUILD_VERSION)
  49         self.Version = "%prog Version " + self.VersionNumber
  50         self.Copyright = "Copyright (c) 2009 - 2018, Intel Corporation  All rights reserved."
  51
  52         self.InitDefaultConfigIni()
  53         self.OutputFile = 'output.txt'
  54         self.ReportFile = 'Report.csv'
  55         self.ExceptionFile = 'exception.xml'
  56         self.IsInit = True
  57         self.ScanSourceCode = True
  58         self.ScanMetaData = True
  59         self.MetaFile = ''
  60         self.OnlyScan = None
  61
  62         # Parse the options and args
  63         self.ParseOption()
  64         EdkLogger.info(time.strftime("%H:%M:%S, %b.%d %Y ", time.localtime()) + "[00:00]" + "\n")
  65
  66         WorkspaceDir = os.path.normcase(os.path.normpath(os.environ["WORKSPACE"]))
  67         os.environ["WORKSPACE"] = WorkspaceDir
  68
  69         # set multiple workspace
  70         PackagesPath = os.getenv("PACKAGES_PATH")
  71         mws.setWs(WorkspaceDir, PackagesPath)
  72
  73         GlobalData.gWorkspace = WorkspaceDir
  74
  75         GlobalData.gGlobalDefines["WORKSPACE"]  = WorkspaceDir
  76
  77         EdkLogger.info("Loading ECC configuration ... done")
  78         # Generate checkpoints list
  79         EccGlobalData.gConfig = Configuration(self.ConfigFile)
  80
  81         # Generate exception list
  82         EccGlobalData.gException = ExceptionCheck(self.ExceptionFile)
  83
  84         # Init Ecc database
  85         EccGlobalData.gDb = Database.Database(Database.DATABASE_PATH)
  86         EccGlobalData.gDb.InitDatabase(self.IsInit)
  87
  88         #
  89         # Get files real name in workspace dir
  90         #
  91         GlobalData.gAllFiles = DirCache(GlobalData.gWorkspace)
  92
  93         # Build ECC database
  94 #         self.BuildDatabase()
  95         self.DetectOnlyScanDirs()
  96
  97         # Start to check
  98         self.Check()
  99
 100         # Show report
 101         self.GenReport()
 102
 103         # Close Database
 104         EccGlobalData.gDb.Close()
 105
 106     def InitDefaultConfigIni(self):
 107         paths = map(lambda p: os.path.join(p, 'Ecc', 'config.ini'), sys.path)
 108         paths = (os.path.abspath('config.ini'),) + tuple(paths)
 109         for path in paths:
 110             if os.path.exists(path):
 111                 self.ConfigFile = path
 112                 return
 113         self.ConfigFile = 'config.ini'
 114
 115
 116     ## DetectOnlyScan
 117     #
 118     # Detect whether only scanned folders have been enabled
 119     #
 120     def DetectOnlyScanDirs(self):
 121         if self.OnlyScan == True:
 122             OnlyScanDirs = []
 123             # Use regex here if multiple spaces or TAB exists in ScanOnlyDirList in config.ini file
 124             for folder in re.finditer(r'\S+', EccGlobalData.gConfig.ScanOnlyDirList):
 125                 OnlyScanDirs.append(folder.group())
 126             if len(OnlyScanDirs) != 0:
 127                 self.BuildDatabase(OnlyScanDirs)
 128             else:
 129                 EdkLogger.error("ECC", BuildToolError.OPTION_VALUE_INVALID, ExtraData="Use -f option need to fill specific folders in config.ini file")
 130         else:
 131             self.BuildDatabase()
 132
 133
 134     ## BuildDatabase
 135     #
 136     # Build the database for target
 137     #
 138     def BuildDatabase(self, SpeciDirs = None):
 139         # Clean report table
 140         EccGlobalData.gDb.TblReport.Drop()
 141         EccGlobalData.gDb.TblReport.Create()
 142
 143         # Build database
 144         if self.IsInit:
 145             if self.ScanMetaData:
 146                 EdkLogger.quiet("Building database for Meta Data File ...")
 147                 self.BuildMetaDataFileDatabase(SpeciDirs)
 148             if self.ScanSourceCode:
 149                 EdkLogger.quiet("Building database for Meta Data File Done!")
 150                 if SpeciDirs is None:
 151                     c.CollectSourceCodeDataIntoDB(EccGlobalData.gTarget)
 152                 else:
 153                     for specificDir in SpeciDirs:
 154                         c.CollectSourceCodeDataIntoDB(os.path.join(EccGlobalData.gTarget, specificDir))
 155
 156         EccGlobalData.gIdentifierTableList = GetTableList((MODEL_FILE_C, MODEL_FILE_H), 'Identifier', EccGlobalData.gDb)
 157         EccGlobalData.gCFileList = GetFileList(MODEL_FILE_C, EccGlobalData.gDb)
 158         EccGlobalData.gHFileList = GetFileList(MODEL_FILE_H, EccGlobalData.gDb)
 159         EccGlobalData.gUFileList = GetFileList(MODEL_FILE_UNI, EccGlobalData.gDb)
 160
 161     ## BuildMetaDataFileDatabase
 162     #
 163     # Build the database for meta data files
 164     #
 165     def BuildMetaDataFileDatabase(self, SpecificDirs = None):
 166         ScanFolders = []
 167         if SpecificDirs is None:
 168             ScanFolders.append(EccGlobalData.gTarget)
 169         else:
 170             for specificDir in SpecificDirs:
 171                 ScanFolders.append(os.path.join(EccGlobalData.gTarget, specificDir))
 172         EdkLogger.quiet("Building database for meta data files ...")
 173         Op = open(EccGlobalData.gConfig.MetaDataFileCheckPathOfGenerateFileList, 'w+')
 174         #SkipDirs = Read from config file
 175         SkipDirs = EccGlobalData.gConfig.SkipDirList
 176         SkipDirString = '|'.join(SkipDirs)
 177 #         p = re.compile(r'.*[\\/](?:%s)[\\/]?.*' % SkipDirString)
 178         p = re.compile(r'.*[\\/](?:%s^\S)[\\/]?.*' % SkipDirString)
 179         for scanFolder in ScanFolders:
 180             for Root, Dirs, Files in os.walk(scanFolder):
 181                 if p.match(Root.upper()):
 182                     continue
 183                 for Dir in Dirs:
 184                     Dirname = os.path.join(Root, Dir)
 185                     if os.path.islink(Dirname):
 186                         Dirname = os.path.realpath(Dirname)
 187                         if os.path.isdir(Dirname):
 188                             # symlinks to directories are treated as directories
 189                             Dirs.remove(Dir)
 190                             Dirs.append(Dirname)
 191
 192                 for File in Files:
 193                     if len(File) > 4 and File[-4:].upper() == ".DEC":
 194                         Filename = os.path.normpath(os.path.join(Root, File))
 195                         EdkLogger.quiet("Parsing %s" % Filename)
 196                         Op.write("%s\r" % Filename)
 197                         #Dec(Filename, True, True, EccGlobalData.gWorkspace, EccGlobalData.gDb)
 198                         self.MetaFile = DecParser(Filename, MODEL_FILE_DEC, EccGlobalData.gDb.TblDec)
 199                         self.MetaFile.Start()
 200                         continue
 201                     if len(File) > 4 and File[-4:].upper() == ".DSC":
 202                         Filename = os.path.normpath(os.path.join(Root, File))
 203                         EdkLogger.quiet("Parsing %s" % Filename)
 204                         Op.write("%s\r" % Filename)
 205                         #Dsc(Filename, True, True, EccGlobalData.gWorkspace, EccGlobalData.gDb)
 206                         self.MetaFile = DscParser(PathClass(Filename, Root), MODEL_FILE_DSC, MetaFileStorage(EccGlobalData.gDb.TblDsc.Cur, Filename, MODEL_FILE_DSC, True))
 207                         # always do post-process, in case of macros change
 208                         self.MetaFile.DoPostProcess()
 209                         self.MetaFile.Start()
 210                         self.MetaFile._PostProcess()
 211                         continue
 212                     if len(File) > 4 and File[-4:].upper() == ".INF":
 213                         Filename = os.path.normpath(os.path.join(Root, File))
 214                         EdkLogger.quiet("Parsing %s" % Filename)
 215                         Op.write("%s\r" % Filename)
 216                         #Inf(Filename, True, True, EccGlobalData.gWorkspace, EccGlobalData.gDb)
 217                         self.MetaFile = InfParser(Filename, MODEL_FILE_INF, EccGlobalData.gDb.TblInf)
 218                         self.MetaFile.Start()
 219                         continue
 220                     if len(File) > 4 and File[-4:].upper() == ".FDF":
 221                         Filename = os.path.normpath(os.path.join(Root, File))
 222                         EdkLogger.quiet("Parsing %s" % Filename)
 223                         Op.write("%s\r" % Filename)
 224                         Fdf(Filename, True, EccGlobalData.gWorkspace, EccGlobalData.gDb)
 225                         continue
 226                     if len(File) > 4 and File[-4:].upper() == ".UNI":
 227                         Filename = os.path.normpath(os.path.join(Root, File))
 228                         EdkLogger.quiet("Parsing %s" % Filename)
 229                         Op.write("%s\r" % Filename)
 230                         FileID = EccGlobalData.gDb.TblFile.InsertFile(Filename, MODEL_FILE_UNI)
 231                         EccGlobalData.gDb.TblReport.UpdateBelongsToItemByFile(FileID, File)
 232                         continue
 233
 234         Op.close()
 235
 236         # Commit to database
 237         EccGlobalData.gDb.Conn.commit()
 238
 239         EdkLogger.quiet("Building database for meta data files done!")
 240
 241     ##
 242     #
 243     # Check each checkpoint
 244     #
 245     def Check(self):
 246         EdkLogger.quiet("Checking ...")
 247         EccCheck = Check()
 248         EccCheck.Check()
 249         EdkLogger.quiet("Checking  done!")
 250
 251     ##
 252     #
 253     # Generate the scan report
 254     #
 255     def GenReport(self):
 256         EdkLogger.quiet("Generating report ...")
 257         EccGlobalData.gDb.TblReport.ToCSV(self.ReportFile)
 258         EdkLogger.quiet("Generating report done!")
 259
 260     def GetRealPathCase(self, path):
 261         TmpPath = path.rstrip(os.sep)
 262         PathParts = TmpPath.split(os.sep)
 263         if len(PathParts) == 0:
 264             return path
 265         if len(PathParts) == 1:
 266             if PathParts[0].strip().endswith(':'):
 267                 return PathParts[0].upper()
 268             # Relative dir, list . current dir
 269             Dirs = os.listdir('.')
 270             for Dir in Dirs:
 271                 if Dir.upper() == PathParts[0].upper():
 272                     return Dir
 273
 274         if PathParts[0].strip().endswith(':'):
 275             PathParts[0] = PathParts[0].upper()
 276         ParentDir = PathParts[0]
 277         RealPath = ParentDir
 278         if PathParts[0] == '':
 279             RealPath = os.sep
 280             ParentDir = os.sep
 281
 282         PathParts.remove(PathParts[0])    # need to remove the parent
 283         for Part in PathParts:
 284             Dirs = os.listdir(ParentDir + os.sep)
 285             for Dir in Dirs:
 286                 if Dir.upper() == Part.upper():
 287                     RealPath += os.sep
 288                     RealPath += Dir
 289                     break
 290             ParentDir += os.sep
 291             ParentDir += Dir
 292
 293         return RealPath
 294
 295     ## ParseOption
 296     #
 297     # Parse options
 298     #
 299     def ParseOption(self):
 300         (Options, Target) = self.EccOptionParser()
 301
 302         if Options.Workspace:
 303             os.environ["WORKSPACE"] = Options.Workspace
 304
 305         # Check workspace environment
 306         if "WORKSPACE" not in os.environ:
 307             EdkLogger.error("ECC", BuildToolError.ATTRIBUTE_NOT_AVAILABLE, "Environment variable not found",
 308                             ExtraData="WORKSPACE")
 309         else:
 310             EccGlobalData.gWorkspace = os.path.normpath(os.getenv("WORKSPACE"))
 311             if not os.path.exists(EccGlobalData.gWorkspace):
 312                 EdkLogger.error("ECC", BuildToolError.FILE_NOT_FOUND, ExtraData="WORKSPACE = %s" % EccGlobalData.gWorkspace)
 313             os.environ["WORKSPACE"] = EccGlobalData.gWorkspace
 314         # Set log level
 315         self.SetLogLevel(Options)
 316
 317         # Set other options
 318         if Options.ConfigFile is not None:
 319             self.ConfigFile = Options.ConfigFile
 320         if Options.OutputFile is not None:
 321             self.OutputFile = Options.OutputFile
 322         if Options.ReportFile is not None:
 323             self.ReportFile = Options.ReportFile
 324         if Options.ExceptionFile is not None:
 325             self.ExceptionFile = Options.ExceptionFile
 326         if Options.Target is not None:
 327             if not os.path.isdir(Options.Target):
 328                 EdkLogger.error("ECC", BuildToolError.OPTION_VALUE_INVALID, ExtraData="Target [%s] does NOT exist" % Options.Target)
 329             else:
 330                 EccGlobalData.gTarget = self.GetRealPathCase(os.path.normpath(Options.Target))
 331         else:
 332             EdkLogger.warn("Ecc", EdkLogger.ECC_ERROR, "The target source tree was not specified, using current WORKSPACE instead!")
 333             EccGlobalData.gTarget = os.path.normpath(os.getenv("WORKSPACE"))
 334         if Options.keepdatabase is not None:
 335             self.IsInit = False
 336         if Options.metadata is not None and Options.sourcecode is not None:
 337             EdkLogger.error("ECC", BuildToolError.OPTION_CONFLICT, ExtraData="-m and -s can't be specified at one time")
 338         if Options.metadata is not None:
 339             self.ScanSourceCode = False
 340         if Options.sourcecode is not None:
 341             self.ScanMetaData = False
 342         if Options.folders is not None:
 343             self.OnlyScan = True
 344
 345     ## SetLogLevel
 346     #
 347     # Set current log level of the tool based on args
 348     #
 349     # @param Option:  The option list including log level setting
 350     #
 351     def SetLogLevel(self, Option):
 352         if Option.verbose is not None:
 353             EdkLogger.SetLevel(EdkLogger.VERBOSE)
 354         elif Option.quiet is not None:
 355             EdkLogger.SetLevel(EdkLogger.QUIET)
 356         elif Option.debug is not None:
 357             EdkLogger.SetLevel(Option.debug + 1)
 358         else:
 359             EdkLogger.SetLevel(EdkLogger.INFO)
 360
 361     ## Parse command line options
 362     #
 363     # Using standard Python module optparse to parse command line option of this tool.
 364     #
 365     # @retval Opt   A optparse.Values object containing the parsed options
 366     # @retval Args  Target of build command
 367     #
 368     def EccOptionParser(self):
 369         Parser = OptionParser(description = self.Copyright, version = self.Version, prog = "Ecc.exe", usage = "%prog [options]")
 370         Parser.add_option("-t", "--target sourcepath", action="store", type="string", dest='Target',
 371             help="Check all files under the target workspace.")
 372         Parser.add_option("-c", "--config filename", action="store", type="string", dest="ConfigFile",
 373             help="Specify a configuration file. Defaultly use config.ini under ECC tool directory.")
 374         Parser.add_option("-o", "--outfile filename", action="store", type="string", dest="OutputFile",
 375             help="Specify the name of an output file, if and only if one filename was specified.")
 376         Parser.add_option("-r", "--reportfile filename", action="store", type="string", dest="ReportFile",
 377             help="Specify the name of an report file, if and only if one filename was specified.")
 378         Parser.add_option("-e", "--exceptionfile filename", action="store", type="string", dest="ExceptionFile",
 379             help="Specify the name of an exception file, if and only if one filename was specified.")
 380         Parser.add_option("-m", "--metadata", action="store_true", type=None, help="Only scan meta-data files information if this option is specified.")
 381         Parser.add_option("-s", "--sourcecode", action="store_true", type=None, help="Only scan source code files information if this option is specified.")
 382         Parser.add_option("-k", "--keepdatabase", action="store_true", type=None, help="The existing Ecc database will not be cleaned except report information if this option is specified.")
 383         Parser.add_option("-l", "--log filename", action="store", dest="LogFile", help="""If specified, the tool should emit the changes that
 384                                                                                           were made by the tool after printing the result message.
 385                                                                                           If filename, the emit to the file, otherwise emit to
 386                                                                                           standard output. If no modifications were made, then do not
 387                                                                                           create a log file, or output a log message.""")
 388         Parser.add_option("-q", "--quiet", action="store_true", type=None, help="Disable all messages except FATAL ERRORS.")
 389         Parser.add_option("-v", "--verbose", action="store_true", type=None, help="Turn on verbose output with informational messages printed, "\
 390                                                                                    "including library instances selected, final dependency expression, "\
 391                                                                                    "and warning messages, etc.")
 392         Parser.add_option("-d", "--debug", action="store", type="int", help="Enable debug messages at specified level.")
 393         Parser.add_option("-w", "--workspace", action="store", type="string", dest='Workspace', help="Specify workspace.")
 394         Parser.add_option("-f", "--folders", action="store_true", type=None, help="Only scanning specified folders which are recorded in config.ini file.")
 395
 396         (Opt, Args)=Parser.parse_args()
 397
 398         return (Opt, Args)
 399
 400 ##
 401 #
 402 # This acts like the main() function for the script, unless it is 'import'ed into another
 403 # script.
 404 #
 405 if __name__ == '__main__':
 406     # Initialize log system
 407     EdkLogger.Initialize()
 408     EdkLogger.IsRaiseError = False
 409
 410     StartTime = time.perf_counter()
 411     Ecc = Ecc()
 412     FinishTime = time.perf_counter()
 413
 414     BuildDuration = time.strftime("%M:%S", time.gmtime(int(round(FinishTime - StartTime))))
 415     EdkLogger.quiet("\n%s [%s]" % (time.strftime("%H:%M:%S, %b.%d %Y", time.localtime()), BuildDuration))