BaseTools/Source/Python/Ecc/EccMain.py

   1 ## @file
   2 # This file is used to be the main entrance of ECC tool
   3 #
   4 # Copyright (c) 2009 - 2018, Intel Corporation. All rights reserved.<BR>
   5 # SPDX-License-Identifier: BSD-2-Clause-Patent
   6 #
   7
   8 ##
   9 # Import Modules
  10 #
  11 from __future__ import absolute_import
  12 import Common.LongFilePathOs as os, time, glob, sys
  13 import Common.EdkLogger as EdkLogger
  14 from Ecc import Database
  15 from Ecc import EccGlobalData
  16 from Ecc.MetaDataParser import *
  17 from optparse import OptionParser
  18 from Ecc.Configuration import Configuration
  19 from Ecc.Check import Check
  20 import Common.GlobalData as GlobalData
  21
  22 from Common.StringUtils import NormPath
  23 from Common.BuildVersion import gBUILD_VERSION
  24 from Common import BuildToolError
  25 from Common.Misc import PathClass
  26 from Common.Misc import DirCache
  27 from Ecc.MetaFileWorkspace.MetaFileParser import DscParser
  28 from Ecc.MetaFileWorkspace.MetaFileParser import DecParser
  29 from Ecc.MetaFileWorkspace.MetaFileParser import InfParser
  30 from Ecc.MetaFileWorkspace.MetaFileParser import Fdf
  31 from Ecc.MetaFileWorkspace.MetaFileTable import MetaFileStorage
  32 from Ecc import c
  33 import re, string
  34 from Ecc.Exception import *
  35 from Common.LongFilePathSupport import OpenLongFilePath as open
  36 from Common.MultipleWorkspace import MultipleWorkspace as mws
  37
  38 ## Ecc
  39 #
  40 # This class is used to define Ecc main entrance
  41 #
  42 # @param object:          Inherited from object class
  43 #
  44 class Ecc(object):
  45     def __init__(self):
  46         # Version and Copyright
  47         self.VersionNumber = ("1.0" + " Build " + gBUILD_VERSION)
  48         self.Version = "%prog Version " + self.VersionNumber
  49         self.Copyright = "Copyright (c) 2009 - 2018, Intel Corporation  All rights reserved."
  50
  51         self.InitDefaultConfigIni()
  52         self.OutputFile = 'output.txt'
  53         self.ReportFile = 'Report.csv'
  54         self.ExceptionFile = 'exception.xml'
  55         self.IsInit = True
  56         self.ScanSourceCode = True
  57         self.ScanMetaData = True
  58         self.MetaFile = ''
  59         self.OnlyScan = None
  60
  61         # Parse the options and args
  62         self.ParseOption()
  63         EdkLogger.info(time.strftime("%H:%M:%S, %b.%d %Y ", time.localtime()) + "[00:00]" + "\n")
  64
  65         WorkspaceDir = os.path.normcase(os.path.normpath(os.environ["WORKSPACE"]))
  66         os.environ["WORKSPACE"] = WorkspaceDir
  67
  68         # set multiple workspace
  69         PackagesPath = os.getenv("PACKAGES_PATH")
  70         mws.setWs(WorkspaceDir, PackagesPath)
  71
  72         GlobalData.gWorkspace = WorkspaceDir
  73
  74         GlobalData.gGlobalDefines["WORKSPACE"]  = WorkspaceDir
  75
  76         EdkLogger.info("Loading ECC configuration ... done")
  77         # Generate checkpoints list
  78         EccGlobalData.gConfig = Configuration(self.ConfigFile)
  79
  80         # Generate exception list
  81         EccGlobalData.gException = ExceptionCheck(self.ExceptionFile)
  82
  83         # Init Ecc database
  84         EccGlobalData.gDb = Database.Database(Database.DATABASE_PATH)
  85         EccGlobalData.gDb.InitDatabase(self.IsInit)
  86
  87         #
  88         # Get files real name in workspace dir
  89         #
  90         GlobalData.gAllFiles = DirCache(GlobalData.gWorkspace)
  91
  92         # Build ECC database
  93 #         self.BuildDatabase()
  94         self.DetectOnlyScanDirs()
  95
  96         # Start to check
  97         self.Check()
  98
  99         # Show report
 100         self.GenReport()
 101
 102         # Close Database
 103         EccGlobalData.gDb.Close()
 104
 105     def InitDefaultConfigIni(self):
 106         paths = map(lambda p: os.path.join(p, 'Ecc', 'config.ini'), sys.path)
 107         paths = (os.path.realpath('config.ini'),) + tuple(paths)
 108         for path in paths:
 109             if os.path.exists(path):
 110                 self.ConfigFile = path
 111                 return
 112         self.ConfigFile = 'config.ini'
 113
 114
 115     ## DetectOnlyScan
 116     #
 117     # Detect whether only scanned folders have been enabled
 118     #
 119     def DetectOnlyScanDirs(self):
 120         if self.OnlyScan == True:
 121             OnlyScanDirs = []
 122             # Use regex here if multiple spaces or TAB exists in ScanOnlyDirList in config.ini file
 123             for folder in re.finditer(r'\S+', EccGlobalData.gConfig.ScanOnlyDirList):
 124                 OnlyScanDirs.append(folder.group())
 125             if len(OnlyScanDirs) != 0:
 126                 self.BuildDatabase(OnlyScanDirs)
 127             else:
 128                 EdkLogger.error("ECC", BuildToolError.OPTION_VALUE_INVALID, ExtraData="Use -f option need to fill specific folders in config.ini file")
 129         else:
 130             self.BuildDatabase()
 131
 132
 133     ## BuildDatabase
 134     #
 135     # Build the database for target
 136     #
 137     def BuildDatabase(self, SpeciDirs = None):
 138         # Clean report table
 139         EccGlobalData.gDb.TblReport.Drop()
 140         EccGlobalData.gDb.TblReport.Create()
 141
 142         # Build database
 143         if self.IsInit:
 144             if self.ScanMetaData:
 145                 EdkLogger.quiet("Building database for Meta Data File ...")
 146                 self.BuildMetaDataFileDatabase(SpeciDirs)
 147             if self.ScanSourceCode:
 148                 EdkLogger.quiet("Building database for Meta Data File Done!")
 149                 if SpeciDirs is None:
 150                     c.CollectSourceCodeDataIntoDB(EccGlobalData.gTarget)
 151                 else:
 152                     for specificDir in SpeciDirs:
 153                         c.CollectSourceCodeDataIntoDB(os.path.join(EccGlobalData.gTarget, specificDir))
 154
 155         EccGlobalData.gIdentifierTableList = GetTableList((MODEL_FILE_C, MODEL_FILE_H), 'Identifier', EccGlobalData.gDb)
 156         EccGlobalData.gCFileList = GetFileList(MODEL_FILE_C, EccGlobalData.gDb)
 157         EccGlobalData.gHFileList = GetFileList(MODEL_FILE_H, EccGlobalData.gDb)
 158         EccGlobalData.gUFileList = GetFileList(MODEL_FILE_UNI, EccGlobalData.gDb)
 159
 160     ## BuildMetaDataFileDatabase
 161     #
 162     # Build the database for meta data files
 163     #
 164     def BuildMetaDataFileDatabase(self, SpecificDirs = None):
 165         ScanFolders = []
 166         if SpecificDirs is None:
 167             ScanFolders.append(EccGlobalData.gTarget)
 168         else:
 169             for specificDir in SpecificDirs:
 170                 ScanFolders.append(os.path.join(EccGlobalData.gTarget, specificDir))
 171         EdkLogger.quiet("Building database for meta data files ...")
 172         Op = open(EccGlobalData.gConfig.MetaDataFileCheckPathOfGenerateFileList, 'w+')
 173         #SkipDirs = Read from config file
 174         SkipDirs = EccGlobalData.gConfig.SkipDirList
 175         SkipDirString = '|'.join(SkipDirs)
 176 #         p = re.compile(r'.*[\\/](?:%s)[\\/]?.*' % SkipDirString)
 177         p = re.compile(r'.*[\\/](?:%s^\S)[\\/]?.*' % SkipDirString)
 178         for scanFolder in ScanFolders:
 179             for Root, Dirs, Files in os.walk(scanFolder):
 180                 if p.match(Root.upper()):
 181                     continue
 182                 for Dir in Dirs:
 183                     Dirname = os.path.join(Root, Dir)
 184                     if os.path.islink(Dirname):
 185                         Dirname = os.path.realpath(Dirname)
 186                         if os.path.isdir(Dirname):
 187                             # symlinks to directories are treated as directories
 188                             Dirs.remove(Dir)
 189                             Dirs.append(Dirname)
 190
 191                 for File in Files:
 192                     if len(File) > 4 and File[-4:].upper() == ".DEC":
 193                         Filename = os.path.normpath(os.path.join(Root, File))
 194                         EdkLogger.quiet("Parsing %s" % Filename)
 195                         Op.write("%s\r" % Filename)
 196                         #Dec(Filename, True, True, EccGlobalData.gWorkspace, EccGlobalData.gDb)
 197                         self.MetaFile = DecParser(Filename, MODEL_FILE_DEC, EccGlobalData.gDb.TblDec)
 198                         self.MetaFile.Start()
 199                         continue
 200                     if len(File) > 4 and File[-4:].upper() == ".DSC":
 201                         Filename = os.path.normpath(os.path.join(Root, File))
 202                         EdkLogger.quiet("Parsing %s" % Filename)
 203                         Op.write("%s\r" % Filename)
 204                         #Dsc(Filename, True, True, EccGlobalData.gWorkspace, EccGlobalData.gDb)
 205                         self.MetaFile = DscParser(PathClass(Filename, Root), MODEL_FILE_DSC, MetaFileStorage(EccGlobalData.gDb.TblDsc.Cur, Filename, MODEL_FILE_DSC, True))
 206                         # always do post-process, in case of macros change
 207                         self.MetaFile.DoPostProcess()
 208                         self.MetaFile.Start()
 209                         self.MetaFile._PostProcess()
 210                         continue
 211                     if len(File) > 4 and File[-4:].upper() == ".INF":
 212                         Filename = os.path.normpath(os.path.join(Root, File))
 213                         EdkLogger.quiet("Parsing %s" % Filename)
 214                         Op.write("%s\r" % Filename)
 215                         #Inf(Filename, True, True, EccGlobalData.gWorkspace, EccGlobalData.gDb)
 216                         self.MetaFile = InfParser(Filename, MODEL_FILE_INF, EccGlobalData.gDb.TblInf)
 217                         self.MetaFile.Start()
 218                         continue
 219                     if len(File) > 4 and File[-4:].upper() == ".FDF":
 220                         Filename = os.path.normpath(os.path.join(Root, File))
 221                         EdkLogger.quiet("Parsing %s" % Filename)
 222                         Op.write("%s\r" % Filename)
 223                         Fdf(Filename, True, EccGlobalData.gWorkspace, EccGlobalData.gDb)
 224                         continue
 225                     if len(File) > 4 and File[-4:].upper() == ".UNI":
 226                         Filename = os.path.normpath(os.path.join(Root, File))
 227                         EdkLogger.quiet("Parsing %s" % Filename)
 228                         Op.write("%s\r" % Filename)
 229                         FileID = EccGlobalData.gDb.TblFile.InsertFile(Filename, MODEL_FILE_UNI)
 230                         EccGlobalData.gDb.TblReport.UpdateBelongsToItemByFile(FileID, File)
 231                         continue
 232
 233         Op.close()
 234
 235         # Commit to database
 236         EccGlobalData.gDb.Conn.commit()
 237
 238         EdkLogger.quiet("Building database for meta data files done!")
 239
 240     ##
 241     #
 242     # Check each checkpoint
 243     #
 244     def Check(self):
 245         EdkLogger.quiet("Checking ...")
 246         EccCheck = Check()
 247         EccCheck.Check()
 248         EdkLogger.quiet("Checking  done!")
 249
 250     ##
 251     #
 252     # Generate the scan report
 253     #
 254     def GenReport(self):
 255         EdkLogger.quiet("Generating report ...")
 256         EccGlobalData.gDb.TblReport.ToCSV(self.ReportFile)
 257         EdkLogger.quiet("Generating report done!")
 258
 259     def GetRealPathCase(self, path):
 260         TmpPath = path.rstrip(os.sep)
 261         PathParts = TmpPath.split(os.sep)
 262         if len(PathParts) == 0:
 263             return path
 264         if len(PathParts) == 1:
 265             if PathParts[0].strip().endswith(':'):
 266                 return PathParts[0].upper()
 267             # Relative dir, list . current dir
 268             Dirs = os.listdir('.')
 269             for Dir in Dirs:
 270                 if Dir.upper() == PathParts[0].upper():
 271                     return Dir
 272
 273         if PathParts[0].strip().endswith(':'):
 274             PathParts[0] = PathParts[0].upper()
 275         ParentDir = PathParts[0]
 276         RealPath = ParentDir
 277         if PathParts[0] == '':
 278             RealPath = os.sep
 279             ParentDir = os.sep
 280
 281         PathParts.remove(PathParts[0])    # need to remove the parent
 282         for Part in PathParts:
 283             Dirs = os.listdir(ParentDir + os.sep)
 284             for Dir in Dirs:
 285                 if Dir.upper() == Part.upper():
 286                     RealPath += os.sep
 287                     RealPath += Dir
 288                     break
 289             ParentDir += os.sep
 290             ParentDir += Dir
 291
 292         return RealPath
 293
 294     ## ParseOption
 295     #
 296     # Parse options
 297     #
 298     def ParseOption(self):
 299         (Options, Target) = self.EccOptionParser()
 300
 301         if Options.Workspace:
 302             os.environ["WORKSPACE"] = Options.Workspace
 303
 304         # Check workspace environment
 305         if "WORKSPACE" not in os.environ:
 306             EdkLogger.error("ECC", BuildToolError.ATTRIBUTE_NOT_AVAILABLE, "Environment variable not found",
 307                             ExtraData="WORKSPACE")
 308         else:
 309             EccGlobalData.gWorkspace = os.path.normpath(os.getenv("WORKSPACE"))
 310             if not os.path.exists(EccGlobalData.gWorkspace):
 311                 EdkLogger.error("ECC", BuildToolError.FILE_NOT_FOUND, ExtraData="WORKSPACE = %s" % EccGlobalData.gWorkspace)
 312             os.environ["WORKSPACE"] = EccGlobalData.gWorkspace
 313         # Set log level
 314         self.SetLogLevel(Options)
 315
 316         # Set other options
 317         if Options.ConfigFile is not None:
 318             self.ConfigFile = Options.ConfigFile
 319         if Options.OutputFile is not None:
 320             self.OutputFile = Options.OutputFile
 321         if Options.ReportFile is not None:
 322             self.ReportFile = Options.ReportFile
 323         if Options.ExceptionFile is not None:
 324             self.ExceptionFile = Options.ExceptionFile
 325         if Options.Target is not None:
 326             if not os.path.isdir(Options.Target):
 327                 EdkLogger.error("ECC", BuildToolError.OPTION_VALUE_INVALID, ExtraData="Target [%s] does NOT exist" % Options.Target)
 328             else:
 329                 EccGlobalData.gTarget = self.GetRealPathCase(os.path.normpath(Options.Target))
 330         else:
 331             EdkLogger.warn("Ecc", EdkLogger.ECC_ERROR, "The target source tree was not specified, using current WORKSPACE instead!")
 332             EccGlobalData.gTarget = os.path.normpath(os.getenv("WORKSPACE"))
 333         if Options.keepdatabase is not None:
 334             self.IsInit = False
 335         if Options.metadata is not None and Options.sourcecode is not None:
 336             EdkLogger.error("ECC", BuildToolError.OPTION_CONFLICT, ExtraData="-m and -s can't be specified at one time")
 337         if Options.metadata is not None:
 338             self.ScanSourceCode = False
 339         if Options.sourcecode is not None:
 340             self.ScanMetaData = False
 341         if Options.folders is not None:
 342             self.OnlyScan = True
 343
 344     ## SetLogLevel
 345     #
 346     # Set current log level of the tool based on args
 347     #
 348     # @param Option:  The option list including log level setting
 349     #
 350     def SetLogLevel(self, Option):
 351         if Option.verbose is not None:
 352             EdkLogger.SetLevel(EdkLogger.VERBOSE)
 353         elif Option.quiet is not None:
 354             EdkLogger.SetLevel(EdkLogger.QUIET)
 355         elif Option.debug is not None:
 356             EdkLogger.SetLevel(Option.debug + 1)
 357         else:
 358             EdkLogger.SetLevel(EdkLogger.INFO)
 359
 360     ## Parse command line options
 361     #
 362     # Using standard Python module optparse to parse command line option of this tool.
 363     #
 364     # @retval Opt   A optparse.Values object containing the parsed options
 365     # @retval Args  Target of build command
 366     #
 367     def EccOptionParser(self):
 368         Parser = OptionParser(description = self.Copyright, version = self.Version, prog = "Ecc.exe", usage = "%prog [options]")
 369         Parser.add_option("-t", "--target sourcepath", action="store", type="string", dest='Target',
 370             help="Check all files under the target workspace.")
 371         Parser.add_option("-c", "--config filename", action="store", type="string", dest="ConfigFile",
 372             help="Specify a configuration file. Defaultly use config.ini under ECC tool directory.")
 373         Parser.add_option("-o", "--outfile filename", action="store", type="string", dest="OutputFile",
 374             help="Specify the name of an output file, if and only if one filename was specified.")
 375         Parser.add_option("-r", "--reportfile filename", action="store", type="string", dest="ReportFile",
 376             help="Specify the name of an report file, if and only if one filename was specified.")
 377         Parser.add_option("-e", "--exceptionfile filename", action="store", type="string", dest="ExceptionFile",
 378             help="Specify the name of an exception file, if and only if one filename was specified.")
 379         Parser.add_option("-m", "--metadata", action="store_true", type=None, help="Only scan meta-data files information if this option is specified.")
 380         Parser.add_option("-s", "--sourcecode", action="store_true", type=None, help="Only scan source code files information if this option is specified.")
 381         Parser.add_option("-k", "--keepdatabase", action="store_true", type=None, help="The existing Ecc database will not be cleaned except report information if this option is specified.")
 382         Parser.add_option("-l", "--log filename", action="store", dest="LogFile", help="""If specified, the tool should emit the changes that
 383                                                                                           were made by the tool after printing the result message.
 384                                                                                           If filename, the emit to the file, otherwise emit to
 385                                                                                           standard output. If no modifications were made, then do not
 386                                                                                           create a log file, or output a log message.""")
 387         Parser.add_option("-q", "--quiet", action="store_true", type=None, help="Disable all messages except FATAL ERRORS.")
 388         Parser.add_option("-v", "--verbose", action="store_true", type=None, help="Turn on verbose output with informational messages printed, "\
 389                                                                                    "including library instances selected, final dependency expression, "\
 390                                                                                    "and warning messages, etc.")
 391         Parser.add_option("-d", "--debug", action="store", type="int", help="Enable debug messages at specified level.")
 392         Parser.add_option("-w", "--workspace", action="store", type="string", dest='Workspace', help="Specify workspace.")
 393         Parser.add_option("-f", "--folders", action="store_true", type=None, help="Only scanning specified folders which are recorded in config.ini file.")
 394
 395         (Opt, Args)=Parser.parse_args()
 396
 397         return (Opt, Args)
 398
 399 ##
 400 #
 401 # This acts like the main() function for the script, unless it is 'import'ed into another
 402 # script.
 403 #
 404 if __name__ == '__main__':
 405     # Initialize log system
 406     EdkLogger.Initialize()
 407     EdkLogger.IsRaiseError = False
 408
 409     StartTime = time.clock()
 410     Ecc = Ecc()
 411     FinishTime = time.clock()
 412
 413     BuildDuration = time.strftime("%M:%S", time.gmtime(int(round(FinishTime - StartTime))))
 414     EdkLogger.quiet("\n%s [%s]" % (time.strftime("%H:%M:%S, %b.%d %Y", time.localtime()), BuildDuration))