BaseTools/Source/Python/Ecc/EccMain.py

   1 ## @file
   2 # This file is used to be the main entrance of ECC tool
   3 #
   4 # Copyright (c) 2009 - 2018, Intel Corporation. All rights reserved.<BR>
   5 # This program and the accompanying materials
   6 # are licensed and made available under the terms and conditions of the BSD License
   7 # which accompanies this distribution.  The full text of the license may be found at
   8 # http://opensource.org/licenses/bsd-license.php
   9 #
  10 # THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS,
  11 # WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED.
  12 #
  13
  14 ##
  15 # Import Modules
  16 #
  17 from __future__ import absolute_import
  18 import Common.LongFilePathOs as os, time, glob, sys
  19 import Common.EdkLogger as EdkLogger
  20 from Ecc import Database
  21 from Ecc import EccGlobalData
  22 from Ecc.MetaDataParser import *
  23 from optparse import OptionParser
  24 from Ecc.Configuration import Configuration
  25 from Ecc.Check import Check
  26 import Common.GlobalData as GlobalData
  27
  28 from Common.StringUtils import NormPath
  29 from Common.BuildVersion import gBUILD_VERSION
  30 from Common import BuildToolError
  31 from Common.Misc import PathClass
  32 from Common.Misc import DirCache
  33 from Ecc.MetaFileWorkspace.MetaFileParser import DscParser
  34 from Ecc.MetaFileWorkspace.MetaFileParser import DecParser
  35 from Ecc.MetaFileWorkspace.MetaFileParser import InfParser
  36 from Ecc.MetaFileWorkspace.MetaFileParser import Fdf
  37 from Ecc.MetaFileWorkspace.MetaFileTable import MetaFileStorage
  38 from Ecc import c
  39 import re, string
  40 from Ecc.Exception import *
  41 from Common.LongFilePathSupport import OpenLongFilePath as open
  42 from Common.MultipleWorkspace import MultipleWorkspace as mws
  43
  44 ## Ecc
  45 #
  46 # This class is used to define Ecc main entrance
  47 #
  48 # @param object:          Inherited from object class
  49 #
  50 class Ecc(object):
  51     def __init__(self):
  52         # Version and Copyright
  53         self.VersionNumber = ("1.0" + " Build " + gBUILD_VERSION)
  54         self.Version = "%prog Version " + self.VersionNumber
  55         self.Copyright = "Copyright (c) 2009 - 2018, Intel Corporation  All rights reserved."
  56
  57         self.InitDefaultConfigIni()
  58         self.OutputFile = 'output.txt'
  59         self.ReportFile = 'Report.csv'
  60         self.ExceptionFile = 'exception.xml'
  61         self.IsInit = True
  62         self.ScanSourceCode = True
  63         self.ScanMetaData = True
  64         self.MetaFile = ''
  65         self.OnlyScan = None
  66
  67         # Parse the options and args
  68         self.ParseOption()
  69         EdkLogger.info(time.strftime("%H:%M:%S, %b.%d %Y ", time.localtime()) + "[00:00]" + "\n")
  70
  71         WorkspaceDir = os.path.normcase(os.path.normpath(os.environ["WORKSPACE"]))
  72         os.environ["WORKSPACE"] = WorkspaceDir
  73
  74         # set multiple workspace
  75         PackagesPath = os.getenv("PACKAGES_PATH")
  76         mws.setWs(WorkspaceDir, PackagesPath)
  77
  78         GlobalData.gWorkspace = WorkspaceDir
  79
  80         GlobalData.gGlobalDefines["WORKSPACE"]  = WorkspaceDir
  81
  82         EdkLogger.info("Loading ECC configuration ... done")
  83         # Generate checkpoints list
  84         EccGlobalData.gConfig = Configuration(self.ConfigFile)
  85
  86         # Generate exception list
  87         EccGlobalData.gException = ExceptionCheck(self.ExceptionFile)
  88
  89         # Init Ecc database
  90         EccGlobalData.gDb = Database.Database(Database.DATABASE_PATH)
  91         EccGlobalData.gDb.InitDatabase(self.IsInit)
  92
  93         #
  94         # Get files real name in workspace dir
  95         #
  96         GlobalData.gAllFiles = DirCache(GlobalData.gWorkspace)
  97
  98         # Build ECC database
  99 #         self.BuildDatabase()
 100         self.DetectOnlyScanDirs()
 101
 102         # Start to check
 103         self.Check()
 104
 105         # Show report
 106         self.GenReport()
 107
 108         # Close Database
 109         EccGlobalData.gDb.Close()
 110
 111     def InitDefaultConfigIni(self):
 112         paths = map(lambda p: os.path.join(p, 'Ecc', 'config.ini'), sys.path)
 113         paths = (os.path.realpath('config.ini'),) + tuple(paths)
 114         for path in paths:
 115             if os.path.exists(path):
 116                 self.ConfigFile = path
 117                 return
 118         self.ConfigFile = 'config.ini'
 119
 120
 121     ## DetectOnlyScan
 122     #
 123     # Detect whether only scanned folders have been enabled
 124     #
 125     def DetectOnlyScanDirs(self):
 126         if self.OnlyScan == True:
 127             OnlyScanDirs = []
 128             # Use regex here if multiple spaces or TAB exists in ScanOnlyDirList in config.ini file
 129             for folder in re.finditer(r'\S+', EccGlobalData.gConfig.ScanOnlyDirList):
 130                 OnlyScanDirs.append(folder.group())
 131             if len(OnlyScanDirs) != 0:
 132                 self.BuildDatabase(OnlyScanDirs)
 133             else:
 134                 EdkLogger.error("ECC", BuildToolError.OPTION_VALUE_INVALID, ExtraData="Use -f option need to fill specific folders in config.ini file")
 135         else:
 136             self.BuildDatabase()
 137
 138
 139     ## BuildDatabase
 140     #
 141     # Build the database for target
 142     #
 143     def BuildDatabase(self, SpeciDirs = None):
 144         # Clean report table
 145         EccGlobalData.gDb.TblReport.Drop()
 146         EccGlobalData.gDb.TblReport.Create()
 147
 148         # Build database
 149         if self.IsInit:
 150             if self.ScanMetaData:
 151                 EdkLogger.quiet("Building database for Meta Data File ...")
 152                 self.BuildMetaDataFileDatabase(SpeciDirs)
 153             if self.ScanSourceCode:
 154                 EdkLogger.quiet("Building database for Meta Data File Done!")
 155                 if SpeciDirs is None:
 156                     c.CollectSourceCodeDataIntoDB(EccGlobalData.gTarget)
 157                 else:
 158                     for specificDir in SpeciDirs:
 159                         c.CollectSourceCodeDataIntoDB(os.path.join(EccGlobalData.gTarget, specificDir))
 160
 161         EccGlobalData.gIdentifierTableList = GetTableList((MODEL_FILE_C, MODEL_FILE_H), 'Identifier', EccGlobalData.gDb)
 162         EccGlobalData.gCFileList = GetFileList(MODEL_FILE_C, EccGlobalData.gDb)
 163         EccGlobalData.gHFileList = GetFileList(MODEL_FILE_H, EccGlobalData.gDb)
 164         EccGlobalData.gUFileList = GetFileList(MODEL_FILE_UNI, EccGlobalData.gDb)
 165
 166     ## BuildMetaDataFileDatabase
 167     #
 168     # Build the database for meta data files
 169     #
 170     def BuildMetaDataFileDatabase(self, SpecificDirs = None):
 171         ScanFolders = []
 172         if SpecificDirs is None:
 173             ScanFolders.append(EccGlobalData.gTarget)
 174         else:
 175             for specificDir in SpecificDirs:
 176                 ScanFolders.append(os.path.join(EccGlobalData.gTarget, specificDir))
 177         EdkLogger.quiet("Building database for meta data files ...")
 178         Op = open(EccGlobalData.gConfig.MetaDataFileCheckPathOfGenerateFileList, 'w+')
 179         #SkipDirs = Read from config file
 180         SkipDirs = EccGlobalData.gConfig.SkipDirList
 181         SkipDirString = '|'.join(SkipDirs)
 182 #         p = re.compile(r'.*[\\/](?:%s)[\\/]?.*' % SkipDirString)
 183         p = re.compile(r'.*[\\/](?:%s^\S)[\\/]?.*' % SkipDirString)
 184         for scanFolder in ScanFolders:
 185             for Root, Dirs, Files in os.walk(scanFolder):
 186                 if p.match(Root.upper()):
 187                     continue
 188                 for Dir in Dirs:
 189                     Dirname = os.path.join(Root, Dir)
 190                     if os.path.islink(Dirname):
 191                         Dirname = os.path.realpath(Dirname)
 192                         if os.path.isdir(Dirname):
 193                             # symlinks to directories are treated as directories
 194                             Dirs.remove(Dir)
 195                             Dirs.append(Dirname)
 196
 197                 for File in Files:
 198                     if len(File) > 4 and File[-4:].upper() == ".DEC":
 199                         Filename = os.path.normpath(os.path.join(Root, File))
 200                         EdkLogger.quiet("Parsing %s" % Filename)
 201                         Op.write("%s\r" % Filename)
 202                         #Dec(Filename, True, True, EccGlobalData.gWorkspace, EccGlobalData.gDb)
 203                         self.MetaFile = DecParser(Filename, MODEL_FILE_DEC, EccGlobalData.gDb.TblDec)
 204                         self.MetaFile.Start()
 205                         continue
 206                     if len(File) > 4 and File[-4:].upper() == ".DSC":
 207                         Filename = os.path.normpath(os.path.join(Root, File))
 208                         EdkLogger.quiet("Parsing %s" % Filename)
 209                         Op.write("%s\r" % Filename)
 210                         #Dsc(Filename, True, True, EccGlobalData.gWorkspace, EccGlobalData.gDb)
 211                         self.MetaFile = DscParser(PathClass(Filename, Root), MODEL_FILE_DSC, MetaFileStorage(EccGlobalData.gDb.TblDsc.Cur, Filename, MODEL_FILE_DSC, True))
 212                         # alwasy do post-process, in case of macros change
 213                         self.MetaFile.DoPostProcess()
 214                         self.MetaFile.Start()
 215                         self.MetaFile._PostProcess()
 216                         continue
 217                     if len(File) > 4 and File[-4:].upper() == ".INF":
 218                         Filename = os.path.normpath(os.path.join(Root, File))
 219                         EdkLogger.quiet("Parsing %s" % Filename)
 220                         Op.write("%s\r" % Filename)
 221                         #Inf(Filename, True, True, EccGlobalData.gWorkspace, EccGlobalData.gDb)
 222                         self.MetaFile = InfParser(Filename, MODEL_FILE_INF, EccGlobalData.gDb.TblInf)
 223                         self.MetaFile.Start()
 224                         continue
 225                     if len(File) > 4 and File[-4:].upper() == ".FDF":
 226                         Filename = os.path.normpath(os.path.join(Root, File))
 227                         EdkLogger.quiet("Parsing %s" % Filename)
 228                         Op.write("%s\r" % Filename)
 229                         Fdf(Filename, True, EccGlobalData.gWorkspace, EccGlobalData.gDb)
 230                         continue
 231                     if len(File) > 4 and File[-4:].upper() == ".UNI":
 232                         Filename = os.path.normpath(os.path.join(Root, File))
 233                         EdkLogger.quiet("Parsing %s" % Filename)
 234                         Op.write("%s\r" % Filename)
 235                         FileID = EccGlobalData.gDb.TblFile.InsertFile(Filename, MODEL_FILE_UNI)
 236                         EccGlobalData.gDb.TblReport.UpdateBelongsToItemByFile(FileID, File)
 237                         continue
 238
 239         Op.close()
 240
 241         # Commit to database
 242         EccGlobalData.gDb.Conn.commit()
 243
 244         EdkLogger.quiet("Building database for meta data files done!")
 245
 246     ##
 247     #
 248     # Check each checkpoint
 249     #
 250     def Check(self):
 251         EdkLogger.quiet("Checking ...")
 252         EccCheck = Check()
 253         EccCheck.Check()
 254         EdkLogger.quiet("Checking  done!")
 255
 256     ##
 257     #
 258     # Generate the scan report
 259     #
 260     def GenReport(self):
 261         EdkLogger.quiet("Generating report ...")
 262         EccGlobalData.gDb.TblReport.ToCSV(self.ReportFile)
 263         EdkLogger.quiet("Generating report done!")
 264
 265     def GetRealPathCase(self, path):
 266         TmpPath = path.rstrip(os.sep)
 267         PathParts = TmpPath.split(os.sep)
 268         if len(PathParts) == 0:
 269             return path
 270         if len(PathParts) == 1:
 271             if PathParts[0].strip().endswith(':'):
 272                 return PathParts[0].upper()
 273             # Relative dir, list . current dir
 274             Dirs = os.listdir('.')
 275             for Dir in Dirs:
 276                 if Dir.upper() == PathParts[0].upper():
 277                     return Dir
 278
 279         if PathParts[0].strip().endswith(':'):
 280             PathParts[0] = PathParts[0].upper()
 281         ParentDir = PathParts[0]
 282         RealPath = ParentDir
 283         if PathParts[0] == '':
 284             RealPath = os.sep
 285             ParentDir = os.sep
 286
 287         PathParts.remove(PathParts[0])    # need to remove the parent
 288         for Part in PathParts:
 289             Dirs = os.listdir(ParentDir + os.sep)
 290             for Dir in Dirs:
 291                 if Dir.upper() == Part.upper():
 292                     RealPath += os.sep
 293                     RealPath += Dir
 294                     break
 295             ParentDir += os.sep
 296             ParentDir += Dir
 297
 298         return RealPath
 299
 300     ## ParseOption
 301     #
 302     # Parse options
 303     #
 304     def ParseOption(self):
 305         (Options, Target) = self.EccOptionParser()
 306
 307         if Options.Workspace:
 308             os.environ["WORKSPACE"] = Options.Workspace
 309
 310         # Check workspace envirnoment
 311         if "WORKSPACE" not in os.environ:
 312             EdkLogger.error("ECC", BuildToolError.ATTRIBUTE_NOT_AVAILABLE, "Environment variable not found",
 313                             ExtraData="WORKSPACE")
 314         else:
 315             EccGlobalData.gWorkspace = os.path.normpath(os.getenv("WORKSPACE"))
 316             if not os.path.exists(EccGlobalData.gWorkspace):
 317                 EdkLogger.error("ECC", BuildToolError.FILE_NOT_FOUND, ExtraData="WORKSPACE = %s" % EccGlobalData.gWorkspace)
 318             os.environ["WORKSPACE"] = EccGlobalData.gWorkspace
 319         # Set log level
 320         self.SetLogLevel(Options)
 321
 322         # Set other options
 323         if Options.ConfigFile is not None:
 324             self.ConfigFile = Options.ConfigFile
 325         if Options.OutputFile is not None:
 326             self.OutputFile = Options.OutputFile
 327         if Options.ReportFile is not None:
 328             self.ReportFile = Options.ReportFile
 329         if Options.ExceptionFile is not None:
 330             self.ExceptionFile = Options.ExceptionFile
 331         if Options.Target is not None:
 332             if not os.path.isdir(Options.Target):
 333                 EdkLogger.error("ECC", BuildToolError.OPTION_VALUE_INVALID, ExtraData="Target [%s] does NOT exist" % Options.Target)
 334             else:
 335                 EccGlobalData.gTarget = self.GetRealPathCase(os.path.normpath(Options.Target))
 336         else:
 337             EdkLogger.warn("Ecc", EdkLogger.ECC_ERROR, "The target source tree was not specified, using current WORKSPACE instead!")
 338             EccGlobalData.gTarget = os.path.normpath(os.getenv("WORKSPACE"))
 339         if Options.keepdatabase is not None:
 340             self.IsInit = False
 341         if Options.metadata is not None and Options.sourcecode is not None:
 342             EdkLogger.error("ECC", BuildToolError.OPTION_CONFLICT, ExtraData="-m and -s can't be specified at one time")
 343         if Options.metadata is not None:
 344             self.ScanSourceCode = False
 345         if Options.sourcecode is not None:
 346             self.ScanMetaData = False
 347         if Options.folders is not None:
 348             self.OnlyScan = True
 349
 350     ## SetLogLevel
 351     #
 352     # Set current log level of the tool based on args
 353     #
 354     # @param Option:  The option list including log level setting
 355     #
 356     def SetLogLevel(self, Option):
 357         if Option.verbose is not None:
 358             EdkLogger.SetLevel(EdkLogger.VERBOSE)
 359         elif Option.quiet is not None:
 360             EdkLogger.SetLevel(EdkLogger.QUIET)
 361         elif Option.debug is not None:
 362             EdkLogger.SetLevel(Option.debug + 1)
 363         else:
 364             EdkLogger.SetLevel(EdkLogger.INFO)
 365
 366     ## Parse command line options
 367     #
 368     # Using standard Python module optparse to parse command line option of this tool.
 369     #
 370     # @retval Opt   A optparse.Values object containing the parsed options
 371     # @retval Args  Target of build command
 372     #
 373     def EccOptionParser(self):
 374         Parser = OptionParser(description = self.Copyright, version = self.Version, prog = "Ecc.exe", usage = "%prog [options]")
 375         Parser.add_option("-t", "--target sourcepath", action="store", type="string", dest='Target',
 376             help="Check all files under the target workspace.")
 377         Parser.add_option("-c", "--config filename", action="store", type="string", dest="ConfigFile",
 378             help="Specify a configuration file. Defaultly use config.ini under ECC tool directory.")
 379         Parser.add_option("-o", "--outfile filename", action="store", type="string", dest="OutputFile",
 380             help="Specify the name of an output file, if and only if one filename was specified.")
 381         Parser.add_option("-r", "--reportfile filename", action="store", type="string", dest="ReportFile",
 382             help="Specify the name of an report file, if and only if one filename was specified.")
 383         Parser.add_option("-e", "--exceptionfile filename", action="store", type="string", dest="ExceptionFile",
 384             help="Specify the name of an exception file, if and only if one filename was specified.")
 385         Parser.add_option("-m", "--metadata", action="store_true", type=None, help="Only scan meta-data files information if this option is specified.")
 386         Parser.add_option("-s", "--sourcecode", action="store_true", type=None, help="Only scan source code files information if this option is specified.")
 387         Parser.add_option("-k", "--keepdatabase", action="store_true", type=None, help="The existing Ecc database will not be cleaned except report information if this option is specified.")
 388         Parser.add_option("-l", "--log filename", action="store", dest="LogFile", help="""If specified, the tool should emit the changes that
 389                                                                                           were made by the tool after printing the result message.
 390                                                                                           If filename, the emit to the file, otherwise emit to
 391                                                                                           standard output. If no modifications were made, then do not
 392                                                                                           create a log file, or output a log message.""")
 393         Parser.add_option("-q", "--quiet", action="store_true", type=None, help="Disable all messages except FATAL ERRORS.")
 394         Parser.add_option("-v", "--verbose", action="store_true", type=None, help="Turn on verbose output with informational messages printed, "\
 395                                                                                    "including library instances selected, final dependency expression, "\
 396                                                                                    "and warning messages, etc.")
 397         Parser.add_option("-d", "--debug", action="store", type="int", help="Enable debug messages at specified level.")
 398         Parser.add_option("-w", "--workspace", action="store", type="string", dest='Workspace', help="Specify workspace.")
 399         Parser.add_option("-f", "--folders", action="store_true", type=None, help="Only scanning specified folders which are recorded in config.ini file.")
 400
 401         (Opt, Args)=Parser.parse_args()
 402
 403         return (Opt, Args)
 404
 405 ##
 406 #
 407 # This acts like the main() function for the script, unless it is 'import'ed into another
 408 # script.
 409 #
 410 if __name__ == '__main__':
 411     # Initialize log system
 412     EdkLogger.Initialize()
 413     EdkLogger.IsRaiseError = False
 414
 415     StartTime = time.clock()
 416     Ecc = Ecc()
 417     FinishTime = time.clock()
 418
 419     BuildDuration = time.strftime("%M:%S", time.gmtime(int(round(FinishTime - StartTime))))
 420     EdkLogger.quiet("\n%s [%s]" % (time.strftime("%H:%M:%S, %b.%d %Y", time.localtime()), BuildDuration))