BaseTools/Source/Python/Ecc/Ecc.py

   1 ## @file
   2 # This file is used to be the main entrance of ECC tool
   3 #
   4 # Copyright (c) 2009 - 2014, Intel Corporation. All rights reserved.<BR>
   5 # This program and the accompanying materials
   6 # are licensed and made available under the terms and conditions of the BSD License
   7 # which accompanies this distribution.  The full text of the license may be found at
   8 # http://opensource.org/licenses/bsd-license.php
   9 #
  10 # THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS,
  11 # WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED.
  12 #
  13
  14 ##
  15 # Import Modules
  16 #
  17 import os, time, glob, sys
  18 import Common.EdkLogger as EdkLogger
  19 import Database
  20 import EccGlobalData
  21 from MetaDataParser import *
  22 from optparse import OptionParser
  23 from Configuration import Configuration
  24 from Check import Check
  25 import Common.GlobalData as GlobalData
  26
  27 from Common.String import NormPath
  28 from Common.BuildVersion import gBUILD_VERSION
  29 from Common import BuildToolError
  30 from Common.Misc import PathClass
  31 from Common.Misc import DirCache
  32 from MetaFileWorkspace.MetaFileParser import DscParser
  33 from MetaFileWorkspace.MetaFileParser import DecParser
  34 from MetaFileWorkspace.MetaFileParser import InfParser
  35 from MetaFileWorkspace.MetaFileParser import Fdf
  36 from MetaFileWorkspace.MetaFileTable import MetaFileStorage
  37 import c
  38 import re, string
  39 from Exception import *
  40
  41 ## Ecc
  42 #
  43 # This class is used to define Ecc main entrance
  44 #
  45 # @param object:          Inherited from object class
  46 #
  47 class Ecc(object):
  48     def __init__(self):
  49         # Version and Copyright
  50         self.VersionNumber = ("0.01" + " " + gBUILD_VERSION)
  51         self.Version = "%prog Version " + self.VersionNumber
  52         self.Copyright = "Copyright (c) 2009 - 2010, Intel Corporation  All rights reserved."
  53
  54         self.InitDefaultConfigIni()
  55         self.OutputFile = 'output.txt'
  56         self.ReportFile = 'Report.csv'
  57         self.ExceptionFile = 'exception.xml'
  58         self.IsInit = True
  59         self.ScanSourceCode = True
  60         self.ScanMetaData = True
  61         self.MetaFile = ''
  62         self.OnlyScan = None
  63
  64         # Parse the options and args
  65         self.ParseOption()
  66
  67         #
  68         # Check EFI_SOURCE (Edk build convention). EDK_SOURCE will always point to ECP
  69         #
  70         WorkspaceDir = os.path.normcase(os.path.normpath(os.environ["WORKSPACE"]))
  71         os.environ["WORKSPACE"] = WorkspaceDir
  72         if "ECP_SOURCE" not in os.environ:
  73             os.environ["ECP_SOURCE"] = os.path.join(WorkspaceDir, GlobalData.gEdkCompatibilityPkg)
  74         if "EFI_SOURCE" not in os.environ:
  75             os.environ["EFI_SOURCE"] = os.environ["ECP_SOURCE"]
  76         if "EDK_SOURCE" not in os.environ:
  77             os.environ["EDK_SOURCE"] = os.environ["ECP_SOURCE"]
  78
  79         #
  80         # Unify case of characters on case-insensitive systems
  81         #
  82         EfiSourceDir = os.path.normcase(os.path.normpath(os.environ["EFI_SOURCE"]))
  83         EdkSourceDir = os.path.normcase(os.path.normpath(os.environ["EDK_SOURCE"]))
  84         EcpSourceDir = os.path.normcase(os.path.normpath(os.environ["ECP_SOURCE"]))
  85
  86         os.environ["EFI_SOURCE"] = EfiSourceDir
  87         os.environ["EDK_SOURCE"] = EdkSourceDir
  88         os.environ["ECP_SOURCE"] = EcpSourceDir
  89
  90         GlobalData.gWorkspace = WorkspaceDir
  91         GlobalData.gEfiSource = EfiSourceDir
  92         GlobalData.gEdkSource = EdkSourceDir
  93         GlobalData.gEcpSource = EcpSourceDir
  94
  95         GlobalData.gGlobalDefines["WORKSPACE"]  = WorkspaceDir
  96         GlobalData.gGlobalDefines["EFI_SOURCE"] = EfiSourceDir
  97         GlobalData.gGlobalDefines["EDK_SOURCE"] = EdkSourceDir
  98         GlobalData.gGlobalDefines["ECP_SOURCE"] = EcpSourceDir
  99
 100
 101         # Generate checkpoints list
 102         EccGlobalData.gConfig = Configuration(self.ConfigFile)
 103
 104         # Generate exception list
 105         EccGlobalData.gException = ExceptionCheck(self.ExceptionFile)
 106
 107         # Init Ecc database
 108         EccGlobalData.gDb = Database.Database(Database.DATABASE_PATH)
 109         EccGlobalData.gDb.InitDatabase(self.IsInit)
 110
 111         #
 112         # Get files real name in workspace dir
 113         #
 114         GlobalData.gAllFiles = DirCache(GlobalData.gWorkspace)
 115
 116         # Build ECC database
 117 #         self.BuildDatabase()
 118         self.DetectOnlyScanDirs()
 119
 120         # Start to check
 121         self.Check()
 122
 123         # Show report
 124         self.GenReport()
 125
 126         # Close Database
 127         EccGlobalData.gDb.Close()
 128
 129     def InitDefaultConfigIni(self):
 130         paths = map(lambda p: os.path.join(p, 'Ecc', 'config.ini'), sys.path)
 131         paths = (os.path.realpath('config.ini'),) + tuple(paths)
 132         for path in paths:
 133             if os.path.exists(path):
 134                 self.ConfigFile = path
 135                 return
 136         self.ConfigFile = 'config.ini'
 137
 138
 139     ## DetectOnlyScan
 140     #
 141     # Detect whether only scanned folders have been enabled
 142     #
 143     def DetectOnlyScanDirs(self):
 144         if self.OnlyScan == True:
 145             OnlyScanDirs = []
 146             # Use regex here if multiple spaces or TAB exists in ScanOnlyDirList in config.ini file
 147             for folder in re.finditer(r'\S+', EccGlobalData.gConfig.ScanOnlyDirList):
 148                 OnlyScanDirs.append(folder.group())
 149             if len(OnlyScanDirs) != 0:
 150                 self.BuildDatabase(OnlyScanDirs)
 151             else:
 152                 EdkLogger.error("ECC", BuildToolError.OPTION_VALUE_INVALID, ExtraData="Use -f option need to fill specific folders in config.ini file")
 153         else:
 154             self.BuildDatabase()
 155
 156
 157     ## BuildDatabase
 158     #
 159     # Build the database for target
 160     #
 161     def BuildDatabase(self, SpeciDirs = None):
 162         # Clean report table
 163         EccGlobalData.gDb.TblReport.Drop()
 164         EccGlobalData.gDb.TblReport.Create()
 165
 166         # Build database
 167         if self.IsInit:
 168             if self.ScanMetaData:
 169                 EdkLogger.quiet("Building database for Meta Data File ...")
 170                 self.BuildMetaDataFileDatabase(SpeciDirs)
 171             if self.ScanSourceCode:
 172                 EdkLogger.quiet("Building database for Meta Data File Done!")
 173                 if SpeciDirs == None:
 174                     c.CollectSourceCodeDataIntoDB(EccGlobalData.gTarget)
 175                 else:
 176                     for specificDir in SpeciDirs:
 177                         c.CollectSourceCodeDataIntoDB(os.path.join(EccGlobalData.gTarget, specificDir))
 178
 179         EccGlobalData.gIdentifierTableList = GetTableList((MODEL_FILE_C, MODEL_FILE_H), 'Identifier', EccGlobalData.gDb)
 180         EccGlobalData.gCFileList = GetFileList(MODEL_FILE_C, EccGlobalData.gDb)
 181         EccGlobalData.gHFileList = GetFileList(MODEL_FILE_H, EccGlobalData.gDb)
 182
 183     ## BuildMetaDataFileDatabase
 184     #
 185     # Build the database for meta data files
 186     #
 187     def BuildMetaDataFileDatabase(self, SpecificDirs = None):
 188         ScanFolders = []
 189         if SpecificDirs == None:
 190             ScanFolders.append(EccGlobalData.gTarget)
 191         else:
 192             for specificDir in SpecificDirs:
 193                 ScanFolders.append(os.path.join(EccGlobalData.gTarget, specificDir))
 194         EdkLogger.quiet("Building database for meta data files ...")
 195         Op = open(EccGlobalData.gConfig.MetaDataFileCheckPathOfGenerateFileList, 'w+')
 196         #SkipDirs = Read from config file
 197         SkipDirs = EccGlobalData.gConfig.SkipDirList
 198         SkipDirString = string.join(SkipDirs, '|')
 199 #         p = re.compile(r'.*[\\/](?:%s)[\\/]?.*' % SkipDirString)
 200         p = re.compile(r'.*[\\/](?:%s^\S)[\\/]?.*' % SkipDirString)
 201         for scanFolder in ScanFolders:
 202             for Root, Dirs, Files in os.walk(scanFolder):
 203                 if p.match(Root.upper()):
 204                     continue
 205                 for Dir in Dirs:
 206                     Dirname = os.path.join(Root, Dir)
 207                     if os.path.islink(Dirname):
 208                         Dirname = os.path.realpath(Dirname)
 209                         if os.path.isdir(Dirname):
 210                             # symlinks to directories are treated as directories
 211                             Dirs.remove(Dir)
 212                             Dirs.append(Dirname)
 213
 214                 for File in Files:
 215                     if len(File) > 4 and File[-4:].upper() == ".DEC":
 216                         Filename = os.path.normpath(os.path.join(Root, File))
 217                         EdkLogger.quiet("Parsing %s" % Filename)
 218                         Op.write("%s\r" % Filename)
 219                         #Dec(Filename, True, True, EccGlobalData.gWorkspace, EccGlobalData.gDb)
 220                         self.MetaFile = DecParser(Filename, MODEL_FILE_DEC, EccGlobalData.gDb.TblDec)
 221                         self.MetaFile.Start()
 222                         continue
 223                     if len(File) > 4 and File[-4:].upper() == ".DSC":
 224                         Filename = os.path.normpath(os.path.join(Root, File))
 225                         EdkLogger.quiet("Parsing %s" % Filename)
 226                         Op.write("%s\r" % Filename)
 227                         #Dsc(Filename, True, True, EccGlobalData.gWorkspace, EccGlobalData.gDb)
 228                         self.MetaFile = DscParser(PathClass(Filename, Root), MODEL_FILE_DSC, MetaFileStorage(EccGlobalData.gDb.TblDsc.Cur, Filename, MODEL_FILE_DSC, True))
 229                         # alwasy do post-process, in case of macros change
 230                         self.MetaFile.DoPostProcess()
 231                         self.MetaFile.Start()
 232                         self.MetaFile._PostProcess()
 233                         continue
 234                     if len(File) > 4 and File[-4:].upper() == ".INF":
 235                         Filename = os.path.normpath(os.path.join(Root, File))
 236                         EdkLogger.quiet("Parsing %s" % Filename)
 237                         Op.write("%s\r" % Filename)
 238                         #Inf(Filename, True, True, EccGlobalData.gWorkspace, EccGlobalData.gDb)
 239                         self.MetaFile = InfParser(Filename, MODEL_FILE_INF, EccGlobalData.gDb.TblInf)
 240                         self.MetaFile.Start()
 241                         continue
 242                     if len(File) > 4 and File[-4:].upper() == ".FDF":
 243                         Filename = os.path.normpath(os.path.join(Root, File))
 244                         EdkLogger.quiet("Parsing %s" % Filename)
 245                         Op.write("%s\r" % Filename)
 246                         Fdf(Filename, True, EccGlobalData.gWorkspace, EccGlobalData.gDb)
 247                         continue
 248         Op.close()
 249
 250         # Commit to database
 251         EccGlobalData.gDb.Conn.commit()
 252
 253         EdkLogger.quiet("Building database for meta data files done!")
 254
 255     ##
 256     #
 257     # Check each checkpoint
 258     #
 259     def Check(self):
 260         EdkLogger.quiet("Checking ...")
 261         EccCheck = Check()
 262         EccCheck.Check()
 263         EdkLogger.quiet("Checking  done!")
 264
 265     ##
 266     #
 267     # Generate the scan report
 268     #
 269     def GenReport(self):
 270         EdkLogger.quiet("Generating report ...")
 271         EccGlobalData.gDb.TblReport.ToCSV(self.ReportFile)
 272         EdkLogger.quiet("Generating report done!")
 273
 274     def GetRealPathCase(self, path):
 275         TmpPath = path.rstrip(os.sep)
 276         PathParts = TmpPath.split(os.sep)
 277         if len(PathParts) == 0:
 278             return path
 279         if len(PathParts) == 1:
 280             if PathParts[0].strip().endswith(':'):
 281                 return PathParts[0].upper()
 282             # Relative dir, list . current dir
 283             Dirs = os.listdir('.')
 284             for Dir in Dirs:
 285                 if Dir.upper() == PathParts[0].upper():
 286                     return Dir
 287
 288         if PathParts[0].strip().endswith(':'):
 289             PathParts[0] = PathParts[0].upper()
 290         ParentDir = PathParts[0]
 291         RealPath = ParentDir
 292         if PathParts[0] == '':
 293             RealPath = os.sep
 294             ParentDir = os.sep
 295
 296         PathParts.remove(PathParts[0])    # need to remove the parent
 297         for Part in PathParts:
 298             Dirs = os.listdir(ParentDir + os.sep)
 299             for Dir in Dirs:
 300                 if Dir.upper() == Part.upper():
 301                     RealPath += os.sep
 302                     RealPath += Dir
 303                     break
 304             ParentDir += os.sep
 305             ParentDir += Dir
 306
 307         return RealPath
 308
 309     ## ParseOption
 310     #
 311     # Parse options
 312     #
 313     def ParseOption(self):
 314         EdkLogger.quiet("Loading ECC configuration ... done")
 315         (Options, Target) = self.EccOptionParser()
 316
 317         if Options.Workspace:
 318             os.environ["WORKSPACE"] = Options.Workspace
 319
 320         # Check workspace envirnoment
 321         if "WORKSPACE" not in os.environ:
 322             EdkLogger.error("ECC", BuildToolError.ATTRIBUTE_NOT_AVAILABLE, "Environment variable not found",
 323                             ExtraData="WORKSPACE")
 324         else:
 325             EccGlobalData.gWorkspace = os.path.normpath(os.getenv("WORKSPACE"))
 326             if not os.path.exists(EccGlobalData.gWorkspace):
 327                 EdkLogger.error("ECC", BuildToolError.FILE_NOT_FOUND, ExtraData="WORKSPACE = %s" % EccGlobalData.gWorkspace)
 328             os.environ["WORKSPACE"] = EccGlobalData.gWorkspace
 329         # Set log level
 330         self.SetLogLevel(Options)
 331
 332         # Set other options
 333         if Options.ConfigFile != None:
 334             self.ConfigFile = Options.ConfigFile
 335         if Options.OutputFile != None:
 336             self.OutputFile = Options.OutputFile
 337         if Options.ReportFile != None:
 338             self.ReportFile = Options.ReportFile
 339         if Options.ExceptionFile != None:
 340             self.ExceptionFile = Options.ExceptionFile
 341         if Options.Target != None:
 342             if not os.path.isdir(Options.Target):
 343                 EdkLogger.error("ECC", BuildToolError.OPTION_VALUE_INVALID, ExtraData="Target [%s] does NOT exist" % Options.Target)
 344             else:
 345                 EccGlobalData.gTarget = self.GetRealPathCase(os.path.normpath(Options.Target))
 346         else:
 347             EdkLogger.warn("Ecc", EdkLogger.ECC_ERROR, "The target source tree was not specified, using current WORKSPACE instead!")
 348             EccGlobalData.gTarget = os.path.normpath(os.getenv("WORKSPACE"))
 349         if Options.keepdatabase != None:
 350             self.IsInit = False
 351         if Options.metadata != None and Options.sourcecode != None:
 352             EdkLogger.error("ECC", BuildToolError.OPTION_CONFLICT, ExtraData="-m and -s can't be specified at one time")
 353         if Options.metadata != None:
 354             self.ScanSourceCode = False
 355         if Options.sourcecode != None:
 356             self.ScanMetaData = False
 357         if Options.folders != None:
 358             self.OnlyScan = True
 359
 360     ## SetLogLevel
 361     #
 362     # Set current log level of the tool based on args
 363     #
 364     # @param Option:  The option list including log level setting
 365     #
 366     def SetLogLevel(self, Option):
 367         if Option.verbose != None:
 368             EdkLogger.SetLevel(EdkLogger.VERBOSE)
 369         elif Option.quiet != None:
 370             EdkLogger.SetLevel(EdkLogger.QUIET)
 371         elif Option.debug != None:
 372             EdkLogger.SetLevel(Option.debug + 1)
 373         else:
 374             EdkLogger.SetLevel(EdkLogger.INFO)
 375
 376     ## Parse command line options
 377     #
 378     # Using standard Python module optparse to parse command line option of this tool.
 379     #
 380     # @retval Opt   A optparse.Values object containing the parsed options
 381     # @retval Args  Target of build command
 382     #
 383     def EccOptionParser(self):
 384         Parser = OptionParser(description = self.Copyright, version = self.Version, prog = "Ecc.exe", usage = "%prog [options]")
 385         Parser.add_option("-t", "--target sourcepath", action="store", type="string", dest='Target',
 386             help="Check all files under the target workspace.")
 387         Parser.add_option("-c", "--config filename", action="store", type="string", dest="ConfigFile",
 388             help="Specify a configuration file. Defaultly use config.ini under ECC tool directory.")
 389         Parser.add_option("-o", "--outfile filename", action="store", type="string", dest="OutputFile",
 390             help="Specify the name of an output file, if and only if one filename was specified.")
 391         Parser.add_option("-r", "--reportfile filename", action="store", type="string", dest="ReportFile",
 392             help="Specify the name of an report file, if and only if one filename was specified.")
 393         Parser.add_option("-e", "--exceptionfile filename", action="store", type="string", dest="ExceptionFile",
 394             help="Specify the name of an exception file, if and only if one filename was specified.")
 395         Parser.add_option("-m", "--metadata", action="store_true", type=None, help="Only scan meta-data files information if this option is specified.")
 396         Parser.add_option("-s", "--sourcecode", action="store_true", type=None, help="Only scan source code files information if this option is specified.")
 397         Parser.add_option("-k", "--keepdatabase", action="store_true", type=None, help="The existing Ecc database will not be cleaned except report information if this option is specified.")
 398         Parser.add_option("-l", "--log filename", action="store", dest="LogFile", help="""If specified, the tool should emit the changes that
 399                                                                                           were made by the tool after printing the result message.
 400                                                                                           If filename, the emit to the file, otherwise emit to
 401                                                                                           standard output. If no modifications were made, then do not
 402                                                                                           create a log file, or output a log message.""")
 403         Parser.add_option("-q", "--quiet", action="store_true", type=None, help="Disable all messages except FATAL ERRORS.")
 404         Parser.add_option("-v", "--verbose", action="store_true", type=None, help="Turn on verbose output with informational messages printed, "\
 405                                                                                    "including library instances selected, final dependency expression, "\
 406                                                                                    "and warning messages, etc.")
 407         Parser.add_option("-d", "--debug", action="store", type="int", help="Enable debug messages at specified level.")
 408         Parser.add_option("-w", "--workspace", action="store", type="string", dest='Workspace', help="Specify workspace.")
 409         Parser.add_option("-f", "--folders", action="store_true", type=None, help="Only scanning specified folders which are recorded in config.ini file.")
 410
 411         (Opt, Args)=Parser.parse_args()
 412
 413         return (Opt, Args)
 414
 415 ##
 416 #
 417 # This acts like the main() function for the script, unless it is 'import'ed into another
 418 # script.
 419 #
 420 if __name__ == '__main__':
 421     # Initialize log system
 422     EdkLogger.Initialize()
 423     EdkLogger.IsRaiseError = False
 424     EdkLogger.quiet(time.strftime("%H:%M:%S, %b.%d %Y ", time.localtime()) + "[00:00]" + "\n")
 425
 426     StartTime = time.clock()
 427     Ecc = Ecc()
 428     FinishTime = time.clock()
 429
 430     BuildDuration = time.strftime("%M:%S", time.gmtime(int(round(FinishTime - StartTime))))
 431     EdkLogger.quiet("\n%s [%s]" % (time.strftime("%H:%M:%S, %b.%d %Y", time.localtime()), BuildDuration))