BaseTools/Source/Python/Ecc/Ecc.py

   1 ## @file
   2 # This file is used to be the main entrance of ECC tool
   3 #
   4 # Copyright (c) 2009 - 2014, Intel Corporation. All rights reserved.<BR>
   5 # This program and the accompanying materials
   6 # are licensed and made available under the terms and conditions of the BSD License
   7 # which accompanies this distribution.  The full text of the license may be found at
   8 # http://opensource.org/licenses/bsd-license.php
   9 #
  10 # THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS,
  11 # WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED.
  12 #
  13
  14 ##
  15 # Import Modules
  16 #
  17 import Common.LongFilePathOs as os, time, glob, sys
  18 import Common.EdkLogger as EdkLogger
  19 import Database
  20 import EccGlobalData
  21 from MetaDataParser import *
  22 from optparse import OptionParser
  23 from Configuration import Configuration
  24 from Check import Check
  25 import Common.GlobalData as GlobalData
  26
  27 from Common.String import NormPath
  28 from Common.BuildVersion import gBUILD_VERSION
  29 from Common import BuildToolError
  30 from Common.Misc import PathClass
  31 from Common.Misc import DirCache
  32 from MetaFileWorkspace.MetaFileParser import DscParser
  33 from MetaFileWorkspace.MetaFileParser import DecParser
  34 from MetaFileWorkspace.MetaFileParser import InfParser
  35 from MetaFileWorkspace.MetaFileParser import Fdf
  36 from MetaFileWorkspace.MetaFileTable import MetaFileStorage
  37 import c
  38 import re, string
  39 from Exception import *
  40 from Common.LongFilePathSupport import OpenLongFilePath as open
  41
  42 ## Ecc
  43 #
  44 # This class is used to define Ecc main entrance
  45 #
  46 # @param object:          Inherited from object class
  47 #
  48 class Ecc(object):
  49     def __init__(self):
  50         # Version and Copyright
  51         self.VersionNumber = ("0.01" + " " + gBUILD_VERSION)
  52         self.Version = "%prog Version " + self.VersionNumber
  53         self.Copyright = "Copyright (c) 2009 - 2010, Intel Corporation  All rights reserved."
  54
  55         self.InitDefaultConfigIni()
  56         self.OutputFile = 'output.txt'
  57         self.ReportFile = 'Report.csv'
  58         self.ExceptionFile = 'exception.xml'
  59         self.IsInit = True
  60         self.ScanSourceCode = True
  61         self.ScanMetaData = True
  62         self.MetaFile = ''
  63         self.OnlyScan = None
  64
  65         # Parse the options and args
  66         self.ParseOption()
  67
  68         #
  69         # Check EFI_SOURCE (Edk build convention). EDK_SOURCE will always point to ECP
  70         #
  71         WorkspaceDir = os.path.normcase(os.path.normpath(os.environ["WORKSPACE"]))
  72         os.environ["WORKSPACE"] = WorkspaceDir
  73         if "ECP_SOURCE" not in os.environ:
  74             os.environ["ECP_SOURCE"] = os.path.join(WorkspaceDir, GlobalData.gEdkCompatibilityPkg)
  75         if "EFI_SOURCE" not in os.environ:
  76             os.environ["EFI_SOURCE"] = os.environ["ECP_SOURCE"]
  77         if "EDK_SOURCE" not in os.environ:
  78             os.environ["EDK_SOURCE"] = os.environ["ECP_SOURCE"]
  79
  80         #
  81         # Unify case of characters on case-insensitive systems
  82         #
  83         EfiSourceDir = os.path.normcase(os.path.normpath(os.environ["EFI_SOURCE"]))
  84         EdkSourceDir = os.path.normcase(os.path.normpath(os.environ["EDK_SOURCE"]))
  85         EcpSourceDir = os.path.normcase(os.path.normpath(os.environ["ECP_SOURCE"]))
  86
  87         os.environ["EFI_SOURCE"] = EfiSourceDir
  88         os.environ["EDK_SOURCE"] = EdkSourceDir
  89         os.environ["ECP_SOURCE"] = EcpSourceDir
  90
  91         GlobalData.gWorkspace = WorkspaceDir
  92         GlobalData.gEfiSource = EfiSourceDir
  93         GlobalData.gEdkSource = EdkSourceDir
  94         GlobalData.gEcpSource = EcpSourceDir
  95
  96         GlobalData.gGlobalDefines["WORKSPACE"]  = WorkspaceDir
  97         GlobalData.gGlobalDefines["EFI_SOURCE"] = EfiSourceDir
  98         GlobalData.gGlobalDefines["EDK_SOURCE"] = EdkSourceDir
  99         GlobalData.gGlobalDefines["ECP_SOURCE"] = EcpSourceDir
 100
 101
 102         # Generate checkpoints list
 103         EccGlobalData.gConfig = Configuration(self.ConfigFile)
 104
 105         # Generate exception list
 106         EccGlobalData.gException = ExceptionCheck(self.ExceptionFile)
 107
 108         # Init Ecc database
 109         EccGlobalData.gDb = Database.Database(Database.DATABASE_PATH)
 110         EccGlobalData.gDb.InitDatabase(self.IsInit)
 111
 112         #
 113         # Get files real name in workspace dir
 114         #
 115         GlobalData.gAllFiles = DirCache(GlobalData.gWorkspace)
 116
 117         # Build ECC database
 118 #         self.BuildDatabase()
 119         self.DetectOnlyScanDirs()
 120
 121         # Start to check
 122         self.Check()
 123
 124         # Show report
 125         self.GenReport()
 126
 127         # Close Database
 128         EccGlobalData.gDb.Close()
 129
 130     def InitDefaultConfigIni(self):
 131         paths = map(lambda p: os.path.join(p, 'Ecc', 'config.ini'), sys.path)
 132         paths = (os.path.realpath('config.ini'),) + tuple(paths)
 133         for path in paths:
 134             if os.path.exists(path):
 135                 self.ConfigFile = path
 136                 return
 137         self.ConfigFile = 'config.ini'
 138
 139
 140     ## DetectOnlyScan
 141     #
 142     # Detect whether only scanned folders have been enabled
 143     #
 144     def DetectOnlyScanDirs(self):
 145         if self.OnlyScan == True:
 146             OnlyScanDirs = []
 147             # Use regex here if multiple spaces or TAB exists in ScanOnlyDirList in config.ini file
 148             for folder in re.finditer(r'\S+', EccGlobalData.gConfig.ScanOnlyDirList):
 149                 OnlyScanDirs.append(folder.group())
 150             if len(OnlyScanDirs) != 0:
 151                 self.BuildDatabase(OnlyScanDirs)
 152             else:
 153                 EdkLogger.error("ECC", BuildToolError.OPTION_VALUE_INVALID, ExtraData="Use -f option need to fill specific folders in config.ini file")
 154         else:
 155             self.BuildDatabase()
 156
 157
 158     ## BuildDatabase
 159     #
 160     # Build the database for target
 161     #
 162     def BuildDatabase(self, SpeciDirs = None):
 163         # Clean report table
 164         EccGlobalData.gDb.TblReport.Drop()
 165         EccGlobalData.gDb.TblReport.Create()
 166
 167         # Build database
 168         if self.IsInit:
 169             if self.ScanMetaData:
 170                 EdkLogger.quiet("Building database for Meta Data File ...")
 171                 self.BuildMetaDataFileDatabase(SpeciDirs)
 172             if self.ScanSourceCode:
 173                 EdkLogger.quiet("Building database for Meta Data File Done!")
 174                 if SpeciDirs == None:
 175                     c.CollectSourceCodeDataIntoDB(EccGlobalData.gTarget)
 176                 else:
 177                     for specificDir in SpeciDirs:
 178                         c.CollectSourceCodeDataIntoDB(os.path.join(EccGlobalData.gTarget, specificDir))
 179
 180         EccGlobalData.gIdentifierTableList = GetTableList((MODEL_FILE_C, MODEL_FILE_H), 'Identifier', EccGlobalData.gDb)
 181         EccGlobalData.gCFileList = GetFileList(MODEL_FILE_C, EccGlobalData.gDb)
 182         EccGlobalData.gHFileList = GetFileList(MODEL_FILE_H, EccGlobalData.gDb)
 183
 184     ## BuildMetaDataFileDatabase
 185     #
 186     # Build the database for meta data files
 187     #
 188     def BuildMetaDataFileDatabase(self, SpecificDirs = None):
 189         ScanFolders = []
 190         if SpecificDirs == None:
 191             ScanFolders.append(EccGlobalData.gTarget)
 192         else:
 193             for specificDir in SpecificDirs:
 194                 ScanFolders.append(os.path.join(EccGlobalData.gTarget, specificDir))
 195         EdkLogger.quiet("Building database for meta data files ...")
 196         Op = open(EccGlobalData.gConfig.MetaDataFileCheckPathOfGenerateFileList, 'w+')
 197         #SkipDirs = Read from config file
 198         SkipDirs = EccGlobalData.gConfig.SkipDirList
 199         SkipDirString = string.join(SkipDirs, '|')
 200 #         p = re.compile(r'.*[\\/](?:%s)[\\/]?.*' % SkipDirString)
 201         p = re.compile(r'.*[\\/](?:%s^\S)[\\/]?.*' % SkipDirString)
 202         for scanFolder in ScanFolders:
 203             for Root, Dirs, Files in os.walk(scanFolder):
 204                 if p.match(Root.upper()):
 205                     continue
 206                 for Dir in Dirs:
 207                     Dirname = os.path.join(Root, Dir)
 208                     if os.path.islink(Dirname):
 209                         Dirname = os.path.realpath(Dirname)
 210                         if os.path.isdir(Dirname):
 211                             # symlinks to directories are treated as directories
 212                             Dirs.remove(Dir)
 213                             Dirs.append(Dirname)
 214
 215                 for File in Files:
 216                     if len(File) > 4 and File[-4:].upper() == ".DEC":
 217                         Filename = os.path.normpath(os.path.join(Root, File))
 218                         EdkLogger.quiet("Parsing %s" % Filename)
 219                         Op.write("%s\r" % Filename)
 220                         #Dec(Filename, True, True, EccGlobalData.gWorkspace, EccGlobalData.gDb)
 221                         self.MetaFile = DecParser(Filename, MODEL_FILE_DEC, EccGlobalData.gDb.TblDec)
 222                         self.MetaFile.Start()
 223                         continue
 224                     if len(File) > 4 and File[-4:].upper() == ".DSC":
 225                         Filename = os.path.normpath(os.path.join(Root, File))
 226                         EdkLogger.quiet("Parsing %s" % Filename)
 227                         Op.write("%s\r" % Filename)
 228                         #Dsc(Filename, True, True, EccGlobalData.gWorkspace, EccGlobalData.gDb)
 229                         self.MetaFile = DscParser(PathClass(Filename, Root), MODEL_FILE_DSC, MetaFileStorage(EccGlobalData.gDb.TblDsc.Cur, Filename, MODEL_FILE_DSC, True))
 230                         # alwasy do post-process, in case of macros change
 231                         self.MetaFile.DoPostProcess()
 232                         self.MetaFile.Start()
 233                         self.MetaFile._PostProcess()
 234                         continue
 235                     if len(File) > 4 and File[-4:].upper() == ".INF":
 236                         Filename = os.path.normpath(os.path.join(Root, File))
 237                         EdkLogger.quiet("Parsing %s" % Filename)
 238                         Op.write("%s\r" % Filename)
 239                         #Inf(Filename, True, True, EccGlobalData.gWorkspace, EccGlobalData.gDb)
 240                         self.MetaFile = InfParser(Filename, MODEL_FILE_INF, EccGlobalData.gDb.TblInf)
 241                         self.MetaFile.Start()
 242                         continue
 243                     if len(File) > 4 and File[-4:].upper() == ".FDF":
 244                         Filename = os.path.normpath(os.path.join(Root, File))
 245                         EdkLogger.quiet("Parsing %s" % Filename)
 246                         Op.write("%s\r" % Filename)
 247                         Fdf(Filename, True, EccGlobalData.gWorkspace, EccGlobalData.gDb)
 248                         continue
 249         Op.close()
 250
 251         # Commit to database
 252         EccGlobalData.gDb.Conn.commit()
 253
 254         EdkLogger.quiet("Building database for meta data files done!")
 255
 256     ##
 257     #
 258     # Check each checkpoint
 259     #
 260     def Check(self):
 261         EdkLogger.quiet("Checking ...")
 262         EccCheck = Check()
 263         EccCheck.Check()
 264         EdkLogger.quiet("Checking  done!")
 265
 266     ##
 267     #
 268     # Generate the scan report
 269     #
 270     def GenReport(self):
 271         EdkLogger.quiet("Generating report ...")
 272         EccGlobalData.gDb.TblReport.ToCSV(self.ReportFile)
 273         EdkLogger.quiet("Generating report done!")
 274
 275     def GetRealPathCase(self, path):
 276         TmpPath = path.rstrip(os.sep)
 277         PathParts = TmpPath.split(os.sep)
 278         if len(PathParts) == 0:
 279             return path
 280         if len(PathParts) == 1:
 281             if PathParts[0].strip().endswith(':'):
 282                 return PathParts[0].upper()
 283             # Relative dir, list . current dir
 284             Dirs = os.listdir('.')
 285             for Dir in Dirs:
 286                 if Dir.upper() == PathParts[0].upper():
 287                     return Dir
 288
 289         if PathParts[0].strip().endswith(':'):
 290             PathParts[0] = PathParts[0].upper()
 291         ParentDir = PathParts[0]
 292         RealPath = ParentDir
 293         if PathParts[0] == '':
 294             RealPath = os.sep
 295             ParentDir = os.sep
 296
 297         PathParts.remove(PathParts[0])    # need to remove the parent
 298         for Part in PathParts:
 299             Dirs = os.listdir(ParentDir + os.sep)
 300             for Dir in Dirs:
 301                 if Dir.upper() == Part.upper():
 302                     RealPath += os.sep
 303                     RealPath += Dir
 304                     break
 305             ParentDir += os.sep
 306             ParentDir += Dir
 307
 308         return RealPath
 309
 310     ## ParseOption
 311     #
 312     # Parse options
 313     #
 314     def ParseOption(self):
 315         EdkLogger.quiet("Loading ECC configuration ... done")
 316         (Options, Target) = self.EccOptionParser()
 317
 318         if Options.Workspace:
 319             os.environ["WORKSPACE"] = Options.Workspace
 320
 321         # Check workspace envirnoment
 322         if "WORKSPACE" not in os.environ:
 323             EdkLogger.error("ECC", BuildToolError.ATTRIBUTE_NOT_AVAILABLE, "Environment variable not found",
 324                             ExtraData="WORKSPACE")
 325         else:
 326             EccGlobalData.gWorkspace = os.path.normpath(os.getenv("WORKSPACE"))
 327             if not os.path.exists(EccGlobalData.gWorkspace):
 328                 EdkLogger.error("ECC", BuildToolError.FILE_NOT_FOUND, ExtraData="WORKSPACE = %s" % EccGlobalData.gWorkspace)
 329             os.environ["WORKSPACE"] = EccGlobalData.gWorkspace
 330         # Set log level
 331         self.SetLogLevel(Options)
 332
 333         # Set other options
 334         if Options.ConfigFile != None:
 335             self.ConfigFile = Options.ConfigFile
 336         if Options.OutputFile != None:
 337             self.OutputFile = Options.OutputFile
 338         if Options.ReportFile != None:
 339             self.ReportFile = Options.ReportFile
 340         if Options.ExceptionFile != None:
 341             self.ExceptionFile = Options.ExceptionFile
 342         if Options.Target != None:
 343             if not os.path.isdir(Options.Target):
 344                 EdkLogger.error("ECC", BuildToolError.OPTION_VALUE_INVALID, ExtraData="Target [%s] does NOT exist" % Options.Target)
 345             else:
 346                 EccGlobalData.gTarget = self.GetRealPathCase(os.path.normpath(Options.Target))
 347         else:
 348             EdkLogger.warn("Ecc", EdkLogger.ECC_ERROR, "The target source tree was not specified, using current WORKSPACE instead!")
 349             EccGlobalData.gTarget = os.path.normpath(os.getenv("WORKSPACE"))
 350         if Options.keepdatabase != None:
 351             self.IsInit = False
 352         if Options.metadata != None and Options.sourcecode != None:
 353             EdkLogger.error("ECC", BuildToolError.OPTION_CONFLICT, ExtraData="-m and -s can't be specified at one time")
 354         if Options.metadata != None:
 355             self.ScanSourceCode = False
 356         if Options.sourcecode != None:
 357             self.ScanMetaData = False
 358         if Options.folders != None:
 359             self.OnlyScan = True
 360
 361     ## SetLogLevel
 362     #
 363     # Set current log level of the tool based on args
 364     #
 365     # @param Option:  The option list including log level setting
 366     #
 367     def SetLogLevel(self, Option):
 368         if Option.verbose != None:
 369             EdkLogger.SetLevel(EdkLogger.VERBOSE)
 370         elif Option.quiet != None:
 371             EdkLogger.SetLevel(EdkLogger.QUIET)
 372         elif Option.debug != None:
 373             EdkLogger.SetLevel(Option.debug + 1)
 374         else:
 375             EdkLogger.SetLevel(EdkLogger.INFO)
 376
 377     ## Parse command line options
 378     #
 379     # Using standard Python module optparse to parse command line option of this tool.
 380     #
 381     # @retval Opt   A optparse.Values object containing the parsed options
 382     # @retval Args  Target of build command
 383     #
 384     def EccOptionParser(self):
 385         Parser = OptionParser(description = self.Copyright, version = self.Version, prog = "Ecc.exe", usage = "%prog [options]")
 386         Parser.add_option("-t", "--target sourcepath", action="store", type="string", dest='Target',
 387             help="Check all files under the target workspace.")
 388         Parser.add_option("-c", "--config filename", action="store", type="string", dest="ConfigFile",
 389             help="Specify a configuration file. Defaultly use config.ini under ECC tool directory.")
 390         Parser.add_option("-o", "--outfile filename", action="store", type="string", dest="OutputFile",
 391             help="Specify the name of an output file, if and only if one filename was specified.")
 392         Parser.add_option("-r", "--reportfile filename", action="store", type="string", dest="ReportFile",
 393             help="Specify the name of an report file, if and only if one filename was specified.")
 394         Parser.add_option("-e", "--exceptionfile filename", action="store", type="string", dest="ExceptionFile",
 395             help="Specify the name of an exception file, if and only if one filename was specified.")
 396         Parser.add_option("-m", "--metadata", action="store_true", type=None, help="Only scan meta-data files information if this option is specified.")
 397         Parser.add_option("-s", "--sourcecode", action="store_true", type=None, help="Only scan source code files information if this option is specified.")
 398         Parser.add_option("-k", "--keepdatabase", action="store_true", type=None, help="The existing Ecc database will not be cleaned except report information if this option is specified.")
 399         Parser.add_option("-l", "--log filename", action="store", dest="LogFile", help="""If specified, the tool should emit the changes that
 400                                                                                           were made by the tool after printing the result message.
 401                                                                                           If filename, the emit to the file, otherwise emit to
 402                                                                                           standard output. If no modifications were made, then do not
 403                                                                                           create a log file, or output a log message.""")
 404         Parser.add_option("-q", "--quiet", action="store_true", type=None, help="Disable all messages except FATAL ERRORS.")
 405         Parser.add_option("-v", "--verbose", action="store_true", type=None, help="Turn on verbose output with informational messages printed, "\
 406                                                                                    "including library instances selected, final dependency expression, "\
 407                                                                                    "and warning messages, etc.")
 408         Parser.add_option("-d", "--debug", action="store", type="int", help="Enable debug messages at specified level.")
 409         Parser.add_option("-w", "--workspace", action="store", type="string", dest='Workspace', help="Specify workspace.")
 410         Parser.add_option("-f", "--folders", action="store_true", type=None, help="Only scanning specified folders which are recorded in config.ini file.")
 411
 412         (Opt, Args)=Parser.parse_args()
 413
 414         return (Opt, Args)
 415
 416 ##
 417 #
 418 # This acts like the main() function for the script, unless it is 'import'ed into another
 419 # script.
 420 #
 421 if __name__ == '__main__':
 422     # Initialize log system
 423     EdkLogger.Initialize()
 424     EdkLogger.IsRaiseError = False
 425     EdkLogger.quiet(time.strftime("%H:%M:%S, %b.%d %Y ", time.localtime()) + "[00:00]" + "\n")
 426
 427     StartTime = time.clock()
 428     Ecc = Ecc()
 429     FinishTime = time.clock()
 430
 431     BuildDuration = time.strftime("%M:%S", time.gmtime(int(round(FinishTime - StartTime))))
 432     EdkLogger.quiet("\n%s [%s]" % (time.strftime("%H:%M:%S, %b.%d %Y", time.localtime()), BuildDuration))