BaseTools/Source/Python/Ecc/Ecc.py

   1 ## @file
   2 # This file is used to be the main entrance of ECC tool
   3 #
   4 # Copyright (c) 2009 - 2010, Intel Corporation. All rights reserved.<BR>
   5 # This program and the accompanying materials
   6 # are licensed and made available under the terms and conditions of the BSD License
   7 # which accompanies this distribution.  The full text of the license may be found at
   8 # http://opensource.org/licenses/bsd-license.php
   9 #
  10 # THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS,
  11 # WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED.
  12 #
  13
  14 ##
  15 # Import Modules
  16 #
  17 import os, time, glob, sys
  18 import Common.EdkLogger as EdkLogger
  19 import Database
  20 import EccGlobalData
  21 from MetaDataParser import *
  22 from optparse import OptionParser
  23 from Configuration import Configuration
  24 from Check import Check
  25 import Common.GlobalData as GlobalData
  26
  27 from Common.String import NormPath
  28 from Common.BuildVersion import gBUILD_VERSION
  29 from Common import BuildToolError
  30 from Common.Misc import PathClass
  31 from Common.Misc import DirCache
  32 from MetaFileWorkspace.MetaFileParser import DscParser
  33 from MetaFileWorkspace.MetaFileParser import DecParser
  34 from MetaFileWorkspace.MetaFileParser import InfParser
  35 from MetaFileWorkspace.MetaFileParser import Fdf
  36 from MetaFileWorkspace.MetaFileTable import MetaFileStorage
  37 import c
  38 import re, string
  39 from Exception import *
  40
  41 ## Ecc
  42 #
  43 # This class is used to define Ecc main entrance
  44 #
  45 # @param object:          Inherited from object class
  46 #
  47 class Ecc(object):
  48     def __init__(self):
  49         # Version and Copyright
  50         self.VersionNumber = ("0.01" + " " + gBUILD_VERSION)
  51         self.Version = "%prog Version " + self.VersionNumber
  52         self.Copyright = "Copyright (c) 2009 - 2010, Intel Corporation  All rights reserved."
  53
  54         self.InitDefaultConfigIni()
  55         self.OutputFile = 'output.txt'
  56         self.ReportFile = 'Report.csv'
  57         self.ExceptionFile = 'exception.xml'
  58         self.IsInit = True
  59         self.ScanSourceCode = True
  60         self.ScanMetaData = True
  61         self.MetaFile = ''
  62
  63         # Parse the options and args
  64         self.ParseOption()
  65
  66         #
  67         # Check EFI_SOURCE (Edk build convention). EDK_SOURCE will always point to ECP
  68         #
  69         WorkspaceDir = os.path.normcase(os.path.normpath(os.environ["WORKSPACE"]))
  70         os.environ["WORKSPACE"] = WorkspaceDir
  71         if "ECP_SOURCE" not in os.environ:
  72             os.environ["ECP_SOURCE"] = os.path.join(WorkspaceDir, GlobalData.gEdkCompatibilityPkg)
  73         if "EFI_SOURCE" not in os.environ:
  74             os.environ["EFI_SOURCE"] = os.environ["ECP_SOURCE"]
  75         if "EDK_SOURCE" not in os.environ:
  76             os.environ["EDK_SOURCE"] = os.environ["ECP_SOURCE"]
  77
  78         #
  79         # Unify case of characters on case-insensitive systems
  80         #
  81         EfiSourceDir = os.path.normcase(os.path.normpath(os.environ["EFI_SOURCE"]))
  82         EdkSourceDir = os.path.normcase(os.path.normpath(os.environ["EDK_SOURCE"]))
  83         EcpSourceDir = os.path.normcase(os.path.normpath(os.environ["ECP_SOURCE"]))
  84
  85         os.environ["EFI_SOURCE"] = EfiSourceDir
  86         os.environ["EDK_SOURCE"] = EdkSourceDir
  87         os.environ["ECP_SOURCE"] = EcpSourceDir
  88
  89         GlobalData.gWorkspace = WorkspaceDir
  90         GlobalData.gEfiSource = EfiSourceDir
  91         GlobalData.gEdkSource = EdkSourceDir
  92         GlobalData.gEcpSource = EcpSourceDir
  93
  94         GlobalData.gGlobalDefines["WORKSPACE"]  = WorkspaceDir
  95         GlobalData.gGlobalDefines["EFI_SOURCE"] = EfiSourceDir
  96         GlobalData.gGlobalDefines["EDK_SOURCE"] = EdkSourceDir
  97         GlobalData.gGlobalDefines["ECP_SOURCE"] = EcpSourceDir
  98
  99
 100         # Generate checkpoints list
 101         EccGlobalData.gConfig = Configuration(self.ConfigFile)
 102
 103         # Generate exception list
 104         EccGlobalData.gException = ExceptionCheck(self.ExceptionFile)
 105
 106         # Init Ecc database
 107         EccGlobalData.gDb = Database.Database(Database.DATABASE_PATH)
 108         EccGlobalData.gDb.InitDatabase(self.IsInit)
 109
 110         #
 111         # Get files real name in workspace dir
 112         #
 113         GlobalData.gAllFiles = DirCache(GlobalData.gWorkspace)
 114
 115         # Build ECC database
 116         self.BuildDatabase()
 117
 118         # Start to check
 119         self.Check()
 120
 121         # Show report
 122         self.GenReport()
 123
 124         # Close Database
 125         EccGlobalData.gDb.Close()
 126
 127     def InitDefaultConfigIni(self):
 128         paths = map(lambda p: os.path.join(p, 'Ecc', 'config.ini'), sys.path)
 129         paths = (os.path.realpath('config.ini'),) + tuple(paths)
 130         for path in paths:
 131             if os.path.exists(path):
 132                 self.ConfigFile = path
 133                 return
 134         self.ConfigFile = 'config.ini'
 135
 136     ## BuildDatabase
 137     #
 138     # Build the database for target
 139     #
 140     def BuildDatabase(self):
 141         # Clean report table
 142         EccGlobalData.gDb.TblReport.Drop()
 143         EccGlobalData.gDb.TblReport.Create()
 144
 145         # Build database
 146         if self.IsInit:
 147             if self.ScanMetaData:
 148                 EdkLogger.quiet("Building database for Meta Data File ...")
 149                 self.BuildMetaDataFileDatabase()
 150             if self.ScanSourceCode:
 151                 EdkLogger.quiet("Building database for Meta Data File Done!")
 152                 c.CollectSourceCodeDataIntoDB(EccGlobalData.gTarget)
 153
 154         EccGlobalData.gIdentifierTableList = GetTableList((MODEL_FILE_C, MODEL_FILE_H), 'Identifier', EccGlobalData.gDb)
 155         EccGlobalData.gCFileList = GetFileList(MODEL_FILE_C, EccGlobalData.gDb)
 156         EccGlobalData.gHFileList = GetFileList(MODEL_FILE_H, EccGlobalData.gDb)
 157
 158     ## BuildMetaDataFileDatabase
 159     #
 160     # Build the database for meta data files
 161     #
 162     def BuildMetaDataFileDatabase(self):
 163         EdkLogger.quiet("Building database for meta data files ...")
 164         Op = open(EccGlobalData.gConfig.MetaDataFileCheckPathOfGenerateFileList, 'w+')
 165         #SkipDirs = Read from config file
 166         SkipDirs = EccGlobalData.gConfig.SkipDirList
 167         SkipDirString = string.join(SkipDirs, '|')
 168         p = re.compile(r'.*[\\/](?:%s)[\\/]?.*' % SkipDirString)
 169         for Root, Dirs, Files in os.walk(EccGlobalData.gTarget):
 170             if p.match(Root.upper()):
 171                 continue
 172             for Dir in Dirs:
 173                 Dirname = os.path.join(Root, Dir)
 174                 if os.path.islink(Dirname):
 175                     Dirname = os.path.realpath(Dirname)
 176                     if os.path.isdir(Dirname):
 177                         # symlinks to directories are treated as directories
 178                         Dirs.remove(Dir)
 179                         Dirs.append(Dirname)
 180
 181             for File in Files:
 182                 if len(File) > 4 and File[-4:].upper() == ".DEC":
 183                     Filename = os.path.normpath(os.path.join(Root, File))
 184                     EdkLogger.quiet("Parsing %s" % Filename)
 185                     Op.write("%s\r" % Filename)
 186                     #Dec(Filename, True, True, EccGlobalData.gWorkspace, EccGlobalData.gDb)
 187                     self.MetaFile = DecParser(Filename, MODEL_FILE_DEC, EccGlobalData.gDb.TblDec)
 188                     self.MetaFile.Start()
 189                     continue
 190                 if len(File) > 4 and File[-4:].upper() == ".DSC":
 191                     Filename = os.path.normpath(os.path.join(Root, File))
 192                     EdkLogger.quiet("Parsing %s" % Filename)
 193                     Op.write("%s\r" % Filename)
 194                     #Dsc(Filename, True, True, EccGlobalData.gWorkspace, EccGlobalData.gDb)
 195                     self.MetaFile = DscParser(PathClass(Filename, Root), MODEL_FILE_DSC, MetaFileStorage(EccGlobalData.gDb.TblDsc.Cur, Filename, MODEL_FILE_DSC, True))
 196                     # alwasy do post-process, in case of macros change
 197                     self.MetaFile.DoPostProcess()
 198                     self.MetaFile.Start()
 199                     self.MetaFile._PostProcess()
 200                     continue
 201                 if len(File) > 4 and File[-4:].upper() == ".INF":
 202                     Filename = os.path.normpath(os.path.join(Root, File))
 203                     EdkLogger.quiet("Parsing %s" % Filename)
 204                     Op.write("%s\r" % Filename)
 205                     #Inf(Filename, True, True, EccGlobalData.gWorkspace, EccGlobalData.gDb)
 206                     self.MetaFile = InfParser(Filename, MODEL_FILE_INF, EccGlobalData.gDb.TblInf)
 207                     self.MetaFile.Start()
 208                     continue
 209                 if len(File) > 4 and File[-4:].upper() == ".FDF":
 210                     Filename = os.path.normpath(os.path.join(Root, File))
 211                     EdkLogger.quiet("Parsing %s" % Filename)
 212                     Op.write("%s\r" % Filename)
 213                     Fdf(Filename, True, EccGlobalData.gWorkspace, EccGlobalData.gDb)
 214                     continue
 215         Op.close()
 216
 217         # Commit to database
 218         EccGlobalData.gDb.Conn.commit()
 219
 220         EdkLogger.quiet("Building database for meta data files done!")
 221
 222     ##
 223     #
 224     # Check each checkpoint
 225     #
 226     def Check(self):
 227         EdkLogger.quiet("Checking ...")
 228         EccCheck = Check()
 229         EccCheck.Check()
 230         EdkLogger.quiet("Checking  done!")
 231
 232     ##
 233     #
 234     # Generate the scan report
 235     #
 236     def GenReport(self):
 237         EdkLogger.quiet("Generating report ...")
 238         EccGlobalData.gDb.TblReport.ToCSV(self.ReportFile)
 239         EdkLogger.quiet("Generating report done!")
 240
 241     def GetRealPathCase(self, path):
 242         TmpPath = path.rstrip(os.sep)
 243         PathParts = TmpPath.split(os.sep)
 244         if len(PathParts) == 0:
 245             return path
 246         if len(PathParts) == 1:
 247             if PathParts[0].strip().endswith(':'):
 248                 return PathParts[0].upper()
 249             # Relative dir, list . current dir
 250             Dirs = os.listdir('.')
 251             for Dir in Dirs:
 252                 if Dir.upper() == PathParts[0].upper():
 253                     return Dir
 254
 255         if PathParts[0].strip().endswith(':'):
 256             PathParts[0] = PathParts[0].upper()
 257         ParentDir = PathParts[0]
 258         RealPath = ParentDir
 259         if PathParts[0] == '':
 260             RealPath = os.sep
 261             ParentDir = os.sep
 262
 263         PathParts.remove(PathParts[0])    # need to remove the parent
 264         for Part in PathParts:
 265             Dirs = os.listdir(ParentDir + os.sep)
 266             for Dir in Dirs:
 267                 if Dir.upper() == Part.upper():
 268                     RealPath += os.sep
 269                     RealPath += Dir
 270                     break
 271             ParentDir += os.sep
 272             ParentDir += Dir
 273
 274         return RealPath
 275
 276     ## ParseOption
 277     #
 278     # Parse options
 279     #
 280     def ParseOption(self):
 281         EdkLogger.quiet("Loading ECC configuration ... done")
 282         (Options, Target) = self.EccOptionParser()
 283
 284         if Options.Workspace:
 285             os.environ["WORKSPACE"] = Options.Workspace
 286
 287         # Check workspace envirnoment
 288         if "WORKSPACE" not in os.environ:
 289             EdkLogger.error("ECC", BuildToolError.ATTRIBUTE_NOT_AVAILABLE, "Environment variable not found",
 290                             ExtraData="WORKSPACE")
 291         else:
 292             EccGlobalData.gWorkspace = os.path.normpath(os.getenv("WORKSPACE"))
 293             if not os.path.exists(EccGlobalData.gWorkspace):
 294                 EdkLogger.error("ECC", BuildToolError.FILE_NOT_FOUND, ExtraData="WORKSPACE = %s" % EccGlobalData.gWorkspace)
 295             os.environ["WORKSPACE"] = EccGlobalData.gWorkspace
 296         # Set log level
 297         self.SetLogLevel(Options)
 298
 299         # Set other options
 300         if Options.ConfigFile != None:
 301             self.ConfigFile = Options.ConfigFile
 302         if Options.OutputFile != None:
 303             self.OutputFile = Options.OutputFile
 304         if Options.ReportFile != None:
 305             self.ReportFile = Options.ReportFile
 306         if Options.ExceptionFile != None:
 307             self.ExceptionFile = Options.ExceptionFile
 308         if Options.Target != None:
 309             if not os.path.isdir(Options.Target):
 310                 EdkLogger.error("ECC", BuildToolError.OPTION_VALUE_INVALID, ExtraData="Target [%s] does NOT exist" % Options.Target)
 311             else:
 312                 EccGlobalData.gTarget = self.GetRealPathCase(os.path.normpath(Options.Target))
 313         else:
 314             EdkLogger.warn("Ecc", EdkLogger.ECC_ERROR, "The target source tree was not specified, using current WORKSPACE instead!")
 315             EccGlobalData.gTarget = os.path.normpath(os.getenv("WORKSPACE"))
 316         if Options.keepdatabase != None:
 317             self.IsInit = False
 318         if Options.metadata != None and Options.sourcecode != None:
 319             EdkLogger.error("ECC", BuildToolError.OPTION_CONFLICT, ExtraData="-m and -s can't be specified at one time")
 320         if Options.metadata != None:
 321             self.ScanSourceCode = False
 322         if Options.sourcecode != None:
 323             self.ScanMetaData = False
 324
 325     ## SetLogLevel
 326     #
 327     # Set current log level of the tool based on args
 328     #
 329     # @param Option:  The option list including log level setting
 330     #
 331     def SetLogLevel(self, Option):
 332         if Option.verbose != None:
 333             EdkLogger.SetLevel(EdkLogger.VERBOSE)
 334         elif Option.quiet != None:
 335             EdkLogger.SetLevel(EdkLogger.QUIET)
 336         elif Option.debug != None:
 337             EdkLogger.SetLevel(Option.debug + 1)
 338         else:
 339             EdkLogger.SetLevel(EdkLogger.INFO)
 340
 341     ## Parse command line options
 342     #
 343     # Using standard Python module optparse to parse command line option of this tool.
 344     #
 345     # @retval Opt   A optparse.Values object containing the parsed options
 346     # @retval Args  Target of build command
 347     #
 348     def EccOptionParser(self):
 349         Parser = OptionParser(description = self.Copyright, version = self.Version, prog = "Ecc.exe", usage = "%prog [options]")
 350         Parser.add_option("-t", "--target sourcepath", action="store", type="string", dest='Target',
 351             help="Check all files under the target workspace.")
 352         Parser.add_option("-c", "--config filename", action="store", type="string", dest="ConfigFile",
 353             help="Specify a configuration file. Defaultly use config.ini under ECC tool directory.")
 354         Parser.add_option("-o", "--outfile filename", action="store", type="string", dest="OutputFile",
 355             help="Specify the name of an output file, if and only if one filename was specified.")
 356         Parser.add_option("-r", "--reportfile filename", action="store", type="string", dest="ReportFile",
 357             help="Specify the name of an report file, if and only if one filename was specified.")
 358         Parser.add_option("-e", "--exceptionfile filename", action="store", type="string", dest="ExceptionFile",
 359             help="Specify the name of an exception file, if and only if one filename was specified.")
 360         Parser.add_option("-m", "--metadata", action="store_true", type=None, help="Only scan meta-data files information if this option is specified.")
 361         Parser.add_option("-s", "--sourcecode", action="store_true", type=None, help="Only scan source code files information if this option is specified.")
 362         Parser.add_option("-k", "--keepdatabase", action="store_true", type=None, help="The existing Ecc database will not be cleaned except report information if this option is specified.")
 363         Parser.add_option("-l", "--log filename", action="store", dest="LogFile", help="""If specified, the tool should emit the changes that
 364                                                                                           were made by the tool after printing the result message.
 365                                                                                           If filename, the emit to the file, otherwise emit to
 366                                                                                           standard output. If no modifications were made, then do not
 367                                                                                           create a log file, or output a log message.""")
 368         Parser.add_option("-q", "--quiet", action="store_true", type=None, help="Disable all messages except FATAL ERRORS.")
 369         Parser.add_option("-v", "--verbose", action="store_true", type=None, help="Turn on verbose output with informational messages printed, "\
 370                                                                                    "including library instances selected, final dependency expression, "\
 371                                                                                    "and warning messages, etc.")
 372         Parser.add_option("-d", "--debug", action="store", type="int", help="Enable debug messages at specified level.")
 373         Parser.add_option("-w", "--workspace", action="store", type="string", dest='Workspace', help="Specify workspace.")
 374
 375         (Opt, Args)=Parser.parse_args()
 376
 377         return (Opt, Args)
 378
 379 ##
 380 #
 381 # This acts like the main() function for the script, unless it is 'import'ed into another
 382 # script.
 383 #
 384 if __name__ == '__main__':
 385     # Initialize log system
 386     EdkLogger.Initialize()
 387     EdkLogger.IsRaiseError = False
 388     EdkLogger.quiet(time.strftime("%H:%M:%S, %b.%d %Y ", time.localtime()) + "[00:00]" + "\n")
 389
 390     StartTime = time.clock()
 391     Ecc = Ecc()
 392     FinishTime = time.clock()
 393
 394     BuildDuration = time.strftime("%M:%S", time.gmtime(int(round(FinishTime - StartTime))))
 395     EdkLogger.quiet("\n%s [%s]" % (time.strftime("%H:%M:%S, %b.%d %Y", time.localtime()), BuildDuration))