BaseTools/Source/Python/Ecc/Ecc.py

   1 ## @file
   2 # This file is used to be the main entrance of ECC tool
   3 #
   4 # Copyright (c) 2009 - 2014, Intel Corporation. All rights reserved.<BR>
   5 # This program and the accompanying materials
   6 # are licensed and made available under the terms and conditions of the BSD License
   7 # which accompanies this distribution.  The full text of the license may be found at
   8 # http://opensource.org/licenses/bsd-license.php
   9 #
  10 # THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS,
  11 # WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED.
  12 #
  13
  14 ##
  15 # Import Modules
  16 #
  17 import Common.LongFilePathOs as os, time, glob, sys
  18 import Common.EdkLogger as EdkLogger
  19 import Database
  20 import EccGlobalData
  21 from MetaDataParser import *
  22 from optparse import OptionParser
  23 from Configuration import Configuration
  24 from Check import Check
  25 import Common.GlobalData as GlobalData
  26
  27 from Common.String import NormPath
  28 from Common.BuildVersion import gBUILD_VERSION
  29 from Common import BuildToolError
  30 from Common.Misc import PathClass
  31 from Common.Misc import DirCache
  32 from MetaFileWorkspace.MetaFileParser import DscParser
  33 from MetaFileWorkspace.MetaFileParser import DecParser
  34 from MetaFileWorkspace.MetaFileParser import InfParser
  35 from MetaFileWorkspace.MetaFileParser import Fdf
  36 from MetaFileWorkspace.MetaFileTable import MetaFileStorage
  37 import c
  38 import re, string
  39 from Exception import *
  40 from Common.LongFilePathSupport import OpenLongFilePath as open
  41
  42 ## Ecc
  43 #
  44 # This class is used to define Ecc main entrance
  45 #
  46 # @param object:          Inherited from object class
  47 #
  48 class Ecc(object):
  49     def __init__(self):
  50         # Version and Copyright
  51         self.VersionNumber = ("0.01" + " " + gBUILD_VERSION)
  52         self.Version = "%prog Version " + self.VersionNumber
  53         self.Copyright = "Copyright (c) 2009 - 2010, Intel Corporation  All rights reserved."
  54
  55         self.InitDefaultConfigIni()
  56         self.OutputFile = 'output.txt'
  57         self.ReportFile = 'Report.csv'
  58         self.ExceptionFile = 'exception.xml'
  59         self.IsInit = True
  60         self.ScanSourceCode = True
  61         self.ScanMetaData = True
  62         self.MetaFile = ''
  63         self.OnlyScan = None
  64
  65         # Parse the options and args
  66         self.ParseOption()
  67
  68         #
  69         # Check EFI_SOURCE (Edk build convention). EDK_SOURCE will always point to ECP
  70         #
  71         WorkspaceDir = os.path.normcase(os.path.normpath(os.environ["WORKSPACE"]))
  72         os.environ["WORKSPACE"] = WorkspaceDir
  73         if "ECP_SOURCE" not in os.environ:
  74             os.environ["ECP_SOURCE"] = os.path.join(WorkspaceDir, GlobalData.gEdkCompatibilityPkg)
  75         if "EFI_SOURCE" not in os.environ:
  76             os.environ["EFI_SOURCE"] = os.environ["ECP_SOURCE"]
  77         if "EDK_SOURCE" not in os.environ:
  78             os.environ["EDK_SOURCE"] = os.environ["ECP_SOURCE"]
  79
  80         #
  81         # Unify case of characters on case-insensitive systems
  82         #
  83         EfiSourceDir = os.path.normcase(os.path.normpath(os.environ["EFI_SOURCE"]))
  84         EdkSourceDir = os.path.normcase(os.path.normpath(os.environ["EDK_SOURCE"]))
  85         EcpSourceDir = os.path.normcase(os.path.normpath(os.environ["ECP_SOURCE"]))
  86
  87         os.environ["EFI_SOURCE"] = EfiSourceDir
  88         os.environ["EDK_SOURCE"] = EdkSourceDir
  89         os.environ["ECP_SOURCE"] = EcpSourceDir
  90
  91         GlobalData.gWorkspace = WorkspaceDir
  92         GlobalData.gEfiSource = EfiSourceDir
  93         GlobalData.gEdkSource = EdkSourceDir
  94         GlobalData.gEcpSource = EcpSourceDir
  95
  96         GlobalData.gGlobalDefines["WORKSPACE"]  = WorkspaceDir
  97         GlobalData.gGlobalDefines["EFI_SOURCE"] = EfiSourceDir
  98         GlobalData.gGlobalDefines["EDK_SOURCE"] = EdkSourceDir
  99         GlobalData.gGlobalDefines["ECP_SOURCE"] = EcpSourceDir
 100
 101
 102         # Generate checkpoints list
 103         EccGlobalData.gConfig = Configuration(self.ConfigFile)
 104
 105         # Generate exception list
 106         EccGlobalData.gException = ExceptionCheck(self.ExceptionFile)
 107
 108         # Init Ecc database
 109         EccGlobalData.gDb = Database.Database(Database.DATABASE_PATH)
 110         EccGlobalData.gDb.InitDatabase(self.IsInit)
 111
 112         #
 113         # Get files real name in workspace dir
 114         #
 115         GlobalData.gAllFiles = DirCache(GlobalData.gWorkspace)
 116
 117         # Build ECC database
 118 #         self.BuildDatabase()
 119         self.DetectOnlyScanDirs()
 120
 121         # Start to check
 122         self.Check()
 123
 124         # Show report
 125         self.GenReport()
 126
 127         # Close Database
 128         EccGlobalData.gDb.Close()
 129
 130     def InitDefaultConfigIni(self):
 131         paths = map(lambda p: os.path.join(p, 'Ecc', 'config.ini'), sys.path)
 132         paths = (os.path.realpath('config.ini'),) + tuple(paths)
 133         for path in paths:
 134             if os.path.exists(path):
 135                 self.ConfigFile = path
 136                 return
 137         self.ConfigFile = 'config.ini'
 138
 139
 140     ## DetectOnlyScan
 141     #
 142     # Detect whether only scanned folders have been enabled
 143     #
 144     def DetectOnlyScanDirs(self):
 145         if self.OnlyScan == True:
 146             OnlyScanDirs = []
 147             # Use regex here if multiple spaces or TAB exists in ScanOnlyDirList in config.ini file
 148             for folder in re.finditer(r'\S+', EccGlobalData.gConfig.ScanOnlyDirList):
 149                 OnlyScanDirs.append(folder.group())
 150             if len(OnlyScanDirs) != 0:
 151                 self.BuildDatabase(OnlyScanDirs)
 152             else:
 153                 EdkLogger.error("ECC", BuildToolError.OPTION_VALUE_INVALID, ExtraData="Use -f option need to fill specific folders in config.ini file")
 154         else:
 155             self.BuildDatabase()
 156
 157
 158     ## BuildDatabase
 159     #
 160     # Build the database for target
 161     #
 162     def BuildDatabase(self, SpeciDirs = None):
 163         # Clean report table
 164         EccGlobalData.gDb.TblReport.Drop()
 165         EccGlobalData.gDb.TblReport.Create()
 166
 167         # Build database
 168         if self.IsInit:
 169             if self.ScanMetaData:
 170                 EdkLogger.quiet("Building database for Meta Data File ...")
 171                 self.BuildMetaDataFileDatabase(SpeciDirs)
 172             if self.ScanSourceCode:
 173                 EdkLogger.quiet("Building database for Meta Data File Done!")
 174                 if SpeciDirs == None:
 175                     c.CollectSourceCodeDataIntoDB(EccGlobalData.gTarget)
 176                 else:
 177                     for specificDir in SpeciDirs:
 178                         c.CollectSourceCodeDataIntoDB(os.path.join(EccGlobalData.gTarget, specificDir))
 179
 180         EccGlobalData.gIdentifierTableList = GetTableList((MODEL_FILE_C, MODEL_FILE_H), 'Identifier', EccGlobalData.gDb)
 181         EccGlobalData.gCFileList = GetFileList(MODEL_FILE_C, EccGlobalData.gDb)
 182         EccGlobalData.gHFileList = GetFileList(MODEL_FILE_H, EccGlobalData.gDb)
 183         EccGlobalData.gUFileList = GetFileList(MODEL_FILE_UNI, EccGlobalData.gDb)
 184
 185     ## BuildMetaDataFileDatabase
 186     #
 187     # Build the database for meta data files
 188     #
 189     def BuildMetaDataFileDatabase(self, SpecificDirs = None):
 190         ScanFolders = []
 191         if SpecificDirs == None:
 192             ScanFolders.append(EccGlobalData.gTarget)
 193         else:
 194             for specificDir in SpecificDirs:
 195                 ScanFolders.append(os.path.join(EccGlobalData.gTarget, specificDir))
 196         EdkLogger.quiet("Building database for meta data files ...")
 197         Op = open(EccGlobalData.gConfig.MetaDataFileCheckPathOfGenerateFileList, 'w+')
 198         #SkipDirs = Read from config file
 199         SkipDirs = EccGlobalData.gConfig.SkipDirList
 200         SkipDirString = string.join(SkipDirs, '|')
 201 #         p = re.compile(r'.*[\\/](?:%s)[\\/]?.*' % SkipDirString)
 202         p = re.compile(r'.*[\\/](?:%s^\S)[\\/]?.*' % SkipDirString)
 203         for scanFolder in ScanFolders:
 204             for Root, Dirs, Files in os.walk(scanFolder):
 205                 if p.match(Root.upper()):
 206                     continue
 207                 for Dir in Dirs:
 208                     Dirname = os.path.join(Root, Dir)
 209                     if os.path.islink(Dirname):
 210                         Dirname = os.path.realpath(Dirname)
 211                         if os.path.isdir(Dirname):
 212                             # symlinks to directories are treated as directories
 213                             Dirs.remove(Dir)
 214                             Dirs.append(Dirname)
 215
 216                 for File in Files:
 217                     if len(File) > 4 and File[-4:].upper() == ".DEC":
 218                         Filename = os.path.normpath(os.path.join(Root, File))
 219                         EdkLogger.quiet("Parsing %s" % Filename)
 220                         Op.write("%s\r" % Filename)
 221                         #Dec(Filename, True, True, EccGlobalData.gWorkspace, EccGlobalData.gDb)
 222                         self.MetaFile = DecParser(Filename, MODEL_FILE_DEC, EccGlobalData.gDb.TblDec)
 223                         self.MetaFile.Start()
 224                         continue
 225                     if len(File) > 4 and File[-4:].upper() == ".DSC":
 226                         Filename = os.path.normpath(os.path.join(Root, File))
 227                         EdkLogger.quiet("Parsing %s" % Filename)
 228                         Op.write("%s\r" % Filename)
 229                         #Dsc(Filename, True, True, EccGlobalData.gWorkspace, EccGlobalData.gDb)
 230                         self.MetaFile = DscParser(PathClass(Filename, Root), MODEL_FILE_DSC, MetaFileStorage(EccGlobalData.gDb.TblDsc.Cur, Filename, MODEL_FILE_DSC, True))
 231                         # alwasy do post-process, in case of macros change
 232                         self.MetaFile.DoPostProcess()
 233                         self.MetaFile.Start()
 234                         self.MetaFile._PostProcess()
 235                         continue
 236                     if len(File) > 4 and File[-4:].upper() == ".INF":
 237                         Filename = os.path.normpath(os.path.join(Root, File))
 238                         EdkLogger.quiet("Parsing %s" % Filename)
 239                         Op.write("%s\r" % Filename)
 240                         #Inf(Filename, True, True, EccGlobalData.gWorkspace, EccGlobalData.gDb)
 241                         self.MetaFile = InfParser(Filename, MODEL_FILE_INF, EccGlobalData.gDb.TblInf)
 242                         self.MetaFile.Start()
 243                         continue
 244                     if len(File) > 4 and File[-4:].upper() == ".FDF":
 245                         Filename = os.path.normpath(os.path.join(Root, File))
 246                         EdkLogger.quiet("Parsing %s" % Filename)
 247                         Op.write("%s\r" % Filename)
 248                         Fdf(Filename, True, EccGlobalData.gWorkspace, EccGlobalData.gDb)
 249                         continue
 250                     if len(File) > 4 and File[-4:].upper() == ".UNI":
 251                         Filename = os.path.normpath(os.path.join(Root, File))
 252                         EdkLogger.quiet("Parsing %s" % Filename)
 253                         Op.write("%s\r" % Filename)
 254                         FileID = EccGlobalData.gDb.TblFile.InsertFile(Filename, MODEL_FILE_UNI)
 255                         EccGlobalData.gDb.TblReport.UpdateBelongsToItemByFile(FileID, File)
 256                         continue
 257
 258         Op.close()
 259
 260         # Commit to database
 261         EccGlobalData.gDb.Conn.commit()
 262
 263         EdkLogger.quiet("Building database for meta data files done!")
 264
 265     ##
 266     #
 267     # Check each checkpoint
 268     #
 269     def Check(self):
 270         EdkLogger.quiet("Checking ...")
 271         EccCheck = Check()
 272         EccCheck.Check()
 273         EdkLogger.quiet("Checking  done!")
 274
 275     ##
 276     #
 277     # Generate the scan report
 278     #
 279     def GenReport(self):
 280         EdkLogger.quiet("Generating report ...")
 281         EccGlobalData.gDb.TblReport.ToCSV(self.ReportFile)
 282         EdkLogger.quiet("Generating report done!")
 283
 284     def GetRealPathCase(self, path):
 285         TmpPath = path.rstrip(os.sep)
 286         PathParts = TmpPath.split(os.sep)
 287         if len(PathParts) == 0:
 288             return path
 289         if len(PathParts) == 1:
 290             if PathParts[0].strip().endswith(':'):
 291                 return PathParts[0].upper()
 292             # Relative dir, list . current dir
 293             Dirs = os.listdir('.')
 294             for Dir in Dirs:
 295                 if Dir.upper() == PathParts[0].upper():
 296                     return Dir
 297
 298         if PathParts[0].strip().endswith(':'):
 299             PathParts[0] = PathParts[0].upper()
 300         ParentDir = PathParts[0]
 301         RealPath = ParentDir
 302         if PathParts[0] == '':
 303             RealPath = os.sep
 304             ParentDir = os.sep
 305
 306         PathParts.remove(PathParts[0])    # need to remove the parent
 307         for Part in PathParts:
 308             Dirs = os.listdir(ParentDir + os.sep)
 309             for Dir in Dirs:
 310                 if Dir.upper() == Part.upper():
 311                     RealPath += os.sep
 312                     RealPath += Dir
 313                     break
 314             ParentDir += os.sep
 315             ParentDir += Dir
 316
 317         return RealPath
 318
 319     ## ParseOption
 320     #
 321     # Parse options
 322     #
 323     def ParseOption(self):
 324         EdkLogger.quiet("Loading ECC configuration ... done")
 325         (Options, Target) = self.EccOptionParser()
 326
 327         if Options.Workspace:
 328             os.environ["WORKSPACE"] = Options.Workspace
 329
 330         # Check workspace envirnoment
 331         if "WORKSPACE" not in os.environ:
 332             EdkLogger.error("ECC", BuildToolError.ATTRIBUTE_NOT_AVAILABLE, "Environment variable not found",
 333                             ExtraData="WORKSPACE")
 334         else:
 335             EccGlobalData.gWorkspace = os.path.normpath(os.getenv("WORKSPACE"))
 336             if not os.path.exists(EccGlobalData.gWorkspace):
 337                 EdkLogger.error("ECC", BuildToolError.FILE_NOT_FOUND, ExtraData="WORKSPACE = %s" % EccGlobalData.gWorkspace)
 338             os.environ["WORKSPACE"] = EccGlobalData.gWorkspace
 339         # Set log level
 340         self.SetLogLevel(Options)
 341
 342         # Set other options
 343         if Options.ConfigFile != None:
 344             self.ConfigFile = Options.ConfigFile
 345         if Options.OutputFile != None:
 346             self.OutputFile = Options.OutputFile
 347         if Options.ReportFile != None:
 348             self.ReportFile = Options.ReportFile
 349         if Options.ExceptionFile != None:
 350             self.ExceptionFile = Options.ExceptionFile
 351         if Options.Target != None:
 352             if not os.path.isdir(Options.Target):
 353                 EdkLogger.error("ECC", BuildToolError.OPTION_VALUE_INVALID, ExtraData="Target [%s] does NOT exist" % Options.Target)
 354             else:
 355                 EccGlobalData.gTarget = self.GetRealPathCase(os.path.normpath(Options.Target))
 356         else:
 357             EdkLogger.warn("Ecc", EdkLogger.ECC_ERROR, "The target source tree was not specified, using current WORKSPACE instead!")
 358             EccGlobalData.gTarget = os.path.normpath(os.getenv("WORKSPACE"))
 359         if Options.keepdatabase != None:
 360             self.IsInit = False
 361         if Options.metadata != None and Options.sourcecode != None:
 362             EdkLogger.error("ECC", BuildToolError.OPTION_CONFLICT, ExtraData="-m and -s can't be specified at one time")
 363         if Options.metadata != None:
 364             self.ScanSourceCode = False
 365         if Options.sourcecode != None:
 366             self.ScanMetaData = False
 367         if Options.folders != None:
 368             self.OnlyScan = True
 369
 370     ## SetLogLevel
 371     #
 372     # Set current log level of the tool based on args
 373     #
 374     # @param Option:  The option list including log level setting
 375     #
 376     def SetLogLevel(self, Option):
 377         if Option.verbose != None:
 378             EdkLogger.SetLevel(EdkLogger.VERBOSE)
 379         elif Option.quiet != None:
 380             EdkLogger.SetLevel(EdkLogger.QUIET)
 381         elif Option.debug != None:
 382             EdkLogger.SetLevel(Option.debug + 1)
 383         else:
 384             EdkLogger.SetLevel(EdkLogger.INFO)
 385
 386     ## Parse command line options
 387     #
 388     # Using standard Python module optparse to parse command line option of this tool.
 389     #
 390     # @retval Opt   A optparse.Values object containing the parsed options
 391     # @retval Args  Target of build command
 392     #
 393     def EccOptionParser(self):
 394         Parser = OptionParser(description = self.Copyright, version = self.Version, prog = "Ecc.exe", usage = "%prog [options]")
 395         Parser.add_option("-t", "--target sourcepath", action="store", type="string", dest='Target',
 396             help="Check all files under the target workspace.")
 397         Parser.add_option("-c", "--config filename", action="store", type="string", dest="ConfigFile",
 398             help="Specify a configuration file. Defaultly use config.ini under ECC tool directory.")
 399         Parser.add_option("-o", "--outfile filename", action="store", type="string", dest="OutputFile",
 400             help="Specify the name of an output file, if and only if one filename was specified.")
 401         Parser.add_option("-r", "--reportfile filename", action="store", type="string", dest="ReportFile",
 402             help="Specify the name of an report file, if and only if one filename was specified.")
 403         Parser.add_option("-e", "--exceptionfile filename", action="store", type="string", dest="ExceptionFile",
 404             help="Specify the name of an exception file, if and only if one filename was specified.")
 405         Parser.add_option("-m", "--metadata", action="store_true", type=None, help="Only scan meta-data files information if this option is specified.")
 406         Parser.add_option("-s", "--sourcecode", action="store_true", type=None, help="Only scan source code files information if this option is specified.")
 407         Parser.add_option("-k", "--keepdatabase", action="store_true", type=None, help="The existing Ecc database will not be cleaned except report information if this option is specified.")
 408         Parser.add_option("-l", "--log filename", action="store", dest="LogFile", help="""If specified, the tool should emit the changes that
 409                                                                                           were made by the tool after printing the result message.
 410                                                                                           If filename, the emit to the file, otherwise emit to
 411                                                                                           standard output. If no modifications were made, then do not
 412                                                                                           create a log file, or output a log message.""")
 413         Parser.add_option("-q", "--quiet", action="store_true", type=None, help="Disable all messages except FATAL ERRORS.")
 414         Parser.add_option("-v", "--verbose", action="store_true", type=None, help="Turn on verbose output with informational messages printed, "\
 415                                                                                    "including library instances selected, final dependency expression, "\
 416                                                                                    "and warning messages, etc.")
 417         Parser.add_option("-d", "--debug", action="store", type="int", help="Enable debug messages at specified level.")
 418         Parser.add_option("-w", "--workspace", action="store", type="string", dest='Workspace', help="Specify workspace.")
 419         Parser.add_option("-f", "--folders", action="store_true", type=None, help="Only scanning specified folders which are recorded in config.ini file.")
 420
 421         (Opt, Args)=Parser.parse_args()
 422
 423         return (Opt, Args)
 424
 425 ##
 426 #
 427 # This acts like the main() function for the script, unless it is 'import'ed into another
 428 # script.
 429 #
 430 if __name__ == '__main__':
 431     # Initialize log system
 432     EdkLogger.Initialize()
 433     EdkLogger.IsRaiseError = False
 434     EdkLogger.quiet(time.strftime("%H:%M:%S, %b.%d %Y ", time.localtime()) + "[00:00]" + "\n")
 435
 436     StartTime = time.clock()
 437     Ecc = Ecc()
 438     FinishTime = time.clock()
 439
 440     BuildDuration = time.strftime("%M:%S", time.gmtime(int(round(FinishTime - StartTime))))
 441     EdkLogger.quiet("\n%s [%s]" % (time.strftime("%H:%M:%S, %b.%d %Y", time.localtime()), BuildDuration))