BaseTools/Source/Python/Ecc/Ecc.py

   1 ## @file
   2 # This file is used to be the main entrance of ECC tool
   3 #
   4 # Copyright (c) 2009 - 2014, Intel Corporation. All rights reserved.<BR>
   5 # This program and the accompanying materials
   6 # are licensed and made available under the terms and conditions of the BSD License
   7 # which accompanies this distribution.  The full text of the license may be found at
   8 # http://opensource.org/licenses/bsd-license.php
   9 #
  10 # THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS,
  11 # WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED.
  12 #
  13
  14 ##
  15 # Import Modules
  16 #
  17 import Common.LongFilePathOs as os, time, glob, sys
  18 import Common.EdkLogger as EdkLogger
  19 import Database
  20 import EccGlobalData
  21 from MetaDataParser import *
  22 from optparse import OptionParser
  23 from Configuration import Configuration
  24 from Check import Check
  25 import Common.GlobalData as GlobalData
  26
  27 from Common.String import NormPath
  28 from Common.BuildVersion import gBUILD_VERSION
  29 from Common import BuildToolError
  30 from Common.Misc import PathClass
  31 from Common.Misc import DirCache
  32 from MetaFileWorkspace.MetaFileParser import DscParser
  33 from MetaFileWorkspace.MetaFileParser import DecParser
  34 from MetaFileWorkspace.MetaFileParser import InfParser
  35 from MetaFileWorkspace.MetaFileParser import Fdf
  36 from MetaFileWorkspace.MetaFileTable import MetaFileStorage
  37 import c
  38 import re, string
  39 from Exception import *
  40 from Common.LongFilePathSupport import OpenLongFilePath as open
  41
  42 ## Ecc
  43 #
  44 # This class is used to define Ecc main entrance
  45 #
  46 # @param object:          Inherited from object class
  47 #
  48 class Ecc(object):
  49     def __init__(self):
  50         # Version and Copyright
  51         self.VersionNumber = ("0.01" + " " + gBUILD_VERSION)
  52         self.Version = "%prog Version " + self.VersionNumber
  53         self.Copyright = "Copyright (c) 2009 - 2010, Intel Corporation  All rights reserved."
  54
  55         self.InitDefaultConfigIni()
  56         self.OutputFile = 'output.txt'
  57         self.ReportFile = 'Report.csv'
  58         self.ExceptionFile = 'exception.xml'
  59         self.IsInit = True
  60         self.ScanSourceCode = True
  61         self.ScanMetaData = True
  62         self.MetaFile = ''
  63         self.OnlyScan = None
  64
  65         # Parse the options and args
  66         self.ParseOption()
  67
  68         #
  69         # Check EFI_SOURCE (Edk build convention). EDK_SOURCE will always point to ECP
  70         #
  71         WorkspaceDir = os.path.normcase(os.path.normpath(os.environ["WORKSPACE"]))
  72         os.environ["WORKSPACE"] = WorkspaceDir
  73         if "ECP_SOURCE" not in os.environ:
  74             os.environ["ECP_SOURCE"] = os.path.join(WorkspaceDir, GlobalData.gEdkCompatibilityPkg)
  75         if "EFI_SOURCE" not in os.environ:
  76             os.environ["EFI_SOURCE"] = os.environ["ECP_SOURCE"]
  77         if "EDK_SOURCE" not in os.environ:
  78             os.environ["EDK_SOURCE"] = os.environ["ECP_SOURCE"]
  79
  80         #
  81         # Unify case of characters on case-insensitive systems
  82         #
  83         EfiSourceDir = os.path.normcase(os.path.normpath(os.environ["EFI_SOURCE"]))
  84         EdkSourceDir = os.path.normcase(os.path.normpath(os.environ["EDK_SOURCE"]))
  85         EcpSourceDir = os.path.normcase(os.path.normpath(os.environ["ECP_SOURCE"]))
  86
  87         os.environ["EFI_SOURCE"] = EfiSourceDir
  88         os.environ["EDK_SOURCE"] = EdkSourceDir
  89         os.environ["ECP_SOURCE"] = EcpSourceDir
  90
  91         GlobalData.gWorkspace = WorkspaceDir
  92         GlobalData.gEfiSource = EfiSourceDir
  93         GlobalData.gEdkSource = EdkSourceDir
  94         GlobalData.gEcpSource = EcpSourceDir
  95
  96         GlobalData.gGlobalDefines["WORKSPACE"]  = WorkspaceDir
  97         GlobalData.gGlobalDefines["EFI_SOURCE"] = EfiSourceDir
  98         GlobalData.gGlobalDefines["EDK_SOURCE"] = EdkSourceDir
  99         GlobalData.gGlobalDefines["ECP_SOURCE"] = EcpSourceDir
 100
 101
 102         # Generate checkpoints list
 103         EccGlobalData.gConfig = Configuration(self.ConfigFile)
 104
 105         # Generate exception list
 106         EccGlobalData.gException = ExceptionCheck(self.ExceptionFile)
 107
 108         # Init Ecc database
 109         EccGlobalData.gDb = Database.Database(Database.DATABASE_PATH)
 110         EccGlobalData.gDb.InitDatabase(self.IsInit)
 111
 112         #
 113         # Get files real name in workspace dir
 114         #
 115         GlobalData.gAllFiles = DirCache(GlobalData.gWorkspace)
 116
 117         # Build ECC database
 118 #         self.BuildDatabase()
 119         self.DetectOnlyScanDirs()
 120
 121         # Start to check
 122         self.Check()
 123
 124         # Show report
 125         self.GenReport()
 126
 127         # Close Database
 128         EccGlobalData.gDb.Close()
 129
 130     def InitDefaultConfigIni(self):
 131         paths = map(lambda p: os.path.join(p, 'Ecc', 'config.ini'), sys.path)
 132         paths = (os.path.realpath('config.ini'),) + tuple(paths)
 133         for path in paths:
 134             if os.path.exists(path):
 135                 self.ConfigFile = path
 136                 return
 137         self.ConfigFile = 'config.ini'
 138
 139
 140     ## DetectOnlyScan
 141     #
 142     # Detect whether only scanned folders have been enabled
 143     #
 144     def DetectOnlyScanDirs(self):
 145         if self.OnlyScan == True:
 146             OnlyScanDirs = []
 147             # Use regex here if multiple spaces or TAB exists in ScanOnlyDirList in config.ini file
 148             for folder in re.finditer(r'\S+', EccGlobalData.gConfig.ScanOnlyDirList):
 149                 OnlyScanDirs.append(folder.group())
 150             if len(OnlyScanDirs) != 0:
 151                 self.BuildDatabase(OnlyScanDirs)
 152             else:
 153                 EdkLogger.error("ECC", BuildToolError.OPTION_VALUE_INVALID, ExtraData="Use -f option need to fill specific folders in config.ini file")
 154         else:
 155             self.BuildDatabase()
 156
 157
 158     ## BuildDatabase
 159     #
 160     # Build the database for target
 161     #
 162     def BuildDatabase(self, SpeciDirs = None):
 163         # Clean report table
 164         EccGlobalData.gDb.TblReport.Drop()
 165         EccGlobalData.gDb.TblReport.Create()
 166
 167         # Build database
 168         if self.IsInit:
 169             if self.ScanMetaData:
 170                 EdkLogger.quiet("Building database for Meta Data File ...")
 171                 self.BuildMetaDataFileDatabase(SpeciDirs)
 172             if self.ScanSourceCode:
 173                 EdkLogger.quiet("Building database for Meta Data File Done!")
 174                 if SpeciDirs == None:
 175                     c.CollectSourceCodeDataIntoDB(EccGlobalData.gTarget)
 176                 else:
 177                     for specificDir in SpeciDirs:
 178                         c.CollectSourceCodeDataIntoDB(os.path.join(EccGlobalData.gTarget, specificDir))
 179
 180         EccGlobalData.gIdentifierTableList = GetTableList((MODEL_FILE_C, MODEL_FILE_H), 'Identifier', EccGlobalData.gDb)
 181         EccGlobalData.gCFileList = GetFileList(MODEL_FILE_C, EccGlobalData.gDb)
 182         EccGlobalData.gHFileList = GetFileList(MODEL_FILE_H, EccGlobalData.gDb)
 183         EccGlobalData.gUFileList = GetFileList(MODEL_FILE_UNI, EccGlobalData.gDb)
 184
 185     ## BuildMetaDataFileDatabase
 186     #
 187     # Build the database for meta data files
 188     #
 189     def BuildMetaDataFileDatabase(self, SpecificDirs = None):
 190         ScanFolders = []
 191         if SpecificDirs == None:
 192             ScanFolders.append(EccGlobalData.gTarget)
 193         else:
 194             for specificDir in SpecificDirs:
 195                 ScanFolders.append(os.path.join(EccGlobalData.gTarget, specificDir))
 196         EdkLogger.quiet("Building database for meta data files ...")
 197         Op = open(EccGlobalData.gConfig.MetaDataFileCheckPathOfGenerateFileList, 'w+')
 198         #SkipDirs = Read from config file
 199         SkipDirs = EccGlobalData.gConfig.SkipDirList
 200         SkipDirString = string.join(SkipDirs, '|')
 201 #         p = re.compile(r'.*[\\/](?:%s)[\\/]?.*' % SkipDirString)
 202         p = re.compile(r'.*[\\/](?:%s^\S)[\\/]?.*' % SkipDirString)
 203         for scanFolder in ScanFolders:
 204             for Root, Dirs, Files in os.walk(scanFolder):
 205                 if p.match(Root.upper()):
 206                     continue
 207                 for Dir in Dirs:
 208                     Dirname = os.path.join(Root, Dir)
 209                     if os.path.islink(Dirname):
 210                         Dirname = os.path.realpath(Dirname)
 211                         if os.path.isdir(Dirname):
 212                             # symlinks to directories are treated as directories
 213                             Dirs.remove(Dir)
 214                             Dirs.append(Dirname)
 215
 216                 for File in Files:
 217                     if len(File) > 4 and File[-4:].upper() == ".DEC":
 218                         Filename = os.path.normpath(os.path.join(Root, File))
 219                         EdkLogger.quiet("Parsing %s" % Filename)
 220                         Op.write("%s\r" % Filename)
 221                         #Dec(Filename, True, True, EccGlobalData.gWorkspace, EccGlobalData.gDb)
 222                         self.MetaFile = DecParser(Filename, MODEL_FILE_DEC, EccGlobalData.gDb.TblDec)
 223                         self.MetaFile.Start()
 224                         continue
 225                     if len(File) > 4 and File[-4:].upper() == ".DSC":
 226                         Filename = os.path.normpath(os.path.join(Root, File))
 227                         EdkLogger.quiet("Parsing %s" % Filename)
 228                         Op.write("%s\r" % Filename)
 229                         #Dsc(Filename, True, True, EccGlobalData.gWorkspace, EccGlobalData.gDb)
 230                         self.MetaFile = DscParser(PathClass(Filename, Root), MODEL_FILE_DSC, MetaFileStorage(EccGlobalData.gDb.TblDsc.Cur, Filename, MODEL_FILE_DSC, True))
 231                         # alwasy do post-process, in case of macros change
 232                         self.MetaFile.DoPostProcess()
 233                         self.MetaFile.Start()
 234                         self.MetaFile._PostProcess()
 235                         continue
 236                     if len(File) > 4 and File[-4:].upper() == ".INF":
 237                         Filename = os.path.normpath(os.path.join(Root, File))
 238                         EdkLogger.quiet("Parsing %s" % Filename)
 239                         Op.write("%s\r" % Filename)
 240                         #Inf(Filename, True, True, EccGlobalData.gWorkspace, EccGlobalData.gDb)
 241                         self.MetaFile = InfParser(Filename, MODEL_FILE_INF, EccGlobalData.gDb.TblInf)
 242                         self.MetaFile.Start()
 243                         continue
 244                     if len(File) > 4 and File[-4:].upper() == ".FDF":
 245                         Filename = os.path.normpath(os.path.join(Root, File))
 246                         EdkLogger.quiet("Parsing %s" % Filename)
 247                         Op.write("%s\r" % Filename)
 248                         Fdf(Filename, True, EccGlobalData.gWorkspace, EccGlobalData.gDb)
 249                         continue
 250                     if len(File) > 4 and File[-4:].upper() == ".UNI":
 251                         Filename = os.path.normpath(os.path.join(Root, File))
 252                         EdkLogger.quiet("Parsing %s" % Filename)
 253                         Op.write("%s\r" % Filename)
 254                         EccGlobalData.gDb.TblFile.InsertFile(Filename, MODEL_FILE_UNI)
 255                         continue
 256
 257         Op.close()
 258
 259         # Commit to database
 260         EccGlobalData.gDb.Conn.commit()
 261
 262         EdkLogger.quiet("Building database for meta data files done!")
 263
 264     ##
 265     #
 266     # Check each checkpoint
 267     #
 268     def Check(self):
 269         EdkLogger.quiet("Checking ...")
 270         EccCheck = Check()
 271         EccCheck.Check()
 272         EdkLogger.quiet("Checking  done!")
 273
 274     ##
 275     #
 276     # Generate the scan report
 277     #
 278     def GenReport(self):
 279         EdkLogger.quiet("Generating report ...")
 280         EccGlobalData.gDb.TblReport.ToCSV(self.ReportFile)
 281         EdkLogger.quiet("Generating report done!")
 282
 283     def GetRealPathCase(self, path):
 284         TmpPath = path.rstrip(os.sep)
 285         PathParts = TmpPath.split(os.sep)
 286         if len(PathParts) == 0:
 287             return path
 288         if len(PathParts) == 1:
 289             if PathParts[0].strip().endswith(':'):
 290                 return PathParts[0].upper()
 291             # Relative dir, list . current dir
 292             Dirs = os.listdir('.')
 293             for Dir in Dirs:
 294                 if Dir.upper() == PathParts[0].upper():
 295                     return Dir
 296
 297         if PathParts[0].strip().endswith(':'):
 298             PathParts[0] = PathParts[0].upper()
 299         ParentDir = PathParts[0]
 300         RealPath = ParentDir
 301         if PathParts[0] == '':
 302             RealPath = os.sep
 303             ParentDir = os.sep
 304
 305         PathParts.remove(PathParts[0])    # need to remove the parent
 306         for Part in PathParts:
 307             Dirs = os.listdir(ParentDir + os.sep)
 308             for Dir in Dirs:
 309                 if Dir.upper() == Part.upper():
 310                     RealPath += os.sep
 311                     RealPath += Dir
 312                     break
 313             ParentDir += os.sep
 314             ParentDir += Dir
 315
 316         return RealPath
 317
 318     ## ParseOption
 319     #
 320     # Parse options
 321     #
 322     def ParseOption(self):
 323         EdkLogger.quiet("Loading ECC configuration ... done")
 324         (Options, Target) = self.EccOptionParser()
 325
 326         if Options.Workspace:
 327             os.environ["WORKSPACE"] = Options.Workspace
 328
 329         # Check workspace envirnoment
 330         if "WORKSPACE" not in os.environ:
 331             EdkLogger.error("ECC", BuildToolError.ATTRIBUTE_NOT_AVAILABLE, "Environment variable not found",
 332                             ExtraData="WORKSPACE")
 333         else:
 334             EccGlobalData.gWorkspace = os.path.normpath(os.getenv("WORKSPACE"))
 335             if not os.path.exists(EccGlobalData.gWorkspace):
 336                 EdkLogger.error("ECC", BuildToolError.FILE_NOT_FOUND, ExtraData="WORKSPACE = %s" % EccGlobalData.gWorkspace)
 337             os.environ["WORKSPACE"] = EccGlobalData.gWorkspace
 338         # Set log level
 339         self.SetLogLevel(Options)
 340
 341         # Set other options
 342         if Options.ConfigFile != None:
 343             self.ConfigFile = Options.ConfigFile
 344         if Options.OutputFile != None:
 345             self.OutputFile = Options.OutputFile
 346         if Options.ReportFile != None:
 347             self.ReportFile = Options.ReportFile
 348         if Options.ExceptionFile != None:
 349             self.ExceptionFile = Options.ExceptionFile
 350         if Options.Target != None:
 351             if not os.path.isdir(Options.Target):
 352                 EdkLogger.error("ECC", BuildToolError.OPTION_VALUE_INVALID, ExtraData="Target [%s] does NOT exist" % Options.Target)
 353             else:
 354                 EccGlobalData.gTarget = self.GetRealPathCase(os.path.normpath(Options.Target))
 355         else:
 356             EdkLogger.warn("Ecc", EdkLogger.ECC_ERROR, "The target source tree was not specified, using current WORKSPACE instead!")
 357             EccGlobalData.gTarget = os.path.normpath(os.getenv("WORKSPACE"))
 358         if Options.keepdatabase != None:
 359             self.IsInit = False
 360         if Options.metadata != None and Options.sourcecode != None:
 361             EdkLogger.error("ECC", BuildToolError.OPTION_CONFLICT, ExtraData="-m and -s can't be specified at one time")
 362         if Options.metadata != None:
 363             self.ScanSourceCode = False
 364         if Options.sourcecode != None:
 365             self.ScanMetaData = False
 366         if Options.folders != None:
 367             self.OnlyScan = True
 368
 369     ## SetLogLevel
 370     #
 371     # Set current log level of the tool based on args
 372     #
 373     # @param Option:  The option list including log level setting
 374     #
 375     def SetLogLevel(self, Option):
 376         if Option.verbose != None:
 377             EdkLogger.SetLevel(EdkLogger.VERBOSE)
 378         elif Option.quiet != None:
 379             EdkLogger.SetLevel(EdkLogger.QUIET)
 380         elif Option.debug != None:
 381             EdkLogger.SetLevel(Option.debug + 1)
 382         else:
 383             EdkLogger.SetLevel(EdkLogger.INFO)
 384
 385     ## Parse command line options
 386     #
 387     # Using standard Python module optparse to parse command line option of this tool.
 388     #
 389     # @retval Opt   A optparse.Values object containing the parsed options
 390     # @retval Args  Target of build command
 391     #
 392     def EccOptionParser(self):
 393         Parser = OptionParser(description = self.Copyright, version = self.Version, prog = "Ecc.exe", usage = "%prog [options]")
 394         Parser.add_option("-t", "--target sourcepath", action="store", type="string", dest='Target',
 395             help="Check all files under the target workspace.")
 396         Parser.add_option("-c", "--config filename", action="store", type="string", dest="ConfigFile",
 397             help="Specify a configuration file. Defaultly use config.ini under ECC tool directory.")
 398         Parser.add_option("-o", "--outfile filename", action="store", type="string", dest="OutputFile",
 399             help="Specify the name of an output file, if and only if one filename was specified.")
 400         Parser.add_option("-r", "--reportfile filename", action="store", type="string", dest="ReportFile",
 401             help="Specify the name of an report file, if and only if one filename was specified.")
 402         Parser.add_option("-e", "--exceptionfile filename", action="store", type="string", dest="ExceptionFile",
 403             help="Specify the name of an exception file, if and only if one filename was specified.")
 404         Parser.add_option("-m", "--metadata", action="store_true", type=None, help="Only scan meta-data files information if this option is specified.")
 405         Parser.add_option("-s", "--sourcecode", action="store_true", type=None, help="Only scan source code files information if this option is specified.")
 406         Parser.add_option("-k", "--keepdatabase", action="store_true", type=None, help="The existing Ecc database will not be cleaned except report information if this option is specified.")
 407         Parser.add_option("-l", "--log filename", action="store", dest="LogFile", help="""If specified, the tool should emit the changes that
 408                                                                                           were made by the tool after printing the result message.
 409                                                                                           If filename, the emit to the file, otherwise emit to
 410                                                                                           standard output. If no modifications were made, then do not
 411                                                                                           create a log file, or output a log message.""")
 412         Parser.add_option("-q", "--quiet", action="store_true", type=None, help="Disable all messages except FATAL ERRORS.")
 413         Parser.add_option("-v", "--verbose", action="store_true", type=None, help="Turn on verbose output with informational messages printed, "\
 414                                                                                    "including library instances selected, final dependency expression, "\
 415                                                                                    "and warning messages, etc.")
 416         Parser.add_option("-d", "--debug", action="store", type="int", help="Enable debug messages at specified level.")
 417         Parser.add_option("-w", "--workspace", action="store", type="string", dest='Workspace', help="Specify workspace.")
 418         Parser.add_option("-f", "--folders", action="store_true", type=None, help="Only scanning specified folders which are recorded in config.ini file.")
 419
 420         (Opt, Args)=Parser.parse_args()
 421
 422         return (Opt, Args)
 423
 424 ##
 425 #
 426 # This acts like the main() function for the script, unless it is 'import'ed into another
 427 # script.
 428 #
 429 if __name__ == '__main__':
 430     # Initialize log system
 431     EdkLogger.Initialize()
 432     EdkLogger.IsRaiseError = False
 433     EdkLogger.quiet(time.strftime("%H:%M:%S, %b.%d %Y ", time.localtime()) + "[00:00]" + "\n")
 434
 435     StartTime = time.clock()
 436     Ecc = Ecc()
 437     FinishTime = time.clock()
 438
 439     BuildDuration = time.strftime("%M:%S", time.gmtime(int(round(FinishTime - StartTime))))
 440     EdkLogger.quiet("\n%s [%s]" % (time.strftime("%H:%M:%S, %b.%d %Y", time.localtime()), BuildDuration))