BaseTools/Source/Python/Ecc/Ecc.py

   1 ## @file
   2 # This file is used to be the main entrance of ECC tool
   3 #
   4 # Copyright (c) 2009 - 2014, Intel Corporation. All rights reserved.<BR>
   5 # This program and the accompanying materials
   6 # are licensed and made available under the terms and conditions of the BSD License
   7 # which accompanies this distribution.  The full text of the license may be found at
   8 # http://opensource.org/licenses/bsd-license.php
   9 #
  10 # THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS,
  11 # WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED.
  12 #
  13
  14 ##
  15 # Import Modules
  16 #
  17 import Common.LongFilePathOs as os, time, glob, sys
  18 import Common.EdkLogger as EdkLogger
  19 import Database
  20 import EccGlobalData
  21 from MetaDataParser import *
  22 from optparse import OptionParser
  23 from Configuration import Configuration
  24 from Check import Check
  25 import Common.GlobalData as GlobalData
  26
  27 from Common.String import NormPath
  28 from Common.BuildVersion import gBUILD_VERSION
  29 from Common import BuildToolError
  30 from Common.Misc import PathClass
  31 from Common.Misc import DirCache
  32 from MetaFileWorkspace.MetaFileParser import DscParser
  33 from MetaFileWorkspace.MetaFileParser import DecParser
  34 from MetaFileWorkspace.MetaFileParser import InfParser
  35 from MetaFileWorkspace.MetaFileParser import Fdf
  36 from MetaFileWorkspace.MetaFileTable import MetaFileStorage
  37 import c
  38 import re, string
  39 from Exception import *
  40 from Common.LongFilePathSupport import OpenLongFilePath as open
  41 from Common.MultipleWorkspace import MultipleWorkspace as mws
  42
  43 ## Ecc
  44 #
  45 # This class is used to define Ecc main entrance
  46 #
  47 # @param object:          Inherited from object class
  48 #
  49 class Ecc(object):
  50     def __init__(self):
  51         # Version and Copyright
  52         self.VersionNumber = ("0.01" + " " + gBUILD_VERSION)
  53         self.Version = "%prog Version " + self.VersionNumber
  54         self.Copyright = "Copyright (c) 2009 - 2010, Intel Corporation  All rights reserved."
  55
  56         self.InitDefaultConfigIni()
  57         self.OutputFile = 'output.txt'
  58         self.ReportFile = 'Report.csv'
  59         self.ExceptionFile = 'exception.xml'
  60         self.IsInit = True
  61         self.ScanSourceCode = True
  62         self.ScanMetaData = True
  63         self.MetaFile = ''
  64         self.OnlyScan = None
  65
  66         # Parse the options and args
  67         self.ParseOption()
  68
  69         #
  70         # Check EFI_SOURCE (Edk build convention). EDK_SOURCE will always point to ECP
  71         #
  72         WorkspaceDir = os.path.normcase(os.path.normpath(os.environ["WORKSPACE"]))
  73         os.environ["WORKSPACE"] = WorkspaceDir
  74
  75         # set multiple workspace
  76         PackagesPath = os.getenv("PACKAGES_PATH")
  77         mws.setWs(WorkspaceDir, PackagesPath)
  78
  79         if "ECP_SOURCE" not in os.environ:
  80             os.environ["ECP_SOURCE"] = mws.join(WorkspaceDir, GlobalData.gEdkCompatibilityPkg)
  81         if "EFI_SOURCE" not in os.environ:
  82             os.environ["EFI_SOURCE"] = os.environ["ECP_SOURCE"]
  83         if "EDK_SOURCE" not in os.environ:
  84             os.environ["EDK_SOURCE"] = os.environ["ECP_SOURCE"]
  85
  86         #
  87         # Unify case of characters on case-insensitive systems
  88         #
  89         EfiSourceDir = os.path.normcase(os.path.normpath(os.environ["EFI_SOURCE"]))
  90         EdkSourceDir = os.path.normcase(os.path.normpath(os.environ["EDK_SOURCE"]))
  91         EcpSourceDir = os.path.normcase(os.path.normpath(os.environ["ECP_SOURCE"]))
  92
  93         os.environ["EFI_SOURCE"] = EfiSourceDir
  94         os.environ["EDK_SOURCE"] = EdkSourceDir
  95         os.environ["ECP_SOURCE"] = EcpSourceDir
  96
  97         GlobalData.gWorkspace = WorkspaceDir
  98         GlobalData.gEfiSource = EfiSourceDir
  99         GlobalData.gEdkSource = EdkSourceDir
 100         GlobalData.gEcpSource = EcpSourceDir
 101
 102         GlobalData.gGlobalDefines["WORKSPACE"]  = WorkspaceDir
 103         GlobalData.gGlobalDefines["EFI_SOURCE"] = EfiSourceDir
 104         GlobalData.gGlobalDefines["EDK_SOURCE"] = EdkSourceDir
 105         GlobalData.gGlobalDefines["ECP_SOURCE"] = EcpSourceDir
 106
 107
 108         # Generate checkpoints list
 109         EccGlobalData.gConfig = Configuration(self.ConfigFile)
 110
 111         # Generate exception list
 112         EccGlobalData.gException = ExceptionCheck(self.ExceptionFile)
 113
 114         # Init Ecc database
 115         EccGlobalData.gDb = Database.Database(Database.DATABASE_PATH)
 116         EccGlobalData.gDb.InitDatabase(self.IsInit)
 117
 118         #
 119         # Get files real name in workspace dir
 120         #
 121         GlobalData.gAllFiles = DirCache(GlobalData.gWorkspace)
 122
 123         # Build ECC database
 124 #         self.BuildDatabase()
 125         self.DetectOnlyScanDirs()
 126
 127         # Start to check
 128         self.Check()
 129
 130         # Show report
 131         self.GenReport()
 132
 133         # Close Database
 134         EccGlobalData.gDb.Close()
 135
 136     def InitDefaultConfigIni(self):
 137         paths = map(lambda p: os.path.join(p, 'Ecc', 'config.ini'), sys.path)
 138         paths = (os.path.realpath('config.ini'),) + tuple(paths)
 139         for path in paths:
 140             if os.path.exists(path):
 141                 self.ConfigFile = path
 142                 return
 143         self.ConfigFile = 'config.ini'
 144
 145
 146     ## DetectOnlyScan
 147     #
 148     # Detect whether only scanned folders have been enabled
 149     #
 150     def DetectOnlyScanDirs(self):
 151         if self.OnlyScan == True:
 152             OnlyScanDirs = []
 153             # Use regex here if multiple spaces or TAB exists in ScanOnlyDirList in config.ini file
 154             for folder in re.finditer(r'\S+', EccGlobalData.gConfig.ScanOnlyDirList):
 155                 OnlyScanDirs.append(folder.group())
 156             if len(OnlyScanDirs) != 0:
 157                 self.BuildDatabase(OnlyScanDirs)
 158             else:
 159                 EdkLogger.error("ECC", BuildToolError.OPTION_VALUE_INVALID, ExtraData="Use -f option need to fill specific folders in config.ini file")
 160         else:
 161             self.BuildDatabase()
 162
 163
 164     ## BuildDatabase
 165     #
 166     # Build the database for target
 167     #
 168     def BuildDatabase(self, SpeciDirs = None):
 169         # Clean report table
 170         EccGlobalData.gDb.TblReport.Drop()
 171         EccGlobalData.gDb.TblReport.Create()
 172
 173         # Build database
 174         if self.IsInit:
 175             if self.ScanMetaData:
 176                 EdkLogger.quiet("Building database for Meta Data File ...")
 177                 self.BuildMetaDataFileDatabase(SpeciDirs)
 178             if self.ScanSourceCode:
 179                 EdkLogger.quiet("Building database for Meta Data File Done!")
 180                 if SpeciDirs == None:
 181                     c.CollectSourceCodeDataIntoDB(EccGlobalData.gTarget)
 182                 else:
 183                     for specificDir in SpeciDirs:
 184                         c.CollectSourceCodeDataIntoDB(os.path.join(EccGlobalData.gTarget, specificDir))
 185
 186         EccGlobalData.gIdentifierTableList = GetTableList((MODEL_FILE_C, MODEL_FILE_H), 'Identifier', EccGlobalData.gDb)
 187         EccGlobalData.gCFileList = GetFileList(MODEL_FILE_C, EccGlobalData.gDb)
 188         EccGlobalData.gHFileList = GetFileList(MODEL_FILE_H, EccGlobalData.gDb)
 189         EccGlobalData.gUFileList = GetFileList(MODEL_FILE_UNI, EccGlobalData.gDb)
 190
 191     ## BuildMetaDataFileDatabase
 192     #
 193     # Build the database for meta data files
 194     #
 195     def BuildMetaDataFileDatabase(self, SpecificDirs = None):
 196         ScanFolders = []
 197         if SpecificDirs == None:
 198             ScanFolders.append(EccGlobalData.gTarget)
 199         else:
 200             for specificDir in SpecificDirs:
 201                 ScanFolders.append(os.path.join(EccGlobalData.gTarget, specificDir))
 202         EdkLogger.quiet("Building database for meta data files ...")
 203         Op = open(EccGlobalData.gConfig.MetaDataFileCheckPathOfGenerateFileList, 'w+')
 204         #SkipDirs = Read from config file
 205         SkipDirs = EccGlobalData.gConfig.SkipDirList
 206         SkipDirString = string.join(SkipDirs, '|')
 207 #         p = re.compile(r'.*[\\/](?:%s)[\\/]?.*' % SkipDirString)
 208         p = re.compile(r'.*[\\/](?:%s^\S)[\\/]?.*' % SkipDirString)
 209         for scanFolder in ScanFolders:
 210             for Root, Dirs, Files in os.walk(scanFolder):
 211                 if p.match(Root.upper()):
 212                     continue
 213                 for Dir in Dirs:
 214                     Dirname = os.path.join(Root, Dir)
 215                     if os.path.islink(Dirname):
 216                         Dirname = os.path.realpath(Dirname)
 217                         if os.path.isdir(Dirname):
 218                             # symlinks to directories are treated as directories
 219                             Dirs.remove(Dir)
 220                             Dirs.append(Dirname)
 221
 222                 for File in Files:
 223                     if len(File) > 4 and File[-4:].upper() == ".DEC":
 224                         Filename = os.path.normpath(os.path.join(Root, File))
 225                         EdkLogger.quiet("Parsing %s" % Filename)
 226                         Op.write("%s\r" % Filename)
 227                         #Dec(Filename, True, True, EccGlobalData.gWorkspace, EccGlobalData.gDb)
 228                         self.MetaFile = DecParser(Filename, MODEL_FILE_DEC, EccGlobalData.gDb.TblDec)
 229                         self.MetaFile.Start()
 230                         continue
 231                     if len(File) > 4 and File[-4:].upper() == ".DSC":
 232                         Filename = os.path.normpath(os.path.join(Root, File))
 233                         EdkLogger.quiet("Parsing %s" % Filename)
 234                         Op.write("%s\r" % Filename)
 235                         #Dsc(Filename, True, True, EccGlobalData.gWorkspace, EccGlobalData.gDb)
 236                         self.MetaFile = DscParser(PathClass(Filename, Root), MODEL_FILE_DSC, MetaFileStorage(EccGlobalData.gDb.TblDsc.Cur, Filename, MODEL_FILE_DSC, True))
 237                         # alwasy do post-process, in case of macros change
 238                         self.MetaFile.DoPostProcess()
 239                         self.MetaFile.Start()
 240                         self.MetaFile._PostProcess()
 241                         continue
 242                     if len(File) > 4 and File[-4:].upper() == ".INF":
 243                         Filename = os.path.normpath(os.path.join(Root, File))
 244                         EdkLogger.quiet("Parsing %s" % Filename)
 245                         Op.write("%s\r" % Filename)
 246                         #Inf(Filename, True, True, EccGlobalData.gWorkspace, EccGlobalData.gDb)
 247                         self.MetaFile = InfParser(Filename, MODEL_FILE_INF, EccGlobalData.gDb.TblInf)
 248                         self.MetaFile.Start()
 249                         continue
 250                     if len(File) > 4 and File[-4:].upper() == ".FDF":
 251                         Filename = os.path.normpath(os.path.join(Root, File))
 252                         EdkLogger.quiet("Parsing %s" % Filename)
 253                         Op.write("%s\r" % Filename)
 254                         Fdf(Filename, True, EccGlobalData.gWorkspace, EccGlobalData.gDb)
 255                         continue
 256                     if len(File) > 4 and File[-4:].upper() == ".UNI":
 257                         Filename = os.path.normpath(os.path.join(Root, File))
 258                         EdkLogger.quiet("Parsing %s" % Filename)
 259                         Op.write("%s\r" % Filename)
 260                         FileID = EccGlobalData.gDb.TblFile.InsertFile(Filename, MODEL_FILE_UNI)
 261                         EccGlobalData.gDb.TblReport.UpdateBelongsToItemByFile(FileID, File)
 262                         continue
 263
 264         Op.close()
 265
 266         # Commit to database
 267         EccGlobalData.gDb.Conn.commit()
 268
 269         EdkLogger.quiet("Building database for meta data files done!")
 270
 271     ##
 272     #
 273     # Check each checkpoint
 274     #
 275     def Check(self):
 276         EdkLogger.quiet("Checking ...")
 277         EccCheck = Check()
 278         EccCheck.Check()
 279         EdkLogger.quiet("Checking  done!")
 280
 281     ##
 282     #
 283     # Generate the scan report
 284     #
 285     def GenReport(self):
 286         EdkLogger.quiet("Generating report ...")
 287         EccGlobalData.gDb.TblReport.ToCSV(self.ReportFile)
 288         EdkLogger.quiet("Generating report done!")
 289
 290     def GetRealPathCase(self, path):
 291         TmpPath = path.rstrip(os.sep)
 292         PathParts = TmpPath.split(os.sep)
 293         if len(PathParts) == 0:
 294             return path
 295         if len(PathParts) == 1:
 296             if PathParts[0].strip().endswith(':'):
 297                 return PathParts[0].upper()
 298             # Relative dir, list . current dir
 299             Dirs = os.listdir('.')
 300             for Dir in Dirs:
 301                 if Dir.upper() == PathParts[0].upper():
 302                     return Dir
 303
 304         if PathParts[0].strip().endswith(':'):
 305             PathParts[0] = PathParts[0].upper()
 306         ParentDir = PathParts[0]
 307         RealPath = ParentDir
 308         if PathParts[0] == '':
 309             RealPath = os.sep
 310             ParentDir = os.sep
 311
 312         PathParts.remove(PathParts[0])    # need to remove the parent
 313         for Part in PathParts:
 314             Dirs = os.listdir(ParentDir + os.sep)
 315             for Dir in Dirs:
 316                 if Dir.upper() == Part.upper():
 317                     RealPath += os.sep
 318                     RealPath += Dir
 319                     break
 320             ParentDir += os.sep
 321             ParentDir += Dir
 322
 323         return RealPath
 324
 325     ## ParseOption
 326     #
 327     # Parse options
 328     #
 329     def ParseOption(self):
 330         EdkLogger.quiet("Loading ECC configuration ... done")
 331         (Options, Target) = self.EccOptionParser()
 332
 333         if Options.Workspace:
 334             os.environ["WORKSPACE"] = Options.Workspace
 335
 336         # Check workspace envirnoment
 337         if "WORKSPACE" not in os.environ:
 338             EdkLogger.error("ECC", BuildToolError.ATTRIBUTE_NOT_AVAILABLE, "Environment variable not found",
 339                             ExtraData="WORKSPACE")
 340         else:
 341             EccGlobalData.gWorkspace = os.path.normpath(os.getenv("WORKSPACE"))
 342             if not os.path.exists(EccGlobalData.gWorkspace):
 343                 EdkLogger.error("ECC", BuildToolError.FILE_NOT_FOUND, ExtraData="WORKSPACE = %s" % EccGlobalData.gWorkspace)
 344             os.environ["WORKSPACE"] = EccGlobalData.gWorkspace
 345         # Set log level
 346         self.SetLogLevel(Options)
 347
 348         # Set other options
 349         if Options.ConfigFile != None:
 350             self.ConfigFile = Options.ConfigFile
 351         if Options.OutputFile != None:
 352             self.OutputFile = Options.OutputFile
 353         if Options.ReportFile != None:
 354             self.ReportFile = Options.ReportFile
 355         if Options.ExceptionFile != None:
 356             self.ExceptionFile = Options.ExceptionFile
 357         if Options.Target != None:
 358             if not os.path.isdir(Options.Target):
 359                 EdkLogger.error("ECC", BuildToolError.OPTION_VALUE_INVALID, ExtraData="Target [%s] does NOT exist" % Options.Target)
 360             else:
 361                 EccGlobalData.gTarget = self.GetRealPathCase(os.path.normpath(Options.Target))
 362         else:
 363             EdkLogger.warn("Ecc", EdkLogger.ECC_ERROR, "The target source tree was not specified, using current WORKSPACE instead!")
 364             EccGlobalData.gTarget = os.path.normpath(os.getenv("WORKSPACE"))
 365         if Options.keepdatabase != None:
 366             self.IsInit = False
 367         if Options.metadata != None and Options.sourcecode != None:
 368             EdkLogger.error("ECC", BuildToolError.OPTION_CONFLICT, ExtraData="-m and -s can't be specified at one time")
 369         if Options.metadata != None:
 370             self.ScanSourceCode = False
 371         if Options.sourcecode != None:
 372             self.ScanMetaData = False
 373         if Options.folders != None:
 374             self.OnlyScan = True
 375
 376     ## SetLogLevel
 377     #
 378     # Set current log level of the tool based on args
 379     #
 380     # @param Option:  The option list including log level setting
 381     #
 382     def SetLogLevel(self, Option):
 383         if Option.verbose != None:
 384             EdkLogger.SetLevel(EdkLogger.VERBOSE)
 385         elif Option.quiet != None:
 386             EdkLogger.SetLevel(EdkLogger.QUIET)
 387         elif Option.debug != None:
 388             EdkLogger.SetLevel(Option.debug + 1)
 389         else:
 390             EdkLogger.SetLevel(EdkLogger.INFO)
 391
 392     ## Parse command line options
 393     #
 394     # Using standard Python module optparse to parse command line option of this tool.
 395     #
 396     # @retval Opt   A optparse.Values object containing the parsed options
 397     # @retval Args  Target of build command
 398     #
 399     def EccOptionParser(self):
 400         Parser = OptionParser(description = self.Copyright, version = self.Version, prog = "Ecc.exe", usage = "%prog [options]")
 401         Parser.add_option("-t", "--target sourcepath", action="store", type="string", dest='Target',
 402             help="Check all files under the target workspace.")
 403         Parser.add_option("-c", "--config filename", action="store", type="string", dest="ConfigFile",
 404             help="Specify a configuration file. Defaultly use config.ini under ECC tool directory.")
 405         Parser.add_option("-o", "--outfile filename", action="store", type="string", dest="OutputFile",
 406             help="Specify the name of an output file, if and only if one filename was specified.")
 407         Parser.add_option("-r", "--reportfile filename", action="store", type="string", dest="ReportFile",
 408             help="Specify the name of an report file, if and only if one filename was specified.")
 409         Parser.add_option("-e", "--exceptionfile filename", action="store", type="string", dest="ExceptionFile",
 410             help="Specify the name of an exception file, if and only if one filename was specified.")
 411         Parser.add_option("-m", "--metadata", action="store_true", type=None, help="Only scan meta-data files information if this option is specified.")
 412         Parser.add_option("-s", "--sourcecode", action="store_true", type=None, help="Only scan source code files information if this option is specified.")
 413         Parser.add_option("-k", "--keepdatabase", action="store_true", type=None, help="The existing Ecc database will not be cleaned except report information if this option is specified.")
 414         Parser.add_option("-l", "--log filename", action="store", dest="LogFile", help="""If specified, the tool should emit the changes that
 415                                                                                           were made by the tool after printing the result message.
 416                                                                                           If filename, the emit to the file, otherwise emit to
 417                                                                                           standard output. If no modifications were made, then do not
 418                                                                                           create a log file, or output a log message.""")
 419         Parser.add_option("-q", "--quiet", action="store_true", type=None, help="Disable all messages except FATAL ERRORS.")
 420         Parser.add_option("-v", "--verbose", action="store_true", type=None, help="Turn on verbose output with informational messages printed, "\
 421                                                                                    "including library instances selected, final dependency expression, "\
 422                                                                                    "and warning messages, etc.")
 423         Parser.add_option("-d", "--debug", action="store", type="int", help="Enable debug messages at specified level.")
 424         Parser.add_option("-w", "--workspace", action="store", type="string", dest='Workspace', help="Specify workspace.")
 425         Parser.add_option("-f", "--folders", action="store_true", type=None, help="Only scanning specified folders which are recorded in config.ini file.")
 426
 427         (Opt, Args)=Parser.parse_args()
 428
 429         return (Opt, Args)
 430
 431 ##
 432 #
 433 # This acts like the main() function for the script, unless it is 'import'ed into another
 434 # script.
 435 #
 436 if __name__ == '__main__':
 437     # Initialize log system
 438     EdkLogger.Initialize()
 439     EdkLogger.IsRaiseError = False
 440     EdkLogger.quiet(time.strftime("%H:%M:%S, %b.%d %Y ", time.localtime()) + "[00:00]" + "\n")
 441
 442     StartTime = time.clock()
 443     Ecc = Ecc()
 444     FinishTime = time.clock()
 445
 446     BuildDuration = time.strftime("%M:%S", time.gmtime(int(round(FinishTime - StartTime))))
 447     EdkLogger.quiet("\n%s [%s]" % (time.strftime("%H:%M:%S, %b.%d %Y", time.localtime()), BuildDuration))