BaseTools/Source/Python/Ecc/Ecc.py

   1 ## @file
   2 # This file is used to be the main entrance of ECC tool
   3 #
   4 # Copyright (c) 2009 - 2018, Intel Corporation. All rights reserved.<BR>
   5 # This program and the accompanying materials
   6 # are licensed and made available under the terms and conditions of the BSD License
   7 # which accompanies this distribution.  The full text of the license may be found at
   8 # http://opensource.org/licenses/bsd-license.php
   9 #
  10 # THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS,
  11 # WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED.
  12 #
  13
  14 ##
  15 # Import Modules
  16 #
  17 from __future__ import absolute_import
  18 import Common.LongFilePathOs as os, time, glob, sys
  19 import Common.EdkLogger as EdkLogger
  20 from . import Database
  21 from . import EccGlobalData
  22 from .MetaDataParser import *
  23 from optparse import OptionParser
  24 from .Configuration import Configuration
  25 from .Check import Check
  26 import Common.GlobalData as GlobalData
  27
  28 from Common.StringUtils import NormPath
  29 from Common.BuildVersion import gBUILD_VERSION
  30 from Common import BuildToolError
  31 from Common.Misc import PathClass
  32 from Common.Misc import DirCache
  33 from .MetaFileWorkspace.MetaFileParser import DscParser
  34 from .MetaFileWorkspace.MetaFileParser import DecParser
  35 from .MetaFileWorkspace.MetaFileParser import InfParser
  36 from .MetaFileWorkspace.MetaFileParser import Fdf
  37 from .MetaFileWorkspace.MetaFileTable import MetaFileStorage
  38 from . import c
  39 import re, string
  40 from .Exception import *
  41 from Common.LongFilePathSupport import OpenLongFilePath as open
  42 from Common.MultipleWorkspace import MultipleWorkspace as mws
  43
  44 ## Ecc
  45 #
  46 # This class is used to define Ecc main entrance
  47 #
  48 # @param object:          Inherited from object class
  49 #
  50 class Ecc(object):
  51     def __init__(self):
  52         # Version and Copyright
  53         self.VersionNumber = ("1.0" + " Build " + gBUILD_VERSION)
  54         self.Version = "%prog Version " + self.VersionNumber
  55         self.Copyright = "Copyright (c) 2009 - 2018, Intel Corporation  All rights reserved."
  56
  57         self.InitDefaultConfigIni()
  58         self.OutputFile = 'output.txt'
  59         self.ReportFile = 'Report.csv'
  60         self.ExceptionFile = 'exception.xml'
  61         self.IsInit = True
  62         self.ScanSourceCode = True
  63         self.ScanMetaData = True
  64         self.MetaFile = ''
  65         self.OnlyScan = None
  66
  67         # Parse the options and args
  68         self.ParseOption()
  69         EdkLogger.info(time.strftime("%H:%M:%S, %b.%d %Y ", time.localtime()) + "[00:00]" + "\n")
  70
  71         #
  72         # Check EFI_SOURCE (Edk build convention). EDK_SOURCE will always point to ECP
  73         #
  74         WorkspaceDir = os.path.normcase(os.path.normpath(os.environ["WORKSPACE"]))
  75         os.environ["WORKSPACE"] = WorkspaceDir
  76
  77         # set multiple workspace
  78         PackagesPath = os.getenv("PACKAGES_PATH")
  79         mws.setWs(WorkspaceDir, PackagesPath)
  80
  81         if "ECP_SOURCE" not in os.environ:
  82             os.environ["ECP_SOURCE"] = mws.join(WorkspaceDir, GlobalData.gEdkCompatibilityPkg)
  83         if "EFI_SOURCE" not in os.environ:
  84             os.environ["EFI_SOURCE"] = os.environ["ECP_SOURCE"]
  85         if "EDK_SOURCE" not in os.environ:
  86             os.environ["EDK_SOURCE"] = os.environ["ECP_SOURCE"]
  87
  88         #
  89         # Unify case of characters on case-insensitive systems
  90         #
  91         EfiSourceDir = os.path.normcase(os.path.normpath(os.environ["EFI_SOURCE"]))
  92         EdkSourceDir = os.path.normcase(os.path.normpath(os.environ["EDK_SOURCE"]))
  93         EcpSourceDir = os.path.normcase(os.path.normpath(os.environ["ECP_SOURCE"]))
  94
  95         os.environ["EFI_SOURCE"] = EfiSourceDir
  96         os.environ["EDK_SOURCE"] = EdkSourceDir
  97         os.environ["ECP_SOURCE"] = EcpSourceDir
  98
  99         GlobalData.gWorkspace = WorkspaceDir
 100         GlobalData.gEfiSource = EfiSourceDir
 101         GlobalData.gEdkSource = EdkSourceDir
 102         GlobalData.gEcpSource = EcpSourceDir
 103
 104         GlobalData.gGlobalDefines["WORKSPACE"]  = WorkspaceDir
 105         GlobalData.gGlobalDefines["EFI_SOURCE"] = EfiSourceDir
 106         GlobalData.gGlobalDefines["EDK_SOURCE"] = EdkSourceDir
 107         GlobalData.gGlobalDefines["ECP_SOURCE"] = EcpSourceDir
 108
 109         EdkLogger.info("Loading ECC configuration ... done")
 110         # Generate checkpoints list
 111         EccGlobalData.gConfig = Configuration(self.ConfigFile)
 112
 113         # Generate exception list
 114         EccGlobalData.gException = ExceptionCheck(self.ExceptionFile)
 115
 116         # Init Ecc database
 117         EccGlobalData.gDb = Database.Database(Database.DATABASE_PATH)
 118         EccGlobalData.gDb.InitDatabase(self.IsInit)
 119
 120         #
 121         # Get files real name in workspace dir
 122         #
 123         GlobalData.gAllFiles = DirCache(GlobalData.gWorkspace)
 124
 125         # Build ECC database
 126 #         self.BuildDatabase()
 127         self.DetectOnlyScanDirs()
 128
 129         # Start to check
 130         self.Check()
 131
 132         # Show report
 133         self.GenReport()
 134
 135         # Close Database
 136         EccGlobalData.gDb.Close()
 137
 138     def InitDefaultConfigIni(self):
 139         paths = map(lambda p: os.path.join(p, 'Ecc', 'config.ini'), sys.path)
 140         paths = (os.path.realpath('config.ini'),) + tuple(paths)
 141         for path in paths:
 142             if os.path.exists(path):
 143                 self.ConfigFile = path
 144                 return
 145         self.ConfigFile = 'config.ini'
 146
 147
 148     ## DetectOnlyScan
 149     #
 150     # Detect whether only scanned folders have been enabled
 151     #
 152     def DetectOnlyScanDirs(self):
 153         if self.OnlyScan == True:
 154             OnlyScanDirs = []
 155             # Use regex here if multiple spaces or TAB exists in ScanOnlyDirList in config.ini file
 156             for folder in re.finditer(r'\S+', EccGlobalData.gConfig.ScanOnlyDirList):
 157                 OnlyScanDirs.append(folder.group())
 158             if len(OnlyScanDirs) != 0:
 159                 self.BuildDatabase(OnlyScanDirs)
 160             else:
 161                 EdkLogger.error("ECC", BuildToolError.OPTION_VALUE_INVALID, ExtraData="Use -f option need to fill specific folders in config.ini file")
 162         else:
 163             self.BuildDatabase()
 164
 165
 166     ## BuildDatabase
 167     #
 168     # Build the database for target
 169     #
 170     def BuildDatabase(self, SpeciDirs = None):
 171         # Clean report table
 172         EccGlobalData.gDb.TblReport.Drop()
 173         EccGlobalData.gDb.TblReport.Create()
 174
 175         # Build database
 176         if self.IsInit:
 177             if self.ScanMetaData:
 178                 EdkLogger.quiet("Building database for Meta Data File ...")
 179                 self.BuildMetaDataFileDatabase(SpeciDirs)
 180             if self.ScanSourceCode:
 181                 EdkLogger.quiet("Building database for Meta Data File Done!")
 182                 if SpeciDirs is None:
 183                     c.CollectSourceCodeDataIntoDB(EccGlobalData.gTarget)
 184                 else:
 185                     for specificDir in SpeciDirs:
 186                         c.CollectSourceCodeDataIntoDB(os.path.join(EccGlobalData.gTarget, specificDir))
 187
 188         EccGlobalData.gIdentifierTableList = GetTableList((MODEL_FILE_C, MODEL_FILE_H), 'Identifier', EccGlobalData.gDb)
 189         EccGlobalData.gCFileList = GetFileList(MODEL_FILE_C, EccGlobalData.gDb)
 190         EccGlobalData.gHFileList = GetFileList(MODEL_FILE_H, EccGlobalData.gDb)
 191         EccGlobalData.gUFileList = GetFileList(MODEL_FILE_UNI, EccGlobalData.gDb)
 192
 193     ## BuildMetaDataFileDatabase
 194     #
 195     # Build the database for meta data files
 196     #
 197     def BuildMetaDataFileDatabase(self, SpecificDirs = None):
 198         ScanFolders = []
 199         if SpecificDirs is None:
 200             ScanFolders.append(EccGlobalData.gTarget)
 201         else:
 202             for specificDir in SpecificDirs:
 203                 ScanFolders.append(os.path.join(EccGlobalData.gTarget, specificDir))
 204         EdkLogger.quiet("Building database for meta data files ...")
 205         Op = open(EccGlobalData.gConfig.MetaDataFileCheckPathOfGenerateFileList, 'w+')
 206         #SkipDirs = Read from config file
 207         SkipDirs = EccGlobalData.gConfig.SkipDirList
 208         SkipDirString = string.join(SkipDirs, '|')
 209 #         p = re.compile(r'.*[\\/](?:%s)[\\/]?.*' % SkipDirString)
 210         p = re.compile(r'.*[\\/](?:%s^\S)[\\/]?.*' % SkipDirString)
 211         for scanFolder in ScanFolders:
 212             for Root, Dirs, Files in os.walk(scanFolder):
 213                 if p.match(Root.upper()):
 214                     continue
 215                 for Dir in Dirs:
 216                     Dirname = os.path.join(Root, Dir)
 217                     if os.path.islink(Dirname):
 218                         Dirname = os.path.realpath(Dirname)
 219                         if os.path.isdir(Dirname):
 220                             # symlinks to directories are treated as directories
 221                             Dirs.remove(Dir)
 222                             Dirs.append(Dirname)
 223
 224                 for File in Files:
 225                     if len(File) > 4 and File[-4:].upper() == ".DEC":
 226                         Filename = os.path.normpath(os.path.join(Root, File))
 227                         EdkLogger.quiet("Parsing %s" % Filename)
 228                         Op.write("%s\r" % Filename)
 229                         #Dec(Filename, True, True, EccGlobalData.gWorkspace, EccGlobalData.gDb)
 230                         self.MetaFile = DecParser(Filename, MODEL_FILE_DEC, EccGlobalData.gDb.TblDec)
 231                         self.MetaFile.Start()
 232                         continue
 233                     if len(File) > 4 and File[-4:].upper() == ".DSC":
 234                         Filename = os.path.normpath(os.path.join(Root, File))
 235                         EdkLogger.quiet("Parsing %s" % Filename)
 236                         Op.write("%s\r" % Filename)
 237                         #Dsc(Filename, True, True, EccGlobalData.gWorkspace, EccGlobalData.gDb)
 238                         self.MetaFile = DscParser(PathClass(Filename, Root), MODEL_FILE_DSC, MetaFileStorage(EccGlobalData.gDb.TblDsc.Cur, Filename, MODEL_FILE_DSC, True))
 239                         # alwasy do post-process, in case of macros change
 240                         self.MetaFile.DoPostProcess()
 241                         self.MetaFile.Start()
 242                         self.MetaFile._PostProcess()
 243                         continue
 244                     if len(File) > 4 and File[-4:].upper() == ".INF":
 245                         Filename = os.path.normpath(os.path.join(Root, File))
 246                         EdkLogger.quiet("Parsing %s" % Filename)
 247                         Op.write("%s\r" % Filename)
 248                         #Inf(Filename, True, True, EccGlobalData.gWorkspace, EccGlobalData.gDb)
 249                         self.MetaFile = InfParser(Filename, MODEL_FILE_INF, EccGlobalData.gDb.TblInf)
 250                         self.MetaFile.Start()
 251                         continue
 252                     if len(File) > 4 and File[-4:].upper() == ".FDF":
 253                         Filename = os.path.normpath(os.path.join(Root, File))
 254                         EdkLogger.quiet("Parsing %s" % Filename)
 255                         Op.write("%s\r" % Filename)
 256                         Fdf(Filename, True, EccGlobalData.gWorkspace, EccGlobalData.gDb)
 257                         continue
 258                     if len(File) > 4 and File[-4:].upper() == ".UNI":
 259                         Filename = os.path.normpath(os.path.join(Root, File))
 260                         EdkLogger.quiet("Parsing %s" % Filename)
 261                         Op.write("%s\r" % Filename)
 262                         FileID = EccGlobalData.gDb.TblFile.InsertFile(Filename, MODEL_FILE_UNI)
 263                         EccGlobalData.gDb.TblReport.UpdateBelongsToItemByFile(FileID, File)
 264                         continue
 265
 266         Op.close()
 267
 268         # Commit to database
 269         EccGlobalData.gDb.Conn.commit()
 270
 271         EdkLogger.quiet("Building database for meta data files done!")
 272
 273     ##
 274     #
 275     # Check each checkpoint
 276     #
 277     def Check(self):
 278         EdkLogger.quiet("Checking ...")
 279         EccCheck = Check()
 280         EccCheck.Check()
 281         EdkLogger.quiet("Checking  done!")
 282
 283     ##
 284     #
 285     # Generate the scan report
 286     #
 287     def GenReport(self):
 288         EdkLogger.quiet("Generating report ...")
 289         EccGlobalData.gDb.TblReport.ToCSV(self.ReportFile)
 290         EdkLogger.quiet("Generating report done!")
 291
 292     def GetRealPathCase(self, path):
 293         TmpPath = path.rstrip(os.sep)
 294         PathParts = TmpPath.split(os.sep)
 295         if len(PathParts) == 0:
 296             return path
 297         if len(PathParts) == 1:
 298             if PathParts[0].strip().endswith(':'):
 299                 return PathParts[0].upper()
 300             # Relative dir, list . current dir
 301             Dirs = os.listdir('.')
 302             for Dir in Dirs:
 303                 if Dir.upper() == PathParts[0].upper():
 304                     return Dir
 305
 306         if PathParts[0].strip().endswith(':'):
 307             PathParts[0] = PathParts[0].upper()
 308         ParentDir = PathParts[0]
 309         RealPath = ParentDir
 310         if PathParts[0] == '':
 311             RealPath = os.sep
 312             ParentDir = os.sep
 313
 314         PathParts.remove(PathParts[0])    # need to remove the parent
 315         for Part in PathParts:
 316             Dirs = os.listdir(ParentDir + os.sep)
 317             for Dir in Dirs:
 318                 if Dir.upper() == Part.upper():
 319                     RealPath += os.sep
 320                     RealPath += Dir
 321                     break
 322             ParentDir += os.sep
 323             ParentDir += Dir
 324
 325         return RealPath
 326
 327     ## ParseOption
 328     #
 329     # Parse options
 330     #
 331     def ParseOption(self):
 332         (Options, Target) = self.EccOptionParser()
 333
 334         if Options.Workspace:
 335             os.environ["WORKSPACE"] = Options.Workspace
 336
 337         # Check workspace envirnoment
 338         if "WORKSPACE" not in os.environ:
 339             EdkLogger.error("ECC", BuildToolError.ATTRIBUTE_NOT_AVAILABLE, "Environment variable not found",
 340                             ExtraData="WORKSPACE")
 341         else:
 342             EccGlobalData.gWorkspace = os.path.normpath(os.getenv("WORKSPACE"))
 343             if not os.path.exists(EccGlobalData.gWorkspace):
 344                 EdkLogger.error("ECC", BuildToolError.FILE_NOT_FOUND, ExtraData="WORKSPACE = %s" % EccGlobalData.gWorkspace)
 345             os.environ["WORKSPACE"] = EccGlobalData.gWorkspace
 346         # Set log level
 347         self.SetLogLevel(Options)
 348
 349         # Set other options
 350         if Options.ConfigFile is not None:
 351             self.ConfigFile = Options.ConfigFile
 352         if Options.OutputFile is not None:
 353             self.OutputFile = Options.OutputFile
 354         if Options.ReportFile is not None:
 355             self.ReportFile = Options.ReportFile
 356         if Options.ExceptionFile is not None:
 357             self.ExceptionFile = Options.ExceptionFile
 358         if Options.Target is not None:
 359             if not os.path.isdir(Options.Target):
 360                 EdkLogger.error("ECC", BuildToolError.OPTION_VALUE_INVALID, ExtraData="Target [%s] does NOT exist" % Options.Target)
 361             else:
 362                 EccGlobalData.gTarget = self.GetRealPathCase(os.path.normpath(Options.Target))
 363         else:
 364             EdkLogger.warn("Ecc", EdkLogger.ECC_ERROR, "The target source tree was not specified, using current WORKSPACE instead!")
 365             EccGlobalData.gTarget = os.path.normpath(os.getenv("WORKSPACE"))
 366         if Options.keepdatabase is not None:
 367             self.IsInit = False
 368         if Options.metadata is not None and Options.sourcecode is not None:
 369             EdkLogger.error("ECC", BuildToolError.OPTION_CONFLICT, ExtraData="-m and -s can't be specified at one time")
 370         if Options.metadata is not None:
 371             self.ScanSourceCode = False
 372         if Options.sourcecode is not None:
 373             self.ScanMetaData = False
 374         if Options.folders is not None:
 375             self.OnlyScan = True
 376
 377     ## SetLogLevel
 378     #
 379     # Set current log level of the tool based on args
 380     #
 381     # @param Option:  The option list including log level setting
 382     #
 383     def SetLogLevel(self, Option):
 384         if Option.verbose is not None:
 385             EdkLogger.SetLevel(EdkLogger.VERBOSE)
 386         elif Option.quiet is not None:
 387             EdkLogger.SetLevel(EdkLogger.QUIET)
 388         elif Option.debug is not None:
 389             EdkLogger.SetLevel(Option.debug + 1)
 390         else:
 391             EdkLogger.SetLevel(EdkLogger.INFO)
 392
 393     ## Parse command line options
 394     #
 395     # Using standard Python module optparse to parse command line option of this tool.
 396     #
 397     # @retval Opt   A optparse.Values object containing the parsed options
 398     # @retval Args  Target of build command
 399     #
 400     def EccOptionParser(self):
 401         Parser = OptionParser(description = self.Copyright, version = self.Version, prog = "Ecc.exe", usage = "%prog [options]")
 402         Parser.add_option("-t", "--target sourcepath", action="store", type="string", dest='Target',
 403             help="Check all files under the target workspace.")
 404         Parser.add_option("-c", "--config filename", action="store", type="string", dest="ConfigFile",
 405             help="Specify a configuration file. Defaultly use config.ini under ECC tool directory.")
 406         Parser.add_option("-o", "--outfile filename", action="store", type="string", dest="OutputFile",
 407             help="Specify the name of an output file, if and only if one filename was specified.")
 408         Parser.add_option("-r", "--reportfile filename", action="store", type="string", dest="ReportFile",
 409             help="Specify the name of an report file, if and only if one filename was specified.")
 410         Parser.add_option("-e", "--exceptionfile filename", action="store", type="string", dest="ExceptionFile",
 411             help="Specify the name of an exception file, if and only if one filename was specified.")
 412         Parser.add_option("-m", "--metadata", action="store_true", type=None, help="Only scan meta-data files information if this option is specified.")
 413         Parser.add_option("-s", "--sourcecode", action="store_true", type=None, help="Only scan source code files information if this option is specified.")
 414         Parser.add_option("-k", "--keepdatabase", action="store_true", type=None, help="The existing Ecc database will not be cleaned except report information if this option is specified.")
 415         Parser.add_option("-l", "--log filename", action="store", dest="LogFile", help="""If specified, the tool should emit the changes that
 416                                                                                           were made by the tool after printing the result message.
 417                                                                                           If filename, the emit to the file, otherwise emit to
 418                                                                                           standard output. If no modifications were made, then do not
 419                                                                                           create a log file, or output a log message.""")
 420         Parser.add_option("-q", "--quiet", action="store_true", type=None, help="Disable all messages except FATAL ERRORS.")
 421         Parser.add_option("-v", "--verbose", action="store_true", type=None, help="Turn on verbose output with informational messages printed, "\
 422                                                                                    "including library instances selected, final dependency expression, "\
 423                                                                                    "and warning messages, etc.")
 424         Parser.add_option("-d", "--debug", action="store", type="int", help="Enable debug messages at specified level.")
 425         Parser.add_option("-w", "--workspace", action="store", type="string", dest='Workspace', help="Specify workspace.")
 426         Parser.add_option("-f", "--folders", action="store_true", type=None, help="Only scanning specified folders which are recorded in config.ini file.")
 427
 428         (Opt, Args)=Parser.parse_args()
 429
 430         return (Opt, Args)
 431
 432 ##
 433 #
 434 # This acts like the main() function for the script, unless it is 'import'ed into another
 435 # script.
 436 #
 437 if __name__ == '__main__':
 438     # Initialize log system
 439     EdkLogger.Initialize()
 440     EdkLogger.IsRaiseError = False
 441
 442     StartTime = time.clock()
 443     Ecc = Ecc()
 444     FinishTime = time.clock()
 445
 446     BuildDuration = time.strftime("%M:%S", time.gmtime(int(round(FinishTime - StartTime))))
 447     EdkLogger.quiet("\n%s [%s]" % (time.strftime("%H:%M:%S, %b.%d %Y", time.localtime()), BuildDuration))