BaseTools/Source/Python/Ecc/Ecc.py

   1 ## @file
   2 # This file is used to be the main entrance of ECC tool
   3 #
   4 # Copyright (c) 2009 - 2016, Intel Corporation. All rights reserved.<BR>
   5 # This program and the accompanying materials
   6 # are licensed and made available under the terms and conditions of the BSD License
   7 # which accompanies this distribution.  The full text of the license may be found at
   8 # http://opensource.org/licenses/bsd-license.php
   9 #
  10 # THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS,
  11 # WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED.
  12 #
  13
  14 ##
  15 # Import Modules
  16 #
  17 import Common.LongFilePathOs as os, time, glob, sys
  18 import Common.EdkLogger as EdkLogger
  19 import Database
  20 import EccGlobalData
  21 from MetaDataParser import *
  22 from optparse import OptionParser
  23 from Configuration import Configuration
  24 from Check import Check
  25 import Common.GlobalData as GlobalData
  26
  27 from Common.String import NormPath
  28 from Common.BuildVersion import gBUILD_VERSION
  29 from Common import BuildToolError
  30 from Common.Misc import PathClass
  31 from Common.Misc import DirCache
  32 from MetaFileWorkspace.MetaFileParser import DscParser
  33 from MetaFileWorkspace.MetaFileParser import DecParser
  34 from MetaFileWorkspace.MetaFileParser import InfParser
  35 from MetaFileWorkspace.MetaFileParser import Fdf
  36 from MetaFileWorkspace.MetaFileTable import MetaFileStorage
  37 import c
  38 import re, string
  39 from Exception import *
  40 from Common.LongFilePathSupport import OpenLongFilePath as open
  41 from Common.MultipleWorkspace import MultipleWorkspace as mws
  42
  43 ## Ecc
  44 #
  45 # This class is used to define Ecc main entrance
  46 #
  47 # @param object:          Inherited from object class
  48 #
  49 class Ecc(object):
  50     def __init__(self):
  51         # Version and Copyright
  52         self.VersionNumber = ("1.0" + " Build " + gBUILD_VERSION)
  53         self.Version = "%prog Version " + self.VersionNumber
  54         self.Copyright = "Copyright (c) 2009 - 2016, Intel Corporation  All rights reserved."
  55
  56         self.InitDefaultConfigIni()
  57         self.OutputFile = 'output.txt'
  58         self.ReportFile = 'Report.csv'
  59         self.ExceptionFile = 'exception.xml'
  60         self.IsInit = True
  61         self.ScanSourceCode = True
  62         self.ScanMetaData = True
  63         self.MetaFile = ''
  64         self.OnlyScan = None
  65
  66         # Parse the options and args
  67         self.ParseOption()
  68         EdkLogger.info(time.strftime("%H:%M:%S, %b.%d %Y ", time.localtime()) + "[00:00]" + "\n")
  69
  70         #
  71         # Check EFI_SOURCE (Edk build convention). EDK_SOURCE will always point to ECP
  72         #
  73         WorkspaceDir = os.path.normcase(os.path.normpath(os.environ["WORKSPACE"]))
  74         os.environ["WORKSPACE"] = WorkspaceDir
  75
  76         # set multiple workspace
  77         PackagesPath = os.getenv("PACKAGES_PATH")
  78         mws.setWs(WorkspaceDir, PackagesPath)
  79
  80         if "ECP_SOURCE" not in os.environ:
  81             os.environ["ECP_SOURCE"] = mws.join(WorkspaceDir, GlobalData.gEdkCompatibilityPkg)
  82         if "EFI_SOURCE" not in os.environ:
  83             os.environ["EFI_SOURCE"] = os.environ["ECP_SOURCE"]
  84         if "EDK_SOURCE" not in os.environ:
  85             os.environ["EDK_SOURCE"] = os.environ["ECP_SOURCE"]
  86
  87         #
  88         # Unify case of characters on case-insensitive systems
  89         #
  90         EfiSourceDir = os.path.normcase(os.path.normpath(os.environ["EFI_SOURCE"]))
  91         EdkSourceDir = os.path.normcase(os.path.normpath(os.environ["EDK_SOURCE"]))
  92         EcpSourceDir = os.path.normcase(os.path.normpath(os.environ["ECP_SOURCE"]))
  93
  94         os.environ["EFI_SOURCE"] = EfiSourceDir
  95         os.environ["EDK_SOURCE"] = EdkSourceDir
  96         os.environ["ECP_SOURCE"] = EcpSourceDir
  97
  98         GlobalData.gWorkspace = WorkspaceDir
  99         GlobalData.gEfiSource = EfiSourceDir
 100         GlobalData.gEdkSource = EdkSourceDir
 101         GlobalData.gEcpSource = EcpSourceDir
 102
 103         GlobalData.gGlobalDefines["WORKSPACE"]  = WorkspaceDir
 104         GlobalData.gGlobalDefines["EFI_SOURCE"] = EfiSourceDir
 105         GlobalData.gGlobalDefines["EDK_SOURCE"] = EdkSourceDir
 106         GlobalData.gGlobalDefines["ECP_SOURCE"] = EcpSourceDir
 107
 108         EdkLogger.info("Loading ECC configuration ... done")
 109         # Generate checkpoints list
 110         EccGlobalData.gConfig = Configuration(self.ConfigFile)
 111
 112         # Generate exception list
 113         EccGlobalData.gException = ExceptionCheck(self.ExceptionFile)
 114
 115         # Init Ecc database
 116         EccGlobalData.gDb = Database.Database(Database.DATABASE_PATH)
 117         EccGlobalData.gDb.InitDatabase(self.IsInit)
 118
 119         #
 120         # Get files real name in workspace dir
 121         #
 122         GlobalData.gAllFiles = DirCache(GlobalData.gWorkspace)
 123
 124         # Build ECC database
 125 #         self.BuildDatabase()
 126         self.DetectOnlyScanDirs()
 127
 128         # Start to check
 129         self.Check()
 130
 131         # Show report
 132         self.GenReport()
 133
 134         # Close Database
 135         EccGlobalData.gDb.Close()
 136
 137     def InitDefaultConfigIni(self):
 138         paths = map(lambda p: os.path.join(p, 'Ecc', 'config.ini'), sys.path)
 139         paths = (os.path.realpath('config.ini'),) + tuple(paths)
 140         for path in paths:
 141             if os.path.exists(path):
 142                 self.ConfigFile = path
 143                 return
 144         self.ConfigFile = 'config.ini'
 145
 146
 147     ## DetectOnlyScan
 148     #
 149     # Detect whether only scanned folders have been enabled
 150     #
 151     def DetectOnlyScanDirs(self):
 152         if self.OnlyScan == True:
 153             OnlyScanDirs = []
 154             # Use regex here if multiple spaces or TAB exists in ScanOnlyDirList in config.ini file
 155             for folder in re.finditer(r'\S+', EccGlobalData.gConfig.ScanOnlyDirList):
 156                 OnlyScanDirs.append(folder.group())
 157             if len(OnlyScanDirs) != 0:
 158                 self.BuildDatabase(OnlyScanDirs)
 159             else:
 160                 EdkLogger.error("ECC", BuildToolError.OPTION_VALUE_INVALID, ExtraData="Use -f option need to fill specific folders in config.ini file")
 161         else:
 162             self.BuildDatabase()
 163
 164
 165     ## BuildDatabase
 166     #
 167     # Build the database for target
 168     #
 169     def BuildDatabase(self, SpeciDirs = None):
 170         # Clean report table
 171         EccGlobalData.gDb.TblReport.Drop()
 172         EccGlobalData.gDb.TblReport.Create()
 173
 174         # Build database
 175         if self.IsInit:
 176             if self.ScanMetaData:
 177                 EdkLogger.quiet("Building database for Meta Data File ...")
 178                 self.BuildMetaDataFileDatabase(SpeciDirs)
 179             if self.ScanSourceCode:
 180                 EdkLogger.quiet("Building database for Meta Data File Done!")
 181                 if SpeciDirs is None:
 182                     c.CollectSourceCodeDataIntoDB(EccGlobalData.gTarget)
 183                 else:
 184                     for specificDir in SpeciDirs:
 185                         c.CollectSourceCodeDataIntoDB(os.path.join(EccGlobalData.gTarget, specificDir))
 186
 187         EccGlobalData.gIdentifierTableList = GetTableList((MODEL_FILE_C, MODEL_FILE_H), 'Identifier', EccGlobalData.gDb)
 188         EccGlobalData.gCFileList = GetFileList(MODEL_FILE_C, EccGlobalData.gDb)
 189         EccGlobalData.gHFileList = GetFileList(MODEL_FILE_H, EccGlobalData.gDb)
 190         EccGlobalData.gUFileList = GetFileList(MODEL_FILE_UNI, EccGlobalData.gDb)
 191
 192     ## BuildMetaDataFileDatabase
 193     #
 194     # Build the database for meta data files
 195     #
 196     def BuildMetaDataFileDatabase(self, SpecificDirs = None):
 197         ScanFolders = []
 198         if SpecificDirs is None:
 199             ScanFolders.append(EccGlobalData.gTarget)
 200         else:
 201             for specificDir in SpecificDirs:
 202                 ScanFolders.append(os.path.join(EccGlobalData.gTarget, specificDir))
 203         EdkLogger.quiet("Building database for meta data files ...")
 204         Op = open(EccGlobalData.gConfig.MetaDataFileCheckPathOfGenerateFileList, 'w+')
 205         #SkipDirs = Read from config file
 206         SkipDirs = EccGlobalData.gConfig.SkipDirList
 207         SkipDirString = string.join(SkipDirs, '|')
 208 #         p = re.compile(r'.*[\\/](?:%s)[\\/]?.*' % SkipDirString)
 209         p = re.compile(r'.*[\\/](?:%s^\S)[\\/]?.*' % SkipDirString)
 210         for scanFolder in ScanFolders:
 211             for Root, Dirs, Files in os.walk(scanFolder):
 212                 if p.match(Root.upper()):
 213                     continue
 214                 for Dir in Dirs:
 215                     Dirname = os.path.join(Root, Dir)
 216                     if os.path.islink(Dirname):
 217                         Dirname = os.path.realpath(Dirname)
 218                         if os.path.isdir(Dirname):
 219                             # symlinks to directories are treated as directories
 220                             Dirs.remove(Dir)
 221                             Dirs.append(Dirname)
 222
 223                 for File in Files:
 224                     if len(File) > 4 and File[-4:].upper() == ".DEC":
 225                         Filename = os.path.normpath(os.path.join(Root, File))
 226                         EdkLogger.quiet("Parsing %s" % Filename)
 227                         Op.write("%s\r" % Filename)
 228                         #Dec(Filename, True, True, EccGlobalData.gWorkspace, EccGlobalData.gDb)
 229                         self.MetaFile = DecParser(Filename, MODEL_FILE_DEC, EccGlobalData.gDb.TblDec)
 230                         self.MetaFile.Start()
 231                         continue
 232                     if len(File) > 4 and File[-4:].upper() == ".DSC":
 233                         Filename = os.path.normpath(os.path.join(Root, File))
 234                         EdkLogger.quiet("Parsing %s" % Filename)
 235                         Op.write("%s\r" % Filename)
 236                         #Dsc(Filename, True, True, EccGlobalData.gWorkspace, EccGlobalData.gDb)
 237                         self.MetaFile = DscParser(PathClass(Filename, Root), MODEL_FILE_DSC, MetaFileStorage(EccGlobalData.gDb.TblDsc.Cur, Filename, MODEL_FILE_DSC, True))
 238                         # alwasy do post-process, in case of macros change
 239                         self.MetaFile.DoPostProcess()
 240                         self.MetaFile.Start()
 241                         self.MetaFile._PostProcess()
 242                         continue
 243                     if len(File) > 4 and File[-4:].upper() == ".INF":
 244                         Filename = os.path.normpath(os.path.join(Root, File))
 245                         EdkLogger.quiet("Parsing %s" % Filename)
 246                         Op.write("%s\r" % Filename)
 247                         #Inf(Filename, True, True, EccGlobalData.gWorkspace, EccGlobalData.gDb)
 248                         self.MetaFile = InfParser(Filename, MODEL_FILE_INF, EccGlobalData.gDb.TblInf)
 249                         self.MetaFile.Start()
 250                         continue
 251                     if len(File) > 4 and File[-4:].upper() == ".FDF":
 252                         Filename = os.path.normpath(os.path.join(Root, File))
 253                         EdkLogger.quiet("Parsing %s" % Filename)
 254                         Op.write("%s\r" % Filename)
 255                         Fdf(Filename, True, EccGlobalData.gWorkspace, EccGlobalData.gDb)
 256                         continue
 257                     if len(File) > 4 and File[-4:].upper() == ".UNI":
 258                         Filename = os.path.normpath(os.path.join(Root, File))
 259                         EdkLogger.quiet("Parsing %s" % Filename)
 260                         Op.write("%s\r" % Filename)
 261                         FileID = EccGlobalData.gDb.TblFile.InsertFile(Filename, MODEL_FILE_UNI)
 262                         EccGlobalData.gDb.TblReport.UpdateBelongsToItemByFile(FileID, File)
 263                         continue
 264
 265         Op.close()
 266
 267         # Commit to database
 268         EccGlobalData.gDb.Conn.commit()
 269
 270         EdkLogger.quiet("Building database for meta data files done!")
 271
 272     ##
 273     #
 274     # Check each checkpoint
 275     #
 276     def Check(self):
 277         EdkLogger.quiet("Checking ...")
 278         EccCheck = Check()
 279         EccCheck.Check()
 280         EdkLogger.quiet("Checking  done!")
 281
 282     ##
 283     #
 284     # Generate the scan report
 285     #
 286     def GenReport(self):
 287         EdkLogger.quiet("Generating report ...")
 288         EccGlobalData.gDb.TblReport.ToCSV(self.ReportFile)
 289         EdkLogger.quiet("Generating report done!")
 290
 291     def GetRealPathCase(self, path):
 292         TmpPath = path.rstrip(os.sep)
 293         PathParts = TmpPath.split(os.sep)
 294         if len(PathParts) == 0:
 295             return path
 296         if len(PathParts) == 1:
 297             if PathParts[0].strip().endswith(':'):
 298                 return PathParts[0].upper()
 299             # Relative dir, list . current dir
 300             Dirs = os.listdir('.')
 301             for Dir in Dirs:
 302                 if Dir.upper() == PathParts[0].upper():
 303                     return Dir
 304
 305         if PathParts[0].strip().endswith(':'):
 306             PathParts[0] = PathParts[0].upper()
 307         ParentDir = PathParts[0]
 308         RealPath = ParentDir
 309         if PathParts[0] == '':
 310             RealPath = os.sep
 311             ParentDir = os.sep
 312
 313         PathParts.remove(PathParts[0])    # need to remove the parent
 314         for Part in PathParts:
 315             Dirs = os.listdir(ParentDir + os.sep)
 316             for Dir in Dirs:
 317                 if Dir.upper() == Part.upper():
 318                     RealPath += os.sep
 319                     RealPath += Dir
 320                     break
 321             ParentDir += os.sep
 322             ParentDir += Dir
 323
 324         return RealPath
 325
 326     ## ParseOption
 327     #
 328     # Parse options
 329     #
 330     def ParseOption(self):
 331         (Options, Target) = self.EccOptionParser()
 332
 333         if Options.Workspace:
 334             os.environ["WORKSPACE"] = Options.Workspace
 335
 336         # Check workspace envirnoment
 337         if "WORKSPACE" not in os.environ:
 338             EdkLogger.error("ECC", BuildToolError.ATTRIBUTE_NOT_AVAILABLE, "Environment variable not found",
 339                             ExtraData="WORKSPACE")
 340         else:
 341             EccGlobalData.gWorkspace = os.path.normpath(os.getenv("WORKSPACE"))
 342             if not os.path.exists(EccGlobalData.gWorkspace):
 343                 EdkLogger.error("ECC", BuildToolError.FILE_NOT_FOUND, ExtraData="WORKSPACE = %s" % EccGlobalData.gWorkspace)
 344             os.environ["WORKSPACE"] = EccGlobalData.gWorkspace
 345         # Set log level
 346         self.SetLogLevel(Options)
 347
 348         # Set other options
 349         if Options.ConfigFile is not None:
 350             self.ConfigFile = Options.ConfigFile
 351         if Options.OutputFile is not None:
 352             self.OutputFile = Options.OutputFile
 353         if Options.ReportFile is not None:
 354             self.ReportFile = Options.ReportFile
 355         if Options.ExceptionFile is not None:
 356             self.ExceptionFile = Options.ExceptionFile
 357         if Options.Target is not None:
 358             if not os.path.isdir(Options.Target):
 359                 EdkLogger.error("ECC", BuildToolError.OPTION_VALUE_INVALID, ExtraData="Target [%s] does NOT exist" % Options.Target)
 360             else:
 361                 EccGlobalData.gTarget = self.GetRealPathCase(os.path.normpath(Options.Target))
 362         else:
 363             EdkLogger.warn("Ecc", EdkLogger.ECC_ERROR, "The target source tree was not specified, using current WORKSPACE instead!")
 364             EccGlobalData.gTarget = os.path.normpath(os.getenv("WORKSPACE"))
 365         if Options.keepdatabase is not None:
 366             self.IsInit = False
 367         if Options.metadata is not None and Options.sourcecode is not None:
 368             EdkLogger.error("ECC", BuildToolError.OPTION_CONFLICT, ExtraData="-m and -s can't be specified at one time")
 369         if Options.metadata is not None:
 370             self.ScanSourceCode = False
 371         if Options.sourcecode is not None:
 372             self.ScanMetaData = False
 373         if Options.folders is not None:
 374             self.OnlyScan = True
 375
 376     ## SetLogLevel
 377     #
 378     # Set current log level of the tool based on args
 379     #
 380     # @param Option:  The option list including log level setting
 381     #
 382     def SetLogLevel(self, Option):
 383         if Option.verbose is not None:
 384             EdkLogger.SetLevel(EdkLogger.VERBOSE)
 385         elif Option.quiet is not None:
 386             EdkLogger.SetLevel(EdkLogger.QUIET)
 387         elif Option.debug is not None:
 388             EdkLogger.SetLevel(Option.debug + 1)
 389         else:
 390             EdkLogger.SetLevel(EdkLogger.INFO)
 391
 392     ## Parse command line options
 393     #
 394     # Using standard Python module optparse to parse command line option of this tool.
 395     #
 396     # @retval Opt   A optparse.Values object containing the parsed options
 397     # @retval Args  Target of build command
 398     #
 399     def EccOptionParser(self):
 400         Parser = OptionParser(description = self.Copyright, version = self.Version, prog = "Ecc.exe", usage = "%prog [options]")
 401         Parser.add_option("-t", "--target sourcepath", action="store", type="string", dest='Target',
 402             help="Check all files under the target workspace.")
 403         Parser.add_option("-c", "--config filename", action="store", type="string", dest="ConfigFile",
 404             help="Specify a configuration file. Defaultly use config.ini under ECC tool directory.")
 405         Parser.add_option("-o", "--outfile filename", action="store", type="string", dest="OutputFile",
 406             help="Specify the name of an output file, if and only if one filename was specified.")
 407         Parser.add_option("-r", "--reportfile filename", action="store", type="string", dest="ReportFile",
 408             help="Specify the name of an report file, if and only if one filename was specified.")
 409         Parser.add_option("-e", "--exceptionfile filename", action="store", type="string", dest="ExceptionFile",
 410             help="Specify the name of an exception file, if and only if one filename was specified.")
 411         Parser.add_option("-m", "--metadata", action="store_true", type=None, help="Only scan meta-data files information if this option is specified.")
 412         Parser.add_option("-s", "--sourcecode", action="store_true", type=None, help="Only scan source code files information if this option is specified.")
 413         Parser.add_option("-k", "--keepdatabase", action="store_true", type=None, help="The existing Ecc database will not be cleaned except report information if this option is specified.")
 414         Parser.add_option("-l", "--log filename", action="store", dest="LogFile", help="""If specified, the tool should emit the changes that
 415                                                                                           were made by the tool after printing the result message.
 416                                                                                           If filename, the emit to the file, otherwise emit to
 417                                                                                           standard output. If no modifications were made, then do not
 418                                                                                           create a log file, or output a log message.""")
 419         Parser.add_option("-q", "--quiet", action="store_true", type=None, help="Disable all messages except FATAL ERRORS.")
 420         Parser.add_option("-v", "--verbose", action="store_true", type=None, help="Turn on verbose output with informational messages printed, "\
 421                                                                                    "including library instances selected, final dependency expression, "\
 422                                                                                    "and warning messages, etc.")
 423         Parser.add_option("-d", "--debug", action="store", type="int", help="Enable debug messages at specified level.")
 424         Parser.add_option("-w", "--workspace", action="store", type="string", dest='Workspace', help="Specify workspace.")
 425         Parser.add_option("-f", "--folders", action="store_true", type=None, help="Only scanning specified folders which are recorded in config.ini file.")
 426
 427         (Opt, Args)=Parser.parse_args()
 428
 429         return (Opt, Args)
 430
 431 ##
 432 #
 433 # This acts like the main() function for the script, unless it is 'import'ed into another
 434 # script.
 435 #
 436 if __name__ == '__main__':
 437     # Initialize log system
 438     EdkLogger.Initialize()
 439     EdkLogger.IsRaiseError = False
 440
 441     StartTime = time.clock()
 442     Ecc = Ecc()
 443     FinishTime = time.clock()
 444
 445     BuildDuration = time.strftime("%M:%S", time.gmtime(int(round(FinishTime - StartTime))))
 446     EdkLogger.quiet("\n%s [%s]" % (time.strftime("%H:%M:%S, %b.%d %Y", time.localtime()), BuildDuration))