c2ad4faff4466b82b08c279a9858ef5c4ac870c7
[mirror_edk2.git] / BaseTools / Source / Python / Ecc / Ecc.py
1 ## @file
2 # This file is used to be the main entrance of ECC tool
3 #
4 # Copyright (c) 2009 - 2014, Intel Corporation. All rights reserved.<BR>
5 # This program and the accompanying materials
6 # are licensed and made available under the terms and conditions of the BSD License
7 # which accompanies this distribution. The full text of the license may be found at
8 # http://opensource.org/licenses/bsd-license.php
9 #
10 # THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS,
11 # WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED.
12 #
13
14 ##
15 # Import Modules
16 #
17 import Common.LongFilePathOs as os, time, glob, sys
18 import Common.EdkLogger as EdkLogger
19 import Database
20 import EccGlobalData
21 from MetaDataParser import *
22 from optparse import OptionParser
23 from Configuration import Configuration
24 from Check import Check
25 import Common.GlobalData as GlobalData
26
27 from Common.String import NormPath
28 from Common.BuildVersion import gBUILD_VERSION
29 from Common import BuildToolError
30 from Common.Misc import PathClass
31 from Common.Misc import DirCache
32 from MetaFileWorkspace.MetaFileParser import DscParser
33 from MetaFileWorkspace.MetaFileParser import DecParser
34 from MetaFileWorkspace.MetaFileParser import InfParser
35 from MetaFileWorkspace.MetaFileParser import Fdf
36 from MetaFileWorkspace.MetaFileTable import MetaFileStorage
37 import c
38 import re, string
39 from Exception import *
40 from Common.LongFilePathSupport import OpenLongFilePath as open
41 from Common.MultipleWorkspace import MultipleWorkspace as mws
42
43 ## Ecc
44 #
45 # This class is used to define Ecc main entrance
46 #
47 # @param object: Inherited from object class
48 #
49 class Ecc(object):
50 def __init__(self):
51 # Version and Copyright
52 self.VersionNumber = ("0.01" + " " + gBUILD_VERSION)
53 self.Version = "%prog Version " + self.VersionNumber
54 self.Copyright = "Copyright (c) 2009 - 2010, Intel Corporation All rights reserved."
55
56 self.InitDefaultConfigIni()
57 self.OutputFile = 'output.txt'
58 self.ReportFile = 'Report.csv'
59 self.ExceptionFile = 'exception.xml'
60 self.IsInit = True
61 self.ScanSourceCode = True
62 self.ScanMetaData = True
63 self.MetaFile = ''
64 self.OnlyScan = None
65
66 # Parse the options and args
67 self.ParseOption()
68
69 #
70 # Check EFI_SOURCE (Edk build convention). EDK_SOURCE will always point to ECP
71 #
72 WorkspaceDir = os.path.normcase(os.path.normpath(os.environ["WORKSPACE"]))
73 os.environ["WORKSPACE"] = WorkspaceDir
74
75 # set multiple workspace
76 PackagesPath = os.getenv("PACKAGES_PATH")
77 mws.setWs(WorkspaceDir, PackagesPath)
78
79 if "ECP_SOURCE" not in os.environ:
80 os.environ["ECP_SOURCE"] = mws.join(WorkspaceDir, GlobalData.gEdkCompatibilityPkg)
81 if "EFI_SOURCE" not in os.environ:
82 os.environ["EFI_SOURCE"] = os.environ["ECP_SOURCE"]
83 if "EDK_SOURCE" not in os.environ:
84 os.environ["EDK_SOURCE"] = os.environ["ECP_SOURCE"]
85
86 #
87 # Unify case of characters on case-insensitive systems
88 #
89 EfiSourceDir = os.path.normcase(os.path.normpath(os.environ["EFI_SOURCE"]))
90 EdkSourceDir = os.path.normcase(os.path.normpath(os.environ["EDK_SOURCE"]))
91 EcpSourceDir = os.path.normcase(os.path.normpath(os.environ["ECP_SOURCE"]))
92
93 os.environ["EFI_SOURCE"] = EfiSourceDir
94 os.environ["EDK_SOURCE"] = EdkSourceDir
95 os.environ["ECP_SOURCE"] = EcpSourceDir
96
97 GlobalData.gWorkspace = WorkspaceDir
98 GlobalData.gEfiSource = EfiSourceDir
99 GlobalData.gEdkSource = EdkSourceDir
100 GlobalData.gEcpSource = EcpSourceDir
101
102 GlobalData.gGlobalDefines["WORKSPACE"] = WorkspaceDir
103 GlobalData.gGlobalDefines["EFI_SOURCE"] = EfiSourceDir
104 GlobalData.gGlobalDefines["EDK_SOURCE"] = EdkSourceDir
105 GlobalData.gGlobalDefines["ECP_SOURCE"] = EcpSourceDir
106
107
108 # Generate checkpoints list
109 EccGlobalData.gConfig = Configuration(self.ConfigFile)
110
111 # Generate exception list
112 EccGlobalData.gException = ExceptionCheck(self.ExceptionFile)
113
114 # Init Ecc database
115 EccGlobalData.gDb = Database.Database(Database.DATABASE_PATH)
116 EccGlobalData.gDb.InitDatabase(self.IsInit)
117
118 #
119 # Get files real name in workspace dir
120 #
121 GlobalData.gAllFiles = DirCache(GlobalData.gWorkspace)
122
123 # Build ECC database
124 # self.BuildDatabase()
125 self.DetectOnlyScanDirs()
126
127 # Start to check
128 self.Check()
129
130 # Show report
131 self.GenReport()
132
133 # Close Database
134 EccGlobalData.gDb.Close()
135
136 def InitDefaultConfigIni(self):
137 paths = map(lambda p: os.path.join(p, 'Ecc', 'config.ini'), sys.path)
138 paths = (os.path.realpath('config.ini'),) + tuple(paths)
139 for path in paths:
140 if os.path.exists(path):
141 self.ConfigFile = path
142 return
143 self.ConfigFile = 'config.ini'
144
145
146 ## DetectOnlyScan
147 #
148 # Detect whether only scanned folders have been enabled
149 #
150 def DetectOnlyScanDirs(self):
151 if self.OnlyScan == True:
152 OnlyScanDirs = []
153 # Use regex here if multiple spaces or TAB exists in ScanOnlyDirList in config.ini file
154 for folder in re.finditer(r'\S+', EccGlobalData.gConfig.ScanOnlyDirList):
155 OnlyScanDirs.append(folder.group())
156 if len(OnlyScanDirs) != 0:
157 self.BuildDatabase(OnlyScanDirs)
158 else:
159 EdkLogger.error("ECC", BuildToolError.OPTION_VALUE_INVALID, ExtraData="Use -f option need to fill specific folders in config.ini file")
160 else:
161 self.BuildDatabase()
162
163
164 ## BuildDatabase
165 #
166 # Build the database for target
167 #
168 def BuildDatabase(self, SpeciDirs = None):
169 # Clean report table
170 EccGlobalData.gDb.TblReport.Drop()
171 EccGlobalData.gDb.TblReport.Create()
172
173 # Build database
174 if self.IsInit:
175 if self.ScanMetaData:
176 EdkLogger.quiet("Building database for Meta Data File ...")
177 self.BuildMetaDataFileDatabase(SpeciDirs)
178 if self.ScanSourceCode:
179 EdkLogger.quiet("Building database for Meta Data File Done!")
180 if SpeciDirs == None:
181 c.CollectSourceCodeDataIntoDB(EccGlobalData.gTarget)
182 else:
183 for specificDir in SpeciDirs:
184 c.CollectSourceCodeDataIntoDB(os.path.join(EccGlobalData.gTarget, specificDir))
185
186 EccGlobalData.gIdentifierTableList = GetTableList((MODEL_FILE_C, MODEL_FILE_H), 'Identifier', EccGlobalData.gDb)
187 EccGlobalData.gCFileList = GetFileList(MODEL_FILE_C, EccGlobalData.gDb)
188 EccGlobalData.gHFileList = GetFileList(MODEL_FILE_H, EccGlobalData.gDb)
189 EccGlobalData.gUFileList = GetFileList(MODEL_FILE_UNI, EccGlobalData.gDb)
190
191 ## BuildMetaDataFileDatabase
192 #
193 # Build the database for meta data files
194 #
195 def BuildMetaDataFileDatabase(self, SpecificDirs = None):
196 ScanFolders = []
197 if SpecificDirs == None:
198 ScanFolders.append(EccGlobalData.gTarget)
199 else:
200 for specificDir in SpecificDirs:
201 ScanFolders.append(os.path.join(EccGlobalData.gTarget, specificDir))
202 EdkLogger.quiet("Building database for meta data files ...")
203 Op = open(EccGlobalData.gConfig.MetaDataFileCheckPathOfGenerateFileList, 'w+')
204 #SkipDirs = Read from config file
205 SkipDirs = EccGlobalData.gConfig.SkipDirList
206 SkipDirString = string.join(SkipDirs, '|')
207 # p = re.compile(r'.*[\\/](?:%s)[\\/]?.*' % SkipDirString)
208 p = re.compile(r'.*[\\/](?:%s^\S)[\\/]?.*' % SkipDirString)
209 for scanFolder in ScanFolders:
210 for Root, Dirs, Files in os.walk(scanFolder):
211 if p.match(Root.upper()):
212 continue
213 for Dir in Dirs:
214 Dirname = os.path.join(Root, Dir)
215 if os.path.islink(Dirname):
216 Dirname = os.path.realpath(Dirname)
217 if os.path.isdir(Dirname):
218 # symlinks to directories are treated as directories
219 Dirs.remove(Dir)
220 Dirs.append(Dirname)
221
222 for File in Files:
223 if len(File) > 4 and File[-4:].upper() == ".DEC":
224 Filename = os.path.normpath(os.path.join(Root, File))
225 EdkLogger.quiet("Parsing %s" % Filename)
226 Op.write("%s\r" % Filename)
227 #Dec(Filename, True, True, EccGlobalData.gWorkspace, EccGlobalData.gDb)
228 self.MetaFile = DecParser(Filename, MODEL_FILE_DEC, EccGlobalData.gDb.TblDec)
229 self.MetaFile.Start()
230 continue
231 if len(File) > 4 and File[-4:].upper() == ".DSC":
232 Filename = os.path.normpath(os.path.join(Root, File))
233 EdkLogger.quiet("Parsing %s" % Filename)
234 Op.write("%s\r" % Filename)
235 #Dsc(Filename, True, True, EccGlobalData.gWorkspace, EccGlobalData.gDb)
236 self.MetaFile = DscParser(PathClass(Filename, Root), MODEL_FILE_DSC, MetaFileStorage(EccGlobalData.gDb.TblDsc.Cur, Filename, MODEL_FILE_DSC, True))
237 # alwasy do post-process, in case of macros change
238 self.MetaFile.DoPostProcess()
239 self.MetaFile.Start()
240 self.MetaFile._PostProcess()
241 continue
242 if len(File) > 4 and File[-4:].upper() == ".INF":
243 Filename = os.path.normpath(os.path.join(Root, File))
244 EdkLogger.quiet("Parsing %s" % Filename)
245 Op.write("%s\r" % Filename)
246 #Inf(Filename, True, True, EccGlobalData.gWorkspace, EccGlobalData.gDb)
247 self.MetaFile = InfParser(Filename, MODEL_FILE_INF, EccGlobalData.gDb.TblInf)
248 self.MetaFile.Start()
249 continue
250 if len(File) > 4 and File[-4:].upper() == ".FDF":
251 Filename = os.path.normpath(os.path.join(Root, File))
252 EdkLogger.quiet("Parsing %s" % Filename)
253 Op.write("%s\r" % Filename)
254 Fdf(Filename, True, EccGlobalData.gWorkspace, EccGlobalData.gDb)
255 continue
256 if len(File) > 4 and File[-4:].upper() == ".UNI":
257 Filename = os.path.normpath(os.path.join(Root, File))
258 EdkLogger.quiet("Parsing %s" % Filename)
259 Op.write("%s\r" % Filename)
260 FileID = EccGlobalData.gDb.TblFile.InsertFile(Filename, MODEL_FILE_UNI)
261 EccGlobalData.gDb.TblReport.UpdateBelongsToItemByFile(FileID, File)
262 continue
263
264 Op.close()
265
266 # Commit to database
267 EccGlobalData.gDb.Conn.commit()
268
269 EdkLogger.quiet("Building database for meta data files done!")
270
271 ##
272 #
273 # Check each checkpoint
274 #
275 def Check(self):
276 EdkLogger.quiet("Checking ...")
277 EccCheck = Check()
278 EccCheck.Check()
279 EdkLogger.quiet("Checking done!")
280
281 ##
282 #
283 # Generate the scan report
284 #
285 def GenReport(self):
286 EdkLogger.quiet("Generating report ...")
287 EccGlobalData.gDb.TblReport.ToCSV(self.ReportFile)
288 EdkLogger.quiet("Generating report done!")
289
290 def GetRealPathCase(self, path):
291 TmpPath = path.rstrip(os.sep)
292 PathParts = TmpPath.split(os.sep)
293 if len(PathParts) == 0:
294 return path
295 if len(PathParts) == 1:
296 if PathParts[0].strip().endswith(':'):
297 return PathParts[0].upper()
298 # Relative dir, list . current dir
299 Dirs = os.listdir('.')
300 for Dir in Dirs:
301 if Dir.upper() == PathParts[0].upper():
302 return Dir
303
304 if PathParts[0].strip().endswith(':'):
305 PathParts[0] = PathParts[0].upper()
306 ParentDir = PathParts[0]
307 RealPath = ParentDir
308 if PathParts[0] == '':
309 RealPath = os.sep
310 ParentDir = os.sep
311
312 PathParts.remove(PathParts[0]) # need to remove the parent
313 for Part in PathParts:
314 Dirs = os.listdir(ParentDir + os.sep)
315 for Dir in Dirs:
316 if Dir.upper() == Part.upper():
317 RealPath += os.sep
318 RealPath += Dir
319 break
320 ParentDir += os.sep
321 ParentDir += Dir
322
323 return RealPath
324
325 ## ParseOption
326 #
327 # Parse options
328 #
329 def ParseOption(self):
330 EdkLogger.quiet("Loading ECC configuration ... done")
331 (Options, Target) = self.EccOptionParser()
332
333 if Options.Workspace:
334 os.environ["WORKSPACE"] = Options.Workspace
335
336 # Check workspace envirnoment
337 if "WORKSPACE" not in os.environ:
338 EdkLogger.error("ECC", BuildToolError.ATTRIBUTE_NOT_AVAILABLE, "Environment variable not found",
339 ExtraData="WORKSPACE")
340 else:
341 EccGlobalData.gWorkspace = os.path.normpath(os.getenv("WORKSPACE"))
342 if not os.path.exists(EccGlobalData.gWorkspace):
343 EdkLogger.error("ECC", BuildToolError.FILE_NOT_FOUND, ExtraData="WORKSPACE = %s" % EccGlobalData.gWorkspace)
344 os.environ["WORKSPACE"] = EccGlobalData.gWorkspace
345 # Set log level
346 self.SetLogLevel(Options)
347
348 # Set other options
349 if Options.ConfigFile != None:
350 self.ConfigFile = Options.ConfigFile
351 if Options.OutputFile != None:
352 self.OutputFile = Options.OutputFile
353 if Options.ReportFile != None:
354 self.ReportFile = Options.ReportFile
355 if Options.ExceptionFile != None:
356 self.ExceptionFile = Options.ExceptionFile
357 if Options.Target != None:
358 if not os.path.isdir(Options.Target):
359 EdkLogger.error("ECC", BuildToolError.OPTION_VALUE_INVALID, ExtraData="Target [%s] does NOT exist" % Options.Target)
360 else:
361 EccGlobalData.gTarget = self.GetRealPathCase(os.path.normpath(Options.Target))
362 else:
363 EdkLogger.warn("Ecc", EdkLogger.ECC_ERROR, "The target source tree was not specified, using current WORKSPACE instead!")
364 EccGlobalData.gTarget = os.path.normpath(os.getenv("WORKSPACE"))
365 if Options.keepdatabase != None:
366 self.IsInit = False
367 if Options.metadata != None and Options.sourcecode != None:
368 EdkLogger.error("ECC", BuildToolError.OPTION_CONFLICT, ExtraData="-m and -s can't be specified at one time")
369 if Options.metadata != None:
370 self.ScanSourceCode = False
371 if Options.sourcecode != None:
372 self.ScanMetaData = False
373 if Options.folders != None:
374 self.OnlyScan = True
375
376 ## SetLogLevel
377 #
378 # Set current log level of the tool based on args
379 #
380 # @param Option: The option list including log level setting
381 #
382 def SetLogLevel(self, Option):
383 if Option.verbose != None:
384 EdkLogger.SetLevel(EdkLogger.VERBOSE)
385 elif Option.quiet != None:
386 EdkLogger.SetLevel(EdkLogger.QUIET)
387 elif Option.debug != None:
388 EdkLogger.SetLevel(Option.debug + 1)
389 else:
390 EdkLogger.SetLevel(EdkLogger.INFO)
391
392 ## Parse command line options
393 #
394 # Using standard Python module optparse to parse command line option of this tool.
395 #
396 # @retval Opt A optparse.Values object containing the parsed options
397 # @retval Args Target of build command
398 #
399 def EccOptionParser(self):
400 Parser = OptionParser(description = self.Copyright, version = self.Version, prog = "Ecc.exe", usage = "%prog [options]")
401 Parser.add_option("-t", "--target sourcepath", action="store", type="string", dest='Target',
402 help="Check all files under the target workspace.")
403 Parser.add_option("-c", "--config filename", action="store", type="string", dest="ConfigFile",
404 help="Specify a configuration file. Defaultly use config.ini under ECC tool directory.")
405 Parser.add_option("-o", "--outfile filename", action="store", type="string", dest="OutputFile",
406 help="Specify the name of an output file, if and only if one filename was specified.")
407 Parser.add_option("-r", "--reportfile filename", action="store", type="string", dest="ReportFile",
408 help="Specify the name of an report file, if and only if one filename was specified.")
409 Parser.add_option("-e", "--exceptionfile filename", action="store", type="string", dest="ExceptionFile",
410 help="Specify the name of an exception file, if and only if one filename was specified.")
411 Parser.add_option("-m", "--metadata", action="store_true", type=None, help="Only scan meta-data files information if this option is specified.")
412 Parser.add_option("-s", "--sourcecode", action="store_true", type=None, help="Only scan source code files information if this option is specified.")
413 Parser.add_option("-k", "--keepdatabase", action="store_true", type=None, help="The existing Ecc database will not be cleaned except report information if this option is specified.")
414 Parser.add_option("-l", "--log filename", action="store", dest="LogFile", help="""If specified, the tool should emit the changes that
415 were made by the tool after printing the result message.
416 If filename, the emit to the file, otherwise emit to
417 standard output. If no modifications were made, then do not
418 create a log file, or output a log message.""")
419 Parser.add_option("-q", "--quiet", action="store_true", type=None, help="Disable all messages except FATAL ERRORS.")
420 Parser.add_option("-v", "--verbose", action="store_true", type=None, help="Turn on verbose output with informational messages printed, "\
421 "including library instances selected, final dependency expression, "\
422 "and warning messages, etc.")
423 Parser.add_option("-d", "--debug", action="store", type="int", help="Enable debug messages at specified level.")
424 Parser.add_option("-w", "--workspace", action="store", type="string", dest='Workspace', help="Specify workspace.")
425 Parser.add_option("-f", "--folders", action="store_true", type=None, help="Only scanning specified folders which are recorded in config.ini file.")
426
427 (Opt, Args)=Parser.parse_args()
428
429 return (Opt, Args)
430
431 ##
432 #
433 # This acts like the main() function for the script, unless it is 'import'ed into another
434 # script.
435 #
436 if __name__ == '__main__':
437 # Initialize log system
438 EdkLogger.Initialize()
439 EdkLogger.IsRaiseError = False
440 EdkLogger.quiet(time.strftime("%H:%M:%S, %b.%d %Y ", time.localtime()) + "[00:00]" + "\n")
441
442 StartTime = time.clock()
443 Ecc = Ecc()
444 FinishTime = time.clock()
445
446 BuildDuration = time.strftime("%M:%S", time.gmtime(int(round(FinishTime - StartTime))))
447 EdkLogger.quiet("\n%s [%s]" % (time.strftime("%H:%M:%S, %b.%d %Y", time.localtime()), BuildDuration))