BaseTools: Use absolute import in Ecc
[mirror_edk2.git] / BaseTools / Source / Python / Ecc / Ecc.py
1 ## @file
2 # This file is used to be the main entrance of ECC tool
3 #
4 # Copyright (c) 2009 - 2018, Intel Corporation. All rights reserved.<BR>
5 # This program and the accompanying materials
6 # are licensed and made available under the terms and conditions of the BSD License
7 # which accompanies this distribution. The full text of the license may be found at
8 # http://opensource.org/licenses/bsd-license.php
9 #
10 # THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS,
11 # WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED.
12 #
13
14 ##
15 # Import Modules
16 #
17 from __future__ import absolute_import
18 import Common.LongFilePathOs as os, time, glob, sys
19 import Common.EdkLogger as EdkLogger
20 from . import Database
21 from . import EccGlobalData
22 from .MetaDataParser import *
23 from optparse import OptionParser
24 from .Configuration import Configuration
25 from .Check import Check
26 import Common.GlobalData as GlobalData
27
28 from Common.StringUtils import NormPath
29 from Common.BuildVersion import gBUILD_VERSION
30 from Common import BuildToolError
31 from Common.Misc import PathClass
32 from Common.Misc import DirCache
33 from .MetaFileWorkspace.MetaFileParser import DscParser
34 from .MetaFileWorkspace.MetaFileParser import DecParser
35 from .MetaFileWorkspace.MetaFileParser import InfParser
36 from .MetaFileWorkspace.MetaFileParser import Fdf
37 from .MetaFileWorkspace.MetaFileTable import MetaFileStorage
38 from . import c
39 import re, string
40 from .Exception import *
41 from Common.LongFilePathSupport import OpenLongFilePath as open
42 from Common.MultipleWorkspace import MultipleWorkspace as mws
43
44 ## Ecc
45 #
46 # This class is used to define Ecc main entrance
47 #
48 # @param object: Inherited from object class
49 #
50 class Ecc(object):
51 def __init__(self):
52 # Version and Copyright
53 self.VersionNumber = ("1.0" + " Build " + gBUILD_VERSION)
54 self.Version = "%prog Version " + self.VersionNumber
55 self.Copyright = "Copyright (c) 2009 - 2018, Intel Corporation All rights reserved."
56
57 self.InitDefaultConfigIni()
58 self.OutputFile = 'output.txt'
59 self.ReportFile = 'Report.csv'
60 self.ExceptionFile = 'exception.xml'
61 self.IsInit = True
62 self.ScanSourceCode = True
63 self.ScanMetaData = True
64 self.MetaFile = ''
65 self.OnlyScan = None
66
67 # Parse the options and args
68 self.ParseOption()
69 EdkLogger.info(time.strftime("%H:%M:%S, %b.%d %Y ", time.localtime()) + "[00:00]" + "\n")
70
71 #
72 # Check EFI_SOURCE (Edk build convention). EDK_SOURCE will always point to ECP
73 #
74 WorkspaceDir = os.path.normcase(os.path.normpath(os.environ["WORKSPACE"]))
75 os.environ["WORKSPACE"] = WorkspaceDir
76
77 # set multiple workspace
78 PackagesPath = os.getenv("PACKAGES_PATH")
79 mws.setWs(WorkspaceDir, PackagesPath)
80
81 if "ECP_SOURCE" not in os.environ:
82 os.environ["ECP_SOURCE"] = mws.join(WorkspaceDir, GlobalData.gEdkCompatibilityPkg)
83 if "EFI_SOURCE" not in os.environ:
84 os.environ["EFI_SOURCE"] = os.environ["ECP_SOURCE"]
85 if "EDK_SOURCE" not in os.environ:
86 os.environ["EDK_SOURCE"] = os.environ["ECP_SOURCE"]
87
88 #
89 # Unify case of characters on case-insensitive systems
90 #
91 EfiSourceDir = os.path.normcase(os.path.normpath(os.environ["EFI_SOURCE"]))
92 EdkSourceDir = os.path.normcase(os.path.normpath(os.environ["EDK_SOURCE"]))
93 EcpSourceDir = os.path.normcase(os.path.normpath(os.environ["ECP_SOURCE"]))
94
95 os.environ["EFI_SOURCE"] = EfiSourceDir
96 os.environ["EDK_SOURCE"] = EdkSourceDir
97 os.environ["ECP_SOURCE"] = EcpSourceDir
98
99 GlobalData.gWorkspace = WorkspaceDir
100 GlobalData.gEfiSource = EfiSourceDir
101 GlobalData.gEdkSource = EdkSourceDir
102 GlobalData.gEcpSource = EcpSourceDir
103
104 GlobalData.gGlobalDefines["WORKSPACE"] = WorkspaceDir
105 GlobalData.gGlobalDefines["EFI_SOURCE"] = EfiSourceDir
106 GlobalData.gGlobalDefines["EDK_SOURCE"] = EdkSourceDir
107 GlobalData.gGlobalDefines["ECP_SOURCE"] = EcpSourceDir
108
109 EdkLogger.info("Loading ECC configuration ... done")
110 # Generate checkpoints list
111 EccGlobalData.gConfig = Configuration(self.ConfigFile)
112
113 # Generate exception list
114 EccGlobalData.gException = ExceptionCheck(self.ExceptionFile)
115
116 # Init Ecc database
117 EccGlobalData.gDb = Database.Database(Database.DATABASE_PATH)
118 EccGlobalData.gDb.InitDatabase(self.IsInit)
119
120 #
121 # Get files real name in workspace dir
122 #
123 GlobalData.gAllFiles = DirCache(GlobalData.gWorkspace)
124
125 # Build ECC database
126 # self.BuildDatabase()
127 self.DetectOnlyScanDirs()
128
129 # Start to check
130 self.Check()
131
132 # Show report
133 self.GenReport()
134
135 # Close Database
136 EccGlobalData.gDb.Close()
137
138 def InitDefaultConfigIni(self):
139 paths = map(lambda p: os.path.join(p, 'Ecc', 'config.ini'), sys.path)
140 paths = (os.path.realpath('config.ini'),) + tuple(paths)
141 for path in paths:
142 if os.path.exists(path):
143 self.ConfigFile = path
144 return
145 self.ConfigFile = 'config.ini'
146
147
148 ## DetectOnlyScan
149 #
150 # Detect whether only scanned folders have been enabled
151 #
152 def DetectOnlyScanDirs(self):
153 if self.OnlyScan == True:
154 OnlyScanDirs = []
155 # Use regex here if multiple spaces or TAB exists in ScanOnlyDirList in config.ini file
156 for folder in re.finditer(r'\S+', EccGlobalData.gConfig.ScanOnlyDirList):
157 OnlyScanDirs.append(folder.group())
158 if len(OnlyScanDirs) != 0:
159 self.BuildDatabase(OnlyScanDirs)
160 else:
161 EdkLogger.error("ECC", BuildToolError.OPTION_VALUE_INVALID, ExtraData="Use -f option need to fill specific folders in config.ini file")
162 else:
163 self.BuildDatabase()
164
165
166 ## BuildDatabase
167 #
168 # Build the database for target
169 #
170 def BuildDatabase(self, SpeciDirs = None):
171 # Clean report table
172 EccGlobalData.gDb.TblReport.Drop()
173 EccGlobalData.gDb.TblReport.Create()
174
175 # Build database
176 if self.IsInit:
177 if self.ScanMetaData:
178 EdkLogger.quiet("Building database for Meta Data File ...")
179 self.BuildMetaDataFileDatabase(SpeciDirs)
180 if self.ScanSourceCode:
181 EdkLogger.quiet("Building database for Meta Data File Done!")
182 if SpeciDirs is None:
183 c.CollectSourceCodeDataIntoDB(EccGlobalData.gTarget)
184 else:
185 for specificDir in SpeciDirs:
186 c.CollectSourceCodeDataIntoDB(os.path.join(EccGlobalData.gTarget, specificDir))
187
188 EccGlobalData.gIdentifierTableList = GetTableList((MODEL_FILE_C, MODEL_FILE_H), 'Identifier', EccGlobalData.gDb)
189 EccGlobalData.gCFileList = GetFileList(MODEL_FILE_C, EccGlobalData.gDb)
190 EccGlobalData.gHFileList = GetFileList(MODEL_FILE_H, EccGlobalData.gDb)
191 EccGlobalData.gUFileList = GetFileList(MODEL_FILE_UNI, EccGlobalData.gDb)
192
193 ## BuildMetaDataFileDatabase
194 #
195 # Build the database for meta data files
196 #
197 def BuildMetaDataFileDatabase(self, SpecificDirs = None):
198 ScanFolders = []
199 if SpecificDirs is None:
200 ScanFolders.append(EccGlobalData.gTarget)
201 else:
202 for specificDir in SpecificDirs:
203 ScanFolders.append(os.path.join(EccGlobalData.gTarget, specificDir))
204 EdkLogger.quiet("Building database for meta data files ...")
205 Op = open(EccGlobalData.gConfig.MetaDataFileCheckPathOfGenerateFileList, 'w+')
206 #SkipDirs = Read from config file
207 SkipDirs = EccGlobalData.gConfig.SkipDirList
208 SkipDirString = string.join(SkipDirs, '|')
209 # p = re.compile(r'.*[\\/](?:%s)[\\/]?.*' % SkipDirString)
210 p = re.compile(r'.*[\\/](?:%s^\S)[\\/]?.*' % SkipDirString)
211 for scanFolder in ScanFolders:
212 for Root, Dirs, Files in os.walk(scanFolder):
213 if p.match(Root.upper()):
214 continue
215 for Dir in Dirs:
216 Dirname = os.path.join(Root, Dir)
217 if os.path.islink(Dirname):
218 Dirname = os.path.realpath(Dirname)
219 if os.path.isdir(Dirname):
220 # symlinks to directories are treated as directories
221 Dirs.remove(Dir)
222 Dirs.append(Dirname)
223
224 for File in Files:
225 if len(File) > 4 and File[-4:].upper() == ".DEC":
226 Filename = os.path.normpath(os.path.join(Root, File))
227 EdkLogger.quiet("Parsing %s" % Filename)
228 Op.write("%s\r" % Filename)
229 #Dec(Filename, True, True, EccGlobalData.gWorkspace, EccGlobalData.gDb)
230 self.MetaFile = DecParser(Filename, MODEL_FILE_DEC, EccGlobalData.gDb.TblDec)
231 self.MetaFile.Start()
232 continue
233 if len(File) > 4 and File[-4:].upper() == ".DSC":
234 Filename = os.path.normpath(os.path.join(Root, File))
235 EdkLogger.quiet("Parsing %s" % Filename)
236 Op.write("%s\r" % Filename)
237 #Dsc(Filename, True, True, EccGlobalData.gWorkspace, EccGlobalData.gDb)
238 self.MetaFile = DscParser(PathClass(Filename, Root), MODEL_FILE_DSC, MetaFileStorage(EccGlobalData.gDb.TblDsc.Cur, Filename, MODEL_FILE_DSC, True))
239 # alwasy do post-process, in case of macros change
240 self.MetaFile.DoPostProcess()
241 self.MetaFile.Start()
242 self.MetaFile._PostProcess()
243 continue
244 if len(File) > 4 and File[-4:].upper() == ".INF":
245 Filename = os.path.normpath(os.path.join(Root, File))
246 EdkLogger.quiet("Parsing %s" % Filename)
247 Op.write("%s\r" % Filename)
248 #Inf(Filename, True, True, EccGlobalData.gWorkspace, EccGlobalData.gDb)
249 self.MetaFile = InfParser(Filename, MODEL_FILE_INF, EccGlobalData.gDb.TblInf)
250 self.MetaFile.Start()
251 continue
252 if len(File) > 4 and File[-4:].upper() == ".FDF":
253 Filename = os.path.normpath(os.path.join(Root, File))
254 EdkLogger.quiet("Parsing %s" % Filename)
255 Op.write("%s\r" % Filename)
256 Fdf(Filename, True, EccGlobalData.gWorkspace, EccGlobalData.gDb)
257 continue
258 if len(File) > 4 and File[-4:].upper() == ".UNI":
259 Filename = os.path.normpath(os.path.join(Root, File))
260 EdkLogger.quiet("Parsing %s" % Filename)
261 Op.write("%s\r" % Filename)
262 FileID = EccGlobalData.gDb.TblFile.InsertFile(Filename, MODEL_FILE_UNI)
263 EccGlobalData.gDb.TblReport.UpdateBelongsToItemByFile(FileID, File)
264 continue
265
266 Op.close()
267
268 # Commit to database
269 EccGlobalData.gDb.Conn.commit()
270
271 EdkLogger.quiet("Building database for meta data files done!")
272
273 ##
274 #
275 # Check each checkpoint
276 #
277 def Check(self):
278 EdkLogger.quiet("Checking ...")
279 EccCheck = Check()
280 EccCheck.Check()
281 EdkLogger.quiet("Checking done!")
282
283 ##
284 #
285 # Generate the scan report
286 #
287 def GenReport(self):
288 EdkLogger.quiet("Generating report ...")
289 EccGlobalData.gDb.TblReport.ToCSV(self.ReportFile)
290 EdkLogger.quiet("Generating report done!")
291
292 def GetRealPathCase(self, path):
293 TmpPath = path.rstrip(os.sep)
294 PathParts = TmpPath.split(os.sep)
295 if len(PathParts) == 0:
296 return path
297 if len(PathParts) == 1:
298 if PathParts[0].strip().endswith(':'):
299 return PathParts[0].upper()
300 # Relative dir, list . current dir
301 Dirs = os.listdir('.')
302 for Dir in Dirs:
303 if Dir.upper() == PathParts[0].upper():
304 return Dir
305
306 if PathParts[0].strip().endswith(':'):
307 PathParts[0] = PathParts[0].upper()
308 ParentDir = PathParts[0]
309 RealPath = ParentDir
310 if PathParts[0] == '':
311 RealPath = os.sep
312 ParentDir = os.sep
313
314 PathParts.remove(PathParts[0]) # need to remove the parent
315 for Part in PathParts:
316 Dirs = os.listdir(ParentDir + os.sep)
317 for Dir in Dirs:
318 if Dir.upper() == Part.upper():
319 RealPath += os.sep
320 RealPath += Dir
321 break
322 ParentDir += os.sep
323 ParentDir += Dir
324
325 return RealPath
326
327 ## ParseOption
328 #
329 # Parse options
330 #
331 def ParseOption(self):
332 (Options, Target) = self.EccOptionParser()
333
334 if Options.Workspace:
335 os.environ["WORKSPACE"] = Options.Workspace
336
337 # Check workspace envirnoment
338 if "WORKSPACE" not in os.environ:
339 EdkLogger.error("ECC", BuildToolError.ATTRIBUTE_NOT_AVAILABLE, "Environment variable not found",
340 ExtraData="WORKSPACE")
341 else:
342 EccGlobalData.gWorkspace = os.path.normpath(os.getenv("WORKSPACE"))
343 if not os.path.exists(EccGlobalData.gWorkspace):
344 EdkLogger.error("ECC", BuildToolError.FILE_NOT_FOUND, ExtraData="WORKSPACE = %s" % EccGlobalData.gWorkspace)
345 os.environ["WORKSPACE"] = EccGlobalData.gWorkspace
346 # Set log level
347 self.SetLogLevel(Options)
348
349 # Set other options
350 if Options.ConfigFile is not None:
351 self.ConfigFile = Options.ConfigFile
352 if Options.OutputFile is not None:
353 self.OutputFile = Options.OutputFile
354 if Options.ReportFile is not None:
355 self.ReportFile = Options.ReportFile
356 if Options.ExceptionFile is not None:
357 self.ExceptionFile = Options.ExceptionFile
358 if Options.Target is not None:
359 if not os.path.isdir(Options.Target):
360 EdkLogger.error("ECC", BuildToolError.OPTION_VALUE_INVALID, ExtraData="Target [%s] does NOT exist" % Options.Target)
361 else:
362 EccGlobalData.gTarget = self.GetRealPathCase(os.path.normpath(Options.Target))
363 else:
364 EdkLogger.warn("Ecc", EdkLogger.ECC_ERROR, "The target source tree was not specified, using current WORKSPACE instead!")
365 EccGlobalData.gTarget = os.path.normpath(os.getenv("WORKSPACE"))
366 if Options.keepdatabase is not None:
367 self.IsInit = False
368 if Options.metadata is not None and Options.sourcecode is not None:
369 EdkLogger.error("ECC", BuildToolError.OPTION_CONFLICT, ExtraData="-m and -s can't be specified at one time")
370 if Options.metadata is not None:
371 self.ScanSourceCode = False
372 if Options.sourcecode is not None:
373 self.ScanMetaData = False
374 if Options.folders is not None:
375 self.OnlyScan = True
376
377 ## SetLogLevel
378 #
379 # Set current log level of the tool based on args
380 #
381 # @param Option: The option list including log level setting
382 #
383 def SetLogLevel(self, Option):
384 if Option.verbose is not None:
385 EdkLogger.SetLevel(EdkLogger.VERBOSE)
386 elif Option.quiet is not None:
387 EdkLogger.SetLevel(EdkLogger.QUIET)
388 elif Option.debug is not None:
389 EdkLogger.SetLevel(Option.debug + 1)
390 else:
391 EdkLogger.SetLevel(EdkLogger.INFO)
392
393 ## Parse command line options
394 #
395 # Using standard Python module optparse to parse command line option of this tool.
396 #
397 # @retval Opt A optparse.Values object containing the parsed options
398 # @retval Args Target of build command
399 #
400 def EccOptionParser(self):
401 Parser = OptionParser(description = self.Copyright, version = self.Version, prog = "Ecc.exe", usage = "%prog [options]")
402 Parser.add_option("-t", "--target sourcepath", action="store", type="string", dest='Target',
403 help="Check all files under the target workspace.")
404 Parser.add_option("-c", "--config filename", action="store", type="string", dest="ConfigFile",
405 help="Specify a configuration file. Defaultly use config.ini under ECC tool directory.")
406 Parser.add_option("-o", "--outfile filename", action="store", type="string", dest="OutputFile",
407 help="Specify the name of an output file, if and only if one filename was specified.")
408 Parser.add_option("-r", "--reportfile filename", action="store", type="string", dest="ReportFile",
409 help="Specify the name of an report file, if and only if one filename was specified.")
410 Parser.add_option("-e", "--exceptionfile filename", action="store", type="string", dest="ExceptionFile",
411 help="Specify the name of an exception file, if and only if one filename was specified.")
412 Parser.add_option("-m", "--metadata", action="store_true", type=None, help="Only scan meta-data files information if this option is specified.")
413 Parser.add_option("-s", "--sourcecode", action="store_true", type=None, help="Only scan source code files information if this option is specified.")
414 Parser.add_option("-k", "--keepdatabase", action="store_true", type=None, help="The existing Ecc database will not be cleaned except report information if this option is specified.")
415 Parser.add_option("-l", "--log filename", action="store", dest="LogFile", help="""If specified, the tool should emit the changes that
416 were made by the tool after printing the result message.
417 If filename, the emit to the file, otherwise emit to
418 standard output. If no modifications were made, then do not
419 create a log file, or output a log message.""")
420 Parser.add_option("-q", "--quiet", action="store_true", type=None, help="Disable all messages except FATAL ERRORS.")
421 Parser.add_option("-v", "--verbose", action="store_true", type=None, help="Turn on verbose output with informational messages printed, "\
422 "including library instances selected, final dependency expression, "\
423 "and warning messages, etc.")
424 Parser.add_option("-d", "--debug", action="store", type="int", help="Enable debug messages at specified level.")
425 Parser.add_option("-w", "--workspace", action="store", type="string", dest='Workspace', help="Specify workspace.")
426 Parser.add_option("-f", "--folders", action="store_true", type=None, help="Only scanning specified folders which are recorded in config.ini file.")
427
428 (Opt, Args)=Parser.parse_args()
429
430 return (Opt, Args)
431
432 ##
433 #
434 # This acts like the main() function for the script, unless it is 'import'ed into another
435 # script.
436 #
437 if __name__ == '__main__':
438 # Initialize log system
439 EdkLogger.Initialize()
440 EdkLogger.IsRaiseError = False
441
442 StartTime = time.clock()
443 Ecc = Ecc()
444 FinishTime = time.clock()
445
446 BuildDuration = time.strftime("%M:%S", time.gmtime(int(round(FinishTime - StartTime))))
447 EdkLogger.quiet("\n%s [%s]" % (time.strftime("%H:%M:%S, %b.%d %Y", time.localtime()), BuildDuration))