ccd563a4660e87f9da6b9a131c37f855b7134d4a
[mirror_edk2.git] / BaseTools / Source / Python / Ecc / Ecc.py
1 ## @file
2 # This file is used to be the main entrance of ECC tool
3 #
4 # Copyright (c) 2009 - 2018, Intel Corporation. All rights reserved.<BR>
5 # This program and the accompanying materials
6 # are licensed and made available under the terms and conditions of the BSD License
7 # which accompanies this distribution. The full text of the license may be found at
8 # http://opensource.org/licenses/bsd-license.php
9 #
10 # THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS,
11 # WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED.
12 #
13
14 ##
15 # Import Modules
16 #
17 import Common.LongFilePathOs as os, time, glob, sys
18 import Common.EdkLogger as EdkLogger
19 import Database
20 import EccGlobalData
21 from MetaDataParser import *
22 from optparse import OptionParser
23 from Configuration import Configuration
24 from Check import Check
25 import Common.GlobalData as GlobalData
26
27 from Common.StringUtils import NormPath
28 from Common.BuildVersion import gBUILD_VERSION
29 from Common import BuildToolError
30 from Common.Misc import PathClass
31 from Common.Misc import DirCache
32 from MetaFileWorkspace.MetaFileParser import DscParser
33 from MetaFileWorkspace.MetaFileParser import DecParser
34 from MetaFileWorkspace.MetaFileParser import InfParser
35 from MetaFileWorkspace.MetaFileParser import Fdf
36 from MetaFileWorkspace.MetaFileTable import MetaFileStorage
37 import c
38 import re, string
39 from Exception import *
40 from Common.LongFilePathSupport import OpenLongFilePath as open
41 from Common.MultipleWorkspace import MultipleWorkspace as mws
42
43 ## Ecc
44 #
45 # This class is used to define Ecc main entrance
46 #
47 # @param object: Inherited from object class
48 #
49 class Ecc(object):
50 def __init__(self):
51 # Version and Copyright
52 self.VersionNumber = ("1.0" + " Build " + gBUILD_VERSION)
53 self.Version = "%prog Version " + self.VersionNumber
54 self.Copyright = "Copyright (c) 2009 - 2018, Intel Corporation All rights reserved."
55
56 self.InitDefaultConfigIni()
57 self.OutputFile = 'output.txt'
58 self.ReportFile = 'Report.csv'
59 self.ExceptionFile = 'exception.xml'
60 self.IsInit = True
61 self.ScanSourceCode = True
62 self.ScanMetaData = True
63 self.MetaFile = ''
64 self.OnlyScan = None
65
66 # Parse the options and args
67 self.ParseOption()
68 EdkLogger.info(time.strftime("%H:%M:%S, %b.%d %Y ", time.localtime()) + "[00:00]" + "\n")
69
70 #
71 # Check EFI_SOURCE (Edk build convention). EDK_SOURCE will always point to ECP
72 #
73 WorkspaceDir = os.path.normcase(os.path.normpath(os.environ["WORKSPACE"]))
74 os.environ["WORKSPACE"] = WorkspaceDir
75
76 # set multiple workspace
77 PackagesPath = os.getenv("PACKAGES_PATH")
78 mws.setWs(WorkspaceDir, PackagesPath)
79
80 if "ECP_SOURCE" not in os.environ:
81 os.environ["ECP_SOURCE"] = mws.join(WorkspaceDir, GlobalData.gEdkCompatibilityPkg)
82 if "EFI_SOURCE" not in os.environ:
83 os.environ["EFI_SOURCE"] = os.environ["ECP_SOURCE"]
84 if "EDK_SOURCE" not in os.environ:
85 os.environ["EDK_SOURCE"] = os.environ["ECP_SOURCE"]
86
87 #
88 # Unify case of characters on case-insensitive systems
89 #
90 EfiSourceDir = os.path.normcase(os.path.normpath(os.environ["EFI_SOURCE"]))
91 EdkSourceDir = os.path.normcase(os.path.normpath(os.environ["EDK_SOURCE"]))
92 EcpSourceDir = os.path.normcase(os.path.normpath(os.environ["ECP_SOURCE"]))
93
94 os.environ["EFI_SOURCE"] = EfiSourceDir
95 os.environ["EDK_SOURCE"] = EdkSourceDir
96 os.environ["ECP_SOURCE"] = EcpSourceDir
97
98 GlobalData.gWorkspace = WorkspaceDir
99 GlobalData.gEfiSource = EfiSourceDir
100 GlobalData.gEdkSource = EdkSourceDir
101 GlobalData.gEcpSource = EcpSourceDir
102
103 GlobalData.gGlobalDefines["WORKSPACE"] = WorkspaceDir
104 GlobalData.gGlobalDefines["EFI_SOURCE"] = EfiSourceDir
105 GlobalData.gGlobalDefines["EDK_SOURCE"] = EdkSourceDir
106 GlobalData.gGlobalDefines["ECP_SOURCE"] = EcpSourceDir
107
108 EdkLogger.info("Loading ECC configuration ... done")
109 # Generate checkpoints list
110 EccGlobalData.gConfig = Configuration(self.ConfigFile)
111
112 # Generate exception list
113 EccGlobalData.gException = ExceptionCheck(self.ExceptionFile)
114
115 # Init Ecc database
116 EccGlobalData.gDb = Database.Database(Database.DATABASE_PATH)
117 EccGlobalData.gDb.InitDatabase(self.IsInit)
118
119 #
120 # Get files real name in workspace dir
121 #
122 GlobalData.gAllFiles = DirCache(GlobalData.gWorkspace)
123
124 # Build ECC database
125 # self.BuildDatabase()
126 self.DetectOnlyScanDirs()
127
128 # Start to check
129 self.Check()
130
131 # Show report
132 self.GenReport()
133
134 # Close Database
135 EccGlobalData.gDb.Close()
136
137 def InitDefaultConfigIni(self):
138 paths = map(lambda p: os.path.join(p, 'Ecc', 'config.ini'), sys.path)
139 paths = (os.path.realpath('config.ini'),) + tuple(paths)
140 for path in paths:
141 if os.path.exists(path):
142 self.ConfigFile = path
143 return
144 self.ConfigFile = 'config.ini'
145
146
147 ## DetectOnlyScan
148 #
149 # Detect whether only scanned folders have been enabled
150 #
151 def DetectOnlyScanDirs(self):
152 if self.OnlyScan == True:
153 OnlyScanDirs = []
154 # Use regex here if multiple spaces or TAB exists in ScanOnlyDirList in config.ini file
155 for folder in re.finditer(r'\S+', EccGlobalData.gConfig.ScanOnlyDirList):
156 OnlyScanDirs.append(folder.group())
157 if len(OnlyScanDirs) != 0:
158 self.BuildDatabase(OnlyScanDirs)
159 else:
160 EdkLogger.error("ECC", BuildToolError.OPTION_VALUE_INVALID, ExtraData="Use -f option need to fill specific folders in config.ini file")
161 else:
162 self.BuildDatabase()
163
164
165 ## BuildDatabase
166 #
167 # Build the database for target
168 #
169 def BuildDatabase(self, SpeciDirs = None):
170 # Clean report table
171 EccGlobalData.gDb.TblReport.Drop()
172 EccGlobalData.gDb.TblReport.Create()
173
174 # Build database
175 if self.IsInit:
176 if self.ScanMetaData:
177 EdkLogger.quiet("Building database for Meta Data File ...")
178 self.BuildMetaDataFileDatabase(SpeciDirs)
179 if self.ScanSourceCode:
180 EdkLogger.quiet("Building database for Meta Data File Done!")
181 if SpeciDirs is None:
182 c.CollectSourceCodeDataIntoDB(EccGlobalData.gTarget)
183 else:
184 for specificDir in SpeciDirs:
185 c.CollectSourceCodeDataIntoDB(os.path.join(EccGlobalData.gTarget, specificDir))
186
187 EccGlobalData.gIdentifierTableList = GetTableList((MODEL_FILE_C, MODEL_FILE_H), 'Identifier', EccGlobalData.gDb)
188 EccGlobalData.gCFileList = GetFileList(MODEL_FILE_C, EccGlobalData.gDb)
189 EccGlobalData.gHFileList = GetFileList(MODEL_FILE_H, EccGlobalData.gDb)
190 EccGlobalData.gUFileList = GetFileList(MODEL_FILE_UNI, EccGlobalData.gDb)
191
192 ## BuildMetaDataFileDatabase
193 #
194 # Build the database for meta data files
195 #
196 def BuildMetaDataFileDatabase(self, SpecificDirs = None):
197 ScanFolders = []
198 if SpecificDirs is None:
199 ScanFolders.append(EccGlobalData.gTarget)
200 else:
201 for specificDir in SpecificDirs:
202 ScanFolders.append(os.path.join(EccGlobalData.gTarget, specificDir))
203 EdkLogger.quiet("Building database for meta data files ...")
204 Op = open(EccGlobalData.gConfig.MetaDataFileCheckPathOfGenerateFileList, 'w+')
205 #SkipDirs = Read from config file
206 SkipDirs = EccGlobalData.gConfig.SkipDirList
207 SkipDirString = string.join(SkipDirs, '|')
208 # p = re.compile(r'.*[\\/](?:%s)[\\/]?.*' % SkipDirString)
209 p = re.compile(r'.*[\\/](?:%s^\S)[\\/]?.*' % SkipDirString)
210 for scanFolder in ScanFolders:
211 for Root, Dirs, Files in os.walk(scanFolder):
212 if p.match(Root.upper()):
213 continue
214 for Dir in Dirs:
215 Dirname = os.path.join(Root, Dir)
216 if os.path.islink(Dirname):
217 Dirname = os.path.realpath(Dirname)
218 if os.path.isdir(Dirname):
219 # symlinks to directories are treated as directories
220 Dirs.remove(Dir)
221 Dirs.append(Dirname)
222
223 for File in Files:
224 if len(File) > 4 and File[-4:].upper() == ".DEC":
225 Filename = os.path.normpath(os.path.join(Root, File))
226 EdkLogger.quiet("Parsing %s" % Filename)
227 Op.write("%s\r" % Filename)
228 #Dec(Filename, True, True, EccGlobalData.gWorkspace, EccGlobalData.gDb)
229 self.MetaFile = DecParser(Filename, MODEL_FILE_DEC, EccGlobalData.gDb.TblDec)
230 self.MetaFile.Start()
231 continue
232 if len(File) > 4 and File[-4:].upper() == ".DSC":
233 Filename = os.path.normpath(os.path.join(Root, File))
234 EdkLogger.quiet("Parsing %s" % Filename)
235 Op.write("%s\r" % Filename)
236 #Dsc(Filename, True, True, EccGlobalData.gWorkspace, EccGlobalData.gDb)
237 self.MetaFile = DscParser(PathClass(Filename, Root), MODEL_FILE_DSC, MetaFileStorage(EccGlobalData.gDb.TblDsc.Cur, Filename, MODEL_FILE_DSC, True))
238 # alwasy do post-process, in case of macros change
239 self.MetaFile.DoPostProcess()
240 self.MetaFile.Start()
241 self.MetaFile._PostProcess()
242 continue
243 if len(File) > 4 and File[-4:].upper() == ".INF":
244 Filename = os.path.normpath(os.path.join(Root, File))
245 EdkLogger.quiet("Parsing %s" % Filename)
246 Op.write("%s\r" % Filename)
247 #Inf(Filename, True, True, EccGlobalData.gWorkspace, EccGlobalData.gDb)
248 self.MetaFile = InfParser(Filename, MODEL_FILE_INF, EccGlobalData.gDb.TblInf)
249 self.MetaFile.Start()
250 continue
251 if len(File) > 4 and File[-4:].upper() == ".FDF":
252 Filename = os.path.normpath(os.path.join(Root, File))
253 EdkLogger.quiet("Parsing %s" % Filename)
254 Op.write("%s\r" % Filename)
255 Fdf(Filename, True, EccGlobalData.gWorkspace, EccGlobalData.gDb)
256 continue
257 if len(File) > 4 and File[-4:].upper() == ".UNI":
258 Filename = os.path.normpath(os.path.join(Root, File))
259 EdkLogger.quiet("Parsing %s" % Filename)
260 Op.write("%s\r" % Filename)
261 FileID = EccGlobalData.gDb.TblFile.InsertFile(Filename, MODEL_FILE_UNI)
262 EccGlobalData.gDb.TblReport.UpdateBelongsToItemByFile(FileID, File)
263 continue
264
265 Op.close()
266
267 # Commit to database
268 EccGlobalData.gDb.Conn.commit()
269
270 EdkLogger.quiet("Building database for meta data files done!")
271
272 ##
273 #
274 # Check each checkpoint
275 #
276 def Check(self):
277 EdkLogger.quiet("Checking ...")
278 EccCheck = Check()
279 EccCheck.Check()
280 EdkLogger.quiet("Checking done!")
281
282 ##
283 #
284 # Generate the scan report
285 #
286 def GenReport(self):
287 EdkLogger.quiet("Generating report ...")
288 EccGlobalData.gDb.TblReport.ToCSV(self.ReportFile)
289 EdkLogger.quiet("Generating report done!")
290
291 def GetRealPathCase(self, path):
292 TmpPath = path.rstrip(os.sep)
293 PathParts = TmpPath.split(os.sep)
294 if len(PathParts) == 0:
295 return path
296 if len(PathParts) == 1:
297 if PathParts[0].strip().endswith(':'):
298 return PathParts[0].upper()
299 # Relative dir, list . current dir
300 Dirs = os.listdir('.')
301 for Dir in Dirs:
302 if Dir.upper() == PathParts[0].upper():
303 return Dir
304
305 if PathParts[0].strip().endswith(':'):
306 PathParts[0] = PathParts[0].upper()
307 ParentDir = PathParts[0]
308 RealPath = ParentDir
309 if PathParts[0] == '':
310 RealPath = os.sep
311 ParentDir = os.sep
312
313 PathParts.remove(PathParts[0]) # need to remove the parent
314 for Part in PathParts:
315 Dirs = os.listdir(ParentDir + os.sep)
316 for Dir in Dirs:
317 if Dir.upper() == Part.upper():
318 RealPath += os.sep
319 RealPath += Dir
320 break
321 ParentDir += os.sep
322 ParentDir += Dir
323
324 return RealPath
325
326 ## ParseOption
327 #
328 # Parse options
329 #
330 def ParseOption(self):
331 (Options, Target) = self.EccOptionParser()
332
333 if Options.Workspace:
334 os.environ["WORKSPACE"] = Options.Workspace
335
336 # Check workspace envirnoment
337 if "WORKSPACE" not in os.environ:
338 EdkLogger.error("ECC", BuildToolError.ATTRIBUTE_NOT_AVAILABLE, "Environment variable not found",
339 ExtraData="WORKSPACE")
340 else:
341 EccGlobalData.gWorkspace = os.path.normpath(os.getenv("WORKSPACE"))
342 if not os.path.exists(EccGlobalData.gWorkspace):
343 EdkLogger.error("ECC", BuildToolError.FILE_NOT_FOUND, ExtraData="WORKSPACE = %s" % EccGlobalData.gWorkspace)
344 os.environ["WORKSPACE"] = EccGlobalData.gWorkspace
345 # Set log level
346 self.SetLogLevel(Options)
347
348 # Set other options
349 if Options.ConfigFile is not None:
350 self.ConfigFile = Options.ConfigFile
351 if Options.OutputFile is not None:
352 self.OutputFile = Options.OutputFile
353 if Options.ReportFile is not None:
354 self.ReportFile = Options.ReportFile
355 if Options.ExceptionFile is not None:
356 self.ExceptionFile = Options.ExceptionFile
357 if Options.Target is not None:
358 if not os.path.isdir(Options.Target):
359 EdkLogger.error("ECC", BuildToolError.OPTION_VALUE_INVALID, ExtraData="Target [%s] does NOT exist" % Options.Target)
360 else:
361 EccGlobalData.gTarget = self.GetRealPathCase(os.path.normpath(Options.Target))
362 else:
363 EdkLogger.warn("Ecc", EdkLogger.ECC_ERROR, "The target source tree was not specified, using current WORKSPACE instead!")
364 EccGlobalData.gTarget = os.path.normpath(os.getenv("WORKSPACE"))
365 if Options.keepdatabase is not None:
366 self.IsInit = False
367 if Options.metadata is not None and Options.sourcecode is not None:
368 EdkLogger.error("ECC", BuildToolError.OPTION_CONFLICT, ExtraData="-m and -s can't be specified at one time")
369 if Options.metadata is not None:
370 self.ScanSourceCode = False
371 if Options.sourcecode is not None:
372 self.ScanMetaData = False
373 if Options.folders is not None:
374 self.OnlyScan = True
375
376 ## SetLogLevel
377 #
378 # Set current log level of the tool based on args
379 #
380 # @param Option: The option list including log level setting
381 #
382 def SetLogLevel(self, Option):
383 if Option.verbose is not None:
384 EdkLogger.SetLevel(EdkLogger.VERBOSE)
385 elif Option.quiet is not None:
386 EdkLogger.SetLevel(EdkLogger.QUIET)
387 elif Option.debug is not None:
388 EdkLogger.SetLevel(Option.debug + 1)
389 else:
390 EdkLogger.SetLevel(EdkLogger.INFO)
391
392 ## Parse command line options
393 #
394 # Using standard Python module optparse to parse command line option of this tool.
395 #
396 # @retval Opt A optparse.Values object containing the parsed options
397 # @retval Args Target of build command
398 #
399 def EccOptionParser(self):
400 Parser = OptionParser(description = self.Copyright, version = self.Version, prog = "Ecc.exe", usage = "%prog [options]")
401 Parser.add_option("-t", "--target sourcepath", action="store", type="string", dest='Target',
402 help="Check all files under the target workspace.")
403 Parser.add_option("-c", "--config filename", action="store", type="string", dest="ConfigFile",
404 help="Specify a configuration file. Defaultly use config.ini under ECC tool directory.")
405 Parser.add_option("-o", "--outfile filename", action="store", type="string", dest="OutputFile",
406 help="Specify the name of an output file, if and only if one filename was specified.")
407 Parser.add_option("-r", "--reportfile filename", action="store", type="string", dest="ReportFile",
408 help="Specify the name of an report file, if and only if one filename was specified.")
409 Parser.add_option("-e", "--exceptionfile filename", action="store", type="string", dest="ExceptionFile",
410 help="Specify the name of an exception file, if and only if one filename was specified.")
411 Parser.add_option("-m", "--metadata", action="store_true", type=None, help="Only scan meta-data files information if this option is specified.")
412 Parser.add_option("-s", "--sourcecode", action="store_true", type=None, help="Only scan source code files information if this option is specified.")
413 Parser.add_option("-k", "--keepdatabase", action="store_true", type=None, help="The existing Ecc database will not be cleaned except report information if this option is specified.")
414 Parser.add_option("-l", "--log filename", action="store", dest="LogFile", help="""If specified, the tool should emit the changes that
415 were made by the tool after printing the result message.
416 If filename, the emit to the file, otherwise emit to
417 standard output. If no modifications were made, then do not
418 create a log file, or output a log message.""")
419 Parser.add_option("-q", "--quiet", action="store_true", type=None, help="Disable all messages except FATAL ERRORS.")
420 Parser.add_option("-v", "--verbose", action="store_true", type=None, help="Turn on verbose output with informational messages printed, "\
421 "including library instances selected, final dependency expression, "\
422 "and warning messages, etc.")
423 Parser.add_option("-d", "--debug", action="store", type="int", help="Enable debug messages at specified level.")
424 Parser.add_option("-w", "--workspace", action="store", type="string", dest='Workspace', help="Specify workspace.")
425 Parser.add_option("-f", "--folders", action="store_true", type=None, help="Only scanning specified folders which are recorded in config.ini file.")
426
427 (Opt, Args)=Parser.parse_args()
428
429 return (Opt, Args)
430
431 ##
432 #
433 # This acts like the main() function for the script, unless it is 'import'ed into another
434 # script.
435 #
436 if __name__ == '__main__':
437 # Initialize log system
438 EdkLogger.Initialize()
439 EdkLogger.IsRaiseError = False
440
441 StartTime = time.clock()
442 Ecc = Ecc()
443 FinishTime = time.clock()
444
445 BuildDuration = time.strftime("%M:%S", time.gmtime(int(round(FinishTime - StartTime))))
446 EdkLogger.quiet("\n%s [%s]" % (time.strftime("%H:%M:%S, %b.%d %Y", time.localtime()), BuildDuration))