a40ab8fc8c882c49aa0c32041870655f337ac834
[mirror_edk2.git] / BaseTools / Source / Python / Workspace / WorkspaceDatabase.py
1 ## @file
2 # This file is used to create a database used by build tool
3 #
4 # Copyright (c) 2008 - 2018, Intel Corporation. All rights reserved.<BR>
5 # (C) Copyright 2016 Hewlett Packard Enterprise Development LP<BR>
6 # This program and the accompanying materials
7 # are licensed and made available under the terms and conditions of the BSD License
8 # which accompanies this distribution. The full text of the license may be found at
9 # http://opensource.org/licenses/bsd-license.php
10 #
11 # THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS,
12 # WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED.
13 #
14
15 ##
16 # Import Modules
17 #
18 import sqlite3
19 from Common.StringUtils import *
20 from Common.DataType import *
21 from Common.Misc import *
22 from types import *
23
24 from MetaDataTable import *
25 from MetaFileTable import *
26 from MetaFileParser import *
27
28 from Workspace.DecBuildData import DecBuildData
29 from Workspace.DscBuildData import DscBuildData
30 from Workspace.InfBuildData import InfBuildData
31
32 ## Database
33 #
34 # This class defined the build database for all modules, packages and platform.
35 # It will call corresponding parser for the given file if it cannot find it in
36 # the database.
37 #
38 # @param DbPath Path of database file
39 # @param GlobalMacros Global macros used for replacement during file parsing
40 # @prarm RenewDb=False Create new database file if it's already there
41 #
42 class WorkspaceDatabase(object):
43
44 #
45 # internal class used for call corresponding file parser and caching the result
46 # to avoid unnecessary re-parsing
47 #
48 class BuildObjectFactory(object):
49
50 _FILE_TYPE_ = {
51 ".inf" : MODEL_FILE_INF,
52 ".dec" : MODEL_FILE_DEC,
53 ".dsc" : MODEL_FILE_DSC,
54 }
55
56 # file parser
57 _FILE_PARSER_ = {
58 MODEL_FILE_INF : InfParser,
59 MODEL_FILE_DEC : DecParser,
60 MODEL_FILE_DSC : DscParser,
61 }
62
63 # convert to xxxBuildData object
64 _GENERATOR_ = {
65 MODEL_FILE_INF : InfBuildData,
66 MODEL_FILE_DEC : DecBuildData,
67 MODEL_FILE_DSC : DscBuildData,
68 }
69
70 _CACHE_ = {} # (FilePath, Arch) : <object>
71
72 # constructor
73 def __init__(self, WorkspaceDb):
74 self.WorkspaceDb = WorkspaceDb
75
76 # key = (FilePath, Arch=None)
77 def __contains__(self, Key):
78 FilePath = Key[0]
79 if len(Key) > 1:
80 Arch = Key[1]
81 else:
82 Arch = None
83 return (FilePath, Arch) in self._CACHE_
84
85 # key = (FilePath, Arch=None, Target=None, Toochain=None)
86 def __getitem__(self, Key):
87 FilePath = Key[0]
88 KeyLength = len(Key)
89 if KeyLength > 1:
90 Arch = Key[1]
91 else:
92 Arch = None
93 if KeyLength > 2:
94 Target = Key[2]
95 else:
96 Target = None
97 if KeyLength > 3:
98 Toolchain = Key[3]
99 else:
100 Toolchain = None
101
102 # if it's generated before, just return the cached one
103 Key = (FilePath, Arch, Target, Toolchain)
104 if Key in self._CACHE_:
105 return self._CACHE_[Key]
106
107 # check file type
108 Ext = FilePath.Type
109 if Ext not in self._FILE_TYPE_:
110 return None
111 FileType = self._FILE_TYPE_[Ext]
112 if FileType not in self._GENERATOR_:
113 return None
114
115 # get the parser ready for this file
116 MetaFile = self._FILE_PARSER_[FileType](
117 FilePath,
118 FileType,
119 Arch,
120 MetaFileStorage(self.WorkspaceDb.Cur, FilePath, FileType)
121 )
122 # alwasy do post-process, in case of macros change
123 MetaFile.DoPostProcess()
124 # object the build is based on
125 BuildObject = self._GENERATOR_[FileType](
126 FilePath,
127 MetaFile,
128 self,
129 Arch,
130 Target,
131 Toolchain
132 )
133 self._CACHE_[Key] = BuildObject
134 return BuildObject
135
136 # placeholder for file format conversion
137 class TransformObjectFactory:
138 def __init__(self, WorkspaceDb):
139 self.WorkspaceDb = WorkspaceDb
140
141 # key = FilePath, Arch
142 def __getitem__(self, Key):
143 pass
144
145 ## Constructor of WorkspaceDatabase
146 #
147 # @param DbPath Path of database file
148 # @param GlobalMacros Global macros used for replacement during file parsing
149 # @prarm RenewDb=False Create new database file if it's already there
150 #
151 def __init__(self, DbPath, RenewDb=False):
152 self._DbClosedFlag = False
153 if not DbPath:
154 DbPath = os.path.normpath(mws.join(GlobalData.gWorkspace, 'Conf', GlobalData.gDatabasePath))
155
156 # don't create necessary path for db in memory
157 if DbPath != ':memory:':
158 DbDir = os.path.split(DbPath)[0]
159 if not os.path.exists(DbDir):
160 os.makedirs(DbDir)
161
162 # remove db file in case inconsistency between db and file in file system
163 if self._CheckWhetherDbNeedRenew(RenewDb, DbPath):
164 os.remove(DbPath)
165
166 # create db with optimized parameters
167 self.Conn = sqlite3.connect(DbPath, isolation_level='DEFERRED')
168 self.Conn.execute("PRAGMA synchronous=OFF")
169 self.Conn.execute("PRAGMA temp_store=MEMORY")
170 self.Conn.execute("PRAGMA count_changes=OFF")
171 self.Conn.execute("PRAGMA cache_size=8192")
172 #self.Conn.execute("PRAGMA page_size=8192")
173
174 # to avoid non-ascii character conversion issue
175 self.Conn.text_factory = str
176 self.Cur = self.Conn.cursor()
177
178 # create table for internal uses
179 self.TblDataModel = TableDataModel(self.Cur)
180 self.TblFile = TableFile(self.Cur)
181 self.Platform = None
182
183 # conversion object for build or file format conversion purpose
184 self.BuildObject = WorkspaceDatabase.BuildObjectFactory(self)
185 self.TransformObject = WorkspaceDatabase.TransformObjectFactory(self)
186
187 ## Check whether workspace database need to be renew.
188 # The renew reason maybe:
189 # 1) If user force to renew;
190 # 2) If user do not force renew, and
191 # a) If the time of last modified python source is newer than database file;
192 # b) If the time of last modified frozen executable file is newer than database file;
193 #
194 # @param force User force renew database
195 # @param DbPath The absolute path of workspace database file
196 #
197 # @return Bool value for whether need renew workspace databse
198 #
199 def _CheckWhetherDbNeedRenew (self, force, DbPath):
200 # if database does not exist, we need do nothing
201 if not os.path.exists(DbPath): return False
202
203 # if user force to renew database, then not check whether database is out of date
204 if force: return True
205
206 #
207 # Check the time of last modified source file or build.exe
208 # if is newer than time of database, then database need to be re-created.
209 #
210 timeOfToolModified = 0
211 if hasattr(sys, "frozen"):
212 exePath = os.path.abspath(sys.executable)
213 timeOfToolModified = os.stat(exePath).st_mtime
214 else:
215 curPath = os.path.dirname(__file__) # curPath is the path of WorkspaceDatabase.py
216 rootPath = os.path.split(curPath)[0] # rootPath is root path of python source, such as /BaseTools/Source/Python
217 if rootPath == "" or rootPath is None:
218 EdkLogger.verbose("\nFail to find the root path of build.exe or python sources, so can not \
219 determine whether database file is out of date!\n")
220
221 # walk the root path of source or build's binary to get the time last modified.
222
223 for root, dirs, files in os.walk (rootPath):
224 for dir in dirs:
225 # bypass source control folder
226 if dir.lower() in [".svn", "_svn", "cvs"]:
227 dirs.remove(dir)
228
229 for file in files:
230 ext = os.path.splitext(file)[1]
231 if ext.lower() == ".py": # only check .py files
232 fd = os.stat(os.path.join(root, file))
233 if timeOfToolModified < fd.st_mtime:
234 timeOfToolModified = fd.st_mtime
235 if timeOfToolModified > os.stat(DbPath).st_mtime:
236 EdkLogger.verbose("\nWorkspace database is out of data!")
237 return True
238
239 return False
240
241 ## Initialize build database
242 def InitDatabase(self):
243 EdkLogger.verbose("\nInitialize build database started ...")
244
245 #
246 # Create new tables
247 #
248 self.TblDataModel.Create(False)
249 self.TblFile.Create(False)
250
251 #
252 # Initialize table DataModel
253 #
254 self.TblDataModel.InitTable()
255 EdkLogger.verbose("Initialize build database ... DONE!")
256
257 ## Query a table
258 #
259 # @param Table: The instance of the table to be queried
260 #
261 def QueryTable(self, Table):
262 Table.Query()
263
264 def __del__(self):
265 self.Close()
266
267 ## Close entire database
268 #
269 # Commit all first
270 # Close the connection and cursor
271 #
272 def Close(self):
273 if not self._DbClosedFlag:
274 self.Conn.commit()
275 self.Cur.close()
276 self.Conn.close()
277 self._DbClosedFlag = True
278
279 ## Summarize all packages in the database
280 def GetPackageList(self, Platform, Arch, TargetName, ToolChainTag):
281 self.Platform = Platform
282 PackageList = []
283 Pa = self.BuildObject[self.Platform, Arch, TargetName, ToolChainTag]
284 #
285 # Get Package related to Modules
286 #
287 for Module in Pa.Modules:
288 ModuleObj = self.BuildObject[Module, Arch, TargetName, ToolChainTag]
289 for Package in ModuleObj.Packages:
290 if Package not in PackageList:
291 PackageList.append(Package)
292 #
293 # Get Packages related to Libraries
294 #
295 for Lib in Pa.LibraryInstances:
296 LibObj = self.BuildObject[Lib, Arch, TargetName, ToolChainTag]
297 for Package in LibObj.Packages:
298 if Package not in PackageList:
299 PackageList.append(Package)
300
301 return PackageList
302
303 ## Summarize all platforms in the database
304 def _GetPlatformList(self):
305 PlatformList = []
306 for PlatformFile in self.TblFile.GetFileList(MODEL_FILE_DSC):
307 try:
308 Platform = self.BuildObject[PathClass(PlatformFile), TAB_COMMON]
309 except:
310 Platform = None
311 if Platform is not None:
312 PlatformList.append(Platform)
313 return PlatformList
314
315 def _MapPlatform(self, Dscfile):
316 Platform = self.BuildObject[PathClass(Dscfile), TAB_COMMON]
317 if Platform is None:
318 EdkLogger.error('build', PARSER_ERROR, "Failed to parser DSC file: %s" % Dscfile)
319 return Platform
320
321 PlatformList = property(_GetPlatformList)
322
323 ##
324 #
325 # This acts like the main() function for the script, unless it is 'import'ed into another
326 # script.
327 #
328 if __name__ == '__main__':
329 pass
330