BaseTools/Source/Python/Workspace/WorkspaceDatabase.py

   1 ## @file
   2 # This file is used to create a database used by build tool
   3 #
   4 # Copyright (c) 2008 - 2018, Intel Corporation. All rights reserved.<BR>
   5 # (C) Copyright 2016 Hewlett Packard Enterprise Development LP<BR>
   6 # This program and the accompanying materials
   7 # are licensed and made available under the terms and conditions of the BSD License
   8 # which accompanies this distribution.  The full text of the license may be found at
   9 # http://opensource.org/licenses/bsd-license.php
  10 #
  11 # THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS,
  12 # WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED.
  13 #
  14
  15 ##
  16 # Import Modules
  17 #
  18 from __future__ import absolute_import
  19 import sqlite3
  20 from Common.StringUtils import *
  21 from Common.DataType import *
  22 from Common.Misc import *
  23 from types import *
  24
  25 from .MetaDataTable import *
  26 from .MetaFileTable import *
  27 from .MetaFileParser import *
  28
  29 from Workspace.DecBuildData import DecBuildData
  30 from Workspace.DscBuildData import DscBuildData
  31 from Workspace.InfBuildData import InfBuildData
  32
  33 ## Database
  34 #
  35 #   This class defined the build database for all modules, packages and platform.
  36 # It will call corresponding parser for the given file if it cannot find it in
  37 # the database.
  38 #
  39 # @param DbPath             Path of database file
  40 # @param GlobalMacros       Global macros used for replacement during file parsing
  41 # @prarm RenewDb=False      Create new database file if it's already there
  42 #
  43 class WorkspaceDatabase(object):
  44
  45     #
  46     # internal class used for call corresponding file parser and caching the result
  47     # to avoid unnecessary re-parsing
  48     #
  49     class BuildObjectFactory(object):
  50
  51         _FILE_TYPE_ = {
  52             ".inf"  : MODEL_FILE_INF,
  53             ".dec"  : MODEL_FILE_DEC,
  54             ".dsc"  : MODEL_FILE_DSC,
  55         }
  56
  57         # file parser
  58         _FILE_PARSER_ = {
  59             MODEL_FILE_INF  :   InfParser,
  60             MODEL_FILE_DEC  :   DecParser,
  61             MODEL_FILE_DSC  :   DscParser,
  62         }
  63
  64         # convert to xxxBuildData object
  65         _GENERATOR_ = {
  66             MODEL_FILE_INF  :   InfBuildData,
  67             MODEL_FILE_DEC  :   DecBuildData,
  68             MODEL_FILE_DSC  :   DscBuildData,
  69         }
  70
  71         _CACHE_ = {}    # (FilePath, Arch)  : <object>
  72
  73         # constructor
  74         def __init__(self, WorkspaceDb):
  75             self.WorkspaceDb = WorkspaceDb
  76
  77         # key = (FilePath, Arch=None)
  78         def __contains__(self, Key):
  79             FilePath = Key[0]
  80             if len(Key) > 1:
  81                 Arch = Key[1]
  82             else:
  83                 Arch = None
  84             return (FilePath, Arch) in self._CACHE_
  85
  86         # key = (FilePath, Arch=None, Target=None, Toochain=None)
  87         def __getitem__(self, Key):
  88             FilePath = Key[0]
  89             KeyLength = len(Key)
  90             if KeyLength > 1:
  91                 Arch = Key[1]
  92             else:
  93                 Arch = None
  94             if KeyLength > 2:
  95                 Target = Key[2]
  96             else:
  97                 Target = None
  98             if KeyLength > 3:
  99                 Toolchain = Key[3]
 100             else:
 101                 Toolchain = None
 102
 103             # if it's generated before, just return the cached one
 104             Key = (FilePath, Arch, Target, Toolchain)
 105             if Key in self._CACHE_:
 106                 return self._CACHE_[Key]
 107
 108             # check file type
 109             BuildObject = self.CreateBuildObject(FilePath, Arch, Target, Toolchain)
 110             self._CACHE_[Key] = BuildObject
 111             return BuildObject
 112         def CreateBuildObject(self,FilePath, Arch, Target, Toolchain):
 113             Ext = FilePath.Type
 114             if Ext not in self._FILE_TYPE_:
 115                 return None
 116             FileType = self._FILE_TYPE_[Ext]
 117             if FileType not in self._GENERATOR_:
 118                 return None
 119
 120             # get the parser ready for this file
 121             MetaFile = self._FILE_PARSER_[FileType](
 122                                 FilePath,
 123                                 FileType,
 124                                 Arch,
 125                                 MetaFileStorage(self.WorkspaceDb.Cur, FilePath, FileType)
 126                                 )
 127             # alwasy do post-process, in case of macros change
 128             MetaFile.DoPostProcess()
 129             # object the build is based on
 130             BuildObject = self._GENERATOR_[FileType](
 131                                     FilePath,
 132                                     MetaFile,
 133                                     self,
 134                                     Arch,
 135                                     Target,
 136                                     Toolchain
 137                                     )
 138             return BuildObject
 139
 140     # placeholder for file format conversion
 141     class TransformObjectFactory:
 142         def __init__(self, WorkspaceDb):
 143             self.WorkspaceDb = WorkspaceDb
 144
 145         # key = FilePath, Arch
 146         def __getitem__(self, Key):
 147             pass
 148
 149     ## Constructor of WorkspaceDatabase
 150     #
 151     # @param DbPath             Path of database file
 152     # @param GlobalMacros       Global macros used for replacement during file parsing
 153     # @prarm RenewDb=False      Create new database file if it's already there
 154     #
 155     def __init__(self, DbPath, RenewDb=False):
 156         self._DbClosedFlag = False
 157         if not DbPath:
 158             DbPath = os.path.normpath(mws.join(GlobalData.gWorkspace, 'Conf', GlobalData.gDatabasePath))
 159
 160         # don't create necessary path for db in memory
 161         if DbPath != ':memory:':
 162             DbDir = os.path.split(DbPath)[0]
 163             if not os.path.exists(DbDir):
 164                 os.makedirs(DbDir)
 165
 166             # remove db file in case inconsistency between db and file in file system
 167             if self._CheckWhetherDbNeedRenew(RenewDb, DbPath):
 168                 os.remove(DbPath)
 169
 170         # create db with optimized parameters
 171         self.Conn = sqlite3.connect(DbPath, isolation_level='DEFERRED')
 172         self.Conn.execute("PRAGMA synchronous=OFF")
 173         self.Conn.execute("PRAGMA temp_store=MEMORY")
 174         self.Conn.execute("PRAGMA count_changes=OFF")
 175         self.Conn.execute("PRAGMA cache_size=8192")
 176         #self.Conn.execute("PRAGMA page_size=8192")
 177
 178         # to avoid non-ascii character conversion issue
 179         self.Conn.text_factory = str
 180         self.Cur = self.Conn.cursor()
 181
 182         # create table for internal uses
 183         self.TblDataModel = TableDataModel(self.Cur)
 184         self.TblFile = TableFile(self.Cur)
 185         self.Platform = None
 186
 187         # conversion object for build or file format conversion purpose
 188         self.BuildObject = WorkspaceDatabase.BuildObjectFactory(self)
 189         self.TransformObject = WorkspaceDatabase.TransformObjectFactory(self)
 190
 191     ## Check whether workspace database need to be renew.
 192     #  The renew reason maybe:
 193     #  1) If user force to renew;
 194     #  2) If user do not force renew, and
 195     #     a) If the time of last modified python source is newer than database file;
 196     #     b) If the time of last modified frozen executable file is newer than database file;
 197     #
 198     #  @param force     User force renew database
 199     #  @param DbPath    The absolute path of workspace database file
 200     #
 201     #  @return Bool value for whether need renew workspace databse
 202     #
 203     def _CheckWhetherDbNeedRenew (self, force, DbPath):
 204         # if database does not exist, we need do nothing
 205         if not os.path.exists(DbPath): return False
 206
 207         # if user force to renew database, then not check whether database is out of date
 208         if force: return True
 209
 210         #
 211         # Check the time of last modified source file or build.exe
 212         # if is newer than time of database, then database need to be re-created.
 213         #
 214         timeOfToolModified = 0
 215         if hasattr(sys, "frozen"):
 216             exePath             = os.path.abspath(sys.executable)
 217             timeOfToolModified  = os.stat(exePath).st_mtime
 218         else:
 219             curPath  = os.path.dirname(__file__) # curPath is the path of WorkspaceDatabase.py
 220             rootPath = os.path.split(curPath)[0] # rootPath is root path of python source, such as /BaseTools/Source/Python
 221             if rootPath == "" or rootPath is None:
 222                 EdkLogger.verbose("\nFail to find the root path of build.exe or python sources, so can not \
 223 determine whether database file is out of date!\n")
 224
 225             # walk the root path of source or build's binary to get the time last modified.
 226
 227             for root, dirs, files in os.walk (rootPath):
 228                 for dir in dirs:
 229                     # bypass source control folder
 230                     if dir.lower() in [".svn", "_svn", "cvs"]:
 231                         dirs.remove(dir)
 232
 233                 for file in files:
 234                     ext = os.path.splitext(file)[1]
 235                     if ext.lower() == ".py":            # only check .py files
 236                         fd = os.stat(os.path.join(root, file))
 237                         if timeOfToolModified < fd.st_mtime:
 238                             timeOfToolModified = fd.st_mtime
 239         if timeOfToolModified > os.stat(DbPath).st_mtime:
 240             EdkLogger.verbose("\nWorkspace database is out of data!")
 241             return True
 242
 243         return False
 244
 245     ## Initialize build database
 246     def InitDatabase(self):
 247         EdkLogger.verbose("\nInitialize build database started ...")
 248
 249         #
 250         # Create new tables
 251         #
 252         self.TblDataModel.Create(False)
 253         self.TblFile.Create(False)
 254
 255         #
 256         # Initialize table DataModel
 257         #
 258         self.TblDataModel.InitTable()
 259         EdkLogger.verbose("Initialize build database ... DONE!")
 260
 261     ## Query a table
 262     #
 263     # @param Table:  The instance of the table to be queried
 264     #
 265     def QueryTable(self, Table):
 266         Table.Query()
 267
 268     def __del__(self):
 269         self.Close()
 270
 271     ## Close entire database
 272     #
 273     # Commit all first
 274     # Close the connection and cursor
 275     #
 276     def Close(self):
 277         if not self._DbClosedFlag:
 278             self.Conn.commit()
 279             self.Cur.close()
 280             self.Conn.close()
 281             self._DbClosedFlag = True
 282
 283     ## Summarize all packages in the database
 284     def GetPackageList(self, Platform, Arch, TargetName, ToolChainTag):
 285         self.Platform = Platform
 286         PackageList = []
 287         Pa = self.BuildObject[self.Platform, Arch, TargetName, ToolChainTag]
 288         #
 289         # Get Package related to Modules
 290         #
 291         for Module in Pa.Modules:
 292             ModuleObj = self.BuildObject[Module, Arch, TargetName, ToolChainTag]
 293             for Package in ModuleObj.Packages:
 294                 if Package not in PackageList:
 295                     PackageList.append(Package)
 296         #
 297         # Get Packages related to Libraries
 298         #
 299         for Lib in Pa.LibraryInstances:
 300             LibObj = self.BuildObject[Lib, Arch, TargetName, ToolChainTag]
 301             for Package in LibObj.Packages:
 302                 if Package not in PackageList:
 303                     PackageList.append(Package)
 304
 305         return PackageList
 306
 307     ## Summarize all platforms in the database
 308     def PlatformList(self):
 309         RetVal = []
 310         for PlatformFile in self.TblFile.GetFileList(MODEL_FILE_DSC):
 311             try:
 312                 RetVal.append(self.BuildObject[PathClass(PlatformFile), TAB_COMMON])
 313             except:
 314                 pass
 315         return RetVal
 316
 317     def MapPlatform(self, Dscfile):
 318         Platform = self.BuildObject[PathClass(Dscfile), TAB_COMMON]
 319         if Platform is None:
 320             EdkLogger.error('build', PARSER_ERROR, "Failed to parser DSC file: %s" % Dscfile)
 321         return Platform
 322
 323 ##
 324 #
 325 # This acts like the main() function for the script, unless it is 'import'ed into another
 326 # script.
 327 #
 328 if __name__ == '__main__':
 329     pass
 330