BaseTools/Source/Python/Workspace/WorkspaceDatabase.py

   1 ## @file
   2 # This file is used to create a database used by build tool
   3 #
   4 # Copyright (c) 2008 - 2018, Intel Corporation. All rights reserved.<BR>
   5 # (C) Copyright 2016 Hewlett Packard Enterprise Development LP<BR>
   6 # This program and the accompanying materials
   7 # are licensed and made available under the terms and conditions of the BSD License
   8 # which accompanies this distribution.  The full text of the license may be found at
   9 # http://opensource.org/licenses/bsd-license.php
  10 #
  11 # THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS,
  12 # WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED.
  13 #
  14
  15 ##
  16 # Import Modules
  17 #
  18 from __future__ import absolute_import
  19 import sqlite3
  20 from Common.StringUtils import *
  21 from Common.DataType import *
  22 from Common.Misc import *
  23 from types import *
  24
  25 from .MetaDataTable import *
  26 from .MetaFileTable import *
  27 from .MetaFileParser import *
  28
  29 from Workspace.DecBuildData import DecBuildData
  30 from Workspace.DscBuildData import DscBuildData
  31 from Workspace.InfBuildData import InfBuildData
  32
  33 ## Database
  34 #
  35 #   This class defined the build database for all modules, packages and platform.
  36 # It will call corresponding parser for the given file if it cannot find it in
  37 # the database.
  38 #
  39 # @param DbPath             Path of database file
  40 # @param GlobalMacros       Global macros used for replacement during file parsing
  41 # @prarm RenewDb=False      Create new database file if it's already there
  42 #
  43 class WorkspaceDatabase(object):
  44
  45     #
  46     # internal class used for call corresponding file parser and caching the result
  47     # to avoid unnecessary re-parsing
  48     #
  49     class BuildObjectFactory(object):
  50
  51         _FILE_TYPE_ = {
  52             ".inf"  : MODEL_FILE_INF,
  53             ".dec"  : MODEL_FILE_DEC,
  54             ".dsc"  : MODEL_FILE_DSC,
  55         }
  56
  57         # file parser
  58         _FILE_PARSER_ = {
  59             MODEL_FILE_INF  :   InfParser,
  60             MODEL_FILE_DEC  :   DecParser,
  61             MODEL_FILE_DSC  :   DscParser,
  62         }
  63
  64         # convert to xxxBuildData object
  65         _GENERATOR_ = {
  66             MODEL_FILE_INF  :   InfBuildData,
  67             MODEL_FILE_DEC  :   DecBuildData,
  68             MODEL_FILE_DSC  :   DscBuildData,
  69         }
  70
  71         _CACHE_ = {}    # (FilePath, Arch)  : <object>
  72
  73         # constructor
  74         def __init__(self, WorkspaceDb):
  75             self.WorkspaceDb = WorkspaceDb
  76
  77         # key = (FilePath, Arch=None)
  78         def __contains__(self, Key):
  79             FilePath = Key[0]
  80             if len(Key) > 1:
  81                 Arch = Key[1]
  82             else:
  83                 Arch = None
  84             return (FilePath, Arch) in self._CACHE_
  85
  86         # key = (FilePath, Arch=None, Target=None, Toochain=None)
  87         def __getitem__(self, Key):
  88             FilePath = Key[0]
  89             KeyLength = len(Key)
  90             if KeyLength > 1:
  91                 Arch = Key[1]
  92             else:
  93                 Arch = None
  94             if KeyLength > 2:
  95                 Target = Key[2]
  96             else:
  97                 Target = None
  98             if KeyLength > 3:
  99                 Toolchain = Key[3]
 100             else:
 101                 Toolchain = None
 102
 103             # if it's generated before, just return the cached one
 104             Key = (FilePath, Arch, Target, Toolchain)
 105             if Key in self._CACHE_:
 106                 return self._CACHE_[Key]
 107
 108             # check file type
 109             Ext = FilePath.Type
 110             if Ext not in self._FILE_TYPE_:
 111                 return None
 112             FileType = self._FILE_TYPE_[Ext]
 113             if FileType not in self._GENERATOR_:
 114                 return None
 115
 116             # get the parser ready for this file
 117             MetaFile = self._FILE_PARSER_[FileType](
 118                                 FilePath,
 119                                 FileType,
 120                                 Arch,
 121                                 MetaFileStorage(self.WorkspaceDb.Cur, FilePath, FileType)
 122                                 )
 123             # alwasy do post-process, in case of macros change
 124             MetaFile.DoPostProcess()
 125             # object the build is based on
 126             BuildObject = self._GENERATOR_[FileType](
 127                                     FilePath,
 128                                     MetaFile,
 129                                     self,
 130                                     Arch,
 131                                     Target,
 132                                     Toolchain
 133                                     )
 134             self._CACHE_[Key] = BuildObject
 135             return BuildObject
 136
 137     # placeholder for file format conversion
 138     class TransformObjectFactory:
 139         def __init__(self, WorkspaceDb):
 140             self.WorkspaceDb = WorkspaceDb
 141
 142         # key = FilePath, Arch
 143         def __getitem__(self, Key):
 144             pass
 145
 146     ## Constructor of WorkspaceDatabase
 147     #
 148     # @param DbPath             Path of database file
 149     # @param GlobalMacros       Global macros used for replacement during file parsing
 150     # @prarm RenewDb=False      Create new database file if it's already there
 151     #
 152     def __init__(self, DbPath, RenewDb=False):
 153         self._DbClosedFlag = False
 154         if not DbPath:
 155             DbPath = os.path.normpath(mws.join(GlobalData.gWorkspace, 'Conf', GlobalData.gDatabasePath))
 156
 157         # don't create necessary path for db in memory
 158         if DbPath != ':memory:':
 159             DbDir = os.path.split(DbPath)[0]
 160             if not os.path.exists(DbDir):
 161                 os.makedirs(DbDir)
 162
 163             # remove db file in case inconsistency between db and file in file system
 164             if self._CheckWhetherDbNeedRenew(RenewDb, DbPath):
 165                 os.remove(DbPath)
 166
 167         # create db with optimized parameters
 168         self.Conn = sqlite3.connect(DbPath, isolation_level='DEFERRED')
 169         self.Conn.execute("PRAGMA synchronous=OFF")
 170         self.Conn.execute("PRAGMA temp_store=MEMORY")
 171         self.Conn.execute("PRAGMA count_changes=OFF")
 172         self.Conn.execute("PRAGMA cache_size=8192")
 173         #self.Conn.execute("PRAGMA page_size=8192")
 174
 175         # to avoid non-ascii character conversion issue
 176         self.Conn.text_factory = str
 177         self.Cur = self.Conn.cursor()
 178
 179         # create table for internal uses
 180         self.TblDataModel = TableDataModel(self.Cur)
 181         self.TblFile = TableFile(self.Cur)
 182         self.Platform = None
 183
 184         # conversion object for build or file format conversion purpose
 185         self.BuildObject = WorkspaceDatabase.BuildObjectFactory(self)
 186         self.TransformObject = WorkspaceDatabase.TransformObjectFactory(self)
 187
 188     ## Check whether workspace database need to be renew.
 189     #  The renew reason maybe:
 190     #  1) If user force to renew;
 191     #  2) If user do not force renew, and
 192     #     a) If the time of last modified python source is newer than database file;
 193     #     b) If the time of last modified frozen executable file is newer than database file;
 194     #
 195     #  @param force     User force renew database
 196     #  @param DbPath    The absolute path of workspace database file
 197     #
 198     #  @return Bool value for whether need renew workspace databse
 199     #
 200     def _CheckWhetherDbNeedRenew (self, force, DbPath):
 201         # if database does not exist, we need do nothing
 202         if not os.path.exists(DbPath): return False
 203
 204         # if user force to renew database, then not check whether database is out of date
 205         if force: return True
 206
 207         #
 208         # Check the time of last modified source file or build.exe
 209         # if is newer than time of database, then database need to be re-created.
 210         #
 211         timeOfToolModified = 0
 212         if hasattr(sys, "frozen"):
 213             exePath             = os.path.abspath(sys.executable)
 214             timeOfToolModified  = os.stat(exePath).st_mtime
 215         else:
 216             curPath  = os.path.dirname(__file__) # curPath is the path of WorkspaceDatabase.py
 217             rootPath = os.path.split(curPath)[0] # rootPath is root path of python source, such as /BaseTools/Source/Python
 218             if rootPath == "" or rootPath is None:
 219                 EdkLogger.verbose("\nFail to find the root path of build.exe or python sources, so can not \
 220 determine whether database file is out of date!\n")
 221
 222             # walk the root path of source or build's binary to get the time last modified.
 223
 224             for root, dirs, files in os.walk (rootPath):
 225                 for dir in dirs:
 226                     # bypass source control folder
 227                     if dir.lower() in [".svn", "_svn", "cvs"]:
 228                         dirs.remove(dir)
 229
 230                 for file in files:
 231                     ext = os.path.splitext(file)[1]
 232                     if ext.lower() == ".py":            # only check .py files
 233                         fd = os.stat(os.path.join(root, file))
 234                         if timeOfToolModified < fd.st_mtime:
 235                             timeOfToolModified = fd.st_mtime
 236         if timeOfToolModified > os.stat(DbPath).st_mtime:
 237             EdkLogger.verbose("\nWorkspace database is out of data!")
 238             return True
 239
 240         return False
 241
 242     ## Initialize build database
 243     def InitDatabase(self):
 244         EdkLogger.verbose("\nInitialize build database started ...")
 245
 246         #
 247         # Create new tables
 248         #
 249         self.TblDataModel.Create(False)
 250         self.TblFile.Create(False)
 251
 252         #
 253         # Initialize table DataModel
 254         #
 255         self.TblDataModel.InitTable()
 256         EdkLogger.verbose("Initialize build database ... DONE!")
 257
 258     ## Query a table
 259     #
 260     # @param Table:  The instance of the table to be queried
 261     #
 262     def QueryTable(self, Table):
 263         Table.Query()
 264
 265     def __del__(self):
 266         self.Close()
 267
 268     ## Close entire database
 269     #
 270     # Commit all first
 271     # Close the connection and cursor
 272     #
 273     def Close(self):
 274         if not self._DbClosedFlag:
 275             self.Conn.commit()
 276             self.Cur.close()
 277             self.Conn.close()
 278             self._DbClosedFlag = True
 279
 280     ## Summarize all packages in the database
 281     def GetPackageList(self, Platform, Arch, TargetName, ToolChainTag):
 282         self.Platform = Platform
 283         PackageList = []
 284         Pa = self.BuildObject[self.Platform, Arch, TargetName, ToolChainTag]
 285         #
 286         # Get Package related to Modules
 287         #
 288         for Module in Pa.Modules:
 289             ModuleObj = self.BuildObject[Module, Arch, TargetName, ToolChainTag]
 290             for Package in ModuleObj.Packages:
 291                 if Package not in PackageList:
 292                     PackageList.append(Package)
 293         #
 294         # Get Packages related to Libraries
 295         #
 296         for Lib in Pa.LibraryInstances:
 297             LibObj = self.BuildObject[Lib, Arch, TargetName, ToolChainTag]
 298             for Package in LibObj.Packages:
 299                 if Package not in PackageList:
 300                     PackageList.append(Package)
 301
 302         return PackageList
 303
 304     ## Summarize all platforms in the database
 305     def _GetPlatformList(self):
 306         PlatformList = []
 307         for PlatformFile in self.TblFile.GetFileList(MODEL_FILE_DSC):
 308             try:
 309                 Platform = self.BuildObject[PathClass(PlatformFile), TAB_COMMON]
 310             except:
 311                 Platform = None
 312             if Platform is not None:
 313                 PlatformList.append(Platform)
 314         return PlatformList
 315
 316     def _MapPlatform(self, Dscfile):
 317         Platform = self.BuildObject[PathClass(Dscfile), TAB_COMMON]
 318         if Platform is None:
 319             EdkLogger.error('build', PARSER_ERROR, "Failed to parser DSC file: %s" % Dscfile)
 320         return Platform
 321
 322     PlatformList = property(_GetPlatformList)
 323
 324 ##
 325 #
 326 # This acts like the main() function for the script, unless it is 'import'ed into another
 327 # script.
 328 #
 329 if __name__ == '__main__':
 330     pass
 331