BaseTools/Source/Python/Workspace/WorkspaceDatabase.py

   1 ## @file
   2 # This file is used to create a database used by build tool
   3 #
   4 # Copyright (c) 2008 - 2018, Intel Corporation. All rights reserved.<BR>
   5 # (C) Copyright 2016 Hewlett Packard Enterprise Development LP<BR>
   6 # This program and the accompanying materials
   7 # are licensed and made available under the terms and conditions of the BSD License
   8 # which accompanies this distribution.  The full text of the license may be found at
   9 # http://opensource.org/licenses/bsd-license.php
  10 #
  11 # THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS,
  12 # WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED.
  13 #
  14
  15 ##
  16 # Import Modules
  17 #
  18 import sqlite3
  19 from Common.String import *
  20 from Common.DataType import *
  21 from Common.Misc import *
  22 from types import *
  23
  24 from MetaDataTable import *
  25 from MetaFileTable import *
  26 from MetaFileParser import *
  27
  28 from Workspace.DecBuildData import DecBuildData
  29 from Workspace.DscBuildData import DscBuildData
  30 from Workspace.InfBuildData import InfBuildData
  31
  32 ## Database
  33 #
  34 #   This class defined the build database for all modules, packages and platform.
  35 # It will call corresponding parser for the given file if it cannot find it in
  36 # the database.
  37 #
  38 # @param DbPath             Path of database file
  39 # @param GlobalMacros       Global macros used for replacement during file parsing
  40 # @prarm RenewDb=False      Create new database file if it's already there
  41 #
  42 class WorkspaceDatabase(object):
  43
  44     #
  45     # internal class used for call corresponding file parser and caching the result
  46     # to avoid unnecessary re-parsing
  47     #
  48     class BuildObjectFactory(object):
  49
  50         _FILE_TYPE_ = {
  51             ".inf"  : MODEL_FILE_INF,
  52             ".dec"  : MODEL_FILE_DEC,
  53             ".dsc"  : MODEL_FILE_DSC,
  54         }
  55
  56         # file parser
  57         _FILE_PARSER_ = {
  58             MODEL_FILE_INF  :   InfParser,
  59             MODEL_FILE_DEC  :   DecParser,
  60             MODEL_FILE_DSC  :   DscParser,
  61         }
  62
  63         # convert to xxxBuildData object
  64         _GENERATOR_ = {
  65             MODEL_FILE_INF  :   InfBuildData,
  66             MODEL_FILE_DEC  :   DecBuildData,
  67             MODEL_FILE_DSC  :   DscBuildData,
  68         }
  69
  70         _CACHE_ = {}    # (FilePath, Arch)  : <object>
  71
  72         # constructor
  73         def __init__(self, WorkspaceDb):
  74             self.WorkspaceDb = WorkspaceDb
  75
  76         # key = (FilePath, Arch=None)
  77         def __contains__(self, Key):
  78             FilePath = Key[0]
  79             if len(Key) > 1:
  80                 Arch = Key[1]
  81             else:
  82                 Arch = None
  83             return (FilePath, Arch) in self._CACHE_
  84
  85         # key = (FilePath, Arch=None, Target=None, Toochain=None)
  86         def __getitem__(self, Key):
  87             FilePath = Key[0]
  88             KeyLength = len(Key)
  89             if KeyLength > 1:
  90                 Arch = Key[1]
  91             else:
  92                 Arch = None
  93             if KeyLength > 2:
  94                 Target = Key[2]
  95             else:
  96                 Target = None
  97             if KeyLength > 3:
  98                 Toolchain = Key[3]
  99             else:
 100                 Toolchain = None
 101
 102             # if it's generated before, just return the cached one
 103             Key = (FilePath, Arch, Target, Toolchain)
 104             if Key in self._CACHE_:
 105                 return self._CACHE_[Key]
 106
 107             # check file type
 108             Ext = FilePath.Type
 109             if Ext not in self._FILE_TYPE_:
 110                 return None
 111             FileType = self._FILE_TYPE_[Ext]
 112             if FileType not in self._GENERATOR_:
 113                 return None
 114
 115             # get the parser ready for this file
 116             MetaFile = self._FILE_PARSER_[FileType](
 117                                 FilePath,
 118                                 FileType,
 119                                 Arch,
 120                                 MetaFileStorage(self.WorkspaceDb.Cur, FilePath, FileType)
 121                                 )
 122             # alwasy do post-process, in case of macros change
 123             MetaFile.DoPostProcess()
 124             # object the build is based on
 125             BuildObject = self._GENERATOR_[FileType](
 126                                     FilePath,
 127                                     MetaFile,
 128                                     self,
 129                                     Arch,
 130                                     Target,
 131                                     Toolchain
 132                                     )
 133             self._CACHE_[Key] = BuildObject
 134             return BuildObject
 135
 136     # placeholder for file format conversion
 137     class TransformObjectFactory:
 138         def __init__(self, WorkspaceDb):
 139             self.WorkspaceDb = WorkspaceDb
 140
 141         # key = FilePath, Arch
 142         def __getitem__(self, Key):
 143             pass
 144
 145     ## Constructor of WorkspaceDatabase
 146     #
 147     # @param DbPath             Path of database file
 148     # @param GlobalMacros       Global macros used for replacement during file parsing
 149     # @prarm RenewDb=False      Create new database file if it's already there
 150     #
 151     def __init__(self, DbPath, RenewDb=False):
 152         self._DbClosedFlag = False
 153         if not DbPath:
 154             DbPath = os.path.normpath(mws.join(GlobalData.gWorkspace, 'Conf', GlobalData.gDatabasePath))
 155
 156         # don't create necessary path for db in memory
 157         if DbPath != ':memory:':
 158             DbDir = os.path.split(DbPath)[0]
 159             if not os.path.exists(DbDir):
 160                 os.makedirs(DbDir)
 161
 162             # remove db file in case inconsistency between db and file in file system
 163             if self._CheckWhetherDbNeedRenew(RenewDb, DbPath):
 164                 os.remove(DbPath)
 165
 166         # create db with optimized parameters
 167         self.Conn = sqlite3.connect(DbPath, isolation_level='DEFERRED')
 168         self.Conn.execute("PRAGMA synchronous=OFF")
 169         self.Conn.execute("PRAGMA temp_store=MEMORY")
 170         self.Conn.execute("PRAGMA count_changes=OFF")
 171         self.Conn.execute("PRAGMA cache_size=8192")
 172         #self.Conn.execute("PRAGMA page_size=8192")
 173
 174         # to avoid non-ascii character conversion issue
 175         self.Conn.text_factory = str
 176         self.Cur = self.Conn.cursor()
 177
 178         # create table for internal uses
 179         self.TblDataModel = TableDataModel(self.Cur)
 180         self.TblFile = TableFile(self.Cur)
 181         self.Platform = None
 182
 183         # conversion object for build or file format conversion purpose
 184         self.BuildObject = WorkspaceDatabase.BuildObjectFactory(self)
 185         self.TransformObject = WorkspaceDatabase.TransformObjectFactory(self)
 186
 187     ## Check whether workspace database need to be renew.
 188     #  The renew reason maybe:
 189     #  1) If user force to renew;
 190     #  2) If user do not force renew, and
 191     #     a) If the time of last modified python source is newer than database file;
 192     #     b) If the time of last modified frozen executable file is newer than database file;
 193     #
 194     #  @param force     User force renew database
 195     #  @param DbPath    The absolute path of workspace database file
 196     #
 197     #  @return Bool value for whether need renew workspace databse
 198     #
 199     def _CheckWhetherDbNeedRenew (self, force, DbPath):
 200         # if database does not exist, we need do nothing
 201         if not os.path.exists(DbPath): return False
 202
 203         # if user force to renew database, then not check whether database is out of date
 204         if force: return True
 205
 206         #
 207         # Check the time of last modified source file or build.exe
 208         # if is newer than time of database, then database need to be re-created.
 209         #
 210         timeOfToolModified = 0
 211         if hasattr(sys, "frozen"):
 212             exePath             = os.path.abspath(sys.executable)
 213             timeOfToolModified  = os.stat(exePath).st_mtime
 214         else:
 215             curPath  = os.path.dirname(__file__) # curPath is the path of WorkspaceDatabase.py
 216             rootPath = os.path.split(curPath)[0] # rootPath is root path of python source, such as /BaseTools/Source/Python
 217             if rootPath == "" or rootPath is None:
 218                 EdkLogger.verbose("\nFail to find the root path of build.exe or python sources, so can not \
 219 determine whether database file is out of date!\n")
 220
 221             # walk the root path of source or build's binary to get the time last modified.
 222
 223             for root, dirs, files in os.walk (rootPath):
 224                 for dir in dirs:
 225                     # bypass source control folder
 226                     if dir.lower() in [".svn", "_svn", "cvs"]:
 227                         dirs.remove(dir)
 228
 229                 for file in files:
 230                     ext = os.path.splitext(file)[1]
 231                     if ext.lower() == ".py":            # only check .py files
 232                         fd = os.stat(os.path.join(root, file))
 233                         if timeOfToolModified < fd.st_mtime:
 234                             timeOfToolModified = fd.st_mtime
 235         if timeOfToolModified > os.stat(DbPath).st_mtime:
 236             EdkLogger.verbose("\nWorkspace database is out of data!")
 237             return True
 238
 239         return False
 240
 241     ## Initialize build database
 242     def InitDatabase(self):
 243         EdkLogger.verbose("\nInitialize build database started ...")
 244
 245         #
 246         # Create new tables
 247         #
 248         self.TblDataModel.Create(False)
 249         self.TblFile.Create(False)
 250
 251         #
 252         # Initialize table DataModel
 253         #
 254         self.TblDataModel.InitTable()
 255         EdkLogger.verbose("Initialize build database ... DONE!")
 256
 257     ## Query a table
 258     #
 259     # @param Table:  The instance of the table to be queried
 260     #
 261     def QueryTable(self, Table):
 262         Table.Query()
 263
 264     def __del__(self):
 265         self.Close()
 266
 267     ## Close entire database
 268     #
 269     # Commit all first
 270     # Close the connection and cursor
 271     #
 272     def Close(self):
 273         if not self._DbClosedFlag:
 274             self.Conn.commit()
 275             self.Cur.close()
 276             self.Conn.close()
 277             self._DbClosedFlag = True
 278
 279     ## Summarize all packages in the database
 280     def GetPackageList(self, Platform, Arch, TargetName, ToolChainTag):
 281         self.Platform = Platform
 282         PackageList = []
 283         Pa = self.BuildObject[self.Platform, Arch, TargetName, ToolChainTag]
 284         #
 285         # Get Package related to Modules
 286         #
 287         for Module in Pa.Modules:
 288             ModuleObj = self.BuildObject[Module, Arch, TargetName, ToolChainTag]
 289             for Package in ModuleObj.Packages:
 290                 if Package not in PackageList:
 291                     PackageList.append(Package)
 292         #
 293         # Get Packages related to Libraries
 294         #
 295         for Lib in Pa.LibraryInstances:
 296             LibObj = self.BuildObject[Lib, Arch, TargetName, ToolChainTag]
 297             for Package in LibObj.Packages:
 298                 if Package not in PackageList:
 299                     PackageList.append(Package)
 300
 301         return PackageList
 302
 303     ## Summarize all platforms in the database
 304     def _GetPlatformList(self):
 305         PlatformList = []
 306         for PlatformFile in self.TblFile.GetFileList(MODEL_FILE_DSC):
 307             try:
 308                 Platform = self.BuildObject[PathClass(PlatformFile), TAB_COMMON]
 309             except:
 310                 Platform = None
 311             if Platform is not None:
 312                 PlatformList.append(Platform)
 313         return PlatformList
 314
 315     def _MapPlatform(self, Dscfile):
 316         Platform = self.BuildObject[PathClass(Dscfile), TAB_COMMON]
 317         if Platform is None:
 318             EdkLogger.error('build', PARSER_ERROR, "Failed to parser DSC file: %s" % Dscfile)
 319         return Platform
 320
 321     PlatformList = property(_GetPlatformList)
 322
 323 ##
 324 #
 325 # This acts like the main() function for the script, unless it is 'import'ed into another
 326 # script.
 327 #
 328 if __name__ == '__main__':
 329     pass
 330