BaseTools/Source/Python/Common/Misc.py

   1 ## @file
   2 # Common routines used by all tools
   3 #
   4 # Copyright (c) 2007 - 2014, Intel Corporation. All rights reserved.<BR>
   5 # This program and the accompanying materials
   6 # are licensed and made available under the terms and conditions of the BSD License
   7 # which accompanies this distribution.  The full text of the license may be found at
   8 # http://opensource.org/licenses/bsd-license.php
   9 #
  10 # THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS,
  11 # WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED.
  12 #
  13
  14 ##
  15 # Import Modules
  16 #
  17 import Common.LongFilePathOs as os
  18 import sys
  19 import string
  20 import thread
  21 import threading
  22 import time
  23 import re
  24 import cPickle
  25 import array
  26 import shutil
  27 from UserDict import IterableUserDict
  28 from UserList import UserList
  29
  30 from Common import EdkLogger as EdkLogger
  31 from Common import GlobalData as GlobalData
  32 from DataType import *
  33 from BuildToolError import *
  34 from CommonDataClass.DataClass import *
  35 from Parsing import GetSplitValueList
  36 from Common.LongFilePathSupport import OpenLongFilePath as open
  37
  38 ## Regular expression used to find out place holders in string template
  39 gPlaceholderPattern = re.compile("\$\{([^$()\s]+)\}", re.MULTILINE|re.UNICODE)
  40
  41 ## Dictionary used to store file time stamp for quick re-access
  42 gFileTimeStampCache = {}    # {file path : file time stamp}
  43
  44 ## Dictionary used to store dependencies of files
  45 gDependencyDatabase = {}    # arch : {file path : [dependent files list]}
  46
  47 ## Routine to process duplicated INF
  48 #
  49 #  This function is called by following two cases:
  50 #  Case 1 in DSC:
  51 #    [components.arch]
  52 #    Pkg/module/module.inf
  53 #    Pkg/module/module.inf {
  54 #      <Defines>
  55 #        FILE_GUID = 0D1B936F-68F3-4589-AFCC-FB8B7AEBC836
  56 #    }
  57 #  Case 2 in FDF:
  58 #    INF Pkg/module/module.inf
  59 #    INF FILE_GUID = 0D1B936F-68F3-4589-AFCC-FB8B7AEBC836 Pkg/module/module.inf
  60 #
  61 #  This function copies Pkg/module/module.inf to
  62 #  Conf/.cache/0D1B936F-68F3-4589-AFCC-FB8B7AEBC836module.inf
  63 #
  64 #  @param Path     Original PathClass object
  65 #  @param BaseName New file base name
  66 #
  67 #  @retval         return the new PathClass object
  68 #
  69 def ProcessDuplicatedInf(Path, BaseName, Workspace):
  70     Filename = os.path.split(Path.File)[1]
  71     if '.' in Filename:
  72         Filename = BaseName + Path.BaseName + Filename[Filename.rfind('.'):]
  73     else:
  74         Filename = BaseName + Path.BaseName
  75
  76     #
  77     # If -N is specified on command line, cache is disabled
  78     # The directory has to be created
  79     #
  80     DbDir = os.path.split(GlobalData.gDatabasePath)[0]
  81     if not os.path.exists(DbDir):
  82         os.makedirs(DbDir)
  83     #
  84     # A temporary INF is copied to database path which must have write permission
  85     # The temporary will be removed at the end of build
  86     # In case of name conflict, the file name is
  87     # FILE_GUIDBaseName (0D1B936F-68F3-4589-AFCC-FB8B7AEBC836module.inf)
  88     #
  89     TempFullPath = os.path.join(DbDir,
  90                                 Filename)
  91     RtPath = PathClass(Path.File, Workspace)
  92     #
  93     # Modify the full path to temporary path, keep other unchanged
  94     #
  95     # To build same module more than once, the module path with FILE_GUID overridden has
  96     # the file name FILE_GUIDmodule.inf, but the relative path (self.MetaFile.File) is the real path
  97     # in DSC which is used as relative path by C files and other files in INF.
  98     # A trick was used: all module paths are PathClass instances, after the initialization
  99     # of PathClass, the PathClass.Path is overridden by the temporary INF path.
 100     #
 101     # The reason for creating a temporary INF is:
 102     # Platform.Modules which is the base to create ModuleAutoGen objects is a dictionary,
 103     # the key is the full path of INF, the value is an object to save overridden library instances, PCDs.
 104     # A different key for the same module is needed to create different output directory,
 105     # retrieve overridden PCDs, library instances.
 106     #
 107     # The BaseName is the FILE_GUID which is also the output directory name.
 108     #
 109     #
 110     RtPath.Path = TempFullPath
 111     RtPath.BaseName = BaseName
 112     #
 113     # If file exists, compare contents
 114     #
 115     if os.path.exists(TempFullPath):
 116         with open(str(Path), 'rb') as f1: Src = f1.read()
 117         with open(TempFullPath, 'rb') as f2: Dst = f2.read()
 118         if Src == Dst:
 119             return RtPath
 120     GlobalData.gTempInfs.append(TempFullPath)
 121     shutil.copy2(str(Path), TempFullPath)
 122     return RtPath
 123
 124 ## Remove temporary created INFs whose paths were saved in gTempInfs
 125 #
 126 def ClearDuplicatedInf():
 127     for File in GlobalData.gTempInfs:
 128         if os.path.exists(File):
 129             os.remove(File)
 130
 131 ## callback routine for processing variable option
 132 #
 133 # This function can be used to process variable number of option values. The
 134 # typical usage of it is specify architecure list on command line.
 135 # (e.g. <tool> -a IA32 X64 IPF)
 136 #
 137 # @param  Option        Standard callback function parameter
 138 # @param  OptionString  Standard callback function parameter
 139 # @param  Value         Standard callback function parameter
 140 # @param  Parser        Standard callback function parameter
 141 #
 142 # @retval
 143 #
 144 def ProcessVariableArgument(Option, OptionString, Value, Parser):
 145     assert Value is None
 146     Value = []
 147     RawArgs = Parser.rargs
 148     while RawArgs:
 149         Arg = RawArgs[0]
 150         if (Arg[:2] == "--" and len(Arg) > 2) or \
 151            (Arg[:1] == "-" and len(Arg) > 1 and Arg[1] != "-"):
 152             break
 153         Value.append(Arg)
 154         del RawArgs[0]
 155     setattr(Parser.values, Option.dest, Value)
 156
 157 ## Convert GUID string in xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx style to C structure style
 158 #
 159 #   @param      Guid    The GUID string
 160 #
 161 #   @retval     string  The GUID string in C structure style
 162 #
 163 def GuidStringToGuidStructureString(Guid):
 164     GuidList = Guid.split('-')
 165     Result = '{'
 166     for Index in range(0,3,1):
 167         Result = Result + '0x' + GuidList[Index] + ', '
 168     Result = Result + '{0x' + GuidList[3][0:2] + ', 0x' + GuidList[3][2:4]
 169     for Index in range(0,12,2):
 170         Result = Result + ', 0x' + GuidList[4][Index:Index+2]
 171     Result += '}}'
 172     return Result
 173
 174 ## Convert GUID structure in byte array to xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
 175 #
 176 #   @param      GuidValue   The GUID value in byte array
 177 #
 178 #   @retval     string      The GUID value in xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx format
 179 #
 180 def GuidStructureByteArrayToGuidString(GuidValue):
 181     guidValueString = GuidValue.lower().replace("{", "").replace("}", "").replace(" ", "").replace(";", "")
 182     guidValueList = guidValueString.split(",")
 183     if len(guidValueList) != 16:
 184         return ''
 185         #EdkLogger.error(None, None, "Invalid GUID value string %s" % GuidValue)
 186     try:
 187         return "%02x%02x%02x%02x-%02x%02x-%02x%02x-%02x%02x-%02x%02x%02x%02x%02x%02x" % (
 188                 int(guidValueList[3], 16),
 189                 int(guidValueList[2], 16),
 190                 int(guidValueList[1], 16),
 191                 int(guidValueList[0], 16),
 192                 int(guidValueList[5], 16),
 193                 int(guidValueList[4], 16),
 194                 int(guidValueList[7], 16),
 195                 int(guidValueList[6], 16),
 196                 int(guidValueList[8], 16),
 197                 int(guidValueList[9], 16),
 198                 int(guidValueList[10], 16),
 199                 int(guidValueList[11], 16),
 200                 int(guidValueList[12], 16),
 201                 int(guidValueList[13], 16),
 202                 int(guidValueList[14], 16),
 203                 int(guidValueList[15], 16)
 204                 )
 205     except:
 206         return ''
 207
 208 ## Convert GUID string in C structure style to xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
 209 #
 210 #   @param      GuidValue   The GUID value in C structure format
 211 #
 212 #   @retval     string      The GUID value in xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx format
 213 #
 214 def GuidStructureStringToGuidString(GuidValue):
 215     guidValueString = GuidValue.lower().replace("{", "").replace("}", "").replace(" ", "").replace(";", "")
 216     guidValueList = guidValueString.split(",")
 217     if len(guidValueList) != 11:
 218         return ''
 219         #EdkLogger.error(None, None, "Invalid GUID value string %s" % GuidValue)
 220     try:
 221         return "%08x-%04x-%04x-%02x%02x-%02x%02x%02x%02x%02x%02x" % (
 222                 int(guidValueList[0], 16),
 223                 int(guidValueList[1], 16),
 224                 int(guidValueList[2], 16),
 225                 int(guidValueList[3], 16),
 226                 int(guidValueList[4], 16),
 227                 int(guidValueList[5], 16),
 228                 int(guidValueList[6], 16),
 229                 int(guidValueList[7], 16),
 230                 int(guidValueList[8], 16),
 231                 int(guidValueList[9], 16),
 232                 int(guidValueList[10], 16)
 233                 )
 234     except:
 235         return ''
 236
 237 ## Convert GUID string in C structure style to xxxxxxxx_xxxx_xxxx_xxxx_xxxxxxxxxxxx
 238 #
 239 #   @param      GuidValue   The GUID value in C structure format
 240 #
 241 #   @retval     string      The GUID value in xxxxxxxx_xxxx_xxxx_xxxx_xxxxxxxxxxxx format
 242 #
 243 def GuidStructureStringToGuidValueName(GuidValue):
 244     guidValueString = GuidValue.lower().replace("{", "").replace("}", "").replace(" ", "")
 245     guidValueList = guidValueString.split(",")
 246     if len(guidValueList) != 11:
 247         EdkLogger.error(None, FORMAT_INVALID, "Invalid GUID value string [%s]" % GuidValue)
 248     return "%08x_%04x_%04x_%02x%02x_%02x%02x%02x%02x%02x%02x" % (
 249             int(guidValueList[0], 16),
 250             int(guidValueList[1], 16),
 251             int(guidValueList[2], 16),
 252             int(guidValueList[3], 16),
 253             int(guidValueList[4], 16),
 254             int(guidValueList[5], 16),
 255             int(guidValueList[6], 16),
 256             int(guidValueList[7], 16),
 257             int(guidValueList[8], 16),
 258             int(guidValueList[9], 16),
 259             int(guidValueList[10], 16)
 260             )
 261
 262 ## Create directories
 263 #
 264 #   @param      Directory   The directory name
 265 #
 266 def CreateDirectory(Directory):
 267     if Directory == None or Directory.strip() == "":
 268         return True
 269     try:
 270         if not os.access(Directory, os.F_OK):
 271             os.makedirs(Directory)
 272     except:
 273         return False
 274     return True
 275
 276 ## Remove directories, including files and sub-directories in it
 277 #
 278 #   @param      Directory   The directory name
 279 #
 280 def RemoveDirectory(Directory, Recursively=False):
 281     if Directory == None or Directory.strip() == "" or not os.path.exists(Directory):
 282         return
 283     if Recursively:
 284         CurrentDirectory = os.getcwd()
 285         os.chdir(Directory)
 286         for File in os.listdir("."):
 287             if os.path.isdir(File):
 288                 RemoveDirectory(File, Recursively)
 289             else:
 290                 os.remove(File)
 291         os.chdir(CurrentDirectory)
 292     os.rmdir(Directory)
 293
 294 ## Check if given file is changed or not
 295 #
 296 #  This method is used to check if a file is changed or not between two build
 297 #  actions. It makes use a cache to store files timestamp.
 298 #
 299 #   @param      File    The path of file
 300 #
 301 #   @retval     True    If the given file is changed, doesn't exist, or can't be
 302 #                       found in timestamp cache
 303 #   @retval     False   If the given file is changed
 304 #
 305 def IsChanged(File):
 306     if not os.path.exists(File):
 307         return True
 308
 309     FileState = os.stat(File)
 310     TimeStamp = FileState[-2]
 311
 312     if File in gFileTimeStampCache and TimeStamp == gFileTimeStampCache[File]:
 313         FileChanged = False
 314     else:
 315         FileChanged = True
 316         gFileTimeStampCache[File] = TimeStamp
 317
 318     return FileChanged
 319
 320 ## Store content in file
 321 #
 322 #  This method is used to save file only when its content is changed. This is
 323 #  quite useful for "make" system to decide what will be re-built and what won't.
 324 #
 325 #   @param      File            The path of file
 326 #   @param      Content         The new content of the file
 327 #   @param      IsBinaryFile    The flag indicating if the file is binary file or not
 328 #
 329 #   @retval     True            If the file content is changed and the file is renewed
 330 #   @retval     False           If the file content is the same
 331 #
 332 def SaveFileOnChange(File, Content, IsBinaryFile=True):
 333     if not IsBinaryFile:
 334         Content = Content.replace("\n", os.linesep)
 335
 336     if os.path.exists(File):
 337         try:
 338             if Content == open(File, "rb").read():
 339                 return False
 340         except:
 341             EdkLogger.error(None, FILE_OPEN_FAILURE, ExtraData=File)
 342
 343     DirName = os.path.dirname(File)
 344     if not CreateDirectory(DirName):
 345         EdkLogger.error(None, FILE_CREATE_FAILURE, "Could not create directory %s" % DirName)
 346     else:
 347         if DirName == '':
 348             DirName = os.getcwd()
 349         if not os.access(DirName, os.W_OK):
 350             EdkLogger.error(None, PERMISSION_FAILURE, "Do not have write permission on directory %s" % DirName)
 351
 352     try:
 353         if GlobalData.gIsWindows:
 354             try:
 355                 from PyUtility import SaveFileToDisk
 356                 if not SaveFileToDisk(File, Content):
 357                     EdkLogger.error(None, FILE_CREATE_FAILURE, ExtraData=File)
 358             except:
 359                 Fd = open(File, "wb")
 360                 Fd.write(Content)
 361                 Fd.close()
 362         else:
 363             Fd = open(File, "wb")
 364             Fd.write(Content)
 365             Fd.close()
 366     except IOError, X:
 367         EdkLogger.error(None, FILE_CREATE_FAILURE, ExtraData='IOError %s'%X)
 368
 369     return True
 370
 371 ## Make a Python object persistent on file system
 372 #
 373 #   @param      Data    The object to be stored in file
 374 #   @param      File    The path of file to store the object
 375 #
 376 def DataDump(Data, File):
 377     Fd = None
 378     try:
 379         Fd = open(File, 'wb')
 380         cPickle.dump(Data, Fd, cPickle.HIGHEST_PROTOCOL)
 381     except:
 382         EdkLogger.error("", FILE_OPEN_FAILURE, ExtraData=File, RaiseError=False)
 383     finally:
 384         if Fd != None:
 385             Fd.close()
 386
 387 ## Restore a Python object from a file
 388 #
 389 #   @param      File    The path of file stored the object
 390 #
 391 #   @retval     object  A python object
 392 #   @retval     None    If failure in file operation
 393 #
 394 def DataRestore(File):
 395     Data = None
 396     Fd = None
 397     try:
 398         Fd = open(File, 'rb')
 399         Data = cPickle.load(Fd)
 400     except Exception, e:
 401         EdkLogger.verbose("Failed to load [%s]\n\t%s" % (File, str(e)))
 402         Data = None
 403     finally:
 404         if Fd != None:
 405             Fd.close()
 406     return Data
 407
 408 ## Retrieve and cache the real path name in file system
 409 #
 410 #   @param      Root    The root directory of path relative to
 411 #
 412 #   @retval     str     The path string if the path exists
 413 #   @retval     None    If path doesn't exist
 414 #
 415 class DirCache:
 416     _CACHE_ = set()
 417     _UPPER_CACHE_ = {}
 418
 419     def __init__(self, Root):
 420         self._Root = Root
 421         for F in os.listdir(Root):
 422             self._CACHE_.add(F)
 423             self._UPPER_CACHE_[F.upper()] = F
 424
 425     # =[] operator
 426     def __getitem__(self, Path):
 427         Path = Path[len(os.path.commonprefix([Path, self._Root])):]
 428         if not Path:
 429             return self._Root
 430         if Path and Path[0] == os.path.sep:
 431             Path = Path[1:]
 432         if Path in self._CACHE_:
 433             return os.path.join(self._Root, Path)
 434         UpperPath = Path.upper()
 435         if UpperPath in self._UPPER_CACHE_:
 436             return os.path.join(self._Root, self._UPPER_CACHE_[UpperPath])
 437
 438         IndexList = []
 439         LastSepIndex = -1
 440         SepIndex = Path.find(os.path.sep)
 441         while SepIndex > -1:
 442             Parent = UpperPath[:SepIndex]
 443             if Parent not in self._UPPER_CACHE_:
 444                 break
 445             LastSepIndex = SepIndex
 446             SepIndex = Path.find(os.path.sep, LastSepIndex + 1)
 447
 448         if LastSepIndex == -1:
 449             return None
 450
 451         Cwd = os.getcwd()
 452         os.chdir(self._Root)
 453         SepIndex = LastSepIndex
 454         while SepIndex > -1:
 455             Parent = Path[:SepIndex]
 456             ParentKey = UpperPath[:SepIndex]
 457             if ParentKey not in self._UPPER_CACHE_:
 458                 os.chdir(Cwd)
 459                 return None
 460
 461             if Parent in self._CACHE_:
 462                 ParentDir = Parent
 463             else:
 464                 ParentDir = self._UPPER_CACHE_[ParentKey]
 465             for F in os.listdir(ParentDir):
 466                 Dir = os.path.join(ParentDir, F)
 467                 self._CACHE_.add(Dir)
 468                 self._UPPER_CACHE_[Dir.upper()] = Dir
 469
 470             SepIndex = Path.find(os.path.sep, SepIndex + 1)
 471
 472         os.chdir(Cwd)
 473         if Path in self._CACHE_:
 474             return os.path.join(self._Root, Path)
 475         elif UpperPath in self._UPPER_CACHE_:
 476             return os.path.join(self._Root, self._UPPER_CACHE_[UpperPath])
 477         return None
 478
 479 ## Get all files of a directory
 480 #
 481 # @param Root:       Root dir
 482 # @param SkipList :  The files need be skipped
 483 #
 484 # @retval  A list of all files
 485 #
 486 def GetFiles(Root, SkipList=None, FullPath = True):
 487     OriPath = Root
 488     FileList = []
 489     for Root, Dirs, Files in os.walk(Root):
 490         if SkipList:
 491             for Item in SkipList:
 492                 if Item in Dirs:
 493                     Dirs.remove(Item)
 494
 495         for File in Files:
 496             File = os.path.normpath(os.path.join(Root, File))
 497             if not FullPath:
 498                 File = File[len(OriPath) + 1:]
 499             FileList.append(File)
 500
 501     return FileList
 502
 503 ## Check if gvien file exists or not
 504 #
 505 #   @param      File    File name or path to be checked
 506 #   @param      Dir     The directory the file is relative to
 507 #
 508 #   @retval     True    if file exists
 509 #   @retval     False   if file doesn't exists
 510 #
 511 def ValidFile(File, Ext=None):
 512     if Ext != None:
 513         Dummy, FileExt = os.path.splitext(File)
 514         if FileExt.lower() != Ext.lower():
 515             return False
 516     if not os.path.exists(File):
 517         return False
 518     return True
 519
 520 def RealPath(File, Dir='', OverrideDir=''):
 521     NewFile = os.path.normpath(os.path.join(Dir, File))
 522     NewFile = GlobalData.gAllFiles[NewFile]
 523     if not NewFile and OverrideDir:
 524         NewFile = os.path.normpath(os.path.join(OverrideDir, File))
 525         NewFile = GlobalData.gAllFiles[NewFile]
 526     return NewFile
 527
 528 def RealPath2(File, Dir='', OverrideDir=''):
 529     if OverrideDir:
 530         NewFile = GlobalData.gAllFiles[os.path.normpath(os.path.join(OverrideDir, File))]
 531         if NewFile:
 532             if OverrideDir[-1] == os.path.sep:
 533                 return NewFile[len(OverrideDir):], NewFile[0:len(OverrideDir)]
 534             else:
 535                 return NewFile[len(OverrideDir)+1:], NewFile[0:len(OverrideDir)]
 536     if GlobalData.gAllFiles:
 537         NewFile = GlobalData.gAllFiles[os.path.normpath(os.path.join(Dir, File))]
 538     else:
 539         NewFile = os.path.normpath(os.path.join(Dir, File))
 540     if NewFile:
 541         if Dir:
 542             if Dir[-1] == os.path.sep:
 543                 return NewFile[len(Dir):], NewFile[0:len(Dir)]
 544             else:
 545                 return NewFile[len(Dir)+1:], NewFile[0:len(Dir)]
 546         else:
 547             return NewFile, ''
 548
 549     return None, None
 550
 551 ## Check if gvien file exists or not
 552 #
 553 #
 554 def ValidFile2(AllFiles, File, Ext=None, Workspace='', EfiSource='', EdkSource='', Dir='.', OverrideDir=''):
 555     NewFile = File
 556     if Ext != None:
 557         Dummy, FileExt = os.path.splitext(File)
 558         if FileExt.lower() != Ext.lower():
 559             return False, File
 560
 561     # Replace the Edk macros
 562     if OverrideDir != '' and OverrideDir != None:
 563         if OverrideDir.find('$(EFI_SOURCE)') > -1:
 564             OverrideDir = OverrideDir.replace('$(EFI_SOURCE)', EfiSource)
 565         if OverrideDir.find('$(EDK_SOURCE)') > -1:
 566             OverrideDir = OverrideDir.replace('$(EDK_SOURCE)', EdkSource)
 567
 568     # Replace the default dir to current dir
 569     if Dir == '.':
 570         Dir = os.getcwd()
 571         Dir = Dir[len(Workspace)+1:]
 572
 573     # First check if File has Edk definition itself
 574     if File.find('$(EFI_SOURCE)') > -1 or File.find('$(EDK_SOURCE)') > -1:
 575         NewFile = File.replace('$(EFI_SOURCE)', EfiSource)
 576         NewFile = NewFile.replace('$(EDK_SOURCE)', EdkSource)
 577         NewFile = AllFiles[os.path.normpath(NewFile)]
 578         if NewFile != None:
 579             return True, NewFile
 580
 581     # Second check the path with override value
 582     if OverrideDir != '' and OverrideDir != None:
 583         NewFile = AllFiles[os.path.normpath(os.path.join(OverrideDir, File))]
 584         if NewFile != None:
 585             return True, NewFile
 586
 587     # Last check the path with normal definitions
 588     File = os.path.join(Dir, File)
 589     NewFile = AllFiles[os.path.normpath(File)]
 590     if NewFile != None:
 591         return True, NewFile
 592
 593     return False, File
 594
 595 ## Check if gvien file exists or not
 596 #
 597 #
 598 def ValidFile3(AllFiles, File, Workspace='', EfiSource='', EdkSource='', Dir='.', OverrideDir=''):
 599     # Replace the Edk macros
 600     if OverrideDir != '' and OverrideDir != None:
 601         if OverrideDir.find('$(EFI_SOURCE)') > -1:
 602             OverrideDir = OverrideDir.replace('$(EFI_SOURCE)', EfiSource)
 603         if OverrideDir.find('$(EDK_SOURCE)') > -1:
 604             OverrideDir = OverrideDir.replace('$(EDK_SOURCE)', EdkSource)
 605
 606     # Replace the default dir to current dir
 607     # Dir is current module dir related to workspace
 608     if Dir == '.':
 609         Dir = os.getcwd()
 610         Dir = Dir[len(Workspace)+1:]
 611
 612     NewFile = File
 613     RelaPath = AllFiles[os.path.normpath(Dir)]
 614     NewRelaPath = RelaPath
 615
 616     while(True):
 617         # First check if File has Edk definition itself
 618         if File.find('$(EFI_SOURCE)') > -1 or File.find('$(EDK_SOURCE)') > -1:
 619             File = File.replace('$(EFI_SOURCE)', EfiSource)
 620             File = File.replace('$(EDK_SOURCE)', EdkSource)
 621             NewFile = AllFiles[os.path.normpath(File)]
 622             if NewFile != None:
 623                 NewRelaPath = os.path.dirname(NewFile)
 624                 File = os.path.basename(NewFile)
 625                 #NewRelaPath = NewFile[:len(NewFile) - len(File.replace("..\\", '').replace("../", '')) - 1]
 626                 break
 627
 628         # Second check the path with override value
 629         if OverrideDir != '' and OverrideDir != None:
 630             NewFile = AllFiles[os.path.normpath(os.path.join(OverrideDir, File))]
 631             if NewFile != None:
 632                 #NewRelaPath = os.path.dirname(NewFile)
 633                 NewRelaPath = NewFile[:len(NewFile) - len(File.replace("..\\", '').replace("../", '')) - 1]
 634                 break
 635
 636         # Last check the path with normal definitions
 637         NewFile = AllFiles[os.path.normpath(os.path.join(Dir, File))]
 638         if NewFile != None:
 639             break
 640
 641         # No file found
 642         break
 643
 644     return NewRelaPath, RelaPath, File
 645
 646
 647 def GetRelPath(Path1, Path2):
 648     FileName = os.path.basename(Path2)
 649     L1 = os.path.normpath(Path1).split(os.path.normpath('/'))
 650     L2 = os.path.normpath(Path2).split(os.path.normpath('/'))
 651     for Index in range(0, len(L1)):
 652         if L1[Index] != L2[Index]:
 653             FileName = '../' * (len(L1) - Index)
 654             for Index2 in range(Index, len(L2)):
 655                 FileName = os.path.join(FileName, L2[Index2])
 656             break
 657     return os.path.normpath(FileName)
 658
 659
 660 ## Get GUID value from given packages
 661 #
 662 #   @param      CName           The CName of the GUID
 663 #   @param      PackageList     List of packages looking-up in
 664 #
 665 #   @retval     GuidValue   if the CName is found in any given package
 666 #   @retval     None        if the CName is not found in all given packages
 667 #
 668 def GuidValue(CName, PackageList):
 669     for P in PackageList:
 670         if CName in P.Guids:
 671             return P.Guids[CName]
 672     return None
 673
 674 ## Get Protocol value from given packages
 675 #
 676 #   @param      CName           The CName of the GUID
 677 #   @param      PackageList     List of packages looking-up in
 678 #
 679 #   @retval     GuidValue   if the CName is found in any given package
 680 #   @retval     None        if the CName is not found in all given packages
 681 #
 682 def ProtocolValue(CName, PackageList):
 683     for P in PackageList:
 684         if CName in P.Protocols:
 685             return P.Protocols[CName]
 686     return None
 687
 688 ## Get PPI value from given packages
 689 #
 690 #   @param      CName           The CName of the GUID
 691 #   @param      PackageList     List of packages looking-up in
 692 #
 693 #   @retval     GuidValue   if the CName is found in any given package
 694 #   @retval     None        if the CName is not found in all given packages
 695 #
 696 def PpiValue(CName, PackageList):
 697     for P in PackageList:
 698         if CName in P.Ppis:
 699             return P.Ppis[CName]
 700     return None
 701
 702 ## A string template class
 703 #
 704 #  This class implements a template for string replacement. A string template
 705 #  looks like following
 706 #
 707 #       ${BEGIN} other_string ${placeholder_name} other_string ${END}
 708 #
 709 #  The string between ${BEGIN} and ${END} will be repeated as many times as the
 710 #  length of "placeholder_name", which is a list passed through a dict. The
 711 #  "placeholder_name" is the key name of the dict. The ${BEGIN} and ${END} can
 712 #  be not used and, in this case, the "placeholder_name" must not a list and it
 713 #  will just be replaced once.
 714 #
 715 class TemplateString(object):
 716     _REPEAT_START_FLAG = "BEGIN"
 717     _REPEAT_END_FLAG = "END"
 718
 719     class Section(object):
 720         _LIST_TYPES = [type([]), type(set()), type((0,))]
 721
 722         def __init__(self, TemplateSection, PlaceHolderList):
 723             self._Template = TemplateSection
 724             self._PlaceHolderList = []
 725
 726             # Split the section into sub-sections according to the position of placeholders
 727             if PlaceHolderList:
 728                 self._SubSectionList = []
 729                 SubSectionStart = 0
 730                 #
 731                 # The placeholders passed in must be in the format of
 732                 #
 733                 #   PlaceHolderName, PlaceHolderStartPoint, PlaceHolderEndPoint
 734                 #
 735                 for PlaceHolder,Start,End in PlaceHolderList:
 736                     self._SubSectionList.append(TemplateSection[SubSectionStart:Start])
 737                     self._SubSectionList.append(TemplateSection[Start:End])
 738                     self._PlaceHolderList.append(PlaceHolder)
 739                     SubSectionStart = End
 740                 if SubSectionStart < len(TemplateSection):
 741                     self._SubSectionList.append(TemplateSection[SubSectionStart:])
 742             else:
 743                 self._SubSectionList = [TemplateSection]
 744
 745         def __str__(self):
 746             return self._Template + " : " + str(self._PlaceHolderList)
 747
 748         def Instantiate(self, PlaceHolderValues):
 749             RepeatTime = -1
 750             RepeatPlaceHolders = {}
 751             NonRepeatPlaceHolders = {}
 752
 753             for PlaceHolder in self._PlaceHolderList:
 754                 if PlaceHolder not in PlaceHolderValues:
 755                     continue
 756                 Value = PlaceHolderValues[PlaceHolder]
 757                 if type(Value) in self._LIST_TYPES:
 758                     if RepeatTime < 0:
 759                         RepeatTime = len(Value)
 760                     elif RepeatTime != len(Value):
 761                         EdkLogger.error(
 762                                     "TemplateString",
 763                                     PARAMETER_INVALID,
 764                                     "${%s} has different repeat time from others!" % PlaceHolder,
 765                                     ExtraData=str(self._Template)
 766                                     )
 767                     RepeatPlaceHolders["${%s}" % PlaceHolder] = Value
 768                 else:
 769                     NonRepeatPlaceHolders["${%s}" % PlaceHolder] = Value
 770
 771             if NonRepeatPlaceHolders:
 772                 StringList = []
 773                 for S in self._SubSectionList:
 774                     if S not in NonRepeatPlaceHolders:
 775                         StringList.append(S)
 776                     else:
 777                         StringList.append(str(NonRepeatPlaceHolders[S]))
 778             else:
 779                 StringList = self._SubSectionList
 780
 781             if RepeatPlaceHolders:
 782                 TempStringList = []
 783                 for Index in range(RepeatTime):
 784                     for S in StringList:
 785                         if S not in RepeatPlaceHolders:
 786                             TempStringList.append(S)
 787                         else:
 788                             TempStringList.append(str(RepeatPlaceHolders[S][Index]))
 789                 StringList = TempStringList
 790
 791             return "".join(StringList)
 792
 793     ## Constructor
 794     def __init__(self, Template=None):
 795         self.String = ''
 796         self.IsBinary = False
 797         self._Template = Template
 798         self._TemplateSectionList = self._Parse(Template)
 799
 800     ## str() operator
 801     #
 802     #   @retval     string  The string replaced
 803     #
 804     def __str__(self):
 805         return self.String
 806
 807     ## Split the template string into fragments per the ${BEGIN} and ${END} flags
 808     #
 809     #   @retval     list    A list of TemplateString.Section objects
 810     #
 811     def _Parse(self, Template):
 812         SectionStart = 0
 813         SearchFrom = 0
 814         MatchEnd = 0
 815         PlaceHolderList = []
 816         TemplateSectionList = []
 817         while Template:
 818             MatchObj = gPlaceholderPattern.search(Template, SearchFrom)
 819             if not MatchObj:
 820                 if MatchEnd <= len(Template):
 821                     TemplateSection = TemplateString.Section(Template[SectionStart:], PlaceHolderList)
 822                     TemplateSectionList.append(TemplateSection)
 823                 break
 824
 825             MatchString = MatchObj.group(1)
 826             MatchStart = MatchObj.start()
 827             MatchEnd = MatchObj.end()
 828
 829             if MatchString == self._REPEAT_START_FLAG:
 830                 if MatchStart > SectionStart:
 831                     TemplateSection = TemplateString.Section(Template[SectionStart:MatchStart], PlaceHolderList)
 832                     TemplateSectionList.append(TemplateSection)
 833                 SectionStart = MatchEnd
 834                 PlaceHolderList = []
 835             elif MatchString == self._REPEAT_END_FLAG:
 836                 TemplateSection = TemplateString.Section(Template[SectionStart:MatchStart], PlaceHolderList)
 837                 TemplateSectionList.append(TemplateSection)
 838                 SectionStart = MatchEnd
 839                 PlaceHolderList = []
 840             else:
 841                 PlaceHolderList.append((MatchString, MatchStart - SectionStart, MatchEnd - SectionStart))
 842             SearchFrom = MatchEnd
 843         return TemplateSectionList
 844
 845     ## Replace the string template with dictionary of placeholders and append it to previous one
 846     #
 847     #   @param      AppendString    The string template to append
 848     #   @param      Dictionary      The placeholder dictionaries
 849     #
 850     def Append(self, AppendString, Dictionary=None):
 851         if Dictionary:
 852             SectionList = self._Parse(AppendString)
 853             self.String += "".join([S.Instantiate(Dictionary) for S in SectionList])
 854         else:
 855             self.String += AppendString
 856
 857     ## Replace the string template with dictionary of placeholders
 858     #
 859     #   @param      Dictionary      The placeholder dictionaries
 860     #
 861     #   @retval     str             The string replaced with placeholder values
 862     #
 863     def Replace(self, Dictionary=None):
 864         return "".join([S.Instantiate(Dictionary) for S in self._TemplateSectionList])
 865
 866 ## Progress indicator class
 867 #
 868 #  This class makes use of thread to print progress on console.
 869 #
 870 class Progressor:
 871     # for avoiding deadloop
 872     _StopFlag = None
 873     _ProgressThread = None
 874     _CheckInterval = 0.25
 875
 876     ## Constructor
 877     #
 878     #   @param      OpenMessage     The string printed before progress charaters
 879     #   @param      CloseMessage    The string printed after progress charaters
 880     #   @param      ProgressChar    The charater used to indicate the progress
 881     #   @param      Interval        The interval in seconds between two progress charaters
 882     #
 883     def __init__(self, OpenMessage="", CloseMessage="", ProgressChar='.', Interval=1.0):
 884         self.PromptMessage = OpenMessage
 885         self.CodaMessage = CloseMessage
 886         self.ProgressChar = ProgressChar
 887         self.Interval = Interval
 888         if Progressor._StopFlag == None:
 889             Progressor._StopFlag = threading.Event()
 890
 891     ## Start to print progress charater
 892     #
 893     #   @param      OpenMessage     The string printed before progress charaters
 894     #
 895     def Start(self, OpenMessage=None):
 896         if OpenMessage != None:
 897             self.PromptMessage = OpenMessage
 898         Progressor._StopFlag.clear()
 899         if Progressor._ProgressThread == None:
 900             Progressor._ProgressThread = threading.Thread(target=self._ProgressThreadEntry)
 901             Progressor._ProgressThread.setDaemon(False)
 902             Progressor._ProgressThread.start()
 903
 904     ## Stop printing progress charater
 905     #
 906     #   @param      CloseMessage    The string printed after progress charaters
 907     #
 908     def Stop(self, CloseMessage=None):
 909         OriginalCodaMessage = self.CodaMessage
 910         if CloseMessage != None:
 911             self.CodaMessage = CloseMessage
 912         self.Abort()
 913         self.CodaMessage = OriginalCodaMessage
 914
 915     ## Thread entry method
 916     def _ProgressThreadEntry(self):
 917         sys.stdout.write(self.PromptMessage + " ")
 918         sys.stdout.flush()
 919         TimeUp = 0.0
 920         while not Progressor._StopFlag.isSet():
 921             if TimeUp <= 0.0:
 922                 sys.stdout.write(self.ProgressChar)
 923                 sys.stdout.flush()
 924                 TimeUp = self.Interval
 925             time.sleep(self._CheckInterval)
 926             TimeUp -= self._CheckInterval
 927         sys.stdout.write(" " + self.CodaMessage + "\n")
 928         sys.stdout.flush()
 929
 930     ## Abort the progress display
 931     @staticmethod
 932     def Abort():
 933         if Progressor._StopFlag != None:
 934             Progressor._StopFlag.set()
 935         if Progressor._ProgressThread != None:
 936             Progressor._ProgressThread.join()
 937             Progressor._ProgressThread = None
 938
 939 ## A dict which can access its keys and/or values orderly
 940 #
 941 #  The class implements a new kind of dict which its keys or values can be
 942 #  accessed in the order they are added into the dict. It guarantees the order
 943 #  by making use of an internal list to keep a copy of keys.
 944 #
 945 class sdict(IterableUserDict):
 946     ## Constructor
 947     def __init__(self):
 948         IterableUserDict.__init__(self)
 949         self._key_list = []
 950
 951     ## [] operator
 952     def __setitem__(self, key, value):
 953         if key not in self._key_list:
 954             self._key_list.append(key)
 955         IterableUserDict.__setitem__(self, key, value)
 956
 957     ## del operator
 958     def __delitem__(self, key):
 959         self._key_list.remove(key)
 960         IterableUserDict.__delitem__(self, key)
 961
 962     ## used in "for k in dict" loop to ensure the correct order
 963     def __iter__(self):
 964         return self.iterkeys()
 965
 966     ## len() support
 967     def __len__(self):
 968         return len(self._key_list)
 969
 970     ## "in" test support
 971     def __contains__(self, key):
 972         return key in self._key_list
 973
 974     ## indexof support
 975     def index(self, key):
 976         return self._key_list.index(key)
 977
 978     ## insert support
 979     def insert(self, key, newkey, newvalue, order):
 980         index = self._key_list.index(key)
 981         if order == 'BEFORE':
 982             self._key_list.insert(index, newkey)
 983             IterableUserDict.__setitem__(self, newkey, newvalue)
 984         elif order == 'AFTER':
 985             self._key_list.insert(index + 1, newkey)
 986             IterableUserDict.__setitem__(self, newkey, newvalue)
 987
 988     ## append support
 989     def append(self, sdict):
 990         for key in sdict:
 991             if key not in self._key_list:
 992                 self._key_list.append(key)
 993             IterableUserDict.__setitem__(self, key, sdict[key])
 994
 995     def has_key(self, key):
 996         return key in self._key_list
 997
 998     ## Empty the dict
 999     def clear(self):
1000         self._key_list = []
1001         IterableUserDict.clear(self)
1002
1003     ## Return a copy of keys
1004     def keys(self):
1005         keys = []
1006         for key in self._key_list:
1007             keys.append(key)
1008         return keys
1009
1010     ## Return a copy of values
1011     def values(self):
1012         values = []
1013         for key in self._key_list:
1014             values.append(self[key])
1015         return values
1016
1017     ## Return a copy of (key, value) list
1018     def items(self):
1019         items = []
1020         for key in self._key_list:
1021             items.append((key, self[key]))
1022         return items
1023
1024     ## Iteration support
1025     def iteritems(self):
1026         return iter(self.items())
1027
1028     ## Keys interation support
1029     def iterkeys(self):
1030         return iter(self.keys())
1031
1032     ## Values interation support
1033     def itervalues(self):
1034         return iter(self.values())
1035
1036     ## Return value related to a key, and remove the (key, value) from the dict
1037     def pop(self, key, *dv):
1038         value = None
1039         if key in self._key_list:
1040             value = self[key]
1041             self.__delitem__(key)
1042         elif len(dv) != 0 :
1043             value = kv[0]
1044         return value
1045
1046     ## Return (key, value) pair, and remove the (key, value) from the dict
1047     def popitem(self):
1048         key = self._key_list[-1]
1049         value = self[key]
1050         self.__delitem__(key)
1051         return key, value
1052
1053     def update(self, dict=None, **kwargs):
1054         if dict != None:
1055             for k, v in dict.items():
1056                 self[k] = v
1057         if len(kwargs):
1058             for k, v in kwargs.items():
1059                 self[k] = v
1060
1061 ## Dictionary with restricted keys
1062 #
1063 class rdict(dict):
1064     ## Constructor
1065     def __init__(self, KeyList):
1066         for Key in KeyList:
1067             dict.__setitem__(self, Key, "")
1068
1069     ## []= operator
1070     def __setitem__(self, key, value):
1071         if key not in self:
1072             EdkLogger.error("RestrictedDict", ATTRIBUTE_SET_FAILURE, "Key [%s] is not allowed" % key,
1073                             ExtraData=", ".join(dict.keys(self)))
1074         dict.__setitem__(self, key, value)
1075
1076     ## =[] operator
1077     def __getitem__(self, key):
1078         if key not in self:
1079             return ""
1080         return dict.__getitem__(self, key)
1081
1082     ## del operator
1083     def __delitem__(self, key):
1084         EdkLogger.error("RestrictedDict", ATTRIBUTE_ACCESS_DENIED, ExtraData="del")
1085
1086     ## Empty the dict
1087     def clear(self):
1088         for Key in self:
1089             self.__setitem__(Key, "")
1090
1091     ## Return value related to a key, and remove the (key, value) from the dict
1092     def pop(self, key, *dv):
1093         EdkLogger.error("RestrictedDict", ATTRIBUTE_ACCESS_DENIED, ExtraData="pop")
1094
1095     ## Return (key, value) pair, and remove the (key, value) from the dict
1096     def popitem(self):
1097         EdkLogger.error("RestrictedDict", ATTRIBUTE_ACCESS_DENIED, ExtraData="popitem")
1098
1099 ## Dictionary using prioritized list as key
1100 #
1101 class tdict:
1102     _ListType = type([])
1103     _TupleType = type(())
1104     _Wildcard = 'COMMON'
1105     _ValidWildcardList = ['COMMON', 'DEFAULT', 'ALL', '*', 'PLATFORM']
1106
1107     def __init__(self, _Single_=False, _Level_=2):
1108         self._Level_ = _Level_
1109         self.data = {}
1110         self._Single_ = _Single_
1111
1112     # =[] operator
1113     def __getitem__(self, key):
1114         KeyType = type(key)
1115         RestKeys = None
1116         if KeyType == self._ListType or KeyType == self._TupleType:
1117             FirstKey = key[0]
1118             if len(key) > 1:
1119                 RestKeys = key[1:]
1120             elif self._Level_ > 1:
1121                 RestKeys = [self._Wildcard for i in range(0, self._Level_-1)]
1122         else:
1123             FirstKey = key
1124             if self._Level_ > 1:
1125                 RestKeys = [self._Wildcard for i in range(0, self._Level_-1)]
1126
1127         if FirstKey == None or str(FirstKey).upper() in self._ValidWildcardList:
1128             FirstKey = self._Wildcard
1129
1130         if self._Single_:
1131             return self._GetSingleValue(FirstKey, RestKeys)
1132         else:
1133             return self._GetAllValues(FirstKey, RestKeys)
1134
1135     def _GetSingleValue(self, FirstKey, RestKeys):
1136         Value = None
1137         #print "%s-%s" % (FirstKey, self._Level_) ,
1138         if self._Level_ > 1:
1139             if FirstKey == self._Wildcard:
1140                 if FirstKey in self.data:
1141                     Value = self.data[FirstKey][RestKeys]
1142                 if Value == None:
1143                     for Key in self.data:
1144                         Value = self.data[Key][RestKeys]
1145                         if Value != None: break
1146             else:
1147                 if FirstKey in self.data:
1148                     Value = self.data[FirstKey][RestKeys]
1149                 if Value == None and self._Wildcard in self.data:
1150                     #print "Value=None"
1151                     Value = self.data[self._Wildcard][RestKeys]
1152         else:
1153             if FirstKey == self._Wildcard:
1154                 if FirstKey in self.data:
1155                     Value = self.data[FirstKey]
1156                 if Value == None:
1157                     for Key in self.data:
1158                         Value = self.data[Key]
1159                         if Value != None: break
1160             else:
1161                 if FirstKey in self.data:
1162                     Value = self.data[FirstKey]
1163                 elif self._Wildcard in self.data:
1164                     Value = self.data[self._Wildcard]
1165         return Value
1166
1167     def _GetAllValues(self, FirstKey, RestKeys):
1168         Value = []
1169         if self._Level_ > 1:
1170             if FirstKey == self._Wildcard:
1171                 for Key in self.data:
1172                     Value += self.data[Key][RestKeys]
1173             else:
1174                 if FirstKey in self.data:
1175                     Value += self.data[FirstKey][RestKeys]
1176                 if self._Wildcard in self.data:
1177                     Value += self.data[self._Wildcard][RestKeys]
1178         else:
1179             if FirstKey == self._Wildcard:
1180                 for Key in self.data:
1181                     Value.append(self.data[Key])
1182             else:
1183                 if FirstKey in self.data:
1184                     Value.append(self.data[FirstKey])
1185                 if self._Wildcard in self.data:
1186                     Value.append(self.data[self._Wildcard])
1187         return Value
1188
1189     ## []= operator
1190     def __setitem__(self, key, value):
1191         KeyType = type(key)
1192         RestKeys = None
1193         if KeyType == self._ListType or KeyType == self._TupleType:
1194             FirstKey = key[0]
1195             if len(key) > 1:
1196                 RestKeys = key[1:]
1197             else:
1198                 RestKeys = [self._Wildcard for i in range(0, self._Level_-1)]
1199         else:
1200             FirstKey = key
1201             if self._Level_ > 1:
1202                 RestKeys = [self._Wildcard for i in range(0, self._Level_-1)]
1203
1204         if FirstKey in self._ValidWildcardList:
1205             FirstKey = self._Wildcard
1206
1207         if FirstKey not in self.data and self._Level_ > 0:
1208             self.data[FirstKey] = tdict(self._Single_, self._Level_ - 1)
1209
1210         if self._Level_ > 1:
1211             self.data[FirstKey][RestKeys] = value
1212         else:
1213             self.data[FirstKey] = value
1214
1215     def SetGreedyMode(self):
1216         self._Single_ = False
1217         if self._Level_ > 1:
1218             for Key in self.data:
1219                 self.data[Key].SetGreedyMode()
1220
1221     def SetSingleMode(self):
1222         self._Single_ = True
1223         if self._Level_ > 1:
1224             for Key in self.data:
1225                 self.data[Key].SetSingleMode()
1226
1227     def GetKeys(self, KeyIndex=0):
1228         assert KeyIndex >= 0
1229         if KeyIndex == 0:
1230             return set(self.data.keys())
1231         else:
1232             keys = set()
1233             for Key in self.data:
1234                 keys |= self.data[Key].GetKeys(KeyIndex - 1)
1235             return keys
1236
1237 ## Boolean chain list
1238 #
1239 class Blist(UserList):
1240     def __init__(self, initlist=None):
1241         UserList.__init__(self, initlist)
1242     def __setitem__(self, i, item):
1243         if item not in [True, False]:
1244             if item == 0:
1245                 item = False
1246             else:
1247                 item = True
1248         self.data[i] = item
1249     def _GetResult(self):
1250         Value = True
1251         for item in self.data:
1252             Value &= item
1253         return Value
1254     Result = property(_GetResult)
1255
1256 def ParseConsoleLog(Filename):
1257     Opr = open(os.path.normpath(Filename), 'r')
1258     Opw = open(os.path.normpath(Filename + '.New'), 'w+')
1259     for Line in Opr.readlines():
1260         if Line.find('.efi') > -1:
1261             Line = Line[Line.rfind(' ') : Line.rfind('.efi')].strip()
1262             Opw.write('%s\n' % Line)
1263
1264     Opr.close()
1265     Opw.close()
1266
1267 ## AnalyzeDscPcd
1268 #
1269 #  Analyze DSC PCD value, since there is no data type info in DSC
1270 #  This fuction is used to match functions (AnalyzePcdData, AnalyzeHiiPcdData, AnalyzeVpdPcdData) used for retrieving PCD value from database
1271 #  1. Feature flag: TokenSpace.PcdCName|PcdValue
1272 #  2. Fix and Patch:TokenSpace.PcdCName|PcdValue[|MaxSize]
1273 #  3. Dynamic default:
1274 #     TokenSpace.PcdCName|PcdValue[|VOID*[|MaxSize]]
1275 #     TokenSpace.PcdCName|PcdValue
1276 #  4. Dynamic VPD:
1277 #     TokenSpace.PcdCName|VpdOffset[|VpdValue]
1278 #     TokenSpace.PcdCName|VpdOffset[|MaxSize[|VpdValue]]
1279 #  5. Dynamic HII:
1280 #     TokenSpace.PcdCName|HiiString|VaiableGuid|VariableOffset[|HiiValue]
1281 #  PCD value needs to be located in such kind of string, and the PCD value might be an expression in which
1282 #    there might have "|" operator, also in string value.
1283 #
1284 #  @param Setting: String contain information described above with "TokenSpace.PcdCName|" stripped
1285 #  @param PcdType: PCD type: feature, fixed, dynamic default VPD HII
1286 #  @param DataType: The datum type of PCD: VOID*, UNIT, BOOL
1287 #  @retval:
1288 #    ValueList: A List contain fields described above
1289 #    IsValid:   True if conforming EBNF, otherwise False
1290 #    Index:     The index where PcdValue is in ValueList
1291 #
1292 def AnalyzeDscPcd(Setting, PcdType, DataType=''):
1293     Setting = Setting.strip()
1294     # There might be escaped quote in a string: \", \\\"
1295     Data = Setting.replace('\\\\', '//').replace('\\\"', '\\\'')
1296     # There might be '|' in string and in ( ... | ... ), replace it with '-'
1297     NewStr = ''
1298     InStr = False
1299     Pair = 0
1300     for ch in Data:
1301         if ch == '"':
1302             InStr = not InStr
1303         elif ch == '(' and not InStr:
1304             Pair += 1
1305         elif ch == ')' and not InStr:
1306             Pair -= 1
1307
1308         if (Pair > 0 or InStr) and ch == TAB_VALUE_SPLIT:
1309             NewStr += '-'
1310         else:
1311             NewStr += ch
1312     FieldList = []
1313     StartPos = 0
1314     while True:
1315         Pos = NewStr.find(TAB_VALUE_SPLIT, StartPos)
1316         if Pos < 0:
1317             FieldList.append(Setting[StartPos:].strip())
1318             break
1319         FieldList.append(Setting[StartPos:Pos].strip())
1320         StartPos = Pos + 1
1321
1322     IsValid = True
1323     if PcdType in (MODEL_PCD_FIXED_AT_BUILD, MODEL_PCD_PATCHABLE_IN_MODULE, MODEL_PCD_FEATURE_FLAG):
1324         Value = FieldList[0]
1325         Size = ''
1326         if len(FieldList) > 1:
1327             Type = FieldList[1]
1328             # Fix the PCD type when no DataType input
1329             if Type == 'VOID*':
1330                 DataType = 'VOID*'
1331             else:
1332                 Size = FieldList[1]
1333         if len(FieldList) > 2:
1334             Size = FieldList[2]
1335         if DataType == 'VOID*':
1336             IsValid = (len(FieldList) <= 3)
1337         else:
1338             IsValid = (len(FieldList) <= 1)
1339         return [Value, '', Size], IsValid, 0
1340     elif PcdType in (MODEL_PCD_DYNAMIC_DEFAULT, MODEL_PCD_DYNAMIC_EX_DEFAULT):
1341         Value = FieldList[0]
1342         Size = Type = ''
1343         if len(FieldList) > 1:
1344             Type = FieldList[1]
1345         else:
1346             Type = DataType
1347         if len(FieldList) > 2:
1348             Size = FieldList[2]
1349         else:
1350             if Type == 'VOID*':
1351                 if Value.startswith("L"):
1352                     Size = str((len(Value)- 3 + 1) * 2)
1353                 elif Value.startswith("{"):
1354                     Size = str(len(Value.split(",")))
1355                 else:
1356                     Size = str(len(Value) -2 + 1 )
1357         if DataType == 'VOID*':
1358             IsValid = (len(FieldList) <= 3)
1359         else:
1360             IsValid = (len(FieldList) <= 1)
1361         return [Value, Type, Size], IsValid, 0
1362     elif PcdType in (MODEL_PCD_DYNAMIC_VPD, MODEL_PCD_DYNAMIC_EX_VPD):
1363         VpdOffset = FieldList[0]
1364         Value = Size = ''
1365         if not DataType == 'VOID*':
1366             if len(FieldList) > 1:
1367                 Value = FieldList[1]
1368         else:
1369             if len(FieldList) > 1:
1370                 Size = FieldList[1]
1371             if len(FieldList) > 2:
1372                 Value = FieldList[2]
1373         if DataType == 'VOID*':
1374             IsValid = (len(FieldList) <= 3)
1375         else:
1376             IsValid = (len(FieldList) <= 2)
1377         return [VpdOffset, Size, Value], IsValid, 2
1378     elif PcdType in (MODEL_PCD_DYNAMIC_HII, MODEL_PCD_DYNAMIC_EX_HII):
1379         HiiString = FieldList[0]
1380         Guid = Offset = Value = ''
1381         if len(FieldList) > 1:
1382             Guid = FieldList[1]
1383         if len(FieldList) > 2:
1384             Offset = FieldList[2]
1385         if len(FieldList) > 3:
1386             Value = FieldList[3]
1387         IsValid = (3 <= len(FieldList) <= 4)
1388         return [HiiString, Guid, Offset, Value], IsValid, 3
1389     return [], False, 0
1390
1391 ## AnalyzePcdData
1392 #
1393 #  Analyze the pcd Value, Datum type and TokenNumber.
1394 #  Used to avoid split issue while the value string contain "|" character
1395 #
1396 #  @param[in] Setting:  A String contain value/datum type/token number information;
1397 #
1398 #  @retval   ValueList: A List contain value, datum type and toke number.
1399 #
1400 def AnalyzePcdData(Setting):
1401     ValueList = ['', '', '']
1402
1403     ValueRe  = re.compile(r'^\s*L?\".*\|.*\"')
1404     PtrValue = ValueRe.findall(Setting)
1405
1406     ValueUpdateFlag = False
1407
1408     if len(PtrValue) >= 1:
1409         Setting = re.sub(ValueRe, '', Setting)
1410         ValueUpdateFlag = True
1411
1412     TokenList = Setting.split(TAB_VALUE_SPLIT)
1413     ValueList[0:len(TokenList)] = TokenList
1414
1415     if ValueUpdateFlag:
1416         ValueList[0] = PtrValue[0]
1417
1418     return ValueList
1419
1420 ## AnalyzeHiiPcdData
1421 #
1422 #  Analyze the pcd Value, variable name, variable Guid and variable offset.
1423 #  Used to avoid split issue while the value string contain "|" character
1424 #
1425 #  @param[in] Setting:  A String contain VariableName, VariableGuid, VariableOffset, DefaultValue information;
1426 #
1427 #  @retval   ValueList: A List contaian VariableName, VariableGuid, VariableOffset, DefaultValue.
1428 #
1429 def AnalyzeHiiPcdData(Setting):
1430     ValueList = ['', '', '', '']
1431
1432     TokenList = GetSplitValueList(Setting)
1433     ValueList[0:len(TokenList)] = TokenList
1434
1435     return ValueList
1436
1437 ## AnalyzeVpdPcdData
1438 #
1439 #  Analyze the vpd pcd VpdOffset, MaxDatumSize and InitialValue.
1440 #  Used to avoid split issue while the value string contain "|" character
1441 #
1442 #  @param[in] Setting:  A String contain VpdOffset/MaxDatumSize/InitialValue information;
1443 #
1444 #  @retval   ValueList: A List contain VpdOffset, MaxDatumSize and InitialValue.
1445 #
1446 def AnalyzeVpdPcdData(Setting):
1447     ValueList = ['', '', '']
1448
1449     ValueRe  = re.compile(r'\s*L?\".*\|.*\"\s*$')
1450     PtrValue = ValueRe.findall(Setting)
1451
1452     ValueUpdateFlag = False
1453
1454     if len(PtrValue) >= 1:
1455         Setting = re.sub(ValueRe, '', Setting)
1456         ValueUpdateFlag = True
1457
1458     TokenList = Setting.split(TAB_VALUE_SPLIT)
1459     ValueList[0:len(TokenList)] = TokenList
1460
1461     if ValueUpdateFlag:
1462         ValueList[2] = PtrValue[0]
1463
1464     return ValueList
1465
1466 ## check format of PCD value against its the datum type
1467 #
1468 # For PCD value setting
1469 #
1470 def CheckPcdDatum(Type, Value):
1471     if Type == "VOID*":
1472         ValueRe  = re.compile(r'\s*L?\".*\"\s*$')
1473         if not (((Value.startswith('L"') or Value.startswith('"')) and Value.endswith('"'))
1474                 or (Value.startswith('{') and Value.endswith('}'))
1475                ):
1476             return False, "Invalid value [%s] of type [%s]; must be in the form of {...} for array"\
1477                           ", or \"...\" for string, or L\"...\" for unicode string" % (Value, Type)
1478         elif ValueRe.match(Value):
1479             # Check the chars in UnicodeString or CString is printable
1480             if Value.startswith("L"):
1481                 Value = Value[2:-1]
1482             else:
1483                 Value = Value[1:-1]
1484             Printset = set(string.printable)
1485             Printset.remove(TAB_PRINTCHAR_VT)
1486             Printset.add(TAB_PRINTCHAR_BS)
1487             Printset.add(TAB_PRINTCHAR_NUL)
1488             if not set(Value).issubset(Printset):
1489                 PrintList = list(Printset)
1490                 PrintList.sort()
1491                 return False, "Invalid PCD string value of type [%s]; must be printable chars %s." % (Type, PrintList)
1492     elif Type == 'BOOLEAN':
1493         if Value not in ['TRUE', 'True', 'true', '0x1', '0x01', '1', 'FALSE', 'False', 'false', '0x0', '0x00', '0']:
1494             return False, "Invalid value [%s] of type [%s]; must be one of TRUE, True, true, 0x1, 0x01, 1"\
1495                           ", FALSE, False, false, 0x0, 0x00, 0" % (Value, Type)
1496     elif Type in [TAB_UINT8, TAB_UINT16, TAB_UINT32, TAB_UINT64]:
1497         try:
1498             Value = long(Value, 0)
1499         except:
1500             return False, "Invalid value [%s] of type [%s];"\
1501                           " must be a hexadecimal, decimal or octal in C language format." % (Value, Type)
1502     else:
1503         return False, "Invalid type [%s]; must be one of VOID*, BOOLEAN, UINT8, UINT16, UINT32, UINT64." % (Type)
1504
1505     return True, ""
1506
1507 ## Split command line option string to list
1508 #
1509 # subprocess.Popen needs the args to be a sequence. Otherwise there's problem
1510 # in non-windows platform to launch command
1511 #
1512 def SplitOption(OptionString):
1513     OptionList = []
1514     LastChar = " "
1515     OptionStart = 0
1516     QuotationMark = ""
1517     for Index in range(0, len(OptionString)):
1518         CurrentChar = OptionString[Index]
1519         if CurrentChar in ['"', "'"]:
1520             if QuotationMark == CurrentChar:
1521                 QuotationMark = ""
1522             elif QuotationMark == "":
1523                 QuotationMark = CurrentChar
1524             continue
1525         elif QuotationMark:
1526             continue
1527
1528         if CurrentChar in ["/", "-"] and LastChar in [" ", "\t", "\r", "\n"]:
1529             if Index > OptionStart:
1530                 OptionList.append(OptionString[OptionStart:Index-1])
1531             OptionStart = Index
1532         LastChar = CurrentChar
1533     OptionList.append(OptionString[OptionStart:])
1534     return OptionList
1535
1536 def CommonPath(PathList):
1537     P1 = min(PathList).split(os.path.sep)
1538     P2 = max(PathList).split(os.path.sep)
1539     for Index in xrange(min(len(P1), len(P2))):
1540         if P1[Index] != P2[Index]:
1541             return os.path.sep.join(P1[:Index])
1542     return os.path.sep.join(P1)
1543
1544 #
1545 # Convert string to C format array
1546 #
1547 def ConvertStringToByteArray(Value):
1548     Value = Value.strip()
1549     if not Value:
1550         return None
1551     if Value[0] == '{':
1552         if not Value.endswith('}'):
1553             return None
1554         Value = Value.replace(' ', '').replace('{', '').replace('}', '')
1555         ValFields = Value.split(',')
1556         try:
1557             for Index in range(len(ValFields)):
1558                 ValFields[Index] = str(int(ValFields[Index], 0))
1559         except ValueError:
1560             return None
1561         Value = '{' + ','.join(ValFields) + '}'
1562         return Value
1563
1564     Unicode = False
1565     if Value.startswith('L"'):
1566         if not Value.endswith('"'):
1567             return None
1568         Value = Value[1:]
1569         Unicode = True
1570     elif not Value.startswith('"') or not Value.endswith('"'):
1571         return None
1572
1573     Value = eval(Value)         # translate escape character
1574     NewValue = '{'
1575     for Index in range(0,len(Value)):
1576         if Unicode:
1577             NewValue = NewValue + str(ord(Value[Index]) % 0x10000) + ','
1578         else:
1579             NewValue = NewValue + str(ord(Value[Index]) % 0x100) + ','
1580     Value = NewValue + '0}'
1581     return Value
1582
1583 class PathClass(object):
1584     def __init__(self, File='', Root='', AlterRoot='', Type='', IsBinary=False,
1585                  Arch='COMMON', ToolChainFamily='', Target='', TagName='', ToolCode=''):
1586         self.Arch = Arch
1587         self.File = str(File)
1588         if os.path.isabs(self.File):
1589             self.Root = ''
1590             self.AlterRoot = ''
1591         else:
1592             self.Root = str(Root)
1593             self.AlterRoot = str(AlterRoot)
1594
1595         # Remove any '.' and '..' in path
1596         if self.Root:
1597             self.Path = os.path.normpath(os.path.join(self.Root, self.File))
1598             self.Root = os.path.normpath(CommonPath([self.Root, self.Path]))
1599             # eliminate the side-effect of 'C:'
1600             if self.Root[-1] == ':':
1601                 self.Root += os.path.sep
1602             # file path should not start with path separator
1603             if self.Root[-1] == os.path.sep:
1604                 self.File = self.Path[len(self.Root):]
1605             else:
1606                 self.File = self.Path[len(self.Root)+1:]
1607         else:
1608             self.Path = os.path.normpath(self.File)
1609
1610         self.SubDir, self.Name = os.path.split(self.File)
1611         self.BaseName, self.Ext = os.path.splitext(self.Name)
1612
1613         if self.Root:
1614             if self.SubDir:
1615                 self.Dir = os.path.join(self.Root, self.SubDir)
1616             else:
1617                 self.Dir = self.Root
1618         else:
1619             self.Dir = self.SubDir
1620
1621         if IsBinary:
1622             self.Type = Type
1623         else:
1624             self.Type = self.Ext.lower()
1625
1626         self.IsBinary = IsBinary
1627         self.Target = Target
1628         self.TagName = TagName
1629         self.ToolCode = ToolCode
1630         self.ToolChainFamily = ToolChainFamily
1631
1632         self._Key = None
1633
1634     ## Convert the object of this class to a string
1635     #
1636     #  Convert member Path of the class to a string
1637     #
1638     #  @retval string Formatted String
1639     #
1640     def __str__(self):
1641         return self.Path
1642
1643     ## Override __eq__ function
1644     #
1645     # Check whether PathClass are the same
1646     #
1647     # @retval False The two PathClass are different
1648     # @retval True  The two PathClass are the same
1649     #
1650     def __eq__(self, Other):
1651         if type(Other) == type(self):
1652             return self.Path == Other.Path
1653         else:
1654             return self.Path == str(Other)
1655
1656     ## Override __cmp__ function
1657     #
1658     # Customize the comparsion operation of two PathClass
1659     #
1660     # @retval 0     The two PathClass are different
1661     # @retval -1    The first PathClass is less than the second PathClass
1662     # @retval 1     The first PathClass is Bigger than the second PathClass
1663     def __cmp__(self, Other):
1664         if type(Other) == type(self):
1665             OtherKey = Other.Path
1666         else:
1667             OtherKey = str(Other)
1668
1669         SelfKey = self.Path
1670         if SelfKey == OtherKey:
1671             return 0
1672         elif SelfKey > OtherKey:
1673             return 1
1674         else:
1675             return -1
1676
1677     ## Override __hash__ function
1678     #
1679     # Use Path as key in hash table
1680     #
1681     # @retval string Key for hash table
1682     #
1683     def __hash__(self):
1684         return hash(self.Path)
1685
1686     def _GetFileKey(self):
1687         if self._Key == None:
1688             self._Key = self.Path.upper()   # + self.ToolChainFamily + self.TagName + self.ToolCode + self.Target
1689         return self._Key
1690
1691     def _GetTimeStamp(self):
1692         return os.stat(self.Path)[8]
1693
1694     def Validate(self, Type='', CaseSensitive=True):
1695         if GlobalData.gCaseInsensitive:
1696             CaseSensitive = False
1697         if Type and Type.lower() != self.Type:
1698             return FILE_TYPE_MISMATCH, '%s (expect %s but got %s)' % (self.File, Type, self.Type)
1699
1700         RealFile, RealRoot = RealPath2(self.File, self.Root, self.AlterRoot)
1701         if not RealRoot and not RealFile:
1702             RealFile = self.File
1703             if self.AlterRoot:
1704                 RealFile = os.path.join(self.AlterRoot, self.File)
1705             elif self.Root:
1706                 RealFile = os.path.join(self.Root, self.File)
1707             return FILE_NOT_FOUND, os.path.join(self.AlterRoot, RealFile)
1708
1709         ErrorCode = 0
1710         ErrorInfo = ''
1711         if RealRoot != self.Root or RealFile != self.File:
1712             if CaseSensitive and (RealFile != self.File or (RealRoot != self.Root and RealRoot != self.AlterRoot)):
1713                 ErrorCode = FILE_CASE_MISMATCH
1714                 ErrorInfo = self.File + '\n\t' + RealFile + " [in file system]"
1715
1716             self.SubDir, self.Name = os.path.split(RealFile)
1717             self.BaseName, self.Ext = os.path.splitext(self.Name)
1718             if self.SubDir:
1719                 self.Dir = os.path.join(RealRoot, self.SubDir)
1720             else:
1721                 self.Dir = RealRoot
1722             self.File = RealFile
1723             self.Root = RealRoot
1724             self.Path = os.path.join(RealRoot, RealFile)
1725         return ErrorCode, ErrorInfo
1726
1727     Key = property(_GetFileKey)
1728     TimeStamp = property(_GetTimeStamp)
1729
1730 ## Parse PE image to get the required PE informaion.
1731 #
1732 class PeImageClass():
1733     ## Constructor
1734     #
1735     #   @param  File FilePath of PeImage
1736     #
1737     def __init__(self, PeFile):
1738         self.FileName   = PeFile
1739         self.IsValid    = False
1740         self.Size       = 0
1741         self.EntryPoint = 0
1742         self.SectionAlignment  = 0
1743         self.SectionHeaderList = []
1744         self.ErrorInfo = ''
1745         try:
1746             PeObject = open(PeFile, 'rb')
1747         except:
1748             self.ErrorInfo = self.FileName + ' can not be found\n'
1749             return
1750         # Read DOS header
1751         ByteArray = array.array('B')
1752         ByteArray.fromfile(PeObject, 0x3E)
1753         ByteList = ByteArray.tolist()
1754         # DOS signature should be 'MZ'
1755         if self._ByteListToStr (ByteList[0x0:0x2]) != 'MZ':
1756             self.ErrorInfo = self.FileName + ' has no valid DOS signature MZ'
1757             return
1758
1759         # Read 4 byte PE Signature
1760         PeOffset = self._ByteListToInt(ByteList[0x3C:0x3E])
1761         PeObject.seek(PeOffset)
1762         ByteArray = array.array('B')
1763         ByteArray.fromfile(PeObject, 4)
1764         # PE signature should be 'PE\0\0'
1765         if ByteArray.tostring() != 'PE\0\0':
1766             self.ErrorInfo = self.FileName + ' has no valid PE signature PE00'
1767             return
1768
1769         # Read PE file header
1770         ByteArray = array.array('B')
1771         ByteArray.fromfile(PeObject, 0x14)
1772         ByteList = ByteArray.tolist()
1773         SecNumber = self._ByteListToInt(ByteList[0x2:0x4])
1774         if SecNumber == 0:
1775             self.ErrorInfo = self.FileName + ' has no section header'
1776             return
1777
1778         # Read PE optional header
1779         OptionalHeaderSize = self._ByteListToInt(ByteArray[0x10:0x12])
1780         ByteArray = array.array('B')
1781         ByteArray.fromfile(PeObject, OptionalHeaderSize)
1782         ByteList = ByteArray.tolist()
1783         self.EntryPoint       = self._ByteListToInt(ByteList[0x10:0x14])
1784         self.SectionAlignment = self._ByteListToInt(ByteList[0x20:0x24])
1785         self.Size             = self._ByteListToInt(ByteList[0x38:0x3C])
1786
1787         # Read each Section Header
1788         for Index in range(SecNumber):
1789             ByteArray = array.array('B')
1790             ByteArray.fromfile(PeObject, 0x28)
1791             ByteList = ByteArray.tolist()
1792             SecName  = self._ByteListToStr(ByteList[0:8])
1793             SecVirtualSize = self._ByteListToInt(ByteList[8:12])
1794             SecRawAddress  = self._ByteListToInt(ByteList[20:24])
1795             SecVirtualAddress = self._ByteListToInt(ByteList[12:16])
1796             self.SectionHeaderList.append((SecName, SecVirtualAddress, SecRawAddress, SecVirtualSize))
1797         self.IsValid = True
1798         PeObject.close()
1799
1800     def _ByteListToStr(self, ByteList):
1801         String = ''
1802         for index in range(len(ByteList)):
1803             if ByteList[index] == 0:
1804                 break
1805             String += chr(ByteList[index])
1806         return String
1807
1808     def _ByteListToInt(self, ByteList):
1809         Value = 0
1810         for index in range(len(ByteList) - 1, -1, -1):
1811             Value = (Value << 8) | int(ByteList[index])
1812         return Value
1813
1814
1815 class SkuClass():
1816
1817     DEFAULT = 0
1818     SINGLE = 1
1819     MULTIPLE =2
1820
1821     def __init__(self,SkuIdentifier='', SkuIds={}):
1822
1823         self.AvailableSkuIds = sdict()
1824         self.SkuIdSet = []
1825
1826         if SkuIdentifier == '' or SkuIdentifier is None:
1827             self.SkuIdSet = ['DEFAULT']
1828         elif SkuIdentifier == 'ALL':
1829             self.SkuIdSet = SkuIds.keys()
1830         else:
1831             r = SkuIdentifier.split('|')
1832             self.SkuIdSet=[r[k].strip() for k in range(len(r))]
1833         if len(self.SkuIdSet) == 2 and 'DEFAULT' in self.SkuIdSet and SkuIdentifier != 'ALL':
1834             self.SkuIdSet.remove('DEFAULT')
1835
1836         for each in self.SkuIdSet:
1837             if each in SkuIds:
1838                 self.AvailableSkuIds[each] = SkuIds[each]
1839             else:
1840                 EdkLogger.error("build", PARAMETER_INVALID,
1841                             ExtraData="SKU-ID [%s] is not supported by the platform. [Valid SKU-ID: %s]"
1842                                       % (each, " ".join(SkuIds.keys())))
1843
1844     def __SkuUsageType(self):
1845
1846         if len(self.SkuIdSet) == 1:
1847             if self.SkuIdSet[0] == 'DEFAULT':
1848                 return SkuClass.DEFAULT
1849             else:
1850                 return SkuClass.SINGLE
1851         else:
1852             return SkuClass.MULTIPLE
1853
1854     def __GetAvailableSkuIds(self):
1855         return self.AvailableSkuIds
1856
1857     def __GetSystemSkuID(self):
1858         if self.__SkuUsageType() == SkuClass.SINGLE:
1859             return self.SkuIdSet[0]
1860         else:
1861             return 'DEFAULT'
1862
1863     SystemSkuId = property(__GetSystemSkuID)
1864     AvailableSkuIdSet = property(__GetAvailableSkuIds)
1865     SkuUsageType = property(__SkuUsageType)
1866
1867 ##
1868 #
1869 # This acts like the main() function for the script, unless it is 'import'ed into another
1870 # script.
1871 #
1872 if __name__ == '__main__':
1873     pass
1874