BaseTools/Source/Python/Common/Misc.py

   1 ## @file
   2 # Common routines used by all tools
   3 #
   4 # Copyright (c) 2007 - 2014, Intel Corporation. All rights reserved.<BR>
   5 # This program and the accompanying materials
   6 # are licensed and made available under the terms and conditions of the BSD License
   7 # which accompanies this distribution.  The full text of the license may be found at
   8 # http://opensource.org/licenses/bsd-license.php
   9 #
  10 # THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS,
  11 # WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED.
  12 #
  13
  14 ##
  15 # Import Modules
  16 #
  17 import Common.LongFilePathOs as os
  18 import sys
  19 import string
  20 import thread
  21 import threading
  22 import time
  23 import re
  24 import cPickle
  25 import array
  26 from UserDict import IterableUserDict
  27 from UserList import UserList
  28
  29 from Common import EdkLogger as EdkLogger
  30 from Common import GlobalData as GlobalData
  31 from DataType import *
  32 from BuildToolError import *
  33 from CommonDataClass.DataClass import *
  34 from Parsing import GetSplitValueList
  35 from Common.LongFilePathSupport import OpenLongFilePath as open
  36
  37 ## Regular expression used to find out place holders in string template
  38 gPlaceholderPattern = re.compile("\$\{([^$()\s]+)\}", re.MULTILINE|re.UNICODE)
  39
  40 ## Dictionary used to store file time stamp for quick re-access
  41 gFileTimeStampCache = {}    # {file path : file time stamp}
  42
  43 ## Dictionary used to store dependencies of files
  44 gDependencyDatabase = {}    # arch : {file path : [dependent files list]}
  45
  46 ## callback routine for processing variable option
  47 #
  48 # This function can be used to process variable number of option values. The
  49 # typical usage of it is specify architecure list on command line.
  50 # (e.g. <tool> -a IA32 X64 IPF)
  51 #
  52 # @param  Option        Standard callback function parameter
  53 # @param  OptionString  Standard callback function parameter
  54 # @param  Value         Standard callback function parameter
  55 # @param  Parser        Standard callback function parameter
  56 #
  57 # @retval
  58 #
  59 def ProcessVariableArgument(Option, OptionString, Value, Parser):
  60     assert Value is None
  61     Value = []
  62     RawArgs = Parser.rargs
  63     while RawArgs:
  64         Arg = RawArgs[0]
  65         if (Arg[:2] == "--" and len(Arg) > 2) or \
  66            (Arg[:1] == "-" and len(Arg) > 1 and Arg[1] != "-"):
  67             break
  68         Value.append(Arg)
  69         del RawArgs[0]
  70     setattr(Parser.values, Option.dest, Value)
  71
  72 ## Convert GUID string in xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx style to C structure style
  73 #
  74 #   @param      Guid    The GUID string
  75 #
  76 #   @retval     string  The GUID string in C structure style
  77 #
  78 def GuidStringToGuidStructureString(Guid):
  79     GuidList = Guid.split('-')
  80     Result = '{'
  81     for Index in range(0,3,1):
  82         Result = Result + '0x' + GuidList[Index] + ', '
  83     Result = Result + '{0x' + GuidList[3][0:2] + ', 0x' + GuidList[3][2:4]
  84     for Index in range(0,12,2):
  85         Result = Result + ', 0x' + GuidList[4][Index:Index+2]
  86     Result += '}}'
  87     return Result
  88
  89 ## Convert GUID structure in byte array to xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
  90 #
  91 #   @param      GuidValue   The GUID value in byte array
  92 #
  93 #   @retval     string      The GUID value in xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx format
  94 #
  95 def GuidStructureByteArrayToGuidString(GuidValue):
  96     guidValueString = GuidValue.lower().replace("{", "").replace("}", "").replace(" ", "").replace(";", "")
  97     guidValueList = guidValueString.split(",")
  98     if len(guidValueList) != 16:
  99         return ''
 100         #EdkLogger.error(None, None, "Invalid GUID value string %s" % GuidValue)
 101     try:
 102         return "%02x%02x%02x%02x-%02x%02x-%02x%02x-%02x%02x-%02x%02x%02x%02x%02x%02x" % (
 103                 int(guidValueList[3], 16),
 104                 int(guidValueList[2], 16),
 105                 int(guidValueList[1], 16),
 106                 int(guidValueList[0], 16),
 107                 int(guidValueList[5], 16),
 108                 int(guidValueList[4], 16),
 109                 int(guidValueList[7], 16),
 110                 int(guidValueList[6], 16),
 111                 int(guidValueList[8], 16),
 112                 int(guidValueList[9], 16),
 113                 int(guidValueList[10], 16),
 114                 int(guidValueList[11], 16),
 115                 int(guidValueList[12], 16),
 116                 int(guidValueList[13], 16),
 117                 int(guidValueList[14], 16),
 118                 int(guidValueList[15], 16)
 119                 )
 120     except:
 121         return ''
 122
 123 ## Convert GUID string in C structure style to xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
 124 #
 125 #   @param      GuidValue   The GUID value in C structure format
 126 #
 127 #   @retval     string      The GUID value in xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx format
 128 #
 129 def GuidStructureStringToGuidString(GuidValue):
 130     guidValueString = GuidValue.lower().replace("{", "").replace("}", "").replace(" ", "").replace(";", "")
 131     guidValueList = guidValueString.split(",")
 132     if len(guidValueList) != 11:
 133         return ''
 134         #EdkLogger.error(None, None, "Invalid GUID value string %s" % GuidValue)
 135     try:
 136         return "%08x-%04x-%04x-%02x%02x-%02x%02x%02x%02x%02x%02x" % (
 137                 int(guidValueList[0], 16),
 138                 int(guidValueList[1], 16),
 139                 int(guidValueList[2], 16),
 140                 int(guidValueList[3], 16),
 141                 int(guidValueList[4], 16),
 142                 int(guidValueList[5], 16),
 143                 int(guidValueList[6], 16),
 144                 int(guidValueList[7], 16),
 145                 int(guidValueList[8], 16),
 146                 int(guidValueList[9], 16),
 147                 int(guidValueList[10], 16)
 148                 )
 149     except:
 150         return ''
 151
 152 ## Convert GUID string in C structure style to xxxxxxxx_xxxx_xxxx_xxxx_xxxxxxxxxxxx
 153 #
 154 #   @param      GuidValue   The GUID value in C structure format
 155 #
 156 #   @retval     string      The GUID value in xxxxxxxx_xxxx_xxxx_xxxx_xxxxxxxxxxxx format
 157 #
 158 def GuidStructureStringToGuidValueName(GuidValue):
 159     guidValueString = GuidValue.lower().replace("{", "").replace("}", "").replace(" ", "")
 160     guidValueList = guidValueString.split(",")
 161     if len(guidValueList) != 11:
 162         EdkLogger.error(None, FORMAT_INVALID, "Invalid GUID value string [%s]" % GuidValue)
 163     return "%08x_%04x_%04x_%02x%02x_%02x%02x%02x%02x%02x%02x" % (
 164             int(guidValueList[0], 16),
 165             int(guidValueList[1], 16),
 166             int(guidValueList[2], 16),
 167             int(guidValueList[3], 16),
 168             int(guidValueList[4], 16),
 169             int(guidValueList[5], 16),
 170             int(guidValueList[6], 16),
 171             int(guidValueList[7], 16),
 172             int(guidValueList[8], 16),
 173             int(guidValueList[9], 16),
 174             int(guidValueList[10], 16)
 175             )
 176
 177 ## Create directories
 178 #
 179 #   @param      Directory   The directory name
 180 #
 181 def CreateDirectory(Directory):
 182     if Directory == None or Directory.strip() == "":
 183         return True
 184     try:
 185         if not os.access(Directory, os.F_OK):
 186             os.makedirs(Directory)
 187     except:
 188         return False
 189     return True
 190
 191 ## Remove directories, including files and sub-directories in it
 192 #
 193 #   @param      Directory   The directory name
 194 #
 195 def RemoveDirectory(Directory, Recursively=False):
 196     if Directory == None or Directory.strip() == "" or not os.path.exists(Directory):
 197         return
 198     if Recursively:
 199         CurrentDirectory = os.getcwd()
 200         os.chdir(Directory)
 201         for File in os.listdir("."):
 202             if os.path.isdir(File):
 203                 RemoveDirectory(File, Recursively)
 204             else:
 205                 os.remove(File)
 206         os.chdir(CurrentDirectory)
 207     os.rmdir(Directory)
 208
 209 ## Check if given file is changed or not
 210 #
 211 #  This method is used to check if a file is changed or not between two build
 212 #  actions. It makes use a cache to store files timestamp.
 213 #
 214 #   @param      File    The path of file
 215 #
 216 #   @retval     True    If the given file is changed, doesn't exist, or can't be
 217 #                       found in timestamp cache
 218 #   @retval     False   If the given file is changed
 219 #
 220 def IsChanged(File):
 221     if not os.path.exists(File):
 222         return True
 223
 224     FileState = os.stat(File)
 225     TimeStamp = FileState[-2]
 226
 227     if File in gFileTimeStampCache and TimeStamp == gFileTimeStampCache[File]:
 228         FileChanged = False
 229     else:
 230         FileChanged = True
 231         gFileTimeStampCache[File] = TimeStamp
 232
 233     return FileChanged
 234
 235 ## Store content in file
 236 #
 237 #  This method is used to save file only when its content is changed. This is
 238 #  quite useful for "make" system to decide what will be re-built and what won't.
 239 #
 240 #   @param      File            The path of file
 241 #   @param      Content         The new content of the file
 242 #   @param      IsBinaryFile    The flag indicating if the file is binary file or not
 243 #
 244 #   @retval     True            If the file content is changed and the file is renewed
 245 #   @retval     False           If the file content is the same
 246 #
 247 def SaveFileOnChange(File, Content, IsBinaryFile=True):
 248     if not IsBinaryFile:
 249         Content = Content.replace("\n", os.linesep)
 250
 251     if os.path.exists(File):
 252         try:
 253             if Content == open(File, "rb").read():
 254                 return False
 255         except:
 256             EdkLogger.error(None, FILE_OPEN_FAILURE, ExtraData=File)
 257
 258     DirName = os.path.dirname(File)
 259     if not CreateDirectory(DirName):
 260         EdkLogger.error(None, FILE_CREATE_FAILURE, "Could not create directory %s" % DirName)
 261     else:
 262         if DirName == '':
 263             DirName = os.getcwd()
 264         if not os.access(DirName, os.W_OK):
 265             EdkLogger.error(None, PERMISSION_FAILURE, "Do not have write permission on directory %s" % DirName)
 266
 267     try:
 268         if GlobalData.gIsWindows:
 269             try:
 270                 from PyUtility import SaveFileToDisk
 271                 if not SaveFileToDisk(File, Content):
 272                     EdkLogger.error(None, FILE_CREATE_FAILURE, ExtraData=File)
 273             except:
 274                 Fd = open(File, "wb")
 275                 Fd.write(Content)
 276                 Fd.close()
 277         else:
 278             Fd = open(File, "wb")
 279             Fd.write(Content)
 280             Fd.close()
 281     except IOError, X:
 282         EdkLogger.error(None, FILE_CREATE_FAILURE, ExtraData='IOError %s'%X)
 283
 284     return True
 285
 286 ## Make a Python object persistent on file system
 287 #
 288 #   @param      Data    The object to be stored in file
 289 #   @param      File    The path of file to store the object
 290 #
 291 def DataDump(Data, File):
 292     Fd = None
 293     try:
 294         Fd = open(File, 'wb')
 295         cPickle.dump(Data, Fd, cPickle.HIGHEST_PROTOCOL)
 296     except:
 297         EdkLogger.error("", FILE_OPEN_FAILURE, ExtraData=File, RaiseError=False)
 298     finally:
 299         if Fd != None:
 300             Fd.close()
 301
 302 ## Restore a Python object from a file
 303 #
 304 #   @param      File    The path of file stored the object
 305 #
 306 #   @retval     object  A python object
 307 #   @retval     None    If failure in file operation
 308 #
 309 def DataRestore(File):
 310     Data = None
 311     Fd = None
 312     try:
 313         Fd = open(File, 'rb')
 314         Data = cPickle.load(Fd)
 315     except Exception, e:
 316         EdkLogger.verbose("Failed to load [%s]\n\t%s" % (File, str(e)))
 317         Data = None
 318     finally:
 319         if Fd != None:
 320             Fd.close()
 321     return Data
 322
 323 ## Retrieve and cache the real path name in file system
 324 #
 325 #   @param      Root    The root directory of path relative to
 326 #
 327 #   @retval     str     The path string if the path exists
 328 #   @retval     None    If path doesn't exist
 329 #
 330 class DirCache:
 331     _CACHE_ = set()
 332     _UPPER_CACHE_ = {}
 333
 334     def __init__(self, Root):
 335         self._Root = Root
 336         for F in os.listdir(Root):
 337             self._CACHE_.add(F)
 338             self._UPPER_CACHE_[F.upper()] = F
 339
 340     # =[] operator
 341     def __getitem__(self, Path):
 342         Path = Path[len(os.path.commonprefix([Path, self._Root])):]
 343         if not Path:
 344             return self._Root
 345         if Path and Path[0] == os.path.sep:
 346             Path = Path[1:]
 347         if Path in self._CACHE_:
 348             return os.path.join(self._Root, Path)
 349         UpperPath = Path.upper()
 350         if UpperPath in self._UPPER_CACHE_:
 351             return os.path.join(self._Root, self._UPPER_CACHE_[UpperPath])
 352
 353         IndexList = []
 354         LastSepIndex = -1
 355         SepIndex = Path.find(os.path.sep)
 356         while SepIndex > -1:
 357             Parent = UpperPath[:SepIndex]
 358             if Parent not in self._UPPER_CACHE_:
 359                 break
 360             LastSepIndex = SepIndex
 361             SepIndex = Path.find(os.path.sep, LastSepIndex + 1)
 362
 363         if LastSepIndex == -1:
 364             return None
 365
 366         Cwd = os.getcwd()
 367         os.chdir(self._Root)
 368         SepIndex = LastSepIndex
 369         while SepIndex > -1:
 370             Parent = Path[:SepIndex]
 371             ParentKey = UpperPath[:SepIndex]
 372             if ParentKey not in self._UPPER_CACHE_:
 373                 os.chdir(Cwd)
 374                 return None
 375
 376             if Parent in self._CACHE_:
 377                 ParentDir = Parent
 378             else:
 379                 ParentDir = self._UPPER_CACHE_[ParentKey]
 380             for F in os.listdir(ParentDir):
 381                 Dir = os.path.join(ParentDir, F)
 382                 self._CACHE_.add(Dir)
 383                 self._UPPER_CACHE_[Dir.upper()] = Dir
 384
 385             SepIndex = Path.find(os.path.sep, SepIndex + 1)
 386
 387         os.chdir(Cwd)
 388         if Path in self._CACHE_:
 389             return os.path.join(self._Root, Path)
 390         elif UpperPath in self._UPPER_CACHE_:
 391             return os.path.join(self._Root, self._UPPER_CACHE_[UpperPath])
 392         return None
 393
 394 ## Get all files of a directory
 395 #
 396 # @param Root:       Root dir
 397 # @param SkipList :  The files need be skipped
 398 #
 399 # @retval  A list of all files
 400 #
 401 def GetFiles(Root, SkipList=None, FullPath = True):
 402     OriPath = Root
 403     FileList = []
 404     for Root, Dirs, Files in os.walk(Root):
 405         if SkipList:
 406             for Item in SkipList:
 407                 if Item in Dirs:
 408                     Dirs.remove(Item)
 409
 410         for File in Files:
 411             File = os.path.normpath(os.path.join(Root, File))
 412             if not FullPath:
 413                 File = File[len(OriPath) + 1:]
 414             FileList.append(File)
 415
 416     return FileList
 417
 418 ## Check if gvien file exists or not
 419 #
 420 #   @param      File    File name or path to be checked
 421 #   @param      Dir     The directory the file is relative to
 422 #
 423 #   @retval     True    if file exists
 424 #   @retval     False   if file doesn't exists
 425 #
 426 def ValidFile(File, Ext=None):
 427     if Ext != None:
 428         Dummy, FileExt = os.path.splitext(File)
 429         if FileExt.lower() != Ext.lower():
 430             return False
 431     if not os.path.exists(File):
 432         return False
 433     return True
 434
 435 def RealPath(File, Dir='', OverrideDir=''):
 436     NewFile = os.path.normpath(os.path.join(Dir, File))
 437     NewFile = GlobalData.gAllFiles[NewFile]
 438     if not NewFile and OverrideDir:
 439         NewFile = os.path.normpath(os.path.join(OverrideDir, File))
 440         NewFile = GlobalData.gAllFiles[NewFile]
 441     return NewFile
 442
 443 def RealPath2(File, Dir='', OverrideDir=''):
 444     if OverrideDir:
 445         NewFile = GlobalData.gAllFiles[os.path.normpath(os.path.join(OverrideDir, File))]
 446         if NewFile:
 447             if OverrideDir[-1] == os.path.sep:
 448                 return NewFile[len(OverrideDir):], NewFile[0:len(OverrideDir)]
 449             else:
 450                 return NewFile[len(OverrideDir)+1:], NewFile[0:len(OverrideDir)]
 451     if GlobalData.gAllFiles:
 452         NewFile = GlobalData.gAllFiles[os.path.normpath(os.path.join(Dir, File))]
 453     else:
 454         NewFile = os.path.normpath(os.path.join(Dir, File))
 455     if NewFile:
 456         if Dir:
 457             if Dir[-1] == os.path.sep:
 458                 return NewFile[len(Dir):], NewFile[0:len(Dir)]
 459             else:
 460                 return NewFile[len(Dir)+1:], NewFile[0:len(Dir)]
 461         else:
 462             return NewFile, ''
 463
 464     return None, None
 465
 466 ## Check if gvien file exists or not
 467 #
 468 #
 469 def ValidFile2(AllFiles, File, Ext=None, Workspace='', EfiSource='', EdkSource='', Dir='.', OverrideDir=''):
 470     NewFile = File
 471     if Ext != None:
 472         Dummy, FileExt = os.path.splitext(File)
 473         if FileExt.lower() != Ext.lower():
 474             return False, File
 475
 476     # Replace the Edk macros
 477     if OverrideDir != '' and OverrideDir != None:
 478         if OverrideDir.find('$(EFI_SOURCE)') > -1:
 479             OverrideDir = OverrideDir.replace('$(EFI_SOURCE)', EfiSource)
 480         if OverrideDir.find('$(EDK_SOURCE)') > -1:
 481             OverrideDir = OverrideDir.replace('$(EDK_SOURCE)', EdkSource)
 482
 483     # Replace the default dir to current dir
 484     if Dir == '.':
 485         Dir = os.getcwd()
 486         Dir = Dir[len(Workspace)+1:]
 487
 488     # First check if File has Edk definition itself
 489     if File.find('$(EFI_SOURCE)') > -1 or File.find('$(EDK_SOURCE)') > -1:
 490         NewFile = File.replace('$(EFI_SOURCE)', EfiSource)
 491         NewFile = NewFile.replace('$(EDK_SOURCE)', EdkSource)
 492         NewFile = AllFiles[os.path.normpath(NewFile)]
 493         if NewFile != None:
 494             return True, NewFile
 495
 496     # Second check the path with override value
 497     if OverrideDir != '' and OverrideDir != None:
 498         NewFile = AllFiles[os.path.normpath(os.path.join(OverrideDir, File))]
 499         if NewFile != None:
 500             return True, NewFile
 501
 502     # Last check the path with normal definitions
 503     File = os.path.join(Dir, File)
 504     NewFile = AllFiles[os.path.normpath(File)]
 505     if NewFile != None:
 506         return True, NewFile
 507
 508     return False, File
 509
 510 ## Check if gvien file exists or not
 511 #
 512 #
 513 def ValidFile3(AllFiles, File, Workspace='', EfiSource='', EdkSource='', Dir='.', OverrideDir=''):
 514     # Replace the Edk macros
 515     if OverrideDir != '' and OverrideDir != None:
 516         if OverrideDir.find('$(EFI_SOURCE)') > -1:
 517             OverrideDir = OverrideDir.replace('$(EFI_SOURCE)', EfiSource)
 518         if OverrideDir.find('$(EDK_SOURCE)') > -1:
 519             OverrideDir = OverrideDir.replace('$(EDK_SOURCE)', EdkSource)
 520
 521     # Replace the default dir to current dir
 522     # Dir is current module dir related to workspace
 523     if Dir == '.':
 524         Dir = os.getcwd()
 525         Dir = Dir[len(Workspace)+1:]
 526
 527     NewFile = File
 528     RelaPath = AllFiles[os.path.normpath(Dir)]
 529     NewRelaPath = RelaPath
 530
 531     while(True):
 532         # First check if File has Edk definition itself
 533         if File.find('$(EFI_SOURCE)') > -1 or File.find('$(EDK_SOURCE)') > -1:
 534             File = File.replace('$(EFI_SOURCE)', EfiSource)
 535             File = File.replace('$(EDK_SOURCE)', EdkSource)
 536             NewFile = AllFiles[os.path.normpath(File)]
 537             if NewFile != None:
 538                 NewRelaPath = os.path.dirname(NewFile)
 539                 File = os.path.basename(NewFile)
 540                 #NewRelaPath = NewFile[:len(NewFile) - len(File.replace("..\\", '').replace("../", '')) - 1]
 541                 break
 542
 543         # Second check the path with override value
 544         if OverrideDir != '' and OverrideDir != None:
 545             NewFile = AllFiles[os.path.normpath(os.path.join(OverrideDir, File))]
 546             if NewFile != None:
 547                 #NewRelaPath = os.path.dirname(NewFile)
 548                 NewRelaPath = NewFile[:len(NewFile) - len(File.replace("..\\", '').replace("../", '')) - 1]
 549                 break
 550
 551         # Last check the path with normal definitions
 552         NewFile = AllFiles[os.path.normpath(os.path.join(Dir, File))]
 553         if NewFile != None:
 554             break
 555
 556         # No file found
 557         break
 558
 559     return NewRelaPath, RelaPath, File
 560
 561
 562 def GetRelPath(Path1, Path2):
 563     FileName = os.path.basename(Path2)
 564     L1 = os.path.normpath(Path1).split(os.path.normpath('/'))
 565     L2 = os.path.normpath(Path2).split(os.path.normpath('/'))
 566     for Index in range(0, len(L1)):
 567         if L1[Index] != L2[Index]:
 568             FileName = '../' * (len(L1) - Index)
 569             for Index2 in range(Index, len(L2)):
 570                 FileName = os.path.join(FileName, L2[Index2])
 571             break
 572     return os.path.normpath(FileName)
 573
 574
 575 ## Get GUID value from given packages
 576 #
 577 #   @param      CName           The CName of the GUID
 578 #   @param      PackageList     List of packages looking-up in
 579 #
 580 #   @retval     GuidValue   if the CName is found in any given package
 581 #   @retval     None        if the CName is not found in all given packages
 582 #
 583 def GuidValue(CName, PackageList):
 584     for P in PackageList:
 585         if CName in P.Guids:
 586             return P.Guids[CName]
 587     return None
 588
 589 ## Get Protocol value from given packages
 590 #
 591 #   @param      CName           The CName of the GUID
 592 #   @param      PackageList     List of packages looking-up in
 593 #
 594 #   @retval     GuidValue   if the CName is found in any given package
 595 #   @retval     None        if the CName is not found in all given packages
 596 #
 597 def ProtocolValue(CName, PackageList):
 598     for P in PackageList:
 599         if CName in P.Protocols:
 600             return P.Protocols[CName]
 601     return None
 602
 603 ## Get PPI value from given packages
 604 #
 605 #   @param      CName           The CName of the GUID
 606 #   @param      PackageList     List of packages looking-up in
 607 #
 608 #   @retval     GuidValue   if the CName is found in any given package
 609 #   @retval     None        if the CName is not found in all given packages
 610 #
 611 def PpiValue(CName, PackageList):
 612     for P in PackageList:
 613         if CName in P.Ppis:
 614             return P.Ppis[CName]
 615     return None
 616
 617 ## A string template class
 618 #
 619 #  This class implements a template for string replacement. A string template
 620 #  looks like following
 621 #
 622 #       ${BEGIN} other_string ${placeholder_name} other_string ${END}
 623 #
 624 #  The string between ${BEGIN} and ${END} will be repeated as many times as the
 625 #  length of "placeholder_name", which is a list passed through a dict. The
 626 #  "placeholder_name" is the key name of the dict. The ${BEGIN} and ${END} can
 627 #  be not used and, in this case, the "placeholder_name" must not a list and it
 628 #  will just be replaced once.
 629 #
 630 class TemplateString(object):
 631     _REPEAT_START_FLAG = "BEGIN"
 632     _REPEAT_END_FLAG = "END"
 633
 634     class Section(object):
 635         _LIST_TYPES = [type([]), type(set()), type((0,))]
 636
 637         def __init__(self, TemplateSection, PlaceHolderList):
 638             self._Template = TemplateSection
 639             self._PlaceHolderList = []
 640
 641             # Split the section into sub-sections according to the position of placeholders
 642             if PlaceHolderList:
 643                 self._SubSectionList = []
 644                 SubSectionStart = 0
 645                 #
 646                 # The placeholders passed in must be in the format of
 647                 #
 648                 #   PlaceHolderName, PlaceHolderStartPoint, PlaceHolderEndPoint
 649                 #
 650                 for PlaceHolder,Start,End in PlaceHolderList:
 651                     self._SubSectionList.append(TemplateSection[SubSectionStart:Start])
 652                     self._SubSectionList.append(TemplateSection[Start:End])
 653                     self._PlaceHolderList.append(PlaceHolder)
 654                     SubSectionStart = End
 655                 if SubSectionStart < len(TemplateSection):
 656                     self._SubSectionList.append(TemplateSection[SubSectionStart:])
 657             else:
 658                 self._SubSectionList = [TemplateSection]
 659
 660         def __str__(self):
 661             return self._Template + " : " + str(self._PlaceHolderList)
 662
 663         def Instantiate(self, PlaceHolderValues):
 664             RepeatTime = -1
 665             RepeatPlaceHolders = {}
 666             NonRepeatPlaceHolders = {}
 667
 668             for PlaceHolder in self._PlaceHolderList:
 669                 if PlaceHolder not in PlaceHolderValues:
 670                     continue
 671                 Value = PlaceHolderValues[PlaceHolder]
 672                 if type(Value) in self._LIST_TYPES:
 673                     if RepeatTime < 0:
 674                         RepeatTime = len(Value)
 675                     elif RepeatTime != len(Value):
 676                         EdkLogger.error(
 677                                     "TemplateString",
 678                                     PARAMETER_INVALID,
 679                                     "${%s} has different repeat time from others!" % PlaceHolder,
 680                                     ExtraData=str(self._Template)
 681                                     )
 682                     RepeatPlaceHolders["${%s}" % PlaceHolder] = Value
 683                 else:
 684                     NonRepeatPlaceHolders["${%s}" % PlaceHolder] = Value
 685
 686             if NonRepeatPlaceHolders:
 687                 StringList = []
 688                 for S in self._SubSectionList:
 689                     if S not in NonRepeatPlaceHolders:
 690                         StringList.append(S)
 691                     else:
 692                         StringList.append(str(NonRepeatPlaceHolders[S]))
 693             else:
 694                 StringList = self._SubSectionList
 695
 696             if RepeatPlaceHolders:
 697                 TempStringList = []
 698                 for Index in range(RepeatTime):
 699                     for S in StringList:
 700                         if S not in RepeatPlaceHolders:
 701                             TempStringList.append(S)
 702                         else:
 703                             TempStringList.append(str(RepeatPlaceHolders[S][Index]))
 704                 StringList = TempStringList
 705
 706             return "".join(StringList)
 707
 708     ## Constructor
 709     def __init__(self, Template=None):
 710         self.String = ''
 711         self.IsBinary = False
 712         self._Template = Template
 713         self._TemplateSectionList = self._Parse(Template)
 714
 715     ## str() operator
 716     #
 717     #   @retval     string  The string replaced
 718     #
 719     def __str__(self):
 720         return self.String
 721
 722     ## Split the template string into fragments per the ${BEGIN} and ${END} flags
 723     #
 724     #   @retval     list    A list of TemplateString.Section objects
 725     #
 726     def _Parse(self, Template):
 727         SectionStart = 0
 728         SearchFrom = 0
 729         MatchEnd = 0
 730         PlaceHolderList = []
 731         TemplateSectionList = []
 732         while Template:
 733             MatchObj = gPlaceholderPattern.search(Template, SearchFrom)
 734             if not MatchObj:
 735                 if MatchEnd <= len(Template):
 736                     TemplateSection = TemplateString.Section(Template[SectionStart:], PlaceHolderList)
 737                     TemplateSectionList.append(TemplateSection)
 738                 break
 739
 740             MatchString = MatchObj.group(1)
 741             MatchStart = MatchObj.start()
 742             MatchEnd = MatchObj.end()
 743
 744             if MatchString == self._REPEAT_START_FLAG:
 745                 if MatchStart > SectionStart:
 746                     TemplateSection = TemplateString.Section(Template[SectionStart:MatchStart], PlaceHolderList)
 747                     TemplateSectionList.append(TemplateSection)
 748                 SectionStart = MatchEnd
 749                 PlaceHolderList = []
 750             elif MatchString == self._REPEAT_END_FLAG:
 751                 TemplateSection = TemplateString.Section(Template[SectionStart:MatchStart], PlaceHolderList)
 752                 TemplateSectionList.append(TemplateSection)
 753                 SectionStart = MatchEnd
 754                 PlaceHolderList = []
 755             else:
 756                 PlaceHolderList.append((MatchString, MatchStart - SectionStart, MatchEnd - SectionStart))
 757             SearchFrom = MatchEnd
 758         return TemplateSectionList
 759
 760     ## Replace the string template with dictionary of placeholders and append it to previous one
 761     #
 762     #   @param      AppendString    The string template to append
 763     #   @param      Dictionary      The placeholder dictionaries
 764     #
 765     def Append(self, AppendString, Dictionary=None):
 766         if Dictionary:
 767             SectionList = self._Parse(AppendString)
 768             self.String += "".join([S.Instantiate(Dictionary) for S in SectionList])
 769         else:
 770             self.String += AppendString
 771
 772     ## Replace the string template with dictionary of placeholders
 773     #
 774     #   @param      Dictionary      The placeholder dictionaries
 775     #
 776     #   @retval     str             The string replaced with placeholder values
 777     #
 778     def Replace(self, Dictionary=None):
 779         return "".join([S.Instantiate(Dictionary) for S in self._TemplateSectionList])
 780
 781 ## Progress indicator class
 782 #
 783 #  This class makes use of thread to print progress on console.
 784 #
 785 class Progressor:
 786     # for avoiding deadloop
 787     _StopFlag = None
 788     _ProgressThread = None
 789     _CheckInterval = 0.25
 790
 791     ## Constructor
 792     #
 793     #   @param      OpenMessage     The string printed before progress charaters
 794     #   @param      CloseMessage    The string printed after progress charaters
 795     #   @param      ProgressChar    The charater used to indicate the progress
 796     #   @param      Interval        The interval in seconds between two progress charaters
 797     #
 798     def __init__(self, OpenMessage="", CloseMessage="", ProgressChar='.', Interval=1.0):
 799         self.PromptMessage = OpenMessage
 800         self.CodaMessage = CloseMessage
 801         self.ProgressChar = ProgressChar
 802         self.Interval = Interval
 803         if Progressor._StopFlag == None:
 804             Progressor._StopFlag = threading.Event()
 805
 806     ## Start to print progress charater
 807     #
 808     #   @param      OpenMessage     The string printed before progress charaters
 809     #
 810     def Start(self, OpenMessage=None):
 811         if OpenMessage != None:
 812             self.PromptMessage = OpenMessage
 813         Progressor._StopFlag.clear()
 814         if Progressor._ProgressThread == None:
 815             Progressor._ProgressThread = threading.Thread(target=self._ProgressThreadEntry)
 816             Progressor._ProgressThread.setDaemon(False)
 817             Progressor._ProgressThread.start()
 818
 819     ## Stop printing progress charater
 820     #
 821     #   @param      CloseMessage    The string printed after progress charaters
 822     #
 823     def Stop(self, CloseMessage=None):
 824         OriginalCodaMessage = self.CodaMessage
 825         if CloseMessage != None:
 826             self.CodaMessage = CloseMessage
 827         self.Abort()
 828         self.CodaMessage = OriginalCodaMessage
 829
 830     ## Thread entry method
 831     def _ProgressThreadEntry(self):
 832         sys.stdout.write(self.PromptMessage + " ")
 833         sys.stdout.flush()
 834         TimeUp = 0.0
 835         while not Progressor._StopFlag.isSet():
 836             if TimeUp <= 0.0:
 837                 sys.stdout.write(self.ProgressChar)
 838                 sys.stdout.flush()
 839                 TimeUp = self.Interval
 840             time.sleep(self._CheckInterval)
 841             TimeUp -= self._CheckInterval
 842         sys.stdout.write(" " + self.CodaMessage + "\n")
 843         sys.stdout.flush()
 844
 845     ## Abort the progress display
 846     @staticmethod
 847     def Abort():
 848         if Progressor._StopFlag != None:
 849             Progressor._StopFlag.set()
 850         if Progressor._ProgressThread != None:
 851             Progressor._ProgressThread.join()
 852             Progressor._ProgressThread = None
 853
 854 ## A dict which can access its keys and/or values orderly
 855 #
 856 #  The class implements a new kind of dict which its keys or values can be
 857 #  accessed in the order they are added into the dict. It guarantees the order
 858 #  by making use of an internal list to keep a copy of keys.
 859 #
 860 class sdict(IterableUserDict):
 861     ## Constructor
 862     def __init__(self):
 863         IterableUserDict.__init__(self)
 864         self._key_list = []
 865
 866     ## [] operator
 867     def __setitem__(self, key, value):
 868         if key not in self._key_list:
 869             self._key_list.append(key)
 870         IterableUserDict.__setitem__(self, key, value)
 871
 872     ## del operator
 873     def __delitem__(self, key):
 874         self._key_list.remove(key)
 875         IterableUserDict.__delitem__(self, key)
 876
 877     ## used in "for k in dict" loop to ensure the correct order
 878     def __iter__(self):
 879         return self.iterkeys()
 880
 881     ## len() support
 882     def __len__(self):
 883         return len(self._key_list)
 884
 885     ## "in" test support
 886     def __contains__(self, key):
 887         return key in self._key_list
 888
 889     ## indexof support
 890     def index(self, key):
 891         return self._key_list.index(key)
 892
 893     ## insert support
 894     def insert(self, key, newkey, newvalue, order):
 895         index = self._key_list.index(key)
 896         if order == 'BEFORE':
 897             self._key_list.insert(index, newkey)
 898             IterableUserDict.__setitem__(self, newkey, newvalue)
 899         elif order == 'AFTER':
 900             self._key_list.insert(index + 1, newkey)
 901             IterableUserDict.__setitem__(self, newkey, newvalue)
 902
 903     ## append support
 904     def append(self, sdict):
 905         for key in sdict:
 906             if key not in self._key_list:
 907                 self._key_list.append(key)
 908             IterableUserDict.__setitem__(self, key, sdict[key])
 909
 910     def has_key(self, key):
 911         return key in self._key_list
 912
 913     ## Empty the dict
 914     def clear(self):
 915         self._key_list = []
 916         IterableUserDict.clear(self)
 917
 918     ## Return a copy of keys
 919     def keys(self):
 920         keys = []
 921         for key in self._key_list:
 922             keys.append(key)
 923         return keys
 924
 925     ## Return a copy of values
 926     def values(self):
 927         values = []
 928         for key in self._key_list:
 929             values.append(self[key])
 930         return values
 931
 932     ## Return a copy of (key, value) list
 933     def items(self):
 934         items = []
 935         for key in self._key_list:
 936             items.append((key, self[key]))
 937         return items
 938
 939     ## Iteration support
 940     def iteritems(self):
 941         return iter(self.items())
 942
 943     ## Keys interation support
 944     def iterkeys(self):
 945         return iter(self.keys())
 946
 947     ## Values interation support
 948     def itervalues(self):
 949         return iter(self.values())
 950
 951     ## Return value related to a key, and remove the (key, value) from the dict
 952     def pop(self, key, *dv):
 953         value = None
 954         if key in self._key_list:
 955             value = self[key]
 956             self.__delitem__(key)
 957         elif len(dv) != 0 :
 958             value = kv[0]
 959         return value
 960
 961     ## Return (key, value) pair, and remove the (key, value) from the dict
 962     def popitem(self):
 963         key = self._key_list[-1]
 964         value = self[key]
 965         self.__delitem__(key)
 966         return key, value
 967
 968     def update(self, dict=None, **kwargs):
 969         if dict != None:
 970             for k, v in dict.items():
 971                 self[k] = v
 972         if len(kwargs):
 973             for k, v in kwargs.items():
 974                 self[k] = v
 975
 976 ## Dictionary with restricted keys
 977 #
 978 class rdict(dict):
 979     ## Constructor
 980     def __init__(self, KeyList):
 981         for Key in KeyList:
 982             dict.__setitem__(self, Key, "")
 983
 984     ## []= operator
 985     def __setitem__(self, key, value):
 986         if key not in self:
 987             EdkLogger.error("RestrictedDict", ATTRIBUTE_SET_FAILURE, "Key [%s] is not allowed" % key,
 988                             ExtraData=", ".join(dict.keys(self)))
 989         dict.__setitem__(self, key, value)
 990
 991     ## =[] operator
 992     def __getitem__(self, key):
 993         if key not in self:
 994             return ""
 995         return dict.__getitem__(self, key)
 996
 997     ## del operator
 998     def __delitem__(self, key):
 999         EdkLogger.error("RestrictedDict", ATTRIBUTE_ACCESS_DENIED, ExtraData="del")
1000
1001     ## Empty the dict
1002     def clear(self):
1003         for Key in self:
1004             self.__setitem__(Key, "")
1005
1006     ## Return value related to a key, and remove the (key, value) from the dict
1007     def pop(self, key, *dv):
1008         EdkLogger.error("RestrictedDict", ATTRIBUTE_ACCESS_DENIED, ExtraData="pop")
1009
1010     ## Return (key, value) pair, and remove the (key, value) from the dict
1011     def popitem(self):
1012         EdkLogger.error("RestrictedDict", ATTRIBUTE_ACCESS_DENIED, ExtraData="popitem")
1013
1014 ## Dictionary using prioritized list as key
1015 #
1016 class tdict:
1017     _ListType = type([])
1018     _TupleType = type(())
1019     _Wildcard = 'COMMON'
1020     _ValidWildcardList = ['COMMON', 'DEFAULT', 'ALL', '*', 'PLATFORM']
1021
1022     def __init__(self, _Single_=False, _Level_=2):
1023         self._Level_ = _Level_
1024         self.data = {}
1025         self._Single_ = _Single_
1026
1027     # =[] operator
1028     def __getitem__(self, key):
1029         KeyType = type(key)
1030         RestKeys = None
1031         if KeyType == self._ListType or KeyType == self._TupleType:
1032             FirstKey = key[0]
1033             if len(key) > 1:
1034                 RestKeys = key[1:]
1035             elif self._Level_ > 1:
1036                 RestKeys = [self._Wildcard for i in range(0, self._Level_-1)]
1037         else:
1038             FirstKey = key
1039             if self._Level_ > 1:
1040                 RestKeys = [self._Wildcard for i in range(0, self._Level_-1)]
1041
1042         if FirstKey == None or str(FirstKey).upper() in self._ValidWildcardList:
1043             FirstKey = self._Wildcard
1044
1045         if self._Single_:
1046             return self._GetSingleValue(FirstKey, RestKeys)
1047         else:
1048             return self._GetAllValues(FirstKey, RestKeys)
1049
1050     def _GetSingleValue(self, FirstKey, RestKeys):
1051         Value = None
1052         #print "%s-%s" % (FirstKey, self._Level_) ,
1053         if self._Level_ > 1:
1054             if FirstKey == self._Wildcard:
1055                 if FirstKey in self.data:
1056                     Value = self.data[FirstKey][RestKeys]
1057                 if Value == None:
1058                     for Key in self.data:
1059                         Value = self.data[Key][RestKeys]
1060                         if Value != None: break
1061             else:
1062                 if FirstKey in self.data:
1063                     Value = self.data[FirstKey][RestKeys]
1064                 if Value == None and self._Wildcard in self.data:
1065                     #print "Value=None"
1066                     Value = self.data[self._Wildcard][RestKeys]
1067         else:
1068             if FirstKey == self._Wildcard:
1069                 if FirstKey in self.data:
1070                     Value = self.data[FirstKey]
1071                 if Value == None:
1072                     for Key in self.data:
1073                         Value = self.data[Key]
1074                         if Value != None: break
1075             else:
1076                 if FirstKey in self.data:
1077                     Value = self.data[FirstKey]
1078                 elif self._Wildcard in self.data:
1079                     Value = self.data[self._Wildcard]
1080         return Value
1081
1082     def _GetAllValues(self, FirstKey, RestKeys):
1083         Value = []
1084         if self._Level_ > 1:
1085             if FirstKey == self._Wildcard:
1086                 for Key in self.data:
1087                     Value += self.data[Key][RestKeys]
1088             else:
1089                 if FirstKey in self.data:
1090                     Value += self.data[FirstKey][RestKeys]
1091                 if self._Wildcard in self.data:
1092                     Value += self.data[self._Wildcard][RestKeys]
1093         else:
1094             if FirstKey == self._Wildcard:
1095                 for Key in self.data:
1096                     Value.append(self.data[Key])
1097             else:
1098                 if FirstKey in self.data:
1099                     Value.append(self.data[FirstKey])
1100                 if self._Wildcard in self.data:
1101                     Value.append(self.data[self._Wildcard])
1102         return Value
1103
1104     ## []= operator
1105     def __setitem__(self, key, value):
1106         KeyType = type(key)
1107         RestKeys = None
1108         if KeyType == self._ListType or KeyType == self._TupleType:
1109             FirstKey = key[0]
1110             if len(key) > 1:
1111                 RestKeys = key[1:]
1112             else:
1113                 RestKeys = [self._Wildcard for i in range(0, self._Level_-1)]
1114         else:
1115             FirstKey = key
1116             if self._Level_ > 1:
1117                 RestKeys = [self._Wildcard for i in range(0, self._Level_-1)]
1118
1119         if FirstKey in self._ValidWildcardList:
1120             FirstKey = self._Wildcard
1121
1122         if FirstKey not in self.data and self._Level_ > 0:
1123             self.data[FirstKey] = tdict(self._Single_, self._Level_ - 1)
1124
1125         if self._Level_ > 1:
1126             self.data[FirstKey][RestKeys] = value
1127         else:
1128             self.data[FirstKey] = value
1129
1130     def SetGreedyMode(self):
1131         self._Single_ = False
1132         if self._Level_ > 1:
1133             for Key in self.data:
1134                 self.data[Key].SetGreedyMode()
1135
1136     def SetSingleMode(self):
1137         self._Single_ = True
1138         if self._Level_ > 1:
1139             for Key in self.data:
1140                 self.data[Key].SetSingleMode()
1141
1142     def GetKeys(self, KeyIndex=0):
1143         assert KeyIndex >= 0
1144         if KeyIndex == 0:
1145             return set(self.data.keys())
1146         else:
1147             keys = set()
1148             for Key in self.data:
1149                 keys |= self.data[Key].GetKeys(KeyIndex - 1)
1150             return keys
1151
1152 ## Boolean chain list
1153 #
1154 class Blist(UserList):
1155     def __init__(self, initlist=None):
1156         UserList.__init__(self, initlist)
1157     def __setitem__(self, i, item):
1158         if item not in [True, False]:
1159             if item == 0:
1160                 item = False
1161             else:
1162                 item = True
1163         self.data[i] = item
1164     def _GetResult(self):
1165         Value = True
1166         for item in self.data:
1167             Value &= item
1168         return Value
1169     Result = property(_GetResult)
1170
1171 def ParseConsoleLog(Filename):
1172     Opr = open(os.path.normpath(Filename), 'r')
1173     Opw = open(os.path.normpath(Filename + '.New'), 'w+')
1174     for Line in Opr.readlines():
1175         if Line.find('.efi') > -1:
1176             Line = Line[Line.rfind(' ') : Line.rfind('.efi')].strip()
1177             Opw.write('%s\n' % Line)
1178
1179     Opr.close()
1180     Opw.close()
1181
1182 ## AnalyzeDscPcd
1183 #
1184 #  Analyze DSC PCD value, since there is no data type info in DSC
1185 #  This fuction is used to match functions (AnalyzePcdData, AnalyzeHiiPcdData, AnalyzeVpdPcdData) used for retrieving PCD value from database
1186 #  1. Feature flag: TokenSpace.PcdCName|PcdValue
1187 #  2. Fix and Patch:TokenSpace.PcdCName|PcdValue[|MaxSize]
1188 #  3. Dynamic default:
1189 #     TokenSpace.PcdCName|PcdValue[|VOID*[|MaxSize]]
1190 #     TokenSpace.PcdCName|PcdValue
1191 #  4. Dynamic VPD:
1192 #     TokenSpace.PcdCName|VpdOffset[|VpdValue]
1193 #     TokenSpace.PcdCName|VpdOffset[|MaxSize[|VpdValue]]
1194 #  5. Dynamic HII:
1195 #     TokenSpace.PcdCName|HiiString|VaiableGuid|VariableOffset[|HiiValue]
1196 #  PCD value needs to be located in such kind of string, and the PCD value might be an expression in which
1197 #    there might have "|" operator, also in string value.
1198 #
1199 #  @param Setting: String contain information described above with "TokenSpace.PcdCName|" stripped
1200 #  @param PcdType: PCD type: feature, fixed, dynamic default VPD HII
1201 #  @param DataType: The datum type of PCD: VOID*, UNIT, BOOL
1202 #  @retval:
1203 #    ValueList: A List contain fields described above
1204 #    IsValid:   True if conforming EBNF, otherwise False
1205 #    Index:     The index where PcdValue is in ValueList
1206 #
1207 def AnalyzeDscPcd(Setting, PcdType, DataType=''):
1208     Setting = Setting.strip()
1209     # There might be escaped quote in a string: \", \\\"
1210     Data = Setting.replace('\\\\', '//').replace('\\\"', '\\\'')
1211     # There might be '|' in string and in ( ... | ... ), replace it with '-'
1212     NewStr = ''
1213     InStr = False
1214     Pair = 0
1215     for ch in Data:
1216         if ch == '"':
1217             InStr = not InStr
1218         elif ch == '(' and not InStr:
1219             Pair += 1
1220         elif ch == ')' and not InStr:
1221             Pair -= 1
1222
1223         if (Pair > 0 or InStr) and ch == TAB_VALUE_SPLIT:
1224             NewStr += '-'
1225         else:
1226             NewStr += ch
1227     FieldList = []
1228     StartPos = 0
1229     while True:
1230         Pos = NewStr.find(TAB_VALUE_SPLIT, StartPos)
1231         if Pos < 0:
1232             FieldList.append(Setting[StartPos:].strip())
1233             break
1234         FieldList.append(Setting[StartPos:Pos].strip())
1235         StartPos = Pos + 1
1236
1237     IsValid = True
1238     if PcdType in (MODEL_PCD_FIXED_AT_BUILD, MODEL_PCD_PATCHABLE_IN_MODULE, MODEL_PCD_FEATURE_FLAG):
1239         Value = FieldList[0]
1240         Size = ''
1241         if len(FieldList) > 1:
1242             Type = FieldList[1]
1243             # Fix the PCD type when no DataType input
1244             if Type == 'VOID*':
1245                 DataType = 'VOID*'
1246             else:
1247                 Size = FieldList[1]
1248         if len(FieldList) > 2:
1249             Size = FieldList[2]
1250         if DataType == 'VOID*':
1251             IsValid = (len(FieldList) <= 3)
1252         else:
1253             IsValid = (len(FieldList) <= 1)
1254         return [Value, '', Size], IsValid, 0
1255     elif PcdType in (MODEL_PCD_DYNAMIC_DEFAULT, MODEL_PCD_DYNAMIC_EX_DEFAULT):
1256         Value = FieldList[0]
1257         Size = Type = ''
1258         if len(FieldList) > 1:
1259             Type = FieldList[1]
1260         else:
1261             Type = DataType
1262         if len(FieldList) > 2:
1263             Size = FieldList[2]
1264         else:
1265             if Type == 'VOID*':
1266                 if Value.startswith("L"):
1267                     Size = str((len(Value)- 3 + 1) * 2)
1268                 elif Value.startswith("{"):
1269                     Size = str(len(Value.split(",")))
1270                 else:
1271                     Size = str(len(Value) -2 + 1 )
1272         if DataType == 'VOID*':
1273             IsValid = (len(FieldList) <= 3)
1274         else:
1275             IsValid = (len(FieldList) <= 1)
1276         return [Value, Type, Size], IsValid, 0
1277     elif PcdType in (MODEL_PCD_DYNAMIC_VPD, MODEL_PCD_DYNAMIC_EX_VPD):
1278         VpdOffset = FieldList[0]
1279         Value = Size = ''
1280         if not DataType == 'VOID*':
1281             if len(FieldList) > 1:
1282                 Value = FieldList[1]
1283         else:
1284             if len(FieldList) > 1:
1285                 Size = FieldList[1]
1286             if len(FieldList) > 2:
1287                 Value = FieldList[2]
1288         if DataType == 'VOID*':
1289             IsValid = (len(FieldList) <= 3)
1290         else:
1291             IsValid = (len(FieldList) <= 2)
1292         return [VpdOffset, Size, Value], IsValid, 2
1293     elif PcdType in (MODEL_PCD_DYNAMIC_HII, MODEL_PCD_DYNAMIC_EX_HII):
1294         HiiString = FieldList[0]
1295         Guid = Offset = Value = ''
1296         if len(FieldList) > 1:
1297             Guid = FieldList[1]
1298         if len(FieldList) > 2:
1299             Offset = FieldList[2]
1300         if len(FieldList) > 3:
1301             Value = FieldList[3]
1302         IsValid = (3 <= len(FieldList) <= 4)
1303         return [HiiString, Guid, Offset, Value], IsValid, 3
1304     return [], False, 0
1305
1306 ## AnalyzePcdData
1307 #
1308 #  Analyze the pcd Value, Datum type and TokenNumber.
1309 #  Used to avoid split issue while the value string contain "|" character
1310 #
1311 #  @param[in] Setting:  A String contain value/datum type/token number information;
1312 #
1313 #  @retval   ValueList: A List contain value, datum type and toke number.
1314 #
1315 def AnalyzePcdData(Setting):
1316     ValueList = ['', '', '']
1317
1318     ValueRe  = re.compile(r'^\s*L?\".*\|.*\"')
1319     PtrValue = ValueRe.findall(Setting)
1320
1321     ValueUpdateFlag = False
1322
1323     if len(PtrValue) >= 1:
1324         Setting = re.sub(ValueRe, '', Setting)
1325         ValueUpdateFlag = True
1326
1327     TokenList = Setting.split(TAB_VALUE_SPLIT)
1328     ValueList[0:len(TokenList)] = TokenList
1329
1330     if ValueUpdateFlag:
1331         ValueList[0] = PtrValue[0]
1332
1333     return ValueList
1334
1335 ## AnalyzeHiiPcdData
1336 #
1337 #  Analyze the pcd Value, variable name, variable Guid and variable offset.
1338 #  Used to avoid split issue while the value string contain "|" character
1339 #
1340 #  @param[in] Setting:  A String contain VariableName, VariableGuid, VariableOffset, DefaultValue information;
1341 #
1342 #  @retval   ValueList: A List contaian VariableName, VariableGuid, VariableOffset, DefaultValue.
1343 #
1344 def AnalyzeHiiPcdData(Setting):
1345     ValueList = ['', '', '', '']
1346
1347     TokenList = GetSplitValueList(Setting)
1348     ValueList[0:len(TokenList)] = TokenList
1349
1350     return ValueList
1351
1352 ## AnalyzeVpdPcdData
1353 #
1354 #  Analyze the vpd pcd VpdOffset, MaxDatumSize and InitialValue.
1355 #  Used to avoid split issue while the value string contain "|" character
1356 #
1357 #  @param[in] Setting:  A String contain VpdOffset/MaxDatumSize/InitialValue information;
1358 #
1359 #  @retval   ValueList: A List contain VpdOffset, MaxDatumSize and InitialValue.
1360 #
1361 def AnalyzeVpdPcdData(Setting):
1362     ValueList = ['', '', '']
1363
1364     ValueRe  = re.compile(r'\s*L?\".*\|.*\"\s*$')
1365     PtrValue = ValueRe.findall(Setting)
1366
1367     ValueUpdateFlag = False
1368
1369     if len(PtrValue) >= 1:
1370         Setting = re.sub(ValueRe, '', Setting)
1371         ValueUpdateFlag = True
1372
1373     TokenList = Setting.split(TAB_VALUE_SPLIT)
1374     ValueList[0:len(TokenList)] = TokenList
1375
1376     if ValueUpdateFlag:
1377         ValueList[2] = PtrValue[0]
1378
1379     return ValueList
1380
1381 ## check format of PCD value against its the datum type
1382 #
1383 # For PCD value setting
1384 #
1385 def CheckPcdDatum(Type, Value):
1386     if Type == "VOID*":
1387         ValueRe  = re.compile(r'\s*L?\".*\"\s*$')
1388         if not (((Value.startswith('L"') or Value.startswith('"')) and Value.endswith('"'))
1389                 or (Value.startswith('{') and Value.endswith('}'))
1390                ):
1391             return False, "Invalid value [%s] of type [%s]; must be in the form of {...} for array"\
1392                           ", or \"...\" for string, or L\"...\" for unicode string" % (Value, Type)
1393         elif ValueRe.match(Value):
1394             # Check the chars in UnicodeString or CString is printable
1395             if Value.startswith("L"):
1396                 Value = Value[2:-1]
1397             else:
1398                 Value = Value[1:-1]
1399             Printset = set(string.printable)
1400             Printset.remove(TAB_PRINTCHAR_VT)
1401             Printset.add(TAB_PRINTCHAR_BS)
1402             Printset.add(TAB_PRINTCHAR_NUL)
1403             if not set(Value).issubset(Printset):
1404                 PrintList = list(Printset)
1405                 PrintList.sort()
1406                 return False, "Invalid PCD string value of type [%s]; must be printable chars %s." % (Type, PrintList)
1407     elif Type == 'BOOLEAN':
1408         if Value not in ['TRUE', 'True', 'true', '0x1', '0x01', '1', 'FALSE', 'False', 'false', '0x0', '0x00', '0']:
1409             return False, "Invalid value [%s] of type [%s]; must be one of TRUE, True, true, 0x1, 0x01, 1"\
1410                           ", FALSE, False, false, 0x0, 0x00, 0" % (Value, Type)
1411     elif Type in [TAB_UINT8, TAB_UINT16, TAB_UINT32, TAB_UINT64]:
1412         try:
1413             Value = long(Value, 0)
1414         except:
1415             return False, "Invalid value [%s] of type [%s];"\
1416                           " must be a hexadecimal, decimal or octal in C language format." % (Value, Type)
1417     else:
1418         return False, "Invalid type [%s]; must be one of VOID*, BOOLEAN, UINT8, UINT16, UINT32, UINT64." % (Type)
1419
1420     return True, ""
1421
1422 ## Split command line option string to list
1423 #
1424 # subprocess.Popen needs the args to be a sequence. Otherwise there's problem
1425 # in non-windows platform to launch command
1426 #
1427 def SplitOption(OptionString):
1428     OptionList = []
1429     LastChar = " "
1430     OptionStart = 0
1431     QuotationMark = ""
1432     for Index in range(0, len(OptionString)):
1433         CurrentChar = OptionString[Index]
1434         if CurrentChar in ['"', "'"]:
1435             if QuotationMark == CurrentChar:
1436                 QuotationMark = ""
1437             elif QuotationMark == "":
1438                 QuotationMark = CurrentChar
1439             continue
1440         elif QuotationMark:
1441             continue
1442
1443         if CurrentChar in ["/", "-"] and LastChar in [" ", "\t", "\r", "\n"]:
1444             if Index > OptionStart:
1445                 OptionList.append(OptionString[OptionStart:Index-1])
1446             OptionStart = Index
1447         LastChar = CurrentChar
1448     OptionList.append(OptionString[OptionStart:])
1449     return OptionList
1450
1451 def CommonPath(PathList):
1452     P1 = min(PathList).split(os.path.sep)
1453     P2 = max(PathList).split(os.path.sep)
1454     for Index in xrange(min(len(P1), len(P2))):
1455         if P1[Index] != P2[Index]:
1456             return os.path.sep.join(P1[:Index])
1457     return os.path.sep.join(P1)
1458
1459 class PathClass(object):
1460     def __init__(self, File='', Root='', AlterRoot='', Type='', IsBinary=False,
1461                  Arch='COMMON', ToolChainFamily='', Target='', TagName='', ToolCode=''):
1462         self.Arch = Arch
1463         self.File = str(File)
1464         if os.path.isabs(self.File):
1465             self.Root = ''
1466             self.AlterRoot = ''
1467         else:
1468             self.Root = str(Root)
1469             self.AlterRoot = str(AlterRoot)
1470
1471         # Remove any '.' and '..' in path
1472         if self.Root:
1473             self.Path = os.path.normpath(os.path.join(self.Root, self.File))
1474             self.Root = os.path.normpath(CommonPath([self.Root, self.Path]))
1475             # eliminate the side-effect of 'C:'
1476             if self.Root[-1] == ':':
1477                 self.Root += os.path.sep
1478             # file path should not start with path separator
1479             if self.Root[-1] == os.path.sep:
1480                 self.File = self.Path[len(self.Root):]
1481             else:
1482                 self.File = self.Path[len(self.Root)+1:]
1483         else:
1484             self.Path = os.path.normpath(self.File)
1485
1486         self.SubDir, self.Name = os.path.split(self.File)
1487         self.BaseName, self.Ext = os.path.splitext(self.Name)
1488
1489         if self.Root:
1490             if self.SubDir:
1491                 self.Dir = os.path.join(self.Root, self.SubDir)
1492             else:
1493                 self.Dir = self.Root
1494         else:
1495             self.Dir = self.SubDir
1496
1497         if IsBinary:
1498             self.Type = Type
1499         else:
1500             self.Type = self.Ext.lower()
1501
1502         self.IsBinary = IsBinary
1503         self.Target = Target
1504         self.TagName = TagName
1505         self.ToolCode = ToolCode
1506         self.ToolChainFamily = ToolChainFamily
1507
1508         self._Key = None
1509
1510     ## Convert the object of this class to a string
1511     #
1512     #  Convert member Path of the class to a string
1513     #
1514     #  @retval string Formatted String
1515     #
1516     def __str__(self):
1517         return self.Path
1518
1519     ## Override __eq__ function
1520     #
1521     # Check whether PathClass are the same
1522     #
1523     # @retval False The two PathClass are different
1524     # @retval True  The two PathClass are the same
1525     #
1526     def __eq__(self, Other):
1527         if type(Other) == type(self):
1528             return self.Path == Other.Path
1529         else:
1530             return self.Path == str(Other)
1531
1532     ## Override __cmp__ function
1533     #
1534     # Customize the comparsion operation of two PathClass
1535     #
1536     # @retval 0     The two PathClass are different
1537     # @retval -1    The first PathClass is less than the second PathClass
1538     # @retval 1     The first PathClass is Bigger than the second PathClass
1539     def __cmp__(self, Other):
1540         if type(Other) == type(self):
1541             OtherKey = Other.Path
1542         else:
1543             OtherKey = str(Other)
1544
1545         SelfKey = self.Path
1546         if SelfKey == OtherKey:
1547             return 0
1548         elif SelfKey > OtherKey:
1549             return 1
1550         else:
1551             return -1
1552
1553     ## Override __hash__ function
1554     #
1555     # Use Path as key in hash table
1556     #
1557     # @retval string Key for hash table
1558     #
1559     def __hash__(self):
1560         return hash(self.Path)
1561
1562     def _GetFileKey(self):
1563         if self._Key == None:
1564             self._Key = self.Path.upper()   # + self.ToolChainFamily + self.TagName + self.ToolCode + self.Target
1565         return self._Key
1566
1567     def _GetTimeStamp(self):
1568         return os.stat(self.Path)[8]
1569
1570     def Validate(self, Type='', CaseSensitive=True):
1571         if GlobalData.gCaseInsensitive:
1572             CaseSensitive = False
1573         if Type and Type.lower() != self.Type:
1574             return FILE_TYPE_MISMATCH, '%s (expect %s but got %s)' % (self.File, Type, self.Type)
1575
1576         RealFile, RealRoot = RealPath2(self.File, self.Root, self.AlterRoot)
1577         if not RealRoot and not RealFile:
1578             RealFile = self.File
1579             if self.AlterRoot:
1580                 RealFile = os.path.join(self.AlterRoot, self.File)
1581             elif self.Root:
1582                 RealFile = os.path.join(self.Root, self.File)
1583             return FILE_NOT_FOUND, os.path.join(self.AlterRoot, RealFile)
1584
1585         ErrorCode = 0
1586         ErrorInfo = ''
1587         if RealRoot != self.Root or RealFile != self.File:
1588             if CaseSensitive and (RealFile != self.File or (RealRoot != self.Root and RealRoot != self.AlterRoot)):
1589                 ErrorCode = FILE_CASE_MISMATCH
1590                 ErrorInfo = self.File + '\n\t' + RealFile + " [in file system]"
1591
1592             self.SubDir, self.Name = os.path.split(RealFile)
1593             self.BaseName, self.Ext = os.path.splitext(self.Name)
1594             if self.SubDir:
1595                 self.Dir = os.path.join(RealRoot, self.SubDir)
1596             else:
1597                 self.Dir = RealRoot
1598             self.File = RealFile
1599             self.Root = RealRoot
1600             self.Path = os.path.join(RealRoot, RealFile)
1601         return ErrorCode, ErrorInfo
1602
1603     Key = property(_GetFileKey)
1604     TimeStamp = property(_GetTimeStamp)
1605
1606 ## Parse PE image to get the required PE informaion.
1607 #
1608 class PeImageClass():
1609     ## Constructor
1610     #
1611     #   @param  File FilePath of PeImage
1612     #
1613     def __init__(self, PeFile):
1614         self.FileName   = PeFile
1615         self.IsValid    = False
1616         self.Size       = 0
1617         self.EntryPoint = 0
1618         self.SectionAlignment  = 0
1619         self.SectionHeaderList = []
1620         self.ErrorInfo = ''
1621         try:
1622             PeObject = open(PeFile, 'rb')
1623         except:
1624             self.ErrorInfo = self.FileName + ' can not be found\n'
1625             return
1626         # Read DOS header
1627         ByteArray = array.array('B')
1628         ByteArray.fromfile(PeObject, 0x3E)
1629         ByteList = ByteArray.tolist()
1630         # DOS signature should be 'MZ'
1631         if self._ByteListToStr (ByteList[0x0:0x2]) != 'MZ':
1632             self.ErrorInfo = self.FileName + ' has no valid DOS signature MZ'
1633             return
1634
1635         # Read 4 byte PE Signature
1636         PeOffset = self._ByteListToInt(ByteList[0x3C:0x3E])
1637         PeObject.seek(PeOffset)
1638         ByteArray = array.array('B')
1639         ByteArray.fromfile(PeObject, 4)
1640         # PE signature should be 'PE\0\0'
1641         if ByteArray.tostring() != 'PE\0\0':
1642             self.ErrorInfo = self.FileName + ' has no valid PE signature PE00'
1643             return
1644
1645         # Read PE file header
1646         ByteArray = array.array('B')
1647         ByteArray.fromfile(PeObject, 0x14)
1648         ByteList = ByteArray.tolist()
1649         SecNumber = self._ByteListToInt(ByteList[0x2:0x4])
1650         if SecNumber == 0:
1651             self.ErrorInfo = self.FileName + ' has no section header'
1652             return
1653
1654         # Read PE optional header
1655         OptionalHeaderSize = self._ByteListToInt(ByteArray[0x10:0x12])
1656         ByteArray = array.array('B')
1657         ByteArray.fromfile(PeObject, OptionalHeaderSize)
1658         ByteList = ByteArray.tolist()
1659         self.EntryPoint       = self._ByteListToInt(ByteList[0x10:0x14])
1660         self.SectionAlignment = self._ByteListToInt(ByteList[0x20:0x24])
1661         self.Size             = self._ByteListToInt(ByteList[0x38:0x3C])
1662
1663         # Read each Section Header
1664         for Index in range(SecNumber):
1665             ByteArray = array.array('B')
1666             ByteArray.fromfile(PeObject, 0x28)
1667             ByteList = ByteArray.tolist()
1668             SecName  = self._ByteListToStr(ByteList[0:8])
1669             SecVirtualSize = self._ByteListToInt(ByteList[8:12])
1670             SecRawAddress  = self._ByteListToInt(ByteList[20:24])
1671             SecVirtualAddress = self._ByteListToInt(ByteList[12:16])
1672             self.SectionHeaderList.append((SecName, SecVirtualAddress, SecRawAddress, SecVirtualSize))
1673         self.IsValid = True
1674         PeObject.close()
1675
1676     def _ByteListToStr(self, ByteList):
1677         String = ''
1678         for index in range(len(ByteList)):
1679             if ByteList[index] == 0:
1680                 break
1681             String += chr(ByteList[index])
1682         return String
1683
1684     def _ByteListToInt(self, ByteList):
1685         Value = 0
1686         for index in range(len(ByteList) - 1, -1, -1):
1687             Value = (Value << 8) | int(ByteList[index])
1688         return Value
1689
1690
1691 class SkuClass():
1692
1693     DEFAULT = 0
1694     SINGLE = 1
1695     MULTIPLE =2
1696
1697     def __init__(self,SkuIdentifier='', SkuIds={}):
1698
1699         self.AvailableSkuIds = sdict()
1700         self.SkuIdSet = []
1701
1702         if SkuIdentifier == '' or SkuIdentifier is None:
1703             self.SkuIdSet = ['DEFAULT']
1704         elif SkuIdentifier == 'ALL':
1705             self.SkuIdSet = SkuIds.keys()
1706         else:
1707             r = SkuIdentifier.split('|')
1708             self.SkuIdSet=[r[k].strip() for k in range(len(r))]
1709         if len(self.SkuIdSet) == 2 and 'DEFAULT' in self.SkuIdSet and SkuIdentifier != 'ALL':
1710             self.SkuIdSet.remove('DEFAULT')
1711
1712         for each in self.SkuIdSet:
1713             if each in SkuIds:
1714                 self.AvailableSkuIds[each] = SkuIds[each]
1715             else:
1716                 EdkLogger.error("build", PARAMETER_INVALID,
1717                             ExtraData="SKU-ID [%s] is not supported by the platform. [Valid SKU-ID: %s]"
1718                                       % (each, " ".join(SkuIds.keys())))
1719
1720     def __SkuUsageType(self):
1721
1722         if len(self.SkuIdSet) == 1:
1723             if self.SkuIdSet[0] == 'DEFAULT':
1724                 return SkuClass.DEFAULT
1725             else:
1726                 return SkuClass.SINGLE
1727         else:
1728             return SkuClass.MULTIPLE
1729
1730     def __GetAvailableSkuIds(self):
1731         return self.AvailableSkuIds
1732
1733     def __GetSystemSkuID(self):
1734         if self.__SkuUsageType() == SkuClass.SINGLE:
1735             return self.SkuIdSet[0]
1736         else:
1737             return 'DEFAULT'
1738
1739     SystemSkuId = property(__GetSystemSkuID)
1740     AvailableSkuIdSet = property(__GetAvailableSkuIds)
1741     SkuUsageType = property(__SkuUsageType)
1742
1743 ##
1744 #
1745 # This acts like the main() function for the script, unless it is 'import'ed into another
1746 # script.
1747 #
1748 if __name__ == '__main__':
1749     pass
1750