BaseTools/Source/Python/Common/Misc.py

   1 ## @file
   2 # Common routines used by all tools
   3 #
   4 # Copyright (c) 2007 - 2010, Intel Corporation. All rights reserved.<BR>
   5 # This program and the accompanying materials
   6 # are licensed and made available under the terms and conditions of the BSD License
   7 # which accompanies this distribution.  The full text of the license may be found at
   8 # http://opensource.org/licenses/bsd-license.php
   9 #
  10 # THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS,
  11 # WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED.
  12 #
  13
  14 ##
  15 # Import Modules
  16 #
  17 import os
  18 import sys
  19 import string
  20 import thread
  21 import threading
  22 import time
  23 import re
  24 import cPickle
  25 import array
  26 from UserDict import IterableUserDict
  27 from UserList import UserList
  28
  29 from Common import EdkLogger as EdkLogger
  30 from Common import GlobalData as GlobalData
  31 from DataType import *
  32 from BuildToolError import *
  33 from CommonDataClass.DataClass import *
  34
  35 ## Regular expression used to find out place holders in string template
  36 gPlaceholderPattern = re.compile("\$\{([^$()\s]+)\}", re.MULTILINE|re.UNICODE)
  37
  38 ## Dictionary used to store file time stamp for quick re-access
  39 gFileTimeStampCache = {}    # {file path : file time stamp}
  40
  41 ## Dictionary used to store dependencies of files
  42 gDependencyDatabase = {}    # arch : {file path : [dependent files list]}
  43
  44 ## callback routine for processing variable option
  45 #
  46 # This function can be used to process variable number of option values. The
  47 # typical usage of it is specify architecure list on command line.
  48 # (e.g. <tool> -a IA32 X64 IPF)
  49 #
  50 # @param  Option        Standard callback function parameter
  51 # @param  OptionString  Standard callback function parameter
  52 # @param  Value         Standard callback function parameter
  53 # @param  Parser        Standard callback function parameter
  54 #
  55 # @retval
  56 #
  57 def ProcessVariableArgument(Option, OptionString, Value, Parser):
  58     assert Value is None
  59     Value = []
  60     RawArgs = Parser.rargs
  61     while RawArgs:
  62         Arg = RawArgs[0]
  63         if (Arg[:2] == "--" and len(Arg) > 2) or \
  64            (Arg[:1] == "-" and len(Arg) > 1 and Arg[1] != "-"):
  65             break
  66         Value.append(Arg)
  67         del RawArgs[0]
  68     setattr(Parser.values, Option.dest, Value)
  69
  70 ## Convert GUID string in xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx style to C structure style
  71 #
  72 #   @param      Guid    The GUID string
  73 #
  74 #   @retval     string  The GUID string in C structure style
  75 #
  76 def GuidStringToGuidStructureString(Guid):
  77     GuidList = Guid.split('-')
  78     Result = '{'
  79     for Index in range(0,3,1):
  80         Result = Result + '0x' + GuidList[Index] + ', '
  81     Result = Result + '{0x' + GuidList[3][0:2] + ', 0x' + GuidList[3][2:4]
  82     for Index in range(0,12,2):
  83         Result = Result + ', 0x' + GuidList[4][Index:Index+2]
  84     Result += '}}'
  85     return Result
  86
  87 ## Convert GUID structure in byte array to xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
  88 #
  89 #   @param      GuidValue   The GUID value in byte array
  90 #
  91 #   @retval     string      The GUID value in xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx format
  92 #
  93 def GuidStructureByteArrayToGuidString(GuidValue):
  94     guidValueString = GuidValue.lower().replace("{", "").replace("}", "").replace(" ", "").replace(";", "")
  95     guidValueList = guidValueString.split(",")
  96     if len(guidValueList) != 16:
  97         return ''
  98         #EdkLogger.error(None, None, "Invalid GUID value string %s" % GuidValue)
  99     try:
 100         return "%02x%02x%02x%02x-%02x%02x-%02x%02x-%02x%02x-%02x%02x%02x%02x%02x%02x" % (
 101                 int(guidValueList[3], 16),
 102                 int(guidValueList[2], 16),
 103                 int(guidValueList[1], 16),
 104                 int(guidValueList[0], 16),
 105                 int(guidValueList[5], 16),
 106                 int(guidValueList[4], 16),
 107                 int(guidValueList[7], 16),
 108                 int(guidValueList[6], 16),
 109                 int(guidValueList[8], 16),
 110                 int(guidValueList[9], 16),
 111                 int(guidValueList[10], 16),
 112                 int(guidValueList[11], 16),
 113                 int(guidValueList[12], 16),
 114                 int(guidValueList[13], 16),
 115                 int(guidValueList[14], 16),
 116                 int(guidValueList[15], 16)
 117                 )
 118     except:
 119         return ''
 120
 121 ## Convert GUID string in C structure style to xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
 122 #
 123 #   @param      GuidValue   The GUID value in C structure format
 124 #
 125 #   @retval     string      The GUID value in xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx format
 126 #
 127 def GuidStructureStringToGuidString(GuidValue):
 128     guidValueString = GuidValue.lower().replace("{", "").replace("}", "").replace(" ", "").replace(";", "")
 129     guidValueList = guidValueString.split(",")
 130     if len(guidValueList) != 11:
 131         return ''
 132         #EdkLogger.error(None, None, "Invalid GUID value string %s" % GuidValue)
 133     try:
 134         return "%08x-%04x-%04x-%02x%02x-%02x%02x%02x%02x%02x%02x" % (
 135                 int(guidValueList[0], 16),
 136                 int(guidValueList[1], 16),
 137                 int(guidValueList[2], 16),
 138                 int(guidValueList[3], 16),
 139                 int(guidValueList[4], 16),
 140                 int(guidValueList[5], 16),
 141                 int(guidValueList[6], 16),
 142                 int(guidValueList[7], 16),
 143                 int(guidValueList[8], 16),
 144                 int(guidValueList[9], 16),
 145                 int(guidValueList[10], 16)
 146                 )
 147     except:
 148         return ''
 149
 150 ## Convert GUID string in C structure style to xxxxxxxx_xxxx_xxxx_xxxx_xxxxxxxxxxxx
 151 #
 152 #   @param      GuidValue   The GUID value in C structure format
 153 #
 154 #   @retval     string      The GUID value in xxxxxxxx_xxxx_xxxx_xxxx_xxxxxxxxxxxx format
 155 #
 156 def GuidStructureStringToGuidValueName(GuidValue):
 157     guidValueString = GuidValue.lower().replace("{", "").replace("}", "").replace(" ", "")
 158     guidValueList = guidValueString.split(",")
 159     if len(guidValueList) != 11:
 160         EdkLogger.error(None, FORMAT_INVALID, "Invalid GUID value string [%s]" % GuidValue)
 161     return "%08x_%04x_%04x_%02x%02x_%02x%02x%02x%02x%02x%02x" % (
 162             int(guidValueList[0], 16),
 163             int(guidValueList[1], 16),
 164             int(guidValueList[2], 16),
 165             int(guidValueList[3], 16),
 166             int(guidValueList[4], 16),
 167             int(guidValueList[5], 16),
 168             int(guidValueList[6], 16),
 169             int(guidValueList[7], 16),
 170             int(guidValueList[8], 16),
 171             int(guidValueList[9], 16),
 172             int(guidValueList[10], 16)
 173             )
 174
 175 ## Create directories
 176 #
 177 #   @param      Directory   The directory name
 178 #
 179 def CreateDirectory(Directory):
 180     if Directory == None or Directory.strip() == "":
 181         return True
 182     try:
 183         if not os.access(Directory, os.F_OK):
 184             os.makedirs(Directory)
 185     except:
 186         return False
 187     return True
 188
 189 ## Remove directories, including files and sub-directories in it
 190 #
 191 #   @param      Directory   The directory name
 192 #
 193 def RemoveDirectory(Directory, Recursively=False):
 194     if Directory == None or Directory.strip() == "" or not os.path.exists(Directory):
 195         return
 196     if Recursively:
 197         CurrentDirectory = os.getcwd()
 198         os.chdir(Directory)
 199         for File in os.listdir("."):
 200             if os.path.isdir(File):
 201                 RemoveDirectory(File, Recursively)
 202             else:
 203                 os.remove(File)
 204         os.chdir(CurrentDirectory)
 205     os.rmdir(Directory)
 206
 207 ## Check if given file is changed or not
 208 #
 209 #  This method is used to check if a file is changed or not between two build
 210 #  actions. It makes use a cache to store files timestamp.
 211 #
 212 #   @param      File    The path of file
 213 #
 214 #   @retval     True    If the given file is changed, doesn't exist, or can't be
 215 #                       found in timestamp cache
 216 #   @retval     False   If the given file is changed
 217 #
 218 def IsChanged(File):
 219     if not os.path.exists(File):
 220         return True
 221
 222     FileState = os.stat(File)
 223     TimeStamp = FileState[-2]
 224
 225     if File in gFileTimeStampCache and TimeStamp == gFileTimeStampCache[File]:
 226         FileChanged = False
 227     else:
 228         FileChanged = True
 229         gFileTimeStampCache[File] = TimeStamp
 230
 231     return FileChanged
 232
 233 ## Store content in file
 234 #
 235 #  This method is used to save file only when its content is changed. This is
 236 #  quite useful for "make" system to decide what will be re-built and what won't.
 237 #
 238 #   @param      File            The path of file
 239 #   @param      Content         The new content of the file
 240 #   @param      IsBinaryFile    The flag indicating if the file is binary file or not
 241 #
 242 #   @retval     True            If the file content is changed and the file is renewed
 243 #   @retval     False           If the file content is the same
 244 #
 245 def SaveFileOnChange(File, Content, IsBinaryFile=True):
 246     if not IsBinaryFile:
 247         Content = Content.replace("\n", os.linesep)
 248
 249     if os.path.exists(File):
 250         try:
 251             if Content == open(File, "rb").read():
 252                 return False
 253         except:
 254             EdkLogger.error(None, FILE_OPEN_FAILURE, ExtraData=File)
 255
 256     DirName = os.path.dirname(File)
 257     if not CreateDirectory(DirName):
 258         EdkLogger.error(None, FILE_CREATE_FAILURE, "Could not create directory %s" % DirName)
 259     else:
 260         if DirName == '':
 261             DirName = os.getcwd()
 262         if not os.access(DirName, os.W_OK):
 263             EdkLogger.error(None, PERMISSION_FAILURE, "Do not have write permission on directory %s" % DirName)
 264
 265     try:
 266         if GlobalData.gIsWindows:
 267             try:
 268                 from PyUtility import SaveFileToDisk
 269                 if not SaveFileToDisk(File, Content):
 270                     EdkLogger.error(None, FILE_CREATE_FAILURE, ExtraData=File)
 271             except:
 272                 Fd = open(File, "wb")
 273                 Fd.write(Content)
 274                 Fd.close()
 275         else:
 276             Fd = open(File, "wb")
 277             Fd.write(Content)
 278             Fd.close()
 279     except IOError, X:
 280         EdkLogger.error(None, FILE_CREATE_FAILURE, ExtraData='IOError %s'%X)
 281
 282     return True
 283
 284 ## Make a Python object persistent on file system
 285 #
 286 #   @param      Data    The object to be stored in file
 287 #   @param      File    The path of file to store the object
 288 #
 289 def DataDump(Data, File):
 290     Fd = None
 291     try:
 292         Fd = open(File, 'wb')
 293         cPickle.dump(Data, Fd, cPickle.HIGHEST_PROTOCOL)
 294     except:
 295         EdkLogger.error("", FILE_OPEN_FAILURE, ExtraData=File, RaiseError=False)
 296     finally:
 297         if Fd != None:
 298             Fd.close()
 299
 300 ## Restore a Python object from a file
 301 #
 302 #   @param      File    The path of file stored the object
 303 #
 304 #   @retval     object  A python object
 305 #   @retval     None    If failure in file operation
 306 #
 307 def DataRestore(File):
 308     Data = None
 309     Fd = None
 310     try:
 311         Fd = open(File, 'rb')
 312         Data = cPickle.load(Fd)
 313     except Exception, e:
 314         EdkLogger.verbose("Failed to load [%s]\n\t%s" % (File, str(e)))
 315         Data = None
 316     finally:
 317         if Fd != None:
 318             Fd.close()
 319     return Data
 320
 321 ## Retrieve and cache the real path name in file system
 322 #
 323 #   @param      Root    The root directory of path relative to
 324 #
 325 #   @retval     str     The path string if the path exists
 326 #   @retval     None    If path doesn't exist
 327 #
 328 class DirCache:
 329     _CACHE_ = set()
 330     _UPPER_CACHE_ = {}
 331
 332     def __init__(self, Root):
 333         self._Root = Root
 334         for F in os.listdir(Root):
 335             self._CACHE_.add(F)
 336             self._UPPER_CACHE_[F.upper()] = F
 337
 338     # =[] operator
 339     def __getitem__(self, Path):
 340         Path = Path[len(os.path.commonprefix([Path, self._Root])):]
 341         if not Path:
 342             return self._Root
 343         if Path and Path[0] == os.path.sep:
 344             Path = Path[1:]
 345         if Path in self._CACHE_:
 346             return os.path.join(self._Root, Path)
 347         UpperPath = Path.upper()
 348         if UpperPath in self._UPPER_CACHE_:
 349             return os.path.join(self._Root, self._UPPER_CACHE_[UpperPath])
 350
 351         IndexList = []
 352         LastSepIndex = -1
 353         SepIndex = Path.find(os.path.sep)
 354         while SepIndex > -1:
 355             Parent = UpperPath[:SepIndex]
 356             if Parent not in self._UPPER_CACHE_:
 357                 break
 358             LastSepIndex = SepIndex
 359             SepIndex = Path.find(os.path.sep, LastSepIndex + 1)
 360
 361         if LastSepIndex == -1:
 362             return None
 363
 364         Cwd = os.getcwd()
 365         os.chdir(self._Root)
 366         SepIndex = LastSepIndex
 367         while SepIndex > -1:
 368             Parent = Path[:SepIndex]
 369             ParentKey = UpperPath[:SepIndex]
 370             if ParentKey not in self._UPPER_CACHE_:
 371                 os.chdir(Cwd)
 372                 return None
 373
 374             if Parent in self._CACHE_:
 375                 ParentDir = Parent
 376             else:
 377                 ParentDir = self._UPPER_CACHE_[ParentKey]
 378             for F in os.listdir(ParentDir):
 379                 Dir = os.path.join(ParentDir, F)
 380                 self._CACHE_.add(Dir)
 381                 self._UPPER_CACHE_[Dir.upper()] = Dir
 382
 383             SepIndex = Path.find(os.path.sep, SepIndex + 1)
 384
 385         os.chdir(Cwd)
 386         if Path in self._CACHE_:
 387             return os.path.join(self._Root, Path)
 388         elif UpperPath in self._UPPER_CACHE_:
 389             return os.path.join(self._Root, self._UPPER_CACHE_[UpperPath])
 390         return None
 391
 392 ## Get all files of a directory
 393 #
 394 # @param Root:       Root dir
 395 # @param SkipList :  The files need be skipped
 396 #
 397 # @retval  A list of all files
 398 #
 399 def GetFiles(Root, SkipList=None, FullPath = True):
 400     OriPath = Root
 401     FileList = []
 402     for Root, Dirs, Files in os.walk(Root):
 403         if SkipList:
 404             for Item in SkipList:
 405                 if Item in Dirs:
 406                     Dirs.remove(Item)
 407
 408         for File in Files:
 409             File = os.path.normpath(os.path.join(Root, File))
 410             if not FullPath:
 411                 File = File[len(OriPath) + 1:]
 412             FileList.append(File)
 413
 414     return FileList
 415
 416 ## Check if gvien file exists or not
 417 #
 418 #   @param      File    File name or path to be checked
 419 #   @param      Dir     The directory the file is relative to
 420 #
 421 #   @retval     True    if file exists
 422 #   @retval     False   if file doesn't exists
 423 #
 424 def ValidFile(File, Ext=None):
 425     if Ext != None:
 426         Dummy, FileExt = os.path.splitext(File)
 427         if FileExt.lower() != Ext.lower():
 428             return False
 429     if not os.path.exists(File):
 430         return False
 431     return True
 432
 433 def RealPath(File, Dir='', OverrideDir=''):
 434     NewFile = os.path.normpath(os.path.join(Dir, File))
 435     NewFile = GlobalData.gAllFiles[NewFile]
 436     if not NewFile and OverrideDir:
 437         NewFile = os.path.normpath(os.path.join(OverrideDir, File))
 438         NewFile = GlobalData.gAllFiles[NewFile]
 439     return NewFile
 440
 441 def RealPath2(File, Dir='', OverrideDir=''):
 442     if OverrideDir:
 443         NewFile = GlobalData.gAllFiles[os.path.normpath(os.path.join(OverrideDir, File))]
 444         if NewFile:
 445             if OverrideDir[-1] == os.path.sep:
 446                 return NewFile[len(OverrideDir):], NewFile[0:len(OverrideDir)]
 447             else:
 448                 return NewFile[len(OverrideDir)+1:], NewFile[0:len(OverrideDir)]
 449     if GlobalData.gAllFiles:
 450         NewFile = GlobalData.gAllFiles[os.path.normpath(os.path.join(Dir, File))]
 451     else:
 452         NewFile = os.path.normpath(os.path.join(Dir, File))
 453     if NewFile:
 454         if Dir:
 455             if Dir[-1] == os.path.sep:
 456                 return NewFile[len(Dir):], NewFile[0:len(Dir)]
 457             else:
 458                 return NewFile[len(Dir)+1:], NewFile[0:len(Dir)]
 459         else:
 460             return NewFile, ''
 461
 462     return None, None
 463
 464 ## Check if gvien file exists or not
 465 #
 466 #
 467 def ValidFile2(AllFiles, File, Ext=None, Workspace='', EfiSource='', EdkSource='', Dir='.', OverrideDir=''):
 468     NewFile = File
 469     if Ext != None:
 470         Dummy, FileExt = os.path.splitext(File)
 471         if FileExt.lower() != Ext.lower():
 472             return False, File
 473
 474     # Replace the Edk macros
 475     if OverrideDir != '' and OverrideDir != None:
 476         if OverrideDir.find('$(EFI_SOURCE)') > -1:
 477             OverrideDir = OverrideDir.replace('$(EFI_SOURCE)', EfiSource)
 478         if OverrideDir.find('$(EDK_SOURCE)') > -1:
 479             OverrideDir = OverrideDir.replace('$(EDK_SOURCE)', EdkSource)
 480
 481     # Replace the default dir to current dir
 482     if Dir == '.':
 483         Dir = os.getcwd()
 484         Dir = Dir[len(Workspace)+1:]
 485
 486     # First check if File has Edk definition itself
 487     if File.find('$(EFI_SOURCE)') > -1 or File.find('$(EDK_SOURCE)') > -1:
 488         NewFile = File.replace('$(EFI_SOURCE)', EfiSource)
 489         NewFile = NewFile.replace('$(EDK_SOURCE)', EdkSource)
 490         NewFile = AllFiles[os.path.normpath(NewFile)]
 491         if NewFile != None:
 492             return True, NewFile
 493
 494     # Second check the path with override value
 495     if OverrideDir != '' and OverrideDir != None:
 496         NewFile = AllFiles[os.path.normpath(os.path.join(OverrideDir, File))]
 497         if NewFile != None:
 498             return True, NewFile
 499
 500     # Last check the path with normal definitions
 501     File = os.path.join(Dir, File)
 502     NewFile = AllFiles[os.path.normpath(File)]
 503     if NewFile != None:
 504         return True, NewFile
 505
 506     return False, File
 507
 508 ## Check if gvien file exists or not
 509 #
 510 #
 511 def ValidFile3(AllFiles, File, Workspace='', EfiSource='', EdkSource='', Dir='.', OverrideDir=''):
 512     # Replace the Edk macros
 513     if OverrideDir != '' and OverrideDir != None:
 514         if OverrideDir.find('$(EFI_SOURCE)') > -1:
 515             OverrideDir = OverrideDir.replace('$(EFI_SOURCE)', EfiSource)
 516         if OverrideDir.find('$(EDK_SOURCE)') > -1:
 517             OverrideDir = OverrideDir.replace('$(EDK_SOURCE)', EdkSource)
 518
 519     # Replace the default dir to current dir
 520     # Dir is current module dir related to workspace
 521     if Dir == '.':
 522         Dir = os.getcwd()
 523         Dir = Dir[len(Workspace)+1:]
 524
 525     NewFile = File
 526     RelaPath = AllFiles[os.path.normpath(Dir)]
 527     NewRelaPath = RelaPath
 528
 529     while(True):
 530         # First check if File has Edk definition itself
 531         if File.find('$(EFI_SOURCE)') > -1 or File.find('$(EDK_SOURCE)') > -1:
 532             File = File.replace('$(EFI_SOURCE)', EfiSource)
 533             File = File.replace('$(EDK_SOURCE)', EdkSource)
 534             NewFile = AllFiles[os.path.normpath(File)]
 535             if NewFile != None:
 536                 NewRelaPath = os.path.dirname(NewFile)
 537                 File = os.path.basename(NewFile)
 538                 #NewRelaPath = NewFile[:len(NewFile) - len(File.replace("..\\", '').replace("../", '')) - 1]
 539                 break
 540
 541         # Second check the path with override value
 542         if OverrideDir != '' and OverrideDir != None:
 543             NewFile = AllFiles[os.path.normpath(os.path.join(OverrideDir, File))]
 544             if NewFile != None:
 545                 #NewRelaPath = os.path.dirname(NewFile)
 546                 NewRelaPath = NewFile[:len(NewFile) - len(File.replace("..\\", '').replace("../", '')) - 1]
 547                 break
 548
 549         # Last check the path with normal definitions
 550         NewFile = AllFiles[os.path.normpath(os.path.join(Dir, File))]
 551         if NewFile != None:
 552             break
 553
 554         # No file found
 555         break
 556
 557     return NewRelaPath, RelaPath, File
 558
 559
 560 def GetRelPath(Path1, Path2):
 561     FileName = os.path.basename(Path2)
 562     L1 = os.path.normpath(Path1).split(os.path.normpath('/'))
 563     L2 = os.path.normpath(Path2).split(os.path.normpath('/'))
 564     for Index in range(0, len(L1)):
 565         if L1[Index] != L2[Index]:
 566             FileName = '../' * (len(L1) - Index)
 567             for Index2 in range(Index, len(L2)):
 568                 FileName = os.path.join(FileName, L2[Index2])
 569             break
 570     return os.path.normpath(FileName)
 571
 572
 573 ## Get GUID value from given packages
 574 #
 575 #   @param      CName           The CName of the GUID
 576 #   @param      PackageList     List of packages looking-up in
 577 #
 578 #   @retval     GuidValue   if the CName is found in any given package
 579 #   @retval     None        if the CName is not found in all given packages
 580 #
 581 def GuidValue(CName, PackageList):
 582     for P in PackageList:
 583         if CName in P.Guids:
 584             return P.Guids[CName]
 585     return None
 586
 587 ## Get Protocol value from given packages
 588 #
 589 #   @param      CName           The CName of the GUID
 590 #   @param      PackageList     List of packages looking-up in
 591 #
 592 #   @retval     GuidValue   if the CName is found in any given package
 593 #   @retval     None        if the CName is not found in all given packages
 594 #
 595 def ProtocolValue(CName, PackageList):
 596     for P in PackageList:
 597         if CName in P.Protocols:
 598             return P.Protocols[CName]
 599     return None
 600
 601 ## Get PPI value from given packages
 602 #
 603 #   @param      CName           The CName of the GUID
 604 #   @param      PackageList     List of packages looking-up in
 605 #
 606 #   @retval     GuidValue   if the CName is found in any given package
 607 #   @retval     None        if the CName is not found in all given packages
 608 #
 609 def PpiValue(CName, PackageList):
 610     for P in PackageList:
 611         if CName in P.Ppis:
 612             return P.Ppis[CName]
 613     return None
 614
 615 ## A string template class
 616 #
 617 #  This class implements a template for string replacement. A string template
 618 #  looks like following
 619 #
 620 #       ${BEGIN} other_string ${placeholder_name} other_string ${END}
 621 #
 622 #  The string between ${BEGIN} and ${END} will be repeated as many times as the
 623 #  length of "placeholder_name", which is a list passed through a dict. The
 624 #  "placeholder_name" is the key name of the dict. The ${BEGIN} and ${END} can
 625 #  be not used and, in this case, the "placeholder_name" must not a list and it
 626 #  will just be replaced once.
 627 #
 628 class TemplateString(object):
 629     _REPEAT_START_FLAG = "BEGIN"
 630     _REPEAT_END_FLAG = "END"
 631
 632     class Section(object):
 633         _LIST_TYPES = [type([]), type(set()), type((0,))]
 634
 635         def __init__(self, TemplateSection, PlaceHolderList):
 636             self._Template = TemplateSection
 637             self._PlaceHolderList = []
 638
 639             # Split the section into sub-sections according to the position of placeholders
 640             if PlaceHolderList:
 641                 self._SubSectionList = []
 642                 SubSectionStart = 0
 643                 #
 644                 # The placeholders passed in must be in the format of
 645                 #
 646                 #   PlaceHolderName, PlaceHolderStartPoint, PlaceHolderEndPoint
 647                 #
 648                 for PlaceHolder,Start,End in PlaceHolderList:
 649                     self._SubSectionList.append(TemplateSection[SubSectionStart:Start])
 650                     self._SubSectionList.append(TemplateSection[Start:End])
 651                     self._PlaceHolderList.append(PlaceHolder)
 652                     SubSectionStart = End
 653                 if SubSectionStart < len(TemplateSection):
 654                     self._SubSectionList.append(TemplateSection[SubSectionStart:])
 655             else:
 656                 self._SubSectionList = [TemplateSection]
 657
 658         def __str__(self):
 659             return self._Template + " : " + str(self._PlaceHolderList)
 660
 661         def Instantiate(self, PlaceHolderValues):
 662             RepeatTime = -1
 663             RepeatPlaceHolders = {}
 664             NonRepeatPlaceHolders = {}
 665
 666             for PlaceHolder in self._PlaceHolderList:
 667                 if PlaceHolder not in PlaceHolderValues:
 668                     continue
 669                 Value = PlaceHolderValues[PlaceHolder]
 670                 if type(Value) in self._LIST_TYPES:
 671                     if RepeatTime < 0:
 672                         RepeatTime = len(Value)
 673                     elif RepeatTime != len(Value):
 674                         EdkLogger.error(
 675                                     "TemplateString",
 676                                     PARAMETER_INVALID,
 677                                     "${%s} has different repeat time from others!" % PlaceHolder,
 678                                     ExtraData=str(self._Template)
 679                                     )
 680                     RepeatPlaceHolders["${%s}" % PlaceHolder] = Value
 681                 else:
 682                     NonRepeatPlaceHolders["${%s}" % PlaceHolder] = Value
 683
 684             if NonRepeatPlaceHolders:
 685                 StringList = []
 686                 for S in self._SubSectionList:
 687                     if S not in NonRepeatPlaceHolders:
 688                         StringList.append(S)
 689                     else:
 690                         StringList.append(str(NonRepeatPlaceHolders[S]))
 691             else:
 692                 StringList = self._SubSectionList
 693
 694             if RepeatPlaceHolders:
 695                 TempStringList = []
 696                 for Index in range(RepeatTime):
 697                     for S in StringList:
 698                         if S not in RepeatPlaceHolders:
 699                             TempStringList.append(S)
 700                         else:
 701                             TempStringList.append(str(RepeatPlaceHolders[S][Index]))
 702                 StringList = TempStringList
 703
 704             return "".join(StringList)
 705
 706     ## Constructor
 707     def __init__(self, Template=None):
 708         self.String = ''
 709         self.IsBinary = False
 710         self._Template = Template
 711         self._TemplateSectionList = self._Parse(Template)
 712
 713     ## str() operator
 714     #
 715     #   @retval     string  The string replaced
 716     #
 717     def __str__(self):
 718         return self.String
 719
 720     ## Split the template string into fragments per the ${BEGIN} and ${END} flags
 721     #
 722     #   @retval     list    A list of TemplateString.Section objects
 723     #
 724     def _Parse(self, Template):
 725         SectionStart = 0
 726         SearchFrom = 0
 727         MatchEnd = 0
 728         PlaceHolderList = []
 729         TemplateSectionList = []
 730         while Template:
 731             MatchObj = gPlaceholderPattern.search(Template, SearchFrom)
 732             if not MatchObj:
 733                 if MatchEnd <= len(Template):
 734                     TemplateSection = TemplateString.Section(Template[SectionStart:], PlaceHolderList)
 735                     TemplateSectionList.append(TemplateSection)
 736                 break
 737
 738             MatchString = MatchObj.group(1)
 739             MatchStart = MatchObj.start()
 740             MatchEnd = MatchObj.end()
 741
 742             if MatchString == self._REPEAT_START_FLAG:
 743                 if MatchStart > SectionStart:
 744                     TemplateSection = TemplateString.Section(Template[SectionStart:MatchStart], PlaceHolderList)
 745                     TemplateSectionList.append(TemplateSection)
 746                 SectionStart = MatchEnd
 747                 PlaceHolderList = []
 748             elif MatchString == self._REPEAT_END_FLAG:
 749                 TemplateSection = TemplateString.Section(Template[SectionStart:MatchStart], PlaceHolderList)
 750                 TemplateSectionList.append(TemplateSection)
 751                 SectionStart = MatchEnd
 752                 PlaceHolderList = []
 753             else:
 754                 PlaceHolderList.append((MatchString, MatchStart - SectionStart, MatchEnd - SectionStart))
 755             SearchFrom = MatchEnd
 756         return TemplateSectionList
 757
 758     ## Replace the string template with dictionary of placeholders and append it to previous one
 759     #
 760     #   @param      AppendString    The string template to append
 761     #   @param      Dictionary      The placeholder dictionaries
 762     #
 763     def Append(self, AppendString, Dictionary=None):
 764         if Dictionary:
 765             SectionList = self._Parse(AppendString)
 766             self.String += "".join([S.Instantiate(Dictionary) for S in SectionList])
 767         else:
 768             self.String += AppendString
 769
 770     ## Replace the string template with dictionary of placeholders
 771     #
 772     #   @param      Dictionary      The placeholder dictionaries
 773     #
 774     #   @retval     str             The string replaced with placeholder values
 775     #
 776     def Replace(self, Dictionary=None):
 777         return "".join([S.Instantiate(Dictionary) for S in self._TemplateSectionList])
 778
 779 ## Progress indicator class
 780 #
 781 #  This class makes use of thread to print progress on console.
 782 #
 783 class Progressor:
 784     # for avoiding deadloop
 785     _StopFlag = None
 786     _ProgressThread = None
 787     _CheckInterval = 0.25
 788
 789     ## Constructor
 790     #
 791     #   @param      OpenMessage     The string printed before progress charaters
 792     #   @param      CloseMessage    The string printed after progress charaters
 793     #   @param      ProgressChar    The charater used to indicate the progress
 794     #   @param      Interval        The interval in seconds between two progress charaters
 795     #
 796     def __init__(self, OpenMessage="", CloseMessage="", ProgressChar='.', Interval=1.0):
 797         self.PromptMessage = OpenMessage
 798         self.CodaMessage = CloseMessage
 799         self.ProgressChar = ProgressChar
 800         self.Interval = Interval
 801         if Progressor._StopFlag == None:
 802             Progressor._StopFlag = threading.Event()
 803
 804     ## Start to print progress charater
 805     #
 806     #   @param      OpenMessage     The string printed before progress charaters
 807     #
 808     def Start(self, OpenMessage=None):
 809         if OpenMessage != None:
 810             self.PromptMessage = OpenMessage
 811         Progressor._StopFlag.clear()
 812         if Progressor._ProgressThread == None:
 813             Progressor._ProgressThread = threading.Thread(target=self._ProgressThreadEntry)
 814             Progressor._ProgressThread.setDaemon(False)
 815             Progressor._ProgressThread.start()
 816
 817     ## Stop printing progress charater
 818     #
 819     #   @param      CloseMessage    The string printed after progress charaters
 820     #
 821     def Stop(self, CloseMessage=None):
 822         OriginalCodaMessage = self.CodaMessage
 823         if CloseMessage != None:
 824             self.CodaMessage = CloseMessage
 825         self.Abort()
 826         self.CodaMessage = OriginalCodaMessage
 827
 828     ## Thread entry method
 829     def _ProgressThreadEntry(self):
 830         sys.stdout.write(self.PromptMessage + " ")
 831         sys.stdout.flush()
 832         TimeUp = 0.0
 833         while not Progressor._StopFlag.isSet():
 834             if TimeUp <= 0.0:
 835                 sys.stdout.write(self.ProgressChar)
 836                 sys.stdout.flush()
 837                 TimeUp = self.Interval
 838             time.sleep(self._CheckInterval)
 839             TimeUp -= self._CheckInterval
 840         sys.stdout.write(" " + self.CodaMessage + "\n")
 841         sys.stdout.flush()
 842
 843     ## Abort the progress display
 844     @staticmethod
 845     def Abort():
 846         if Progressor._StopFlag != None:
 847             Progressor._StopFlag.set()
 848         if Progressor._ProgressThread != None:
 849             Progressor._ProgressThread.join()
 850             Progressor._ProgressThread = None
 851
 852 ## A dict which can access its keys and/or values orderly
 853 #
 854 #  The class implements a new kind of dict which its keys or values can be
 855 #  accessed in the order they are added into the dict. It guarantees the order
 856 #  by making use of an internal list to keep a copy of keys.
 857 #
 858 class sdict(IterableUserDict):
 859     ## Constructor
 860     def __init__(self):
 861         IterableUserDict.__init__(self)
 862         self._key_list = []
 863
 864     ## [] operator
 865     def __setitem__(self, key, value):
 866         if key not in self._key_list:
 867             self._key_list.append(key)
 868         IterableUserDict.__setitem__(self, key, value)
 869
 870     ## del operator
 871     def __delitem__(self, key):
 872         self._key_list.remove(key)
 873         IterableUserDict.__delitem__(self, key)
 874
 875     ## used in "for k in dict" loop to ensure the correct order
 876     def __iter__(self):
 877         return self.iterkeys()
 878
 879     ## len() support
 880     def __len__(self):
 881         return len(self._key_list)
 882
 883     ## "in" test support
 884     def __contains__(self, key):
 885         return key in self._key_list
 886
 887     ## indexof support
 888     def index(self, key):
 889         return self._key_list.index(key)
 890
 891     ## insert support
 892     def insert(self, key, newkey, newvalue, order):
 893         index = self._key_list.index(key)
 894         if order == 'BEFORE':
 895             self._key_list.insert(index, newkey)
 896             IterableUserDict.__setitem__(self, newkey, newvalue)
 897         elif order == 'AFTER':
 898             self._key_list.insert(index + 1, newkey)
 899             IterableUserDict.__setitem__(self, newkey, newvalue)
 900
 901     ## append support
 902     def append(self, sdict):
 903         for key in sdict:
 904             if key not in self._key_list:
 905                 self._key_list.append(key)
 906             IterableUserDict.__setitem__(self, key, sdict[key])
 907
 908     def has_key(self, key):
 909         return key in self._key_list
 910
 911     ## Empty the dict
 912     def clear(self):
 913         self._key_list = []
 914         IterableUserDict.clear(self)
 915
 916     ## Return a copy of keys
 917     def keys(self):
 918         keys = []
 919         for key in self._key_list:
 920             keys.append(key)
 921         return keys
 922
 923     ## Return a copy of values
 924     def values(self):
 925         values = []
 926         for key in self._key_list:
 927             values.append(self[key])
 928         return values
 929
 930     ## Return a copy of (key, value) list
 931     def items(self):
 932         items = []
 933         for key in self._key_list:
 934             items.append((key, self[key]))
 935         return items
 936
 937     ## Iteration support
 938     def iteritems(self):
 939         return iter(self.items())
 940
 941     ## Keys interation support
 942     def iterkeys(self):
 943         return iter(self.keys())
 944
 945     ## Values interation support
 946     def itervalues(self):
 947         return iter(self.values())
 948
 949     ## Return value related to a key, and remove the (key, value) from the dict
 950     def pop(self, key, *dv):
 951         value = None
 952         if key in self._key_list:
 953             value = self[key]
 954             self.__delitem__(key)
 955         elif len(dv) != 0 :
 956             value = kv[0]
 957         return value
 958
 959     ## Return (key, value) pair, and remove the (key, value) from the dict
 960     def popitem(self):
 961         key = self._key_list[-1]
 962         value = self[key]
 963         self.__delitem__(key)
 964         return key, value
 965
 966     def update(self, dict=None, **kwargs):
 967         if dict != None:
 968             for k, v in dict.items():
 969                 self[k] = v
 970         if len(kwargs):
 971             for k, v in kwargs.items():
 972                 self[k] = v
 973
 974 ## Dictionary with restricted keys
 975 #
 976 class rdict(dict):
 977     ## Constructor
 978     def __init__(self, KeyList):
 979         for Key in KeyList:
 980             dict.__setitem__(self, Key, "")
 981
 982     ## []= operator
 983     def __setitem__(self, key, value):
 984         if key not in self:
 985             EdkLogger.error("RestrictedDict", ATTRIBUTE_SET_FAILURE, "Key [%s] is not allowed" % key,
 986                             ExtraData=", ".join(dict.keys(self)))
 987         dict.__setitem__(self, key, value)
 988
 989     ## =[] operator
 990     def __getitem__(self, key):
 991         if key not in self:
 992             return ""
 993         return dict.__getitem__(self, key)
 994
 995     ## del operator
 996     def __delitem__(self, key):
 997         EdkLogger.error("RestrictedDict", ATTRIBUTE_ACCESS_DENIED, ExtraData="del")
 998
 999     ## Empty the dict
1000     def clear(self):
1001         for Key in self:
1002             self.__setitem__(Key, "")
1003
1004     ## Return value related to a key, and remove the (key, value) from the dict
1005     def pop(self, key, *dv):
1006         EdkLogger.error("RestrictedDict", ATTRIBUTE_ACCESS_DENIED, ExtraData="pop")
1007
1008     ## Return (key, value) pair, and remove the (key, value) from the dict
1009     def popitem(self):
1010         EdkLogger.error("RestrictedDict", ATTRIBUTE_ACCESS_DENIED, ExtraData="popitem")
1011
1012 ## Dictionary using prioritized list as key
1013 #
1014 class tdict:
1015     _ListType = type([])
1016     _TupleType = type(())
1017     _Wildcard = 'COMMON'
1018     _ValidWildcardList = ['COMMON', 'DEFAULT', 'ALL', '*', 'PLATFORM']
1019
1020     def __init__(self, _Single_=False, _Level_=2):
1021         self._Level_ = _Level_
1022         self.data = {}
1023         self._Single_ = _Single_
1024
1025     # =[] operator
1026     def __getitem__(self, key):
1027         KeyType = type(key)
1028         RestKeys = None
1029         if KeyType == self._ListType or KeyType == self._TupleType:
1030             FirstKey = key[0]
1031             if len(key) > 1:
1032                 RestKeys = key[1:]
1033             elif self._Level_ > 1:
1034                 RestKeys = [self._Wildcard for i in range(0, self._Level_-1)]
1035         else:
1036             FirstKey = key
1037             if self._Level_ > 1:
1038                 RestKeys = [self._Wildcard for i in range(0, self._Level_-1)]
1039
1040         if FirstKey == None or str(FirstKey).upper() in self._ValidWildcardList:
1041             FirstKey = self._Wildcard
1042
1043         if self._Single_:
1044             return self._GetSingleValue(FirstKey, RestKeys)
1045         else:
1046             return self._GetAllValues(FirstKey, RestKeys)
1047
1048     def _GetSingleValue(self, FirstKey, RestKeys):
1049         Value = None
1050         #print "%s-%s" % (FirstKey, self._Level_) ,
1051         if self._Level_ > 1:
1052             if FirstKey == self._Wildcard:
1053                 if FirstKey in self.data:
1054                     Value = self.data[FirstKey][RestKeys]
1055                 if Value == None:
1056                     for Key in self.data:
1057                         Value = self.data[Key][RestKeys]
1058                         if Value != None: break
1059             else:
1060                 if FirstKey in self.data:
1061                     Value = self.data[FirstKey][RestKeys]
1062                 if Value == None and self._Wildcard in self.data:
1063                     #print "Value=None"
1064                     Value = self.data[self._Wildcard][RestKeys]
1065         else:
1066             if FirstKey == self._Wildcard:
1067                 if FirstKey in self.data:
1068                     Value = self.data[FirstKey]
1069                 if Value == None:
1070                     for Key in self.data:
1071                         Value = self.data[Key]
1072                         if Value != None: break
1073             else:
1074                 if FirstKey in self.data:
1075                     Value = self.data[FirstKey]
1076                 elif self._Wildcard in self.data:
1077                     Value = self.data[self._Wildcard]
1078         return Value
1079
1080     def _GetAllValues(self, FirstKey, RestKeys):
1081         Value = []
1082         if self._Level_ > 1:
1083             if FirstKey == self._Wildcard:
1084                 for Key in self.data:
1085                     Value += self.data[Key][RestKeys]
1086             else:
1087                 if FirstKey in self.data:
1088                     Value += self.data[FirstKey][RestKeys]
1089                 if self._Wildcard in self.data:
1090                     Value += self.data[self._Wildcard][RestKeys]
1091         else:
1092             if FirstKey == self._Wildcard:
1093                 for Key in self.data:
1094                     Value.append(self.data[Key])
1095             else:
1096                 if FirstKey in self.data:
1097                     Value.append(self.data[FirstKey])
1098                 if self._Wildcard in self.data:
1099                     Value.append(self.data[self._Wildcard])
1100         return Value
1101
1102     ## []= operator
1103     def __setitem__(self, key, value):
1104         KeyType = type(key)
1105         RestKeys = None
1106         if KeyType == self._ListType or KeyType == self._TupleType:
1107             FirstKey = key[0]
1108             if len(key) > 1:
1109                 RestKeys = key[1:]
1110             else:
1111                 RestKeys = [self._Wildcard for i in range(0, self._Level_-1)]
1112         else:
1113             FirstKey = key
1114             if self._Level_ > 1:
1115                 RestKeys = [self._Wildcard for i in range(0, self._Level_-1)]
1116
1117         if FirstKey in self._ValidWildcardList:
1118             FirstKey = self._Wildcard
1119
1120         if FirstKey not in self.data and self._Level_ > 0:
1121             self.data[FirstKey] = tdict(self._Single_, self._Level_ - 1)
1122
1123         if self._Level_ > 1:
1124             self.data[FirstKey][RestKeys] = value
1125         else:
1126             self.data[FirstKey] = value
1127
1128     def SetGreedyMode(self):
1129         self._Single_ = False
1130         if self._Level_ > 1:
1131             for Key in self.data:
1132                 self.data[Key].SetGreedyMode()
1133
1134     def SetSingleMode(self):
1135         self._Single_ = True
1136         if self._Level_ > 1:
1137             for Key in self.data:
1138                 self.data[Key].SetSingleMode()
1139
1140     def GetKeys(self, KeyIndex=0):
1141         assert KeyIndex >= 0
1142         if KeyIndex == 0:
1143             return set(self.data.keys())
1144         else:
1145             keys = set()
1146             for Key in self.data:
1147                 keys |= self.data[Key].GetKeys(KeyIndex - 1)
1148             return keys
1149
1150 ## Boolean chain list
1151 #
1152 class Blist(UserList):
1153     def __init__(self, initlist=None):
1154         UserList.__init__(self, initlist)
1155     def __setitem__(self, i, item):
1156         if item not in [True, False]:
1157             if item == 0:
1158                 item = False
1159             else:
1160                 item = True
1161         self.data[i] = item
1162     def _GetResult(self):
1163         Value = True
1164         for item in self.data:
1165             Value &= item
1166         return Value
1167     Result = property(_GetResult)
1168
1169 def ParseConsoleLog(Filename):
1170     Opr = open(os.path.normpath(Filename), 'r')
1171     Opw = open(os.path.normpath(Filename + '.New'), 'w+')
1172     for Line in Opr.readlines():
1173         if Line.find('.efi') > -1:
1174             Line = Line[Line.rfind(' ') : Line.rfind('.efi')].strip()
1175             Opw.write('%s\n' % Line)
1176
1177     Opr.close()
1178     Opw.close()
1179
1180 ## AnalyzeDscPcd
1181 #
1182 #  Analyze DSC PCD value, since there is no data type info in DSC
1183 #  This fuction is used to match functions (AnalyzePcdData, AnalyzeHiiPcdData, AnalyzeVpdPcdData) used for retrieving PCD value from database
1184 #  1. Feature flag: TokenSpace.PcdCName|PcdValue
1185 #  2. Fix and Patch:TokenSpace.PcdCName|PcdValue[|MaxSize]
1186 #  3. Dynamic default:
1187 #     TokenSpace.PcdCName|PcdValue[|VOID*[|MaxSize]]
1188 #     TokenSpace.PcdCName|PcdValue
1189 #  4. Dynamic VPD:
1190 #     TokenSpace.PcdCName|VpdOffset[|VpdValue]
1191 #     TokenSpace.PcdCName|VpdOffset[|MaxSize[|VpdValue]]
1192 #  5. Dynamic HII:
1193 #     TokenSpace.PcdCName|HiiString|VaiableGuid|VariableOffset[|HiiValue]
1194 #  PCD value needs to be located in such kind of string, and the PCD value might be an expression in which
1195 #    there might have "|" operator, also in string value.
1196 #
1197 #  @param Setting: String contain information described above with "TokenSpace.PcdCName|" stripped
1198 #  @param PcdType: PCD type: feature, fixed, dynamic default VPD HII
1199 #  @param DataType: The datum type of PCD: VOID*, UNIT, BOOL
1200 #  @retval:
1201 #    ValueList: A List contain fields described above
1202 #    IsValid:   True if conforming EBNF, otherwise False
1203 #    Index:     The index where PcdValue is in ValueList
1204 #
1205 def AnalyzeDscPcd(Setting, PcdType, DataType=''):
1206     Setting = Setting.strip()
1207     # There might be escaped quote in a string: \", \\\"
1208     Data = Setting.replace('\\\\', '//').replace('\\\"', '\\\'')
1209     # There might be '|' in string and in ( ... | ... ), replace it with '-'
1210     NewStr = ''
1211     InStr = False
1212     Pair = 0
1213     for ch in Data:
1214         if ch == '"':
1215             InStr = not InStr
1216         elif ch == '(' and not InStr:
1217             Pair += 1
1218         elif ch == ')' and not InStr:
1219             Pair -= 1
1220
1221         if (Pair > 0 or InStr) and ch == TAB_VALUE_SPLIT:
1222             NewStr += '-'
1223         else:
1224             NewStr += ch
1225     FieldList = []
1226     StartPos = 0
1227     while True:
1228         Pos = NewStr.find(TAB_VALUE_SPLIT, StartPos)
1229         if Pos < 0:
1230             FieldList.append(Setting[StartPos:].strip())
1231             break
1232         FieldList.append(Setting[StartPos:Pos].strip())
1233         StartPos = Pos + 1
1234
1235     IsValid = True
1236     if PcdType in (MODEL_PCD_FIXED_AT_BUILD, MODEL_PCD_PATCHABLE_IN_MODULE, MODEL_PCD_FEATURE_FLAG):
1237         Value = FieldList[0]
1238         Size = ''
1239         if len(FieldList) > 1:
1240             Size = FieldList[1]
1241         if DataType == 'VOID*':
1242             IsValid = (len(FieldList) <= 2)
1243         else:
1244             IsValid = (len(FieldList) <= 1)
1245         return [Value, '', Size], IsValid, 0
1246     elif PcdType in (MODEL_PCD_DYNAMIC_DEFAULT, MODEL_PCD_DYNAMIC_EX_DEFAULT):
1247         Value = FieldList[0]
1248         Size = Type = ''
1249         if len(FieldList) > 1:
1250             Type = FieldList[1]
1251         if len(FieldList) > 2:
1252             Size = FieldList[2]
1253         if DataType == 'VOID*':
1254             IsValid = (len(FieldList) <= 3)
1255         else:
1256             IsValid = (len(FieldList) <= 1)
1257         return [Value, Type, Size], IsValid, 0
1258     elif PcdType in (MODEL_PCD_DYNAMIC_VPD, MODEL_PCD_DYNAMIC_EX_VPD):
1259         VpdOffset = FieldList[0]
1260         Value = Size = ''
1261         if not DataType == 'VOID*':
1262             if len(FieldList) > 1:
1263                 Value = FieldList[1]
1264         else:
1265             if len(FieldList) > 1:
1266                 Size = FieldList[1]
1267             if len(FieldList) > 2:
1268                 Value = FieldList[2]
1269         if DataType == 'VOID*':
1270             IsValid = (len(FieldList) <= 3)
1271         else:
1272             IsValid = (len(FieldList) <= 2)
1273         return [VpdOffset, Size, Value], IsValid, 2
1274     elif PcdType in (MODEL_PCD_DYNAMIC_HII, MODEL_PCD_DYNAMIC_EX_HII):
1275         HiiString = FieldList[0]
1276         Guid = Offset = Value = ''
1277         if len(FieldList) > 1:
1278             Guid = FieldList[1]
1279         if len(FieldList) > 2:
1280             Offset = FieldList[2]
1281         if len(FieldList) > 3:
1282             Value = FieldList[3]
1283         IsValid = (3 <= len(FieldList) <= 4)
1284         return [HiiString, Guid, Offset, Value], IsValid, 3
1285     return [], False, 0
1286
1287 ## AnalyzePcdData
1288 #
1289 #  Analyze the pcd Value, Datum type and TokenNumber.
1290 #  Used to avoid split issue while the value string contain "|" character
1291 #
1292 #  @param[in] Setting:  A String contain value/datum type/token number information;
1293 #
1294 #  @retval   ValueList: A List contain value, datum type and toke number.
1295 #
1296 def AnalyzePcdData(Setting):
1297     ValueList = ['', '', '']
1298
1299     ValueRe  = re.compile(r'^\s*L?\".*\|.*\"')
1300     PtrValue = ValueRe.findall(Setting)
1301
1302     ValueUpdateFlag = False
1303
1304     if len(PtrValue) >= 1:
1305         Setting = re.sub(ValueRe, '', Setting)
1306         ValueUpdateFlag = True
1307
1308     TokenList = Setting.split(TAB_VALUE_SPLIT)
1309     ValueList[0:len(TokenList)] = TokenList
1310
1311     if ValueUpdateFlag:
1312         ValueList[0] = PtrValue[0]
1313
1314     return ValueList
1315
1316 ## AnalyzeHiiPcdData
1317 #
1318 #  Analyze the pcd Value, variable name, variable Guid and variable offset.
1319 #  Used to avoid split issue while the value string contain "|" character
1320 #
1321 #  @param[in] Setting:  A String contain VariableName, VariableGuid, VariableOffset, DefaultValue information;
1322 #
1323 #  @retval   ValueList: A List contaian VariableName, VariableGuid, VariableOffset, DefaultValue.
1324 #
1325 def AnalyzeHiiPcdData(Setting):
1326     ValueList = ['', '', '', '']
1327
1328     ValueRe  = re.compile(r'^\s*L?\".*\|.*\"')
1329     PtrValue = ValueRe.findall(Setting)
1330
1331     ValueUpdateFlag = False
1332
1333     if len(PtrValue) >= 1:
1334         Setting = re.sub(ValueRe, '', Setting)
1335         ValueUpdateFlag = True
1336
1337     TokenList = Setting.split(TAB_VALUE_SPLIT)
1338     ValueList[0:len(TokenList)] = TokenList
1339
1340     if ValueUpdateFlag:
1341         ValueList[0] = PtrValue[0]
1342
1343     return ValueList
1344
1345 ## AnalyzeVpdPcdData
1346 #
1347 #  Analyze the vpd pcd VpdOffset, MaxDatumSize and InitialValue.
1348 #  Used to avoid split issue while the value string contain "|" character
1349 #
1350 #  @param[in] Setting:  A String contain VpdOffset/MaxDatumSize/InitialValue information;
1351 #
1352 #  @retval   ValueList: A List contain VpdOffset, MaxDatumSize and InitialValue.
1353 #
1354 def AnalyzeVpdPcdData(Setting):
1355     ValueList = ['', '', '']
1356
1357     ValueRe  = re.compile(r'\s*L?\".*\|.*\"\s*$')
1358     PtrValue = ValueRe.findall(Setting)
1359
1360     ValueUpdateFlag = False
1361
1362     if len(PtrValue) >= 1:
1363         Setting = re.sub(ValueRe, '', Setting)
1364         ValueUpdateFlag = True
1365
1366     TokenList = Setting.split(TAB_VALUE_SPLIT)
1367     ValueList[0:len(TokenList)] = TokenList
1368
1369     if ValueUpdateFlag:
1370         ValueList[2] = PtrValue[0]
1371
1372     return ValueList
1373
1374 ## check format of PCD value against its the datum type
1375 #
1376 # For PCD value setting
1377 #
1378 def CheckPcdDatum(Type, Value):
1379     if Type == "VOID*":
1380         ValueRe  = re.compile(r'\s*L?\".*\"\s*$')
1381         if not (((Value.startswith('L"') or Value.startswith('"')) and Value.endswith('"'))
1382                 or (Value.startswith('{') and Value.endswith('}'))
1383                ):
1384             return False, "Invalid value [%s] of type [%s]; must be in the form of {...} for array"\
1385                           ", or \"...\" for string, or L\"...\" for unicode string" % (Value, Type)
1386         elif ValueRe.match(Value):
1387             # Check the chars in UnicodeString or CString is printable
1388             if Value.startswith("L"):
1389                 Value = Value[2:-1]
1390             else:
1391                 Value = Value[1:-1]
1392             Printset = set(string.printable)
1393             Printset.remove(TAB_PRINTCHAR_VT)
1394             Printset.add(TAB_PRINTCHAR_BS)
1395             Printset.add(TAB_PRINTCHAR_NUL)
1396             if not set(Value).issubset(Printset):
1397                 PrintList = list(Printset)
1398                 PrintList.sort()
1399                 return False, "Invalid PCD string value of type [%s]; must be printable chars %s." % (Type, PrintList)
1400     elif Type == 'BOOLEAN':
1401         if Value not in ['TRUE', 'True', 'true', '0x1', '0x01', '1', 'FALSE', 'False', 'false', '0x0', '0x00', '0']:
1402             return False, "Invalid value [%s] of type [%s]; must be one of TRUE, True, true, 0x1, 0x01, 1"\
1403                           ", FALSE, False, false, 0x0, 0x00, 0" % (Value, Type)
1404     elif Type in [TAB_UINT8, TAB_UINT16, TAB_UINT32, TAB_UINT64]:
1405         try:
1406             Value = long(Value, 0)
1407         except:
1408             return False, "Invalid value [%s] of type [%s];"\
1409                           " must be a hexadecimal, decimal or octal in C language format." % (Value, Type)
1410     else:
1411         return False, "Invalid type [%s]; must be one of VOID*, BOOLEAN, UINT8, UINT16, UINT32, UINT64." % (Type)
1412
1413     return True, ""
1414
1415 ## Split command line option string to list
1416 #
1417 # subprocess.Popen needs the args to be a sequence. Otherwise there's problem
1418 # in non-windows platform to launch command
1419 #
1420 def SplitOption(OptionString):
1421     OptionList = []
1422     LastChar = " "
1423     OptionStart = 0
1424     QuotationMark = ""
1425     for Index in range(0, len(OptionString)):
1426         CurrentChar = OptionString[Index]
1427         if CurrentChar in ['"', "'"]:
1428             if QuotationMark == CurrentChar:
1429                 QuotationMark = ""
1430             elif QuotationMark == "":
1431                 QuotationMark = CurrentChar
1432             continue
1433         elif QuotationMark:
1434             continue
1435
1436         if CurrentChar in ["/", "-"] and LastChar in [" ", "\t", "\r", "\n"]:
1437             if Index > OptionStart:
1438                 OptionList.append(OptionString[OptionStart:Index-1])
1439             OptionStart = Index
1440         LastChar = CurrentChar
1441     OptionList.append(OptionString[OptionStart:])
1442     return OptionList
1443
1444 def CommonPath(PathList):
1445     P1 = min(PathList).split(os.path.sep)
1446     P2 = max(PathList).split(os.path.sep)
1447     for Index in xrange(min(len(P1), len(P2))):
1448         if P1[Index] != P2[Index]:
1449             return os.path.sep.join(P1[:Index])
1450     return os.path.sep.join(P1)
1451
1452 class PathClass(object):
1453     def __init__(self, File='', Root='', AlterRoot='', Type='', IsBinary=False,
1454                  Arch='COMMON', ToolChainFamily='', Target='', TagName='', ToolCode=''):
1455         self.Arch = Arch
1456         self.File = str(File)
1457         if os.path.isabs(self.File):
1458             self.Root = ''
1459             self.AlterRoot = ''
1460         else:
1461             self.Root = str(Root)
1462             self.AlterRoot = str(AlterRoot)
1463
1464         # Remove any '.' and '..' in path
1465         if self.Root:
1466             self.Path = os.path.normpath(os.path.join(self.Root, self.File))
1467             self.Root = os.path.normpath(CommonPath([self.Root, self.Path]))
1468             # eliminate the side-effect of 'C:'
1469             if self.Root[-1] == ':':
1470                 self.Root += os.path.sep
1471             # file path should not start with path separator
1472             if self.Root[-1] == os.path.sep:
1473                 self.File = self.Path[len(self.Root):]
1474             else:
1475                 self.File = self.Path[len(self.Root)+1:]
1476         else:
1477             self.Path = os.path.normpath(self.File)
1478
1479         self.SubDir, self.Name = os.path.split(self.File)
1480         self.BaseName, self.Ext = os.path.splitext(self.Name)
1481
1482         if self.Root:
1483             if self.SubDir:
1484                 self.Dir = os.path.join(self.Root, self.SubDir)
1485             else:
1486                 self.Dir = self.Root
1487         else:
1488             self.Dir = self.SubDir
1489
1490         if IsBinary:
1491             self.Type = Type
1492         else:
1493             self.Type = self.Ext.lower()
1494
1495         self.IsBinary = IsBinary
1496         self.Target = Target
1497         self.TagName = TagName
1498         self.ToolCode = ToolCode
1499         self.ToolChainFamily = ToolChainFamily
1500
1501         self._Key = None
1502
1503     ## Convert the object of this class to a string
1504     #
1505     #  Convert member Path of the class to a string
1506     #
1507     #  @retval string Formatted String
1508     #
1509     def __str__(self):
1510         return self.Path
1511
1512     ## Override __eq__ function
1513     #
1514     # Check whether PathClass are the same
1515     #
1516     # @retval False The two PathClass are different
1517     # @retval True  The two PathClass are the same
1518     #
1519     def __eq__(self, Other):
1520         if type(Other) == type(self):
1521             return self.Path == Other.Path
1522         else:
1523             return self.Path == str(Other)
1524
1525     ## Override __cmp__ function
1526     #
1527     # Customize the comparsion operation of two PathClass
1528     #
1529     # @retval 0     The two PathClass are different
1530     # @retval -1    The first PathClass is less than the second PathClass
1531     # @retval 1     The first PathClass is Bigger than the second PathClass
1532     def __cmp__(self, Other):
1533         if type(Other) == type(self):
1534             OtherKey = Other.Path
1535         else:
1536             OtherKey = str(Other)
1537
1538         SelfKey = self.Path
1539         if SelfKey == OtherKey:
1540             return 0
1541         elif SelfKey > OtherKey:
1542             return 1
1543         else:
1544             return -1
1545
1546     ## Override __hash__ function
1547     #
1548     # Use Path as key in hash table
1549     #
1550     # @retval string Key for hash table
1551     #
1552     def __hash__(self):
1553         return hash(self.Path)
1554
1555     def _GetFileKey(self):
1556         if self._Key == None:
1557             self._Key = self.Path.upper()   # + self.ToolChainFamily + self.TagName + self.ToolCode + self.Target
1558         return self._Key
1559
1560     def _GetTimeStamp(self):
1561         return os.stat(self.Path)[8]
1562
1563     def Validate(self, Type='', CaseSensitive=True):
1564         if GlobalData.gCaseInsensitive:
1565             CaseSensitive = False
1566         if Type and Type.lower() != self.Type:
1567             return FILE_TYPE_MISMATCH, '%s (expect %s but got %s)' % (self.File, Type, self.Type)
1568
1569         RealFile, RealRoot = RealPath2(self.File, self.Root, self.AlterRoot)
1570         if not RealRoot and not RealFile:
1571             RealFile = self.File
1572             if self.AlterRoot:
1573                 RealFile = os.path.join(self.AlterRoot, self.File)
1574             elif self.Root:
1575                 RealFile = os.path.join(self.Root, self.File)
1576             return FILE_NOT_FOUND, os.path.join(self.AlterRoot, RealFile)
1577
1578         ErrorCode = 0
1579         ErrorInfo = ''
1580         if RealRoot != self.Root or RealFile != self.File:
1581             if CaseSensitive and (RealFile != self.File or (RealRoot != self.Root and RealRoot != self.AlterRoot)):
1582                 ErrorCode = FILE_CASE_MISMATCH
1583                 ErrorInfo = self.File + '\n\t' + RealFile + " [in file system]"
1584
1585             self.SubDir, self.Name = os.path.split(RealFile)
1586             self.BaseName, self.Ext = os.path.splitext(self.Name)
1587             if self.SubDir:
1588                 self.Dir = os.path.join(RealRoot, self.SubDir)
1589             else:
1590                 self.Dir = RealRoot
1591             self.File = RealFile
1592             self.Root = RealRoot
1593             self.Path = os.path.join(RealRoot, RealFile)
1594         return ErrorCode, ErrorInfo
1595
1596     Key = property(_GetFileKey)
1597     TimeStamp = property(_GetTimeStamp)
1598
1599 ## Parse PE image to get the required PE informaion.
1600 #
1601 class PeImageClass():
1602     ## Constructor
1603     #
1604     #   @param  File FilePath of PeImage
1605     #
1606     def __init__(self, PeFile):
1607         self.FileName   = PeFile
1608         self.IsValid    = False
1609         self.Size       = 0
1610         self.EntryPoint = 0
1611         self.SectionAlignment  = 0
1612         self.SectionHeaderList = []
1613         self.ErrorInfo = ''
1614         try:
1615             PeObject = open(PeFile, 'rb')
1616         except:
1617             self.ErrorInfo = self.FileName + ' can not be found\n'
1618             return
1619         # Read DOS header
1620         ByteArray = array.array('B')
1621         ByteArray.fromfile(PeObject, 0x3E)
1622         ByteList = ByteArray.tolist()
1623         # DOS signature should be 'MZ'
1624         if self._ByteListToStr (ByteList[0x0:0x2]) != 'MZ':
1625             self.ErrorInfo = self.FileName + ' has no valid DOS signature MZ'
1626             return
1627
1628         # Read 4 byte PE Signature
1629         PeOffset = self._ByteListToInt(ByteList[0x3C:0x3E])
1630         PeObject.seek(PeOffset)
1631         ByteArray = array.array('B')
1632         ByteArray.fromfile(PeObject, 4)
1633         # PE signature should be 'PE\0\0'
1634         if ByteArray.tostring() != 'PE\0\0':
1635             self.ErrorInfo = self.FileName + ' has no valid PE signature PE00'
1636             return
1637
1638         # Read PE file header
1639         ByteArray = array.array('B')
1640         ByteArray.fromfile(PeObject, 0x14)
1641         ByteList = ByteArray.tolist()
1642         SecNumber = self._ByteListToInt(ByteList[0x2:0x4])
1643         if SecNumber == 0:
1644             self.ErrorInfo = self.FileName + ' has no section header'
1645             return
1646
1647         # Read PE optional header
1648         OptionalHeaderSize = self._ByteListToInt(ByteArray[0x10:0x12])
1649         ByteArray = array.array('B')
1650         ByteArray.fromfile(PeObject, OptionalHeaderSize)
1651         ByteList = ByteArray.tolist()
1652         self.EntryPoint       = self._ByteListToInt(ByteList[0x10:0x14])
1653         self.SectionAlignment = self._ByteListToInt(ByteList[0x20:0x24])
1654         self.Size             = self._ByteListToInt(ByteList[0x38:0x3C])
1655
1656         # Read each Section Header
1657         for Index in range(SecNumber):
1658             ByteArray = array.array('B')
1659             ByteArray.fromfile(PeObject, 0x28)
1660             ByteList = ByteArray.tolist()
1661             SecName  = self._ByteListToStr(ByteList[0:8])
1662             SecVirtualSize = self._ByteListToInt(ByteList[8:12])
1663             SecRawAddress  = self._ByteListToInt(ByteList[20:24])
1664             SecVirtualAddress = self._ByteListToInt(ByteList[12:16])
1665             self.SectionHeaderList.append((SecName, SecVirtualAddress, SecRawAddress, SecVirtualSize))
1666         self.IsValid = True
1667         PeObject.close()
1668
1669     def _ByteListToStr(self, ByteList):
1670         String = ''
1671         for index in range(len(ByteList)):
1672             if ByteList[index] == 0:
1673                 break
1674             String += chr(ByteList[index])
1675         return String
1676
1677     def _ByteListToInt(self, ByteList):
1678         Value = 0
1679         for index in range(len(ByteList) - 1, -1, -1):
1680             Value = (Value << 8) | int(ByteList[index])
1681         return Value
1682
1683 ##
1684 #
1685 # This acts like the main() function for the script, unless it is 'import'ed into another
1686 # script.
1687 #
1688 if __name__ == '__main__':
1689     pass
1690