BaseTools/Source/Python/Common/Misc.py

   1 ## @file
   2 # Common routines used by all tools
   3 #
   4 # Copyright (c) 2007 - 2010, Intel Corporation. All rights reserved.<BR>
   5 # This program and the accompanying materials
   6 # are licensed and made available under the terms and conditions of the BSD License
   7 # which accompanies this distribution.  The full text of the license may be found at
   8 # http://opensource.org/licenses/bsd-license.php
   9 #
  10 # THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS,
  11 # WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED.
  12 #
  13
  14 ##
  15 # Import Modules
  16 #
  17 import os
  18 import sys
  19 import string
  20 import thread
  21 import threading
  22 import time
  23 import re
  24 import cPickle
  25 import array
  26 from UserDict import IterableUserDict
  27 from UserList import UserList
  28
  29 from Common import EdkLogger as EdkLogger
  30 from Common import GlobalData as GlobalData
  31 from DataType import *
  32 from BuildToolError import *
  33 from CommonDataClass.DataClass import *
  34 from Parsing import GetSplitValueList
  35
  36 ## Regular expression used to find out place holders in string template
  37 gPlaceholderPattern = re.compile("\$\{([^$()\s]+)\}", re.MULTILINE|re.UNICODE)
  38
  39 ## Dictionary used to store file time stamp for quick re-access
  40 gFileTimeStampCache = {}    # {file path : file time stamp}
  41
  42 ## Dictionary used to store dependencies of files
  43 gDependencyDatabase = {}    # arch : {file path : [dependent files list]}
  44
  45 ## callback routine for processing variable option
  46 #
  47 # This function can be used to process variable number of option values. The
  48 # typical usage of it is specify architecure list on command line.
  49 # (e.g. <tool> -a IA32 X64 IPF)
  50 #
  51 # @param  Option        Standard callback function parameter
  52 # @param  OptionString  Standard callback function parameter
  53 # @param  Value         Standard callback function parameter
  54 # @param  Parser        Standard callback function parameter
  55 #
  56 # @retval
  57 #
  58 def ProcessVariableArgument(Option, OptionString, Value, Parser):
  59     assert Value is None
  60     Value = []
  61     RawArgs = Parser.rargs
  62     while RawArgs:
  63         Arg = RawArgs[0]
  64         if (Arg[:2] == "--" and len(Arg) > 2) or \
  65            (Arg[:1] == "-" and len(Arg) > 1 and Arg[1] != "-"):
  66             break
  67         Value.append(Arg)
  68         del RawArgs[0]
  69     setattr(Parser.values, Option.dest, Value)
  70
  71 ## Convert GUID string in xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx style to C structure style
  72 #
  73 #   @param      Guid    The GUID string
  74 #
  75 #   @retval     string  The GUID string in C structure style
  76 #
  77 def GuidStringToGuidStructureString(Guid):
  78     GuidList = Guid.split('-')
  79     Result = '{'
  80     for Index in range(0,3,1):
  81         Result = Result + '0x' + GuidList[Index] + ', '
  82     Result = Result + '{0x' + GuidList[3][0:2] + ', 0x' + GuidList[3][2:4]
  83     for Index in range(0,12,2):
  84         Result = Result + ', 0x' + GuidList[4][Index:Index+2]
  85     Result += '}}'
  86     return Result
  87
  88 ## Convert GUID structure in byte array to xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
  89 #
  90 #   @param      GuidValue   The GUID value in byte array
  91 #
  92 #   @retval     string      The GUID value in xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx format
  93 #
  94 def GuidStructureByteArrayToGuidString(GuidValue):
  95     guidValueString = GuidValue.lower().replace("{", "").replace("}", "").replace(" ", "").replace(";", "")
  96     guidValueList = guidValueString.split(",")
  97     if len(guidValueList) != 16:
  98         return ''
  99         #EdkLogger.error(None, None, "Invalid GUID value string %s" % GuidValue)
 100     try:
 101         return "%02x%02x%02x%02x-%02x%02x-%02x%02x-%02x%02x-%02x%02x%02x%02x%02x%02x" % (
 102                 int(guidValueList[3], 16),
 103                 int(guidValueList[2], 16),
 104                 int(guidValueList[1], 16),
 105                 int(guidValueList[0], 16),
 106                 int(guidValueList[5], 16),
 107                 int(guidValueList[4], 16),
 108                 int(guidValueList[7], 16),
 109                 int(guidValueList[6], 16),
 110                 int(guidValueList[8], 16),
 111                 int(guidValueList[9], 16),
 112                 int(guidValueList[10], 16),
 113                 int(guidValueList[11], 16),
 114                 int(guidValueList[12], 16),
 115                 int(guidValueList[13], 16),
 116                 int(guidValueList[14], 16),
 117                 int(guidValueList[15], 16)
 118                 )
 119     except:
 120         return ''
 121
 122 ## Convert GUID string in C structure style to xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
 123 #
 124 #   @param      GuidValue   The GUID value in C structure format
 125 #
 126 #   @retval     string      The GUID value in xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx format
 127 #
 128 def GuidStructureStringToGuidString(GuidValue):
 129     guidValueString = GuidValue.lower().replace("{", "").replace("}", "").replace(" ", "").replace(";", "")
 130     guidValueList = guidValueString.split(",")
 131     if len(guidValueList) != 11:
 132         return ''
 133         #EdkLogger.error(None, None, "Invalid GUID value string %s" % GuidValue)
 134     try:
 135         return "%08x-%04x-%04x-%02x%02x-%02x%02x%02x%02x%02x%02x" % (
 136                 int(guidValueList[0], 16),
 137                 int(guidValueList[1], 16),
 138                 int(guidValueList[2], 16),
 139                 int(guidValueList[3], 16),
 140                 int(guidValueList[4], 16),
 141                 int(guidValueList[5], 16),
 142                 int(guidValueList[6], 16),
 143                 int(guidValueList[7], 16),
 144                 int(guidValueList[8], 16),
 145                 int(guidValueList[9], 16),
 146                 int(guidValueList[10], 16)
 147                 )
 148     except:
 149         return ''
 150
 151 ## Convert GUID string in C structure style to xxxxxxxx_xxxx_xxxx_xxxx_xxxxxxxxxxxx
 152 #
 153 #   @param      GuidValue   The GUID value in C structure format
 154 #
 155 #   @retval     string      The GUID value in xxxxxxxx_xxxx_xxxx_xxxx_xxxxxxxxxxxx format
 156 #
 157 def GuidStructureStringToGuidValueName(GuidValue):
 158     guidValueString = GuidValue.lower().replace("{", "").replace("}", "").replace(" ", "")
 159     guidValueList = guidValueString.split(",")
 160     if len(guidValueList) != 11:
 161         EdkLogger.error(None, FORMAT_INVALID, "Invalid GUID value string [%s]" % GuidValue)
 162     return "%08x_%04x_%04x_%02x%02x_%02x%02x%02x%02x%02x%02x" % (
 163             int(guidValueList[0], 16),
 164             int(guidValueList[1], 16),
 165             int(guidValueList[2], 16),
 166             int(guidValueList[3], 16),
 167             int(guidValueList[4], 16),
 168             int(guidValueList[5], 16),
 169             int(guidValueList[6], 16),
 170             int(guidValueList[7], 16),
 171             int(guidValueList[8], 16),
 172             int(guidValueList[9], 16),
 173             int(guidValueList[10], 16)
 174             )
 175
 176 ## Create directories
 177 #
 178 #   @param      Directory   The directory name
 179 #
 180 def CreateDirectory(Directory):
 181     if Directory == None or Directory.strip() == "":
 182         return True
 183     try:
 184         if not os.access(Directory, os.F_OK):
 185             os.makedirs(Directory)
 186     except:
 187         return False
 188     return True
 189
 190 ## Remove directories, including files and sub-directories in it
 191 #
 192 #   @param      Directory   The directory name
 193 #
 194 def RemoveDirectory(Directory, Recursively=False):
 195     if Directory == None or Directory.strip() == "" or not os.path.exists(Directory):
 196         return
 197     if Recursively:
 198         CurrentDirectory = os.getcwd()
 199         os.chdir(Directory)
 200         for File in os.listdir("."):
 201             if os.path.isdir(File):
 202                 RemoveDirectory(File, Recursively)
 203             else:
 204                 os.remove(File)
 205         os.chdir(CurrentDirectory)
 206     os.rmdir(Directory)
 207
 208 ## Check if given file is changed or not
 209 #
 210 #  This method is used to check if a file is changed or not between two build
 211 #  actions. It makes use a cache to store files timestamp.
 212 #
 213 #   @param      File    The path of file
 214 #
 215 #   @retval     True    If the given file is changed, doesn't exist, or can't be
 216 #                       found in timestamp cache
 217 #   @retval     False   If the given file is changed
 218 #
 219 def IsChanged(File):
 220     if not os.path.exists(File):
 221         return True
 222
 223     FileState = os.stat(File)
 224     TimeStamp = FileState[-2]
 225
 226     if File in gFileTimeStampCache and TimeStamp == gFileTimeStampCache[File]:
 227         FileChanged = False
 228     else:
 229         FileChanged = True
 230         gFileTimeStampCache[File] = TimeStamp
 231
 232     return FileChanged
 233
 234 ## Store content in file
 235 #
 236 #  This method is used to save file only when its content is changed. This is
 237 #  quite useful for "make" system to decide what will be re-built and what won't.
 238 #
 239 #   @param      File            The path of file
 240 #   @param      Content         The new content of the file
 241 #   @param      IsBinaryFile    The flag indicating if the file is binary file or not
 242 #
 243 #   @retval     True            If the file content is changed and the file is renewed
 244 #   @retval     False           If the file content is the same
 245 #
 246 def SaveFileOnChange(File, Content, IsBinaryFile=True):
 247     if not IsBinaryFile:
 248         Content = Content.replace("\n", os.linesep)
 249
 250     if os.path.exists(File):
 251         try:
 252             if Content == open(File, "rb").read():
 253                 return False
 254         except:
 255             EdkLogger.error(None, FILE_OPEN_FAILURE, ExtraData=File)
 256
 257     DirName = os.path.dirname(File)
 258     if not CreateDirectory(DirName):
 259         EdkLogger.error(None, FILE_CREATE_FAILURE, "Could not create directory %s" % DirName)
 260     else:
 261         if DirName == '':
 262             DirName = os.getcwd()
 263         if not os.access(DirName, os.W_OK):
 264             EdkLogger.error(None, PERMISSION_FAILURE, "Do not have write permission on directory %s" % DirName)
 265
 266     try:
 267         if GlobalData.gIsWindows:
 268             try:
 269                 from PyUtility import SaveFileToDisk
 270                 if not SaveFileToDisk(File, Content):
 271                     EdkLogger.error(None, FILE_CREATE_FAILURE, ExtraData=File)
 272             except:
 273                 Fd = open(File, "wb")
 274                 Fd.write(Content)
 275                 Fd.close()
 276         else:
 277             Fd = open(File, "wb")
 278             Fd.write(Content)
 279             Fd.close()
 280     except IOError, X:
 281         EdkLogger.error(None, FILE_CREATE_FAILURE, ExtraData='IOError %s'%X)
 282
 283     return True
 284
 285 ## Make a Python object persistent on file system
 286 #
 287 #   @param      Data    The object to be stored in file
 288 #   @param      File    The path of file to store the object
 289 #
 290 def DataDump(Data, File):
 291     Fd = None
 292     try:
 293         Fd = open(File, 'wb')
 294         cPickle.dump(Data, Fd, cPickle.HIGHEST_PROTOCOL)
 295     except:
 296         EdkLogger.error("", FILE_OPEN_FAILURE, ExtraData=File, RaiseError=False)
 297     finally:
 298         if Fd != None:
 299             Fd.close()
 300
 301 ## Restore a Python object from a file
 302 #
 303 #   @param      File    The path of file stored the object
 304 #
 305 #   @retval     object  A python object
 306 #   @retval     None    If failure in file operation
 307 #
 308 def DataRestore(File):
 309     Data = None
 310     Fd = None
 311     try:
 312         Fd = open(File, 'rb')
 313         Data = cPickle.load(Fd)
 314     except Exception, e:
 315         EdkLogger.verbose("Failed to load [%s]\n\t%s" % (File, str(e)))
 316         Data = None
 317     finally:
 318         if Fd != None:
 319             Fd.close()
 320     return Data
 321
 322 ## Retrieve and cache the real path name in file system
 323 #
 324 #   @param      Root    The root directory of path relative to
 325 #
 326 #   @retval     str     The path string if the path exists
 327 #   @retval     None    If path doesn't exist
 328 #
 329 class DirCache:
 330     _CACHE_ = set()
 331     _UPPER_CACHE_ = {}
 332
 333     def __init__(self, Root):
 334         self._Root = Root
 335         for F in os.listdir(Root):
 336             self._CACHE_.add(F)
 337             self._UPPER_CACHE_[F.upper()] = F
 338
 339     # =[] operator
 340     def __getitem__(self, Path):
 341         Path = Path[len(os.path.commonprefix([Path, self._Root])):]
 342         if not Path:
 343             return self._Root
 344         if Path and Path[0] == os.path.sep:
 345             Path = Path[1:]
 346         if Path in self._CACHE_:
 347             return os.path.join(self._Root, Path)
 348         UpperPath = Path.upper()
 349         if UpperPath in self._UPPER_CACHE_:
 350             return os.path.join(self._Root, self._UPPER_CACHE_[UpperPath])
 351
 352         IndexList = []
 353         LastSepIndex = -1
 354         SepIndex = Path.find(os.path.sep)
 355         while SepIndex > -1:
 356             Parent = UpperPath[:SepIndex]
 357             if Parent not in self._UPPER_CACHE_:
 358                 break
 359             LastSepIndex = SepIndex
 360             SepIndex = Path.find(os.path.sep, LastSepIndex + 1)
 361
 362         if LastSepIndex == -1:
 363             return None
 364
 365         Cwd = os.getcwd()
 366         os.chdir(self._Root)
 367         SepIndex = LastSepIndex
 368         while SepIndex > -1:
 369             Parent = Path[:SepIndex]
 370             ParentKey = UpperPath[:SepIndex]
 371             if ParentKey not in self._UPPER_CACHE_:
 372                 os.chdir(Cwd)
 373                 return None
 374
 375             if Parent in self._CACHE_:
 376                 ParentDir = Parent
 377             else:
 378                 ParentDir = self._UPPER_CACHE_[ParentKey]
 379             for F in os.listdir(ParentDir):
 380                 Dir = os.path.join(ParentDir, F)
 381                 self._CACHE_.add(Dir)
 382                 self._UPPER_CACHE_[Dir.upper()] = Dir
 383
 384             SepIndex = Path.find(os.path.sep, SepIndex + 1)
 385
 386         os.chdir(Cwd)
 387         if Path in self._CACHE_:
 388             return os.path.join(self._Root, Path)
 389         elif UpperPath in self._UPPER_CACHE_:
 390             return os.path.join(self._Root, self._UPPER_CACHE_[UpperPath])
 391         return None
 392
 393 ## Get all files of a directory
 394 #
 395 # @param Root:       Root dir
 396 # @param SkipList :  The files need be skipped
 397 #
 398 # @retval  A list of all files
 399 #
 400 def GetFiles(Root, SkipList=None, FullPath = True):
 401     OriPath = Root
 402     FileList = []
 403     for Root, Dirs, Files in os.walk(Root):
 404         if SkipList:
 405             for Item in SkipList:
 406                 if Item in Dirs:
 407                     Dirs.remove(Item)
 408
 409         for File in Files:
 410             File = os.path.normpath(os.path.join(Root, File))
 411             if not FullPath:
 412                 File = File[len(OriPath) + 1:]
 413             FileList.append(File)
 414
 415     return FileList
 416
 417 ## Check if gvien file exists or not
 418 #
 419 #   @param      File    File name or path to be checked
 420 #   @param      Dir     The directory the file is relative to
 421 #
 422 #   @retval     True    if file exists
 423 #   @retval     False   if file doesn't exists
 424 #
 425 def ValidFile(File, Ext=None):
 426     if Ext != None:
 427         Dummy, FileExt = os.path.splitext(File)
 428         if FileExt.lower() != Ext.lower():
 429             return False
 430     if not os.path.exists(File):
 431         return False
 432     return True
 433
 434 def RealPath(File, Dir='', OverrideDir=''):
 435     NewFile = os.path.normpath(os.path.join(Dir, File))
 436     NewFile = GlobalData.gAllFiles[NewFile]
 437     if not NewFile and OverrideDir:
 438         NewFile = os.path.normpath(os.path.join(OverrideDir, File))
 439         NewFile = GlobalData.gAllFiles[NewFile]
 440     return NewFile
 441
 442 def RealPath2(File, Dir='', OverrideDir=''):
 443     if OverrideDir:
 444         NewFile = GlobalData.gAllFiles[os.path.normpath(os.path.join(OverrideDir, File))]
 445         if NewFile:
 446             if OverrideDir[-1] == os.path.sep:
 447                 return NewFile[len(OverrideDir):], NewFile[0:len(OverrideDir)]
 448             else:
 449                 return NewFile[len(OverrideDir)+1:], NewFile[0:len(OverrideDir)]
 450     if GlobalData.gAllFiles:
 451         NewFile = GlobalData.gAllFiles[os.path.normpath(os.path.join(Dir, File))]
 452     else:
 453         NewFile = os.path.normpath(os.path.join(Dir, File))
 454     if NewFile:
 455         if Dir:
 456             if Dir[-1] == os.path.sep:
 457                 return NewFile[len(Dir):], NewFile[0:len(Dir)]
 458             else:
 459                 return NewFile[len(Dir)+1:], NewFile[0:len(Dir)]
 460         else:
 461             return NewFile, ''
 462
 463     return None, None
 464
 465 ## Check if gvien file exists or not
 466 #
 467 #
 468 def ValidFile2(AllFiles, File, Ext=None, Workspace='', EfiSource='', EdkSource='', Dir='.', OverrideDir=''):
 469     NewFile = File
 470     if Ext != None:
 471         Dummy, FileExt = os.path.splitext(File)
 472         if FileExt.lower() != Ext.lower():
 473             return False, File
 474
 475     # Replace the Edk macros
 476     if OverrideDir != '' and OverrideDir != None:
 477         if OverrideDir.find('$(EFI_SOURCE)') > -1:
 478             OverrideDir = OverrideDir.replace('$(EFI_SOURCE)', EfiSource)
 479         if OverrideDir.find('$(EDK_SOURCE)') > -1:
 480             OverrideDir = OverrideDir.replace('$(EDK_SOURCE)', EdkSource)
 481
 482     # Replace the default dir to current dir
 483     if Dir == '.':
 484         Dir = os.getcwd()
 485         Dir = Dir[len(Workspace)+1:]
 486
 487     # First check if File has Edk definition itself
 488     if File.find('$(EFI_SOURCE)') > -1 or File.find('$(EDK_SOURCE)') > -1:
 489         NewFile = File.replace('$(EFI_SOURCE)', EfiSource)
 490         NewFile = NewFile.replace('$(EDK_SOURCE)', EdkSource)
 491         NewFile = AllFiles[os.path.normpath(NewFile)]
 492         if NewFile != None:
 493             return True, NewFile
 494
 495     # Second check the path with override value
 496     if OverrideDir != '' and OverrideDir != None:
 497         NewFile = AllFiles[os.path.normpath(os.path.join(OverrideDir, File))]
 498         if NewFile != None:
 499             return True, NewFile
 500
 501     # Last check the path with normal definitions
 502     File = os.path.join(Dir, File)
 503     NewFile = AllFiles[os.path.normpath(File)]
 504     if NewFile != None:
 505         return True, NewFile
 506
 507     return False, File
 508
 509 ## Check if gvien file exists or not
 510 #
 511 #
 512 def ValidFile3(AllFiles, File, Workspace='', EfiSource='', EdkSource='', Dir='.', OverrideDir=''):
 513     # Replace the Edk macros
 514     if OverrideDir != '' and OverrideDir != None:
 515         if OverrideDir.find('$(EFI_SOURCE)') > -1:
 516             OverrideDir = OverrideDir.replace('$(EFI_SOURCE)', EfiSource)
 517         if OverrideDir.find('$(EDK_SOURCE)') > -1:
 518             OverrideDir = OverrideDir.replace('$(EDK_SOURCE)', EdkSource)
 519
 520     # Replace the default dir to current dir
 521     # Dir is current module dir related to workspace
 522     if Dir == '.':
 523         Dir = os.getcwd()
 524         Dir = Dir[len(Workspace)+1:]
 525
 526     NewFile = File
 527     RelaPath = AllFiles[os.path.normpath(Dir)]
 528     NewRelaPath = RelaPath
 529
 530     while(True):
 531         # First check if File has Edk definition itself
 532         if File.find('$(EFI_SOURCE)') > -1 or File.find('$(EDK_SOURCE)') > -1:
 533             File = File.replace('$(EFI_SOURCE)', EfiSource)
 534             File = File.replace('$(EDK_SOURCE)', EdkSource)
 535             NewFile = AllFiles[os.path.normpath(File)]
 536             if NewFile != None:
 537                 NewRelaPath = os.path.dirname(NewFile)
 538                 File = os.path.basename(NewFile)
 539                 #NewRelaPath = NewFile[:len(NewFile) - len(File.replace("..\\", '').replace("../", '')) - 1]
 540                 break
 541
 542         # Second check the path with override value
 543         if OverrideDir != '' and OverrideDir != None:
 544             NewFile = AllFiles[os.path.normpath(os.path.join(OverrideDir, File))]
 545             if NewFile != None:
 546                 #NewRelaPath = os.path.dirname(NewFile)
 547                 NewRelaPath = NewFile[:len(NewFile) - len(File.replace("..\\", '').replace("../", '')) - 1]
 548                 break
 549
 550         # Last check the path with normal definitions
 551         NewFile = AllFiles[os.path.normpath(os.path.join(Dir, File))]
 552         if NewFile != None:
 553             break
 554
 555         # No file found
 556         break
 557
 558     return NewRelaPath, RelaPath, File
 559
 560
 561 def GetRelPath(Path1, Path2):
 562     FileName = os.path.basename(Path2)
 563     L1 = os.path.normpath(Path1).split(os.path.normpath('/'))
 564     L2 = os.path.normpath(Path2).split(os.path.normpath('/'))
 565     for Index in range(0, len(L1)):
 566         if L1[Index] != L2[Index]:
 567             FileName = '../' * (len(L1) - Index)
 568             for Index2 in range(Index, len(L2)):
 569                 FileName = os.path.join(FileName, L2[Index2])
 570             break
 571     return os.path.normpath(FileName)
 572
 573
 574 ## Get GUID value from given packages
 575 #
 576 #   @param      CName           The CName of the GUID
 577 #   @param      PackageList     List of packages looking-up in
 578 #
 579 #   @retval     GuidValue   if the CName is found in any given package
 580 #   @retval     None        if the CName is not found in all given packages
 581 #
 582 def GuidValue(CName, PackageList):
 583     for P in PackageList:
 584         if CName in P.Guids:
 585             return P.Guids[CName]
 586     return None
 587
 588 ## Get Protocol value from given packages
 589 #
 590 #   @param      CName           The CName of the GUID
 591 #   @param      PackageList     List of packages looking-up in
 592 #
 593 #   @retval     GuidValue   if the CName is found in any given package
 594 #   @retval     None        if the CName is not found in all given packages
 595 #
 596 def ProtocolValue(CName, PackageList):
 597     for P in PackageList:
 598         if CName in P.Protocols:
 599             return P.Protocols[CName]
 600     return None
 601
 602 ## Get PPI value from given packages
 603 #
 604 #   @param      CName           The CName of the GUID
 605 #   @param      PackageList     List of packages looking-up in
 606 #
 607 #   @retval     GuidValue   if the CName is found in any given package
 608 #   @retval     None        if the CName is not found in all given packages
 609 #
 610 def PpiValue(CName, PackageList):
 611     for P in PackageList:
 612         if CName in P.Ppis:
 613             return P.Ppis[CName]
 614     return None
 615
 616 ## A string template class
 617 #
 618 #  This class implements a template for string replacement. A string template
 619 #  looks like following
 620 #
 621 #       ${BEGIN} other_string ${placeholder_name} other_string ${END}
 622 #
 623 #  The string between ${BEGIN} and ${END} will be repeated as many times as the
 624 #  length of "placeholder_name", which is a list passed through a dict. The
 625 #  "placeholder_name" is the key name of the dict. The ${BEGIN} and ${END} can
 626 #  be not used and, in this case, the "placeholder_name" must not a list and it
 627 #  will just be replaced once.
 628 #
 629 class TemplateString(object):
 630     _REPEAT_START_FLAG = "BEGIN"
 631     _REPEAT_END_FLAG = "END"
 632
 633     class Section(object):
 634         _LIST_TYPES = [type([]), type(set()), type((0,))]
 635
 636         def __init__(self, TemplateSection, PlaceHolderList):
 637             self._Template = TemplateSection
 638             self._PlaceHolderList = []
 639
 640             # Split the section into sub-sections according to the position of placeholders
 641             if PlaceHolderList:
 642                 self._SubSectionList = []
 643                 SubSectionStart = 0
 644                 #
 645                 # The placeholders passed in must be in the format of
 646                 #
 647                 #   PlaceHolderName, PlaceHolderStartPoint, PlaceHolderEndPoint
 648                 #
 649                 for PlaceHolder,Start,End in PlaceHolderList:
 650                     self._SubSectionList.append(TemplateSection[SubSectionStart:Start])
 651                     self._SubSectionList.append(TemplateSection[Start:End])
 652                     self._PlaceHolderList.append(PlaceHolder)
 653                     SubSectionStart = End
 654                 if SubSectionStart < len(TemplateSection):
 655                     self._SubSectionList.append(TemplateSection[SubSectionStart:])
 656             else:
 657                 self._SubSectionList = [TemplateSection]
 658
 659         def __str__(self):
 660             return self._Template + " : " + str(self._PlaceHolderList)
 661
 662         def Instantiate(self, PlaceHolderValues):
 663             RepeatTime = -1
 664             RepeatPlaceHolders = {}
 665             NonRepeatPlaceHolders = {}
 666
 667             for PlaceHolder in self._PlaceHolderList:
 668                 if PlaceHolder not in PlaceHolderValues:
 669                     continue
 670                 Value = PlaceHolderValues[PlaceHolder]
 671                 if type(Value) in self._LIST_TYPES:
 672                     if RepeatTime < 0:
 673                         RepeatTime = len(Value)
 674                     elif RepeatTime != len(Value):
 675                         EdkLogger.error(
 676                                     "TemplateString",
 677                                     PARAMETER_INVALID,
 678                                     "${%s} has different repeat time from others!" % PlaceHolder,
 679                                     ExtraData=str(self._Template)
 680                                     )
 681                     RepeatPlaceHolders["${%s}" % PlaceHolder] = Value
 682                 else:
 683                     NonRepeatPlaceHolders["${%s}" % PlaceHolder] = Value
 684
 685             if NonRepeatPlaceHolders:
 686                 StringList = []
 687                 for S in self._SubSectionList:
 688                     if S not in NonRepeatPlaceHolders:
 689                         StringList.append(S)
 690                     else:
 691                         StringList.append(str(NonRepeatPlaceHolders[S]))
 692             else:
 693                 StringList = self._SubSectionList
 694
 695             if RepeatPlaceHolders:
 696                 TempStringList = []
 697                 for Index in range(RepeatTime):
 698                     for S in StringList:
 699                         if S not in RepeatPlaceHolders:
 700                             TempStringList.append(S)
 701                         else:
 702                             TempStringList.append(str(RepeatPlaceHolders[S][Index]))
 703                 StringList = TempStringList
 704
 705             return "".join(StringList)
 706
 707     ## Constructor
 708     def __init__(self, Template=None):
 709         self.String = ''
 710         self.IsBinary = False
 711         self._Template = Template
 712         self._TemplateSectionList = self._Parse(Template)
 713
 714     ## str() operator
 715     #
 716     #   @retval     string  The string replaced
 717     #
 718     def __str__(self):
 719         return self.String
 720
 721     ## Split the template string into fragments per the ${BEGIN} and ${END} flags
 722     #
 723     #   @retval     list    A list of TemplateString.Section objects
 724     #
 725     def _Parse(self, Template):
 726         SectionStart = 0
 727         SearchFrom = 0
 728         MatchEnd = 0
 729         PlaceHolderList = []
 730         TemplateSectionList = []
 731         while Template:
 732             MatchObj = gPlaceholderPattern.search(Template, SearchFrom)
 733             if not MatchObj:
 734                 if MatchEnd <= len(Template):
 735                     TemplateSection = TemplateString.Section(Template[SectionStart:], PlaceHolderList)
 736                     TemplateSectionList.append(TemplateSection)
 737                 break
 738
 739             MatchString = MatchObj.group(1)
 740             MatchStart = MatchObj.start()
 741             MatchEnd = MatchObj.end()
 742
 743             if MatchString == self._REPEAT_START_FLAG:
 744                 if MatchStart > SectionStart:
 745                     TemplateSection = TemplateString.Section(Template[SectionStart:MatchStart], PlaceHolderList)
 746                     TemplateSectionList.append(TemplateSection)
 747                 SectionStart = MatchEnd
 748                 PlaceHolderList = []
 749             elif MatchString == self._REPEAT_END_FLAG:
 750                 TemplateSection = TemplateString.Section(Template[SectionStart:MatchStart], PlaceHolderList)
 751                 TemplateSectionList.append(TemplateSection)
 752                 SectionStart = MatchEnd
 753                 PlaceHolderList = []
 754             else:
 755                 PlaceHolderList.append((MatchString, MatchStart - SectionStart, MatchEnd - SectionStart))
 756             SearchFrom = MatchEnd
 757         return TemplateSectionList
 758
 759     ## Replace the string template with dictionary of placeholders and append it to previous one
 760     #
 761     #   @param      AppendString    The string template to append
 762     #   @param      Dictionary      The placeholder dictionaries
 763     #
 764     def Append(self, AppendString, Dictionary=None):
 765         if Dictionary:
 766             SectionList = self._Parse(AppendString)
 767             self.String += "".join([S.Instantiate(Dictionary) for S in SectionList])
 768         else:
 769             self.String += AppendString
 770
 771     ## Replace the string template with dictionary of placeholders
 772     #
 773     #   @param      Dictionary      The placeholder dictionaries
 774     #
 775     #   @retval     str             The string replaced with placeholder values
 776     #
 777     def Replace(self, Dictionary=None):
 778         return "".join([S.Instantiate(Dictionary) for S in self._TemplateSectionList])
 779
 780 ## Progress indicator class
 781 #
 782 #  This class makes use of thread to print progress on console.
 783 #
 784 class Progressor:
 785     # for avoiding deadloop
 786     _StopFlag = None
 787     _ProgressThread = None
 788     _CheckInterval = 0.25
 789
 790     ## Constructor
 791     #
 792     #   @param      OpenMessage     The string printed before progress charaters
 793     #   @param      CloseMessage    The string printed after progress charaters
 794     #   @param      ProgressChar    The charater used to indicate the progress
 795     #   @param      Interval        The interval in seconds between two progress charaters
 796     #
 797     def __init__(self, OpenMessage="", CloseMessage="", ProgressChar='.', Interval=1.0):
 798         self.PromptMessage = OpenMessage
 799         self.CodaMessage = CloseMessage
 800         self.ProgressChar = ProgressChar
 801         self.Interval = Interval
 802         if Progressor._StopFlag == None:
 803             Progressor._StopFlag = threading.Event()
 804
 805     ## Start to print progress charater
 806     #
 807     #   @param      OpenMessage     The string printed before progress charaters
 808     #
 809     def Start(self, OpenMessage=None):
 810         if OpenMessage != None:
 811             self.PromptMessage = OpenMessage
 812         Progressor._StopFlag.clear()
 813         if Progressor._ProgressThread == None:
 814             Progressor._ProgressThread = threading.Thread(target=self._ProgressThreadEntry)
 815             Progressor._ProgressThread.setDaemon(False)
 816             Progressor._ProgressThread.start()
 817
 818     ## Stop printing progress charater
 819     #
 820     #   @param      CloseMessage    The string printed after progress charaters
 821     #
 822     def Stop(self, CloseMessage=None):
 823         OriginalCodaMessage = self.CodaMessage
 824         if CloseMessage != None:
 825             self.CodaMessage = CloseMessage
 826         self.Abort()
 827         self.CodaMessage = OriginalCodaMessage
 828
 829     ## Thread entry method
 830     def _ProgressThreadEntry(self):
 831         sys.stdout.write(self.PromptMessage + " ")
 832         sys.stdout.flush()
 833         TimeUp = 0.0
 834         while not Progressor._StopFlag.isSet():
 835             if TimeUp <= 0.0:
 836                 sys.stdout.write(self.ProgressChar)
 837                 sys.stdout.flush()
 838                 TimeUp = self.Interval
 839             time.sleep(self._CheckInterval)
 840             TimeUp -= self._CheckInterval
 841         sys.stdout.write(" " + self.CodaMessage + "\n")
 842         sys.stdout.flush()
 843
 844     ## Abort the progress display
 845     @staticmethod
 846     def Abort():
 847         if Progressor._StopFlag != None:
 848             Progressor._StopFlag.set()
 849         if Progressor._ProgressThread != None:
 850             Progressor._ProgressThread.join()
 851             Progressor._ProgressThread = None
 852
 853 ## A dict which can access its keys and/or values orderly
 854 #
 855 #  The class implements a new kind of dict which its keys or values can be
 856 #  accessed in the order they are added into the dict. It guarantees the order
 857 #  by making use of an internal list to keep a copy of keys.
 858 #
 859 class sdict(IterableUserDict):
 860     ## Constructor
 861     def __init__(self):
 862         IterableUserDict.__init__(self)
 863         self._key_list = []
 864
 865     ## [] operator
 866     def __setitem__(self, key, value):
 867         if key not in self._key_list:
 868             self._key_list.append(key)
 869         IterableUserDict.__setitem__(self, key, value)
 870
 871     ## del operator
 872     def __delitem__(self, key):
 873         self._key_list.remove(key)
 874         IterableUserDict.__delitem__(self, key)
 875
 876     ## used in "for k in dict" loop to ensure the correct order
 877     def __iter__(self):
 878         return self.iterkeys()
 879
 880     ## len() support
 881     def __len__(self):
 882         return len(self._key_list)
 883
 884     ## "in" test support
 885     def __contains__(self, key):
 886         return key in self._key_list
 887
 888     ## indexof support
 889     def index(self, key):
 890         return self._key_list.index(key)
 891
 892     ## insert support
 893     def insert(self, key, newkey, newvalue, order):
 894         index = self._key_list.index(key)
 895         if order == 'BEFORE':
 896             self._key_list.insert(index, newkey)
 897             IterableUserDict.__setitem__(self, newkey, newvalue)
 898         elif order == 'AFTER':
 899             self._key_list.insert(index + 1, newkey)
 900             IterableUserDict.__setitem__(self, newkey, newvalue)
 901
 902     ## append support
 903     def append(self, sdict):
 904         for key in sdict:
 905             if key not in self._key_list:
 906                 self._key_list.append(key)
 907             IterableUserDict.__setitem__(self, key, sdict[key])
 908
 909     def has_key(self, key):
 910         return key in self._key_list
 911
 912     ## Empty the dict
 913     def clear(self):
 914         self._key_list = []
 915         IterableUserDict.clear(self)
 916
 917     ## Return a copy of keys
 918     def keys(self):
 919         keys = []
 920         for key in self._key_list:
 921             keys.append(key)
 922         return keys
 923
 924     ## Return a copy of values
 925     def values(self):
 926         values = []
 927         for key in self._key_list:
 928             values.append(self[key])
 929         return values
 930
 931     ## Return a copy of (key, value) list
 932     def items(self):
 933         items = []
 934         for key in self._key_list:
 935             items.append((key, self[key]))
 936         return items
 937
 938     ## Iteration support
 939     def iteritems(self):
 940         return iter(self.items())
 941
 942     ## Keys interation support
 943     def iterkeys(self):
 944         return iter(self.keys())
 945
 946     ## Values interation support
 947     def itervalues(self):
 948         return iter(self.values())
 949
 950     ## Return value related to a key, and remove the (key, value) from the dict
 951     def pop(self, key, *dv):
 952         value = None
 953         if key in self._key_list:
 954             value = self[key]
 955             self.__delitem__(key)
 956         elif len(dv) != 0 :
 957             value = kv[0]
 958         return value
 959
 960     ## Return (key, value) pair, and remove the (key, value) from the dict
 961     def popitem(self):
 962         key = self._key_list[-1]
 963         value = self[key]
 964         self.__delitem__(key)
 965         return key, value
 966
 967     def update(self, dict=None, **kwargs):
 968         if dict != None:
 969             for k, v in dict.items():
 970                 self[k] = v
 971         if len(kwargs):
 972             for k, v in kwargs.items():
 973                 self[k] = v
 974
 975 ## Dictionary with restricted keys
 976 #
 977 class rdict(dict):
 978     ## Constructor
 979     def __init__(self, KeyList):
 980         for Key in KeyList:
 981             dict.__setitem__(self, Key, "")
 982
 983     ## []= operator
 984     def __setitem__(self, key, value):
 985         if key not in self:
 986             EdkLogger.error("RestrictedDict", ATTRIBUTE_SET_FAILURE, "Key [%s] is not allowed" % key,
 987                             ExtraData=", ".join(dict.keys(self)))
 988         dict.__setitem__(self, key, value)
 989
 990     ## =[] operator
 991     def __getitem__(self, key):
 992         if key not in self:
 993             return ""
 994         return dict.__getitem__(self, key)
 995
 996     ## del operator
 997     def __delitem__(self, key):
 998         EdkLogger.error("RestrictedDict", ATTRIBUTE_ACCESS_DENIED, ExtraData="del")
 999
1000     ## Empty the dict
1001     def clear(self):
1002         for Key in self:
1003             self.__setitem__(Key, "")
1004
1005     ## Return value related to a key, and remove the (key, value) from the dict
1006     def pop(self, key, *dv):
1007         EdkLogger.error("RestrictedDict", ATTRIBUTE_ACCESS_DENIED, ExtraData="pop")
1008
1009     ## Return (key, value) pair, and remove the (key, value) from the dict
1010     def popitem(self):
1011         EdkLogger.error("RestrictedDict", ATTRIBUTE_ACCESS_DENIED, ExtraData="popitem")
1012
1013 ## Dictionary using prioritized list as key
1014 #
1015 class tdict:
1016     _ListType = type([])
1017     _TupleType = type(())
1018     _Wildcard = 'COMMON'
1019     _ValidWildcardList = ['COMMON', 'DEFAULT', 'ALL', '*', 'PLATFORM']
1020
1021     def __init__(self, _Single_=False, _Level_=2):
1022         self._Level_ = _Level_
1023         self.data = {}
1024         self._Single_ = _Single_
1025
1026     # =[] operator
1027     def __getitem__(self, key):
1028         KeyType = type(key)
1029         RestKeys = None
1030         if KeyType == self._ListType or KeyType == self._TupleType:
1031             FirstKey = key[0]
1032             if len(key) > 1:
1033                 RestKeys = key[1:]
1034             elif self._Level_ > 1:
1035                 RestKeys = [self._Wildcard for i in range(0, self._Level_-1)]
1036         else:
1037             FirstKey = key
1038             if self._Level_ > 1:
1039                 RestKeys = [self._Wildcard for i in range(0, self._Level_-1)]
1040
1041         if FirstKey == None or str(FirstKey).upper() in self._ValidWildcardList:
1042             FirstKey = self._Wildcard
1043
1044         if self._Single_:
1045             return self._GetSingleValue(FirstKey, RestKeys)
1046         else:
1047             return self._GetAllValues(FirstKey, RestKeys)
1048
1049     def _GetSingleValue(self, FirstKey, RestKeys):
1050         Value = None
1051         #print "%s-%s" % (FirstKey, self._Level_) ,
1052         if self._Level_ > 1:
1053             if FirstKey == self._Wildcard:
1054                 if FirstKey in self.data:
1055                     Value = self.data[FirstKey][RestKeys]
1056                 if Value == None:
1057                     for Key in self.data:
1058                         Value = self.data[Key][RestKeys]
1059                         if Value != None: break
1060             else:
1061                 if FirstKey in self.data:
1062                     Value = self.data[FirstKey][RestKeys]
1063                 if Value == None and self._Wildcard in self.data:
1064                     #print "Value=None"
1065                     Value = self.data[self._Wildcard][RestKeys]
1066         else:
1067             if FirstKey == self._Wildcard:
1068                 if FirstKey in self.data:
1069                     Value = self.data[FirstKey]
1070                 if Value == None:
1071                     for Key in self.data:
1072                         Value = self.data[Key]
1073                         if Value != None: break
1074             else:
1075                 if FirstKey in self.data:
1076                     Value = self.data[FirstKey]
1077                 elif self._Wildcard in self.data:
1078                     Value = self.data[self._Wildcard]
1079         return Value
1080
1081     def _GetAllValues(self, FirstKey, RestKeys):
1082         Value = []
1083         if self._Level_ > 1:
1084             if FirstKey == self._Wildcard:
1085                 for Key in self.data:
1086                     Value += self.data[Key][RestKeys]
1087             else:
1088                 if FirstKey in self.data:
1089                     Value += self.data[FirstKey][RestKeys]
1090                 if self._Wildcard in self.data:
1091                     Value += self.data[self._Wildcard][RestKeys]
1092         else:
1093             if FirstKey == self._Wildcard:
1094                 for Key in self.data:
1095                     Value.append(self.data[Key])
1096             else:
1097                 if FirstKey in self.data:
1098                     Value.append(self.data[FirstKey])
1099                 if self._Wildcard in self.data:
1100                     Value.append(self.data[self._Wildcard])
1101         return Value
1102
1103     ## []= operator
1104     def __setitem__(self, key, value):
1105         KeyType = type(key)
1106         RestKeys = None
1107         if KeyType == self._ListType or KeyType == self._TupleType:
1108             FirstKey = key[0]
1109             if len(key) > 1:
1110                 RestKeys = key[1:]
1111             else:
1112                 RestKeys = [self._Wildcard for i in range(0, self._Level_-1)]
1113         else:
1114             FirstKey = key
1115             if self._Level_ > 1:
1116                 RestKeys = [self._Wildcard for i in range(0, self._Level_-1)]
1117
1118         if FirstKey in self._ValidWildcardList:
1119             FirstKey = self._Wildcard
1120
1121         if FirstKey not in self.data and self._Level_ > 0:
1122             self.data[FirstKey] = tdict(self._Single_, self._Level_ - 1)
1123
1124         if self._Level_ > 1:
1125             self.data[FirstKey][RestKeys] = value
1126         else:
1127             self.data[FirstKey] = value
1128
1129     def SetGreedyMode(self):
1130         self._Single_ = False
1131         if self._Level_ > 1:
1132             for Key in self.data:
1133                 self.data[Key].SetGreedyMode()
1134
1135     def SetSingleMode(self):
1136         self._Single_ = True
1137         if self._Level_ > 1:
1138             for Key in self.data:
1139                 self.data[Key].SetSingleMode()
1140
1141     def GetKeys(self, KeyIndex=0):
1142         assert KeyIndex >= 0
1143         if KeyIndex == 0:
1144             return set(self.data.keys())
1145         else:
1146             keys = set()
1147             for Key in self.data:
1148                 keys |= self.data[Key].GetKeys(KeyIndex - 1)
1149             return keys
1150
1151 ## Boolean chain list
1152 #
1153 class Blist(UserList):
1154     def __init__(self, initlist=None):
1155         UserList.__init__(self, initlist)
1156     def __setitem__(self, i, item):
1157         if item not in [True, False]:
1158             if item == 0:
1159                 item = False
1160             else:
1161                 item = True
1162         self.data[i] = item
1163     def _GetResult(self):
1164         Value = True
1165         for item in self.data:
1166             Value &= item
1167         return Value
1168     Result = property(_GetResult)
1169
1170 def ParseConsoleLog(Filename):
1171     Opr = open(os.path.normpath(Filename), 'r')
1172     Opw = open(os.path.normpath(Filename + '.New'), 'w+')
1173     for Line in Opr.readlines():
1174         if Line.find('.efi') > -1:
1175             Line = Line[Line.rfind(' ') : Line.rfind('.efi')].strip()
1176             Opw.write('%s\n' % Line)
1177
1178     Opr.close()
1179     Opw.close()
1180
1181 ## AnalyzeDscPcd
1182 #
1183 #  Analyze DSC PCD value, since there is no data type info in DSC
1184 #  This fuction is used to match functions (AnalyzePcdData, AnalyzeHiiPcdData, AnalyzeVpdPcdData) used for retrieving PCD value from database
1185 #  1. Feature flag: TokenSpace.PcdCName|PcdValue
1186 #  2. Fix and Patch:TokenSpace.PcdCName|PcdValue[|MaxSize]
1187 #  3. Dynamic default:
1188 #     TokenSpace.PcdCName|PcdValue[|VOID*[|MaxSize]]
1189 #     TokenSpace.PcdCName|PcdValue
1190 #  4. Dynamic VPD:
1191 #     TokenSpace.PcdCName|VpdOffset[|VpdValue]
1192 #     TokenSpace.PcdCName|VpdOffset[|MaxSize[|VpdValue]]
1193 #  5. Dynamic HII:
1194 #     TokenSpace.PcdCName|HiiString|VaiableGuid|VariableOffset[|HiiValue]
1195 #  PCD value needs to be located in such kind of string, and the PCD value might be an expression in which
1196 #    there might have "|" operator, also in string value.
1197 #
1198 #  @param Setting: String contain information described above with "TokenSpace.PcdCName|" stripped
1199 #  @param PcdType: PCD type: feature, fixed, dynamic default VPD HII
1200 #  @param DataType: The datum type of PCD: VOID*, UNIT, BOOL
1201 #  @retval:
1202 #    ValueList: A List contain fields described above
1203 #    IsValid:   True if conforming EBNF, otherwise False
1204 #    Index:     The index where PcdValue is in ValueList
1205 #
1206 def AnalyzeDscPcd(Setting, PcdType, DataType=''):
1207     Setting = Setting.strip()
1208     # There might be escaped quote in a string: \", \\\"
1209     Data = Setting.replace('\\\\', '//').replace('\\\"', '\\\'')
1210     # There might be '|' in string and in ( ... | ... ), replace it with '-'
1211     NewStr = ''
1212     InStr = False
1213     Pair = 0
1214     for ch in Data:
1215         if ch == '"':
1216             InStr = not InStr
1217         elif ch == '(' and not InStr:
1218             Pair += 1
1219         elif ch == ')' and not InStr:
1220             Pair -= 1
1221
1222         if (Pair > 0 or InStr) and ch == TAB_VALUE_SPLIT:
1223             NewStr += '-'
1224         else:
1225             NewStr += ch
1226     FieldList = []
1227     StartPos = 0
1228     while True:
1229         Pos = NewStr.find(TAB_VALUE_SPLIT, StartPos)
1230         if Pos < 0:
1231             FieldList.append(Setting[StartPos:].strip())
1232             break
1233         FieldList.append(Setting[StartPos:Pos].strip())
1234         StartPos = Pos + 1
1235
1236     IsValid = True
1237     if PcdType in (MODEL_PCD_FIXED_AT_BUILD, MODEL_PCD_PATCHABLE_IN_MODULE, MODEL_PCD_FEATURE_FLAG):
1238         Value = FieldList[0]
1239         Size = ''
1240         if len(FieldList) > 1:
1241             Size = FieldList[1]
1242         if DataType == 'VOID*':
1243             IsValid = (len(FieldList) <= 2)
1244         else:
1245             IsValid = (len(FieldList) <= 1)
1246         return [Value, '', Size], IsValid, 0
1247     elif PcdType in (MODEL_PCD_DYNAMIC_DEFAULT, MODEL_PCD_DYNAMIC_EX_DEFAULT):
1248         Value = FieldList[0]
1249         Size = Type = ''
1250         if len(FieldList) > 1:
1251             Type = FieldList[1]
1252         else:
1253             Type = DataType
1254         if len(FieldList) > 2:
1255             Size = FieldList[2]
1256         else:
1257             if Type == 'VOID*':
1258                 Size = str(len(Value))
1259         if DataType == 'VOID*':
1260             IsValid = (len(FieldList) <= 3)
1261         else:
1262             IsValid = (len(FieldList) <= 1)
1263         return [Value, Type, Size], IsValid, 0
1264     elif PcdType in (MODEL_PCD_DYNAMIC_VPD, MODEL_PCD_DYNAMIC_EX_VPD):
1265         VpdOffset = FieldList[0]
1266         Value = Size = ''
1267         if not DataType == 'VOID*':
1268             if len(FieldList) > 1:
1269                 Value = FieldList[1]
1270         else:
1271             if len(FieldList) > 1:
1272                 Size = FieldList[1]
1273             if len(FieldList) > 2:
1274                 Value = FieldList[2]
1275         if DataType == 'VOID*':
1276             IsValid = (len(FieldList) <= 3)
1277         else:
1278             IsValid = (len(FieldList) <= 2)
1279         return [VpdOffset, Size, Value], IsValid, 2
1280     elif PcdType in (MODEL_PCD_DYNAMIC_HII, MODEL_PCD_DYNAMIC_EX_HII):
1281         HiiString = FieldList[0]
1282         Guid = Offset = Value = ''
1283         if len(FieldList) > 1:
1284             Guid = FieldList[1]
1285         if len(FieldList) > 2:
1286             Offset = FieldList[2]
1287         if len(FieldList) > 3:
1288             Value = FieldList[3]
1289         IsValid = (3 <= len(FieldList) <= 4)
1290         return [HiiString, Guid, Offset, Value], IsValid, 3
1291     return [], False, 0
1292
1293 ## AnalyzePcdData
1294 #
1295 #  Analyze the pcd Value, Datum type and TokenNumber.
1296 #  Used to avoid split issue while the value string contain "|" character
1297 #
1298 #  @param[in] Setting:  A String contain value/datum type/token number information;
1299 #
1300 #  @retval   ValueList: A List contain value, datum type and toke number.
1301 #
1302 def AnalyzePcdData(Setting):
1303     ValueList = ['', '', '']
1304
1305     ValueRe  = re.compile(r'^\s*L?\".*\|.*\"')
1306     PtrValue = ValueRe.findall(Setting)
1307
1308     ValueUpdateFlag = False
1309
1310     if len(PtrValue) >= 1:
1311         Setting = re.sub(ValueRe, '', Setting)
1312         ValueUpdateFlag = True
1313
1314     TokenList = Setting.split(TAB_VALUE_SPLIT)
1315     ValueList[0:len(TokenList)] = TokenList
1316
1317     if ValueUpdateFlag:
1318         ValueList[0] = PtrValue[0]
1319
1320     return ValueList
1321
1322 ## AnalyzeHiiPcdData
1323 #
1324 #  Analyze the pcd Value, variable name, variable Guid and variable offset.
1325 #  Used to avoid split issue while the value string contain "|" character
1326 #
1327 #  @param[in] Setting:  A String contain VariableName, VariableGuid, VariableOffset, DefaultValue information;
1328 #
1329 #  @retval   ValueList: A List contaian VariableName, VariableGuid, VariableOffset, DefaultValue.
1330 #
1331 def AnalyzeHiiPcdData(Setting):
1332     ValueList = ['', '', '', '']
1333
1334     TokenList = GetSplitValueList(Setting)
1335     ValueList[0:len(TokenList)] = TokenList
1336
1337     return ValueList
1338
1339 ## AnalyzeVpdPcdData
1340 #
1341 #  Analyze the vpd pcd VpdOffset, MaxDatumSize and InitialValue.
1342 #  Used to avoid split issue while the value string contain "|" character
1343 #
1344 #  @param[in] Setting:  A String contain VpdOffset/MaxDatumSize/InitialValue information;
1345 #
1346 #  @retval   ValueList: A List contain VpdOffset, MaxDatumSize and InitialValue.
1347 #
1348 def AnalyzeVpdPcdData(Setting):
1349     ValueList = ['', '', '']
1350
1351     ValueRe  = re.compile(r'\s*L?\".*\|.*\"\s*$')
1352     PtrValue = ValueRe.findall(Setting)
1353
1354     ValueUpdateFlag = False
1355
1356     if len(PtrValue) >= 1:
1357         Setting = re.sub(ValueRe, '', Setting)
1358         ValueUpdateFlag = True
1359
1360     TokenList = Setting.split(TAB_VALUE_SPLIT)
1361     ValueList[0:len(TokenList)] = TokenList
1362
1363     if ValueUpdateFlag:
1364         ValueList[2] = PtrValue[0]
1365
1366     return ValueList
1367
1368 ## check format of PCD value against its the datum type
1369 #
1370 # For PCD value setting
1371 #
1372 def CheckPcdDatum(Type, Value):
1373     if Type == "VOID*":
1374         ValueRe  = re.compile(r'\s*L?\".*\"\s*$')
1375         if not (((Value.startswith('L"') or Value.startswith('"')) and Value.endswith('"'))
1376                 or (Value.startswith('{') and Value.endswith('}'))
1377                ):
1378             return False, "Invalid value [%s] of type [%s]; must be in the form of {...} for array"\
1379                           ", or \"...\" for string, or L\"...\" for unicode string" % (Value, Type)
1380         elif ValueRe.match(Value):
1381             # Check the chars in UnicodeString or CString is printable
1382             if Value.startswith("L"):
1383                 Value = Value[2:-1]
1384             else:
1385                 Value = Value[1:-1]
1386             Printset = set(string.printable)
1387             Printset.remove(TAB_PRINTCHAR_VT)
1388             Printset.add(TAB_PRINTCHAR_BS)
1389             Printset.add(TAB_PRINTCHAR_NUL)
1390             if not set(Value).issubset(Printset):
1391                 PrintList = list(Printset)
1392                 PrintList.sort()
1393                 return False, "Invalid PCD string value of type [%s]; must be printable chars %s." % (Type, PrintList)
1394     elif Type == 'BOOLEAN':
1395         if Value not in ['TRUE', 'True', 'true', '0x1', '0x01', '1', 'FALSE', 'False', 'false', '0x0', '0x00', '0']:
1396             return False, "Invalid value [%s] of type [%s]; must be one of TRUE, True, true, 0x1, 0x01, 1"\
1397                           ", FALSE, False, false, 0x0, 0x00, 0" % (Value, Type)
1398     elif Type in [TAB_UINT8, TAB_UINT16, TAB_UINT32, TAB_UINT64]:
1399         try:
1400             Value = long(Value, 0)
1401         except:
1402             return False, "Invalid value [%s] of type [%s];"\
1403                           " must be a hexadecimal, decimal or octal in C language format." % (Value, Type)
1404     else:
1405         return False, "Invalid type [%s]; must be one of VOID*, BOOLEAN, UINT8, UINT16, UINT32, UINT64." % (Type)
1406
1407     return True, ""
1408
1409 ## Split command line option string to list
1410 #
1411 # subprocess.Popen needs the args to be a sequence. Otherwise there's problem
1412 # in non-windows platform to launch command
1413 #
1414 def SplitOption(OptionString):
1415     OptionList = []
1416     LastChar = " "
1417     OptionStart = 0
1418     QuotationMark = ""
1419     for Index in range(0, len(OptionString)):
1420         CurrentChar = OptionString[Index]
1421         if CurrentChar in ['"', "'"]:
1422             if QuotationMark == CurrentChar:
1423                 QuotationMark = ""
1424             elif QuotationMark == "":
1425                 QuotationMark = CurrentChar
1426             continue
1427         elif QuotationMark:
1428             continue
1429
1430         if CurrentChar in ["/", "-"] and LastChar in [" ", "\t", "\r", "\n"]:
1431             if Index > OptionStart:
1432                 OptionList.append(OptionString[OptionStart:Index-1])
1433             OptionStart = Index
1434         LastChar = CurrentChar
1435     OptionList.append(OptionString[OptionStart:])
1436     return OptionList
1437
1438 def CommonPath(PathList):
1439     P1 = min(PathList).split(os.path.sep)
1440     P2 = max(PathList).split(os.path.sep)
1441     for Index in xrange(min(len(P1), len(P2))):
1442         if P1[Index] != P2[Index]:
1443             return os.path.sep.join(P1[:Index])
1444     return os.path.sep.join(P1)
1445
1446 class PathClass(object):
1447     def __init__(self, File='', Root='', AlterRoot='', Type='', IsBinary=False,
1448                  Arch='COMMON', ToolChainFamily='', Target='', TagName='', ToolCode=''):
1449         self.Arch = Arch
1450         self.File = str(File)
1451         if os.path.isabs(self.File):
1452             self.Root = ''
1453             self.AlterRoot = ''
1454         else:
1455             self.Root = str(Root)
1456             self.AlterRoot = str(AlterRoot)
1457
1458         # Remove any '.' and '..' in path
1459         if self.Root:
1460             self.Path = os.path.normpath(os.path.join(self.Root, self.File))
1461             self.Root = os.path.normpath(CommonPath([self.Root, self.Path]))
1462             # eliminate the side-effect of 'C:'
1463             if self.Root[-1] == ':':
1464                 self.Root += os.path.sep
1465             # file path should not start with path separator
1466             if self.Root[-1] == os.path.sep:
1467                 self.File = self.Path[len(self.Root):]
1468             else:
1469                 self.File = self.Path[len(self.Root)+1:]
1470         else:
1471             self.Path = os.path.normpath(self.File)
1472
1473         self.SubDir, self.Name = os.path.split(self.File)
1474         self.BaseName, self.Ext = os.path.splitext(self.Name)
1475
1476         if self.Root:
1477             if self.SubDir:
1478                 self.Dir = os.path.join(self.Root, self.SubDir)
1479             else:
1480                 self.Dir = self.Root
1481         else:
1482             self.Dir = self.SubDir
1483
1484         if IsBinary:
1485             self.Type = Type
1486         else:
1487             self.Type = self.Ext.lower()
1488
1489         self.IsBinary = IsBinary
1490         self.Target = Target
1491         self.TagName = TagName
1492         self.ToolCode = ToolCode
1493         self.ToolChainFamily = ToolChainFamily
1494
1495         self._Key = None
1496
1497     ## Convert the object of this class to a string
1498     #
1499     #  Convert member Path of the class to a string
1500     #
1501     #  @retval string Formatted String
1502     #
1503     def __str__(self):
1504         return self.Path
1505
1506     ## Override __eq__ function
1507     #
1508     # Check whether PathClass are the same
1509     #
1510     # @retval False The two PathClass are different
1511     # @retval True  The two PathClass are the same
1512     #
1513     def __eq__(self, Other):
1514         if type(Other) == type(self):
1515             return self.Path == Other.Path
1516         else:
1517             return self.Path == str(Other)
1518
1519     ## Override __cmp__ function
1520     #
1521     # Customize the comparsion operation of two PathClass
1522     #
1523     # @retval 0     The two PathClass are different
1524     # @retval -1    The first PathClass is less than the second PathClass
1525     # @retval 1     The first PathClass is Bigger than the second PathClass
1526     def __cmp__(self, Other):
1527         if type(Other) == type(self):
1528             OtherKey = Other.Path
1529         else:
1530             OtherKey = str(Other)
1531
1532         SelfKey = self.Path
1533         if SelfKey == OtherKey:
1534             return 0
1535         elif SelfKey > OtherKey:
1536             return 1
1537         else:
1538             return -1
1539
1540     ## Override __hash__ function
1541     #
1542     # Use Path as key in hash table
1543     #
1544     # @retval string Key for hash table
1545     #
1546     def __hash__(self):
1547         return hash(self.Path)
1548
1549     def _GetFileKey(self):
1550         if self._Key == None:
1551             self._Key = self.Path.upper()   # + self.ToolChainFamily + self.TagName + self.ToolCode + self.Target
1552         return self._Key
1553
1554     def _GetTimeStamp(self):
1555         return os.stat(self.Path)[8]
1556
1557     def Validate(self, Type='', CaseSensitive=True):
1558         if GlobalData.gCaseInsensitive:
1559             CaseSensitive = False
1560         if Type and Type.lower() != self.Type:
1561             return FILE_TYPE_MISMATCH, '%s (expect %s but got %s)' % (self.File, Type, self.Type)
1562
1563         RealFile, RealRoot = RealPath2(self.File, self.Root, self.AlterRoot)
1564         if not RealRoot and not RealFile:
1565             RealFile = self.File
1566             if self.AlterRoot:
1567                 RealFile = os.path.join(self.AlterRoot, self.File)
1568             elif self.Root:
1569                 RealFile = os.path.join(self.Root, self.File)
1570             return FILE_NOT_FOUND, os.path.join(self.AlterRoot, RealFile)
1571
1572         ErrorCode = 0
1573         ErrorInfo = ''
1574         if RealRoot != self.Root or RealFile != self.File:
1575             if CaseSensitive and (RealFile != self.File or (RealRoot != self.Root and RealRoot != self.AlterRoot)):
1576                 ErrorCode = FILE_CASE_MISMATCH
1577                 ErrorInfo = self.File + '\n\t' + RealFile + " [in file system]"
1578
1579             self.SubDir, self.Name = os.path.split(RealFile)
1580             self.BaseName, self.Ext = os.path.splitext(self.Name)
1581             if self.SubDir:
1582                 self.Dir = os.path.join(RealRoot, self.SubDir)
1583             else:
1584                 self.Dir = RealRoot
1585             self.File = RealFile
1586             self.Root = RealRoot
1587             self.Path = os.path.join(RealRoot, RealFile)
1588         return ErrorCode, ErrorInfo
1589
1590     Key = property(_GetFileKey)
1591     TimeStamp = property(_GetTimeStamp)
1592
1593 ## Parse PE image to get the required PE informaion.
1594 #
1595 class PeImageClass():
1596     ## Constructor
1597     #
1598     #   @param  File FilePath of PeImage
1599     #
1600     def __init__(self, PeFile):
1601         self.FileName   = PeFile
1602         self.IsValid    = False
1603         self.Size       = 0
1604         self.EntryPoint = 0
1605         self.SectionAlignment  = 0
1606         self.SectionHeaderList = []
1607         self.ErrorInfo = ''
1608         try:
1609             PeObject = open(PeFile, 'rb')
1610         except:
1611             self.ErrorInfo = self.FileName + ' can not be found\n'
1612             return
1613         # Read DOS header
1614         ByteArray = array.array('B')
1615         ByteArray.fromfile(PeObject, 0x3E)
1616         ByteList = ByteArray.tolist()
1617         # DOS signature should be 'MZ'
1618         if self._ByteListToStr (ByteList[0x0:0x2]) != 'MZ':
1619             self.ErrorInfo = self.FileName + ' has no valid DOS signature MZ'
1620             return
1621
1622         # Read 4 byte PE Signature
1623         PeOffset = self._ByteListToInt(ByteList[0x3C:0x3E])
1624         PeObject.seek(PeOffset)
1625         ByteArray = array.array('B')
1626         ByteArray.fromfile(PeObject, 4)
1627         # PE signature should be 'PE\0\0'
1628         if ByteArray.tostring() != 'PE\0\0':
1629             self.ErrorInfo = self.FileName + ' has no valid PE signature PE00'
1630             return
1631
1632         # Read PE file header
1633         ByteArray = array.array('B')
1634         ByteArray.fromfile(PeObject, 0x14)
1635         ByteList = ByteArray.tolist()
1636         SecNumber = self._ByteListToInt(ByteList[0x2:0x4])
1637         if SecNumber == 0:
1638             self.ErrorInfo = self.FileName + ' has no section header'
1639             return
1640
1641         # Read PE optional header
1642         OptionalHeaderSize = self._ByteListToInt(ByteArray[0x10:0x12])
1643         ByteArray = array.array('B')
1644         ByteArray.fromfile(PeObject, OptionalHeaderSize)
1645         ByteList = ByteArray.tolist()
1646         self.EntryPoint       = self._ByteListToInt(ByteList[0x10:0x14])
1647         self.SectionAlignment = self._ByteListToInt(ByteList[0x20:0x24])
1648         self.Size             = self._ByteListToInt(ByteList[0x38:0x3C])
1649
1650         # Read each Section Header
1651         for Index in range(SecNumber):
1652             ByteArray = array.array('B')
1653             ByteArray.fromfile(PeObject, 0x28)
1654             ByteList = ByteArray.tolist()
1655             SecName  = self._ByteListToStr(ByteList[0:8])
1656             SecVirtualSize = self._ByteListToInt(ByteList[8:12])
1657             SecRawAddress  = self._ByteListToInt(ByteList[20:24])
1658             SecVirtualAddress = self._ByteListToInt(ByteList[12:16])
1659             self.SectionHeaderList.append((SecName, SecVirtualAddress, SecRawAddress, SecVirtualSize))
1660         self.IsValid = True
1661         PeObject.close()
1662
1663     def _ByteListToStr(self, ByteList):
1664         String = ''
1665         for index in range(len(ByteList)):
1666             if ByteList[index] == 0:
1667                 break
1668             String += chr(ByteList[index])
1669         return String
1670
1671     def _ByteListToInt(self, ByteList):
1672         Value = 0
1673         for index in range(len(ByteList) - 1, -1, -1):
1674             Value = (Value << 8) | int(ByteList[index])
1675         return Value
1676
1677
1678 class SkuClass():
1679
1680     DEFAULT = 0
1681     SINGLE = 1
1682     MULTIPLE =2
1683
1684     def __init__(self,SkuIdentifier='', SkuIds={}):
1685
1686         self.AvailableSkuIds = sdict()
1687         self.SkuIdSet = []
1688
1689         if SkuIdentifier == '' or SkuIdentifier is None:
1690             self.SkuIdSet = ['DEFAULT']
1691         elif SkuIdentifier == 'ALL':
1692             self.SkuIdSet = SkuIds.keys()
1693         else:
1694             r = SkuIdentifier.split('|')
1695             self.SkuIdSet=[r[k].strip() for k in range(len(r))]
1696         if len(self.SkuIdSet) == 2 and 'DEFAULT' in self.SkuIdSet and SkuIdentifier != 'ALL':
1697             self.SkuIdSet.remove('DEFAULT')
1698
1699         for each in self.SkuIdSet:
1700             if each in SkuIds:
1701                 self.AvailableSkuIds[each] = SkuIds[each]
1702             else:
1703                 EdkLogger.error("build", PARAMETER_INVALID,
1704                             ExtraData="SKU-ID [%s] is not supported by the platform. [Valid SKU-ID: %s]"
1705                                       % (each, " ".join(SkuIds.keys())))
1706
1707     def __SkuUsageType(self):
1708
1709         if len(self.SkuIdSet) == 1:
1710             if self.SkuIdSet[0] == 'DEFAULT':
1711                 return SkuClass.DEFAULT
1712             else:
1713                 return SkuClass.SINGLE
1714         else:
1715             return SkuClass.MULTIPLE
1716
1717     def __GetAvailableSkuIds(self):
1718         return self.AvailableSkuIds
1719
1720     def __GetSystemSkuID(self):
1721         if self.__SkuUsageType() == SkuClass.SINGLE:
1722             return self.SkuIdSet[0]
1723         else:
1724             return 'DEFAULT'
1725
1726     SystemSkuId = property(__GetSystemSkuID)
1727     AvailableSkuIdSet = property(__GetAvailableSkuIds)
1728     SkuUsageType = property(__SkuUsageType)
1729
1730 ##
1731 #
1732 # This acts like the main() function for the script, unless it is 'import'ed into another
1733 # script.
1734 #
1735 if __name__ == '__main__':
1736     pass
1737