BaseTools/Source/Python/Common/Misc.py

   1 ## @file
   2 # Common routines used by all tools
   3 #
   4 # Copyright (c) 2007 - 2018, Intel Corporation. All rights reserved.<BR>
   5 # This program and the accompanying materials
   6 # are licensed and made available under the terms and conditions of the BSD License
   7 # which accompanies this distribution.  The full text of the license may be found at
   8 # http://opensource.org/licenses/bsd-license.php
   9 #
  10 # THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS,
  11 # WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED.
  12 #
  13
  14 ##
  15 # Import Modules
  16 #
  17 from __future__ import absolute_import
  18 import Common.LongFilePathOs as os
  19 import sys
  20 import string
  21 import threading
  22 import time
  23 import re
  24 import pickle
  25 import array
  26 import shutil
  27 from struct import pack
  28 from UserDict import IterableUserDict
  29 from UserList import UserList
  30
  31 from Common import EdkLogger as EdkLogger
  32 from Common import GlobalData as GlobalData
  33 from .DataType import *
  34 from .BuildToolError import *
  35 from CommonDataClass.DataClass import *
  36 from .Parsing import GetSplitValueList
  37 from Common.LongFilePathSupport import OpenLongFilePath as open
  38 from Common.MultipleWorkspace import MultipleWorkspace as mws
  39 import uuid
  40 from CommonDataClass.Exceptions import BadExpression
  41 import subprocess
  42 ## Regular expression used to find out place holders in string template
  43 gPlaceholderPattern = re.compile("\$\{([^$()\s]+)\}", re.MULTILINE | re.UNICODE)
  44
  45 ## regular expressions for map file processing
  46 startPatternGeneral = re.compile("^Start[' ']+Length[' ']+Name[' ']+Class")
  47 addressPatternGeneral = re.compile("^Address[' ']+Publics by Value[' ']+Rva\+Base")
  48 valuePatternGcc = re.compile('^([\w_\.]+) +([\da-fA-Fx]+) +([\da-fA-Fx]+)$')
  49 pcdPatternGcc = re.compile('^([\da-fA-Fx]+) +([\da-fA-Fx]+)')
  50 secReGeneral = re.compile('^([\da-fA-F]+):([\da-fA-F]+) +([\da-fA-F]+)[Hh]? +([.\w\$]+) +(\w+)', re.UNICODE)
  51
  52 ## Dictionary used to store file time stamp for quick re-access
  53 gFileTimeStampCache = {}    # {file path : file time stamp}
  54
  55 ## Dictionary used to store dependencies of files
  56 gDependencyDatabase = {}    # arch : {file path : [dependent files list]}
  57
  58 #
  59 # If a module is built more than once with different PCDs or library classes
  60 # a temporary INF file with same content is created, the temporary file is removed
  61 # when build exits.
  62 #
  63 _TempInfs = []
  64
  65 def GetVariableOffset(mapfilepath, efifilepath, varnames):
  66     """ Parse map file to get variable offset in current EFI file
  67     @param mapfilepath    Map file absolution path
  68     @param efifilepath:   EFI binary file full path
  69     @param varnames       iteratable container whose elements are variable names to be searched
  70
  71     @return List whos elements are tuple with variable name and raw offset
  72     """
  73     lines = []
  74     try:
  75         f = open(mapfilepath, 'r')
  76         lines = f.readlines()
  77         f.close()
  78     except:
  79         return None
  80
  81     if len(lines) == 0: return None
  82     firstline = lines[0].strip()
  83     if (firstline.startswith("Archive member included ") and
  84         firstline.endswith(" file (symbol)")):
  85         return _parseForGCC(lines, efifilepath, varnames)
  86     if firstline.startswith("# Path:"):
  87         return _parseForXcode(lines, efifilepath, varnames)
  88     return _parseGeneral(lines, efifilepath, varnames)
  89
  90 def _parseForXcode(lines, efifilepath, varnames):
  91     status = 0
  92     ret = []
  93     for line in lines:
  94         line = line.strip()
  95         if status == 0 and line == "# Symbols:":
  96             status = 1
  97             continue
  98         if status == 1 and len(line) != 0:
  99             for varname in varnames:
 100                 if varname in line:
 101                     # cannot pregenerate this RegEx since it uses varname from varnames.
 102                     m = re.match('^([\da-fA-FxX]+)([\s\S]*)([_]*%s)$' % varname, line)
 103                     if m is not None:
 104                         ret.append((varname, m.group(1)))
 105     return ret
 106
 107 def _parseForGCC(lines, efifilepath, varnames):
 108     """ Parse map file generated by GCC linker """
 109     status = 0
 110     sections = []
 111     varoffset = []
 112     for index, line in enumerate(lines):
 113         line = line.strip()
 114         # status machine transection
 115         if status == 0 and line == "Memory Configuration":
 116             status = 1
 117             continue
 118         elif status == 1 and line == 'Linker script and memory map':
 119             status = 2
 120             continue
 121         elif status ==2 and line == 'START GROUP':
 122             status = 3
 123             continue
 124
 125         # status handler
 126         if status == 3:
 127             m = valuePatternGcc.match(line)
 128             if m is not None:
 129                 sections.append(m.groups(0))
 130             for varname in varnames:
 131                 Str = ''
 132                 m = re.match("^.data.(%s)" % varname, line)
 133                 if m is not None:
 134                     m = re.match(".data.(%s)$" % varname, line)
 135                     if m is not None:
 136                         Str = lines[index + 1]
 137                     else:
 138                         Str = line[len(".data.%s" % varname):]
 139                     if Str:
 140                         m = pcdPatternGcc.match(Str.strip())
 141                         if m is not None:
 142                             varoffset.append((varname, int(m.groups(0)[0], 16), int(sections[-1][1], 16), sections[-1][0]))
 143
 144     if not varoffset:
 145         return []
 146     # get section information from efi file
 147     efisecs = PeImageClass(efifilepath).SectionHeaderList
 148     if efisecs is None or len(efisecs) == 0:
 149         return []
 150     #redirection
 151     redirection = 0
 152     for efisec in efisecs:
 153         for section in sections:
 154             if section[0].strip() == efisec[0].strip() and section[0].strip() == '.text':
 155                 redirection = int(section[1], 16) - efisec[1]
 156
 157     ret = []
 158     for var in varoffset:
 159         for efisec in efisecs:
 160             if var[1] >= efisec[1] and var[1] < efisec[1]+efisec[3]:
 161                 ret.append((var[0], hex(efisec[2] + var[1] - efisec[1] - redirection)))
 162     return ret
 163
 164 def _parseGeneral(lines, efifilepath, varnames):
 165     status = 0    #0 - beginning of file; 1 - PE section definition; 2 - symbol table
 166     secs  = []    # key = section name
 167     varoffset = []
 168     symRe = re.compile('^([\da-fA-F]+):([\da-fA-F]+) +([\.:\\\\\w\?@\$]+) +([\da-fA-F]+)', re.UNICODE)
 169
 170     for line in lines:
 171         line = line.strip()
 172         if startPatternGeneral.match(line):
 173             status = 1
 174             continue
 175         if addressPatternGeneral.match(line):
 176             status = 2
 177             continue
 178         if line.startswith("entry point at"):
 179             status = 3
 180             continue
 181         if status == 1 and len(line) != 0:
 182             m =  secReGeneral.match(line)
 183             assert m is not None, "Fail to parse the section in map file , line is %s" % line
 184             sec_no, sec_start, sec_length, sec_name, sec_class = m.groups(0)
 185             secs.append([int(sec_no, 16), int(sec_start, 16), int(sec_length, 16), sec_name, sec_class])
 186         if status == 2 and len(line) != 0:
 187             for varname in varnames:
 188                 m = symRe.match(line)
 189                 assert m is not None, "Fail to parse the symbol in map file, line is %s" % line
 190                 sec_no, sym_offset, sym_name, vir_addr = m.groups(0)
 191                 sec_no     = int(sec_no,     16)
 192                 sym_offset = int(sym_offset, 16)
 193                 vir_addr   = int(vir_addr,   16)
 194                 # cannot pregenerate this RegEx since it uses varname from varnames.
 195                 m2 = re.match('^[_]*(%s)' % varname, sym_name)
 196                 if m2 is not None:
 197                     # fond a binary pcd entry in map file
 198                     for sec in secs:
 199                         if sec[0] == sec_no and (sym_offset >= sec[1] and sym_offset < sec[1] + sec[2]):
 200                             varoffset.append([varname, sec[3], sym_offset, vir_addr, sec_no])
 201
 202     if not varoffset: return []
 203
 204     # get section information from efi file
 205     efisecs = PeImageClass(efifilepath).SectionHeaderList
 206     if efisecs is None or len(efisecs) == 0:
 207         return []
 208
 209     ret = []
 210     for var in varoffset:
 211         index = 0
 212         for efisec in efisecs:
 213             index = index + 1
 214             if var[1].strip() == efisec[0].strip():
 215                 ret.append((var[0], hex(efisec[2] + var[2])))
 216             elif var[4] == index:
 217                 ret.append((var[0], hex(efisec[2] + var[2])))
 218
 219     return ret
 220
 221 ## Routine to process duplicated INF
 222 #
 223 #  This function is called by following two cases:
 224 #  Case 1 in DSC:
 225 #    [components.arch]
 226 #    Pkg/module/module.inf
 227 #    Pkg/module/module.inf {
 228 #      <Defines>
 229 #        FILE_GUID = 0D1B936F-68F3-4589-AFCC-FB8B7AEBC836
 230 #    }
 231 #  Case 2 in FDF:
 232 #    INF Pkg/module/module.inf
 233 #    INF FILE_GUID = 0D1B936F-68F3-4589-AFCC-FB8B7AEBC836 Pkg/module/module.inf
 234 #
 235 #  This function copies Pkg/module/module.inf to
 236 #  Conf/.cache/0D1B936F-68F3-4589-AFCC-FB8B7AEBC836module.inf
 237 #
 238 #  @param Path     Original PathClass object
 239 #  @param BaseName New file base name
 240 #
 241 #  @retval         return the new PathClass object
 242 #
 243 def ProcessDuplicatedInf(Path, BaseName, Workspace):
 244     Filename = os.path.split(Path.File)[1]
 245     if '.' in Filename:
 246         Filename = BaseName + Path.BaseName + Filename[Filename.rfind('.'):]
 247     else:
 248         Filename = BaseName + Path.BaseName
 249
 250     #
 251     # If -N is specified on command line, cache is disabled
 252     # The directory has to be created
 253     #
 254     DbDir = os.path.split(GlobalData.gDatabasePath)[0]
 255     if not os.path.exists(DbDir):
 256         os.makedirs(DbDir)
 257     #
 258     # A temporary INF is copied to database path which must have write permission
 259     # The temporary will be removed at the end of build
 260     # In case of name conflict, the file name is
 261     # FILE_GUIDBaseName (0D1B936F-68F3-4589-AFCC-FB8B7AEBC836module.inf)
 262     #
 263     TempFullPath = os.path.join(DbDir,
 264                                 Filename)
 265     RtPath = PathClass(Path.File, Workspace)
 266     #
 267     # Modify the full path to temporary path, keep other unchanged
 268     #
 269     # To build same module more than once, the module path with FILE_GUID overridden has
 270     # the file name FILE_GUIDmodule.inf, but the relative path (self.MetaFile.File) is the real path
 271     # in DSC which is used as relative path by C files and other files in INF.
 272     # A trick was used: all module paths are PathClass instances, after the initialization
 273     # of PathClass, the PathClass.Path is overridden by the temporary INF path.
 274     #
 275     # The reason for creating a temporary INF is:
 276     # Platform.Modules which is the base to create ModuleAutoGen objects is a dictionary,
 277     # the key is the full path of INF, the value is an object to save overridden library instances, PCDs.
 278     # A different key for the same module is needed to create different output directory,
 279     # retrieve overridden PCDs, library instances.
 280     #
 281     # The BaseName is the FILE_GUID which is also the output directory name.
 282     #
 283     #
 284     RtPath.Path = TempFullPath
 285     RtPath.BaseName = BaseName
 286     #
 287     # If file exists, compare contents
 288     #
 289     if os.path.exists(TempFullPath):
 290         with open(str(Path), 'rb') as f1, open(TempFullPath, 'rb') as f2:
 291             if f1.read() == f2.read():
 292                 return RtPath
 293     _TempInfs.append(TempFullPath)
 294     shutil.copy2(str(Path), TempFullPath)
 295     return RtPath
 296
 297 ## Remove temporary created INFs whose paths were saved in _TempInfs
 298 #
 299 def ClearDuplicatedInf():
 300     while _TempInfs:
 301         File = _TempInfs.pop()
 302         if os.path.exists(File):
 303             os.remove(File)
 304
 305 ## Convert GUID string in xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx style to C structure style
 306 #
 307 #   @param      Guid    The GUID string
 308 #
 309 #   @retval     string  The GUID string in C structure style
 310 #
 311 def GuidStringToGuidStructureString(Guid):
 312     GuidList = Guid.split('-')
 313     Result = '{'
 314     for Index in range(0, 3, 1):
 315         Result = Result + '0x' + GuidList[Index] + ', '
 316     Result = Result + '{0x' + GuidList[3][0:2] + ', 0x' + GuidList[3][2:4]
 317     for Index in range(0, 12, 2):
 318         Result = Result + ', 0x' + GuidList[4][Index:Index + 2]
 319     Result += '}}'
 320     return Result
 321
 322 ## Convert GUID structure in byte array to xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
 323 #
 324 #   @param      GuidValue   The GUID value in byte array
 325 #
 326 #   @retval     string      The GUID value in xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx format
 327 #
 328 def GuidStructureByteArrayToGuidString(GuidValue):
 329     guidValueString = GuidValue.lower().replace("{", "").replace("}", "").replace(" ", "").replace(";", "")
 330     guidValueList = guidValueString.split(",")
 331     if len(guidValueList) != 16:
 332         return ''
 333         #EdkLogger.error(None, None, "Invalid GUID value string %s" % GuidValue)
 334     try:
 335         return "%02x%02x%02x%02x-%02x%02x-%02x%02x-%02x%02x-%02x%02x%02x%02x%02x%02x" % (
 336                 int(guidValueList[3], 16),
 337                 int(guidValueList[2], 16),
 338                 int(guidValueList[1], 16),
 339                 int(guidValueList[0], 16),
 340                 int(guidValueList[5], 16),
 341                 int(guidValueList[4], 16),
 342                 int(guidValueList[7], 16),
 343                 int(guidValueList[6], 16),
 344                 int(guidValueList[8], 16),
 345                 int(guidValueList[9], 16),
 346                 int(guidValueList[10], 16),
 347                 int(guidValueList[11], 16),
 348                 int(guidValueList[12], 16),
 349                 int(guidValueList[13], 16),
 350                 int(guidValueList[14], 16),
 351                 int(guidValueList[15], 16)
 352                 )
 353     except:
 354         return ''
 355
 356 ## Convert GUID string in C structure style to xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
 357 #
 358 #   @param      GuidValue   The GUID value in C structure format
 359 #
 360 #   @retval     string      The GUID value in xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx format
 361 #
 362 def GuidStructureStringToGuidString(GuidValue):
 363     guidValueString = GuidValue.lower().replace("{", "").replace("}", "").replace(" ", "").replace(";", "")
 364     guidValueList = guidValueString.split(",")
 365     if len(guidValueList) != 11:
 366         return ''
 367         #EdkLogger.error(None, None, "Invalid GUID value string %s" % GuidValue)
 368     try:
 369         return "%08x-%04x-%04x-%02x%02x-%02x%02x%02x%02x%02x%02x" % (
 370                 int(guidValueList[0], 16),
 371                 int(guidValueList[1], 16),
 372                 int(guidValueList[2], 16),
 373                 int(guidValueList[3], 16),
 374                 int(guidValueList[4], 16),
 375                 int(guidValueList[5], 16),
 376                 int(guidValueList[6], 16),
 377                 int(guidValueList[7], 16),
 378                 int(guidValueList[8], 16),
 379                 int(guidValueList[9], 16),
 380                 int(guidValueList[10], 16)
 381                 )
 382     except:
 383         return ''
 384
 385 ## Convert GUID string in C structure style to xxxxxxxx_xxxx_xxxx_xxxx_xxxxxxxxxxxx
 386 #
 387 #   @param      GuidValue   The GUID value in C structure format
 388 #
 389 #   @retval     string      The GUID value in xxxxxxxx_xxxx_xxxx_xxxx_xxxxxxxxxxxx format
 390 #
 391 def GuidStructureStringToGuidValueName(GuidValue):
 392     guidValueString = GuidValue.lower().replace("{", "").replace("}", "").replace(" ", "")
 393     guidValueList = guidValueString.split(",")
 394     if len(guidValueList) != 11:
 395         EdkLogger.error(None, FORMAT_INVALID, "Invalid GUID value string [%s]" % GuidValue)
 396     return "%08x_%04x_%04x_%02x%02x_%02x%02x%02x%02x%02x%02x" % (
 397             int(guidValueList[0], 16),
 398             int(guidValueList[1], 16),
 399             int(guidValueList[2], 16),
 400             int(guidValueList[3], 16),
 401             int(guidValueList[4], 16),
 402             int(guidValueList[5], 16),
 403             int(guidValueList[6], 16),
 404             int(guidValueList[7], 16),
 405             int(guidValueList[8], 16),
 406             int(guidValueList[9], 16),
 407             int(guidValueList[10], 16)
 408             )
 409
 410 ## Create directories
 411 #
 412 #   @param      Directory   The directory name
 413 #
 414 def CreateDirectory(Directory):
 415     if Directory is None or Directory.strip() == "":
 416         return True
 417     try:
 418         if not os.access(Directory, os.F_OK):
 419             os.makedirs(Directory)
 420     except:
 421         return False
 422     return True
 423
 424 ## Remove directories, including files and sub-directories in it
 425 #
 426 #   @param      Directory   The directory name
 427 #
 428 def RemoveDirectory(Directory, Recursively=False):
 429     if Directory is None or Directory.strip() == "" or not os.path.exists(Directory):
 430         return
 431     if Recursively:
 432         CurrentDirectory = os.getcwd()
 433         os.chdir(Directory)
 434         for File in os.listdir("."):
 435             if os.path.isdir(File):
 436                 RemoveDirectory(File, Recursively)
 437             else:
 438                 os.remove(File)
 439         os.chdir(CurrentDirectory)
 440     os.rmdir(Directory)
 441
 442 ## Store content in file
 443 #
 444 #  This method is used to save file only when its content is changed. This is
 445 #  quite useful for "make" system to decide what will be re-built and what won't.
 446 #
 447 #   @param      File            The path of file
 448 #   @param      Content         The new content of the file
 449 #   @param      IsBinaryFile    The flag indicating if the file is binary file or not
 450 #
 451 #   @retval     True            If the file content is changed and the file is renewed
 452 #   @retval     False           If the file content is the same
 453 #
 454 def SaveFileOnChange(File, Content, IsBinaryFile=True):
 455     if not IsBinaryFile:
 456         Content = Content.replace("\n", os.linesep)
 457
 458     if os.path.exists(File):
 459         try:
 460             if Content == open(File, "rb").read():
 461                 return False
 462         except:
 463             EdkLogger.error(None, FILE_OPEN_FAILURE, ExtraData=File)
 464
 465     DirName = os.path.dirname(File)
 466     if not CreateDirectory(DirName):
 467         EdkLogger.error(None, FILE_CREATE_FAILURE, "Could not create directory %s" % DirName)
 468     else:
 469         if DirName == '':
 470             DirName = os.getcwd()
 471         if not os.access(DirName, os.W_OK):
 472             EdkLogger.error(None, PERMISSION_FAILURE, "Do not have write permission on directory %s" % DirName)
 473
 474     try:
 475         if GlobalData.gIsWindows:
 476             try:
 477                 from .PyUtility import SaveFileToDisk
 478                 if not SaveFileToDisk(File, Content):
 479                     EdkLogger.error(None, FILE_CREATE_FAILURE, ExtraData=File)
 480             except:
 481                 Fd = open(File, "wb")
 482                 Fd.write(Content)
 483                 Fd.close()
 484         else:
 485             Fd = open(File, "wb")
 486             Fd.write(Content)
 487             Fd.close()
 488     except IOError as X:
 489         EdkLogger.error(None, FILE_CREATE_FAILURE, ExtraData='IOError %s' % X)
 490
 491     return True
 492
 493 ## Make a Python object persistent on file system
 494 #
 495 #   @param      Data    The object to be stored in file
 496 #   @param      File    The path of file to store the object
 497 #
 498 def DataDump(Data, File):
 499     Fd = None
 500     try:
 501         Fd = open(File, 'wb')
 502         pickle.dump(Data, Fd, pickle.HIGHEST_PROTOCOL)
 503     except:
 504         EdkLogger.error("", FILE_OPEN_FAILURE, ExtraData=File, RaiseError=False)
 505     finally:
 506         if Fd is not None:
 507             Fd.close()
 508
 509 ## Restore a Python object from a file
 510 #
 511 #   @param      File    The path of file stored the object
 512 #
 513 #   @retval     object  A python object
 514 #   @retval     None    If failure in file operation
 515 #
 516 def DataRestore(File):
 517     Data = None
 518     Fd = None
 519     try:
 520         Fd = open(File, 'rb')
 521         Data = pickle.load(Fd)
 522     except Exception as e:
 523         EdkLogger.verbose("Failed to load [%s]\n\t%s" % (File, str(e)))
 524         Data = None
 525     finally:
 526         if Fd is not None:
 527             Fd.close()
 528     return Data
 529
 530 ## Retrieve and cache the real path name in file system
 531 #
 532 #   @param      Root    The root directory of path relative to
 533 #
 534 #   @retval     str     The path string if the path exists
 535 #   @retval     None    If path doesn't exist
 536 #
 537 class DirCache:
 538     _CACHE_ = set()
 539     _UPPER_CACHE_ = {}
 540
 541     def __init__(self, Root):
 542         self._Root = Root
 543         for F in os.listdir(Root):
 544             self._CACHE_.add(F)
 545             self._UPPER_CACHE_[F.upper()] = F
 546
 547     # =[] operator
 548     def __getitem__(self, Path):
 549         Path = Path[len(os.path.commonprefix([Path, self._Root])):]
 550         if not Path:
 551             return self._Root
 552         if Path and Path[0] == os.path.sep:
 553             Path = Path[1:]
 554         if Path in self._CACHE_:
 555             return os.path.join(self._Root, Path)
 556         UpperPath = Path.upper()
 557         if UpperPath in self._UPPER_CACHE_:
 558             return os.path.join(self._Root, self._UPPER_CACHE_[UpperPath])
 559
 560         IndexList = []
 561         LastSepIndex = -1
 562         SepIndex = Path.find(os.path.sep)
 563         while SepIndex > -1:
 564             Parent = UpperPath[:SepIndex]
 565             if Parent not in self._UPPER_CACHE_:
 566                 break
 567             LastSepIndex = SepIndex
 568             SepIndex = Path.find(os.path.sep, LastSepIndex + 1)
 569
 570         if LastSepIndex == -1:
 571             return None
 572
 573         Cwd = os.getcwd()
 574         os.chdir(self._Root)
 575         SepIndex = LastSepIndex
 576         while SepIndex > -1:
 577             Parent = Path[:SepIndex]
 578             ParentKey = UpperPath[:SepIndex]
 579             if ParentKey not in self._UPPER_CACHE_:
 580                 os.chdir(Cwd)
 581                 return None
 582
 583             if Parent in self._CACHE_:
 584                 ParentDir = Parent
 585             else:
 586                 ParentDir = self._UPPER_CACHE_[ParentKey]
 587             for F in os.listdir(ParentDir):
 588                 Dir = os.path.join(ParentDir, F)
 589                 self._CACHE_.add(Dir)
 590                 self._UPPER_CACHE_[Dir.upper()] = Dir
 591
 592             SepIndex = Path.find(os.path.sep, SepIndex + 1)
 593
 594         os.chdir(Cwd)
 595         if Path in self._CACHE_:
 596             return os.path.join(self._Root, Path)
 597         elif UpperPath in self._UPPER_CACHE_:
 598             return os.path.join(self._Root, self._UPPER_CACHE_[UpperPath])
 599         return None
 600
 601 def RealPath(File, Dir='', OverrideDir=''):
 602     NewFile = os.path.normpath(os.path.join(Dir, File))
 603     NewFile = GlobalData.gAllFiles[NewFile]
 604     if not NewFile and OverrideDir:
 605         NewFile = os.path.normpath(os.path.join(OverrideDir, File))
 606         NewFile = GlobalData.gAllFiles[NewFile]
 607     return NewFile
 608
 609 def RealPath2(File, Dir='', OverrideDir=''):
 610     NewFile = None
 611     if OverrideDir:
 612         NewFile = GlobalData.gAllFiles[os.path.normpath(os.path.join(OverrideDir, File))]
 613         if NewFile:
 614             if OverrideDir[-1] == os.path.sep:
 615                 return NewFile[len(OverrideDir):], NewFile[0:len(OverrideDir)]
 616             else:
 617                 return NewFile[len(OverrideDir) + 1:], NewFile[0:len(OverrideDir)]
 618     if GlobalData.gAllFiles:
 619         NewFile = GlobalData.gAllFiles[os.path.normpath(os.path.join(Dir, File))]
 620     if not NewFile:
 621         NewFile = os.path.normpath(os.path.join(Dir, File))
 622         if not os.path.exists(NewFile):
 623             return None, None
 624     if NewFile:
 625         if Dir:
 626             if Dir[-1] == os.path.sep:
 627                 return NewFile[len(Dir):], NewFile[0:len(Dir)]
 628             else:
 629                 return NewFile[len(Dir) + 1:], NewFile[0:len(Dir)]
 630         else:
 631             return NewFile, ''
 632
 633     return None, None
 634
 635 ## Get GUID value from given packages
 636 #
 637 #   @param      CName           The CName of the GUID
 638 #   @param      PackageList     List of packages looking-up in
 639 #   @param      Inffile         The driver file
 640 #
 641 #   @retval     GuidValue   if the CName is found in any given package
 642 #   @retval     None        if the CName is not found in all given packages
 643 #
 644 def GuidValue(CName, PackageList, Inffile = None):
 645     for P in PackageList:
 646         GuidKeys = P.Guids.keys()
 647         if Inffile and P._PrivateGuids:
 648             if not Inffile.startswith(P.MetaFile.Dir):
 649                 GuidKeys = [x for x in P.Guids if x not in P._PrivateGuids]
 650         if CName in GuidKeys:
 651             return P.Guids[CName]
 652     return None
 653
 654 ## Get Protocol value from given packages
 655 #
 656 #   @param      CName           The CName of the GUID
 657 #   @param      PackageList     List of packages looking-up in
 658 #   @param      Inffile         The driver file
 659 #
 660 #   @retval     GuidValue   if the CName is found in any given package
 661 #   @retval     None        if the CName is not found in all given packages
 662 #
 663 def ProtocolValue(CName, PackageList, Inffile = None):
 664     for P in PackageList:
 665         ProtocolKeys = P.Protocols.keys()
 666         if Inffile and P._PrivateProtocols:
 667             if not Inffile.startswith(P.MetaFile.Dir):
 668                 ProtocolKeys = [x for x in P.Protocols if x not in P._PrivateProtocols]
 669         if CName in ProtocolKeys:
 670             return P.Protocols[CName]
 671     return None
 672
 673 ## Get PPI value from given packages
 674 #
 675 #   @param      CName           The CName of the GUID
 676 #   @param      PackageList     List of packages looking-up in
 677 #   @param      Inffile         The driver file
 678 #
 679 #   @retval     GuidValue   if the CName is found in any given package
 680 #   @retval     None        if the CName is not found in all given packages
 681 #
 682 def PpiValue(CName, PackageList, Inffile = None):
 683     for P in PackageList:
 684         PpiKeys = P.Ppis.keys()
 685         if Inffile and P._PrivatePpis:
 686             if not Inffile.startswith(P.MetaFile.Dir):
 687                 PpiKeys = [x for x in P.Ppis if x not in P._PrivatePpis]
 688         if CName in PpiKeys:
 689             return P.Ppis[CName]
 690     return None
 691
 692 ## A string template class
 693 #
 694 #  This class implements a template for string replacement. A string template
 695 #  looks like following
 696 #
 697 #       ${BEGIN} other_string ${placeholder_name} other_string ${END}
 698 #
 699 #  The string between ${BEGIN} and ${END} will be repeated as many times as the
 700 #  length of "placeholder_name", which is a list passed through a dict. The
 701 #  "placeholder_name" is the key name of the dict. The ${BEGIN} and ${END} can
 702 #  be not used and, in this case, the "placeholder_name" must not a list and it
 703 #  will just be replaced once.
 704 #
 705 class TemplateString(object):
 706     _REPEAT_START_FLAG = "BEGIN"
 707     _REPEAT_END_FLAG = "END"
 708
 709     class Section(object):
 710         _LIST_TYPES = [type([]), type(set()), type((0,))]
 711
 712         def __init__(self, TemplateSection, PlaceHolderList):
 713             self._Template = TemplateSection
 714             self._PlaceHolderList = []
 715
 716             # Split the section into sub-sections according to the position of placeholders
 717             if PlaceHolderList:
 718                 self._SubSectionList = []
 719                 SubSectionStart = 0
 720                 #
 721                 # The placeholders passed in must be in the format of
 722                 #
 723                 #   PlaceHolderName, PlaceHolderStartPoint, PlaceHolderEndPoint
 724                 #
 725                 for PlaceHolder, Start, End in PlaceHolderList:
 726                     self._SubSectionList.append(TemplateSection[SubSectionStart:Start])
 727                     self._SubSectionList.append(TemplateSection[Start:End])
 728                     self._PlaceHolderList.append(PlaceHolder)
 729                     SubSectionStart = End
 730                 if SubSectionStart < len(TemplateSection):
 731                     self._SubSectionList.append(TemplateSection[SubSectionStart:])
 732             else:
 733                 self._SubSectionList = [TemplateSection]
 734
 735         def __str__(self):
 736             return self._Template + " : " + str(self._PlaceHolderList)
 737
 738         def Instantiate(self, PlaceHolderValues):
 739             RepeatTime = -1
 740             RepeatPlaceHolders = {}
 741             NonRepeatPlaceHolders = {}
 742
 743             for PlaceHolder in self._PlaceHolderList:
 744                 if PlaceHolder not in PlaceHolderValues:
 745                     continue
 746                 Value = PlaceHolderValues[PlaceHolder]
 747                 if type(Value) in self._LIST_TYPES:
 748                     if RepeatTime < 0:
 749                         RepeatTime = len(Value)
 750                     elif RepeatTime != len(Value):
 751                         EdkLogger.error(
 752                                     "TemplateString",
 753                                     PARAMETER_INVALID,
 754                                     "${%s} has different repeat time from others!" % PlaceHolder,
 755                                     ExtraData=str(self._Template)
 756                                     )
 757                     RepeatPlaceHolders["${%s}" % PlaceHolder] = Value
 758                 else:
 759                     NonRepeatPlaceHolders["${%s}" % PlaceHolder] = Value
 760
 761             if NonRepeatPlaceHolders:
 762                 StringList = []
 763                 for S in self._SubSectionList:
 764                     if S not in NonRepeatPlaceHolders:
 765                         StringList.append(S)
 766                     else:
 767                         StringList.append(str(NonRepeatPlaceHolders[S]))
 768             else:
 769                 StringList = self._SubSectionList
 770
 771             if RepeatPlaceHolders:
 772                 TempStringList = []
 773                 for Index in range(RepeatTime):
 774                     for S in StringList:
 775                         if S not in RepeatPlaceHolders:
 776                             TempStringList.append(S)
 777                         else:
 778                             TempStringList.append(str(RepeatPlaceHolders[S][Index]))
 779                 StringList = TempStringList
 780
 781             return "".join(StringList)
 782
 783     ## Constructor
 784     def __init__(self, Template=None):
 785         self.String = ''
 786         self.IsBinary = False
 787         self._Template = Template
 788         self._TemplateSectionList = self._Parse(Template)
 789
 790     ## str() operator
 791     #
 792     #   @retval     string  The string replaced
 793     #
 794     def __str__(self):
 795         return self.String
 796
 797     ## Split the template string into fragments per the ${BEGIN} and ${END} flags
 798     #
 799     #   @retval     list    A list of TemplateString.Section objects
 800     #
 801     def _Parse(self, Template):
 802         SectionStart = 0
 803         SearchFrom = 0
 804         MatchEnd = 0
 805         PlaceHolderList = []
 806         TemplateSectionList = []
 807         while Template:
 808             MatchObj = gPlaceholderPattern.search(Template, SearchFrom)
 809             if not MatchObj:
 810                 if MatchEnd <= len(Template):
 811                     TemplateSection = TemplateString.Section(Template[SectionStart:], PlaceHolderList)
 812                     TemplateSectionList.append(TemplateSection)
 813                 break
 814
 815             MatchString = MatchObj.group(1)
 816             MatchStart = MatchObj.start()
 817             MatchEnd = MatchObj.end()
 818
 819             if MatchString == self._REPEAT_START_FLAG:
 820                 if MatchStart > SectionStart:
 821                     TemplateSection = TemplateString.Section(Template[SectionStart:MatchStart], PlaceHolderList)
 822                     TemplateSectionList.append(TemplateSection)
 823                 SectionStart = MatchEnd
 824                 PlaceHolderList = []
 825             elif MatchString == self._REPEAT_END_FLAG:
 826                 TemplateSection = TemplateString.Section(Template[SectionStart:MatchStart], PlaceHolderList)
 827                 TemplateSectionList.append(TemplateSection)
 828                 SectionStart = MatchEnd
 829                 PlaceHolderList = []
 830             else:
 831                 PlaceHolderList.append((MatchString, MatchStart - SectionStart, MatchEnd - SectionStart))
 832             SearchFrom = MatchEnd
 833         return TemplateSectionList
 834
 835     ## Replace the string template with dictionary of placeholders and append it to previous one
 836     #
 837     #   @param      AppendString    The string template to append
 838     #   @param      Dictionary      The placeholder dictionaries
 839     #
 840     def Append(self, AppendString, Dictionary=None):
 841         if Dictionary:
 842             SectionList = self._Parse(AppendString)
 843             self.String += "".join(S.Instantiate(Dictionary) for S in SectionList)
 844         else:
 845             self.String += AppendString
 846
 847     ## Replace the string template with dictionary of placeholders
 848     #
 849     #   @param      Dictionary      The placeholder dictionaries
 850     #
 851     #   @retval     str             The string replaced with placeholder values
 852     #
 853     def Replace(self, Dictionary=None):
 854         return "".join(S.Instantiate(Dictionary) for S in self._TemplateSectionList)
 855
 856 ## Progress indicator class
 857 #
 858 #  This class makes use of thread to print progress on console.
 859 #
 860 class Progressor:
 861     # for avoiding deadloop
 862     _StopFlag = None
 863     _ProgressThread = None
 864     _CheckInterval = 0.25
 865
 866     ## Constructor
 867     #
 868     #   @param      OpenMessage     The string printed before progress charaters
 869     #   @param      CloseMessage    The string printed after progress charaters
 870     #   @param      ProgressChar    The charater used to indicate the progress
 871     #   @param      Interval        The interval in seconds between two progress charaters
 872     #
 873     def __init__(self, OpenMessage="", CloseMessage="", ProgressChar='.', Interval=1.0):
 874         self.PromptMessage = OpenMessage
 875         self.CodaMessage = CloseMessage
 876         self.ProgressChar = ProgressChar
 877         self.Interval = Interval
 878         if Progressor._StopFlag is None:
 879             Progressor._StopFlag = threading.Event()
 880
 881     ## Start to print progress charater
 882     #
 883     #   @param      OpenMessage     The string printed before progress charaters
 884     #
 885     def Start(self, OpenMessage=None):
 886         if OpenMessage is not None:
 887             self.PromptMessage = OpenMessage
 888         Progressor._StopFlag.clear()
 889         if Progressor._ProgressThread is None:
 890             Progressor._ProgressThread = threading.Thread(target=self._ProgressThreadEntry)
 891             Progressor._ProgressThread.setDaemon(False)
 892             Progressor._ProgressThread.start()
 893
 894     ## Stop printing progress charater
 895     #
 896     #   @param      CloseMessage    The string printed after progress charaters
 897     #
 898     def Stop(self, CloseMessage=None):
 899         OriginalCodaMessage = self.CodaMessage
 900         if CloseMessage is not None:
 901             self.CodaMessage = CloseMessage
 902         self.Abort()
 903         self.CodaMessage = OriginalCodaMessage
 904
 905     ## Thread entry method
 906     def _ProgressThreadEntry(self):
 907         sys.stdout.write(self.PromptMessage + " ")
 908         sys.stdout.flush()
 909         TimeUp = 0.0
 910         while not Progressor._StopFlag.isSet():
 911             if TimeUp <= 0.0:
 912                 sys.stdout.write(self.ProgressChar)
 913                 sys.stdout.flush()
 914                 TimeUp = self.Interval
 915             time.sleep(self._CheckInterval)
 916             TimeUp -= self._CheckInterval
 917         sys.stdout.write(" " + self.CodaMessage + "\n")
 918         sys.stdout.flush()
 919
 920     ## Abort the progress display
 921     @staticmethod
 922     def Abort():
 923         if Progressor._StopFlag is not None:
 924             Progressor._StopFlag.set()
 925         if Progressor._ProgressThread is not None:
 926             Progressor._ProgressThread.join()
 927             Progressor._ProgressThread = None
 928
 929 ## A dict which can access its keys and/or values orderly
 930 #
 931 #  The class implements a new kind of dict which its keys or values can be
 932 #  accessed in the order they are added into the dict. It guarantees the order
 933 #  by making use of an internal list to keep a copy of keys.
 934 #
 935 class sdict(IterableUserDict):
 936     ## Constructor
 937     def __init__(self):
 938         IterableUserDict.__init__(self)
 939         self._key_list = []
 940
 941     ## [] operator
 942     def __setitem__(self, key, value):
 943         if key not in self._key_list:
 944             self._key_list.append(key)
 945         IterableUserDict.__setitem__(self, key, value)
 946
 947     ## del operator
 948     def __delitem__(self, key):
 949         self._key_list.remove(key)
 950         IterableUserDict.__delitem__(self, key)
 951
 952     ## used in "for k in dict" loop to ensure the correct order
 953     def __iter__(self):
 954         return self.iterkeys()
 955
 956     ## len() support
 957     def __len__(self):
 958         return len(self._key_list)
 959
 960     ## "in" test support
 961     def __contains__(self, key):
 962         return key in self._key_list
 963
 964     ## indexof support
 965     def index(self, key):
 966         return self._key_list.index(key)
 967
 968     ## insert support
 969     def insert(self, key, newkey, newvalue, order):
 970         index = self._key_list.index(key)
 971         if order == 'BEFORE':
 972             self._key_list.insert(index, newkey)
 973             IterableUserDict.__setitem__(self, newkey, newvalue)
 974         elif order == 'AFTER':
 975             self._key_list.insert(index + 1, newkey)
 976             IterableUserDict.__setitem__(self, newkey, newvalue)
 977
 978     ## append support
 979     def append(self, sdict):
 980         for key in sdict:
 981             if key not in self._key_list:
 982                 self._key_list.append(key)
 983             IterableUserDict.__setitem__(self, key, sdict[key])
 984
 985     def has_key(self, key):
 986         return key in self._key_list
 987
 988     ## Empty the dict
 989     def clear(self):
 990         self._key_list = []
 991         IterableUserDict.clear(self)
 992
 993     ## Return a copy of keys
 994     def keys(self):
 995         keys = []
 996         for key in self._key_list:
 997             keys.append(key)
 998         return keys
 999
1000     ## Return a copy of values
1001     def values(self):
1002         values = []
1003         for key in self._key_list:
1004             values.append(self[key])
1005         return values
1006
1007     ## Return a copy of (key, value) list
1008     def items(self):
1009         items = []
1010         for key in self._key_list:
1011             items.append((key, self[key]))
1012         return items
1013
1014     ## Iteration support
1015     def iteritems(self):
1016         return iter(self.items())
1017
1018     ## Keys interation support
1019     def iterkeys(self):
1020         return iter(self.keys())
1021
1022     ## Values interation support
1023     def itervalues(self):
1024         return iter(self.values())
1025
1026     ## Return value related to a key, and remove the (key, value) from the dict
1027     def pop(self, key, *dv):
1028         value = None
1029         if key in self._key_list:
1030             value = self[key]
1031             self.__delitem__(key)
1032         elif len(dv) != 0 :
1033             value = kv[0]
1034         return value
1035
1036     ## Return (key, value) pair, and remove the (key, value) from the dict
1037     def popitem(self):
1038         key = self._key_list[-1]
1039         value = self[key]
1040         self.__delitem__(key)
1041         return key, value
1042
1043     def update(self, dict=None, **kwargs):
1044         if dict is not None:
1045             for k, v in dict.items():
1046                 self[k] = v
1047         if len(kwargs):
1048             for k, v in kwargs.items():
1049                 self[k] = v
1050
1051 ## Dictionary with restricted keys
1052 #
1053 class rdict(dict):
1054     ## Constructor
1055     def __init__(self, KeyList):
1056         for Key in KeyList:
1057             dict.__setitem__(self, Key, "")
1058
1059     ## []= operator
1060     def __setitem__(self, key, value):
1061         if key not in self:
1062             EdkLogger.error("RestrictedDict", ATTRIBUTE_SET_FAILURE, "Key [%s] is not allowed" % key,
1063                             ExtraData=", ".join(dict.keys(self)))
1064         dict.__setitem__(self, key, value)
1065
1066     ## =[] operator
1067     def __getitem__(self, key):
1068         if key not in self:
1069             return ""
1070         return dict.__getitem__(self, key)
1071
1072     ## del operator
1073     def __delitem__(self, key):
1074         EdkLogger.error("RestrictedDict", ATTRIBUTE_ACCESS_DENIED, ExtraData="del")
1075
1076     ## Empty the dict
1077     def clear(self):
1078         for Key in self:
1079             self.__setitem__(Key, "")
1080
1081     ## Return value related to a key, and remove the (key, value) from the dict
1082     def pop(self, key, *dv):
1083         EdkLogger.error("RestrictedDict", ATTRIBUTE_ACCESS_DENIED, ExtraData="pop")
1084
1085     ## Return (key, value) pair, and remove the (key, value) from the dict
1086     def popitem(self):
1087         EdkLogger.error("RestrictedDict", ATTRIBUTE_ACCESS_DENIED, ExtraData="popitem")
1088
1089 ## Dictionary using prioritized list as key
1090 #
1091 class tdict:
1092     _ListType = type([])
1093     _TupleType = type(())
1094     _Wildcard = 'COMMON'
1095     _ValidWildcardList = ['COMMON', 'DEFAULT', 'ALL', '*', 'PLATFORM']
1096
1097     def __init__(self, _Single_=False, _Level_=2):
1098         self._Level_ = _Level_
1099         self.data = {}
1100         self._Single_ = _Single_
1101
1102     # =[] operator
1103     def __getitem__(self, key):
1104         KeyType = type(key)
1105         RestKeys = None
1106         if KeyType == self._ListType or KeyType == self._TupleType:
1107             FirstKey = key[0]
1108             if len(key) > 1:
1109                 RestKeys = key[1:]
1110             elif self._Level_ > 1:
1111                 RestKeys = [self._Wildcard for i in range(0, self._Level_ - 1)]
1112         else:
1113             FirstKey = key
1114             if self._Level_ > 1:
1115                 RestKeys = [self._Wildcard for i in range(0, self._Level_ - 1)]
1116
1117         if FirstKey is None or str(FirstKey).upper() in self._ValidWildcardList:
1118             FirstKey = self._Wildcard
1119
1120         if self._Single_:
1121             return self._GetSingleValue(FirstKey, RestKeys)
1122         else:
1123             return self._GetAllValues(FirstKey, RestKeys)
1124
1125     def _GetSingleValue(self, FirstKey, RestKeys):
1126         Value = None
1127         #print "%s-%s" % (FirstKey, self._Level_) ,
1128         if self._Level_ > 1:
1129             if FirstKey == self._Wildcard:
1130                 if FirstKey in self.data:
1131                     Value = self.data[FirstKey][RestKeys]
1132                 if Value is None:
1133                     for Key in self.data:
1134                         Value = self.data[Key][RestKeys]
1135                         if Value is not None: break
1136             else:
1137                 if FirstKey in self.data:
1138                     Value = self.data[FirstKey][RestKeys]
1139                 if Value is None and self._Wildcard in self.data:
1140                     #print "Value=None"
1141                     Value = self.data[self._Wildcard][RestKeys]
1142         else:
1143             if FirstKey == self._Wildcard:
1144                 if FirstKey in self.data:
1145                     Value = self.data[FirstKey]
1146                 if Value is None:
1147                     for Key in self.data:
1148                         Value = self.data[Key]
1149                         if Value is not None: break
1150             else:
1151                 if FirstKey in self.data:
1152                     Value = self.data[FirstKey]
1153                 elif self._Wildcard in self.data:
1154                     Value = self.data[self._Wildcard]
1155         return Value
1156
1157     def _GetAllValues(self, FirstKey, RestKeys):
1158         Value = []
1159         if self._Level_ > 1:
1160             if FirstKey == self._Wildcard:
1161                 for Key in self.data:
1162                     Value += self.data[Key][RestKeys]
1163             else:
1164                 if FirstKey in self.data:
1165                     Value += self.data[FirstKey][RestKeys]
1166                 if self._Wildcard in self.data:
1167                     Value += self.data[self._Wildcard][RestKeys]
1168         else:
1169             if FirstKey == self._Wildcard:
1170                 for Key in self.data:
1171                     Value.append(self.data[Key])
1172             else:
1173                 if FirstKey in self.data:
1174                     Value.append(self.data[FirstKey])
1175                 if self._Wildcard in self.data:
1176                     Value.append(self.data[self._Wildcard])
1177         return Value
1178
1179     ## []= operator
1180     def __setitem__(self, key, value):
1181         KeyType = type(key)
1182         RestKeys = None
1183         if KeyType == self._ListType or KeyType == self._TupleType:
1184             FirstKey = key[0]
1185             if len(key) > 1:
1186                 RestKeys = key[1:]
1187             else:
1188                 RestKeys = [self._Wildcard for i in range(0, self._Level_ - 1)]
1189         else:
1190             FirstKey = key
1191             if self._Level_ > 1:
1192                 RestKeys = [self._Wildcard for i in range(0, self._Level_ - 1)]
1193
1194         if FirstKey in self._ValidWildcardList:
1195             FirstKey = self._Wildcard
1196
1197         if FirstKey not in self.data and self._Level_ > 0:
1198             self.data[FirstKey] = tdict(self._Single_, self._Level_ - 1)
1199
1200         if self._Level_ > 1:
1201             self.data[FirstKey][RestKeys] = value
1202         else:
1203             self.data[FirstKey] = value
1204
1205     def SetGreedyMode(self):
1206         self._Single_ = False
1207         if self._Level_ > 1:
1208             for Key in self.data:
1209                 self.data[Key].SetGreedyMode()
1210
1211     def SetSingleMode(self):
1212         self._Single_ = True
1213         if self._Level_ > 1:
1214             for Key in self.data:
1215                 self.data[Key].SetSingleMode()
1216
1217     def GetKeys(self, KeyIndex=0):
1218         assert KeyIndex >= 0
1219         if KeyIndex == 0:
1220             return set(self.data.keys())
1221         else:
1222             keys = set()
1223             for Key in self.data:
1224                 keys |= self.data[Key].GetKeys(KeyIndex - 1)
1225             return keys
1226
1227 def IsFieldValueAnArray (Value):
1228     Value = Value.strip()
1229     if Value.startswith(TAB_GUID) and Value.endswith(')'):
1230         return True
1231     if Value.startswith('L"') and Value.endswith('"')  and len(list(Value[2:-1])) > 1:
1232         return True
1233     if Value[0] == '"' and Value[-1] == '"' and len(list(Value[1:-1])) > 1:
1234         return True
1235     if Value[0] == '{' and Value[-1] == '}':
1236         return True
1237     if Value.startswith("L'") and Value.endswith("'") and len(list(Value[2:-1])) > 1:
1238         return True
1239     if Value[0] == "'" and Value[-1] == "'" and len(list(Value[1:-1])) > 1:
1240         return True
1241     return False
1242
1243 def AnalyzePcdExpression(Setting):
1244     Setting = Setting.strip()
1245     # There might be escaped quote in a string: \", \\\" , \', \\\'
1246     Data = Setting
1247     # There might be '|' in string and in ( ... | ... ), replace it with '-'
1248     NewStr = ''
1249     InSingleQuoteStr = False
1250     InDoubleQuoteStr = False
1251     Pair = 0
1252     for Index, ch in enumerate(Data):
1253         if ch == '"' and not InSingleQuoteStr:
1254             if Data[Index - 1] != '\\':
1255                 InDoubleQuoteStr = not InDoubleQuoteStr
1256         elif ch == "'" and not InDoubleQuoteStr:
1257             if Data[Index - 1] != '\\':
1258                 InSingleQuoteStr = not InSingleQuoteStr
1259         elif ch == '(' and not (InSingleQuoteStr or InDoubleQuoteStr):
1260             Pair += 1
1261         elif ch == ')' and not (InSingleQuoteStr or InDoubleQuoteStr):
1262             Pair -= 1
1263
1264         if (Pair > 0 or InSingleQuoteStr or InDoubleQuoteStr) and ch == TAB_VALUE_SPLIT:
1265             NewStr += '-'
1266         else:
1267             NewStr += ch
1268     FieldList = []
1269     StartPos = 0
1270     while True:
1271         Pos = NewStr.find(TAB_VALUE_SPLIT, StartPos)
1272         if Pos < 0:
1273             FieldList.append(Setting[StartPos:].strip())
1274             break
1275         FieldList.append(Setting[StartPos:Pos].strip())
1276         StartPos = Pos + 1
1277
1278     return FieldList
1279
1280 def ParseDevPathValue (Value):
1281     if '\\' in Value:
1282         Value.replace('\\', '/').replace(' ', '')
1283
1284     Cmd = 'DevicePath ' + '"' + Value + '"'
1285     try:
1286         p = subprocess.Popen(Cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
1287         out, err = p.communicate()
1288     except Exception as X:
1289         raise BadExpression("DevicePath: %s" % (str(X)) )
1290     finally:
1291         subprocess._cleanup()
1292         p.stdout.close()
1293         p.stderr.close()
1294     if err:
1295         raise BadExpression("DevicePath: %s" % str(err))
1296     Size = len(out.split())
1297     out = ','.join(out.split())
1298     return '{' + out + '}', Size
1299
1300 def ParseFieldValue (Value):
1301     if isinstance(Value, type(0)):
1302         return Value, (Value.bit_length() + 7) / 8
1303     if not isinstance(Value, type('')):
1304         raise BadExpression('Type %s is %s' %(Value, type(Value)))
1305     Value = Value.strip()
1306     if Value.startswith(TAB_UINT8) and Value.endswith(')'):
1307         Value, Size = ParseFieldValue(Value.split('(', 1)[1][:-1])
1308         if Size > 1:
1309             raise BadExpression('Value (%s) Size larger than %d' %(Value, Size))
1310         return Value, 1
1311     if Value.startswith(TAB_UINT16) and Value.endswith(')'):
1312         Value, Size = ParseFieldValue(Value.split('(', 1)[1][:-1])
1313         if Size > 2:
1314             raise BadExpression('Value (%s) Size larger than %d' %(Value, Size))
1315         return Value, 2
1316     if Value.startswith(TAB_UINT32) and Value.endswith(')'):
1317         Value, Size = ParseFieldValue(Value.split('(', 1)[1][:-1])
1318         if Size > 4:
1319             raise BadExpression('Value (%s) Size larger than %d' %(Value, Size))
1320         return Value, 4
1321     if Value.startswith(TAB_UINT64) and Value.endswith(')'):
1322         Value, Size = ParseFieldValue(Value.split('(', 1)[1][:-1])
1323         if Size > 8:
1324             raise BadExpression('Value (%s) Size larger than %d' % (Value, Size))
1325         return Value, 8
1326     if Value.startswith(TAB_GUID) and Value.endswith(')'):
1327         Value = Value.split('(', 1)[1][:-1].strip()
1328         if Value[0] == '{' and Value[-1] == '}':
1329             TmpValue = GuidStructureStringToGuidString(Value)
1330             if len(TmpValue) == 0:
1331                 raise BadExpression("Invalid GUID value string %s" % Value)
1332             Value = TmpValue
1333         if Value[0] == '"' and Value[-1] == '"':
1334             Value = Value[1:-1]
1335         try:
1336             Value = "'" + uuid.UUID(Value).get_bytes_le() + "'"
1337         except ValueError as Message:
1338             raise BadExpression(Message)
1339         Value, Size = ParseFieldValue(Value)
1340         return Value, 16
1341     if Value.startswith('L"') and Value.endswith('"'):
1342         # Unicode String
1343         # translate escape character
1344         Value = Value[1:]
1345         try:
1346             Value = eval(Value)
1347         except:
1348             Value = Value[1:-1]
1349         List = list(Value)
1350         List.reverse()
1351         Value = 0
1352         for Char in List:
1353             Value = (Value << 16) | ord(Char)
1354         return Value, (len(List) + 1) * 2
1355     if Value.startswith('"') and Value.endswith('"'):
1356         # ASCII String
1357         # translate escape character
1358         try:
1359             Value = eval(Value)
1360         except:
1361             Value = Value[1:-1]
1362         List = list(Value)
1363         List.reverse()
1364         Value = 0
1365         for Char in List:
1366             Value = (Value << 8) | ord(Char)
1367         return Value, len(List) + 1
1368     if Value.startswith("L'") and Value.endswith("'"):
1369         # Unicode Character Constant
1370         # translate escape character
1371         Value = Value[1:]
1372         try:
1373             Value = eval(Value)
1374         except:
1375             Value = Value[1:-1]
1376         List = list(Value)
1377         if len(List) == 0:
1378             raise BadExpression('Length %s is %s' % (Value, len(List)))
1379         List.reverse()
1380         Value = 0
1381         for Char in List:
1382             Value = (Value << 16) | ord(Char)
1383         return Value, len(List) * 2
1384     if Value.startswith("'") and Value.endswith("'"):
1385         # Character constant
1386         # translate escape character
1387         try:
1388             Value = eval(Value)
1389         except:
1390             Value = Value[1:-1]
1391         List = list(Value)
1392         if len(List) == 0:
1393             raise BadExpression('Length %s is %s' % (Value, len(List)))
1394         List.reverse()
1395         Value = 0
1396         for Char in List:
1397             Value = (Value << 8) | ord(Char)
1398         return Value, len(List)
1399     if Value.startswith('{') and Value.endswith('}'):
1400         # Byte array
1401         Value = Value[1:-1]
1402         List = [Item.strip() for Item in Value.split(',')]
1403         List.reverse()
1404         Value = 0
1405         RetSize = 0
1406         for Item in List:
1407             ItemValue, Size = ParseFieldValue(Item)
1408             RetSize += Size
1409             for I in range(Size):
1410                 Value = (Value << 8) | ((ItemValue >> 8 * I) & 0xff)
1411         return Value, RetSize
1412     if Value.startswith('DEVICE_PATH(') and Value.endswith(')'):
1413         Value = Value.replace("DEVICE_PATH(", '').rstrip(')')
1414         Value = Value.strip().strip('"')
1415         return ParseDevPathValue(Value)
1416     if Value.lower().startswith('0x'):
1417         Value = int(Value, 16)
1418         if Value == 0:
1419             return 0, 1
1420         return Value, (Value.bit_length() + 7) / 8
1421     if Value[0].isdigit():
1422         Value = int(Value, 10)
1423         if Value == 0:
1424             return 0, 1
1425         return Value, (Value.bit_length() + 7) / 8
1426     if Value.lower() == 'true':
1427         return 1, 1
1428     if Value.lower() == 'false':
1429         return 0, 1
1430     return Value, 1
1431
1432 ## AnalyzeDscPcd
1433 #
1434 #  Analyze DSC PCD value, since there is no data type info in DSC
1435 #  This function is used to match functions (AnalyzePcdData) used for retrieving PCD value from database
1436 #  1. Feature flag: TokenSpace.PcdCName|PcdValue
1437 #  2. Fix and Patch:TokenSpace.PcdCName|PcdValue[|VOID*[|MaxSize]]
1438 #  3. Dynamic default:
1439 #     TokenSpace.PcdCName|PcdValue[|VOID*[|MaxSize]]
1440 #     TokenSpace.PcdCName|PcdValue
1441 #  4. Dynamic VPD:
1442 #     TokenSpace.PcdCName|VpdOffset[|VpdValue]
1443 #     TokenSpace.PcdCName|VpdOffset[|MaxSize[|VpdValue]]
1444 #  5. Dynamic HII:
1445 #     TokenSpace.PcdCName|HiiString|VariableGuid|VariableOffset[|HiiValue]
1446 #  PCD value needs to be located in such kind of string, and the PCD value might be an expression in which
1447 #    there might have "|" operator, also in string value.
1448 #
1449 #  @param Setting: String contain information described above with "TokenSpace.PcdCName|" stripped
1450 #  @param PcdType: PCD type: feature, fixed, dynamic default VPD HII
1451 #  @param DataType: The datum type of PCD: VOID*, UNIT, BOOL
1452 #  @retval:
1453 #    ValueList: A List contain fields described above
1454 #    IsValid:   True if conforming EBNF, otherwise False
1455 #    Index:     The index where PcdValue is in ValueList
1456 #
1457 def AnalyzeDscPcd(Setting, PcdType, DataType=''):
1458     FieldList = AnalyzePcdExpression(Setting)
1459
1460     IsValid = True
1461     if PcdType in (MODEL_PCD_FIXED_AT_BUILD, MODEL_PCD_PATCHABLE_IN_MODULE, MODEL_PCD_DYNAMIC_DEFAULT, MODEL_PCD_DYNAMIC_EX_DEFAULT):
1462         Value = FieldList[0]
1463         Size = ''
1464         if len(FieldList) > 1 and FieldList[1]:
1465             DataType = FieldList[1]
1466             if FieldList[1] != TAB_VOID:
1467                 IsValid = False
1468         if len(FieldList) > 2:
1469             Size = FieldList[2]
1470         if IsValid:
1471             if DataType == "":
1472                 IsValid = (len(FieldList) <= 1)
1473             else:
1474                 IsValid = (len(FieldList) <= 3)
1475
1476         if Size:
1477             try:
1478                 int(Size, 16) if Size.upper().startswith("0X") else int(Size)
1479             except:
1480                 IsValid = False
1481                 Size = -1
1482         return [str(Value), DataType, str(Size)], IsValid, 0
1483     elif PcdType == MODEL_PCD_FEATURE_FLAG:
1484         Value = FieldList[0]
1485         Size = ''
1486         IsValid = (len(FieldList) <= 1)
1487         return [Value, DataType, str(Size)], IsValid, 0
1488     elif PcdType in (MODEL_PCD_DYNAMIC_VPD, MODEL_PCD_DYNAMIC_EX_VPD):
1489         VpdOffset = FieldList[0]
1490         Value = Size = ''
1491         if not DataType == TAB_VOID:
1492             if len(FieldList) > 1:
1493                 Value = FieldList[1]
1494         else:
1495             if len(FieldList) > 1:
1496                 Size = FieldList[1]
1497             if len(FieldList) > 2:
1498                 Value = FieldList[2]
1499         if DataType == "":
1500             IsValid = (len(FieldList) <= 1)
1501         else:
1502             IsValid = (len(FieldList) <= 3)
1503         if Size:
1504             try:
1505                 int(Size, 16) if Size.upper().startswith("0X") else int(Size)
1506             except:
1507                 IsValid = False
1508                 Size = -1
1509         return [VpdOffset, str(Size), Value], IsValid, 2
1510     elif PcdType in (MODEL_PCD_DYNAMIC_HII, MODEL_PCD_DYNAMIC_EX_HII):
1511         HiiString = FieldList[0]
1512         Guid = Offset = Value = Attribute = ''
1513         if len(FieldList) > 1:
1514             Guid = FieldList[1]
1515         if len(FieldList) > 2:
1516             Offset = FieldList[2]
1517         if len(FieldList) > 3:
1518             Value = FieldList[3]
1519         if len(FieldList) > 4:
1520             Attribute = FieldList[4]
1521         IsValid = (3 <= len(FieldList) <= 5)
1522         return [HiiString, Guid, Offset, Value, Attribute], IsValid, 3
1523     return [], False, 0
1524
1525 ## AnalyzePcdData
1526 #
1527 #  Analyze the pcd Value, Datum type and TokenNumber.
1528 #  Used to avoid split issue while the value string contain "|" character
1529 #
1530 #  @param[in] Setting:  A String contain value/datum type/token number information;
1531 #
1532 #  @retval   ValueList: A List contain value, datum type and toke number.
1533 #
1534 def AnalyzePcdData(Setting):
1535     ValueList = ['', '', '']
1536
1537     ValueRe = re.compile(r'^\s*L?\".*\|.*\"')
1538     PtrValue = ValueRe.findall(Setting)
1539
1540     ValueUpdateFlag = False
1541
1542     if len(PtrValue) >= 1:
1543         Setting = re.sub(ValueRe, '', Setting)
1544         ValueUpdateFlag = True
1545
1546     TokenList = Setting.split(TAB_VALUE_SPLIT)
1547     ValueList[0:len(TokenList)] = TokenList
1548
1549     if ValueUpdateFlag:
1550         ValueList[0] = PtrValue[0]
1551
1552     return ValueList
1553
1554 ## check format of PCD value against its the datum type
1555 #
1556 # For PCD value setting
1557 #
1558 def CheckPcdDatum(Type, Value):
1559     if Type == TAB_VOID:
1560         ValueRe = re.compile(r'\s*L?\".*\"\s*$')
1561         if not (((Value.startswith('L"') or Value.startswith('"')) and Value.endswith('"'))
1562                 or (Value.startswith('{') and Value.endswith('}')) or (Value.startswith("L'") or Value.startswith("'") and Value.endswith("'"))
1563                ):
1564             return False, "Invalid value [%s] of type [%s]; must be in the form of {...} for array"\
1565                           ", \"...\" or \'...\' for string, L\"...\" or L\'...\' for unicode string" % (Value, Type)
1566         elif ValueRe.match(Value):
1567             # Check the chars in UnicodeString or CString is printable
1568             if Value.startswith("L"):
1569                 Value = Value[2:-1]
1570             else:
1571                 Value = Value[1:-1]
1572             Printset = set(string.printable)
1573             Printset.remove(TAB_PRINTCHAR_VT)
1574             Printset.add(TAB_PRINTCHAR_BS)
1575             Printset.add(TAB_PRINTCHAR_NUL)
1576             if not set(Value).issubset(Printset):
1577                 PrintList = sorted(Printset)
1578                 return False, "Invalid PCD string value of type [%s]; must be printable chars %s." % (Type, PrintList)
1579     elif Type == 'BOOLEAN':
1580         if Value not in ['TRUE', 'True', 'true', '0x1', '0x01', '1', 'FALSE', 'False', 'false', '0x0', '0x00', '0']:
1581             return False, "Invalid value [%s] of type [%s]; must be one of TRUE, True, true, 0x1, 0x01, 1"\
1582                           ", FALSE, False, false, 0x0, 0x00, 0" % (Value, Type)
1583     elif Type in [TAB_UINT8, TAB_UINT16, TAB_UINT32, TAB_UINT64]:
1584         try:
1585             Value = long(Value, 0)
1586         except:
1587             return False, "Invalid value [%s] of type [%s];"\
1588                           " must be a hexadecimal, decimal or octal in C language format." % (Value, Type)
1589     else:
1590         return True, "StructurePcd"
1591
1592     return True, ""
1593
1594 ## Split command line option string to list
1595 #
1596 # subprocess.Popen needs the args to be a sequence. Otherwise there's problem
1597 # in non-windows platform to launch command
1598 #
1599 def SplitOption(OptionString):
1600     OptionList = []
1601     LastChar = " "
1602     OptionStart = 0
1603     QuotationMark = ""
1604     for Index in range(0, len(OptionString)):
1605         CurrentChar = OptionString[Index]
1606         if CurrentChar in ['"', "'"]:
1607             if QuotationMark == CurrentChar:
1608                 QuotationMark = ""
1609             elif QuotationMark == "":
1610                 QuotationMark = CurrentChar
1611             continue
1612         elif QuotationMark:
1613             continue
1614
1615         if CurrentChar in ["/", "-"] and LastChar in [" ", "\t", "\r", "\n"]:
1616             if Index > OptionStart:
1617                 OptionList.append(OptionString[OptionStart:Index - 1])
1618             OptionStart = Index
1619         LastChar = CurrentChar
1620     OptionList.append(OptionString[OptionStart:])
1621     return OptionList
1622
1623 def CommonPath(PathList):
1624     P1 = min(PathList).split(os.path.sep)
1625     P2 = max(PathList).split(os.path.sep)
1626     for Index in xrange(min(len(P1), len(P2))):
1627         if P1[Index] != P2[Index]:
1628             return os.path.sep.join(P1[:Index])
1629     return os.path.sep.join(P1)
1630
1631 #
1632 # Convert string to C format array
1633 #
1634 def ConvertStringToByteArray(Value):
1635     Value = Value.strip()
1636     if not Value:
1637         return None
1638     if Value[0] == '{':
1639         if not Value.endswith('}'):
1640             return None
1641         Value = Value.replace(' ', '').replace('{', '').replace('}', '')
1642         ValFields = Value.split(',')
1643         try:
1644             for Index in range(len(ValFields)):
1645                 ValFields[Index] = str(int(ValFields[Index], 0))
1646         except ValueError:
1647             return None
1648         Value = '{' + ','.join(ValFields) + '}'
1649         return Value
1650
1651     Unicode = False
1652     if Value.startswith('L"'):
1653         if not Value.endswith('"'):
1654             return None
1655         Value = Value[1:]
1656         Unicode = True
1657     elif not Value.startswith('"') or not Value.endswith('"'):
1658         return None
1659
1660     Value = eval(Value)         # translate escape character
1661     NewValue = '{'
1662     for Index in range(0, len(Value)):
1663         if Unicode:
1664             NewValue = NewValue + str(ord(Value[Index]) % 0x10000) + ','
1665         else:
1666             NewValue = NewValue + str(ord(Value[Index]) % 0x100) + ','
1667     Value = NewValue + '0}'
1668     return Value
1669
1670 class PathClass(object):
1671     def __init__(self, File='', Root='', AlterRoot='', Type='', IsBinary=False,
1672                  Arch='COMMON', ToolChainFamily='', Target='', TagName='', ToolCode=''):
1673         self.Arch = Arch
1674         self.File = str(File)
1675         if os.path.isabs(self.File):
1676             self.Root = ''
1677             self.AlterRoot = ''
1678         else:
1679             self.Root = str(Root)
1680             self.AlterRoot = str(AlterRoot)
1681
1682         # Remove any '.' and '..' in path
1683         if self.Root:
1684             self.Root = mws.getWs(self.Root, self.File)
1685             self.Path = os.path.normpath(os.path.join(self.Root, self.File))
1686             self.Root = os.path.normpath(CommonPath([self.Root, self.Path]))
1687             # eliminate the side-effect of 'C:'
1688             if self.Root[-1] == ':':
1689                 self.Root += os.path.sep
1690             # file path should not start with path separator
1691             if self.Root[-1] == os.path.sep:
1692                 self.File = self.Path[len(self.Root):]
1693             else:
1694                 self.File = self.Path[len(self.Root) + 1:]
1695         else:
1696             self.Path = os.path.normpath(self.File)
1697
1698         self.SubDir, self.Name = os.path.split(self.File)
1699         self.BaseName, self.Ext = os.path.splitext(self.Name)
1700
1701         if self.Root:
1702             if self.SubDir:
1703                 self.Dir = os.path.join(self.Root, self.SubDir)
1704             else:
1705                 self.Dir = self.Root
1706         else:
1707             self.Dir = self.SubDir
1708
1709         if IsBinary:
1710             self.Type = Type
1711         else:
1712             self.Type = self.Ext.lower()
1713
1714         self.IsBinary = IsBinary
1715         self.Target = Target
1716         self.TagName = TagName
1717         self.ToolCode = ToolCode
1718         self.ToolChainFamily = ToolChainFamily
1719
1720         self._Key = None
1721
1722     ## Convert the object of this class to a string
1723     #
1724     #  Convert member Path of the class to a string
1725     #
1726     #  @retval string Formatted String
1727     #
1728     def __str__(self):
1729         return self.Path
1730
1731     ## Override __eq__ function
1732     #
1733     # Check whether PathClass are the same
1734     #
1735     # @retval False The two PathClass are different
1736     # @retval True  The two PathClass are the same
1737     #
1738     def __eq__(self, Other):
1739         if isinstance(Other, type(self)):
1740             return self.Path == Other.Path
1741         else:
1742             return self.Path == str(Other)
1743
1744     ## Override __cmp__ function
1745     #
1746     # Customize the comparsion operation of two PathClass
1747     #
1748     # @retval 0     The two PathClass are different
1749     # @retval -1    The first PathClass is less than the second PathClass
1750     # @retval 1     The first PathClass is Bigger than the second PathClass
1751     def __cmp__(self, Other):
1752         if isinstance(Other, type(self)):
1753             OtherKey = Other.Path
1754         else:
1755             OtherKey = str(Other)
1756
1757         SelfKey = self.Path
1758         if SelfKey == OtherKey:
1759             return 0
1760         elif SelfKey > OtherKey:
1761             return 1
1762         else:
1763             return -1
1764
1765     ## Override __hash__ function
1766     #
1767     # Use Path as key in hash table
1768     #
1769     # @retval string Key for hash table
1770     #
1771     def __hash__(self):
1772         return hash(self.Path)
1773
1774     def _GetFileKey(self):
1775         if self._Key is None:
1776             self._Key = self.Path.upper()   # + self.ToolChainFamily + self.TagName + self.ToolCode + self.Target
1777         return self._Key
1778
1779     def _GetTimeStamp(self):
1780         return os.stat(self.Path)[8]
1781
1782     def Validate(self, Type='', CaseSensitive=True):
1783         if GlobalData.gCaseInsensitive:
1784             CaseSensitive = False
1785         if Type and Type.lower() != self.Type:
1786             return FILE_TYPE_MISMATCH, '%s (expect %s but got %s)' % (self.File, Type, self.Type)
1787
1788         RealFile, RealRoot = RealPath2(self.File, self.Root, self.AlterRoot)
1789         if not RealRoot and not RealFile:
1790             RealFile = self.File
1791             if self.AlterRoot:
1792                 RealFile = os.path.join(self.AlterRoot, self.File)
1793             elif self.Root:
1794                 RealFile = os.path.join(self.Root, self.File)
1795             if len (mws.getPkgPath()) == 0:
1796                 return FILE_NOT_FOUND, os.path.join(self.AlterRoot, RealFile)
1797             else:
1798                 return FILE_NOT_FOUND, "%s is not found in packages path:\n\t%s" % (self.File, '\n\t'.join(mws.getPkgPath()))
1799
1800         ErrorCode = 0
1801         ErrorInfo = ''
1802         if RealRoot != self.Root or RealFile != self.File:
1803             if CaseSensitive and (RealFile != self.File or (RealRoot != self.Root and RealRoot != self.AlterRoot)):
1804                 ErrorCode = FILE_CASE_MISMATCH
1805                 ErrorInfo = self.File + '\n\t' + RealFile + " [in file system]"
1806
1807             self.SubDir, self.Name = os.path.split(RealFile)
1808             self.BaseName, self.Ext = os.path.splitext(self.Name)
1809             if self.SubDir:
1810                 self.Dir = os.path.join(RealRoot, self.SubDir)
1811             else:
1812                 self.Dir = RealRoot
1813             self.File = RealFile
1814             self.Root = RealRoot
1815             self.Path = os.path.join(RealRoot, RealFile)
1816         return ErrorCode, ErrorInfo
1817
1818     Key = property(_GetFileKey)
1819     TimeStamp = property(_GetTimeStamp)
1820
1821 ## Parse PE image to get the required PE informaion.
1822 #
1823 class PeImageClass():
1824     ## Constructor
1825     #
1826     #   @param  File FilePath of PeImage
1827     #
1828     def __init__(self, PeFile):
1829         self.FileName   = PeFile
1830         self.IsValid    = False
1831         self.Size       = 0
1832         self.EntryPoint = 0
1833         self.SectionAlignment  = 0
1834         self.SectionHeaderList = []
1835         self.ErrorInfo = ''
1836         try:
1837             PeObject = open(PeFile, 'rb')
1838         except:
1839             self.ErrorInfo = self.FileName + ' can not be found\n'
1840             return
1841         # Read DOS header
1842         ByteArray = array.array('B')
1843         ByteArray.fromfile(PeObject, 0x3E)
1844         ByteList = ByteArray.tolist()
1845         # DOS signature should be 'MZ'
1846         if self._ByteListToStr (ByteList[0x0:0x2]) != 'MZ':
1847             self.ErrorInfo = self.FileName + ' has no valid DOS signature MZ'
1848             return
1849
1850         # Read 4 byte PE Signature
1851         PeOffset = self._ByteListToInt(ByteList[0x3C:0x3E])
1852         PeObject.seek(PeOffset)
1853         ByteArray = array.array('B')
1854         ByteArray.fromfile(PeObject, 4)
1855         # PE signature should be 'PE\0\0'
1856         if ByteArray.tostring() != 'PE\0\0':
1857             self.ErrorInfo = self.FileName + ' has no valid PE signature PE00'
1858             return
1859
1860         # Read PE file header
1861         ByteArray = array.array('B')
1862         ByteArray.fromfile(PeObject, 0x14)
1863         ByteList = ByteArray.tolist()
1864         SecNumber = self._ByteListToInt(ByteList[0x2:0x4])
1865         if SecNumber == 0:
1866             self.ErrorInfo = self.FileName + ' has no section header'
1867             return
1868
1869         # Read PE optional header
1870         OptionalHeaderSize = self._ByteListToInt(ByteArray[0x10:0x12])
1871         ByteArray = array.array('B')
1872         ByteArray.fromfile(PeObject, OptionalHeaderSize)
1873         ByteList = ByteArray.tolist()
1874         self.EntryPoint       = self._ByteListToInt(ByteList[0x10:0x14])
1875         self.SectionAlignment = self._ByteListToInt(ByteList[0x20:0x24])
1876         self.Size             = self._ByteListToInt(ByteList[0x38:0x3C])
1877
1878         # Read each Section Header
1879         for Index in range(SecNumber):
1880             ByteArray = array.array('B')
1881             ByteArray.fromfile(PeObject, 0x28)
1882             ByteList = ByteArray.tolist()
1883             SecName  = self._ByteListToStr(ByteList[0:8])
1884             SecVirtualSize = self._ByteListToInt(ByteList[8:12])
1885             SecRawAddress  = self._ByteListToInt(ByteList[20:24])
1886             SecVirtualAddress = self._ByteListToInt(ByteList[12:16])
1887             self.SectionHeaderList.append((SecName, SecVirtualAddress, SecRawAddress, SecVirtualSize))
1888         self.IsValid = True
1889         PeObject.close()
1890
1891     def _ByteListToStr(self, ByteList):
1892         String = ''
1893         for index in range(len(ByteList)):
1894             if ByteList[index] == 0:
1895                 break
1896             String += chr(ByteList[index])
1897         return String
1898
1899     def _ByteListToInt(self, ByteList):
1900         Value = 0
1901         for index in range(len(ByteList) - 1, -1, -1):
1902             Value = (Value << 8) | int(ByteList[index])
1903         return Value
1904
1905 class DefaultStore():
1906     def __init__(self, DefaultStores ):
1907
1908         self.DefaultStores = DefaultStores
1909     def DefaultStoreID(self, DefaultStoreName):
1910         for key, value in self.DefaultStores.items():
1911             if value == DefaultStoreName:
1912                 return key
1913         return None
1914     def GetDefaultDefault(self):
1915         if not self.DefaultStores or "0" in self.DefaultStores:
1916             return "0", TAB_DEFAULT_STORES_DEFAULT
1917         else:
1918             minvalue = min(int(value_str) for value_str in self.DefaultStores)
1919             return (str(minvalue), self.DefaultStores[str(minvalue)])
1920     def GetMin(self, DefaultSIdList):
1921         if not DefaultSIdList:
1922             return TAB_DEFAULT_STORES_DEFAULT
1923         storeidset = {storeid for storeid, storename in self.DefaultStores.values() if storename in DefaultSIdList}
1924         if not storeidset:
1925             return ""
1926         minid = min(storeidset )
1927         for sid, name in self.DefaultStores.values():
1928             if sid == minid:
1929                 return name
1930 class SkuClass():
1931
1932     DEFAULT = 0
1933     SINGLE = 1
1934     MULTIPLE =2
1935
1936     def __init__(self,SkuIdentifier='', SkuIds=None):
1937         if SkuIds is None:
1938             SkuIds = {}
1939
1940         for SkuName in SkuIds:
1941             SkuId = SkuIds[SkuName][0]
1942             skuid_num = int(SkuId, 16) if SkuId.upper().startswith("0X") else int(SkuId)
1943             if skuid_num > 0xFFFFFFFFFFFFFFFF:
1944                 EdkLogger.error("build", PARAMETER_INVALID,
1945                             ExtraData = "SKU-ID [%s] value %s exceeds the max value of UINT64"
1946                                       % (SkuName, SkuId))
1947
1948         self.AvailableSkuIds = sdict()
1949         self.SkuIdSet = []
1950         self.SkuIdNumberSet = []
1951         self.SkuData = SkuIds
1952         self.__SkuInherit = {}
1953         self.__SkuIdentifier = SkuIdentifier
1954         if SkuIdentifier == '' or SkuIdentifier is None:
1955             self.SkuIdSet = ['DEFAULT']
1956             self.SkuIdNumberSet = ['0U']
1957         elif SkuIdentifier == 'ALL':
1958             self.SkuIdSet = SkuIds.keys()
1959             self.SkuIdNumberSet = [num[0].strip() + 'U' for num in SkuIds.values()]
1960         else:
1961             r = SkuIdentifier.split('|')
1962             self.SkuIdSet=[(r[k].strip()).upper() for k in range(len(r))]
1963             k = None
1964             try:
1965                 self.SkuIdNumberSet = [SkuIds[k][0].strip() + 'U' for k in self.SkuIdSet]
1966             except Exception:
1967                 EdkLogger.error("build", PARAMETER_INVALID,
1968                             ExtraData = "SKU-ID [%s] is not supported by the platform. [Valid SKU-ID: %s]"
1969                                       % (k, " | ".join(SkuIds.keys())))
1970         for each in self.SkuIdSet:
1971             if each in SkuIds:
1972                 self.AvailableSkuIds[each] = SkuIds[each][0]
1973             else:
1974                 EdkLogger.error("build", PARAMETER_INVALID,
1975                             ExtraData="SKU-ID [%s] is not supported by the platform. [Valid SKU-ID: %s]"
1976                                       % (each, " | ".join(SkuIds.keys())))
1977         if self.SkuUsageType != self.SINGLE:
1978             self.AvailableSkuIds.update({'DEFAULT':0, 'COMMON':0})
1979         if self.SkuIdSet:
1980             GlobalData.gSkuids = (self.SkuIdSet)
1981             if 'COMMON' in GlobalData.gSkuids:
1982                 GlobalData.gSkuids.remove('COMMON')
1983             if self.SkuUsageType == self.SINGLE:
1984                 if len(GlobalData.gSkuids) != 1:
1985                     if 'DEFAULT' in GlobalData.gSkuids:
1986                         GlobalData.gSkuids.remove('DEFAULT')
1987             if GlobalData.gSkuids:
1988                 GlobalData.gSkuids.sort()
1989
1990     def GetNextSkuId(self, skuname):
1991         if not self.__SkuInherit:
1992             self.__SkuInherit = {}
1993             for item in self.SkuData.values():
1994                 self.__SkuInherit[item[1]]=item[2] if item[2] else "DEFAULT"
1995         return self.__SkuInherit.get(skuname, "DEFAULT")
1996
1997     def GetSkuChain(self, sku):
1998         if sku == "DEFAULT":
1999             return ["DEFAULT"]
2000         skulist = [sku]
2001         nextsku = sku
2002         while True:
2003             nextsku = self.GetNextSkuId(nextsku)
2004             skulist.append(nextsku)
2005             if nextsku == "DEFAULT":
2006                 break
2007         skulist.reverse()
2008         return skulist
2009     def SkuOverrideOrder(self):
2010         skuorderset = []
2011         for skuname in self.SkuIdSet:
2012             skuorderset.append(self.GetSkuChain(skuname))
2013
2014         skuorder = []
2015         for index in range(max(len(item) for item in skuorderset)):
2016             for subset in skuorderset:
2017                 if index > len(subset)-1:
2018                     continue
2019                 if subset[index] in skuorder:
2020                     continue
2021                 skuorder.append(subset[index])
2022
2023         return skuorder
2024
2025     def __SkuUsageType(self):
2026
2027         if self.__SkuIdentifier.upper() == "ALL":
2028             return SkuClass.MULTIPLE
2029
2030         if len(self.SkuIdSet) == 1:
2031             if self.SkuIdSet[0] == 'DEFAULT':
2032                 return SkuClass.DEFAULT
2033             else:
2034                 return SkuClass.SINGLE
2035         elif len(self.SkuIdSet) == 2:
2036             if 'DEFAULT' in self.SkuIdSet:
2037                 return SkuClass.SINGLE
2038             else:
2039                 return SkuClass.MULTIPLE
2040         else:
2041             return SkuClass.MULTIPLE
2042     def DumpSkuIdArrary(self):
2043
2044         ArrayStrList = []
2045         if self.SkuUsageType == SkuClass.SINGLE:
2046             ArrayStr = "{0x0}"
2047         else:
2048             for skuname in self.AvailableSkuIds:
2049                 if skuname == "COMMON":
2050                     continue
2051                 while skuname != "DEFAULT":
2052                     ArrayStrList.append(hex(int(self.AvailableSkuIds[skuname])))
2053                     skuname = self.GetNextSkuId(skuname)
2054                 ArrayStrList.append("0x0")
2055             ArrayStr = "{" + ",".join(ArrayStrList) +  "}"
2056         return ArrayStr
2057     def __GetAvailableSkuIds(self):
2058         return self.AvailableSkuIds
2059
2060     def __GetSystemSkuID(self):
2061         if self.__SkuUsageType() == SkuClass.SINGLE:
2062             if len(self.SkuIdSet) == 1:
2063                 return self.SkuIdSet[0]
2064             else:
2065                 return self.SkuIdSet[0] if self.SkuIdSet[0] != 'DEFAULT' else self.SkuIdSet[1]
2066         else:
2067             return 'DEFAULT'
2068     def __GetAvailableSkuIdNumber(self):
2069         return self.SkuIdNumberSet
2070     SystemSkuId = property(__GetSystemSkuID)
2071     AvailableSkuIdSet = property(__GetAvailableSkuIds)
2072     SkuUsageType = property(__SkuUsageType)
2073     AvailableSkuIdNumSet = property(__GetAvailableSkuIdNumber)
2074
2075 #
2076 # Pack a registry format GUID
2077 #
2078 def PackRegistryFormatGuid(Guid):
2079     return PackGUID(Guid.split('-'))
2080
2081 ##  Get the integer value from string like "14U" or integer like 2
2082 #
2083 #   @param      Input   The object that may be either a integer value or a string
2084 #
2085 #   @retval     Value    The integer value that the input represents
2086 #
2087 def GetIntegerValue(Input):
2088     if type(Input) in (int, long):
2089         return Input
2090     String = Input
2091     if String.endswith("U"):
2092         String = String[:-1]
2093     if String.endswith("ULL"):
2094         String = String[:-3]
2095     if String.endswith("LL"):
2096         String = String[:-2]
2097
2098     if String.startswith("0x") or String.startswith("0X"):
2099         return int(String, 16)
2100     elif String == '':
2101         return 0
2102     else:
2103         return int(String)
2104
2105 #
2106 # Pack a GUID (registry format) list into a buffer and return it
2107 #
2108 def PackGUID(Guid):
2109     return pack(PACK_PATTERN_GUID,
2110                 int(Guid[0], 16),
2111                 int(Guid[1], 16),
2112                 int(Guid[2], 16),
2113                 int(Guid[3][-4:-2], 16),
2114                 int(Guid[3][-2:], 16),
2115                 int(Guid[4][-12:-10], 16),
2116                 int(Guid[4][-10:-8], 16),
2117                 int(Guid[4][-8:-6], 16),
2118                 int(Guid[4][-6:-4], 16),
2119                 int(Guid[4][-4:-2], 16),
2120                 int(Guid[4][-2:], 16)
2121                 )
2122
2123 #
2124 # Pack a GUID (byte) list into a buffer and return it
2125 #
2126 def PackByteFormatGUID(Guid):
2127     return pack(PACK_PATTERN_GUID,
2128                 Guid[0],
2129                 Guid[1],
2130                 Guid[2],
2131                 Guid[3],
2132                 Guid[4],
2133                 Guid[5],
2134                 Guid[6],
2135                 Guid[7],
2136                 Guid[8],
2137                 Guid[9],
2138                 Guid[10],
2139                 )
2140
2141 ##
2142 #
2143 # This acts like the main() function for the script, unless it is 'import'ed into another
2144 # script.
2145 #
2146 if __name__ == '__main__':
2147     pass
2148