BaseTools/Source/Python/Common/Misc.py

   1 ## @file
   2 # Common routines used by all tools
   3 #
   4 # Copyright (c) 2007 - 2018, Intel Corporation. All rights reserved.<BR>
   5 # This program and the accompanying materials
   6 # are licensed and made available under the terms and conditions of the BSD License
   7 # which accompanies this distribution.  The full text of the license may be found at
   8 # http://opensource.org/licenses/bsd-license.php
   9 #
  10 # THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS,
  11 # WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED.
  12 #
  13
  14 ##
  15 # Import Modules
  16 #
  17 from __future__ import absolute_import
  18 import Common.LongFilePathOs as os
  19 import sys
  20 import string
  21 import threading
  22 import time
  23 import re
  24 import pickle
  25 import array
  26 import shutil
  27 from struct import pack
  28 from UserDict import IterableUserDict
  29 from UserList import UserList
  30
  31 from Common import EdkLogger as EdkLogger
  32 from Common import GlobalData as GlobalData
  33 from .DataType import *
  34 from .BuildToolError import *
  35 from CommonDataClass.DataClass import *
  36 from .Parsing import GetSplitValueList
  37 from Common.LongFilePathSupport import OpenLongFilePath as open
  38 from Common.MultipleWorkspace import MultipleWorkspace as mws
  39 import uuid
  40 from CommonDataClass.Exceptions import BadExpression
  41 from Common.caching import cached_property
  42 import subprocess
  43 ## Regular expression used to find out place holders in string template
  44 gPlaceholderPattern = re.compile("\$\{([^$()\s]+)\}", re.MULTILINE | re.UNICODE)
  45
  46 ## regular expressions for map file processing
  47 startPatternGeneral = re.compile("^Start[' ']+Length[' ']+Name[' ']+Class")
  48 addressPatternGeneral = re.compile("^Address[' ']+Publics by Value[' ']+Rva\+Base")
  49 valuePatternGcc = re.compile('^([\w_\.]+) +([\da-fA-Fx]+) +([\da-fA-Fx]+)$')
  50 pcdPatternGcc = re.compile('^([\da-fA-Fx]+) +([\da-fA-Fx]+)')
  51 secReGeneral = re.compile('^([\da-fA-F]+):([\da-fA-F]+) +([\da-fA-F]+)[Hh]? +([.\w\$]+) +(\w+)', re.UNICODE)
  52
  53 ## Dictionary used to store file time stamp for quick re-access
  54 gFileTimeStampCache = {}    # {file path : file time stamp}
  55
  56 ## Dictionary used to store dependencies of files
  57 gDependencyDatabase = {}    # arch : {file path : [dependent files list]}
  58
  59 #
  60 # If a module is built more than once with different PCDs or library classes
  61 # a temporary INF file with same content is created, the temporary file is removed
  62 # when build exits.
  63 #
  64 _TempInfs = []
  65
  66 def GetVariableOffset(mapfilepath, efifilepath, varnames):
  67     """ Parse map file to get variable offset in current EFI file
  68     @param mapfilepath    Map file absolution path
  69     @param efifilepath:   EFI binary file full path
  70     @param varnames       iteratable container whose elements are variable names to be searched
  71
  72     @return List whos elements are tuple with variable name and raw offset
  73     """
  74     lines = []
  75     try:
  76         f = open(mapfilepath, 'r')
  77         lines = f.readlines()
  78         f.close()
  79     except:
  80         return None
  81
  82     if len(lines) == 0: return None
  83     firstline = lines[0].strip()
  84     if (firstline.startswith("Archive member included ") and
  85         firstline.endswith(" file (symbol)")):
  86         return _parseForGCC(lines, efifilepath, varnames)
  87     if firstline.startswith("# Path:"):
  88         return _parseForXcode(lines, efifilepath, varnames)
  89     return _parseGeneral(lines, efifilepath, varnames)
  90
  91 def _parseForXcode(lines, efifilepath, varnames):
  92     status = 0
  93     ret = []
  94     for line in lines:
  95         line = line.strip()
  96         if status == 0 and line == "# Symbols:":
  97             status = 1
  98             continue
  99         if status == 1 and len(line) != 0:
 100             for varname in varnames:
 101                 if varname in line:
 102                     # cannot pregenerate this RegEx since it uses varname from varnames.
 103                     m = re.match('^([\da-fA-FxX]+)([\s\S]*)([_]*%s)$' % varname, line)
 104                     if m is not None:
 105                         ret.append((varname, m.group(1)))
 106     return ret
 107
 108 def _parseForGCC(lines, efifilepath, varnames):
 109     """ Parse map file generated by GCC linker """
 110     status = 0
 111     sections = []
 112     varoffset = []
 113     for index, line in enumerate(lines):
 114         line = line.strip()
 115         # status machine transection
 116         if status == 0 and line == "Memory Configuration":
 117             status = 1
 118             continue
 119         elif status == 1 and line == 'Linker script and memory map':
 120             status = 2
 121             continue
 122         elif status ==2 and line == 'START GROUP':
 123             status = 3
 124             continue
 125
 126         # status handler
 127         if status == 3:
 128             m = valuePatternGcc.match(line)
 129             if m is not None:
 130                 sections.append(m.groups(0))
 131             for varname in varnames:
 132                 Str = ''
 133                 m = re.match("^.data.(%s)" % varname, line)
 134                 if m is not None:
 135                     m = re.match(".data.(%s)$" % varname, line)
 136                     if m is not None:
 137                         Str = lines[index + 1]
 138                     else:
 139                         Str = line[len(".data.%s" % varname):]
 140                     if Str:
 141                         m = pcdPatternGcc.match(Str.strip())
 142                         if m is not None:
 143                             varoffset.append((varname, int(m.groups(0)[0], 16), int(sections[-1][1], 16), sections[-1][0]))
 144
 145     if not varoffset:
 146         return []
 147     # get section information from efi file
 148     efisecs = PeImageClass(efifilepath).SectionHeaderList
 149     if efisecs is None or len(efisecs) == 0:
 150         return []
 151     #redirection
 152     redirection = 0
 153     for efisec in efisecs:
 154         for section in sections:
 155             if section[0].strip() == efisec[0].strip() and section[0].strip() == '.text':
 156                 redirection = int(section[1], 16) - efisec[1]
 157
 158     ret = []
 159     for var in varoffset:
 160         for efisec in efisecs:
 161             if var[1] >= efisec[1] and var[1] < efisec[1]+efisec[3]:
 162                 ret.append((var[0], hex(efisec[2] + var[1] - efisec[1] - redirection)))
 163     return ret
 164
 165 def _parseGeneral(lines, efifilepath, varnames):
 166     status = 0    #0 - beginning of file; 1 - PE section definition; 2 - symbol table
 167     secs  = []    # key = section name
 168     varoffset = []
 169     symRe = re.compile('^([\da-fA-F]+):([\da-fA-F]+) +([\.:\\\\\w\?@\$]+) +([\da-fA-F]+)', re.UNICODE)
 170
 171     for line in lines:
 172         line = line.strip()
 173         if startPatternGeneral.match(line):
 174             status = 1
 175             continue
 176         if addressPatternGeneral.match(line):
 177             status = 2
 178             continue
 179         if line.startswith("entry point at"):
 180             status = 3
 181             continue
 182         if status == 1 and len(line) != 0:
 183             m =  secReGeneral.match(line)
 184             assert m is not None, "Fail to parse the section in map file , line is %s" % line
 185             sec_no, sec_start, sec_length, sec_name, sec_class = m.groups(0)
 186             secs.append([int(sec_no, 16), int(sec_start, 16), int(sec_length, 16), sec_name, sec_class])
 187         if status == 2 and len(line) != 0:
 188             for varname in varnames:
 189                 m = symRe.match(line)
 190                 assert m is not None, "Fail to parse the symbol in map file, line is %s" % line
 191                 sec_no, sym_offset, sym_name, vir_addr = m.groups(0)
 192                 sec_no     = int(sec_no,     16)
 193                 sym_offset = int(sym_offset, 16)
 194                 vir_addr   = int(vir_addr,   16)
 195                 # cannot pregenerate this RegEx since it uses varname from varnames.
 196                 m2 = re.match('^[_]*(%s)' % varname, sym_name)
 197                 if m2 is not None:
 198                     # fond a binary pcd entry in map file
 199                     for sec in secs:
 200                         if sec[0] == sec_no and (sym_offset >= sec[1] and sym_offset < sec[1] + sec[2]):
 201                             varoffset.append([varname, sec[3], sym_offset, vir_addr, sec_no])
 202
 203     if not varoffset: return []
 204
 205     # get section information from efi file
 206     efisecs = PeImageClass(efifilepath).SectionHeaderList
 207     if efisecs is None or len(efisecs) == 0:
 208         return []
 209
 210     ret = []
 211     for var in varoffset:
 212         index = 0
 213         for efisec in efisecs:
 214             index = index + 1
 215             if var[1].strip() == efisec[0].strip():
 216                 ret.append((var[0], hex(efisec[2] + var[2])))
 217             elif var[4] == index:
 218                 ret.append((var[0], hex(efisec[2] + var[2])))
 219
 220     return ret
 221
 222 ## Routine to process duplicated INF
 223 #
 224 #  This function is called by following two cases:
 225 #  Case 1 in DSC:
 226 #    [components.arch]
 227 #    Pkg/module/module.inf
 228 #    Pkg/module/module.inf {
 229 #      <Defines>
 230 #        FILE_GUID = 0D1B936F-68F3-4589-AFCC-FB8B7AEBC836
 231 #    }
 232 #  Case 2 in FDF:
 233 #    INF Pkg/module/module.inf
 234 #    INF FILE_GUID = 0D1B936F-68F3-4589-AFCC-FB8B7AEBC836 Pkg/module/module.inf
 235 #
 236 #  This function copies Pkg/module/module.inf to
 237 #  Conf/.cache/0D1B936F-68F3-4589-AFCC-FB8B7AEBC836module.inf
 238 #
 239 #  @param Path     Original PathClass object
 240 #  @param BaseName New file base name
 241 #
 242 #  @retval         return the new PathClass object
 243 #
 244 def ProcessDuplicatedInf(Path, BaseName, Workspace):
 245     Filename = os.path.split(Path.File)[1]
 246     if '.' in Filename:
 247         Filename = BaseName + Path.BaseName + Filename[Filename.rfind('.'):]
 248     else:
 249         Filename = BaseName + Path.BaseName
 250
 251     #
 252     # If -N is specified on command line, cache is disabled
 253     # The directory has to be created
 254     #
 255     DbDir = os.path.split(GlobalData.gDatabasePath)[0]
 256     if not os.path.exists(DbDir):
 257         os.makedirs(DbDir)
 258     #
 259     # A temporary INF is copied to database path which must have write permission
 260     # The temporary will be removed at the end of build
 261     # In case of name conflict, the file name is
 262     # FILE_GUIDBaseName (0D1B936F-68F3-4589-AFCC-FB8B7AEBC836module.inf)
 263     #
 264     TempFullPath = os.path.join(DbDir,
 265                                 Filename)
 266     RtPath = PathClass(Path.File, Workspace)
 267     #
 268     # Modify the full path to temporary path, keep other unchanged
 269     #
 270     # To build same module more than once, the module path with FILE_GUID overridden has
 271     # the file name FILE_GUIDmodule.inf, but the relative path (self.MetaFile.File) is the real path
 272     # in DSC which is used as relative path by C files and other files in INF.
 273     # A trick was used: all module paths are PathClass instances, after the initialization
 274     # of PathClass, the PathClass.Path is overridden by the temporary INF path.
 275     #
 276     # The reason for creating a temporary INF is:
 277     # Platform.Modules which is the base to create ModuleAutoGen objects is a dictionary,
 278     # the key is the full path of INF, the value is an object to save overridden library instances, PCDs.
 279     # A different key for the same module is needed to create different output directory,
 280     # retrieve overridden PCDs, library instances.
 281     #
 282     # The BaseName is the FILE_GUID which is also the output directory name.
 283     #
 284     #
 285     RtPath.Path = TempFullPath
 286     RtPath.BaseName = BaseName
 287     #
 288     # If file exists, compare contents
 289     #
 290     if os.path.exists(TempFullPath):
 291         with open(str(Path), 'rb') as f1, open(TempFullPath, 'rb') as f2:
 292             if f1.read() == f2.read():
 293                 return RtPath
 294     _TempInfs.append(TempFullPath)
 295     shutil.copy2(str(Path), TempFullPath)
 296     return RtPath
 297
 298 ## Remove temporary created INFs whose paths were saved in _TempInfs
 299 #
 300 def ClearDuplicatedInf():
 301     while _TempInfs:
 302         File = _TempInfs.pop()
 303         if os.path.exists(File):
 304             os.remove(File)
 305
 306 ## Convert GUID string in xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx style to C structure style
 307 #
 308 #   @param      Guid    The GUID string
 309 #
 310 #   @retval     string  The GUID string in C structure style
 311 #
 312 def GuidStringToGuidStructureString(Guid):
 313     GuidList = Guid.split('-')
 314     Result = '{'
 315     for Index in range(0, 3, 1):
 316         Result = Result + '0x' + GuidList[Index] + ', '
 317     Result = Result + '{0x' + GuidList[3][0:2] + ', 0x' + GuidList[3][2:4]
 318     for Index in range(0, 12, 2):
 319         Result = Result + ', 0x' + GuidList[4][Index:Index + 2]
 320     Result += '}}'
 321     return Result
 322
 323 ## Convert GUID structure in byte array to xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
 324 #
 325 #   @param      GuidValue   The GUID value in byte array
 326 #
 327 #   @retval     string      The GUID value in xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx format
 328 #
 329 def GuidStructureByteArrayToGuidString(GuidValue):
 330     guidValueString = GuidValue.lower().replace("{", "").replace("}", "").replace(" ", "").replace(";", "")
 331     guidValueList = guidValueString.split(",")
 332     if len(guidValueList) != 16:
 333         return ''
 334         #EdkLogger.error(None, None, "Invalid GUID value string %s" % GuidValue)
 335     try:
 336         return "%02x%02x%02x%02x-%02x%02x-%02x%02x-%02x%02x-%02x%02x%02x%02x%02x%02x" % (
 337                 int(guidValueList[3], 16),
 338                 int(guidValueList[2], 16),
 339                 int(guidValueList[1], 16),
 340                 int(guidValueList[0], 16),
 341                 int(guidValueList[5], 16),
 342                 int(guidValueList[4], 16),
 343                 int(guidValueList[7], 16),
 344                 int(guidValueList[6], 16),
 345                 int(guidValueList[8], 16),
 346                 int(guidValueList[9], 16),
 347                 int(guidValueList[10], 16),
 348                 int(guidValueList[11], 16),
 349                 int(guidValueList[12], 16),
 350                 int(guidValueList[13], 16),
 351                 int(guidValueList[14], 16),
 352                 int(guidValueList[15], 16)
 353                 )
 354     except:
 355         return ''
 356
 357 ## Convert GUID string in C structure style to xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
 358 #
 359 #   @param      GuidValue   The GUID value in C structure format
 360 #
 361 #   @retval     string      The GUID value in xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx format
 362 #
 363 def GuidStructureStringToGuidString(GuidValue):
 364     if not GlobalData.gGuidCFormatPattern.match(GuidValue):
 365         return ''
 366     guidValueString = GuidValue.lower().replace("{", "").replace("}", "").replace(" ", "").replace(";", "")
 367     guidValueList = guidValueString.split(",")
 368     if len(guidValueList) != 11:
 369         return ''
 370         #EdkLogger.error(None, None, "Invalid GUID value string %s" % GuidValue)
 371     try:
 372         return "%08x-%04x-%04x-%02x%02x-%02x%02x%02x%02x%02x%02x" % (
 373                 int(guidValueList[0], 16),
 374                 int(guidValueList[1], 16),
 375                 int(guidValueList[2], 16),
 376                 int(guidValueList[3], 16),
 377                 int(guidValueList[4], 16),
 378                 int(guidValueList[5], 16),
 379                 int(guidValueList[6], 16),
 380                 int(guidValueList[7], 16),
 381                 int(guidValueList[8], 16),
 382                 int(guidValueList[9], 16),
 383                 int(guidValueList[10], 16)
 384                 )
 385     except:
 386         return ''
 387
 388 ## Convert GUID string in C structure style to xxxxxxxx_xxxx_xxxx_xxxx_xxxxxxxxxxxx
 389 #
 390 #   @param      GuidValue   The GUID value in C structure format
 391 #
 392 #   @retval     string      The GUID value in xxxxxxxx_xxxx_xxxx_xxxx_xxxxxxxxxxxx format
 393 #
 394 def GuidStructureStringToGuidValueName(GuidValue):
 395     guidValueString = GuidValue.lower().replace("{", "").replace("}", "").replace(" ", "")
 396     guidValueList = guidValueString.split(",")
 397     if len(guidValueList) != 11:
 398         EdkLogger.error(None, FORMAT_INVALID, "Invalid GUID value string [%s]" % GuidValue)
 399     return "%08x_%04x_%04x_%02x%02x_%02x%02x%02x%02x%02x%02x" % (
 400             int(guidValueList[0], 16),
 401             int(guidValueList[1], 16),
 402             int(guidValueList[2], 16),
 403             int(guidValueList[3], 16),
 404             int(guidValueList[4], 16),
 405             int(guidValueList[5], 16),
 406             int(guidValueList[6], 16),
 407             int(guidValueList[7], 16),
 408             int(guidValueList[8], 16),
 409             int(guidValueList[9], 16),
 410             int(guidValueList[10], 16)
 411             )
 412
 413 ## Create directories
 414 #
 415 #   @param      Directory   The directory name
 416 #
 417 def CreateDirectory(Directory):
 418     if Directory is None or Directory.strip() == "":
 419         return True
 420     try:
 421         if not os.access(Directory, os.F_OK):
 422             os.makedirs(Directory)
 423     except:
 424         return False
 425     return True
 426
 427 ## Remove directories, including files and sub-directories in it
 428 #
 429 #   @param      Directory   The directory name
 430 #
 431 def RemoveDirectory(Directory, Recursively=False):
 432     if Directory is None or Directory.strip() == "" or not os.path.exists(Directory):
 433         return
 434     if Recursively:
 435         CurrentDirectory = os.getcwd()
 436         os.chdir(Directory)
 437         for File in os.listdir("."):
 438             if os.path.isdir(File):
 439                 RemoveDirectory(File, Recursively)
 440             else:
 441                 os.remove(File)
 442         os.chdir(CurrentDirectory)
 443     os.rmdir(Directory)
 444
 445 ## Store content in file
 446 #
 447 #  This method is used to save file only when its content is changed. This is
 448 #  quite useful for "make" system to decide what will be re-built and what won't.
 449 #
 450 #   @param      File            The path of file
 451 #   @param      Content         The new content of the file
 452 #   @param      IsBinaryFile    The flag indicating if the file is binary file or not
 453 #
 454 #   @retval     True            If the file content is changed and the file is renewed
 455 #   @retval     False           If the file content is the same
 456 #
 457 def SaveFileOnChange(File, Content, IsBinaryFile=True):
 458     if not IsBinaryFile:
 459         Content = Content.replace("\n", os.linesep)
 460
 461     if os.path.exists(File):
 462         try:
 463             if Content == open(File, "rb").read():
 464                 return False
 465         except:
 466             EdkLogger.error(None, FILE_OPEN_FAILURE, ExtraData=File)
 467
 468     DirName = os.path.dirname(File)
 469     if not CreateDirectory(DirName):
 470         EdkLogger.error(None, FILE_CREATE_FAILURE, "Could not create directory %s" % DirName)
 471     else:
 472         if DirName == '':
 473             DirName = os.getcwd()
 474         if not os.access(DirName, os.W_OK):
 475             EdkLogger.error(None, PERMISSION_FAILURE, "Do not have write permission on directory %s" % DirName)
 476
 477     try:
 478         if GlobalData.gIsWindows:
 479             try:
 480                 from .PyUtility import SaveFileToDisk
 481                 if not SaveFileToDisk(File, Content):
 482                     EdkLogger.error(None, FILE_CREATE_FAILURE, ExtraData=File)
 483             except:
 484                 Fd = open(File, "wb")
 485                 Fd.write(Content)
 486                 Fd.close()
 487         else:
 488             Fd = open(File, "wb")
 489             Fd.write(Content)
 490             Fd.close()
 491     except IOError as X:
 492         EdkLogger.error(None, FILE_CREATE_FAILURE, ExtraData='IOError %s' % X)
 493
 494     return True
 495
 496 ## Make a Python object persistent on file system
 497 #
 498 #   @param      Data    The object to be stored in file
 499 #   @param      File    The path of file to store the object
 500 #
 501 def DataDump(Data, File):
 502     Fd = None
 503     try:
 504         Fd = open(File, 'wb')
 505         pickle.dump(Data, Fd, pickle.HIGHEST_PROTOCOL)
 506     except:
 507         EdkLogger.error("", FILE_OPEN_FAILURE, ExtraData=File, RaiseError=False)
 508     finally:
 509         if Fd is not None:
 510             Fd.close()
 511
 512 ## Restore a Python object from a file
 513 #
 514 #   @param      File    The path of file stored the object
 515 #
 516 #   @retval     object  A python object
 517 #   @retval     None    If failure in file operation
 518 #
 519 def DataRestore(File):
 520     Data = None
 521     Fd = None
 522     try:
 523         Fd = open(File, 'rb')
 524         Data = pickle.load(Fd)
 525     except Exception as e:
 526         EdkLogger.verbose("Failed to load [%s]\n\t%s" % (File, str(e)))
 527         Data = None
 528     finally:
 529         if Fd is not None:
 530             Fd.close()
 531     return Data
 532
 533 ## Retrieve and cache the real path name in file system
 534 #
 535 #   @param      Root    The root directory of path relative to
 536 #
 537 #   @retval     str     The path string if the path exists
 538 #   @retval     None    If path doesn't exist
 539 #
 540 class DirCache:
 541     _CACHE_ = set()
 542     _UPPER_CACHE_ = {}
 543
 544     def __init__(self, Root):
 545         self._Root = Root
 546         for F in os.listdir(Root):
 547             self._CACHE_.add(F)
 548             self._UPPER_CACHE_[F.upper()] = F
 549
 550     # =[] operator
 551     def __getitem__(self, Path):
 552         Path = Path[len(os.path.commonprefix([Path, self._Root])):]
 553         if not Path:
 554             return self._Root
 555         if Path and Path[0] == os.path.sep:
 556             Path = Path[1:]
 557         if Path in self._CACHE_:
 558             return os.path.join(self._Root, Path)
 559         UpperPath = Path.upper()
 560         if UpperPath in self._UPPER_CACHE_:
 561             return os.path.join(self._Root, self._UPPER_CACHE_[UpperPath])
 562
 563         IndexList = []
 564         LastSepIndex = -1
 565         SepIndex = Path.find(os.path.sep)
 566         while SepIndex > -1:
 567             Parent = UpperPath[:SepIndex]
 568             if Parent not in self._UPPER_CACHE_:
 569                 break
 570             LastSepIndex = SepIndex
 571             SepIndex = Path.find(os.path.sep, LastSepIndex + 1)
 572
 573         if LastSepIndex == -1:
 574             return None
 575
 576         Cwd = os.getcwd()
 577         os.chdir(self._Root)
 578         SepIndex = LastSepIndex
 579         while SepIndex > -1:
 580             Parent = Path[:SepIndex]
 581             ParentKey = UpperPath[:SepIndex]
 582             if ParentKey not in self._UPPER_CACHE_:
 583                 os.chdir(Cwd)
 584                 return None
 585
 586             if Parent in self._CACHE_:
 587                 ParentDir = Parent
 588             else:
 589                 ParentDir = self._UPPER_CACHE_[ParentKey]
 590             for F in os.listdir(ParentDir):
 591                 Dir = os.path.join(ParentDir, F)
 592                 self._CACHE_.add(Dir)
 593                 self._UPPER_CACHE_[Dir.upper()] = Dir
 594
 595             SepIndex = Path.find(os.path.sep, SepIndex + 1)
 596
 597         os.chdir(Cwd)
 598         if Path in self._CACHE_:
 599             return os.path.join(self._Root, Path)
 600         elif UpperPath in self._UPPER_CACHE_:
 601             return os.path.join(self._Root, self._UPPER_CACHE_[UpperPath])
 602         return None
 603
 604 def RealPath(File, Dir='', OverrideDir=''):
 605     NewFile = os.path.normpath(os.path.join(Dir, File))
 606     NewFile = GlobalData.gAllFiles[NewFile]
 607     if not NewFile and OverrideDir:
 608         NewFile = os.path.normpath(os.path.join(OverrideDir, File))
 609         NewFile = GlobalData.gAllFiles[NewFile]
 610     return NewFile
 611
 612 def RealPath2(File, Dir='', OverrideDir=''):
 613     NewFile = None
 614     if OverrideDir:
 615         NewFile = GlobalData.gAllFiles[os.path.normpath(os.path.join(OverrideDir, File))]
 616         if NewFile:
 617             if OverrideDir[-1] == os.path.sep:
 618                 return NewFile[len(OverrideDir):], NewFile[0:len(OverrideDir)]
 619             else:
 620                 return NewFile[len(OverrideDir) + 1:], NewFile[0:len(OverrideDir)]
 621     if GlobalData.gAllFiles:
 622         NewFile = GlobalData.gAllFiles[os.path.normpath(os.path.join(Dir, File))]
 623     if not NewFile:
 624         NewFile = os.path.normpath(os.path.join(Dir, File))
 625         if not os.path.exists(NewFile):
 626             return None, None
 627     if NewFile:
 628         if Dir:
 629             if Dir[-1] == os.path.sep:
 630                 return NewFile[len(Dir):], NewFile[0:len(Dir)]
 631             else:
 632                 return NewFile[len(Dir) + 1:], NewFile[0:len(Dir)]
 633         else:
 634             return NewFile, ''
 635
 636     return None, None
 637
 638 ## Get GUID value from given packages
 639 #
 640 #   @param      CName           The CName of the GUID
 641 #   @param      PackageList     List of packages looking-up in
 642 #   @param      Inffile         The driver file
 643 #
 644 #   @retval     GuidValue   if the CName is found in any given package
 645 #   @retval     None        if the CName is not found in all given packages
 646 #
 647 def GuidValue(CName, PackageList, Inffile = None):
 648     for P in PackageList:
 649         GuidKeys = P.Guids.keys()
 650         if Inffile and P._PrivateGuids:
 651             if not Inffile.startswith(P.MetaFile.Dir):
 652                 GuidKeys = [x for x in P.Guids if x not in P._PrivateGuids]
 653         if CName in GuidKeys:
 654             return P.Guids[CName]
 655     return None
 656
 657 ## Get Protocol value from given packages
 658 #
 659 #   @param      CName           The CName of the GUID
 660 #   @param      PackageList     List of packages looking-up in
 661 #   @param      Inffile         The driver file
 662 #
 663 #   @retval     GuidValue   if the CName is found in any given package
 664 #   @retval     None        if the CName is not found in all given packages
 665 #
 666 def ProtocolValue(CName, PackageList, Inffile = None):
 667     for P in PackageList:
 668         ProtocolKeys = P.Protocols.keys()
 669         if Inffile and P._PrivateProtocols:
 670             if not Inffile.startswith(P.MetaFile.Dir):
 671                 ProtocolKeys = [x for x in P.Protocols if x not in P._PrivateProtocols]
 672         if CName in ProtocolKeys:
 673             return P.Protocols[CName]
 674     return None
 675
 676 ## Get PPI value from given packages
 677 #
 678 #   @param      CName           The CName of the GUID
 679 #   @param      PackageList     List of packages looking-up in
 680 #   @param      Inffile         The driver file
 681 #
 682 #   @retval     GuidValue   if the CName is found in any given package
 683 #   @retval     None        if the CName is not found in all given packages
 684 #
 685 def PpiValue(CName, PackageList, Inffile = None):
 686     for P in PackageList:
 687         PpiKeys = P.Ppis.keys()
 688         if Inffile and P._PrivatePpis:
 689             if not Inffile.startswith(P.MetaFile.Dir):
 690                 PpiKeys = [x for x in P.Ppis if x not in P._PrivatePpis]
 691         if CName in PpiKeys:
 692             return P.Ppis[CName]
 693     return None
 694
 695 ## A string template class
 696 #
 697 #  This class implements a template for string replacement. A string template
 698 #  looks like following
 699 #
 700 #       ${BEGIN} other_string ${placeholder_name} other_string ${END}
 701 #
 702 #  The string between ${BEGIN} and ${END} will be repeated as many times as the
 703 #  length of "placeholder_name", which is a list passed through a dict. The
 704 #  "placeholder_name" is the key name of the dict. The ${BEGIN} and ${END} can
 705 #  be not used and, in this case, the "placeholder_name" must not a list and it
 706 #  will just be replaced once.
 707 #
 708 class TemplateString(object):
 709     _REPEAT_START_FLAG = "BEGIN"
 710     _REPEAT_END_FLAG = "END"
 711
 712     class Section(object):
 713         _LIST_TYPES = [type([]), type(set()), type((0,))]
 714
 715         def __init__(self, TemplateSection, PlaceHolderList):
 716             self._Template = TemplateSection
 717             self._PlaceHolderList = []
 718
 719             # Split the section into sub-sections according to the position of placeholders
 720             if PlaceHolderList:
 721                 self._SubSectionList = []
 722                 SubSectionStart = 0
 723                 #
 724                 # The placeholders passed in must be in the format of
 725                 #
 726                 #   PlaceHolderName, PlaceHolderStartPoint, PlaceHolderEndPoint
 727                 #
 728                 for PlaceHolder, Start, End in PlaceHolderList:
 729                     self._SubSectionList.append(TemplateSection[SubSectionStart:Start])
 730                     self._SubSectionList.append(TemplateSection[Start:End])
 731                     self._PlaceHolderList.append(PlaceHolder)
 732                     SubSectionStart = End
 733                 if SubSectionStart < len(TemplateSection):
 734                     self._SubSectionList.append(TemplateSection[SubSectionStart:])
 735             else:
 736                 self._SubSectionList = [TemplateSection]
 737
 738         def __str__(self):
 739             return self._Template + " : " + str(self._PlaceHolderList)
 740
 741         def Instantiate(self, PlaceHolderValues):
 742             RepeatTime = -1
 743             RepeatPlaceHolders = {}
 744             NonRepeatPlaceHolders = {}
 745
 746             for PlaceHolder in self._PlaceHolderList:
 747                 if PlaceHolder not in PlaceHolderValues:
 748                     continue
 749                 Value = PlaceHolderValues[PlaceHolder]
 750                 if type(Value) in self._LIST_TYPES:
 751                     if RepeatTime < 0:
 752                         RepeatTime = len(Value)
 753                     elif RepeatTime != len(Value):
 754                         EdkLogger.error(
 755                                     "TemplateString",
 756                                     PARAMETER_INVALID,
 757                                     "${%s} has different repeat time from others!" % PlaceHolder,
 758                                     ExtraData=str(self._Template)
 759                                     )
 760                     RepeatPlaceHolders["${%s}" % PlaceHolder] = Value
 761                 else:
 762                     NonRepeatPlaceHolders["${%s}" % PlaceHolder] = Value
 763
 764             if NonRepeatPlaceHolders:
 765                 StringList = []
 766                 for S in self._SubSectionList:
 767                     if S not in NonRepeatPlaceHolders:
 768                         StringList.append(S)
 769                     else:
 770                         StringList.append(str(NonRepeatPlaceHolders[S]))
 771             else:
 772                 StringList = self._SubSectionList
 773
 774             if RepeatPlaceHolders:
 775                 TempStringList = []
 776                 for Index in range(RepeatTime):
 777                     for S in StringList:
 778                         if S not in RepeatPlaceHolders:
 779                             TempStringList.append(S)
 780                         else:
 781                             TempStringList.append(str(RepeatPlaceHolders[S][Index]))
 782                 StringList = TempStringList
 783
 784             return "".join(StringList)
 785
 786     ## Constructor
 787     def __init__(self, Template=None):
 788         self.String = ''
 789         self.IsBinary = False
 790         self._Template = Template
 791         self._TemplateSectionList = self._Parse(Template)
 792
 793     ## str() operator
 794     #
 795     #   @retval     string  The string replaced
 796     #
 797     def __str__(self):
 798         return self.String
 799
 800     ## Split the template string into fragments per the ${BEGIN} and ${END} flags
 801     #
 802     #   @retval     list    A list of TemplateString.Section objects
 803     #
 804     def _Parse(self, Template):
 805         SectionStart = 0
 806         SearchFrom = 0
 807         MatchEnd = 0
 808         PlaceHolderList = []
 809         TemplateSectionList = []
 810         while Template:
 811             MatchObj = gPlaceholderPattern.search(Template, SearchFrom)
 812             if not MatchObj:
 813                 if MatchEnd <= len(Template):
 814                     TemplateSection = TemplateString.Section(Template[SectionStart:], PlaceHolderList)
 815                     TemplateSectionList.append(TemplateSection)
 816                 break
 817
 818             MatchString = MatchObj.group(1)
 819             MatchStart = MatchObj.start()
 820             MatchEnd = MatchObj.end()
 821
 822             if MatchString == self._REPEAT_START_FLAG:
 823                 if MatchStart > SectionStart:
 824                     TemplateSection = TemplateString.Section(Template[SectionStart:MatchStart], PlaceHolderList)
 825                     TemplateSectionList.append(TemplateSection)
 826                 SectionStart = MatchEnd
 827                 PlaceHolderList = []
 828             elif MatchString == self._REPEAT_END_FLAG:
 829                 TemplateSection = TemplateString.Section(Template[SectionStart:MatchStart], PlaceHolderList)
 830                 TemplateSectionList.append(TemplateSection)
 831                 SectionStart = MatchEnd
 832                 PlaceHolderList = []
 833             else:
 834                 PlaceHolderList.append((MatchString, MatchStart - SectionStart, MatchEnd - SectionStart))
 835             SearchFrom = MatchEnd
 836         return TemplateSectionList
 837
 838     ## Replace the string template with dictionary of placeholders and append it to previous one
 839     #
 840     #   @param      AppendString    The string template to append
 841     #   @param      Dictionary      The placeholder dictionaries
 842     #
 843     def Append(self, AppendString, Dictionary=None):
 844         if Dictionary:
 845             SectionList = self._Parse(AppendString)
 846             self.String += "".join(S.Instantiate(Dictionary) for S in SectionList)
 847         else:
 848             self.String += AppendString
 849
 850     ## Replace the string template with dictionary of placeholders
 851     #
 852     #   @param      Dictionary      The placeholder dictionaries
 853     #
 854     #   @retval     str             The string replaced with placeholder values
 855     #
 856     def Replace(self, Dictionary=None):
 857         return "".join(S.Instantiate(Dictionary) for S in self._TemplateSectionList)
 858
 859 ## Progress indicator class
 860 #
 861 #  This class makes use of thread to print progress on console.
 862 #
 863 class Progressor:
 864     # for avoiding deadloop
 865     _StopFlag = None
 866     _ProgressThread = None
 867     _CheckInterval = 0.25
 868
 869     ## Constructor
 870     #
 871     #   @param      OpenMessage     The string printed before progress charaters
 872     #   @param      CloseMessage    The string printed after progress charaters
 873     #   @param      ProgressChar    The charater used to indicate the progress
 874     #   @param      Interval        The interval in seconds between two progress charaters
 875     #
 876     def __init__(self, OpenMessage="", CloseMessage="", ProgressChar='.', Interval=1.0):
 877         self.PromptMessage = OpenMessage
 878         self.CodaMessage = CloseMessage
 879         self.ProgressChar = ProgressChar
 880         self.Interval = Interval
 881         if Progressor._StopFlag is None:
 882             Progressor._StopFlag = threading.Event()
 883
 884     ## Start to print progress charater
 885     #
 886     #   @param      OpenMessage     The string printed before progress charaters
 887     #
 888     def Start(self, OpenMessage=None):
 889         if OpenMessage is not None:
 890             self.PromptMessage = OpenMessage
 891         Progressor._StopFlag.clear()
 892         if Progressor._ProgressThread is None:
 893             Progressor._ProgressThread = threading.Thread(target=self._ProgressThreadEntry)
 894             Progressor._ProgressThread.setDaemon(False)
 895             Progressor._ProgressThread.start()
 896
 897     ## Stop printing progress charater
 898     #
 899     #   @param      CloseMessage    The string printed after progress charaters
 900     #
 901     def Stop(self, CloseMessage=None):
 902         OriginalCodaMessage = self.CodaMessage
 903         if CloseMessage is not None:
 904             self.CodaMessage = CloseMessage
 905         self.Abort()
 906         self.CodaMessage = OriginalCodaMessage
 907
 908     ## Thread entry method
 909     def _ProgressThreadEntry(self):
 910         sys.stdout.write(self.PromptMessage + " ")
 911         sys.stdout.flush()
 912         TimeUp = 0.0
 913         while not Progressor._StopFlag.isSet():
 914             if TimeUp <= 0.0:
 915                 sys.stdout.write(self.ProgressChar)
 916                 sys.stdout.flush()
 917                 TimeUp = self.Interval
 918             time.sleep(self._CheckInterval)
 919             TimeUp -= self._CheckInterval
 920         sys.stdout.write(" " + self.CodaMessage + "\n")
 921         sys.stdout.flush()
 922
 923     ## Abort the progress display
 924     @staticmethod
 925     def Abort():
 926         if Progressor._StopFlag is not None:
 927             Progressor._StopFlag.set()
 928         if Progressor._ProgressThread is not None:
 929             Progressor._ProgressThread.join()
 930             Progressor._ProgressThread = None
 931
 932 ## A dict which can access its keys and/or values orderly
 933 #
 934 #  The class implements a new kind of dict which its keys or values can be
 935 #  accessed in the order they are added into the dict. It guarantees the order
 936 #  by making use of an internal list to keep a copy of keys.
 937 #
 938 class sdict(IterableUserDict):
 939     ## Constructor
 940     def __init__(self):
 941         IterableUserDict.__init__(self)
 942         self._key_list = []
 943
 944     ## [] operator
 945     def __setitem__(self, key, value):
 946         if key not in self._key_list:
 947             self._key_list.append(key)
 948         IterableUserDict.__setitem__(self, key, value)
 949
 950     ## del operator
 951     def __delitem__(self, key):
 952         self._key_list.remove(key)
 953         IterableUserDict.__delitem__(self, key)
 954
 955     ## used in "for k in dict" loop to ensure the correct order
 956     def __iter__(self):
 957         return self.iterkeys()
 958
 959     ## len() support
 960     def __len__(self):
 961         return len(self._key_list)
 962
 963     ## "in" test support
 964     def __contains__(self, key):
 965         return key in self._key_list
 966
 967     ## indexof support
 968     def index(self, key):
 969         return self._key_list.index(key)
 970
 971     ## insert support
 972     def insert(self, key, newkey, newvalue, order):
 973         index = self._key_list.index(key)
 974         if order == 'BEFORE':
 975             self._key_list.insert(index, newkey)
 976             IterableUserDict.__setitem__(self, newkey, newvalue)
 977         elif order == 'AFTER':
 978             self._key_list.insert(index + 1, newkey)
 979             IterableUserDict.__setitem__(self, newkey, newvalue)
 980
 981     ## append support
 982     def append(self, sdict):
 983         for key in sdict:
 984             if key not in self._key_list:
 985                 self._key_list.append(key)
 986             IterableUserDict.__setitem__(self, key, sdict[key])
 987
 988     def has_key(self, key):
 989         return key in self._key_list
 990
 991     ## Empty the dict
 992     def clear(self):
 993         self._key_list = []
 994         IterableUserDict.clear(self)
 995
 996     ## Return a copy of keys
 997     def keys(self):
 998         keys = []
 999         for key in self._key_list:
1000             keys.append(key)
1001         return keys
1002
1003     ## Return a copy of values
1004     def values(self):
1005         values = []
1006         for key in self._key_list:
1007             values.append(self[key])
1008         return values
1009
1010     ## Return a copy of (key, value) list
1011     def items(self):
1012         items = []
1013         for key in self._key_list:
1014             items.append((key, self[key]))
1015         return items
1016
1017     ## Iteration support
1018     def iteritems(self):
1019         return iter(self.items())
1020
1021     ## Keys interation support
1022     def iterkeys(self):
1023         return iter(self.keys())
1024
1025     ## Values interation support
1026     def itervalues(self):
1027         return iter(self.values())
1028
1029     ## Return value related to a key, and remove the (key, value) from the dict
1030     def pop(self, key, *dv):
1031         value = None
1032         if key in self._key_list:
1033             value = self[key]
1034             self.__delitem__(key)
1035         elif len(dv) != 0 :
1036             value = kv[0]
1037         return value
1038
1039     ## Return (key, value) pair, and remove the (key, value) from the dict
1040     def popitem(self):
1041         key = self._key_list[-1]
1042         value = self[key]
1043         self.__delitem__(key)
1044         return key, value
1045
1046     def update(self, dict=None, **kwargs):
1047         if dict is not None:
1048             for k, v in dict.items():
1049                 self[k] = v
1050         if len(kwargs):
1051             for k, v in kwargs.items():
1052                 self[k] = v
1053
1054 ## Dictionary with restricted keys
1055 #
1056 class rdict(dict):
1057     ## Constructor
1058     def __init__(self, KeyList):
1059         for Key in KeyList:
1060             dict.__setitem__(self, Key, "")
1061
1062     ## []= operator
1063     def __setitem__(self, key, value):
1064         if key not in self:
1065             EdkLogger.error("RestrictedDict", ATTRIBUTE_SET_FAILURE, "Key [%s] is not allowed" % key,
1066                             ExtraData=", ".join(dict.keys(self)))
1067         dict.__setitem__(self, key, value)
1068
1069     ## =[] operator
1070     def __getitem__(self, key):
1071         if key not in self:
1072             return ""
1073         return dict.__getitem__(self, key)
1074
1075     ## del operator
1076     def __delitem__(self, key):
1077         EdkLogger.error("RestrictedDict", ATTRIBUTE_ACCESS_DENIED, ExtraData="del")
1078
1079     ## Empty the dict
1080     def clear(self):
1081         for Key in self:
1082             self.__setitem__(Key, "")
1083
1084     ## Return value related to a key, and remove the (key, value) from the dict
1085     def pop(self, key, *dv):
1086         EdkLogger.error("RestrictedDict", ATTRIBUTE_ACCESS_DENIED, ExtraData="pop")
1087
1088     ## Return (key, value) pair, and remove the (key, value) from the dict
1089     def popitem(self):
1090         EdkLogger.error("RestrictedDict", ATTRIBUTE_ACCESS_DENIED, ExtraData="popitem")
1091
1092 ## Dictionary using prioritized list as key
1093 #
1094 class tdict:
1095     _ListType = type([])
1096     _TupleType = type(())
1097     _Wildcard = 'COMMON'
1098     _ValidWildcardList = ['COMMON', 'DEFAULT', 'ALL', '*', 'PLATFORM']
1099
1100     def __init__(self, _Single_=False, _Level_=2):
1101         self._Level_ = _Level_
1102         self.data = {}
1103         self._Single_ = _Single_
1104
1105     # =[] operator
1106     def __getitem__(self, key):
1107         KeyType = type(key)
1108         RestKeys = None
1109         if KeyType == self._ListType or KeyType == self._TupleType:
1110             FirstKey = key[0]
1111             if len(key) > 1:
1112                 RestKeys = key[1:]
1113             elif self._Level_ > 1:
1114                 RestKeys = [self._Wildcard for i in range(0, self._Level_ - 1)]
1115         else:
1116             FirstKey = key
1117             if self._Level_ > 1:
1118                 RestKeys = [self._Wildcard for i in range(0, self._Level_ - 1)]
1119
1120         if FirstKey is None or str(FirstKey).upper() in self._ValidWildcardList:
1121             FirstKey = self._Wildcard
1122
1123         if self._Single_:
1124             return self._GetSingleValue(FirstKey, RestKeys)
1125         else:
1126             return self._GetAllValues(FirstKey, RestKeys)
1127
1128     def _GetSingleValue(self, FirstKey, RestKeys):
1129         Value = None
1130         #print "%s-%s" % (FirstKey, self._Level_) ,
1131         if self._Level_ > 1:
1132             if FirstKey == self._Wildcard:
1133                 if FirstKey in self.data:
1134                     Value = self.data[FirstKey][RestKeys]
1135                 if Value is None:
1136                     for Key in self.data:
1137                         Value = self.data[Key][RestKeys]
1138                         if Value is not None: break
1139             else:
1140                 if FirstKey in self.data:
1141                     Value = self.data[FirstKey][RestKeys]
1142                 if Value is None and self._Wildcard in self.data:
1143                     #print "Value=None"
1144                     Value = self.data[self._Wildcard][RestKeys]
1145         else:
1146             if FirstKey == self._Wildcard:
1147                 if FirstKey in self.data:
1148                     Value = self.data[FirstKey]
1149                 if Value is None:
1150                     for Key in self.data:
1151                         Value = self.data[Key]
1152                         if Value is not None: break
1153             else:
1154                 if FirstKey in self.data:
1155                     Value = self.data[FirstKey]
1156                 elif self._Wildcard in self.data:
1157                     Value = self.data[self._Wildcard]
1158         return Value
1159
1160     def _GetAllValues(self, FirstKey, RestKeys):
1161         Value = []
1162         if self._Level_ > 1:
1163             if FirstKey == self._Wildcard:
1164                 for Key in self.data:
1165                     Value += self.data[Key][RestKeys]
1166             else:
1167                 if FirstKey in self.data:
1168                     Value += self.data[FirstKey][RestKeys]
1169                 if self._Wildcard in self.data:
1170                     Value += self.data[self._Wildcard][RestKeys]
1171         else:
1172             if FirstKey == self._Wildcard:
1173                 for Key in self.data:
1174                     Value.append(self.data[Key])
1175             else:
1176                 if FirstKey in self.data:
1177                     Value.append(self.data[FirstKey])
1178                 if self._Wildcard in self.data:
1179                     Value.append(self.data[self._Wildcard])
1180         return Value
1181
1182     ## []= operator
1183     def __setitem__(self, key, value):
1184         KeyType = type(key)
1185         RestKeys = None
1186         if KeyType == self._ListType or KeyType == self._TupleType:
1187             FirstKey = key[0]
1188             if len(key) > 1:
1189                 RestKeys = key[1:]
1190             else:
1191                 RestKeys = [self._Wildcard for i in range(0, self._Level_ - 1)]
1192         else:
1193             FirstKey = key
1194             if self._Level_ > 1:
1195                 RestKeys = [self._Wildcard for i in range(0, self._Level_ - 1)]
1196
1197         if FirstKey in self._ValidWildcardList:
1198             FirstKey = self._Wildcard
1199
1200         if FirstKey not in self.data and self._Level_ > 0:
1201             self.data[FirstKey] = tdict(self._Single_, self._Level_ - 1)
1202
1203         if self._Level_ > 1:
1204             self.data[FirstKey][RestKeys] = value
1205         else:
1206             self.data[FirstKey] = value
1207
1208     def SetGreedyMode(self):
1209         self._Single_ = False
1210         if self._Level_ > 1:
1211             for Key in self.data:
1212                 self.data[Key].SetGreedyMode()
1213
1214     def SetSingleMode(self):
1215         self._Single_ = True
1216         if self._Level_ > 1:
1217             for Key in self.data:
1218                 self.data[Key].SetSingleMode()
1219
1220     def GetKeys(self, KeyIndex=0):
1221         assert KeyIndex >= 0
1222         if KeyIndex == 0:
1223             return set(self.data.keys())
1224         else:
1225             keys = set()
1226             for Key in self.data:
1227                 keys |= self.data[Key].GetKeys(KeyIndex - 1)
1228             return keys
1229
1230 def IsFieldValueAnArray (Value):
1231     Value = Value.strip()
1232     if Value.startswith(TAB_GUID) and Value.endswith(')'):
1233         return True
1234     if Value.startswith('L"') and Value.endswith('"')  and len(list(Value[2:-1])) > 1:
1235         return True
1236     if Value[0] == '"' and Value[-1] == '"' and len(list(Value[1:-1])) > 1:
1237         return True
1238     if Value[0] == '{' and Value[-1] == '}':
1239         return True
1240     if Value.startswith("L'") and Value.endswith("'") and len(list(Value[2:-1])) > 1:
1241         return True
1242     if Value[0] == "'" and Value[-1] == "'" and len(list(Value[1:-1])) > 1:
1243         return True
1244     return False
1245
1246 def AnalyzePcdExpression(Setting):
1247     Setting = Setting.strip()
1248     # There might be escaped quote in a string: \", \\\" , \', \\\'
1249     Data = Setting
1250     # There might be '|' in string and in ( ... | ... ), replace it with '-'
1251     NewStr = ''
1252     InSingleQuoteStr = False
1253     InDoubleQuoteStr = False
1254     Pair = 0
1255     for Index, ch in enumerate(Data):
1256         if ch == '"' and not InSingleQuoteStr:
1257             if Data[Index - 1] != '\\':
1258                 InDoubleQuoteStr = not InDoubleQuoteStr
1259         elif ch == "'" and not InDoubleQuoteStr:
1260             if Data[Index - 1] != '\\':
1261                 InSingleQuoteStr = not InSingleQuoteStr
1262         elif ch == '(' and not (InSingleQuoteStr or InDoubleQuoteStr):
1263             Pair += 1
1264         elif ch == ')' and not (InSingleQuoteStr or InDoubleQuoteStr):
1265             Pair -= 1
1266
1267         if (Pair > 0 or InSingleQuoteStr or InDoubleQuoteStr) and ch == TAB_VALUE_SPLIT:
1268             NewStr += '-'
1269         else:
1270             NewStr += ch
1271     FieldList = []
1272     StartPos = 0
1273     while True:
1274         Pos = NewStr.find(TAB_VALUE_SPLIT, StartPos)
1275         if Pos < 0:
1276             FieldList.append(Setting[StartPos:].strip())
1277             break
1278         FieldList.append(Setting[StartPos:Pos].strip())
1279         StartPos = Pos + 1
1280
1281     return FieldList
1282
1283 def ParseDevPathValue (Value):
1284     if '\\' in Value:
1285         Value.replace('\\', '/').replace(' ', '')
1286
1287     Cmd = 'DevicePath ' + '"' + Value + '"'
1288     try:
1289         p = subprocess.Popen(Cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
1290         out, err = p.communicate()
1291     except Exception as X:
1292         raise BadExpression("DevicePath: %s" % (str(X)) )
1293     finally:
1294         subprocess._cleanup()
1295         p.stdout.close()
1296         p.stderr.close()
1297     if err:
1298         raise BadExpression("DevicePath: %s" % str(err))
1299     Size = len(out.split())
1300     out = ','.join(out.split())
1301     return '{' + out + '}', Size
1302
1303 def ParseFieldValue (Value):
1304     if isinstance(Value, type(0)):
1305         return Value, (Value.bit_length() + 7) / 8
1306     if not isinstance(Value, type('')):
1307         raise BadExpression('Type %s is %s' %(Value, type(Value)))
1308     Value = Value.strip()
1309     if Value.startswith(TAB_UINT8) and Value.endswith(')'):
1310         Value, Size = ParseFieldValue(Value.split('(', 1)[1][:-1])
1311         if Size > 1:
1312             raise BadExpression('Value (%s) Size larger than %d' %(Value, Size))
1313         return Value, 1
1314     if Value.startswith(TAB_UINT16) and Value.endswith(')'):
1315         Value, Size = ParseFieldValue(Value.split('(', 1)[1][:-1])
1316         if Size > 2:
1317             raise BadExpression('Value (%s) Size larger than %d' %(Value, Size))
1318         return Value, 2
1319     if Value.startswith(TAB_UINT32) and Value.endswith(')'):
1320         Value, Size = ParseFieldValue(Value.split('(', 1)[1][:-1])
1321         if Size > 4:
1322             raise BadExpression('Value (%s) Size larger than %d' %(Value, Size))
1323         return Value, 4
1324     if Value.startswith(TAB_UINT64) and Value.endswith(')'):
1325         Value, Size = ParseFieldValue(Value.split('(', 1)[1][:-1])
1326         if Size > 8:
1327             raise BadExpression('Value (%s) Size larger than %d' % (Value, Size))
1328         return Value, 8
1329     if Value.startswith(TAB_GUID) and Value.endswith(')'):
1330         Value = Value.split('(', 1)[1][:-1].strip()
1331         if Value[0] == '{' and Value[-1] == '}':
1332             TmpValue = GuidStructureStringToGuidString(Value)
1333             if not TmpValue:
1334                 raise BadExpression("Invalid GUID value string %s" % Value)
1335             Value = TmpValue
1336         if Value[0] == '"' and Value[-1] == '"':
1337             Value = Value[1:-1]
1338         try:
1339             Value = "'" + uuid.UUID(Value).get_bytes_le() + "'"
1340         except ValueError as Message:
1341             raise BadExpression(Message)
1342         Value, Size = ParseFieldValue(Value)
1343         return Value, 16
1344     if Value.startswith('L"') and Value.endswith('"'):
1345         # Unicode String
1346         # translate escape character
1347         Value = Value[1:]
1348         try:
1349             Value = eval(Value)
1350         except:
1351             Value = Value[1:-1]
1352         List = list(Value)
1353         List.reverse()
1354         Value = 0
1355         for Char in List:
1356             Value = (Value << 16) | ord(Char)
1357         return Value, (len(List) + 1) * 2
1358     if Value.startswith('"') and Value.endswith('"'):
1359         # ASCII String
1360         # translate escape character
1361         try:
1362             Value = eval(Value)
1363         except:
1364             Value = Value[1:-1]
1365         List = list(Value)
1366         List.reverse()
1367         Value = 0
1368         for Char in List:
1369             Value = (Value << 8) | ord(Char)
1370         return Value, len(List) + 1
1371     if Value.startswith("L'") and Value.endswith("'"):
1372         # Unicode Character Constant
1373         # translate escape character
1374         Value = Value[1:]
1375         try:
1376             Value = eval(Value)
1377         except:
1378             Value = Value[1:-1]
1379         List = list(Value)
1380         if len(List) == 0:
1381             raise BadExpression('Length %s is %s' % (Value, len(List)))
1382         List.reverse()
1383         Value = 0
1384         for Char in List:
1385             Value = (Value << 16) | ord(Char)
1386         return Value, len(List) * 2
1387     if Value.startswith("'") and Value.endswith("'"):
1388         # Character constant
1389         # translate escape character
1390         try:
1391             Value = eval(Value)
1392         except:
1393             Value = Value[1:-1]
1394         List = list(Value)
1395         if len(List) == 0:
1396             raise BadExpression('Length %s is %s' % (Value, len(List)))
1397         List.reverse()
1398         Value = 0
1399         for Char in List:
1400             Value = (Value << 8) | ord(Char)
1401         return Value, len(List)
1402     if Value.startswith('{') and Value.endswith('}'):
1403         # Byte array
1404         Value = Value[1:-1]
1405         List = [Item.strip() for Item in Value.split(',')]
1406         List.reverse()
1407         Value = 0
1408         RetSize = 0
1409         for Item in List:
1410             ItemValue, Size = ParseFieldValue(Item)
1411             RetSize += Size
1412             for I in range(Size):
1413                 Value = (Value << 8) | ((ItemValue >> 8 * I) & 0xff)
1414         return Value, RetSize
1415     if Value.startswith('DEVICE_PATH(') and Value.endswith(')'):
1416         Value = Value.replace("DEVICE_PATH(", '').rstrip(')')
1417         Value = Value.strip().strip('"')
1418         return ParseDevPathValue(Value)
1419     if Value.lower().startswith('0x'):
1420         try:
1421             Value = int(Value, 16)
1422         except:
1423             raise BadExpression("invalid hex value: %s" % Value)
1424         if Value == 0:
1425             return 0, 1
1426         return Value, (Value.bit_length() + 7) / 8
1427     if Value[0].isdigit():
1428         Value = int(Value, 10)
1429         if Value == 0:
1430             return 0, 1
1431         return Value, (Value.bit_length() + 7) / 8
1432     if Value.lower() == 'true':
1433         return 1, 1
1434     if Value.lower() == 'false':
1435         return 0, 1
1436     return Value, 1
1437
1438 ## AnalyzeDscPcd
1439 #
1440 #  Analyze DSC PCD value, since there is no data type info in DSC
1441 #  This function is used to match functions (AnalyzePcdData) used for retrieving PCD value from database
1442 #  1. Feature flag: TokenSpace.PcdCName|PcdValue
1443 #  2. Fix and Patch:TokenSpace.PcdCName|PcdValue[|VOID*[|MaxSize]]
1444 #  3. Dynamic default:
1445 #     TokenSpace.PcdCName|PcdValue[|VOID*[|MaxSize]]
1446 #     TokenSpace.PcdCName|PcdValue
1447 #  4. Dynamic VPD:
1448 #     TokenSpace.PcdCName|VpdOffset[|VpdValue]
1449 #     TokenSpace.PcdCName|VpdOffset[|MaxSize[|VpdValue]]
1450 #  5. Dynamic HII:
1451 #     TokenSpace.PcdCName|HiiString|VariableGuid|VariableOffset[|HiiValue]
1452 #  PCD value needs to be located in such kind of string, and the PCD value might be an expression in which
1453 #    there might have "|" operator, also in string value.
1454 #
1455 #  @param Setting: String contain information described above with "TokenSpace.PcdCName|" stripped
1456 #  @param PcdType: PCD type: feature, fixed, dynamic default VPD HII
1457 #  @param DataType: The datum type of PCD: VOID*, UNIT, BOOL
1458 #  @retval:
1459 #    ValueList: A List contain fields described above
1460 #    IsValid:   True if conforming EBNF, otherwise False
1461 #    Index:     The index where PcdValue is in ValueList
1462 #
1463 def AnalyzeDscPcd(Setting, PcdType, DataType=''):
1464     FieldList = AnalyzePcdExpression(Setting)
1465
1466     IsValid = True
1467     if PcdType in (MODEL_PCD_FIXED_AT_BUILD, MODEL_PCD_PATCHABLE_IN_MODULE, MODEL_PCD_DYNAMIC_DEFAULT, MODEL_PCD_DYNAMIC_EX_DEFAULT):
1468         Value = FieldList[0]
1469         Size = ''
1470         if len(FieldList) > 1 and FieldList[1]:
1471             DataType = FieldList[1]
1472             if FieldList[1] != TAB_VOID:
1473                 IsValid = False
1474         if len(FieldList) > 2:
1475             Size = FieldList[2]
1476         if IsValid:
1477             if DataType == "":
1478                 IsValid = (len(FieldList) <= 1)
1479             else:
1480                 IsValid = (len(FieldList) <= 3)
1481
1482         if Size:
1483             try:
1484                 int(Size, 16) if Size.upper().startswith("0X") else int(Size)
1485             except:
1486                 IsValid = False
1487                 Size = -1
1488         return [str(Value), DataType, str(Size)], IsValid, 0
1489     elif PcdType == MODEL_PCD_FEATURE_FLAG:
1490         Value = FieldList[0]
1491         Size = ''
1492         IsValid = (len(FieldList) <= 1)
1493         return [Value, DataType, str(Size)], IsValid, 0
1494     elif PcdType in (MODEL_PCD_DYNAMIC_VPD, MODEL_PCD_DYNAMIC_EX_VPD):
1495         VpdOffset = FieldList[0]
1496         Value = Size = ''
1497         if not DataType == TAB_VOID:
1498             if len(FieldList) > 1:
1499                 Value = FieldList[1]
1500         else:
1501             if len(FieldList) > 1:
1502                 Size = FieldList[1]
1503             if len(FieldList) > 2:
1504                 Value = FieldList[2]
1505         if DataType == "":
1506             IsValid = (len(FieldList) <= 1)
1507         else:
1508             IsValid = (len(FieldList) <= 3)
1509         if Size:
1510             try:
1511                 int(Size, 16) if Size.upper().startswith("0X") else int(Size)
1512             except:
1513                 IsValid = False
1514                 Size = -1
1515         return [VpdOffset, str(Size), Value], IsValid, 2
1516     elif PcdType in (MODEL_PCD_DYNAMIC_HII, MODEL_PCD_DYNAMIC_EX_HII):
1517         IsValid = (3 <= len(FieldList) <= 5)
1518         HiiString = FieldList[0]
1519         Guid = Offset = Value = Attribute = ''
1520         if len(FieldList) > 1:
1521             Guid = FieldList[1]
1522         if len(FieldList) > 2:
1523             Offset = FieldList[2]
1524         if len(FieldList) > 3:
1525             Value = FieldList[3]
1526             if not Value:
1527                 IsValid = False
1528         if len(FieldList) > 4:
1529             Attribute = FieldList[4]
1530         return [HiiString, Guid, Offset, Value, Attribute], IsValid, 3
1531     return [], False, 0
1532
1533 ## AnalyzePcdData
1534 #
1535 #  Analyze the pcd Value, Datum type and TokenNumber.
1536 #  Used to avoid split issue while the value string contain "|" character
1537 #
1538 #  @param[in] Setting:  A String contain value/datum type/token number information;
1539 #
1540 #  @retval   ValueList: A List contain value, datum type and toke number.
1541 #
1542 def AnalyzePcdData(Setting):
1543     ValueList = ['', '', '']
1544
1545     ValueRe = re.compile(r'^\s*L?\".*\|.*\"')
1546     PtrValue = ValueRe.findall(Setting)
1547
1548     ValueUpdateFlag = False
1549
1550     if len(PtrValue) >= 1:
1551         Setting = re.sub(ValueRe, '', Setting)
1552         ValueUpdateFlag = True
1553
1554     TokenList = Setting.split(TAB_VALUE_SPLIT)
1555     ValueList[0:len(TokenList)] = TokenList
1556
1557     if ValueUpdateFlag:
1558         ValueList[0] = PtrValue[0]
1559
1560     return ValueList
1561
1562 ## check format of PCD value against its the datum type
1563 #
1564 # For PCD value setting
1565 #
1566 def CheckPcdDatum(Type, Value):
1567     if Type == TAB_VOID:
1568         ValueRe = re.compile(r'\s*L?\".*\"\s*$')
1569         if not (((Value.startswith('L"') or Value.startswith('"')) and Value.endswith('"'))
1570                 or (Value.startswith('{') and Value.endswith('}')) or (Value.startswith("L'") or Value.startswith("'") and Value.endswith("'"))
1571                ):
1572             return False, "Invalid value [%s] of type [%s]; must be in the form of {...} for array"\
1573                           ", \"...\" or \'...\' for string, L\"...\" or L\'...\' for unicode string" % (Value, Type)
1574         elif ValueRe.match(Value):
1575             # Check the chars in UnicodeString or CString is printable
1576             if Value.startswith("L"):
1577                 Value = Value[2:-1]
1578             else:
1579                 Value = Value[1:-1]
1580             Printset = set(string.printable)
1581             Printset.remove(TAB_PRINTCHAR_VT)
1582             Printset.add(TAB_PRINTCHAR_BS)
1583             Printset.add(TAB_PRINTCHAR_NUL)
1584             if not set(Value).issubset(Printset):
1585                 PrintList = sorted(Printset)
1586                 return False, "Invalid PCD string value of type [%s]; must be printable chars %s." % (Type, PrintList)
1587     elif Type == 'BOOLEAN':
1588         if Value not in ['TRUE', 'True', 'true', '0x1', '0x01', '1', 'FALSE', 'False', 'false', '0x0', '0x00', '0']:
1589             return False, "Invalid value [%s] of type [%s]; must be one of TRUE, True, true, 0x1, 0x01, 1"\
1590                           ", FALSE, False, false, 0x0, 0x00, 0" % (Value, Type)
1591     elif Type in [TAB_UINT8, TAB_UINT16, TAB_UINT32, TAB_UINT64]:
1592         try:
1593             Value = long(Value, 0)
1594         except:
1595             return False, "Invalid value [%s] of type [%s];"\
1596                           " must be a hexadecimal, decimal or octal in C language format." % (Value, Type)
1597     else:
1598         return True, "StructurePcd"
1599
1600     return True, ""
1601
1602 ## Split command line option string to list
1603 #
1604 # subprocess.Popen needs the args to be a sequence. Otherwise there's problem
1605 # in non-windows platform to launch command
1606 #
1607 def SplitOption(OptionString):
1608     OptionList = []
1609     LastChar = " "
1610     OptionStart = 0
1611     QuotationMark = ""
1612     for Index in range(0, len(OptionString)):
1613         CurrentChar = OptionString[Index]
1614         if CurrentChar in ['"', "'"]:
1615             if QuotationMark == CurrentChar:
1616                 QuotationMark = ""
1617             elif QuotationMark == "":
1618                 QuotationMark = CurrentChar
1619             continue
1620         elif QuotationMark:
1621             continue
1622
1623         if CurrentChar in ["/", "-"] and LastChar in [" ", "\t", "\r", "\n"]:
1624             if Index > OptionStart:
1625                 OptionList.append(OptionString[OptionStart:Index - 1])
1626             OptionStart = Index
1627         LastChar = CurrentChar
1628     OptionList.append(OptionString[OptionStart:])
1629     return OptionList
1630
1631 def CommonPath(PathList):
1632     P1 = min(PathList).split(os.path.sep)
1633     P2 = max(PathList).split(os.path.sep)
1634     for Index in xrange(min(len(P1), len(P2))):
1635         if P1[Index] != P2[Index]:
1636             return os.path.sep.join(P1[:Index])
1637     return os.path.sep.join(P1)
1638
1639 #
1640 # Convert string to C format array
1641 #
1642 def ConvertStringToByteArray(Value):
1643     Value = Value.strip()
1644     if not Value:
1645         return None
1646     if Value[0] == '{':
1647         if not Value.endswith('}'):
1648             return None
1649         Value = Value.replace(' ', '').replace('{', '').replace('}', '')
1650         ValFields = Value.split(',')
1651         try:
1652             for Index in range(len(ValFields)):
1653                 ValFields[Index] = str(int(ValFields[Index], 0))
1654         except ValueError:
1655             return None
1656         Value = '{' + ','.join(ValFields) + '}'
1657         return Value
1658
1659     Unicode = False
1660     if Value.startswith('L"'):
1661         if not Value.endswith('"'):
1662             return None
1663         Value = Value[1:]
1664         Unicode = True
1665     elif not Value.startswith('"') or not Value.endswith('"'):
1666         return None
1667
1668     Value = eval(Value)         # translate escape character
1669     NewValue = '{'
1670     for Index in range(0, len(Value)):
1671         if Unicode:
1672             NewValue = NewValue + str(ord(Value[Index]) % 0x10000) + ','
1673         else:
1674             NewValue = NewValue + str(ord(Value[Index]) % 0x100) + ','
1675     Value = NewValue + '0}'
1676     return Value
1677
1678 class PathClass(object):
1679     def __init__(self, File='', Root='', AlterRoot='', Type='', IsBinary=False,
1680                  Arch='COMMON', ToolChainFamily='', Target='', TagName='', ToolCode=''):
1681         self.Arch = Arch
1682         self.File = str(File)
1683         if os.path.isabs(self.File):
1684             self.Root = ''
1685             self.AlterRoot = ''
1686         else:
1687             self.Root = str(Root)
1688             self.AlterRoot = str(AlterRoot)
1689
1690         # Remove any '.' and '..' in path
1691         if self.Root:
1692             self.Root = mws.getWs(self.Root, self.File)
1693             self.Path = os.path.normpath(os.path.join(self.Root, self.File))
1694             self.Root = os.path.normpath(CommonPath([self.Root, self.Path]))
1695             # eliminate the side-effect of 'C:'
1696             if self.Root[-1] == ':':
1697                 self.Root += os.path.sep
1698             # file path should not start with path separator
1699             if self.Root[-1] == os.path.sep:
1700                 self.File = self.Path[len(self.Root):]
1701             else:
1702                 self.File = self.Path[len(self.Root) + 1:]
1703         else:
1704             self.Path = os.path.normpath(self.File)
1705
1706         self.SubDir, self.Name = os.path.split(self.File)
1707         self.BaseName, self.Ext = os.path.splitext(self.Name)
1708
1709         if self.Root:
1710             if self.SubDir:
1711                 self.Dir = os.path.join(self.Root, self.SubDir)
1712             else:
1713                 self.Dir = self.Root
1714         else:
1715             self.Dir = self.SubDir
1716
1717         if IsBinary:
1718             self.Type = Type
1719         else:
1720             self.Type = self.Ext.lower()
1721
1722         self.IsBinary = IsBinary
1723         self.Target = Target
1724         self.TagName = TagName
1725         self.ToolCode = ToolCode
1726         self.ToolChainFamily = ToolChainFamily
1727
1728     ## Convert the object of this class to a string
1729     #
1730     #  Convert member Path of the class to a string
1731     #
1732     #  @retval string Formatted String
1733     #
1734     def __str__(self):
1735         return self.Path
1736
1737     ## Override __eq__ function
1738     #
1739     # Check whether PathClass are the same
1740     #
1741     # @retval False The two PathClass are different
1742     # @retval True  The two PathClass are the same
1743     #
1744     def __eq__(self, Other):
1745         if isinstance(Other, type(self)):
1746             return self.Path == Other.Path
1747         else:
1748             return self.Path == str(Other)
1749
1750     ## Override __cmp__ function
1751     #
1752     # Customize the comparsion operation of two PathClass
1753     #
1754     # @retval 0     The two PathClass are different
1755     # @retval -1    The first PathClass is less than the second PathClass
1756     # @retval 1     The first PathClass is Bigger than the second PathClass
1757     def __cmp__(self, Other):
1758         if isinstance(Other, type(self)):
1759             OtherKey = Other.Path
1760         else:
1761             OtherKey = str(Other)
1762
1763         SelfKey = self.Path
1764         if SelfKey == OtherKey:
1765             return 0
1766         elif SelfKey > OtherKey:
1767             return 1
1768         else:
1769             return -1
1770
1771     ## Override __hash__ function
1772     #
1773     # Use Path as key in hash table
1774     #
1775     # @retval string Key for hash table
1776     #
1777     def __hash__(self):
1778         return hash(self.Path)
1779
1780     @cached_property
1781     def Key(self):
1782         return self.Path.upper()
1783
1784     @property
1785     def TimeStamp(self):
1786         return os.stat(self.Path)[8]
1787
1788     def Validate(self, Type='', CaseSensitive=True):
1789         if GlobalData.gCaseInsensitive:
1790             CaseSensitive = False
1791         if Type and Type.lower() != self.Type:
1792             return FILE_TYPE_MISMATCH, '%s (expect %s but got %s)' % (self.File, Type, self.Type)
1793
1794         RealFile, RealRoot = RealPath2(self.File, self.Root, self.AlterRoot)
1795         if not RealRoot and not RealFile:
1796             RealFile = self.File
1797             if self.AlterRoot:
1798                 RealFile = os.path.join(self.AlterRoot, self.File)
1799             elif self.Root:
1800                 RealFile = os.path.join(self.Root, self.File)
1801             if len (mws.getPkgPath()) == 0:
1802                 return FILE_NOT_FOUND, os.path.join(self.AlterRoot, RealFile)
1803             else:
1804                 return FILE_NOT_FOUND, "%s is not found in packages path:\n\t%s" % (self.File, '\n\t'.join(mws.getPkgPath()))
1805
1806         ErrorCode = 0
1807         ErrorInfo = ''
1808         if RealRoot != self.Root or RealFile != self.File:
1809             if CaseSensitive and (RealFile != self.File or (RealRoot != self.Root and RealRoot != self.AlterRoot)):
1810                 ErrorCode = FILE_CASE_MISMATCH
1811                 ErrorInfo = self.File + '\n\t' + RealFile + " [in file system]"
1812
1813             self.SubDir, self.Name = os.path.split(RealFile)
1814             self.BaseName, self.Ext = os.path.splitext(self.Name)
1815             if self.SubDir:
1816                 self.Dir = os.path.join(RealRoot, self.SubDir)
1817             else:
1818                 self.Dir = RealRoot
1819             self.File = RealFile
1820             self.Root = RealRoot
1821             self.Path = os.path.join(RealRoot, RealFile)
1822         return ErrorCode, ErrorInfo
1823
1824 ## Parse PE image to get the required PE informaion.
1825 #
1826 class PeImageClass():
1827     ## Constructor
1828     #
1829     #   @param  File FilePath of PeImage
1830     #
1831     def __init__(self, PeFile):
1832         self.FileName   = PeFile
1833         self.IsValid    = False
1834         self.Size       = 0
1835         self.EntryPoint = 0
1836         self.SectionAlignment  = 0
1837         self.SectionHeaderList = []
1838         self.ErrorInfo = ''
1839         try:
1840             PeObject = open(PeFile, 'rb')
1841         except:
1842             self.ErrorInfo = self.FileName + ' can not be found\n'
1843             return
1844         # Read DOS header
1845         ByteArray = array.array('B')
1846         ByteArray.fromfile(PeObject, 0x3E)
1847         ByteList = ByteArray.tolist()
1848         # DOS signature should be 'MZ'
1849         if self._ByteListToStr (ByteList[0x0:0x2]) != 'MZ':
1850             self.ErrorInfo = self.FileName + ' has no valid DOS signature MZ'
1851             return
1852
1853         # Read 4 byte PE Signature
1854         PeOffset = self._ByteListToInt(ByteList[0x3C:0x3E])
1855         PeObject.seek(PeOffset)
1856         ByteArray = array.array('B')
1857         ByteArray.fromfile(PeObject, 4)
1858         # PE signature should be 'PE\0\0'
1859         if ByteArray.tostring() != 'PE\0\0':
1860             self.ErrorInfo = self.FileName + ' has no valid PE signature PE00'
1861             return
1862
1863         # Read PE file header
1864         ByteArray = array.array('B')
1865         ByteArray.fromfile(PeObject, 0x14)
1866         ByteList = ByteArray.tolist()
1867         SecNumber = self._ByteListToInt(ByteList[0x2:0x4])
1868         if SecNumber == 0:
1869             self.ErrorInfo = self.FileName + ' has no section header'
1870             return
1871
1872         # Read PE optional header
1873         OptionalHeaderSize = self._ByteListToInt(ByteArray[0x10:0x12])
1874         ByteArray = array.array('B')
1875         ByteArray.fromfile(PeObject, OptionalHeaderSize)
1876         ByteList = ByteArray.tolist()
1877         self.EntryPoint       = self._ByteListToInt(ByteList[0x10:0x14])
1878         self.SectionAlignment = self._ByteListToInt(ByteList[0x20:0x24])
1879         self.Size             = self._ByteListToInt(ByteList[0x38:0x3C])
1880
1881         # Read each Section Header
1882         for Index in range(SecNumber):
1883             ByteArray = array.array('B')
1884             ByteArray.fromfile(PeObject, 0x28)
1885             ByteList = ByteArray.tolist()
1886             SecName  = self._ByteListToStr(ByteList[0:8])
1887             SecVirtualSize = self._ByteListToInt(ByteList[8:12])
1888             SecRawAddress  = self._ByteListToInt(ByteList[20:24])
1889             SecVirtualAddress = self._ByteListToInt(ByteList[12:16])
1890             self.SectionHeaderList.append((SecName, SecVirtualAddress, SecRawAddress, SecVirtualSize))
1891         self.IsValid = True
1892         PeObject.close()
1893
1894     def _ByteListToStr(self, ByteList):
1895         String = ''
1896         for index in range(len(ByteList)):
1897             if ByteList[index] == 0:
1898                 break
1899             String += chr(ByteList[index])
1900         return String
1901
1902     def _ByteListToInt(self, ByteList):
1903         Value = 0
1904         for index in range(len(ByteList) - 1, -1, -1):
1905             Value = (Value << 8) | int(ByteList[index])
1906         return Value
1907
1908 class DefaultStore():
1909     def __init__(self, DefaultStores ):
1910
1911         self.DefaultStores = DefaultStores
1912     def DefaultStoreID(self, DefaultStoreName):
1913         for key, value in self.DefaultStores.items():
1914             if value == DefaultStoreName:
1915                 return key
1916         return None
1917     def GetDefaultDefault(self):
1918         if not self.DefaultStores or "0" in self.DefaultStores:
1919             return "0", TAB_DEFAULT_STORES_DEFAULT
1920         else:
1921             minvalue = min(int(value_str) for value_str in self.DefaultStores)
1922             return (str(minvalue), self.DefaultStores[str(minvalue)])
1923     def GetMin(self, DefaultSIdList):
1924         if not DefaultSIdList:
1925             return TAB_DEFAULT_STORES_DEFAULT
1926         storeidset = {storeid for storeid, storename in self.DefaultStores.values() if storename in DefaultSIdList}
1927         if not storeidset:
1928             return ""
1929         minid = min(storeidset )
1930         for sid, name in self.DefaultStores.values():
1931             if sid == minid:
1932                 return name
1933
1934 class SkuClass():
1935     DEFAULT = 0
1936     SINGLE = 1
1937     MULTIPLE =2
1938
1939     def __init__(self,SkuIdentifier='', SkuIds=None):
1940         if SkuIds is None:
1941             SkuIds = {}
1942
1943         for SkuName in SkuIds:
1944             SkuId = SkuIds[SkuName][0]
1945             skuid_num = int(SkuId, 16) if SkuId.upper().startswith("0X") else int(SkuId)
1946             if skuid_num > 0xFFFFFFFFFFFFFFFF:
1947                 EdkLogger.error("build", PARAMETER_INVALID,
1948                             ExtraData = "SKU-ID [%s] value %s exceeds the max value of UINT64"
1949                                       % (SkuName, SkuId))
1950
1951         self.AvailableSkuIds = sdict()
1952         self.SkuIdSet = []
1953         self.SkuIdNumberSet = []
1954         self.SkuData = SkuIds
1955         self._SkuInherit = {}
1956         self._SkuIdentifier = SkuIdentifier
1957         if SkuIdentifier == '' or SkuIdentifier is None:
1958             self.SkuIdSet = ['DEFAULT']
1959             self.SkuIdNumberSet = ['0U']
1960         elif SkuIdentifier == 'ALL':
1961             self.SkuIdSet = SkuIds.keys()
1962             self.SkuIdNumberSet = [num[0].strip() + 'U' for num in SkuIds.values()]
1963         else:
1964             r = SkuIdentifier.split('|')
1965             self.SkuIdSet=[(r[k].strip()).upper() for k in range(len(r))]
1966             k = None
1967             try:
1968                 self.SkuIdNumberSet = [SkuIds[k][0].strip() + 'U' for k in self.SkuIdSet]
1969             except Exception:
1970                 EdkLogger.error("build", PARAMETER_INVALID,
1971                             ExtraData = "SKU-ID [%s] is not supported by the platform. [Valid SKU-ID: %s]"
1972                                       % (k, " | ".join(SkuIds.keys())))
1973         for each in self.SkuIdSet:
1974             if each in SkuIds:
1975                 self.AvailableSkuIds[each] = SkuIds[each][0]
1976             else:
1977                 EdkLogger.error("build", PARAMETER_INVALID,
1978                             ExtraData="SKU-ID [%s] is not supported by the platform. [Valid SKU-ID: %s]"
1979                                       % (each, " | ".join(SkuIds.keys())))
1980         if self.SkuUsageType != SkuClass.SINGLE:
1981             self.AvailableSkuIds.update({'DEFAULT':0, 'COMMON':0})
1982         if self.SkuIdSet:
1983             GlobalData.gSkuids = (self.SkuIdSet)
1984             if 'COMMON' in GlobalData.gSkuids:
1985                 GlobalData.gSkuids.remove('COMMON')
1986             if self.SkuUsageType == self.SINGLE:
1987                 if len(GlobalData.gSkuids) != 1:
1988                     if 'DEFAULT' in GlobalData.gSkuids:
1989                         GlobalData.gSkuids.remove('DEFAULT')
1990             if GlobalData.gSkuids:
1991                 GlobalData.gSkuids.sort()
1992
1993     def GetNextSkuId(self, skuname):
1994         if not self._SkuInherit:
1995             self._SkuInherit = {}
1996             for item in self.SkuData.values():
1997                 self._SkuInherit[item[1]]=item[2] if item[2] else "DEFAULT"
1998         return self._SkuInherit.get(skuname, "DEFAULT")
1999
2000     def GetSkuChain(self, sku):
2001         if sku == "DEFAULT":
2002             return ["DEFAULT"]
2003         skulist = [sku]
2004         nextsku = sku
2005         while True:
2006             nextsku = self.GetNextSkuId(nextsku)
2007             skulist.append(nextsku)
2008             if nextsku == "DEFAULT":
2009                 break
2010         skulist.reverse()
2011         return skulist
2012     def SkuOverrideOrder(self):
2013         skuorderset = []
2014         for skuname in self.SkuIdSet:
2015             skuorderset.append(self.GetSkuChain(skuname))
2016
2017         skuorder = []
2018         for index in range(max(len(item) for item in skuorderset)):
2019             for subset in skuorderset:
2020                 if index > len(subset)-1:
2021                     continue
2022                 if subset[index] in skuorder:
2023                     continue
2024                 skuorder.append(subset[index])
2025
2026         return skuorder
2027
2028     @property
2029     def SkuUsageType(self):
2030         if self._SkuIdentifier.upper() == "ALL":
2031             return SkuClass.MULTIPLE
2032
2033         if len(self.SkuIdSet) == 1:
2034             if self.SkuIdSet[0] == 'DEFAULT':
2035                 return SkuClass.DEFAULT
2036             return SkuClass.SINGLE
2037         if len(self.SkuIdSet) == 2 and 'DEFAULT' in self.SkuIdSet:
2038             return SkuClass.SINGLE
2039         return SkuClass.MULTIPLE
2040
2041     def DumpSkuIdArrary(self):
2042         if self.SkuUsageType == SkuClass.SINGLE:
2043             return "{0x0}"
2044         ArrayStrList = []
2045         for skuname in self.AvailableSkuIds:
2046             if skuname == "COMMON":
2047                 continue
2048             while skuname != "DEFAULT":
2049                 ArrayStrList.append(hex(int(self.AvailableSkuIds[skuname])))
2050                 skuname = self.GetNextSkuId(skuname)
2051             ArrayStrList.append("0x0")
2052         return "{{{myList}}}".format(myList=",".join(ArrayStrList))
2053
2054     @property
2055     def AvailableSkuIdSet(self):
2056         return self.AvailableSkuIds
2057
2058     @property
2059     def SystemSkuId(self):
2060         if self.SkuUsageType == SkuClass.SINGLE:
2061             if len(self.SkuIdSet) == 1:
2062                 return self.SkuIdSet[0]
2063             else:
2064                 return self.SkuIdSet[0] if self.SkuIdSet[0] != 'DEFAULT' else self.SkuIdSet[1]
2065         else:
2066             return 'DEFAULT'
2067
2068 #
2069 # Pack a registry format GUID
2070 #
2071 def PackRegistryFormatGuid(Guid):
2072     return PackGUID(Guid.split('-'))
2073
2074 ##  Get the integer value from string like "14U" or integer like 2
2075 #
2076 #   @param      Input   The object that may be either a integer value or a string
2077 #
2078 #   @retval     Value    The integer value that the input represents
2079 #
2080 def GetIntegerValue(Input):
2081     if type(Input) in (int, long):
2082         return Input
2083     String = Input
2084     if String.endswith("U"):
2085         String = String[:-1]
2086     if String.endswith("ULL"):
2087         String = String[:-3]
2088     if String.endswith("LL"):
2089         String = String[:-2]
2090
2091     if String.startswith("0x") or String.startswith("0X"):
2092         return int(String, 16)
2093     elif String == '':
2094         return 0
2095     else:
2096         return int(String)
2097
2098 #
2099 # Pack a GUID (registry format) list into a buffer and return it
2100 #
2101 def PackGUID(Guid):
2102     return pack(PACK_PATTERN_GUID,
2103                 int(Guid[0], 16),
2104                 int(Guid[1], 16),
2105                 int(Guid[2], 16),
2106                 int(Guid[3][-4:-2], 16),
2107                 int(Guid[3][-2:], 16),
2108                 int(Guid[4][-12:-10], 16),
2109                 int(Guid[4][-10:-8], 16),
2110                 int(Guid[4][-8:-6], 16),
2111                 int(Guid[4][-6:-4], 16),
2112                 int(Guid[4][-4:-2], 16),
2113                 int(Guid[4][-2:], 16)
2114                 )
2115
2116 #
2117 # Pack a GUID (byte) list into a buffer and return it
2118 #
2119 def PackByteFormatGUID(Guid):
2120     return pack(PACK_PATTERN_GUID,
2121                 Guid[0],
2122                 Guid[1],
2123                 Guid[2],
2124                 Guid[3],
2125                 Guid[4],
2126                 Guid[5],
2127                 Guid[6],
2128                 Guid[7],
2129                 Guid[8],
2130                 Guid[9],
2131                 Guid[10],
2132                 )
2133
2134 ##
2135 #
2136 # This acts like the main() function for the script, unless it is 'import'ed into another
2137 # script.
2138 #
2139 if __name__ == '__main__':
2140     pass
2141