BaseTools/Source/Python/Common/Misc.py

   1 ## @file
   2 # Common routines used by all tools
   3 #
   4 # Copyright (c) 2007 - 2018, Intel Corporation. All rights reserved.<BR>
   5 # This program and the accompanying materials
   6 # are licensed and made available under the terms and conditions of the BSD License
   7 # which accompanies this distribution.  The full text of the license may be found at
   8 # http://opensource.org/licenses/bsd-license.php
   9 #
  10 # THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS,
  11 # WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED.
  12 #
  13
  14 ##
  15 # Import Modules
  16 #
  17 from __future__ import absolute_import
  18 import Common.LongFilePathOs as os
  19 import sys
  20 import string
  21 import threading
  22 import time
  23 import re
  24 import pickle
  25 import array
  26 import shutil
  27 from struct import pack
  28 from UserDict import IterableUserDict
  29 from UserList import UserList
  30
  31 from Common import EdkLogger as EdkLogger
  32 from Common import GlobalData as GlobalData
  33 from .DataType import *
  34 from .BuildToolError import *
  35 from CommonDataClass.DataClass import *
  36 from .Parsing import GetSplitValueList
  37 from Common.LongFilePathSupport import OpenLongFilePath as open
  38 from Common.MultipleWorkspace import MultipleWorkspace as mws
  39 import uuid
  40 from CommonDataClass.Exceptions import BadExpression
  41 from Common.caching import cached_property
  42 import subprocess
  43 ## Regular expression used to find out place holders in string template
  44 gPlaceholderPattern = re.compile("\$\{([^$()\s]+)\}", re.MULTILINE | re.UNICODE)
  45
  46 ## regular expressions for map file processing
  47 startPatternGeneral = re.compile("^Start[' ']+Length[' ']+Name[' ']+Class")
  48 addressPatternGeneral = re.compile("^Address[' ']+Publics by Value[' ']+Rva\+Base")
  49 valuePatternGcc = re.compile('^([\w_\.]+) +([\da-fA-Fx]+) +([\da-fA-Fx]+)$')
  50 pcdPatternGcc = re.compile('^([\da-fA-Fx]+) +([\da-fA-Fx]+)')
  51 secReGeneral = re.compile('^([\da-fA-F]+):([\da-fA-F]+) +([\da-fA-F]+)[Hh]? +([.\w\$]+) +(\w+)', re.UNICODE)
  52
  53 ## Dictionary used to store file time stamp for quick re-access
  54 gFileTimeStampCache = {}    # {file path : file time stamp}
  55
  56 ## Dictionary used to store dependencies of files
  57 gDependencyDatabase = {}    # arch : {file path : [dependent files list]}
  58
  59 #
  60 # If a module is built more than once with different PCDs or library classes
  61 # a temporary INF file with same content is created, the temporary file is removed
  62 # when build exits.
  63 #
  64 _TempInfs = []
  65
  66 def GetVariableOffset(mapfilepath, efifilepath, varnames):
  67     """ Parse map file to get variable offset in current EFI file
  68     @param mapfilepath    Map file absolution path
  69     @param efifilepath:   EFI binary file full path
  70     @param varnames       iteratable container whose elements are variable names to be searched
  71
  72     @return List whos elements are tuple with variable name and raw offset
  73     """
  74     lines = []
  75     try:
  76         f = open(mapfilepath, 'r')
  77         lines = f.readlines()
  78         f.close()
  79     except:
  80         return None
  81
  82     if len(lines) == 0: return None
  83     firstline = lines[0].strip()
  84     if (firstline.startswith("Archive member included ") and
  85         firstline.endswith(" file (symbol)")):
  86         return _parseForGCC(lines, efifilepath, varnames)
  87     if firstline.startswith("# Path:"):
  88         return _parseForXcode(lines, efifilepath, varnames)
  89     return _parseGeneral(lines, efifilepath, varnames)
  90
  91 def _parseForXcode(lines, efifilepath, varnames):
  92     status = 0
  93     ret = []
  94     for line in lines:
  95         line = line.strip()
  96         if status == 0 and line == "# Symbols:":
  97             status = 1
  98             continue
  99         if status == 1 and len(line) != 0:
 100             for varname in varnames:
 101                 if varname in line:
 102                     # cannot pregenerate this RegEx since it uses varname from varnames.
 103                     m = re.match('^([\da-fA-FxX]+)([\s\S]*)([_]*%s)$' % varname, line)
 104                     if m is not None:
 105                         ret.append((varname, m.group(1)))
 106     return ret
 107
 108 def _parseForGCC(lines, efifilepath, varnames):
 109     """ Parse map file generated by GCC linker """
 110     status = 0
 111     sections = []
 112     varoffset = []
 113     for index, line in enumerate(lines):
 114         line = line.strip()
 115         # status machine transection
 116         if status == 0 and line == "Memory Configuration":
 117             status = 1
 118             continue
 119         elif status == 1 and line == 'Linker script and memory map':
 120             status = 2
 121             continue
 122         elif status ==2 and line == 'START GROUP':
 123             status = 3
 124             continue
 125
 126         # status handler
 127         if status == 3:
 128             m = valuePatternGcc.match(line)
 129             if m is not None:
 130                 sections.append(m.groups(0))
 131             for varname in varnames:
 132                 Str = ''
 133                 m = re.match("^.data.(%s)" % varname, line)
 134                 if m is not None:
 135                     m = re.match(".data.(%s)$" % varname, line)
 136                     if m is not None:
 137                         Str = lines[index + 1]
 138                     else:
 139                         Str = line[len(".data.%s" % varname):]
 140                     if Str:
 141                         m = pcdPatternGcc.match(Str.strip())
 142                         if m is not None:
 143                             varoffset.append((varname, int(m.groups(0)[0], 16), int(sections[-1][1], 16), sections[-1][0]))
 144
 145     if not varoffset:
 146         return []
 147     # get section information from efi file
 148     efisecs = PeImageClass(efifilepath).SectionHeaderList
 149     if efisecs is None or len(efisecs) == 0:
 150         return []
 151     #redirection
 152     redirection = 0
 153     for efisec in efisecs:
 154         for section in sections:
 155             if section[0].strip() == efisec[0].strip() and section[0].strip() == '.text':
 156                 redirection = int(section[1], 16) - efisec[1]
 157
 158     ret = []
 159     for var in varoffset:
 160         for efisec in efisecs:
 161             if var[1] >= efisec[1] and var[1] < efisec[1]+efisec[3]:
 162                 ret.append((var[0], hex(efisec[2] + var[1] - efisec[1] - redirection)))
 163     return ret
 164
 165 def _parseGeneral(lines, efifilepath, varnames):
 166     status = 0    #0 - beginning of file; 1 - PE section definition; 2 - symbol table
 167     secs  = []    # key = section name
 168     varoffset = []
 169     symRe = re.compile('^([\da-fA-F]+):([\da-fA-F]+) +([\.:\\\\\w\?@\$]+) +([\da-fA-F]+)', re.UNICODE)
 170
 171     for line in lines:
 172         line = line.strip()
 173         if startPatternGeneral.match(line):
 174             status = 1
 175             continue
 176         if addressPatternGeneral.match(line):
 177             status = 2
 178             continue
 179         if line.startswith("entry point at"):
 180             status = 3
 181             continue
 182         if status == 1 and len(line) != 0:
 183             m =  secReGeneral.match(line)
 184             assert m is not None, "Fail to parse the section in map file , line is %s" % line
 185             sec_no, sec_start, sec_length, sec_name, sec_class = m.groups(0)
 186             secs.append([int(sec_no, 16), int(sec_start, 16), int(sec_length, 16), sec_name, sec_class])
 187         if status == 2 and len(line) != 0:
 188             for varname in varnames:
 189                 m = symRe.match(line)
 190                 assert m is not None, "Fail to parse the symbol in map file, line is %s" % line
 191                 sec_no, sym_offset, sym_name, vir_addr = m.groups(0)
 192                 sec_no     = int(sec_no,     16)
 193                 sym_offset = int(sym_offset, 16)
 194                 vir_addr   = int(vir_addr,   16)
 195                 # cannot pregenerate this RegEx since it uses varname from varnames.
 196                 m2 = re.match('^[_]*(%s)' % varname, sym_name)
 197                 if m2 is not None:
 198                     # fond a binary pcd entry in map file
 199                     for sec in secs:
 200                         if sec[0] == sec_no and (sym_offset >= sec[1] and sym_offset < sec[1] + sec[2]):
 201                             varoffset.append([varname, sec[3], sym_offset, vir_addr, sec_no])
 202
 203     if not varoffset: return []
 204
 205     # get section information from efi file
 206     efisecs = PeImageClass(efifilepath).SectionHeaderList
 207     if efisecs is None or len(efisecs) == 0:
 208         return []
 209
 210     ret = []
 211     for var in varoffset:
 212         index = 0
 213         for efisec in efisecs:
 214             index = index + 1
 215             if var[1].strip() == efisec[0].strip():
 216                 ret.append((var[0], hex(efisec[2] + var[2])))
 217             elif var[4] == index:
 218                 ret.append((var[0], hex(efisec[2] + var[2])))
 219
 220     return ret
 221
 222 ## Routine to process duplicated INF
 223 #
 224 #  This function is called by following two cases:
 225 #  Case 1 in DSC:
 226 #    [components.arch]
 227 #    Pkg/module/module.inf
 228 #    Pkg/module/module.inf {
 229 #      <Defines>
 230 #        FILE_GUID = 0D1B936F-68F3-4589-AFCC-FB8B7AEBC836
 231 #    }
 232 #  Case 2 in FDF:
 233 #    INF Pkg/module/module.inf
 234 #    INF FILE_GUID = 0D1B936F-68F3-4589-AFCC-FB8B7AEBC836 Pkg/module/module.inf
 235 #
 236 #  This function copies Pkg/module/module.inf to
 237 #  Conf/.cache/0D1B936F-68F3-4589-AFCC-FB8B7AEBC836module.inf
 238 #
 239 #  @param Path     Original PathClass object
 240 #  @param BaseName New file base name
 241 #
 242 #  @retval         return the new PathClass object
 243 #
 244 def ProcessDuplicatedInf(Path, BaseName, Workspace):
 245     Filename = os.path.split(Path.File)[1]
 246     if '.' in Filename:
 247         Filename = BaseName + Path.BaseName + Filename[Filename.rfind('.'):]
 248     else:
 249         Filename = BaseName + Path.BaseName
 250
 251     #
 252     # If -N is specified on command line, cache is disabled
 253     # The directory has to be created
 254     #
 255     DbDir = os.path.split(GlobalData.gDatabasePath)[0]
 256     if not os.path.exists(DbDir):
 257         os.makedirs(DbDir)
 258     #
 259     # A temporary INF is copied to database path which must have write permission
 260     # The temporary will be removed at the end of build
 261     # In case of name conflict, the file name is
 262     # FILE_GUIDBaseName (0D1B936F-68F3-4589-AFCC-FB8B7AEBC836module.inf)
 263     #
 264     TempFullPath = os.path.join(DbDir,
 265                                 Filename)
 266     RtPath = PathClass(Path.File, Workspace)
 267     #
 268     # Modify the full path to temporary path, keep other unchanged
 269     #
 270     # To build same module more than once, the module path with FILE_GUID overridden has
 271     # the file name FILE_GUIDmodule.inf, but the relative path (self.MetaFile.File) is the real path
 272     # in DSC which is used as relative path by C files and other files in INF.
 273     # A trick was used: all module paths are PathClass instances, after the initialization
 274     # of PathClass, the PathClass.Path is overridden by the temporary INF path.
 275     #
 276     # The reason for creating a temporary INF is:
 277     # Platform.Modules which is the base to create ModuleAutoGen objects is a dictionary,
 278     # the key is the full path of INF, the value is an object to save overridden library instances, PCDs.
 279     # A different key for the same module is needed to create different output directory,
 280     # retrieve overridden PCDs, library instances.
 281     #
 282     # The BaseName is the FILE_GUID which is also the output directory name.
 283     #
 284     #
 285     RtPath.Path = TempFullPath
 286     RtPath.BaseName = BaseName
 287     #
 288     # If file exists, compare contents
 289     #
 290     if os.path.exists(TempFullPath):
 291         with open(str(Path), 'rb') as f1, open(TempFullPath, 'rb') as f2:
 292             if f1.read() == f2.read():
 293                 return RtPath
 294     _TempInfs.append(TempFullPath)
 295     shutil.copy2(str(Path), TempFullPath)
 296     return RtPath
 297
 298 ## Remove temporary created INFs whose paths were saved in _TempInfs
 299 #
 300 def ClearDuplicatedInf():
 301     while _TempInfs:
 302         File = _TempInfs.pop()
 303         if os.path.exists(File):
 304             os.remove(File)
 305
 306 ## Convert GUID string in xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx style to C structure style
 307 #
 308 #   @param      Guid    The GUID string
 309 #
 310 #   @retval     string  The GUID string in C structure style
 311 #
 312 def GuidStringToGuidStructureString(Guid):
 313     GuidList = Guid.split('-')
 314     Result = '{'
 315     for Index in range(0, 3, 1):
 316         Result = Result + '0x' + GuidList[Index] + ', '
 317     Result = Result + '{0x' + GuidList[3][0:2] + ', 0x' + GuidList[3][2:4]
 318     for Index in range(0, 12, 2):
 319         Result = Result + ', 0x' + GuidList[4][Index:Index + 2]
 320     Result += '}}'
 321     return Result
 322
 323 ## Convert GUID structure in byte array to xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
 324 #
 325 #   @param      GuidValue   The GUID value in byte array
 326 #
 327 #   @retval     string      The GUID value in xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx format
 328 #
 329 def GuidStructureByteArrayToGuidString(GuidValue):
 330     guidValueString = GuidValue.lower().replace("{", "").replace("}", "").replace(" ", "").replace(";", "")
 331     guidValueList = guidValueString.split(",")
 332     if len(guidValueList) != 16:
 333         return ''
 334         #EdkLogger.error(None, None, "Invalid GUID value string %s" % GuidValue)
 335     try:
 336         return "%02x%02x%02x%02x-%02x%02x-%02x%02x-%02x%02x-%02x%02x%02x%02x%02x%02x" % (
 337                 int(guidValueList[3], 16),
 338                 int(guidValueList[2], 16),
 339                 int(guidValueList[1], 16),
 340                 int(guidValueList[0], 16),
 341                 int(guidValueList[5], 16),
 342                 int(guidValueList[4], 16),
 343                 int(guidValueList[7], 16),
 344                 int(guidValueList[6], 16),
 345                 int(guidValueList[8], 16),
 346                 int(guidValueList[9], 16),
 347                 int(guidValueList[10], 16),
 348                 int(guidValueList[11], 16),
 349                 int(guidValueList[12], 16),
 350                 int(guidValueList[13], 16),
 351                 int(guidValueList[14], 16),
 352                 int(guidValueList[15], 16)
 353                 )
 354     except:
 355         return ''
 356
 357 ## Convert GUID string in C structure style to xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
 358 #
 359 #   @param      GuidValue   The GUID value in C structure format
 360 #
 361 #   @retval     string      The GUID value in xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx format
 362 #
 363 def GuidStructureStringToGuidString(GuidValue):
 364     if not GlobalData.gGuidCFormatPattern.match(GuidValue):
 365         return ''
 366     guidValueString = GuidValue.lower().replace("{", "").replace("}", "").replace(" ", "").replace(";", "")
 367     guidValueList = guidValueString.split(",")
 368     if len(guidValueList) != 11:
 369         return ''
 370         #EdkLogger.error(None, None, "Invalid GUID value string %s" % GuidValue)
 371     try:
 372         return "%08x-%04x-%04x-%02x%02x-%02x%02x%02x%02x%02x%02x" % (
 373                 int(guidValueList[0], 16),
 374                 int(guidValueList[1], 16),
 375                 int(guidValueList[2], 16),
 376                 int(guidValueList[3], 16),
 377                 int(guidValueList[4], 16),
 378                 int(guidValueList[5], 16),
 379                 int(guidValueList[6], 16),
 380                 int(guidValueList[7], 16),
 381                 int(guidValueList[8], 16),
 382                 int(guidValueList[9], 16),
 383                 int(guidValueList[10], 16)
 384                 )
 385     except:
 386         return ''
 387
 388 ## Convert GUID string in C structure style to xxxxxxxx_xxxx_xxxx_xxxx_xxxxxxxxxxxx
 389 #
 390 #   @param      GuidValue   The GUID value in C structure format
 391 #
 392 #   @retval     string      The GUID value in xxxxxxxx_xxxx_xxxx_xxxx_xxxxxxxxxxxx format
 393 #
 394 def GuidStructureStringToGuidValueName(GuidValue):
 395     guidValueString = GuidValue.lower().replace("{", "").replace("}", "").replace(" ", "")
 396     guidValueList = guidValueString.split(",")
 397     if len(guidValueList) != 11:
 398         EdkLogger.error(None, FORMAT_INVALID, "Invalid GUID value string [%s]" % GuidValue)
 399     return "%08x_%04x_%04x_%02x%02x_%02x%02x%02x%02x%02x%02x" % (
 400             int(guidValueList[0], 16),
 401             int(guidValueList[1], 16),
 402             int(guidValueList[2], 16),
 403             int(guidValueList[3], 16),
 404             int(guidValueList[4], 16),
 405             int(guidValueList[5], 16),
 406             int(guidValueList[6], 16),
 407             int(guidValueList[7], 16),
 408             int(guidValueList[8], 16),
 409             int(guidValueList[9], 16),
 410             int(guidValueList[10], 16)
 411             )
 412
 413 ## Create directories
 414 #
 415 #   @param      Directory   The directory name
 416 #
 417 def CreateDirectory(Directory):
 418     if Directory is None or Directory.strip() == "":
 419         return True
 420     try:
 421         if not os.access(Directory, os.F_OK):
 422             os.makedirs(Directory)
 423     except:
 424         return False
 425     return True
 426
 427 ## Remove directories, including files and sub-directories in it
 428 #
 429 #   @param      Directory   The directory name
 430 #
 431 def RemoveDirectory(Directory, Recursively=False):
 432     if Directory is None or Directory.strip() == "" or not os.path.exists(Directory):
 433         return
 434     if Recursively:
 435         CurrentDirectory = os.getcwd()
 436         os.chdir(Directory)
 437         for File in os.listdir("."):
 438             if os.path.isdir(File):
 439                 RemoveDirectory(File, Recursively)
 440             else:
 441                 os.remove(File)
 442         os.chdir(CurrentDirectory)
 443     os.rmdir(Directory)
 444
 445 ## Store content in file
 446 #
 447 #  This method is used to save file only when its content is changed. This is
 448 #  quite useful for "make" system to decide what will be re-built and what won't.
 449 #
 450 #   @param      File            The path of file
 451 #   @param      Content         The new content of the file
 452 #   @param      IsBinaryFile    The flag indicating if the file is binary file or not
 453 #
 454 #   @retval     True            If the file content is changed and the file is renewed
 455 #   @retval     False           If the file content is the same
 456 #
 457 def SaveFileOnChange(File, Content, IsBinaryFile=True):
 458     if not IsBinaryFile:
 459         Content = Content.replace("\n", os.linesep)
 460
 461     if os.path.exists(File):
 462         try:
 463             if Content == open(File, "rb").read():
 464                 return False
 465         except:
 466             EdkLogger.error(None, FILE_OPEN_FAILURE, ExtraData=File)
 467
 468     DirName = os.path.dirname(File)
 469     if not CreateDirectory(DirName):
 470         EdkLogger.error(None, FILE_CREATE_FAILURE, "Could not create directory %s" % DirName)
 471     else:
 472         if DirName == '':
 473             DirName = os.getcwd()
 474         if not os.access(DirName, os.W_OK):
 475             EdkLogger.error(None, PERMISSION_FAILURE, "Do not have write permission on directory %s" % DirName)
 476
 477     try:
 478         if GlobalData.gIsWindows:
 479             try:
 480                 from .PyUtility import SaveFileToDisk
 481                 if not SaveFileToDisk(File, Content):
 482                     EdkLogger.error(None, FILE_CREATE_FAILURE, ExtraData=File)
 483             except:
 484                 Fd = open(File, "wb")
 485                 Fd.write(Content)
 486                 Fd.close()
 487         else:
 488             Fd = open(File, "wb")
 489             Fd.write(Content)
 490             Fd.close()
 491     except IOError as X:
 492         EdkLogger.error(None, FILE_CREATE_FAILURE, ExtraData='IOError %s' % X)
 493
 494     return True
 495
 496 ## Make a Python object persistent on file system
 497 #
 498 #   @param      Data    The object to be stored in file
 499 #   @param      File    The path of file to store the object
 500 #
 501 def DataDump(Data, File):
 502     Fd = None
 503     try:
 504         Fd = open(File, 'wb')
 505         pickle.dump(Data, Fd, pickle.HIGHEST_PROTOCOL)
 506     except:
 507         EdkLogger.error("", FILE_OPEN_FAILURE, ExtraData=File, RaiseError=False)
 508     finally:
 509         if Fd is not None:
 510             Fd.close()
 511
 512 ## Restore a Python object from a file
 513 #
 514 #   @param      File    The path of file stored the object
 515 #
 516 #   @retval     object  A python object
 517 #   @retval     None    If failure in file operation
 518 #
 519 def DataRestore(File):
 520     Data = None
 521     Fd = None
 522     try:
 523         Fd = open(File, 'rb')
 524         Data = pickle.load(Fd)
 525     except Exception as e:
 526         EdkLogger.verbose("Failed to load [%s]\n\t%s" % (File, str(e)))
 527         Data = None
 528     finally:
 529         if Fd is not None:
 530             Fd.close()
 531     return Data
 532
 533 ## Retrieve and cache the real path name in file system
 534 #
 535 #   @param      Root    The root directory of path relative to
 536 #
 537 #   @retval     str     The path string if the path exists
 538 #   @retval     None    If path doesn't exist
 539 #
 540 class DirCache:
 541     _CACHE_ = set()
 542     _UPPER_CACHE_ = {}
 543
 544     def __init__(self, Root):
 545         self._Root = Root
 546         for F in os.listdir(Root):
 547             self._CACHE_.add(F)
 548             self._UPPER_CACHE_[F.upper()] = F
 549
 550     # =[] operator
 551     def __getitem__(self, Path):
 552         Path = Path[len(os.path.commonprefix([Path, self._Root])):]
 553         if not Path:
 554             return self._Root
 555         if Path and Path[0] == os.path.sep:
 556             Path = Path[1:]
 557         if Path in self._CACHE_:
 558             return os.path.join(self._Root, Path)
 559         UpperPath = Path.upper()
 560         if UpperPath in self._UPPER_CACHE_:
 561             return os.path.join(self._Root, self._UPPER_CACHE_[UpperPath])
 562
 563         IndexList = []
 564         LastSepIndex = -1
 565         SepIndex = Path.find(os.path.sep)
 566         while SepIndex > -1:
 567             Parent = UpperPath[:SepIndex]
 568             if Parent not in self._UPPER_CACHE_:
 569                 break
 570             LastSepIndex = SepIndex
 571             SepIndex = Path.find(os.path.sep, LastSepIndex + 1)
 572
 573         if LastSepIndex == -1:
 574             return None
 575
 576         Cwd = os.getcwd()
 577         os.chdir(self._Root)
 578         SepIndex = LastSepIndex
 579         while SepIndex > -1:
 580             Parent = Path[:SepIndex]
 581             ParentKey = UpperPath[:SepIndex]
 582             if ParentKey not in self._UPPER_CACHE_:
 583                 os.chdir(Cwd)
 584                 return None
 585
 586             if Parent in self._CACHE_:
 587                 ParentDir = Parent
 588             else:
 589                 ParentDir = self._UPPER_CACHE_[ParentKey]
 590             for F in os.listdir(ParentDir):
 591                 Dir = os.path.join(ParentDir, F)
 592                 self._CACHE_.add(Dir)
 593                 self._UPPER_CACHE_[Dir.upper()] = Dir
 594
 595             SepIndex = Path.find(os.path.sep, SepIndex + 1)
 596
 597         os.chdir(Cwd)
 598         if Path in self._CACHE_:
 599             return os.path.join(self._Root, Path)
 600         elif UpperPath in self._UPPER_CACHE_:
 601             return os.path.join(self._Root, self._UPPER_CACHE_[UpperPath])
 602         return None
 603
 604 def RealPath(File, Dir='', OverrideDir=''):
 605     NewFile = os.path.normpath(os.path.join(Dir, File))
 606     NewFile = GlobalData.gAllFiles[NewFile]
 607     if not NewFile and OverrideDir:
 608         NewFile = os.path.normpath(os.path.join(OverrideDir, File))
 609         NewFile = GlobalData.gAllFiles[NewFile]
 610     return NewFile
 611
 612 def RealPath2(File, Dir='', OverrideDir=''):
 613     NewFile = None
 614     if OverrideDir:
 615         NewFile = GlobalData.gAllFiles[os.path.normpath(os.path.join(OverrideDir, File))]
 616         if NewFile:
 617             if OverrideDir[-1] == os.path.sep:
 618                 return NewFile[len(OverrideDir):], NewFile[0:len(OverrideDir)]
 619             else:
 620                 return NewFile[len(OverrideDir) + 1:], NewFile[0:len(OverrideDir)]
 621     if GlobalData.gAllFiles:
 622         NewFile = GlobalData.gAllFiles[os.path.normpath(os.path.join(Dir, File))]
 623     if not NewFile:
 624         NewFile = os.path.normpath(os.path.join(Dir, File))
 625         if not os.path.exists(NewFile):
 626             return None, None
 627     if NewFile:
 628         if Dir:
 629             if Dir[-1] == os.path.sep:
 630                 return NewFile[len(Dir):], NewFile[0:len(Dir)]
 631             else:
 632                 return NewFile[len(Dir) + 1:], NewFile[0:len(Dir)]
 633         else:
 634             return NewFile, ''
 635
 636     return None, None
 637
 638 ## Get GUID value from given packages
 639 #
 640 #   @param      CName           The CName of the GUID
 641 #   @param      PackageList     List of packages looking-up in
 642 #   @param      Inffile         The driver file
 643 #
 644 #   @retval     GuidValue   if the CName is found in any given package
 645 #   @retval     None        if the CName is not found in all given packages
 646 #
 647 def GuidValue(CName, PackageList, Inffile = None):
 648     for P in PackageList:
 649         GuidKeys = P.Guids.keys()
 650         if Inffile and P._PrivateGuids:
 651             if not Inffile.startswith(P.MetaFile.Dir):
 652                 GuidKeys = [x for x in P.Guids if x not in P._PrivateGuids]
 653         if CName in GuidKeys:
 654             return P.Guids[CName]
 655     return None
 656
 657 ## Get Protocol value from given packages
 658 #
 659 #   @param      CName           The CName of the GUID
 660 #   @param      PackageList     List of packages looking-up in
 661 #   @param      Inffile         The driver file
 662 #
 663 #   @retval     GuidValue   if the CName is found in any given package
 664 #   @retval     None        if the CName is not found in all given packages
 665 #
 666 def ProtocolValue(CName, PackageList, Inffile = None):
 667     for P in PackageList:
 668         ProtocolKeys = P.Protocols.keys()
 669         if Inffile and P._PrivateProtocols:
 670             if not Inffile.startswith(P.MetaFile.Dir):
 671                 ProtocolKeys = [x for x in P.Protocols if x not in P._PrivateProtocols]
 672         if CName in ProtocolKeys:
 673             return P.Protocols[CName]
 674     return None
 675
 676 ## Get PPI value from given packages
 677 #
 678 #   @param      CName           The CName of the GUID
 679 #   @param      PackageList     List of packages looking-up in
 680 #   @param      Inffile         The driver file
 681 #
 682 #   @retval     GuidValue   if the CName is found in any given package
 683 #   @retval     None        if the CName is not found in all given packages
 684 #
 685 def PpiValue(CName, PackageList, Inffile = None):
 686     for P in PackageList:
 687         PpiKeys = P.Ppis.keys()
 688         if Inffile and P._PrivatePpis:
 689             if not Inffile.startswith(P.MetaFile.Dir):
 690                 PpiKeys = [x for x in P.Ppis if x not in P._PrivatePpis]
 691         if CName in PpiKeys:
 692             return P.Ppis[CName]
 693     return None
 694
 695 ## A string template class
 696 #
 697 #  This class implements a template for string replacement. A string template
 698 #  looks like following
 699 #
 700 #       ${BEGIN} other_string ${placeholder_name} other_string ${END}
 701 #
 702 #  The string between ${BEGIN} and ${END} will be repeated as many times as the
 703 #  length of "placeholder_name", which is a list passed through a dict. The
 704 #  "placeholder_name" is the key name of the dict. The ${BEGIN} and ${END} can
 705 #  be not used and, in this case, the "placeholder_name" must not a list and it
 706 #  will just be replaced once.
 707 #
 708 class TemplateString(object):
 709     _REPEAT_START_FLAG = "BEGIN"
 710     _REPEAT_END_FLAG = "END"
 711
 712     class Section(object):
 713         _LIST_TYPES = [type([]), type(set()), type((0,))]
 714
 715         def __init__(self, TemplateSection, PlaceHolderList):
 716             self._Template = TemplateSection
 717             self._PlaceHolderList = []
 718
 719             # Split the section into sub-sections according to the position of placeholders
 720             if PlaceHolderList:
 721                 self._SubSectionList = []
 722                 SubSectionStart = 0
 723                 #
 724                 # The placeholders passed in must be in the format of
 725                 #
 726                 #   PlaceHolderName, PlaceHolderStartPoint, PlaceHolderEndPoint
 727                 #
 728                 for PlaceHolder, Start, End in PlaceHolderList:
 729                     self._SubSectionList.append(TemplateSection[SubSectionStart:Start])
 730                     self._SubSectionList.append(TemplateSection[Start:End])
 731                     self._PlaceHolderList.append(PlaceHolder)
 732                     SubSectionStart = End
 733                 if SubSectionStart < len(TemplateSection):
 734                     self._SubSectionList.append(TemplateSection[SubSectionStart:])
 735             else:
 736                 self._SubSectionList = [TemplateSection]
 737
 738         def __str__(self):
 739             return self._Template + " : " + str(self._PlaceHolderList)
 740
 741         def Instantiate(self, PlaceHolderValues):
 742             RepeatTime = -1
 743             RepeatPlaceHolders = {}
 744             NonRepeatPlaceHolders = {}
 745
 746             for PlaceHolder in self._PlaceHolderList:
 747                 if PlaceHolder not in PlaceHolderValues:
 748                     continue
 749                 Value = PlaceHolderValues[PlaceHolder]
 750                 if type(Value) in self._LIST_TYPES:
 751                     if RepeatTime < 0:
 752                         RepeatTime = len(Value)
 753                     elif RepeatTime != len(Value):
 754                         EdkLogger.error(
 755                                     "TemplateString",
 756                                     PARAMETER_INVALID,
 757                                     "${%s} has different repeat time from others!" % PlaceHolder,
 758                                     ExtraData=str(self._Template)
 759                                     )
 760                     RepeatPlaceHolders["${%s}" % PlaceHolder] = Value
 761                 else:
 762                     NonRepeatPlaceHolders["${%s}" % PlaceHolder] = Value
 763
 764             if NonRepeatPlaceHolders:
 765                 StringList = []
 766                 for S in self._SubSectionList:
 767                     if S not in NonRepeatPlaceHolders:
 768                         StringList.append(S)
 769                     else:
 770                         StringList.append(str(NonRepeatPlaceHolders[S]))
 771             else:
 772                 StringList = self._SubSectionList
 773
 774             if RepeatPlaceHolders:
 775                 TempStringList = []
 776                 for Index in range(RepeatTime):
 777                     for S in StringList:
 778                         if S not in RepeatPlaceHolders:
 779                             TempStringList.append(S)
 780                         else:
 781                             TempStringList.append(str(RepeatPlaceHolders[S][Index]))
 782                 StringList = TempStringList
 783
 784             return "".join(StringList)
 785
 786     ## Constructor
 787     def __init__(self, Template=None):
 788         self.String = ''
 789         self.IsBinary = False
 790         self._Template = Template
 791         self._TemplateSectionList = self._Parse(Template)
 792
 793     ## str() operator
 794     #
 795     #   @retval     string  The string replaced
 796     #
 797     def __str__(self):
 798         return self.String
 799
 800     ## Split the template string into fragments per the ${BEGIN} and ${END} flags
 801     #
 802     #   @retval     list    A list of TemplateString.Section objects
 803     #
 804     def _Parse(self, Template):
 805         SectionStart = 0
 806         SearchFrom = 0
 807         MatchEnd = 0
 808         PlaceHolderList = []
 809         TemplateSectionList = []
 810         while Template:
 811             MatchObj = gPlaceholderPattern.search(Template, SearchFrom)
 812             if not MatchObj:
 813                 if MatchEnd <= len(Template):
 814                     TemplateSection = TemplateString.Section(Template[SectionStart:], PlaceHolderList)
 815                     TemplateSectionList.append(TemplateSection)
 816                 break
 817
 818             MatchString = MatchObj.group(1)
 819             MatchStart = MatchObj.start()
 820             MatchEnd = MatchObj.end()
 821
 822             if MatchString == self._REPEAT_START_FLAG:
 823                 if MatchStart > SectionStart:
 824                     TemplateSection = TemplateString.Section(Template[SectionStart:MatchStart], PlaceHolderList)
 825                     TemplateSectionList.append(TemplateSection)
 826                 SectionStart = MatchEnd
 827                 PlaceHolderList = []
 828             elif MatchString == self._REPEAT_END_FLAG:
 829                 TemplateSection = TemplateString.Section(Template[SectionStart:MatchStart], PlaceHolderList)
 830                 TemplateSectionList.append(TemplateSection)
 831                 SectionStart = MatchEnd
 832                 PlaceHolderList = []
 833             else:
 834                 PlaceHolderList.append((MatchString, MatchStart - SectionStart, MatchEnd - SectionStart))
 835             SearchFrom = MatchEnd
 836         return TemplateSectionList
 837
 838     ## Replace the string template with dictionary of placeholders and append it to previous one
 839     #
 840     #   @param      AppendString    The string template to append
 841     #   @param      Dictionary      The placeholder dictionaries
 842     #
 843     def Append(self, AppendString, Dictionary=None):
 844         if Dictionary:
 845             SectionList = self._Parse(AppendString)
 846             self.String += "".join(S.Instantiate(Dictionary) for S in SectionList)
 847         else:
 848             self.String += AppendString
 849
 850     ## Replace the string template with dictionary of placeholders
 851     #
 852     #   @param      Dictionary      The placeholder dictionaries
 853     #
 854     #   @retval     str             The string replaced with placeholder values
 855     #
 856     def Replace(self, Dictionary=None):
 857         return "".join(S.Instantiate(Dictionary) for S in self._TemplateSectionList)
 858
 859 ## Progress indicator class
 860 #
 861 #  This class makes use of thread to print progress on console.
 862 #
 863 class Progressor:
 864     # for avoiding deadloop
 865     _StopFlag = None
 866     _ProgressThread = None
 867     _CheckInterval = 0.25
 868
 869     ## Constructor
 870     #
 871     #   @param      OpenMessage     The string printed before progress charaters
 872     #   @param      CloseMessage    The string printed after progress charaters
 873     #   @param      ProgressChar    The charater used to indicate the progress
 874     #   @param      Interval        The interval in seconds between two progress charaters
 875     #
 876     def __init__(self, OpenMessage="", CloseMessage="", ProgressChar='.', Interval=1.0):
 877         self.PromptMessage = OpenMessage
 878         self.CodaMessage = CloseMessage
 879         self.ProgressChar = ProgressChar
 880         self.Interval = Interval
 881         if Progressor._StopFlag is None:
 882             Progressor._StopFlag = threading.Event()
 883
 884     ## Start to print progress charater
 885     #
 886     #   @param      OpenMessage     The string printed before progress charaters
 887     #
 888     def Start(self, OpenMessage=None):
 889         if OpenMessage is not None:
 890             self.PromptMessage = OpenMessage
 891         Progressor._StopFlag.clear()
 892         if Progressor._ProgressThread is None:
 893             Progressor._ProgressThread = threading.Thread(target=self._ProgressThreadEntry)
 894             Progressor._ProgressThread.setDaemon(False)
 895             Progressor._ProgressThread.start()
 896
 897     ## Stop printing progress charater
 898     #
 899     #   @param      CloseMessage    The string printed after progress charaters
 900     #
 901     def Stop(self, CloseMessage=None):
 902         OriginalCodaMessage = self.CodaMessage
 903         if CloseMessage is not None:
 904             self.CodaMessage = CloseMessage
 905         self.Abort()
 906         self.CodaMessage = OriginalCodaMessage
 907
 908     ## Thread entry method
 909     def _ProgressThreadEntry(self):
 910         sys.stdout.write(self.PromptMessage + " ")
 911         sys.stdout.flush()
 912         TimeUp = 0.0
 913         while not Progressor._StopFlag.isSet():
 914             if TimeUp <= 0.0:
 915                 sys.stdout.write(self.ProgressChar)
 916                 sys.stdout.flush()
 917                 TimeUp = self.Interval
 918             time.sleep(self._CheckInterval)
 919             TimeUp -= self._CheckInterval
 920         sys.stdout.write(" " + self.CodaMessage + "\n")
 921         sys.stdout.flush()
 922
 923     ## Abort the progress display
 924     @staticmethod
 925     def Abort():
 926         if Progressor._StopFlag is not None:
 927             Progressor._StopFlag.set()
 928         if Progressor._ProgressThread is not None:
 929             Progressor._ProgressThread.join()
 930             Progressor._ProgressThread = None
 931
 932 ## A dict which can access its keys and/or values orderly
 933 #
 934 #  The class implements a new kind of dict which its keys or values can be
 935 #  accessed in the order they are added into the dict. It guarantees the order
 936 #  by making use of an internal list to keep a copy of keys.
 937 #
 938 class sdict(IterableUserDict):
 939     ## Constructor
 940     def __init__(self):
 941         IterableUserDict.__init__(self)
 942         self._key_list = []
 943
 944     ## [] operator
 945     def __setitem__(self, key, value):
 946         if key not in self._key_list:
 947             self._key_list.append(key)
 948         IterableUserDict.__setitem__(self, key, value)
 949
 950     ## del operator
 951     def __delitem__(self, key):
 952         self._key_list.remove(key)
 953         IterableUserDict.__delitem__(self, key)
 954
 955     ## used in "for k in dict" loop to ensure the correct order
 956     def __iter__(self):
 957         return self.iterkeys()
 958
 959     ## len() support
 960     def __len__(self):
 961         return len(self._key_list)
 962
 963     ## "in" test support
 964     def __contains__(self, key):
 965         return key in self._key_list
 966
 967     ## indexof support
 968     def index(self, key):
 969         return self._key_list.index(key)
 970
 971     ## insert support
 972     def insert(self, key, newkey, newvalue, order):
 973         index = self._key_list.index(key)
 974         if order == 'BEFORE':
 975             self._key_list.insert(index, newkey)
 976             IterableUserDict.__setitem__(self, newkey, newvalue)
 977         elif order == 'AFTER':
 978             self._key_list.insert(index + 1, newkey)
 979             IterableUserDict.__setitem__(self, newkey, newvalue)
 980
 981     ## append support
 982     def append(self, sdict):
 983         for key in sdict:
 984             if key not in self._key_list:
 985                 self._key_list.append(key)
 986             IterableUserDict.__setitem__(self, key, sdict[key])
 987
 988     def has_key(self, key):
 989         return key in self._key_list
 990
 991     ## Empty the dict
 992     def clear(self):
 993         self._key_list = []
 994         IterableUserDict.clear(self)
 995
 996     ## Return a copy of keys
 997     def keys(self):
 998         keys = []
 999         for key in self._key_list:
1000             keys.append(key)
1001         return keys
1002
1003     ## Return a copy of values
1004     def values(self):
1005         values = []
1006         for key in self._key_list:
1007             values.append(self[key])
1008         return values
1009
1010     ## Return a copy of (key, value) list
1011     def items(self):
1012         items = []
1013         for key in self._key_list:
1014             items.append((key, self[key]))
1015         return items
1016
1017     ## Iteration support
1018     def iteritems(self):
1019         return iter(self.items())
1020
1021     ## Keys interation support
1022     def iterkeys(self):
1023         return iter(self.keys())
1024
1025     ## Values interation support
1026     def itervalues(self):
1027         return iter(self.values())
1028
1029     ## Return value related to a key, and remove the (key, value) from the dict
1030     def pop(self, key, *dv):
1031         value = None
1032         if key in self._key_list:
1033             value = self[key]
1034             self.__delitem__(key)
1035         elif len(dv) != 0 :
1036             value = kv[0]
1037         return value
1038
1039     ## Return (key, value) pair, and remove the (key, value) from the dict
1040     def popitem(self):
1041         key = self._key_list[-1]
1042         value = self[key]
1043         self.__delitem__(key)
1044         return key, value
1045
1046     def update(self, dict=None, **kwargs):
1047         if dict is not None:
1048             for k, v in dict.items():
1049                 self[k] = v
1050         if len(kwargs):
1051             for k, v in kwargs.items():
1052                 self[k] = v
1053
1054 ## Dictionary with restricted keys
1055 #
1056 class rdict(dict):
1057     ## Constructor
1058     def __init__(self, KeyList):
1059         for Key in KeyList:
1060             dict.__setitem__(self, Key, "")
1061
1062     ## []= operator
1063     def __setitem__(self, key, value):
1064         if key not in self:
1065             EdkLogger.error("RestrictedDict", ATTRIBUTE_SET_FAILURE, "Key [%s] is not allowed" % key,
1066                             ExtraData=", ".join(dict.keys(self)))
1067         dict.__setitem__(self, key, value)
1068
1069     ## =[] operator
1070     def __getitem__(self, key):
1071         if key not in self:
1072             return ""
1073         return dict.__getitem__(self, key)
1074
1075     ## del operator
1076     def __delitem__(self, key):
1077         EdkLogger.error("RestrictedDict", ATTRIBUTE_ACCESS_DENIED, ExtraData="del")
1078
1079     ## Empty the dict
1080     def clear(self):
1081         for Key in self:
1082             self.__setitem__(Key, "")
1083
1084     ## Return value related to a key, and remove the (key, value) from the dict
1085     def pop(self, key, *dv):
1086         EdkLogger.error("RestrictedDict", ATTRIBUTE_ACCESS_DENIED, ExtraData="pop")
1087
1088     ## Return (key, value) pair, and remove the (key, value) from the dict
1089     def popitem(self):
1090         EdkLogger.error("RestrictedDict", ATTRIBUTE_ACCESS_DENIED, ExtraData="popitem")
1091
1092 ## Dictionary using prioritized list as key
1093 #
1094 class tdict:
1095     _ListType = type([])
1096     _TupleType = type(())
1097     _Wildcard = 'COMMON'
1098     _ValidWildcardList = ['COMMON', 'DEFAULT', 'ALL', '*', 'PLATFORM']
1099
1100     def __init__(self, _Single_=False, _Level_=2):
1101         self._Level_ = _Level_
1102         self.data = {}
1103         self._Single_ = _Single_
1104
1105     # =[] operator
1106     def __getitem__(self, key):
1107         KeyType = type(key)
1108         RestKeys = None
1109         if KeyType == self._ListType or KeyType == self._TupleType:
1110             FirstKey = key[0]
1111             if len(key) > 1:
1112                 RestKeys = key[1:]
1113             elif self._Level_ > 1:
1114                 RestKeys = [self._Wildcard for i in range(0, self._Level_ - 1)]
1115         else:
1116             FirstKey = key
1117             if self._Level_ > 1:
1118                 RestKeys = [self._Wildcard for i in range(0, self._Level_ - 1)]
1119
1120         if FirstKey is None or str(FirstKey).upper() in self._ValidWildcardList:
1121             FirstKey = self._Wildcard
1122
1123         if self._Single_:
1124             return self._GetSingleValue(FirstKey, RestKeys)
1125         else:
1126             return self._GetAllValues(FirstKey, RestKeys)
1127
1128     def _GetSingleValue(self, FirstKey, RestKeys):
1129         Value = None
1130         #print "%s-%s" % (FirstKey, self._Level_) ,
1131         if self._Level_ > 1:
1132             if FirstKey == self._Wildcard:
1133                 if FirstKey in self.data:
1134                     Value = self.data[FirstKey][RestKeys]
1135                 if Value is None:
1136                     for Key in self.data:
1137                         Value = self.data[Key][RestKeys]
1138                         if Value is not None: break
1139             else:
1140                 if FirstKey in self.data:
1141                     Value = self.data[FirstKey][RestKeys]
1142                 if Value is None and self._Wildcard in self.data:
1143                     #print "Value=None"
1144                     Value = self.data[self._Wildcard][RestKeys]
1145         else:
1146             if FirstKey == self._Wildcard:
1147                 if FirstKey in self.data:
1148                     Value = self.data[FirstKey]
1149                 if Value is None:
1150                     for Key in self.data:
1151                         Value = self.data[Key]
1152                         if Value is not None: break
1153             else:
1154                 if FirstKey in self.data:
1155                     Value = self.data[FirstKey]
1156                 elif self._Wildcard in self.data:
1157                     Value = self.data[self._Wildcard]
1158         return Value
1159
1160     def _GetAllValues(self, FirstKey, RestKeys):
1161         Value = []
1162         if self._Level_ > 1:
1163             if FirstKey == self._Wildcard:
1164                 for Key in self.data:
1165                     Value += self.data[Key][RestKeys]
1166             else:
1167                 if FirstKey in self.data:
1168                     Value += self.data[FirstKey][RestKeys]
1169                 if self._Wildcard in self.data:
1170                     Value += self.data[self._Wildcard][RestKeys]
1171         else:
1172             if FirstKey == self._Wildcard:
1173                 for Key in self.data:
1174                     Value.append(self.data[Key])
1175             else:
1176                 if FirstKey in self.data:
1177                     Value.append(self.data[FirstKey])
1178                 if self._Wildcard in self.data:
1179                     Value.append(self.data[self._Wildcard])
1180         return Value
1181
1182     ## []= operator
1183     def __setitem__(self, key, value):
1184         KeyType = type(key)
1185         RestKeys = None
1186         if KeyType == self._ListType or KeyType == self._TupleType:
1187             FirstKey = key[0]
1188             if len(key) > 1:
1189                 RestKeys = key[1:]
1190             else:
1191                 RestKeys = [self._Wildcard for i in range(0, self._Level_ - 1)]
1192         else:
1193             FirstKey = key
1194             if self._Level_ > 1:
1195                 RestKeys = [self._Wildcard for i in range(0, self._Level_ - 1)]
1196
1197         if FirstKey in self._ValidWildcardList:
1198             FirstKey = self._Wildcard
1199
1200         if FirstKey not in self.data and self._Level_ > 0:
1201             self.data[FirstKey] = tdict(self._Single_, self._Level_ - 1)
1202
1203         if self._Level_ > 1:
1204             self.data[FirstKey][RestKeys] = value
1205         else:
1206             self.data[FirstKey] = value
1207
1208     def SetGreedyMode(self):
1209         self._Single_ = False
1210         if self._Level_ > 1:
1211             for Key in self.data:
1212                 self.data[Key].SetGreedyMode()
1213
1214     def SetSingleMode(self):
1215         self._Single_ = True
1216         if self._Level_ > 1:
1217             for Key in self.data:
1218                 self.data[Key].SetSingleMode()
1219
1220     def GetKeys(self, KeyIndex=0):
1221         assert KeyIndex >= 0
1222         if KeyIndex == 0:
1223             return set(self.data.keys())
1224         else:
1225             keys = set()
1226             for Key in self.data:
1227                 keys |= self.data[Key].GetKeys(KeyIndex - 1)
1228             return keys
1229
1230 def IsFieldValueAnArray (Value):
1231     Value = Value.strip()
1232     if Value.startswith(TAB_GUID) and Value.endswith(')'):
1233         return True
1234     if Value.startswith('L"') and Value.endswith('"')  and len(list(Value[2:-1])) > 1:
1235         return True
1236     if Value[0] == '"' and Value[-1] == '"' and len(list(Value[1:-1])) > 1:
1237         return True
1238     if Value[0] == '{' and Value[-1] == '}':
1239         return True
1240     if Value.startswith("L'") and Value.endswith("'") and len(list(Value[2:-1])) > 1:
1241         return True
1242     if Value[0] == "'" and Value[-1] == "'" and len(list(Value[1:-1])) > 1:
1243         return True
1244     return False
1245
1246 def AnalyzePcdExpression(Setting):
1247     Setting = Setting.strip()
1248     # There might be escaped quote in a string: \", \\\" , \', \\\'
1249     Data = Setting
1250     # There might be '|' in string and in ( ... | ... ), replace it with '-'
1251     NewStr = ''
1252     InSingleQuoteStr = False
1253     InDoubleQuoteStr = False
1254     Pair = 0
1255     for Index, ch in enumerate(Data):
1256         if ch == '"' and not InSingleQuoteStr:
1257             if Data[Index - 1] != '\\':
1258                 InDoubleQuoteStr = not InDoubleQuoteStr
1259         elif ch == "'" and not InDoubleQuoteStr:
1260             if Data[Index - 1] != '\\':
1261                 InSingleQuoteStr = not InSingleQuoteStr
1262         elif ch == '(' and not (InSingleQuoteStr or InDoubleQuoteStr):
1263             Pair += 1
1264         elif ch == ')' and not (InSingleQuoteStr or InDoubleQuoteStr):
1265             Pair -= 1
1266
1267         if (Pair > 0 or InSingleQuoteStr or InDoubleQuoteStr) and ch == TAB_VALUE_SPLIT:
1268             NewStr += '-'
1269         else:
1270             NewStr += ch
1271     FieldList = []
1272     StartPos = 0
1273     while True:
1274         Pos = NewStr.find(TAB_VALUE_SPLIT, StartPos)
1275         if Pos < 0:
1276             FieldList.append(Setting[StartPos:].strip())
1277             break
1278         FieldList.append(Setting[StartPos:Pos].strip())
1279         StartPos = Pos + 1
1280
1281     return FieldList
1282
1283 def ParseDevPathValue (Value):
1284     if '\\' in Value:
1285         Value.replace('\\', '/').replace(' ', '')
1286
1287     Cmd = 'DevicePath ' + '"' + Value + '"'
1288     try:
1289         p = subprocess.Popen(Cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
1290         out, err = p.communicate()
1291     except Exception as X:
1292         raise BadExpression("DevicePath: %s" % (str(X)) )
1293     finally:
1294         subprocess._cleanup()
1295         p.stdout.close()
1296         p.stderr.close()
1297     if err:
1298         raise BadExpression("DevicePath: %s" % str(err))
1299     Size = len(out.split())
1300     out = ','.join(out.split())
1301     return '{' + out + '}', Size
1302
1303 def ParseFieldValue (Value):
1304     if isinstance(Value, type(0)):
1305         return Value, (Value.bit_length() + 7) / 8
1306     if not isinstance(Value, type('')):
1307         raise BadExpression('Type %s is %s' %(Value, type(Value)))
1308     Value = Value.strip()
1309     if Value.startswith(TAB_UINT8) and Value.endswith(')'):
1310         Value, Size = ParseFieldValue(Value.split('(', 1)[1][:-1])
1311         if Size > 1:
1312             raise BadExpression('Value (%s) Size larger than %d' %(Value, Size))
1313         return Value, 1
1314     if Value.startswith(TAB_UINT16) and Value.endswith(')'):
1315         Value, Size = ParseFieldValue(Value.split('(', 1)[1][:-1])
1316         if Size > 2:
1317             raise BadExpression('Value (%s) Size larger than %d' %(Value, Size))
1318         return Value, 2
1319     if Value.startswith(TAB_UINT32) and Value.endswith(')'):
1320         Value, Size = ParseFieldValue(Value.split('(', 1)[1][:-1])
1321         if Size > 4:
1322             raise BadExpression('Value (%s) Size larger than %d' %(Value, Size))
1323         return Value, 4
1324     if Value.startswith(TAB_UINT64) and Value.endswith(')'):
1325         Value, Size = ParseFieldValue(Value.split('(', 1)[1][:-1])
1326         if Size > 8:
1327             raise BadExpression('Value (%s) Size larger than %d' % (Value, Size))
1328         return Value, 8
1329     if Value.startswith(TAB_GUID) and Value.endswith(')'):
1330         Value = Value.split('(', 1)[1][:-1].strip()
1331         if Value[0] == '{' and Value[-1] == '}':
1332             TmpValue = GuidStructureStringToGuidString(Value)
1333             if not TmpValue:
1334                 raise BadExpression("Invalid GUID value string %s" % Value)
1335             Value = TmpValue
1336         if Value[0] == '"' and Value[-1] == '"':
1337             Value = Value[1:-1]
1338         try:
1339             Value = "'" + uuid.UUID(Value).get_bytes_le() + "'"
1340         except ValueError as Message:
1341             raise BadExpression(Message)
1342         Value, Size = ParseFieldValue(Value)
1343         return Value, 16
1344     if Value.startswith('L"') and Value.endswith('"'):
1345         # Unicode String
1346         # translate escape character
1347         Value = Value[1:]
1348         try:
1349             Value = eval(Value)
1350         except:
1351             Value = Value[1:-1]
1352         List = list(Value)
1353         List.reverse()
1354         Value = 0
1355         for Char in List:
1356             Value = (Value << 16) | ord(Char)
1357         return Value, (len(List) + 1) * 2
1358     if Value.startswith('"') and Value.endswith('"'):
1359         # ASCII String
1360         # translate escape character
1361         try:
1362             Value = eval(Value)
1363         except:
1364             Value = Value[1:-1]
1365         List = list(Value)
1366         List.reverse()
1367         Value = 0
1368         for Char in List:
1369             Value = (Value << 8) | ord(Char)
1370         return Value, len(List) + 1
1371     if Value.startswith("L'") and Value.endswith("'"):
1372         # Unicode Character Constant
1373         # translate escape character
1374         Value = Value[1:]
1375         try:
1376             Value = eval(Value)
1377         except:
1378             Value = Value[1:-1]
1379         List = list(Value)
1380         if len(List) == 0:
1381             raise BadExpression('Length %s is %s' % (Value, len(List)))
1382         List.reverse()
1383         Value = 0
1384         for Char in List:
1385             Value = (Value << 16) | ord(Char)
1386         return Value, len(List) * 2
1387     if Value.startswith("'") and Value.endswith("'"):
1388         # Character constant
1389         # translate escape character
1390         try:
1391             Value = eval(Value)
1392         except:
1393             Value = Value[1:-1]
1394         List = list(Value)
1395         if len(List) == 0:
1396             raise BadExpression('Length %s is %s' % (Value, len(List)))
1397         List.reverse()
1398         Value = 0
1399         for Char in List:
1400             Value = (Value << 8) | ord(Char)
1401         return Value, len(List)
1402     if Value.startswith('{') and Value.endswith('}'):
1403         # Byte array
1404         Value = Value[1:-1]
1405         List = [Item.strip() for Item in Value.split(',')]
1406         List.reverse()
1407         Value = 0
1408         RetSize = 0
1409         for Item in List:
1410             ItemValue, Size = ParseFieldValue(Item)
1411             RetSize += Size
1412             for I in range(Size):
1413                 Value = (Value << 8) | ((ItemValue >> 8 * I) & 0xff)
1414         return Value, RetSize
1415     if Value.startswith('DEVICE_PATH(') and Value.endswith(')'):
1416         Value = Value.replace("DEVICE_PATH(", '').rstrip(')')
1417         Value = Value.strip().strip('"')
1418         return ParseDevPathValue(Value)
1419     if Value.lower().startswith('0x'):
1420         try:
1421             Value = int(Value, 16)
1422         except:
1423             raise BadExpression("invalid hex value: %s" % Value)
1424         if Value == 0:
1425             return 0, 1
1426         return Value, (Value.bit_length() + 7) / 8
1427     if Value[0].isdigit():
1428         Value = int(Value, 10)
1429         if Value == 0:
1430             return 0, 1
1431         return Value, (Value.bit_length() + 7) / 8
1432     if Value.lower() == 'true':
1433         return 1, 1
1434     if Value.lower() == 'false':
1435         return 0, 1
1436     return Value, 1
1437
1438 ## AnalyzeDscPcd
1439 #
1440 #  Analyze DSC PCD value, since there is no data type info in DSC
1441 #  This function is used to match functions (AnalyzePcdData) used for retrieving PCD value from database
1442 #  1. Feature flag: TokenSpace.PcdCName|PcdValue
1443 #  2. Fix and Patch:TokenSpace.PcdCName|PcdValue[|VOID*[|MaxSize]]
1444 #  3. Dynamic default:
1445 #     TokenSpace.PcdCName|PcdValue[|VOID*[|MaxSize]]
1446 #     TokenSpace.PcdCName|PcdValue
1447 #  4. Dynamic VPD:
1448 #     TokenSpace.PcdCName|VpdOffset[|VpdValue]
1449 #     TokenSpace.PcdCName|VpdOffset[|MaxSize[|VpdValue]]
1450 #  5. Dynamic HII:
1451 #     TokenSpace.PcdCName|HiiString|VariableGuid|VariableOffset[|HiiValue]
1452 #  PCD value needs to be located in such kind of string, and the PCD value might be an expression in which
1453 #    there might have "|" operator, also in string value.
1454 #
1455 #  @param Setting: String contain information described above with "TokenSpace.PcdCName|" stripped
1456 #  @param PcdType: PCD type: feature, fixed, dynamic default VPD HII
1457 #  @param DataType: The datum type of PCD: VOID*, UNIT, BOOL
1458 #  @retval:
1459 #    ValueList: A List contain fields described above
1460 #    IsValid:   True if conforming EBNF, otherwise False
1461 #    Index:     The index where PcdValue is in ValueList
1462 #
1463 def AnalyzeDscPcd(Setting, PcdType, DataType=''):
1464     FieldList = AnalyzePcdExpression(Setting)
1465
1466     IsValid = True
1467     if PcdType in (MODEL_PCD_FIXED_AT_BUILD, MODEL_PCD_PATCHABLE_IN_MODULE, MODEL_PCD_DYNAMIC_DEFAULT, MODEL_PCD_DYNAMIC_EX_DEFAULT):
1468         Value = FieldList[0]
1469         Size = ''
1470         if len(FieldList) > 1 and FieldList[1]:
1471             DataType = FieldList[1]
1472             if FieldList[1] != TAB_VOID:
1473                 IsValid = False
1474         if len(FieldList) > 2:
1475             Size = FieldList[2]
1476         if IsValid:
1477             if DataType == "":
1478                 IsValid = (len(FieldList) <= 1)
1479             else:
1480                 IsValid = (len(FieldList) <= 3)
1481
1482         if Size:
1483             try:
1484                 int(Size, 16) if Size.upper().startswith("0X") else int(Size)
1485             except:
1486                 IsValid = False
1487                 Size = -1
1488         return [str(Value), DataType, str(Size)], IsValid, 0
1489     elif PcdType == MODEL_PCD_FEATURE_FLAG:
1490         Value = FieldList[0]
1491         Size = ''
1492         IsValid = (len(FieldList) <= 1)
1493         return [Value, DataType, str(Size)], IsValid, 0
1494     elif PcdType in (MODEL_PCD_DYNAMIC_VPD, MODEL_PCD_DYNAMIC_EX_VPD):
1495         VpdOffset = FieldList[0]
1496         Value = Size = ''
1497         if not DataType == TAB_VOID:
1498             if len(FieldList) > 1:
1499                 Value = FieldList[1]
1500         else:
1501             if len(FieldList) > 1:
1502                 Size = FieldList[1]
1503             if len(FieldList) > 2:
1504                 Value = FieldList[2]
1505         if DataType == "":
1506             IsValid = (len(FieldList) <= 1)
1507         else:
1508             IsValid = (len(FieldList) <= 3)
1509         if Size:
1510             try:
1511                 int(Size, 16) if Size.upper().startswith("0X") else int(Size)
1512             except:
1513                 IsValid = False
1514                 Size = -1
1515         return [VpdOffset, str(Size), Value], IsValid, 2
1516     elif PcdType in (MODEL_PCD_DYNAMIC_HII, MODEL_PCD_DYNAMIC_EX_HII):
1517         IsValid = (3 <= len(FieldList) <= 5)
1518         HiiString = FieldList[0]
1519         Guid = Offset = Value = Attribute = ''
1520         if len(FieldList) > 1:
1521             Guid = FieldList[1]
1522         if len(FieldList) > 2:
1523             Offset = FieldList[2]
1524         if len(FieldList) > 3:
1525             Value = FieldList[3]
1526             if not Value:
1527                 IsValid = False
1528         if len(FieldList) > 4:
1529             Attribute = FieldList[4]
1530         return [HiiString, Guid, Offset, Value, Attribute], IsValid, 3
1531     return [], False, 0
1532
1533 ## AnalyzePcdData
1534 #
1535 #  Analyze the pcd Value, Datum type and TokenNumber.
1536 #  Used to avoid split issue while the value string contain "|" character
1537 #
1538 #  @param[in] Setting:  A String contain value/datum type/token number information;
1539 #
1540 #  @retval   ValueList: A List contain value, datum type and toke number.
1541 #
1542 def AnalyzePcdData(Setting):
1543     ValueList = ['', '', '']
1544
1545     ValueRe = re.compile(r'^\s*L?\".*\|.*\"')
1546     PtrValue = ValueRe.findall(Setting)
1547
1548     ValueUpdateFlag = False
1549
1550     if len(PtrValue) >= 1:
1551         Setting = re.sub(ValueRe, '', Setting)
1552         ValueUpdateFlag = True
1553
1554     TokenList = Setting.split(TAB_VALUE_SPLIT)
1555     ValueList[0:len(TokenList)] = TokenList
1556
1557     if ValueUpdateFlag:
1558         ValueList[0] = PtrValue[0]
1559
1560     return ValueList
1561
1562 ## check format of PCD value against its the datum type
1563 #
1564 # For PCD value setting
1565 #
1566 def CheckPcdDatum(Type, Value):
1567     if Type == TAB_VOID:
1568         ValueRe = re.compile(r'\s*L?\".*\"\s*$')
1569         if not (((Value.startswith('L"') or Value.startswith('"')) and Value.endswith('"'))
1570                 or (Value.startswith('{') and Value.endswith('}')) or (Value.startswith("L'") or Value.startswith("'") and Value.endswith("'"))
1571                ):
1572             return False, "Invalid value [%s] of type [%s]; must be in the form of {...} for array"\
1573                           ", \"...\" or \'...\' for string, L\"...\" or L\'...\' for unicode string" % (Value, Type)
1574         elif ValueRe.match(Value):
1575             # Check the chars in UnicodeString or CString is printable
1576             if Value.startswith("L"):
1577                 Value = Value[2:-1]
1578             else:
1579                 Value = Value[1:-1]
1580             Printset = set(string.printable)
1581             Printset.remove(TAB_PRINTCHAR_VT)
1582             Printset.add(TAB_PRINTCHAR_BS)
1583             Printset.add(TAB_PRINTCHAR_NUL)
1584             if not set(Value).issubset(Printset):
1585                 PrintList = sorted(Printset)
1586                 return False, "Invalid PCD string value of type [%s]; must be printable chars %s." % (Type, PrintList)
1587     elif Type == 'BOOLEAN':
1588         if Value not in ['TRUE', 'True', 'true', '0x1', '0x01', '1', 'FALSE', 'False', 'false', '0x0', '0x00', '0']:
1589             return False, "Invalid value [%s] of type [%s]; must be one of TRUE, True, true, 0x1, 0x01, 1"\
1590                           ", FALSE, False, false, 0x0, 0x00, 0" % (Value, Type)
1591     elif Type in [TAB_UINT8, TAB_UINT16, TAB_UINT32, TAB_UINT64]:
1592         if Value and int(Value, 0) < 0:
1593             return False, "PCD can't be set to negative value[%s] for datum type [%s]" % (Value, Type)
1594         try:
1595             Value = long(Value, 0)
1596             if Value > MAX_VAL_TYPE[Type]:
1597                 return False, "Too large PCD value[%s] for datum type [%s]" % (Value, Type)
1598         except:
1599             return False, "Invalid value [%s] of type [%s];"\
1600                           " must be a hexadecimal, decimal or octal in C language format." % (Value, Type)
1601     else:
1602         return True, "StructurePcd"
1603
1604     return True, ""
1605
1606 ## Split command line option string to list
1607 #
1608 # subprocess.Popen needs the args to be a sequence. Otherwise there's problem
1609 # in non-windows platform to launch command
1610 #
1611 def SplitOption(OptionString):
1612     OptionList = []
1613     LastChar = " "
1614     OptionStart = 0
1615     QuotationMark = ""
1616     for Index in range(0, len(OptionString)):
1617         CurrentChar = OptionString[Index]
1618         if CurrentChar in ['"', "'"]:
1619             if QuotationMark == CurrentChar:
1620                 QuotationMark = ""
1621             elif QuotationMark == "":
1622                 QuotationMark = CurrentChar
1623             continue
1624         elif QuotationMark:
1625             continue
1626
1627         if CurrentChar in ["/", "-"] and LastChar in [" ", "\t", "\r", "\n"]:
1628             if Index > OptionStart:
1629                 OptionList.append(OptionString[OptionStart:Index - 1])
1630             OptionStart = Index
1631         LastChar = CurrentChar
1632     OptionList.append(OptionString[OptionStart:])
1633     return OptionList
1634
1635 def CommonPath(PathList):
1636     P1 = min(PathList).split(os.path.sep)
1637     P2 = max(PathList).split(os.path.sep)
1638     for Index in xrange(min(len(P1), len(P2))):
1639         if P1[Index] != P2[Index]:
1640             return os.path.sep.join(P1[:Index])
1641     return os.path.sep.join(P1)
1642
1643 #
1644 # Convert string to C format array
1645 #
1646 def ConvertStringToByteArray(Value):
1647     Value = Value.strip()
1648     if not Value:
1649         return None
1650     if Value[0] == '{':
1651         if not Value.endswith('}'):
1652             return None
1653         Value = Value.replace(' ', '').replace('{', '').replace('}', '')
1654         ValFields = Value.split(',')
1655         try:
1656             for Index in range(len(ValFields)):
1657                 ValFields[Index] = str(int(ValFields[Index], 0))
1658         except ValueError:
1659             return None
1660         Value = '{' + ','.join(ValFields) + '}'
1661         return Value
1662
1663     Unicode = False
1664     if Value.startswith('L"'):
1665         if not Value.endswith('"'):
1666             return None
1667         Value = Value[1:]
1668         Unicode = True
1669     elif not Value.startswith('"') or not Value.endswith('"'):
1670         return None
1671
1672     Value = eval(Value)         # translate escape character
1673     NewValue = '{'
1674     for Index in range(0, len(Value)):
1675         if Unicode:
1676             NewValue = NewValue + str(ord(Value[Index]) % 0x10000) + ','
1677         else:
1678             NewValue = NewValue + str(ord(Value[Index]) % 0x100) + ','
1679     Value = NewValue + '0}'
1680     return Value
1681
1682 class PathClass(object):
1683     def __init__(self, File='', Root='', AlterRoot='', Type='', IsBinary=False,
1684                  Arch='COMMON', ToolChainFamily='', Target='', TagName='', ToolCode=''):
1685         self.Arch = Arch
1686         self.File = str(File)
1687         if os.path.isabs(self.File):
1688             self.Root = ''
1689             self.AlterRoot = ''
1690         else:
1691             self.Root = str(Root)
1692             self.AlterRoot = str(AlterRoot)
1693
1694         # Remove any '.' and '..' in path
1695         if self.Root:
1696             self.Root = mws.getWs(self.Root, self.File)
1697             self.Path = os.path.normpath(os.path.join(self.Root, self.File))
1698             self.Root = os.path.normpath(CommonPath([self.Root, self.Path]))
1699             # eliminate the side-effect of 'C:'
1700             if self.Root[-1] == ':':
1701                 self.Root += os.path.sep
1702             # file path should not start with path separator
1703             if self.Root[-1] == os.path.sep:
1704                 self.File = self.Path[len(self.Root):]
1705             else:
1706                 self.File = self.Path[len(self.Root) + 1:]
1707         else:
1708             self.Path = os.path.normpath(self.File)
1709
1710         self.SubDir, self.Name = os.path.split(self.File)
1711         self.BaseName, self.Ext = os.path.splitext(self.Name)
1712
1713         if self.Root:
1714             if self.SubDir:
1715                 self.Dir = os.path.join(self.Root, self.SubDir)
1716             else:
1717                 self.Dir = self.Root
1718         else:
1719             self.Dir = self.SubDir
1720
1721         if IsBinary:
1722             self.Type = Type
1723         else:
1724             self.Type = self.Ext.lower()
1725
1726         self.IsBinary = IsBinary
1727         self.Target = Target
1728         self.TagName = TagName
1729         self.ToolCode = ToolCode
1730         self.ToolChainFamily = ToolChainFamily
1731
1732     ## Convert the object of this class to a string
1733     #
1734     #  Convert member Path of the class to a string
1735     #
1736     #  @retval string Formatted String
1737     #
1738     def __str__(self):
1739         return self.Path
1740
1741     ## Override __eq__ function
1742     #
1743     # Check whether PathClass are the same
1744     #
1745     # @retval False The two PathClass are different
1746     # @retval True  The two PathClass are the same
1747     #
1748     def __eq__(self, Other):
1749         if isinstance(Other, type(self)):
1750             return self.Path == Other.Path
1751         else:
1752             return self.Path == str(Other)
1753
1754     ## Override __cmp__ function
1755     #
1756     # Customize the comparsion operation of two PathClass
1757     #
1758     # @retval 0     The two PathClass are different
1759     # @retval -1    The first PathClass is less than the second PathClass
1760     # @retval 1     The first PathClass is Bigger than the second PathClass
1761     def __cmp__(self, Other):
1762         if isinstance(Other, type(self)):
1763             OtherKey = Other.Path
1764         else:
1765             OtherKey = str(Other)
1766
1767         SelfKey = self.Path
1768         if SelfKey == OtherKey:
1769             return 0
1770         elif SelfKey > OtherKey:
1771             return 1
1772         else:
1773             return -1
1774
1775     ## Override __hash__ function
1776     #
1777     # Use Path as key in hash table
1778     #
1779     # @retval string Key for hash table
1780     #
1781     def __hash__(self):
1782         return hash(self.Path)
1783
1784     @cached_property
1785     def Key(self):
1786         return self.Path.upper()
1787
1788     @property
1789     def TimeStamp(self):
1790         return os.stat(self.Path)[8]
1791
1792     def Validate(self, Type='', CaseSensitive=True):
1793         if GlobalData.gCaseInsensitive:
1794             CaseSensitive = False
1795         if Type and Type.lower() != self.Type:
1796             return FILE_TYPE_MISMATCH, '%s (expect %s but got %s)' % (self.File, Type, self.Type)
1797
1798         RealFile, RealRoot = RealPath2(self.File, self.Root, self.AlterRoot)
1799         if not RealRoot and not RealFile:
1800             RealFile = self.File
1801             if self.AlterRoot:
1802                 RealFile = os.path.join(self.AlterRoot, self.File)
1803             elif self.Root:
1804                 RealFile = os.path.join(self.Root, self.File)
1805             if len (mws.getPkgPath()) == 0:
1806                 return FILE_NOT_FOUND, os.path.join(self.AlterRoot, RealFile)
1807             else:
1808                 return FILE_NOT_FOUND, "%s is not found in packages path:\n\t%s" % (self.File, '\n\t'.join(mws.getPkgPath()))
1809
1810         ErrorCode = 0
1811         ErrorInfo = ''
1812         if RealRoot != self.Root or RealFile != self.File:
1813             if CaseSensitive and (RealFile != self.File or (RealRoot != self.Root and RealRoot != self.AlterRoot)):
1814                 ErrorCode = FILE_CASE_MISMATCH
1815                 ErrorInfo = self.File + '\n\t' + RealFile + " [in file system]"
1816
1817             self.SubDir, self.Name = os.path.split(RealFile)
1818             self.BaseName, self.Ext = os.path.splitext(self.Name)
1819             if self.SubDir:
1820                 self.Dir = os.path.join(RealRoot, self.SubDir)
1821             else:
1822                 self.Dir = RealRoot
1823             self.File = RealFile
1824             self.Root = RealRoot
1825             self.Path = os.path.join(RealRoot, RealFile)
1826         return ErrorCode, ErrorInfo
1827
1828 ## Parse PE image to get the required PE informaion.
1829 #
1830 class PeImageClass():
1831     ## Constructor
1832     #
1833     #   @param  File FilePath of PeImage
1834     #
1835     def __init__(self, PeFile):
1836         self.FileName   = PeFile
1837         self.IsValid    = False
1838         self.Size       = 0
1839         self.EntryPoint = 0
1840         self.SectionAlignment  = 0
1841         self.SectionHeaderList = []
1842         self.ErrorInfo = ''
1843         try:
1844             PeObject = open(PeFile, 'rb')
1845         except:
1846             self.ErrorInfo = self.FileName + ' can not be found\n'
1847             return
1848         # Read DOS header
1849         ByteArray = array.array('B')
1850         ByteArray.fromfile(PeObject, 0x3E)
1851         ByteList = ByteArray.tolist()
1852         # DOS signature should be 'MZ'
1853         if self._ByteListToStr (ByteList[0x0:0x2]) != 'MZ':
1854             self.ErrorInfo = self.FileName + ' has no valid DOS signature MZ'
1855             return
1856
1857         # Read 4 byte PE Signature
1858         PeOffset = self._ByteListToInt(ByteList[0x3C:0x3E])
1859         PeObject.seek(PeOffset)
1860         ByteArray = array.array('B')
1861         ByteArray.fromfile(PeObject, 4)
1862         # PE signature should be 'PE\0\0'
1863         if ByteArray.tostring() != 'PE\0\0':
1864             self.ErrorInfo = self.FileName + ' has no valid PE signature PE00'
1865             return
1866
1867         # Read PE file header
1868         ByteArray = array.array('B')
1869         ByteArray.fromfile(PeObject, 0x14)
1870         ByteList = ByteArray.tolist()
1871         SecNumber = self._ByteListToInt(ByteList[0x2:0x4])
1872         if SecNumber == 0:
1873             self.ErrorInfo = self.FileName + ' has no section header'
1874             return
1875
1876         # Read PE optional header
1877         OptionalHeaderSize = self._ByteListToInt(ByteArray[0x10:0x12])
1878         ByteArray = array.array('B')
1879         ByteArray.fromfile(PeObject, OptionalHeaderSize)
1880         ByteList = ByteArray.tolist()
1881         self.EntryPoint       = self._ByteListToInt(ByteList[0x10:0x14])
1882         self.SectionAlignment = self._ByteListToInt(ByteList[0x20:0x24])
1883         self.Size             = self._ByteListToInt(ByteList[0x38:0x3C])
1884
1885         # Read each Section Header
1886         for Index in range(SecNumber):
1887             ByteArray = array.array('B')
1888             ByteArray.fromfile(PeObject, 0x28)
1889             ByteList = ByteArray.tolist()
1890             SecName  = self._ByteListToStr(ByteList[0:8])
1891             SecVirtualSize = self._ByteListToInt(ByteList[8:12])
1892             SecRawAddress  = self._ByteListToInt(ByteList[20:24])
1893             SecVirtualAddress = self._ByteListToInt(ByteList[12:16])
1894             self.SectionHeaderList.append((SecName, SecVirtualAddress, SecRawAddress, SecVirtualSize))
1895         self.IsValid = True
1896         PeObject.close()
1897
1898     def _ByteListToStr(self, ByteList):
1899         String = ''
1900         for index in range(len(ByteList)):
1901             if ByteList[index] == 0:
1902                 break
1903             String += chr(ByteList[index])
1904         return String
1905
1906     def _ByteListToInt(self, ByteList):
1907         Value = 0
1908         for index in range(len(ByteList) - 1, -1, -1):
1909             Value = (Value << 8) | int(ByteList[index])
1910         return Value
1911
1912 class DefaultStore():
1913     def __init__(self, DefaultStores ):
1914
1915         self.DefaultStores = DefaultStores
1916     def DefaultStoreID(self, DefaultStoreName):
1917         for key, value in self.DefaultStores.items():
1918             if value == DefaultStoreName:
1919                 return key
1920         return None
1921     def GetDefaultDefault(self):
1922         if not self.DefaultStores or "0" in self.DefaultStores:
1923             return "0", TAB_DEFAULT_STORES_DEFAULT
1924         else:
1925             minvalue = min(int(value_str) for value_str in self.DefaultStores)
1926             return (str(minvalue), self.DefaultStores[str(minvalue)])
1927     def GetMin(self, DefaultSIdList):
1928         if not DefaultSIdList:
1929             return TAB_DEFAULT_STORES_DEFAULT
1930         storeidset = {storeid for storeid, storename in self.DefaultStores.values() if storename in DefaultSIdList}
1931         if not storeidset:
1932             return ""
1933         minid = min(storeidset )
1934         for sid, name in self.DefaultStores.values():
1935             if sid == minid:
1936                 return name
1937
1938 class SkuClass():
1939     DEFAULT = 0
1940     SINGLE = 1
1941     MULTIPLE =2
1942
1943     def __init__(self,SkuIdentifier='', SkuIds=None):
1944         if SkuIds is None:
1945             SkuIds = {}
1946
1947         for SkuName in SkuIds:
1948             SkuId = SkuIds[SkuName][0]
1949             skuid_num = int(SkuId, 16) if SkuId.upper().startswith("0X") else int(SkuId)
1950             if skuid_num > 0xFFFFFFFFFFFFFFFF:
1951                 EdkLogger.error("build", PARAMETER_INVALID,
1952                             ExtraData = "SKU-ID [%s] value %s exceeds the max value of UINT64"
1953                                       % (SkuName, SkuId))
1954
1955         self.AvailableSkuIds = sdict()
1956         self.SkuIdSet = []
1957         self.SkuIdNumberSet = []
1958         self.SkuData = SkuIds
1959         self._SkuInherit = {}
1960         self._SkuIdentifier = SkuIdentifier
1961         if SkuIdentifier == '' or SkuIdentifier is None:
1962             self.SkuIdSet = ['DEFAULT']
1963             self.SkuIdNumberSet = ['0U']
1964         elif SkuIdentifier == 'ALL':
1965             self.SkuIdSet = SkuIds.keys()
1966             self.SkuIdNumberSet = [num[0].strip() + 'U' for num in SkuIds.values()]
1967         else:
1968             r = SkuIdentifier.split('|')
1969             self.SkuIdSet=[(r[k].strip()).upper() for k in range(len(r))]
1970             k = None
1971             try:
1972                 self.SkuIdNumberSet = [SkuIds[k][0].strip() + 'U' for k in self.SkuIdSet]
1973             except Exception:
1974                 EdkLogger.error("build", PARAMETER_INVALID,
1975                             ExtraData = "SKU-ID [%s] is not supported by the platform. [Valid SKU-ID: %s]"
1976                                       % (k, " | ".join(SkuIds.keys())))
1977         for each in self.SkuIdSet:
1978             if each in SkuIds:
1979                 self.AvailableSkuIds[each] = SkuIds[each][0]
1980             else:
1981                 EdkLogger.error("build", PARAMETER_INVALID,
1982                             ExtraData="SKU-ID [%s] is not supported by the platform. [Valid SKU-ID: %s]"
1983                                       % (each, " | ".join(SkuIds.keys())))
1984         if self.SkuUsageType != SkuClass.SINGLE:
1985             self.AvailableSkuIds.update({'DEFAULT':0, 'COMMON':0})
1986         if self.SkuIdSet:
1987             GlobalData.gSkuids = (self.SkuIdSet)
1988             if 'COMMON' in GlobalData.gSkuids:
1989                 GlobalData.gSkuids.remove('COMMON')
1990             if self.SkuUsageType == self.SINGLE:
1991                 if len(GlobalData.gSkuids) != 1:
1992                     if 'DEFAULT' in GlobalData.gSkuids:
1993                         GlobalData.gSkuids.remove('DEFAULT')
1994             if GlobalData.gSkuids:
1995                 GlobalData.gSkuids.sort()
1996
1997     def GetNextSkuId(self, skuname):
1998         if not self._SkuInherit:
1999             self._SkuInherit = {}
2000             for item in self.SkuData.values():
2001                 self._SkuInherit[item[1]]=item[2] if item[2] else "DEFAULT"
2002         return self._SkuInherit.get(skuname, "DEFAULT")
2003
2004     def GetSkuChain(self, sku):
2005         if sku == "DEFAULT":
2006             return ["DEFAULT"]
2007         skulist = [sku]
2008         nextsku = sku
2009         while True:
2010             nextsku = self.GetNextSkuId(nextsku)
2011             skulist.append(nextsku)
2012             if nextsku == "DEFAULT":
2013                 break
2014         skulist.reverse()
2015         return skulist
2016     def SkuOverrideOrder(self):
2017         skuorderset = []
2018         for skuname in self.SkuIdSet:
2019             skuorderset.append(self.GetSkuChain(skuname))
2020
2021         skuorder = []
2022         for index in range(max(len(item) for item in skuorderset)):
2023             for subset in skuorderset:
2024                 if index > len(subset)-1:
2025                     continue
2026                 if subset[index] in skuorder:
2027                     continue
2028                 skuorder.append(subset[index])
2029
2030         return skuorder
2031
2032     @property
2033     def SkuUsageType(self):
2034         if self._SkuIdentifier.upper() == "ALL":
2035             return SkuClass.MULTIPLE
2036
2037         if len(self.SkuIdSet) == 1:
2038             if self.SkuIdSet[0] == 'DEFAULT':
2039                 return SkuClass.DEFAULT
2040             return SkuClass.SINGLE
2041         if len(self.SkuIdSet) == 2 and 'DEFAULT' in self.SkuIdSet:
2042             return SkuClass.SINGLE
2043         return SkuClass.MULTIPLE
2044
2045     def DumpSkuIdArrary(self):
2046         if self.SkuUsageType == SkuClass.SINGLE:
2047             return "{0x0}"
2048         ArrayStrList = []
2049         for skuname in self.AvailableSkuIds:
2050             if skuname == "COMMON":
2051                 continue
2052             while skuname != "DEFAULT":
2053                 ArrayStrList.append(hex(int(self.AvailableSkuIds[skuname])))
2054                 skuname = self.GetNextSkuId(skuname)
2055             ArrayStrList.append("0x0")
2056         return "{{{myList}}}".format(myList=",".join(ArrayStrList))
2057
2058     @property
2059     def AvailableSkuIdSet(self):
2060         return self.AvailableSkuIds
2061
2062     @property
2063     def SystemSkuId(self):
2064         if self.SkuUsageType == SkuClass.SINGLE:
2065             if len(self.SkuIdSet) == 1:
2066                 return self.SkuIdSet[0]
2067             else:
2068                 return self.SkuIdSet[0] if self.SkuIdSet[0] != 'DEFAULT' else self.SkuIdSet[1]
2069         else:
2070             return 'DEFAULT'
2071
2072 #
2073 # Pack a registry format GUID
2074 #
2075 def PackRegistryFormatGuid(Guid):
2076     return PackGUID(Guid.split('-'))
2077
2078 ##  Get the integer value from string like "14U" or integer like 2
2079 #
2080 #   @param      Input   The object that may be either a integer value or a string
2081 #
2082 #   @retval     Value    The integer value that the input represents
2083 #
2084 def GetIntegerValue(Input):
2085     if type(Input) in (int, long):
2086         return Input
2087     String = Input
2088     if String.endswith("U"):
2089         String = String[:-1]
2090     if String.endswith("ULL"):
2091         String = String[:-3]
2092     if String.endswith("LL"):
2093         String = String[:-2]
2094
2095     if String.startswith("0x") or String.startswith("0X"):
2096         return int(String, 16)
2097     elif String == '':
2098         return 0
2099     else:
2100         return int(String)
2101
2102 #
2103 # Pack a GUID (registry format) list into a buffer and return it
2104 #
2105 def PackGUID(Guid):
2106     return pack(PACK_PATTERN_GUID,
2107                 int(Guid[0], 16),
2108                 int(Guid[1], 16),
2109                 int(Guid[2], 16),
2110                 int(Guid[3][-4:-2], 16),
2111                 int(Guid[3][-2:], 16),
2112                 int(Guid[4][-12:-10], 16),
2113                 int(Guid[4][-10:-8], 16),
2114                 int(Guid[4][-8:-6], 16),
2115                 int(Guid[4][-6:-4], 16),
2116                 int(Guid[4][-4:-2], 16),
2117                 int(Guid[4][-2:], 16)
2118                 )
2119
2120 #
2121 # Pack a GUID (byte) list into a buffer and return it
2122 #
2123 def PackByteFormatGUID(Guid):
2124     return pack(PACK_PATTERN_GUID,
2125                 Guid[0],
2126                 Guid[1],
2127                 Guid[2],
2128                 Guid[3],
2129                 Guid[4],
2130                 Guid[5],
2131                 Guid[6],
2132                 Guid[7],
2133                 Guid[8],
2134                 Guid[9],
2135                 Guid[10],
2136                 )
2137
2138 ##
2139 #
2140 # This acts like the main() function for the script, unless it is 'import'ed into another
2141 # script.
2142 #
2143 if __name__ == '__main__':
2144     pass
2145