BaseTools/Source/Python/Common/Misc.py

   1 ## @file
   2 # Common routines used by all tools
   3 #
   4 # Copyright (c) 2007 - 2018, Intel Corporation. All rights reserved.<BR>
   5 # This program and the accompanying materials
   6 # are licensed and made available under the terms and conditions of the BSD License
   7 # which accompanies this distribution.  The full text of the license may be found at
   8 # http://opensource.org/licenses/bsd-license.php
   9 #
  10 # THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS,
  11 # WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED.
  12 #
  13
  14 ##
  15 # Import Modules
  16 #
  17 from __future__ import absolute_import
  18 import Common.LongFilePathOs as os
  19 import sys
  20 import string
  21 import threading
  22 import time
  23 import re
  24 import cPickle
  25 import array
  26 import shutil
  27 from struct import pack
  28 from UserDict import IterableUserDict
  29 from UserList import UserList
  30
  31 from Common import EdkLogger as EdkLogger
  32 from Common import GlobalData as GlobalData
  33 from .DataType import *
  34 from .BuildToolError import *
  35 from CommonDataClass.DataClass import *
  36 from .Parsing import GetSplitValueList
  37 from Common.LongFilePathSupport import OpenLongFilePath as open
  38 from Common.MultipleWorkspace import MultipleWorkspace as mws
  39 import uuid
  40 from CommonDataClass.Exceptions import BadExpression
  41 import subprocess
  42 ## Regular expression used to find out place holders in string template
  43 gPlaceholderPattern = re.compile("\$\{([^$()\s]+)\}", re.MULTILINE | re.UNICODE)
  44
  45 ## regular expressions for map file processing
  46 startPatternGeneral = re.compile("^Start[' ']+Length[' ']+Name[' ']+Class")
  47 addressPatternGeneral = re.compile("^Address[' ']+Publics by Value[' ']+Rva\+Base")
  48 valuePatternGcc = re.compile('^([\w_\.]+) +([\da-fA-Fx]+) +([\da-fA-Fx]+)$')
  49 pcdPatternGcc = re.compile('^([\da-fA-Fx]+) +([\da-fA-Fx]+)')
  50 secReGeneral = re.compile('^([\da-fA-F]+):([\da-fA-F]+) +([\da-fA-F]+)[Hh]? +([.\w\$]+) +(\w+)', re.UNICODE)
  51
  52 ## Dictionary used to store file time stamp for quick re-access
  53 gFileTimeStampCache = {}    # {file path : file time stamp}
  54
  55 ## Dictionary used to store dependencies of files
  56 gDependencyDatabase = {}    # arch : {file path : [dependent files list]}
  57
  58 #
  59 # If a module is built more than once with different PCDs or library classes
  60 # a temporary INF file with same content is created, the temporary file is removed
  61 # when build exits.
  62 #
  63 _TempInfs = []
  64
  65 def GetVariableOffset(mapfilepath, efifilepath, varnames):
  66     """ Parse map file to get variable offset in current EFI file
  67     @param mapfilepath    Map file absolution path
  68     @param efifilepath:   EFI binary file full path
  69     @param varnames       iteratable container whose elements are variable names to be searched
  70
  71     @return List whos elements are tuple with variable name and raw offset
  72     """
  73     lines = []
  74     try:
  75         f = open(mapfilepath, 'r')
  76         lines = f.readlines()
  77         f.close()
  78     except:
  79         return None
  80
  81     if len(lines) == 0: return None
  82     firstline = lines[0].strip()
  83     if (firstline.startswith("Archive member included ") and
  84         firstline.endswith(" file (symbol)")):
  85         return _parseForGCC(lines, efifilepath, varnames)
  86     if firstline.startswith("# Path:"):
  87         return _parseForXcode(lines, efifilepath, varnames)
  88     return _parseGeneral(lines, efifilepath, varnames)
  89
  90 def _parseForXcode(lines, efifilepath, varnames):
  91     status = 0
  92     ret = []
  93     for line in lines:
  94         line = line.strip()
  95         if status == 0 and line == "# Symbols:":
  96             status = 1
  97             continue
  98         if status == 1 and len(line) != 0:
  99             for varname in varnames:
 100                 if varname in line:
 101                     # cannot pregenerate this RegEx since it uses varname from varnames.
 102                     m = re.match('^([\da-fA-FxX]+)([\s\S]*)([_]*%s)$' % varname, line)
 103                     if m is not None:
 104                         ret.append((varname, m.group(1)))
 105     return ret
 106
 107 def _parseForGCC(lines, efifilepath, varnames):
 108     """ Parse map file generated by GCC linker """
 109     status = 0
 110     sections = []
 111     varoffset = []
 112     for index, line in enumerate(lines):
 113         line = line.strip()
 114         # status machine transection
 115         if status == 0 and line == "Memory Configuration":
 116             status = 1
 117             continue
 118         elif status == 1 and line == 'Linker script and memory map':
 119             status = 2
 120             continue
 121         elif status ==2 and line == 'START GROUP':
 122             status = 3
 123             continue
 124
 125         # status handler
 126         if status == 3:
 127             m = valuePatternGcc.match(line)
 128             if m is not None:
 129                 sections.append(m.groups(0))
 130             for varname in varnames:
 131                 Str = ''
 132                 m = re.match("^.data.(%s)" % varname, line)
 133                 if m is not None:
 134                     m = re.match(".data.(%s)$" % varname, line)
 135                     if m is not None:
 136                         Str = lines[index + 1]
 137                     else:
 138                         Str = line[len(".data.%s" % varname):]
 139                     if Str:
 140                         m = pcdPatternGcc.match(Str.strip())
 141                         if m is not None:
 142                             varoffset.append((varname, int(m.groups(0)[0], 16), int(sections[-1][1], 16), sections[-1][0]))
 143
 144     if not varoffset:
 145         return []
 146     # get section information from efi file
 147     efisecs = PeImageClass(efifilepath).SectionHeaderList
 148     if efisecs is None or len(efisecs) == 0:
 149         return []
 150     #redirection
 151     redirection = 0
 152     for efisec in efisecs:
 153         for section in sections:
 154             if section[0].strip() == efisec[0].strip() and section[0].strip() == '.text':
 155                 redirection = int(section[1], 16) - efisec[1]
 156
 157     ret = []
 158     for var in varoffset:
 159         for efisec in efisecs:
 160             if var[1] >= efisec[1] and var[1] < efisec[1]+efisec[3]:
 161                 ret.append((var[0], hex(efisec[2] + var[1] - efisec[1] - redirection)))
 162     return ret
 163
 164 def _parseGeneral(lines, efifilepath, varnames):
 165     status = 0    #0 - beginning of file; 1 - PE section definition; 2 - symbol table
 166     secs  = []    # key = section name
 167     varoffset = []
 168     symRe = re.compile('^([\da-fA-F]+):([\da-fA-F]+) +([\.:\\\\\w\?@\$]+) +([\da-fA-F]+)', re.UNICODE)
 169
 170     for line in lines:
 171         line = line.strip()
 172         if startPatternGeneral.match(line):
 173             status = 1
 174             continue
 175         if addressPatternGeneral.match(line):
 176             status = 2
 177             continue
 178         if line.startswith("entry point at"):
 179             status = 3
 180             continue
 181         if status == 1 and len(line) != 0:
 182             m =  secReGeneral.match(line)
 183             assert m is not None, "Fail to parse the section in map file , line is %s" % line
 184             sec_no, sec_start, sec_length, sec_name, sec_class = m.groups(0)
 185             secs.append([int(sec_no, 16), int(sec_start, 16), int(sec_length, 16), sec_name, sec_class])
 186         if status == 2 and len(line) != 0:
 187             for varname in varnames:
 188                 m = symRe.match(line)
 189                 assert m is not None, "Fail to parse the symbol in map file, line is %s" % line
 190                 sec_no, sym_offset, sym_name, vir_addr = m.groups(0)
 191                 sec_no     = int(sec_no,     16)
 192                 sym_offset = int(sym_offset, 16)
 193                 vir_addr   = int(vir_addr,   16)
 194                 # cannot pregenerate this RegEx since it uses varname from varnames.
 195                 m2 = re.match('^[_]*(%s)' % varname, sym_name)
 196                 if m2 is not None:
 197                     # fond a binary pcd entry in map file
 198                     for sec in secs:
 199                         if sec[0] == sec_no and (sym_offset >= sec[1] and sym_offset < sec[1] + sec[2]):
 200                             varoffset.append([varname, sec[3], sym_offset, vir_addr, sec_no])
 201
 202     if not varoffset: return []
 203
 204     # get section information from efi file
 205     efisecs = PeImageClass(efifilepath).SectionHeaderList
 206     if efisecs is None or len(efisecs) == 0:
 207         return []
 208
 209     ret = []
 210     for var in varoffset:
 211         index = 0
 212         for efisec in efisecs:
 213             index = index + 1
 214             if var[1].strip() == efisec[0].strip():
 215                 ret.append((var[0], hex(efisec[2] + var[2])))
 216             elif var[4] == index:
 217                 ret.append((var[0], hex(efisec[2] + var[2])))
 218
 219     return ret
 220
 221 ## Routine to process duplicated INF
 222 #
 223 #  This function is called by following two cases:
 224 #  Case 1 in DSC:
 225 #    [components.arch]
 226 #    Pkg/module/module.inf
 227 #    Pkg/module/module.inf {
 228 #      <Defines>
 229 #        FILE_GUID = 0D1B936F-68F3-4589-AFCC-FB8B7AEBC836
 230 #    }
 231 #  Case 2 in FDF:
 232 #    INF Pkg/module/module.inf
 233 #    INF FILE_GUID = 0D1B936F-68F3-4589-AFCC-FB8B7AEBC836 Pkg/module/module.inf
 234 #
 235 #  This function copies Pkg/module/module.inf to
 236 #  Conf/.cache/0D1B936F-68F3-4589-AFCC-FB8B7AEBC836module.inf
 237 #
 238 #  @param Path     Original PathClass object
 239 #  @param BaseName New file base name
 240 #
 241 #  @retval         return the new PathClass object
 242 #
 243 def ProcessDuplicatedInf(Path, BaseName, Workspace):
 244     Filename = os.path.split(Path.File)[1]
 245     if '.' in Filename:
 246         Filename = BaseName + Path.BaseName + Filename[Filename.rfind('.'):]
 247     else:
 248         Filename = BaseName + Path.BaseName
 249
 250     #
 251     # If -N is specified on command line, cache is disabled
 252     # The directory has to be created
 253     #
 254     DbDir = os.path.split(GlobalData.gDatabasePath)[0]
 255     if not os.path.exists(DbDir):
 256         os.makedirs(DbDir)
 257     #
 258     # A temporary INF is copied to database path which must have write permission
 259     # The temporary will be removed at the end of build
 260     # In case of name conflict, the file name is
 261     # FILE_GUIDBaseName (0D1B936F-68F3-4589-AFCC-FB8B7AEBC836module.inf)
 262     #
 263     TempFullPath = os.path.join(DbDir,
 264                                 Filename)
 265     RtPath = PathClass(Path.File, Workspace)
 266     #
 267     # Modify the full path to temporary path, keep other unchanged
 268     #
 269     # To build same module more than once, the module path with FILE_GUID overridden has
 270     # the file name FILE_GUIDmodule.inf, but the relative path (self.MetaFile.File) is the real path
 271     # in DSC which is used as relative path by C files and other files in INF.
 272     # A trick was used: all module paths are PathClass instances, after the initialization
 273     # of PathClass, the PathClass.Path is overridden by the temporary INF path.
 274     #
 275     # The reason for creating a temporary INF is:
 276     # Platform.Modules which is the base to create ModuleAutoGen objects is a dictionary,
 277     # the key is the full path of INF, the value is an object to save overridden library instances, PCDs.
 278     # A different key for the same module is needed to create different output directory,
 279     # retrieve overridden PCDs, library instances.
 280     #
 281     # The BaseName is the FILE_GUID which is also the output directory name.
 282     #
 283     #
 284     RtPath.Path = TempFullPath
 285     RtPath.BaseName = BaseName
 286     #
 287     # If file exists, compare contents
 288     #
 289     if os.path.exists(TempFullPath):
 290         with open(str(Path), 'rb') as f1, open(TempFullPath, 'rb') as f2:
 291             if f1.read() == f2.read():
 292                 return RtPath
 293     _TempInfs.append(TempFullPath)
 294     shutil.copy2(str(Path), TempFullPath)
 295     return RtPath
 296
 297 ## Remove temporary created INFs whose paths were saved in _TempInfs
 298 #
 299 def ClearDuplicatedInf():
 300     while _TempInfs:
 301         File = _TempInfs.pop()
 302         if os.path.exists(File):
 303             os.remove(File)
 304
 305 ## Convert GUID string in xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx style to C structure style
 306 #
 307 #   @param      Guid    The GUID string
 308 #
 309 #   @retval     string  The GUID string in C structure style
 310 #
 311 def GuidStringToGuidStructureString(Guid):
 312     GuidList = Guid.split('-')
 313     Result = '{'
 314     for Index in range(0, 3, 1):
 315         Result = Result + '0x' + GuidList[Index] + ', '
 316     Result = Result + '{0x' + GuidList[3][0:2] + ', 0x' + GuidList[3][2:4]
 317     for Index in range(0, 12, 2):
 318         Result = Result + ', 0x' + GuidList[4][Index:Index + 2]
 319     Result += '}}'
 320     return Result
 321
 322 ## Convert GUID structure in byte array to xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
 323 #
 324 #   @param      GuidValue   The GUID value in byte array
 325 #
 326 #   @retval     string      The GUID value in xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx format
 327 #
 328 def GuidStructureByteArrayToGuidString(GuidValue):
 329     guidValueString = GuidValue.lower().replace("{", "").replace("}", "").replace(" ", "").replace(";", "")
 330     guidValueList = guidValueString.split(",")
 331     if len(guidValueList) != 16:
 332         return ''
 333         #EdkLogger.error(None, None, "Invalid GUID value string %s" % GuidValue)
 334     try:
 335         return "%02x%02x%02x%02x-%02x%02x-%02x%02x-%02x%02x-%02x%02x%02x%02x%02x%02x" % (
 336                 int(guidValueList[3], 16),
 337                 int(guidValueList[2], 16),
 338                 int(guidValueList[1], 16),
 339                 int(guidValueList[0], 16),
 340                 int(guidValueList[5], 16),
 341                 int(guidValueList[4], 16),
 342                 int(guidValueList[7], 16),
 343                 int(guidValueList[6], 16),
 344                 int(guidValueList[8], 16),
 345                 int(guidValueList[9], 16),
 346                 int(guidValueList[10], 16),
 347                 int(guidValueList[11], 16),
 348                 int(guidValueList[12], 16),
 349                 int(guidValueList[13], 16),
 350                 int(guidValueList[14], 16),
 351                 int(guidValueList[15], 16)
 352                 )
 353     except:
 354         return ''
 355
 356 ## Convert GUID string in C structure style to xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
 357 #
 358 #   @param      GuidValue   The GUID value in C structure format
 359 #
 360 #   @retval     string      The GUID value in xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx format
 361 #
 362 def GuidStructureStringToGuidString(GuidValue):
 363     guidValueString = GuidValue.lower().replace("{", "").replace("}", "").replace(" ", "").replace(";", "")
 364     guidValueList = guidValueString.split(",")
 365     if len(guidValueList) != 11:
 366         return ''
 367         #EdkLogger.error(None, None, "Invalid GUID value string %s" % GuidValue)
 368     try:
 369         return "%08x-%04x-%04x-%02x%02x-%02x%02x%02x%02x%02x%02x" % (
 370                 int(guidValueList[0], 16),
 371                 int(guidValueList[1], 16),
 372                 int(guidValueList[2], 16),
 373                 int(guidValueList[3], 16),
 374                 int(guidValueList[4], 16),
 375                 int(guidValueList[5], 16),
 376                 int(guidValueList[6], 16),
 377                 int(guidValueList[7], 16),
 378                 int(guidValueList[8], 16),
 379                 int(guidValueList[9], 16),
 380                 int(guidValueList[10], 16)
 381                 )
 382     except:
 383         return ''
 384
 385 ## Convert GUID string in C structure style to xxxxxxxx_xxxx_xxxx_xxxx_xxxxxxxxxxxx
 386 #
 387 #   @param      GuidValue   The GUID value in C structure format
 388 #
 389 #   @retval     string      The GUID value in xxxxxxxx_xxxx_xxxx_xxxx_xxxxxxxxxxxx format
 390 #
 391 def GuidStructureStringToGuidValueName(GuidValue):
 392     guidValueString = GuidValue.lower().replace("{", "").replace("}", "").replace(" ", "")
 393     guidValueList = guidValueString.split(",")
 394     if len(guidValueList) != 11:
 395         EdkLogger.error(None, FORMAT_INVALID, "Invalid GUID value string [%s]" % GuidValue)
 396     return "%08x_%04x_%04x_%02x%02x_%02x%02x%02x%02x%02x%02x" % (
 397             int(guidValueList[0], 16),
 398             int(guidValueList[1], 16),
 399             int(guidValueList[2], 16),
 400             int(guidValueList[3], 16),
 401             int(guidValueList[4], 16),
 402             int(guidValueList[5], 16),
 403             int(guidValueList[6], 16),
 404             int(guidValueList[7], 16),
 405             int(guidValueList[8], 16),
 406             int(guidValueList[9], 16),
 407             int(guidValueList[10], 16)
 408             )
 409
 410 ## Create directories
 411 #
 412 #   @param      Directory   The directory name
 413 #
 414 def CreateDirectory(Directory):
 415     if Directory is None or Directory.strip() == "":
 416         return True
 417     try:
 418         if not os.access(Directory, os.F_OK):
 419             os.makedirs(Directory)
 420     except:
 421         return False
 422     return True
 423
 424 ## Remove directories, including files and sub-directories in it
 425 #
 426 #   @param      Directory   The directory name
 427 #
 428 def RemoveDirectory(Directory, Recursively=False):
 429     if Directory is None or Directory.strip() == "" or not os.path.exists(Directory):
 430         return
 431     if Recursively:
 432         CurrentDirectory = os.getcwd()
 433         os.chdir(Directory)
 434         for File in os.listdir("."):
 435             if os.path.isdir(File):
 436                 RemoveDirectory(File, Recursively)
 437             else:
 438                 os.remove(File)
 439         os.chdir(CurrentDirectory)
 440     os.rmdir(Directory)
 441
 442 ## Store content in file
 443 #
 444 #  This method is used to save file only when its content is changed. This is
 445 #  quite useful for "make" system to decide what will be re-built and what won't.
 446 #
 447 #   @param      File            The path of file
 448 #   @param      Content         The new content of the file
 449 #   @param      IsBinaryFile    The flag indicating if the file is binary file or not
 450 #
 451 #   @retval     True            If the file content is changed and the file is renewed
 452 #   @retval     False           If the file content is the same
 453 #
 454 def SaveFileOnChange(File, Content, IsBinaryFile=True):
 455     if not IsBinaryFile:
 456         Content = Content.replace("\n", os.linesep)
 457
 458     if os.path.exists(File):
 459         try:
 460             if Content == open(File, "rb").read():
 461                 return False
 462         except:
 463             EdkLogger.error(None, FILE_OPEN_FAILURE, ExtraData=File)
 464
 465     DirName = os.path.dirname(File)
 466     if not CreateDirectory(DirName):
 467         EdkLogger.error(None, FILE_CREATE_FAILURE, "Could not create directory %s" % DirName)
 468     else:
 469         if DirName == '':
 470             DirName = os.getcwd()
 471         if not os.access(DirName, os.W_OK):
 472             EdkLogger.error(None, PERMISSION_FAILURE, "Do not have write permission on directory %s" % DirName)
 473
 474     try:
 475         if GlobalData.gIsWindows:
 476             try:
 477                 from .PyUtility import SaveFileToDisk
 478                 if not SaveFileToDisk(File, Content):
 479                     EdkLogger.error(None, FILE_CREATE_FAILURE, ExtraData=File)
 480             except:
 481                 Fd = open(File, "wb")
 482                 Fd.write(Content)
 483                 Fd.close()
 484         else:
 485             Fd = open(File, "wb")
 486             Fd.write(Content)
 487             Fd.close()
 488     except IOError as X:
 489         EdkLogger.error(None, FILE_CREATE_FAILURE, ExtraData='IOError %s' % X)
 490
 491     return True
 492
 493 ## Make a Python object persistent on file system
 494 #
 495 #   @param      Data    The object to be stored in file
 496 #   @param      File    The path of file to store the object
 497 #
 498 def DataDump(Data, File):
 499     Fd = None
 500     try:
 501         Fd = open(File, 'wb')
 502         cPickle.dump(Data, Fd, cPickle.HIGHEST_PROTOCOL)
 503     except:
 504         EdkLogger.error("", FILE_OPEN_FAILURE, ExtraData=File, RaiseError=False)
 505     finally:
 506         if Fd is not None:
 507             Fd.close()
 508
 509 ## Restore a Python object from a file
 510 #
 511 #   @param      File    The path of file stored the object
 512 #
 513 #   @retval     object  A python object
 514 #   @retval     None    If failure in file operation
 515 #
 516 def DataRestore(File):
 517     Data = None
 518     Fd = None
 519     try:
 520         Fd = open(File, 'rb')
 521         Data = cPickle.load(Fd)
 522     except Exception as e:
 523         EdkLogger.verbose("Failed to load [%s]\n\t%s" % (File, str(e)))
 524         Data = None
 525     finally:
 526         if Fd is not None:
 527             Fd.close()
 528     return Data
 529
 530 ## Retrieve and cache the real path name in file system
 531 #
 532 #   @param      Root    The root directory of path relative to
 533 #
 534 #   @retval     str     The path string if the path exists
 535 #   @retval     None    If path doesn't exist
 536 #
 537 class DirCache:
 538     _CACHE_ = set()
 539     _UPPER_CACHE_ = {}
 540
 541     def __init__(self, Root):
 542         self._Root = Root
 543         for F in os.listdir(Root):
 544             self._CACHE_.add(F)
 545             self._UPPER_CACHE_[F.upper()] = F
 546
 547     # =[] operator
 548     def __getitem__(self, Path):
 549         Path = Path[len(os.path.commonprefix([Path, self._Root])):]
 550         if not Path:
 551             return self._Root
 552         if Path and Path[0] == os.path.sep:
 553             Path = Path[1:]
 554         if Path in self._CACHE_:
 555             return os.path.join(self._Root, Path)
 556         UpperPath = Path.upper()
 557         if UpperPath in self._UPPER_CACHE_:
 558             return os.path.join(self._Root, self._UPPER_CACHE_[UpperPath])
 559
 560         IndexList = []
 561         LastSepIndex = -1
 562         SepIndex = Path.find(os.path.sep)
 563         while SepIndex > -1:
 564             Parent = UpperPath[:SepIndex]
 565             if Parent not in self._UPPER_CACHE_:
 566                 break
 567             LastSepIndex = SepIndex
 568             SepIndex = Path.find(os.path.sep, LastSepIndex + 1)
 569
 570         if LastSepIndex == -1:
 571             return None
 572
 573         Cwd = os.getcwd()
 574         os.chdir(self._Root)
 575         SepIndex = LastSepIndex
 576         while SepIndex > -1:
 577             Parent = Path[:SepIndex]
 578             ParentKey = UpperPath[:SepIndex]
 579             if ParentKey not in self._UPPER_CACHE_:
 580                 os.chdir(Cwd)
 581                 return None
 582
 583             if Parent in self._CACHE_:
 584                 ParentDir = Parent
 585             else:
 586                 ParentDir = self._UPPER_CACHE_[ParentKey]
 587             for F in os.listdir(ParentDir):
 588                 Dir = os.path.join(ParentDir, F)
 589                 self._CACHE_.add(Dir)
 590                 self._UPPER_CACHE_[Dir.upper()] = Dir
 591
 592             SepIndex = Path.find(os.path.sep, SepIndex + 1)
 593
 594         os.chdir(Cwd)
 595         if Path in self._CACHE_:
 596             return os.path.join(self._Root, Path)
 597         elif UpperPath in self._UPPER_CACHE_:
 598             return os.path.join(self._Root, self._UPPER_CACHE_[UpperPath])
 599         return None
 600
 601 def RealPath(File, Dir='', OverrideDir=''):
 602     NewFile = os.path.normpath(os.path.join(Dir, File))
 603     NewFile = GlobalData.gAllFiles[NewFile]
 604     if not NewFile and OverrideDir:
 605         NewFile = os.path.normpath(os.path.join(OverrideDir, File))
 606         NewFile = GlobalData.gAllFiles[NewFile]
 607     return NewFile
 608
 609 def RealPath2(File, Dir='', OverrideDir=''):
 610     NewFile = None
 611     if OverrideDir:
 612         NewFile = GlobalData.gAllFiles[os.path.normpath(os.path.join(OverrideDir, File))]
 613         if NewFile:
 614             if OverrideDir[-1] == os.path.sep:
 615                 return NewFile[len(OverrideDir):], NewFile[0:len(OverrideDir)]
 616             else:
 617                 return NewFile[len(OverrideDir) + 1:], NewFile[0:len(OverrideDir)]
 618     if GlobalData.gAllFiles:
 619         NewFile = GlobalData.gAllFiles[os.path.normpath(os.path.join(Dir, File))]
 620     if not NewFile:
 621         NewFile = os.path.normpath(os.path.join(Dir, File))
 622         if not os.path.exists(NewFile):
 623             return None, None
 624     if NewFile:
 625         if Dir:
 626             if Dir[-1] == os.path.sep:
 627                 return NewFile[len(Dir):], NewFile[0:len(Dir)]
 628             else:
 629                 return NewFile[len(Dir) + 1:], NewFile[0:len(Dir)]
 630         else:
 631             return NewFile, ''
 632
 633     return None, None
 634
 635 ## Get GUID value from given packages
 636 #
 637 #   @param      CName           The CName of the GUID
 638 #   @param      PackageList     List of packages looking-up in
 639 #   @param      Inffile         The driver file
 640 #
 641 #   @retval     GuidValue   if the CName is found in any given package
 642 #   @retval     None        if the CName is not found in all given packages
 643 #
 644 def GuidValue(CName, PackageList, Inffile = None):
 645     for P in PackageList:
 646         GuidKeys = P.Guids.keys()
 647         if Inffile and P._PrivateGuids:
 648             if not Inffile.startswith(P.MetaFile.Dir):
 649                 GuidKeys = [x for x in P.Guids if x not in P._PrivateGuids]
 650         if CName in GuidKeys:
 651             return P.Guids[CName]
 652     return None
 653
 654 ## Get Protocol value from given packages
 655 #
 656 #   @param      CName           The CName of the GUID
 657 #   @param      PackageList     List of packages looking-up in
 658 #   @param      Inffile         The driver file
 659 #
 660 #   @retval     GuidValue   if the CName is found in any given package
 661 #   @retval     None        if the CName is not found in all given packages
 662 #
 663 def ProtocolValue(CName, PackageList, Inffile = None):
 664     for P in PackageList:
 665         ProtocolKeys = P.Protocols.keys()
 666         if Inffile and P._PrivateProtocols:
 667             if not Inffile.startswith(P.MetaFile.Dir):
 668                 ProtocolKeys = [x for x in P.Protocols if x not in P._PrivateProtocols]
 669         if CName in ProtocolKeys:
 670             return P.Protocols[CName]
 671     return None
 672
 673 ## Get PPI value from given packages
 674 #
 675 #   @param      CName           The CName of the GUID
 676 #   @param      PackageList     List of packages looking-up in
 677 #   @param      Inffile         The driver file
 678 #
 679 #   @retval     GuidValue   if the CName is found in any given package
 680 #   @retval     None        if the CName is not found in all given packages
 681 #
 682 def PpiValue(CName, PackageList, Inffile = None):
 683     for P in PackageList:
 684         PpiKeys = P.Ppis.keys()
 685         if Inffile and P._PrivatePpis:
 686             if not Inffile.startswith(P.MetaFile.Dir):
 687                 PpiKeys = [x for x in P.Ppis if x not in P._PrivatePpis]
 688         if CName in PpiKeys:
 689             return P.Ppis[CName]
 690     return None
 691
 692 ## A string template class
 693 #
 694 #  This class implements a template for string replacement. A string template
 695 #  looks like following
 696 #
 697 #       ${BEGIN} other_string ${placeholder_name} other_string ${END}
 698 #
 699 #  The string between ${BEGIN} and ${END} will be repeated as many times as the
 700 #  length of "placeholder_name", which is a list passed through a dict. The
 701 #  "placeholder_name" is the key name of the dict. The ${BEGIN} and ${END} can
 702 #  be not used and, in this case, the "placeholder_name" must not a list and it
 703 #  will just be replaced once.
 704 #
 705 class TemplateString(object):
 706     _REPEAT_START_FLAG = "BEGIN"
 707     _REPEAT_END_FLAG = "END"
 708
 709     class Section(object):
 710         _LIST_TYPES = [type([]), type(set()), type((0,))]
 711
 712         def __init__(self, TemplateSection, PlaceHolderList):
 713             self._Template = TemplateSection
 714             self._PlaceHolderList = []
 715
 716             # Split the section into sub-sections according to the position of placeholders
 717             if PlaceHolderList:
 718                 self._SubSectionList = []
 719                 SubSectionStart = 0
 720                 #
 721                 # The placeholders passed in must be in the format of
 722                 #
 723                 #   PlaceHolderName, PlaceHolderStartPoint, PlaceHolderEndPoint
 724                 #
 725                 for PlaceHolder, Start, End in PlaceHolderList:
 726                     self._SubSectionList.append(TemplateSection[SubSectionStart:Start])
 727                     self._SubSectionList.append(TemplateSection[Start:End])
 728                     self._PlaceHolderList.append(PlaceHolder)
 729                     SubSectionStart = End
 730                 if SubSectionStart < len(TemplateSection):
 731                     self._SubSectionList.append(TemplateSection[SubSectionStart:])
 732             else:
 733                 self._SubSectionList = [TemplateSection]
 734
 735         def __str__(self):
 736             return self._Template + " : " + str(self._PlaceHolderList)
 737
 738         def Instantiate(self, PlaceHolderValues):
 739             RepeatTime = -1
 740             RepeatPlaceHolders = {}
 741             NonRepeatPlaceHolders = {}
 742
 743             for PlaceHolder in self._PlaceHolderList:
 744                 if PlaceHolder not in PlaceHolderValues:
 745                     continue
 746                 Value = PlaceHolderValues[PlaceHolder]
 747                 if type(Value) in self._LIST_TYPES:
 748                     if RepeatTime < 0:
 749                         RepeatTime = len(Value)
 750                     elif RepeatTime != len(Value):
 751                         EdkLogger.error(
 752                                     "TemplateString",
 753                                     PARAMETER_INVALID,
 754                                     "${%s} has different repeat time from others!" % PlaceHolder,
 755                                     ExtraData=str(self._Template)
 756                                     )
 757                     RepeatPlaceHolders["${%s}" % PlaceHolder] = Value
 758                 else:
 759                     NonRepeatPlaceHolders["${%s}" % PlaceHolder] = Value
 760
 761             if NonRepeatPlaceHolders:
 762                 StringList = []
 763                 for S in self._SubSectionList:
 764                     if S not in NonRepeatPlaceHolders:
 765                         StringList.append(S)
 766                     else:
 767                         StringList.append(str(NonRepeatPlaceHolders[S]))
 768             else:
 769                 StringList = self._SubSectionList
 770
 771             if RepeatPlaceHolders:
 772                 TempStringList = []
 773                 for Index in range(RepeatTime):
 774                     for S in StringList:
 775                         if S not in RepeatPlaceHolders:
 776                             TempStringList.append(S)
 777                         else:
 778                             TempStringList.append(str(RepeatPlaceHolders[S][Index]))
 779                 StringList = TempStringList
 780
 781             return "".join(StringList)
 782
 783     ## Constructor
 784     def __init__(self, Template=None):
 785         self.String = ''
 786         self.IsBinary = False
 787         self._Template = Template
 788         self._TemplateSectionList = self._Parse(Template)
 789
 790     ## str() operator
 791     #
 792     #   @retval     string  The string replaced
 793     #
 794     def __str__(self):
 795         return self.String
 796
 797     ## Split the template string into fragments per the ${BEGIN} and ${END} flags
 798     #
 799     #   @retval     list    A list of TemplateString.Section objects
 800     #
 801     def _Parse(self, Template):
 802         SectionStart = 0
 803         SearchFrom = 0
 804         MatchEnd = 0
 805         PlaceHolderList = []
 806         TemplateSectionList = []
 807         while Template:
 808             MatchObj = gPlaceholderPattern.search(Template, SearchFrom)
 809             if not MatchObj:
 810                 if MatchEnd <= len(Template):
 811                     TemplateSection = TemplateString.Section(Template[SectionStart:], PlaceHolderList)
 812                     TemplateSectionList.append(TemplateSection)
 813                 break
 814
 815             MatchString = MatchObj.group(1)
 816             MatchStart = MatchObj.start()
 817             MatchEnd = MatchObj.end()
 818
 819             if MatchString == self._REPEAT_START_FLAG:
 820                 if MatchStart > SectionStart:
 821                     TemplateSection = TemplateString.Section(Template[SectionStart:MatchStart], PlaceHolderList)
 822                     TemplateSectionList.append(TemplateSection)
 823                 SectionStart = MatchEnd
 824                 PlaceHolderList = []
 825             elif MatchString == self._REPEAT_END_FLAG:
 826                 TemplateSection = TemplateString.Section(Template[SectionStart:MatchStart], PlaceHolderList)
 827                 TemplateSectionList.append(TemplateSection)
 828                 SectionStart = MatchEnd
 829                 PlaceHolderList = []
 830             else:
 831                 PlaceHolderList.append((MatchString, MatchStart - SectionStart, MatchEnd - SectionStart))
 832             SearchFrom = MatchEnd
 833         return TemplateSectionList
 834
 835     ## Replace the string template with dictionary of placeholders and append it to previous one
 836     #
 837     #   @param      AppendString    The string template to append
 838     #   @param      Dictionary      The placeholder dictionaries
 839     #
 840     def Append(self, AppendString, Dictionary=None):
 841         if Dictionary:
 842             SectionList = self._Parse(AppendString)
 843             self.String += "".join(S.Instantiate(Dictionary) for S in SectionList)
 844         else:
 845             self.String += AppendString
 846
 847     ## Replace the string template with dictionary of placeholders
 848     #
 849     #   @param      Dictionary      The placeholder dictionaries
 850     #
 851     #   @retval     str             The string replaced with placeholder values
 852     #
 853     def Replace(self, Dictionary=None):
 854         return "".join(S.Instantiate(Dictionary) for S in self._TemplateSectionList)
 855
 856 ## Progress indicator class
 857 #
 858 #  This class makes use of thread to print progress on console.
 859 #
 860 class Progressor:
 861     # for avoiding deadloop
 862     _StopFlag = None
 863     _ProgressThread = None
 864     _CheckInterval = 0.25
 865
 866     ## Constructor
 867     #
 868     #   @param      OpenMessage     The string printed before progress charaters
 869     #   @param      CloseMessage    The string printed after progress charaters
 870     #   @param      ProgressChar    The charater used to indicate the progress
 871     #   @param      Interval        The interval in seconds between two progress charaters
 872     #
 873     def __init__(self, OpenMessage="", CloseMessage="", ProgressChar='.', Interval=1.0):
 874         self.PromptMessage = OpenMessage
 875         self.CodaMessage = CloseMessage
 876         self.ProgressChar = ProgressChar
 877         self.Interval = Interval
 878         if Progressor._StopFlag is None:
 879             Progressor._StopFlag = threading.Event()
 880
 881     ## Start to print progress charater
 882     #
 883     #   @param      OpenMessage     The string printed before progress charaters
 884     #
 885     def Start(self, OpenMessage=None):
 886         if OpenMessage is not None:
 887             self.PromptMessage = OpenMessage
 888         Progressor._StopFlag.clear()
 889         if Progressor._ProgressThread is None:
 890             Progressor._ProgressThread = threading.Thread(target=self._ProgressThreadEntry)
 891             Progressor._ProgressThread.setDaemon(False)
 892             Progressor._ProgressThread.start()
 893
 894     ## Stop printing progress charater
 895     #
 896     #   @param      CloseMessage    The string printed after progress charaters
 897     #
 898     def Stop(self, CloseMessage=None):
 899         OriginalCodaMessage = self.CodaMessage
 900         if CloseMessage is not None:
 901             self.CodaMessage = CloseMessage
 902         self.Abort()
 903         self.CodaMessage = OriginalCodaMessage
 904
 905     ## Thread entry method
 906     def _ProgressThreadEntry(self):
 907         sys.stdout.write(self.PromptMessage + " ")
 908         sys.stdout.flush()
 909         TimeUp = 0.0
 910         while not Progressor._StopFlag.isSet():
 911             if TimeUp <= 0.0:
 912                 sys.stdout.write(self.ProgressChar)
 913                 sys.stdout.flush()
 914                 TimeUp = self.Interval
 915             time.sleep(self._CheckInterval)
 916             TimeUp -= self._CheckInterval
 917         sys.stdout.write(" " + self.CodaMessage + "\n")
 918         sys.stdout.flush()
 919
 920     ## Abort the progress display
 921     @staticmethod
 922     def Abort():
 923         if Progressor._StopFlag is not None:
 924             Progressor._StopFlag.set()
 925         if Progressor._ProgressThread is not None:
 926             Progressor._ProgressThread.join()
 927             Progressor._ProgressThread = None
 928
 929 ## A dict which can access its keys and/or values orderly
 930 #
 931 #  The class implements a new kind of dict which its keys or values can be
 932 #  accessed in the order they are added into the dict. It guarantees the order
 933 #  by making use of an internal list to keep a copy of keys.
 934 #
 935 class sdict(IterableUserDict):
 936     ## Constructor
 937     def __init__(self):
 938         IterableUserDict.__init__(self)
 939         self._key_list = []
 940
 941     ## [] operator
 942     def __setitem__(self, key, value):
 943         if key not in self._key_list:
 944             self._key_list.append(key)
 945         IterableUserDict.__setitem__(self, key, value)
 946
 947     ## del operator
 948     def __delitem__(self, key):
 949         self._key_list.remove(key)
 950         IterableUserDict.__delitem__(self, key)
 951
 952     ## used in "for k in dict" loop to ensure the correct order
 953     def __iter__(self):
 954         return self.iterkeys()
 955
 956     ## len() support
 957     def __len__(self):
 958         return len(self._key_list)
 959
 960     ## "in" test support
 961     def __contains__(self, key):
 962         return key in self._key_list
 963
 964     ## indexof support
 965     def index(self, key):
 966         return self._key_list.index(key)
 967
 968     ## insert support
 969     def insert(self, key, newkey, newvalue, order):
 970         index = self._key_list.index(key)
 971         if order == 'BEFORE':
 972             self._key_list.insert(index, newkey)
 973             IterableUserDict.__setitem__(self, newkey, newvalue)
 974         elif order == 'AFTER':
 975             self._key_list.insert(index + 1, newkey)
 976             IterableUserDict.__setitem__(self, newkey, newvalue)
 977
 978     ## append support
 979     def append(self, sdict):
 980         for key in sdict:
 981             if key not in self._key_list:
 982                 self._key_list.append(key)
 983             IterableUserDict.__setitem__(self, key, sdict[key])
 984
 985     def has_key(self, key):
 986         return key in self._key_list
 987
 988     ## Empty the dict
 989     def clear(self):
 990         self._key_list = []
 991         IterableUserDict.clear(self)
 992
 993     ## Return a copy of keys
 994     def keys(self):
 995         keys = []
 996         for key in self._key_list:
 997             keys.append(key)
 998         return keys
 999
1000     ## Return a copy of values
1001     def values(self):
1002         values = []
1003         for key in self._key_list:
1004             values.append(self[key])
1005         return values
1006
1007     ## Return a copy of (key, value) list
1008     def items(self):
1009         items = []
1010         for key in self._key_list:
1011             items.append((key, self[key]))
1012         return items
1013
1014     ## Iteration support
1015     def iteritems(self):
1016         return iter(self.items())
1017
1018     ## Keys interation support
1019     def iterkeys(self):
1020         return iter(self.keys())
1021
1022     ## Values interation support
1023     def itervalues(self):
1024         return iter(self.values())
1025
1026     ## Return value related to a key, and remove the (key, value) from the dict
1027     def pop(self, key, *dv):
1028         value = None
1029         if key in self._key_list:
1030             value = self[key]
1031             self.__delitem__(key)
1032         elif len(dv) != 0 :
1033             value = kv[0]
1034         return value
1035
1036     ## Return (key, value) pair, and remove the (key, value) from the dict
1037     def popitem(self):
1038         key = self._key_list[-1]
1039         value = self[key]
1040         self.__delitem__(key)
1041         return key, value
1042
1043     def update(self, dict=None, **kwargs):
1044         if dict is not None:
1045             for k, v in dict.items():
1046                 self[k] = v
1047         if len(kwargs):
1048             for k, v in kwargs.items():
1049                 self[k] = v
1050
1051 ## Dictionary with restricted keys
1052 #
1053 class rdict(dict):
1054     ## Constructor
1055     def __init__(self, KeyList):
1056         for Key in KeyList:
1057             dict.__setitem__(self, Key, "")
1058
1059     ## []= operator
1060     def __setitem__(self, key, value):
1061         if key not in self:
1062             EdkLogger.error("RestrictedDict", ATTRIBUTE_SET_FAILURE, "Key [%s] is not allowed" % key,
1063                             ExtraData=", ".join(dict.keys(self)))
1064         dict.__setitem__(self, key, value)
1065
1066     ## =[] operator
1067     def __getitem__(self, key):
1068         if key not in self:
1069             return ""
1070         return dict.__getitem__(self, key)
1071
1072     ## del operator
1073     def __delitem__(self, key):
1074         EdkLogger.error("RestrictedDict", ATTRIBUTE_ACCESS_DENIED, ExtraData="del")
1075
1076     ## Empty the dict
1077     def clear(self):
1078         for Key in self:
1079             self.__setitem__(Key, "")
1080
1081     ## Return value related to a key, and remove the (key, value) from the dict
1082     def pop(self, key, *dv):
1083         EdkLogger.error("RestrictedDict", ATTRIBUTE_ACCESS_DENIED, ExtraData="pop")
1084
1085     ## Return (key, value) pair, and remove the (key, value) from the dict
1086     def popitem(self):
1087         EdkLogger.error("RestrictedDict", ATTRIBUTE_ACCESS_DENIED, ExtraData="popitem")
1088
1089 ## Dictionary using prioritized list as key
1090 #
1091 class tdict:
1092     _ListType = type([])
1093     _TupleType = type(())
1094     _Wildcard = 'COMMON'
1095     _ValidWildcardList = ['COMMON', 'DEFAULT', 'ALL', '*', 'PLATFORM']
1096
1097     def __init__(self, _Single_=False, _Level_=2):
1098         self._Level_ = _Level_
1099         self.data = {}
1100         self._Single_ = _Single_
1101
1102     # =[] operator
1103     def __getitem__(self, key):
1104         KeyType = type(key)
1105         RestKeys = None
1106         if KeyType == self._ListType or KeyType == self._TupleType:
1107             FirstKey = key[0]
1108             if len(key) > 1:
1109                 RestKeys = key[1:]
1110             elif self._Level_ > 1:
1111                 RestKeys = [self._Wildcard for i in range(0, self._Level_ - 1)]
1112         else:
1113             FirstKey = key
1114             if self._Level_ > 1:
1115                 RestKeys = [self._Wildcard for i in range(0, self._Level_ - 1)]
1116
1117         if FirstKey is None or str(FirstKey).upper() in self._ValidWildcardList:
1118             FirstKey = self._Wildcard
1119
1120         if self._Single_:
1121             return self._GetSingleValue(FirstKey, RestKeys)
1122         else:
1123             return self._GetAllValues(FirstKey, RestKeys)
1124
1125     def _GetSingleValue(self, FirstKey, RestKeys):
1126         Value = None
1127         #print "%s-%s" % (FirstKey, self._Level_) ,
1128         if self._Level_ > 1:
1129             if FirstKey == self._Wildcard:
1130                 if FirstKey in self.data:
1131                     Value = self.data[FirstKey][RestKeys]
1132                 if Value is None:
1133                     for Key in self.data:
1134                         Value = self.data[Key][RestKeys]
1135                         if Value is not None: break
1136             else:
1137                 if FirstKey in self.data:
1138                     Value = self.data[FirstKey][RestKeys]
1139                 if Value is None and self._Wildcard in self.data:
1140                     #print "Value=None"
1141                     Value = self.data[self._Wildcard][RestKeys]
1142         else:
1143             if FirstKey == self._Wildcard:
1144                 if FirstKey in self.data:
1145                     Value = self.data[FirstKey]
1146                 if Value is None:
1147                     for Key in self.data:
1148                         Value = self.data[Key]
1149                         if Value is not None: break
1150             else:
1151                 if FirstKey in self.data:
1152                     Value = self.data[FirstKey]
1153                 elif self._Wildcard in self.data:
1154                     Value = self.data[self._Wildcard]
1155         return Value
1156
1157     def _GetAllValues(self, FirstKey, RestKeys):
1158         Value = []
1159         if self._Level_ > 1:
1160             if FirstKey == self._Wildcard:
1161                 for Key in self.data:
1162                     Value += self.data[Key][RestKeys]
1163             else:
1164                 if FirstKey in self.data:
1165                     Value += self.data[FirstKey][RestKeys]
1166                 if self._Wildcard in self.data:
1167                     Value += self.data[self._Wildcard][RestKeys]
1168         else:
1169             if FirstKey == self._Wildcard:
1170                 for Key in self.data:
1171                     Value.append(self.data[Key])
1172             else:
1173                 if FirstKey in self.data:
1174                     Value.append(self.data[FirstKey])
1175                 if self._Wildcard in self.data:
1176                     Value.append(self.data[self._Wildcard])
1177         return Value
1178
1179     ## []= operator
1180     def __setitem__(self, key, value):
1181         KeyType = type(key)
1182         RestKeys = None
1183         if KeyType == self._ListType or KeyType == self._TupleType:
1184             FirstKey = key[0]
1185             if len(key) > 1:
1186                 RestKeys = key[1:]
1187             else:
1188                 RestKeys = [self._Wildcard for i in range(0, self._Level_ - 1)]
1189         else:
1190             FirstKey = key
1191             if self._Level_ > 1:
1192                 RestKeys = [self._Wildcard for i in range(0, self._Level_ - 1)]
1193
1194         if FirstKey in self._ValidWildcardList:
1195             FirstKey = self._Wildcard
1196
1197         if FirstKey not in self.data and self._Level_ > 0:
1198             self.data[FirstKey] = tdict(self._Single_, self._Level_ - 1)
1199
1200         if self._Level_ > 1:
1201             self.data[FirstKey][RestKeys] = value
1202         else:
1203             self.data[FirstKey] = value
1204
1205     def SetGreedyMode(self):
1206         self._Single_ = False
1207         if self._Level_ > 1:
1208             for Key in self.data:
1209                 self.data[Key].SetGreedyMode()
1210
1211     def SetSingleMode(self):
1212         self._Single_ = True
1213         if self._Level_ > 1:
1214             for Key in self.data:
1215                 self.data[Key].SetSingleMode()
1216
1217     def GetKeys(self, KeyIndex=0):
1218         assert KeyIndex >= 0
1219         if KeyIndex == 0:
1220             return set(self.data.keys())
1221         else:
1222             keys = set()
1223             for Key in self.data:
1224                 keys |= self.data[Key].GetKeys(KeyIndex - 1)
1225             return keys
1226
1227 def IsFieldValueAnArray (Value):
1228     Value = Value.strip()
1229     if Value.startswith(TAB_GUID) and Value.endswith(')'):
1230         return True
1231     if Value.startswith('L"') and Value.endswith('"')  and len(list(Value[2:-1])) > 1:
1232         return True
1233     if Value[0] == '"' and Value[-1] == '"' and len(list(Value[1:-1])) > 1:
1234         return True
1235     if Value[0] == '{' and Value[-1] == '}':
1236         return True
1237     if Value.startswith("L'") and Value.endswith("'") and len(list(Value[2:-1])) > 1:
1238         return True
1239     if Value[0] == "'" and Value[-1] == "'" and len(list(Value[1:-1])) > 1:
1240         return True
1241     return False
1242
1243 def AnalyzePcdExpression(Setting):
1244     Setting = Setting.strip()
1245     # There might be escaped quote in a string: \", \\\" , \', \\\'
1246     Data = Setting
1247     # There might be '|' in string and in ( ... | ... ), replace it with '-'
1248     NewStr = ''
1249     InSingleQuoteStr = False
1250     InDoubleQuoteStr = False
1251     Pair = 0
1252     for Index, ch in enumerate(Data):
1253         if ch == '"' and not InSingleQuoteStr:
1254             if Data[Index - 1] != '\\':
1255                 InDoubleQuoteStr = not InDoubleQuoteStr
1256         elif ch == "'" and not InDoubleQuoteStr:
1257             if Data[Index - 1] != '\\':
1258                 InSingleQuoteStr = not InSingleQuoteStr
1259         elif ch == '(' and not (InSingleQuoteStr or InDoubleQuoteStr):
1260             Pair += 1
1261         elif ch == ')' and not (InSingleQuoteStr or InDoubleQuoteStr):
1262             Pair -= 1
1263
1264         if (Pair > 0 or InSingleQuoteStr or InDoubleQuoteStr) and ch == TAB_VALUE_SPLIT:
1265             NewStr += '-'
1266         else:
1267             NewStr += ch
1268     FieldList = []
1269     StartPos = 0
1270     while True:
1271         Pos = NewStr.find(TAB_VALUE_SPLIT, StartPos)
1272         if Pos < 0:
1273             FieldList.append(Setting[StartPos:].strip())
1274             break
1275         FieldList.append(Setting[StartPos:Pos].strip())
1276         StartPos = Pos + 1
1277
1278     return FieldList
1279
1280 def ParseDevPathValue (Value):
1281     if '\\' in Value:
1282         Value.replace('\\', '/').replace(' ', '')
1283
1284     Cmd = 'DevicePath ' + '"' + Value + '"'
1285     try:
1286         p = subprocess.Popen(Cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
1287         out, err = p.communicate()
1288     except Exception as X:
1289         raise BadExpression("DevicePath: %s" % (str(X)) )
1290     finally:
1291         subprocess._cleanup()
1292         p.stdout.close()
1293         p.stderr.close()
1294     if err:
1295         raise BadExpression("DevicePath: %s" % str(err))
1296     Size = len(out.split())
1297     out = ','.join(out.split())
1298     return '{' + out + '}', Size
1299
1300 def ParseFieldValue (Value):
1301     if isinstance(Value, type(0)):
1302         return Value, (Value.bit_length() + 7) / 8
1303     if not isinstance(Value, type('')):
1304         raise BadExpression('Type %s is %s' %(Value, type(Value)))
1305     Value = Value.strip()
1306     if Value.startswith(TAB_UINT8) and Value.endswith(')'):
1307         Value, Size = ParseFieldValue(Value.split('(', 1)[1][:-1])
1308         if Size > 1:
1309             raise BadExpression('Value (%s) Size larger than %d' %(Value, Size))
1310         return Value, 1
1311     if Value.startswith(TAB_UINT16) and Value.endswith(')'):
1312         Value, Size = ParseFieldValue(Value.split('(', 1)[1][:-1])
1313         if Size > 2:
1314             raise BadExpression('Value (%s) Size larger than %d' %(Value, Size))
1315         return Value, 2
1316     if Value.startswith(TAB_UINT32) and Value.endswith(')'):
1317         Value, Size = ParseFieldValue(Value.split('(', 1)[1][:-1])
1318         if Size > 4:
1319             raise BadExpression('Value (%s) Size larger than %d' %(Value, Size))
1320         return Value, 4
1321     if Value.startswith(TAB_UINT64) and Value.endswith(')'):
1322         Value, Size = ParseFieldValue(Value.split('(', 1)[1][:-1])
1323         if Size > 8:
1324             raise BadExpression('Value (%s) Size larger than %d' % (Value, Size))
1325         return Value, 8
1326     if Value.startswith(TAB_GUID) and Value.endswith(')'):
1327         Value = Value.split('(', 1)[1][:-1].strip()
1328         if Value[0] == '{' and Value[-1] == '}':
1329             TmpValue = GuidStructureStringToGuidString(Value)
1330             if len(TmpValue) == 0:
1331                 raise BadExpression("Invalid GUID value string %s" % Value)
1332             Value = TmpValue
1333         if Value[0] == '"' and Value[-1] == '"':
1334             Value = Value[1:-1]
1335         try:
1336             Value = "'" + uuid.UUID(Value).get_bytes_le() + "'"
1337         except ValueError as Message:
1338             raise BadExpression(Message)
1339         Value, Size = ParseFieldValue(Value)
1340         return Value, 16
1341     if Value.startswith('L"') and Value.endswith('"'):
1342         # Unicode String
1343         # translate escape character
1344         Value = Value[1:]
1345         try:
1346             Value = eval(Value)
1347         except:
1348             Value = Value[1:-1]
1349         List = list(Value)
1350         List.reverse()
1351         Value = 0
1352         for Char in List:
1353             Value = (Value << 16) | ord(Char)
1354         return Value, (len(List) + 1) * 2
1355     if Value.startswith('"') and Value.endswith('"'):
1356         # ASCII String
1357         # translate escape character
1358         try:
1359             Value = eval(Value)
1360         except:
1361             Value = Value[1:-1]
1362         List = list(Value)
1363         List.reverse()
1364         Value = 0
1365         for Char in List:
1366             Value = (Value << 8) | ord(Char)
1367         return Value, len(List) + 1
1368     if Value.startswith("L'") and Value.endswith("'"):
1369         # Unicode Character Constant
1370         # translate escape character
1371         Value = Value[1:]
1372         try:
1373             Value = eval(Value)
1374         except:
1375             Value = Value[1:-1]
1376         List = list(Value)
1377         if len(List) == 0:
1378             raise BadExpression('Length %s is %s' % (Value, len(List)))
1379         List.reverse()
1380         Value = 0
1381         for Char in List:
1382             Value = (Value << 16) | ord(Char)
1383         return Value, len(List) * 2
1384     if Value.startswith("'") and Value.endswith("'"):
1385         # Character constant
1386         # translate escape character
1387         try:
1388             Value = eval(Value)
1389         except:
1390             Value = Value[1:-1]
1391         List = list(Value)
1392         if len(List) == 0:
1393             raise BadExpression('Length %s is %s' % (Value, len(List)))
1394         List.reverse()
1395         Value = 0
1396         for Char in List:
1397             Value = (Value << 8) | ord(Char)
1398         return Value, len(List)
1399     if Value.startswith('{') and Value.endswith('}'):
1400         # Byte array
1401         Value = Value[1:-1]
1402         List = [Item.strip() for Item in Value.split(',')]
1403         List.reverse()
1404         Value = 0
1405         RetSize = 0
1406         for Item in List:
1407             ItemValue, Size = ParseFieldValue(Item)
1408             RetSize += Size
1409             for I in range(Size):
1410                 Value = (Value << 8) | ((ItemValue >> 8 * I) & 0xff)
1411         return Value, RetSize
1412     if Value.startswith('DEVICE_PATH(') and Value.endswith(')'):
1413         Value = Value.replace("DEVICE_PATH(", '').rstrip(')')
1414         Value = Value.strip().strip('"')
1415         return ParseDevPathValue(Value)
1416     if Value.lower().startswith('0x'):
1417         Value = int(Value, 16)
1418         if Value == 0:
1419             return 0, 1
1420         return Value, (Value.bit_length() + 7) / 8
1421     if Value[0].isdigit():
1422         Value = int(Value, 10)
1423         if Value == 0:
1424             return 0, 1
1425         return Value, (Value.bit_length() + 7) / 8
1426     if Value.lower() == 'true':
1427         return 1, 1
1428     if Value.lower() == 'false':
1429         return 0, 1
1430     return Value, 1
1431
1432 ## AnalyzeDscPcd
1433 #
1434 #  Analyze DSC PCD value, since there is no data type info in DSC
1435 #  This fuction is used to match functions (AnalyzePcdData) used for retrieving PCD value from database
1436 #  1. Feature flag: TokenSpace.PcdCName|PcdValue
1437 #  2. Fix and Patch:TokenSpace.PcdCName|PcdValue[|MaxSize]
1438 #  3. Dynamic default:
1439 #     TokenSpace.PcdCName|PcdValue[|VOID*[|MaxSize]]
1440 #     TokenSpace.PcdCName|PcdValue
1441 #  4. Dynamic VPD:
1442 #     TokenSpace.PcdCName|VpdOffset[|VpdValue]
1443 #     TokenSpace.PcdCName|VpdOffset[|MaxSize[|VpdValue]]
1444 #  5. Dynamic HII:
1445 #     TokenSpace.PcdCName|HiiString|VaiableGuid|VariableOffset[|HiiValue]
1446 #  PCD value needs to be located in such kind of string, and the PCD value might be an expression in which
1447 #    there might have "|" operator, also in string value.
1448 #
1449 #  @param Setting: String contain information described above with "TokenSpace.PcdCName|" stripped
1450 #  @param PcdType: PCD type: feature, fixed, dynamic default VPD HII
1451 #  @param DataType: The datum type of PCD: VOID*, UNIT, BOOL
1452 #  @retval:
1453 #    ValueList: A List contain fields described above
1454 #    IsValid:   True if conforming EBNF, otherwise False
1455 #    Index:     The index where PcdValue is in ValueList
1456 #
1457 def AnalyzeDscPcd(Setting, PcdType, DataType=''):
1458     FieldList = AnalyzePcdExpression(Setting)
1459
1460     IsValid = True
1461     if PcdType in (MODEL_PCD_FIXED_AT_BUILD, MODEL_PCD_PATCHABLE_IN_MODULE, MODEL_PCD_FEATURE_FLAG):
1462         Value = FieldList[0]
1463         Size = ''
1464         if len(FieldList) > 1:
1465             if FieldList[1].upper().startswith("0X") or FieldList[1].isdigit():
1466                 Size = FieldList[1]
1467             else:
1468                 DataType = FieldList[1]
1469
1470         if len(FieldList) > 2:
1471             Size = FieldList[2]
1472         if DataType == "":
1473             IsValid = (len(FieldList) <= 1)
1474         else:
1475             IsValid = (len(FieldList) <= 3)
1476 #         Value, Size = ParseFieldValue(Value)
1477         if Size:
1478             try:
1479                 int(Size, 16) if Size.upper().startswith("0X") else int(Size)
1480             except:
1481                 IsValid = False
1482                 Size = -1
1483         return [str(Value), '', str(Size)], IsValid, 0
1484     elif PcdType in (MODEL_PCD_DYNAMIC_DEFAULT, MODEL_PCD_DYNAMIC_EX_DEFAULT):
1485         Value = FieldList[0]
1486         Size = Type = ''
1487         if len(FieldList) > 1:
1488             Type = FieldList[1]
1489         else:
1490             Type = DataType
1491         if len(FieldList) > 2:
1492             Size = FieldList[2]
1493         if DataType == "":
1494             IsValid = (len(FieldList) <= 1)
1495         else:
1496             IsValid = (len(FieldList) <= 3)
1497
1498         if Size:
1499             try:
1500                 int(Size, 16) if Size.upper().startswith("0X") else int(Size)
1501             except:
1502                 IsValid = False
1503                 Size = -1
1504         return [Value, Type, str(Size)], IsValid, 0
1505     elif PcdType in (MODEL_PCD_DYNAMIC_VPD, MODEL_PCD_DYNAMIC_EX_VPD):
1506         VpdOffset = FieldList[0]
1507         Value = Size = ''
1508         if not DataType == TAB_VOID:
1509             if len(FieldList) > 1:
1510                 Value = FieldList[1]
1511         else:
1512             if len(FieldList) > 1:
1513                 Size = FieldList[1]
1514             if len(FieldList) > 2:
1515                 Value = FieldList[2]
1516         if DataType == "":
1517             IsValid = (len(FieldList) <= 1)
1518         else:
1519             IsValid = (len(FieldList) <= 3)
1520         if Size:
1521             try:
1522                 int(Size, 16) if Size.upper().startswith("0X") else int(Size)
1523             except:
1524                 IsValid = False
1525                 Size = -1
1526         return [VpdOffset, str(Size), Value], IsValid, 2
1527     elif PcdType in (MODEL_PCD_DYNAMIC_HII, MODEL_PCD_DYNAMIC_EX_HII):
1528         HiiString = FieldList[0]
1529         Guid = Offset = Value = Attribute = ''
1530         if len(FieldList) > 1:
1531             Guid = FieldList[1]
1532         if len(FieldList) > 2:
1533             Offset = FieldList[2]
1534         if len(FieldList) > 3:
1535             Value = FieldList[3]
1536         if len(FieldList) > 4:
1537             Attribute = FieldList[4]
1538         IsValid = (3 <= len(FieldList) <= 5)
1539         return [HiiString, Guid, Offset, Value, Attribute], IsValid, 3
1540     return [], False, 0
1541
1542 ## AnalyzePcdData
1543 #
1544 #  Analyze the pcd Value, Datum type and TokenNumber.
1545 #  Used to avoid split issue while the value string contain "|" character
1546 #
1547 #  @param[in] Setting:  A String contain value/datum type/token number information;
1548 #
1549 #  @retval   ValueList: A List contain value, datum type and toke number.
1550 #
1551 def AnalyzePcdData(Setting):
1552     ValueList = ['', '', '']
1553
1554     ValueRe = re.compile(r'^\s*L?\".*\|.*\"')
1555     PtrValue = ValueRe.findall(Setting)
1556
1557     ValueUpdateFlag = False
1558
1559     if len(PtrValue) >= 1:
1560         Setting = re.sub(ValueRe, '', Setting)
1561         ValueUpdateFlag = True
1562
1563     TokenList = Setting.split(TAB_VALUE_SPLIT)
1564     ValueList[0:len(TokenList)] = TokenList
1565
1566     if ValueUpdateFlag:
1567         ValueList[0] = PtrValue[0]
1568
1569     return ValueList
1570
1571 ## check format of PCD value against its the datum type
1572 #
1573 # For PCD value setting
1574 #
1575 def CheckPcdDatum(Type, Value):
1576     if Type == TAB_VOID:
1577         ValueRe = re.compile(r'\s*L?\".*\"\s*$')
1578         if not (((Value.startswith('L"') or Value.startswith('"')) and Value.endswith('"'))
1579                 or (Value.startswith('{') and Value.endswith('}')) or (Value.startswith("L'") or Value.startswith("'") and Value.endswith("'"))
1580                ):
1581             return False, "Invalid value [%s] of type [%s]; must be in the form of {...} for array"\
1582                           ", \"...\" or \'...\' for string, L\"...\" or L\'...\' for unicode string" % (Value, Type)
1583         elif ValueRe.match(Value):
1584             # Check the chars in UnicodeString or CString is printable
1585             if Value.startswith("L"):
1586                 Value = Value[2:-1]
1587             else:
1588                 Value = Value[1:-1]
1589             Printset = set(string.printable)
1590             Printset.remove(TAB_PRINTCHAR_VT)
1591             Printset.add(TAB_PRINTCHAR_BS)
1592             Printset.add(TAB_PRINTCHAR_NUL)
1593             if not set(Value).issubset(Printset):
1594                 PrintList = sorted(Printset)
1595                 return False, "Invalid PCD string value of type [%s]; must be printable chars %s." % (Type, PrintList)
1596     elif Type == 'BOOLEAN':
1597         if Value not in ['TRUE', 'True', 'true', '0x1', '0x01', '1', 'FALSE', 'False', 'false', '0x0', '0x00', '0']:
1598             return False, "Invalid value [%s] of type [%s]; must be one of TRUE, True, true, 0x1, 0x01, 1"\
1599                           ", FALSE, False, false, 0x0, 0x00, 0" % (Value, Type)
1600     elif Type in [TAB_UINT8, TAB_UINT16, TAB_UINT32, TAB_UINT64]:
1601         try:
1602             Value = long(Value, 0)
1603         except:
1604             return False, "Invalid value [%s] of type [%s];"\
1605                           " must be a hexadecimal, decimal or octal in C language format." % (Value, Type)
1606     else:
1607         return True, "StructurePcd"
1608
1609     return True, ""
1610
1611 ## Split command line option string to list
1612 #
1613 # subprocess.Popen needs the args to be a sequence. Otherwise there's problem
1614 # in non-windows platform to launch command
1615 #
1616 def SplitOption(OptionString):
1617     OptionList = []
1618     LastChar = " "
1619     OptionStart = 0
1620     QuotationMark = ""
1621     for Index in range(0, len(OptionString)):
1622         CurrentChar = OptionString[Index]
1623         if CurrentChar in ['"', "'"]:
1624             if QuotationMark == CurrentChar:
1625                 QuotationMark = ""
1626             elif QuotationMark == "":
1627                 QuotationMark = CurrentChar
1628             continue
1629         elif QuotationMark:
1630             continue
1631
1632         if CurrentChar in ["/", "-"] and LastChar in [" ", "\t", "\r", "\n"]:
1633             if Index > OptionStart:
1634                 OptionList.append(OptionString[OptionStart:Index - 1])
1635             OptionStart = Index
1636         LastChar = CurrentChar
1637     OptionList.append(OptionString[OptionStart:])
1638     return OptionList
1639
1640 def CommonPath(PathList):
1641     P1 = min(PathList).split(os.path.sep)
1642     P2 = max(PathList).split(os.path.sep)
1643     for Index in xrange(min(len(P1), len(P2))):
1644         if P1[Index] != P2[Index]:
1645             return os.path.sep.join(P1[:Index])
1646     return os.path.sep.join(P1)
1647
1648 #
1649 # Convert string to C format array
1650 #
1651 def ConvertStringToByteArray(Value):
1652     Value = Value.strip()
1653     if not Value:
1654         return None
1655     if Value[0] == '{':
1656         if not Value.endswith('}'):
1657             return None
1658         Value = Value.replace(' ', '').replace('{', '').replace('}', '')
1659         ValFields = Value.split(',')
1660         try:
1661             for Index in range(len(ValFields)):
1662                 ValFields[Index] = str(int(ValFields[Index], 0))
1663         except ValueError:
1664             return None
1665         Value = '{' + ','.join(ValFields) + '}'
1666         return Value
1667
1668     Unicode = False
1669     if Value.startswith('L"'):
1670         if not Value.endswith('"'):
1671             return None
1672         Value = Value[1:]
1673         Unicode = True
1674     elif not Value.startswith('"') or not Value.endswith('"'):
1675         return None
1676
1677     Value = eval(Value)         # translate escape character
1678     NewValue = '{'
1679     for Index in range(0, len(Value)):
1680         if Unicode:
1681             NewValue = NewValue + str(ord(Value[Index]) % 0x10000) + ','
1682         else:
1683             NewValue = NewValue + str(ord(Value[Index]) % 0x100) + ','
1684     Value = NewValue + '0}'
1685     return Value
1686
1687 class PathClass(object):
1688     def __init__(self, File='', Root='', AlterRoot='', Type='', IsBinary=False,
1689                  Arch='COMMON', ToolChainFamily='', Target='', TagName='', ToolCode=''):
1690         self.Arch = Arch
1691         self.File = str(File)
1692         if os.path.isabs(self.File):
1693             self.Root = ''
1694             self.AlterRoot = ''
1695         else:
1696             self.Root = str(Root)
1697             self.AlterRoot = str(AlterRoot)
1698
1699         # Remove any '.' and '..' in path
1700         if self.Root:
1701             self.Root = mws.getWs(self.Root, self.File)
1702             self.Path = os.path.normpath(os.path.join(self.Root, self.File))
1703             self.Root = os.path.normpath(CommonPath([self.Root, self.Path]))
1704             # eliminate the side-effect of 'C:'
1705             if self.Root[-1] == ':':
1706                 self.Root += os.path.sep
1707             # file path should not start with path separator
1708             if self.Root[-1] == os.path.sep:
1709                 self.File = self.Path[len(self.Root):]
1710             else:
1711                 self.File = self.Path[len(self.Root) + 1:]
1712         else:
1713             self.Path = os.path.normpath(self.File)
1714
1715         self.SubDir, self.Name = os.path.split(self.File)
1716         self.BaseName, self.Ext = os.path.splitext(self.Name)
1717
1718         if self.Root:
1719             if self.SubDir:
1720                 self.Dir = os.path.join(self.Root, self.SubDir)
1721             else:
1722                 self.Dir = self.Root
1723         else:
1724             self.Dir = self.SubDir
1725
1726         if IsBinary:
1727             self.Type = Type
1728         else:
1729             self.Type = self.Ext.lower()
1730
1731         self.IsBinary = IsBinary
1732         self.Target = Target
1733         self.TagName = TagName
1734         self.ToolCode = ToolCode
1735         self.ToolChainFamily = ToolChainFamily
1736
1737         self._Key = None
1738
1739     ## Convert the object of this class to a string
1740     #
1741     #  Convert member Path of the class to a string
1742     #
1743     #  @retval string Formatted String
1744     #
1745     def __str__(self):
1746         return self.Path
1747
1748     ## Override __eq__ function
1749     #
1750     # Check whether PathClass are the same
1751     #
1752     # @retval False The two PathClass are different
1753     # @retval True  The two PathClass are the same
1754     #
1755     def __eq__(self, Other):
1756         if isinstance(Other, type(self)):
1757             return self.Path == Other.Path
1758         else:
1759             return self.Path == str(Other)
1760
1761     ## Override __cmp__ function
1762     #
1763     # Customize the comparsion operation of two PathClass
1764     #
1765     # @retval 0     The two PathClass are different
1766     # @retval -1    The first PathClass is less than the second PathClass
1767     # @retval 1     The first PathClass is Bigger than the second PathClass
1768     def __cmp__(self, Other):
1769         if isinstance(Other, type(self)):
1770             OtherKey = Other.Path
1771         else:
1772             OtherKey = str(Other)
1773
1774         SelfKey = self.Path
1775         if SelfKey == OtherKey:
1776             return 0
1777         elif SelfKey > OtherKey:
1778             return 1
1779         else:
1780             return -1
1781
1782     ## Override __hash__ function
1783     #
1784     # Use Path as key in hash table
1785     #
1786     # @retval string Key for hash table
1787     #
1788     def __hash__(self):
1789         return hash(self.Path)
1790
1791     def _GetFileKey(self):
1792         if self._Key is None:
1793             self._Key = self.Path.upper()   # + self.ToolChainFamily + self.TagName + self.ToolCode + self.Target
1794         return self._Key
1795
1796     def _GetTimeStamp(self):
1797         return os.stat(self.Path)[8]
1798
1799     def Validate(self, Type='', CaseSensitive=True):
1800         if GlobalData.gCaseInsensitive:
1801             CaseSensitive = False
1802         if Type and Type.lower() != self.Type:
1803             return FILE_TYPE_MISMATCH, '%s (expect %s but got %s)' % (self.File, Type, self.Type)
1804
1805         RealFile, RealRoot = RealPath2(self.File, self.Root, self.AlterRoot)
1806         if not RealRoot and not RealFile:
1807             RealFile = self.File
1808             if self.AlterRoot:
1809                 RealFile = os.path.join(self.AlterRoot, self.File)
1810             elif self.Root:
1811                 RealFile = os.path.join(self.Root, self.File)
1812             if len (mws.getPkgPath()) == 0:
1813                 return FILE_NOT_FOUND, os.path.join(self.AlterRoot, RealFile)
1814             else:
1815                 return FILE_NOT_FOUND, "%s is not found in packages path:\n\t%s" % (self.File, '\n\t'.join(mws.getPkgPath()))
1816
1817         ErrorCode = 0
1818         ErrorInfo = ''
1819         if RealRoot != self.Root or RealFile != self.File:
1820             if CaseSensitive and (RealFile != self.File or (RealRoot != self.Root and RealRoot != self.AlterRoot)):
1821                 ErrorCode = FILE_CASE_MISMATCH
1822                 ErrorInfo = self.File + '\n\t' + RealFile + " [in file system]"
1823
1824             self.SubDir, self.Name = os.path.split(RealFile)
1825             self.BaseName, self.Ext = os.path.splitext(self.Name)
1826             if self.SubDir:
1827                 self.Dir = os.path.join(RealRoot, self.SubDir)
1828             else:
1829                 self.Dir = RealRoot
1830             self.File = RealFile
1831             self.Root = RealRoot
1832             self.Path = os.path.join(RealRoot, RealFile)
1833         return ErrorCode, ErrorInfo
1834
1835     Key = property(_GetFileKey)
1836     TimeStamp = property(_GetTimeStamp)
1837
1838 ## Parse PE image to get the required PE informaion.
1839 #
1840 class PeImageClass():
1841     ## Constructor
1842     #
1843     #   @param  File FilePath of PeImage
1844     #
1845     def __init__(self, PeFile):
1846         self.FileName   = PeFile
1847         self.IsValid    = False
1848         self.Size       = 0
1849         self.EntryPoint = 0
1850         self.SectionAlignment  = 0
1851         self.SectionHeaderList = []
1852         self.ErrorInfo = ''
1853         try:
1854             PeObject = open(PeFile, 'rb')
1855         except:
1856             self.ErrorInfo = self.FileName + ' can not be found\n'
1857             return
1858         # Read DOS header
1859         ByteArray = array.array('B')
1860         ByteArray.fromfile(PeObject, 0x3E)
1861         ByteList = ByteArray.tolist()
1862         # DOS signature should be 'MZ'
1863         if self._ByteListToStr (ByteList[0x0:0x2]) != 'MZ':
1864             self.ErrorInfo = self.FileName + ' has no valid DOS signature MZ'
1865             return
1866
1867         # Read 4 byte PE Signature
1868         PeOffset = self._ByteListToInt(ByteList[0x3C:0x3E])
1869         PeObject.seek(PeOffset)
1870         ByteArray = array.array('B')
1871         ByteArray.fromfile(PeObject, 4)
1872         # PE signature should be 'PE\0\0'
1873         if ByteArray.tostring() != 'PE\0\0':
1874             self.ErrorInfo = self.FileName + ' has no valid PE signature PE00'
1875             return
1876
1877         # Read PE file header
1878         ByteArray = array.array('B')
1879         ByteArray.fromfile(PeObject, 0x14)
1880         ByteList = ByteArray.tolist()
1881         SecNumber = self._ByteListToInt(ByteList[0x2:0x4])
1882         if SecNumber == 0:
1883             self.ErrorInfo = self.FileName + ' has no section header'
1884             return
1885
1886         # Read PE optional header
1887         OptionalHeaderSize = self._ByteListToInt(ByteArray[0x10:0x12])
1888         ByteArray = array.array('B')
1889         ByteArray.fromfile(PeObject, OptionalHeaderSize)
1890         ByteList = ByteArray.tolist()
1891         self.EntryPoint       = self._ByteListToInt(ByteList[0x10:0x14])
1892         self.SectionAlignment = self._ByteListToInt(ByteList[0x20:0x24])
1893         self.Size             = self._ByteListToInt(ByteList[0x38:0x3C])
1894
1895         # Read each Section Header
1896         for Index in range(SecNumber):
1897             ByteArray = array.array('B')
1898             ByteArray.fromfile(PeObject, 0x28)
1899             ByteList = ByteArray.tolist()
1900             SecName  = self._ByteListToStr(ByteList[0:8])
1901             SecVirtualSize = self._ByteListToInt(ByteList[8:12])
1902             SecRawAddress  = self._ByteListToInt(ByteList[20:24])
1903             SecVirtualAddress = self._ByteListToInt(ByteList[12:16])
1904             self.SectionHeaderList.append((SecName, SecVirtualAddress, SecRawAddress, SecVirtualSize))
1905         self.IsValid = True
1906         PeObject.close()
1907
1908     def _ByteListToStr(self, ByteList):
1909         String = ''
1910         for index in range(len(ByteList)):
1911             if ByteList[index] == 0:
1912                 break
1913             String += chr(ByteList[index])
1914         return String
1915
1916     def _ByteListToInt(self, ByteList):
1917         Value = 0
1918         for index in range(len(ByteList) - 1, -1, -1):
1919             Value = (Value << 8) | int(ByteList[index])
1920         return Value
1921
1922 class DefaultStore():
1923     def __init__(self, DefaultStores ):
1924
1925         self.DefaultStores = DefaultStores
1926     def DefaultStoreID(self, DefaultStoreName):
1927         for key, value in self.DefaultStores.items():
1928             if value == DefaultStoreName:
1929                 return key
1930         return None
1931     def GetDefaultDefault(self):
1932         if not self.DefaultStores or "0" in self.DefaultStores:
1933             return "0", TAB_DEFAULT_STORES_DEFAULT
1934         else:
1935             minvalue = min(int(value_str) for value_str in self.DefaultStores)
1936             return (str(minvalue), self.DefaultStores[str(minvalue)])
1937     def GetMin(self, DefaultSIdList):
1938         if not DefaultSIdList:
1939             return TAB_DEFAULT_STORES_DEFAULT
1940         storeidset = {storeid for storeid, storename in self.DefaultStores.values() if storename in DefaultSIdList}
1941         if not storeidset:
1942             return ""
1943         minid = min(storeidset )
1944         for sid, name in self.DefaultStores.values():
1945             if sid == minid:
1946                 return name
1947 class SkuClass():
1948
1949     DEFAULT = 0
1950     SINGLE = 1
1951     MULTIPLE =2
1952
1953     def __init__(self,SkuIdentifier='', SkuIds=None):
1954         if SkuIds is None:
1955             SkuIds = {}
1956
1957         for SkuName in SkuIds:
1958             SkuId = SkuIds[SkuName][0]
1959             skuid_num = int(SkuId, 16) if SkuId.upper().startswith("0X") else int(SkuId)
1960             if skuid_num > 0xFFFFFFFFFFFFFFFF:
1961                 EdkLogger.error("build", PARAMETER_INVALID,
1962                             ExtraData = "SKU-ID [%s] value %s exceeds the max value of UINT64"
1963                                       % (SkuName, SkuId))
1964
1965         self.AvailableSkuIds = sdict()
1966         self.SkuIdSet = []
1967         self.SkuIdNumberSet = []
1968         self.SkuData = SkuIds
1969         self.__SkuInherit = {}
1970         self.__SkuIdentifier = SkuIdentifier
1971         if SkuIdentifier == '' or SkuIdentifier is None:
1972             self.SkuIdSet = ['DEFAULT']
1973             self.SkuIdNumberSet = ['0U']
1974         elif SkuIdentifier == 'ALL':
1975             self.SkuIdSet = SkuIds.keys()
1976             self.SkuIdNumberSet = [num[0].strip() + 'U' for num in SkuIds.values()]
1977         else:
1978             r = SkuIdentifier.split('|')
1979             self.SkuIdSet=[(r[k].strip()).upper() for k in range(len(r))]
1980             k = None
1981             try:
1982                 self.SkuIdNumberSet = [SkuIds[k][0].strip() + 'U' for k in self.SkuIdSet]
1983             except Exception:
1984                 EdkLogger.error("build", PARAMETER_INVALID,
1985                             ExtraData = "SKU-ID [%s] is not supported by the platform. [Valid SKU-ID: %s]"
1986                                       % (k, " | ".join(SkuIds.keys())))
1987         for each in self.SkuIdSet:
1988             if each in SkuIds:
1989                 self.AvailableSkuIds[each] = SkuIds[each][0]
1990             else:
1991                 EdkLogger.error("build", PARAMETER_INVALID,
1992                             ExtraData="SKU-ID [%s] is not supported by the platform. [Valid SKU-ID: %s]"
1993                                       % (each, " | ".join(SkuIds.keys())))
1994         if self.SkuUsageType != self.SINGLE:
1995             self.AvailableSkuIds.update({'DEFAULT':0, 'COMMON':0})
1996         if self.SkuIdSet:
1997             GlobalData.gSkuids = (self.SkuIdSet)
1998             if 'COMMON' in GlobalData.gSkuids:
1999                 GlobalData.gSkuids.remove('COMMON')
2000             if self.SkuUsageType == self.SINGLE:
2001                 if len(GlobalData.gSkuids) != 1:
2002                     if 'DEFAULT' in GlobalData.gSkuids:
2003                         GlobalData.gSkuids.remove('DEFAULT')
2004             if GlobalData.gSkuids:
2005                 GlobalData.gSkuids.sort()
2006
2007     def GetNextSkuId(self, skuname):
2008         if not self.__SkuInherit:
2009             self.__SkuInherit = {}
2010             for item in self.SkuData.values():
2011                 self.__SkuInherit[item[1]]=item[2] if item[2] else "DEFAULT"
2012         return self.__SkuInherit.get(skuname, "DEFAULT")
2013
2014     def GetSkuChain(self, sku):
2015         if sku == "DEFAULT":
2016             return ["DEFAULT"]
2017         skulist = [sku]
2018         nextsku = sku
2019         while True:
2020             nextsku = self.GetNextSkuId(nextsku)
2021             skulist.append(nextsku)
2022             if nextsku == "DEFAULT":
2023                 break
2024         skulist.reverse()
2025         return skulist
2026     def SkuOverrideOrder(self):
2027         skuorderset = []
2028         for skuname in self.SkuIdSet:
2029             skuorderset.append(self.GetSkuChain(skuname))
2030
2031         skuorder = []
2032         for index in range(max(len(item) for item in skuorderset)):
2033             for subset in skuorderset:
2034                 if index > len(subset)-1:
2035                     continue
2036                 if subset[index] in skuorder:
2037                     continue
2038                 skuorder.append(subset[index])
2039
2040         return skuorder
2041
2042     def __SkuUsageType(self):
2043
2044         if self.__SkuIdentifier.upper() == "ALL":
2045             return SkuClass.MULTIPLE
2046
2047         if len(self.SkuIdSet) == 1:
2048             if self.SkuIdSet[0] == 'DEFAULT':
2049                 return SkuClass.DEFAULT
2050             else:
2051                 return SkuClass.SINGLE
2052         elif len(self.SkuIdSet) == 2:
2053             if 'DEFAULT' in self.SkuIdSet:
2054                 return SkuClass.SINGLE
2055             else:
2056                 return SkuClass.MULTIPLE
2057         else:
2058             return SkuClass.MULTIPLE
2059     def DumpSkuIdArrary(self):
2060
2061         ArrayStrList = []
2062         if self.SkuUsageType == SkuClass.SINGLE:
2063             ArrayStr = "{0x0}"
2064         else:
2065             for skuname in self.AvailableSkuIds:
2066                 if skuname == "COMMON":
2067                     continue
2068                 while skuname != "DEFAULT":
2069                     ArrayStrList.append(hex(int(self.AvailableSkuIds[skuname])))
2070                     skuname = self.GetNextSkuId(skuname)
2071                 ArrayStrList.append("0x0")
2072             ArrayStr = "{" + ",".join(ArrayStrList) +  "}"
2073         return ArrayStr
2074     def __GetAvailableSkuIds(self):
2075         return self.AvailableSkuIds
2076
2077     def __GetSystemSkuID(self):
2078         if self.__SkuUsageType() == SkuClass.SINGLE:
2079             if len(self.SkuIdSet) == 1:
2080                 return self.SkuIdSet[0]
2081             else:
2082                 return self.SkuIdSet[0] if self.SkuIdSet[0] != 'DEFAULT' else self.SkuIdSet[1]
2083         else:
2084             return 'DEFAULT'
2085     def __GetAvailableSkuIdNumber(self):
2086         return self.SkuIdNumberSet
2087     SystemSkuId = property(__GetSystemSkuID)
2088     AvailableSkuIdSet = property(__GetAvailableSkuIds)
2089     SkuUsageType = property(__SkuUsageType)
2090     AvailableSkuIdNumSet = property(__GetAvailableSkuIdNumber)
2091
2092 #
2093 # Pack a registry format GUID
2094 #
2095 def PackRegistryFormatGuid(Guid):
2096     return PackGUID(Guid.split('-'))
2097
2098 ##  Get the integer value from string like "14U" or integer like 2
2099 #
2100 #   @param      Input   The object that may be either a integer value or a string
2101 #
2102 #   @retval     Value    The integer value that the input represents
2103 #
2104 def GetIntegerValue(Input):
2105     if type(Input) in (int, long):
2106         return Input
2107     String = Input
2108     if String.endswith("U"):
2109         String = String[:-1]
2110     if String.endswith("ULL"):
2111         String = String[:-3]
2112     if String.endswith("LL"):
2113         String = String[:-2]
2114
2115     if String.startswith("0x") or String.startswith("0X"):
2116         return int(String, 16)
2117     elif String == '':
2118         return 0
2119     else:
2120         return int(String)
2121
2122 #
2123 # Pack a GUID (registry format) list into a buffer and return it
2124 #
2125 def PackGUID(Guid):
2126     return pack(PACK_PATTERN_GUID,
2127                 int(Guid[0], 16),
2128                 int(Guid[1], 16),
2129                 int(Guid[2], 16),
2130                 int(Guid[3][-4:-2], 16),
2131                 int(Guid[3][-2:], 16),
2132                 int(Guid[4][-12:-10], 16),
2133                 int(Guid[4][-10:-8], 16),
2134                 int(Guid[4][-8:-6], 16),
2135                 int(Guid[4][-6:-4], 16),
2136                 int(Guid[4][-4:-2], 16),
2137                 int(Guid[4][-2:], 16)
2138                 )
2139
2140 #
2141 # Pack a GUID (byte) list into a buffer and return it
2142 #
2143 def PackByteFormatGUID(Guid):
2144     return pack(PACK_PATTERN_GUID,
2145                 Guid[0],
2146                 Guid[1],
2147                 Guid[2],
2148                 Guid[3],
2149                 Guid[4],
2150                 Guid[5],
2151                 Guid[6],
2152                 Guid[7],
2153                 Guid[8],
2154                 Guid[9],
2155                 Guid[10],
2156                 )
2157
2158 ##
2159 #
2160 # This acts like the main() function for the script, unless it is 'import'ed into another
2161 # script.
2162 #
2163 if __name__ == '__main__':
2164     pass
2165