BaseTools/Source/Python/Common/Misc.py

   1 ## @file
   2 # Common routines used by all tools
   3 #
   4 # Copyright (c) 2007 - 2018, Intel Corporation. All rights reserved.<BR>
   5 # This program and the accompanying materials
   6 # are licensed and made available under the terms and conditions of the BSD License
   7 # which accompanies this distribution.  The full text of the license may be found at
   8 # http://opensource.org/licenses/bsd-license.php
   9 #
  10 # THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS,
  11 # WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED.
  12 #
  13
  14 ##
  15 # Import Modules
  16 #
  17 from __future__ import absolute_import
  18 import Common.LongFilePathOs as os
  19 import sys
  20 import string
  21 import thread
  22 import threading
  23 import time
  24 import re
  25 import cPickle
  26 import array
  27 import shutil
  28 from struct import pack
  29 from UserDict import IterableUserDict
  30 from UserList import UserList
  31
  32 from Common import EdkLogger as EdkLogger
  33 from Common import GlobalData as GlobalData
  34 from .DataType import *
  35 from .BuildToolError import *
  36 from CommonDataClass.DataClass import *
  37 from .Parsing import GetSplitValueList
  38 from Common.LongFilePathSupport import OpenLongFilePath as open
  39 from Common.MultipleWorkspace import MultipleWorkspace as mws
  40 import uuid
  41 from CommonDataClass.Exceptions import BadExpression
  42 import subprocess
  43 ## Regular expression used to find out place holders in string template
  44 gPlaceholderPattern = re.compile("\$\{([^$()\s]+)\}", re.MULTILINE | re.UNICODE)
  45
  46 ## regular expressions for map file processing
  47 startPatternGeneral = re.compile("^Start[' ']+Length[' ']+Name[' ']+Class")
  48 addressPatternGeneral = re.compile("^Address[' ']+Publics by Value[' ']+Rva\+Base")
  49 valuePatternGcc = re.compile('^([\w_\.]+) +([\da-fA-Fx]+) +([\da-fA-Fx]+)$')
  50 pcdPatternGcc = re.compile('^([\da-fA-Fx]+) +([\da-fA-Fx]+)')
  51 secReGeneral = re.compile('^([\da-fA-F]+):([\da-fA-F]+) +([\da-fA-F]+)[Hh]? +([.\w\$]+) +(\w+)', re.UNICODE)
  52
  53 ## Dictionary used to store file time stamp for quick re-access
  54 gFileTimeStampCache = {}    # {file path : file time stamp}
  55
  56 ## Dictionary used to store dependencies of files
  57 gDependencyDatabase = {}    # arch : {file path : [dependent files list]}
  58
  59 #
  60 # If a module is built more than once with different PCDs or library classes
  61 # a temporary INF file with same content is created, the temporary file is removed
  62 # when build exits.
  63 #
  64 _TempInfs = []
  65
  66 def GetVariableOffset(mapfilepath, efifilepath, varnames):
  67     """ Parse map file to get variable offset in current EFI file
  68     @param mapfilepath    Map file absolution path
  69     @param efifilepath:   EFI binary file full path
  70     @param varnames       iteratable container whose elements are variable names to be searched
  71
  72     @return List whos elements are tuple with variable name and raw offset
  73     """
  74     lines = []
  75     try:
  76         f = open(mapfilepath, 'r')
  77         lines = f.readlines()
  78         f.close()
  79     except:
  80         return None
  81
  82     if len(lines) == 0: return None
  83     firstline = lines[0].strip()
  84     if (firstline.startswith("Archive member included ") and
  85         firstline.endswith(" file (symbol)")):
  86         return _parseForGCC(lines, efifilepath, varnames)
  87     if firstline.startswith("# Path:"):
  88         return _parseForXcode(lines, efifilepath, varnames)
  89     return _parseGeneral(lines, efifilepath, varnames)
  90
  91 def _parseForXcode(lines, efifilepath, varnames):
  92     status = 0
  93     ret = []
  94     for line in lines:
  95         line = line.strip()
  96         if status == 0 and line == "# Symbols:":
  97             status = 1
  98             continue
  99         if status == 1 and len(line) != 0:
 100             for varname in varnames:
 101                 if varname in line:
 102                     # cannot pregenerate this RegEx since it uses varname from varnames.
 103                     m = re.match('^([\da-fA-FxX]+)([\s\S]*)([_]*%s)$' % varname, line)
 104                     if m is not None:
 105                         ret.append((varname, m.group(1)))
 106     return ret
 107
 108 def _parseForGCC(lines, efifilepath, varnames):
 109     """ Parse map file generated by GCC linker """
 110     status = 0
 111     sections = []
 112     varoffset = []
 113     for index, line in enumerate(lines):
 114         line = line.strip()
 115         # status machine transection
 116         if status == 0 and line == "Memory Configuration":
 117             status = 1
 118             continue
 119         elif status == 1 and line == 'Linker script and memory map':
 120             status = 2
 121             continue
 122         elif status ==2 and line == 'START GROUP':
 123             status = 3
 124             continue
 125
 126         # status handler
 127         if status == 3:
 128             m = valuePatternGcc.match(line)
 129             if m is not None:
 130                 sections.append(m.groups(0))
 131             for varname in varnames:
 132                 Str = ''
 133                 m = re.match("^.data.(%s)" % varname, line)
 134                 if m is not None:
 135                     m = re.match(".data.(%s)$" % varname, line)
 136                     if m is not None:
 137                         Str = lines[index + 1]
 138                     else:
 139                         Str = line[len(".data.%s" % varname):]
 140                     if Str:
 141                         m = pcdPatternGcc.match(Str.strip())
 142                         if m is not None:
 143                             varoffset.append((varname, int(m.groups(0)[0], 16), int(sections[-1][1], 16), sections[-1][0]))
 144
 145     if not varoffset:
 146         return []
 147     # get section information from efi file
 148     efisecs = PeImageClass(efifilepath).SectionHeaderList
 149     if efisecs is None or len(efisecs) == 0:
 150         return []
 151     #redirection
 152     redirection = 0
 153     for efisec in efisecs:
 154         for section in sections:
 155             if section[0].strip() == efisec[0].strip() and section[0].strip() == '.text':
 156                 redirection = int(section[1], 16) - efisec[1]
 157
 158     ret = []
 159     for var in varoffset:
 160         for efisec in efisecs:
 161             if var[1] >= efisec[1] and var[1] < efisec[1]+efisec[3]:
 162                 ret.append((var[0], hex(efisec[2] + var[1] - efisec[1] - redirection)))
 163     return ret
 164
 165 def _parseGeneral(lines, efifilepath, varnames):
 166     status = 0    #0 - beginning of file; 1 - PE section definition; 2 - symbol table
 167     secs  = []    # key = section name
 168     varoffset = []
 169     symRe = re.compile('^([\da-fA-F]+):([\da-fA-F]+) +([\.:\\\\\w\?@\$]+) +([\da-fA-F]+)', re.UNICODE)
 170
 171     for line in lines:
 172         line = line.strip()
 173         if startPatternGeneral.match(line):
 174             status = 1
 175             continue
 176         if addressPatternGeneral.match(line):
 177             status = 2
 178             continue
 179         if line.startswith("entry point at"):
 180             status = 3
 181             continue
 182         if status == 1 and len(line) != 0:
 183             m =  secReGeneral.match(line)
 184             assert m is not None, "Fail to parse the section in map file , line is %s" % line
 185             sec_no, sec_start, sec_length, sec_name, sec_class = m.groups(0)
 186             secs.append([int(sec_no, 16), int(sec_start, 16), int(sec_length, 16), sec_name, sec_class])
 187         if status == 2 and len(line) != 0:
 188             for varname in varnames:
 189                 m = symRe.match(line)
 190                 assert m is not None, "Fail to parse the symbol in map file, line is %s" % line
 191                 sec_no, sym_offset, sym_name, vir_addr = m.groups(0)
 192                 sec_no     = int(sec_no,     16)
 193                 sym_offset = int(sym_offset, 16)
 194                 vir_addr   = int(vir_addr,   16)
 195                 # cannot pregenerate this RegEx since it uses varname from varnames.
 196                 m2 = re.match('^[_]*(%s)' % varname, sym_name)
 197                 if m2 is not None:
 198                     # fond a binary pcd entry in map file
 199                     for sec in secs:
 200                         if sec[0] == sec_no and (sym_offset >= sec[1] and sym_offset < sec[1] + sec[2]):
 201                             varoffset.append([varname, sec[3], sym_offset, vir_addr, sec_no])
 202
 203     if not varoffset: return []
 204
 205     # get section information from efi file
 206     efisecs = PeImageClass(efifilepath).SectionHeaderList
 207     if efisecs is None or len(efisecs) == 0:
 208         return []
 209
 210     ret = []
 211     for var in varoffset:
 212         index = 0
 213         for efisec in efisecs:
 214             index = index + 1
 215             if var[1].strip() == efisec[0].strip():
 216                 ret.append((var[0], hex(efisec[2] + var[2])))
 217             elif var[4] == index:
 218                 ret.append((var[0], hex(efisec[2] + var[2])))
 219
 220     return ret
 221
 222 ## Routine to process duplicated INF
 223 #
 224 #  This function is called by following two cases:
 225 #  Case 1 in DSC:
 226 #    [components.arch]
 227 #    Pkg/module/module.inf
 228 #    Pkg/module/module.inf {
 229 #      <Defines>
 230 #        FILE_GUID = 0D1B936F-68F3-4589-AFCC-FB8B7AEBC836
 231 #    }
 232 #  Case 2 in FDF:
 233 #    INF Pkg/module/module.inf
 234 #    INF FILE_GUID = 0D1B936F-68F3-4589-AFCC-FB8B7AEBC836 Pkg/module/module.inf
 235 #
 236 #  This function copies Pkg/module/module.inf to
 237 #  Conf/.cache/0D1B936F-68F3-4589-AFCC-FB8B7AEBC836module.inf
 238 #
 239 #  @param Path     Original PathClass object
 240 #  @param BaseName New file base name
 241 #
 242 #  @retval         return the new PathClass object
 243 #
 244 def ProcessDuplicatedInf(Path, BaseName, Workspace):
 245     Filename = os.path.split(Path.File)[1]
 246     if '.' in Filename:
 247         Filename = BaseName + Path.BaseName + Filename[Filename.rfind('.'):]
 248     else:
 249         Filename = BaseName + Path.BaseName
 250
 251     #
 252     # If -N is specified on command line, cache is disabled
 253     # The directory has to be created
 254     #
 255     DbDir = os.path.split(GlobalData.gDatabasePath)[0]
 256     if not os.path.exists(DbDir):
 257         os.makedirs(DbDir)
 258     #
 259     # A temporary INF is copied to database path which must have write permission
 260     # The temporary will be removed at the end of build
 261     # In case of name conflict, the file name is
 262     # FILE_GUIDBaseName (0D1B936F-68F3-4589-AFCC-FB8B7AEBC836module.inf)
 263     #
 264     TempFullPath = os.path.join(DbDir,
 265                                 Filename)
 266     RtPath = PathClass(Path.File, Workspace)
 267     #
 268     # Modify the full path to temporary path, keep other unchanged
 269     #
 270     # To build same module more than once, the module path with FILE_GUID overridden has
 271     # the file name FILE_GUIDmodule.inf, but the relative path (self.MetaFile.File) is the real path
 272     # in DSC which is used as relative path by C files and other files in INF.
 273     # A trick was used: all module paths are PathClass instances, after the initialization
 274     # of PathClass, the PathClass.Path is overridden by the temporary INF path.
 275     #
 276     # The reason for creating a temporary INF is:
 277     # Platform.Modules which is the base to create ModuleAutoGen objects is a dictionary,
 278     # the key is the full path of INF, the value is an object to save overridden library instances, PCDs.
 279     # A different key for the same module is needed to create different output directory,
 280     # retrieve overridden PCDs, library instances.
 281     #
 282     # The BaseName is the FILE_GUID which is also the output directory name.
 283     #
 284     #
 285     RtPath.Path = TempFullPath
 286     RtPath.BaseName = BaseName
 287     #
 288     # If file exists, compare contents
 289     #
 290     if os.path.exists(TempFullPath):
 291         with open(str(Path), 'rb') as f1, open(TempFullPath, 'rb') as f2:
 292             if f1.read() == f2.read():
 293                 return RtPath
 294     _TempInfs.append(TempFullPath)
 295     shutil.copy2(str(Path), TempFullPath)
 296     return RtPath
 297
 298 ## Remove temporary created INFs whose paths were saved in _TempInfs
 299 #
 300 def ClearDuplicatedInf():
 301     while _TempInfs:
 302         File = _TempInfs.pop()
 303         if os.path.exists(File):
 304             os.remove(File)
 305
 306 ## Convert GUID string in xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx style to C structure style
 307 #
 308 #   @param      Guid    The GUID string
 309 #
 310 #   @retval     string  The GUID string in C structure style
 311 #
 312 def GuidStringToGuidStructureString(Guid):
 313     GuidList = Guid.split('-')
 314     Result = '{'
 315     for Index in range(0, 3, 1):
 316         Result = Result + '0x' + GuidList[Index] + ', '
 317     Result = Result + '{0x' + GuidList[3][0:2] + ', 0x' + GuidList[3][2:4]
 318     for Index in range(0, 12, 2):
 319         Result = Result + ', 0x' + GuidList[4][Index:Index + 2]
 320     Result += '}}'
 321     return Result
 322
 323 ## Convert GUID structure in byte array to xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
 324 #
 325 #   @param      GuidValue   The GUID value in byte array
 326 #
 327 #   @retval     string      The GUID value in xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx format
 328 #
 329 def GuidStructureByteArrayToGuidString(GuidValue):
 330     guidValueString = GuidValue.lower().replace("{", "").replace("}", "").replace(" ", "").replace(";", "")
 331     guidValueList = guidValueString.split(",")
 332     if len(guidValueList) != 16:
 333         return ''
 334         #EdkLogger.error(None, None, "Invalid GUID value string %s" % GuidValue)
 335     try:
 336         return "%02x%02x%02x%02x-%02x%02x-%02x%02x-%02x%02x-%02x%02x%02x%02x%02x%02x" % (
 337                 int(guidValueList[3], 16),
 338                 int(guidValueList[2], 16),
 339                 int(guidValueList[1], 16),
 340                 int(guidValueList[0], 16),
 341                 int(guidValueList[5], 16),
 342                 int(guidValueList[4], 16),
 343                 int(guidValueList[7], 16),
 344                 int(guidValueList[6], 16),
 345                 int(guidValueList[8], 16),
 346                 int(guidValueList[9], 16),
 347                 int(guidValueList[10], 16),
 348                 int(guidValueList[11], 16),
 349                 int(guidValueList[12], 16),
 350                 int(guidValueList[13], 16),
 351                 int(guidValueList[14], 16),
 352                 int(guidValueList[15], 16)
 353                 )
 354     except:
 355         return ''
 356
 357 ## Convert GUID string in C structure style to xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
 358 #
 359 #   @param      GuidValue   The GUID value in C structure format
 360 #
 361 #   @retval     string      The GUID value in xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx format
 362 #
 363 def GuidStructureStringToGuidString(GuidValue):
 364     guidValueString = GuidValue.lower().replace("{", "").replace("}", "").replace(" ", "").replace(";", "")
 365     guidValueList = guidValueString.split(",")
 366     if len(guidValueList) != 11:
 367         return ''
 368         #EdkLogger.error(None, None, "Invalid GUID value string %s" % GuidValue)
 369     try:
 370         return "%08x-%04x-%04x-%02x%02x-%02x%02x%02x%02x%02x%02x" % (
 371                 int(guidValueList[0], 16),
 372                 int(guidValueList[1], 16),
 373                 int(guidValueList[2], 16),
 374                 int(guidValueList[3], 16),
 375                 int(guidValueList[4], 16),
 376                 int(guidValueList[5], 16),
 377                 int(guidValueList[6], 16),
 378                 int(guidValueList[7], 16),
 379                 int(guidValueList[8], 16),
 380                 int(guidValueList[9], 16),
 381                 int(guidValueList[10], 16)
 382                 )
 383     except:
 384         return ''
 385
 386 ## Convert GUID string in C structure style to xxxxxxxx_xxxx_xxxx_xxxx_xxxxxxxxxxxx
 387 #
 388 #   @param      GuidValue   The GUID value in C structure format
 389 #
 390 #   @retval     string      The GUID value in xxxxxxxx_xxxx_xxxx_xxxx_xxxxxxxxxxxx format
 391 #
 392 def GuidStructureStringToGuidValueName(GuidValue):
 393     guidValueString = GuidValue.lower().replace("{", "").replace("}", "").replace(" ", "")
 394     guidValueList = guidValueString.split(",")
 395     if len(guidValueList) != 11:
 396         EdkLogger.error(None, FORMAT_INVALID, "Invalid GUID value string [%s]" % GuidValue)
 397     return "%08x_%04x_%04x_%02x%02x_%02x%02x%02x%02x%02x%02x" % (
 398             int(guidValueList[0], 16),
 399             int(guidValueList[1], 16),
 400             int(guidValueList[2], 16),
 401             int(guidValueList[3], 16),
 402             int(guidValueList[4], 16),
 403             int(guidValueList[5], 16),
 404             int(guidValueList[6], 16),
 405             int(guidValueList[7], 16),
 406             int(guidValueList[8], 16),
 407             int(guidValueList[9], 16),
 408             int(guidValueList[10], 16)
 409             )
 410
 411 ## Create directories
 412 #
 413 #   @param      Directory   The directory name
 414 #
 415 def CreateDirectory(Directory):
 416     if Directory is None or Directory.strip() == "":
 417         return True
 418     try:
 419         if not os.access(Directory, os.F_OK):
 420             os.makedirs(Directory)
 421     except:
 422         return False
 423     return True
 424
 425 ## Remove directories, including files and sub-directories in it
 426 #
 427 #   @param      Directory   The directory name
 428 #
 429 def RemoveDirectory(Directory, Recursively=False):
 430     if Directory is None or Directory.strip() == "" or not os.path.exists(Directory):
 431         return
 432     if Recursively:
 433         CurrentDirectory = os.getcwd()
 434         os.chdir(Directory)
 435         for File in os.listdir("."):
 436             if os.path.isdir(File):
 437                 RemoveDirectory(File, Recursively)
 438             else:
 439                 os.remove(File)
 440         os.chdir(CurrentDirectory)
 441     os.rmdir(Directory)
 442
 443 ## Store content in file
 444 #
 445 #  This method is used to save file only when its content is changed. This is
 446 #  quite useful for "make" system to decide what will be re-built and what won't.
 447 #
 448 #   @param      File            The path of file
 449 #   @param      Content         The new content of the file
 450 #   @param      IsBinaryFile    The flag indicating if the file is binary file or not
 451 #
 452 #   @retval     True            If the file content is changed and the file is renewed
 453 #   @retval     False           If the file content is the same
 454 #
 455 def SaveFileOnChange(File, Content, IsBinaryFile=True):
 456     if not IsBinaryFile:
 457         Content = Content.replace("\n", os.linesep)
 458
 459     if os.path.exists(File):
 460         try:
 461             if Content == open(File, "rb").read():
 462                 return False
 463         except:
 464             EdkLogger.error(None, FILE_OPEN_FAILURE, ExtraData=File)
 465
 466     DirName = os.path.dirname(File)
 467     if not CreateDirectory(DirName):
 468         EdkLogger.error(None, FILE_CREATE_FAILURE, "Could not create directory %s" % DirName)
 469     else:
 470         if DirName == '':
 471             DirName = os.getcwd()
 472         if not os.access(DirName, os.W_OK):
 473             EdkLogger.error(None, PERMISSION_FAILURE, "Do not have write permission on directory %s" % DirName)
 474
 475     try:
 476         if GlobalData.gIsWindows:
 477             try:
 478                 from .PyUtility import SaveFileToDisk
 479                 if not SaveFileToDisk(File, Content):
 480                     EdkLogger.error(None, FILE_CREATE_FAILURE, ExtraData=File)
 481             except:
 482                 Fd = open(File, "wb")
 483                 Fd.write(Content)
 484                 Fd.close()
 485         else:
 486             Fd = open(File, "wb")
 487             Fd.write(Content)
 488             Fd.close()
 489     except IOError as X:
 490         EdkLogger.error(None, FILE_CREATE_FAILURE, ExtraData='IOError %s' % X)
 491
 492     return True
 493
 494 ## Make a Python object persistent on file system
 495 #
 496 #   @param      Data    The object to be stored in file
 497 #   @param      File    The path of file to store the object
 498 #
 499 def DataDump(Data, File):
 500     Fd = None
 501     try:
 502         Fd = open(File, 'wb')
 503         cPickle.dump(Data, Fd, cPickle.HIGHEST_PROTOCOL)
 504     except:
 505         EdkLogger.error("", FILE_OPEN_FAILURE, ExtraData=File, RaiseError=False)
 506     finally:
 507         if Fd is not None:
 508             Fd.close()
 509
 510 ## Restore a Python object from a file
 511 #
 512 #   @param      File    The path of file stored the object
 513 #
 514 #   @retval     object  A python object
 515 #   @retval     None    If failure in file operation
 516 #
 517 def DataRestore(File):
 518     Data = None
 519     Fd = None
 520     try:
 521         Fd = open(File, 'rb')
 522         Data = cPickle.load(Fd)
 523     except Exception as e:
 524         EdkLogger.verbose("Failed to load [%s]\n\t%s" % (File, str(e)))
 525         Data = None
 526     finally:
 527         if Fd is not None:
 528             Fd.close()
 529     return Data
 530
 531 ## Retrieve and cache the real path name in file system
 532 #
 533 #   @param      Root    The root directory of path relative to
 534 #
 535 #   @retval     str     The path string if the path exists
 536 #   @retval     None    If path doesn't exist
 537 #
 538 class DirCache:
 539     _CACHE_ = set()
 540     _UPPER_CACHE_ = {}
 541
 542     def __init__(self, Root):
 543         self._Root = Root
 544         for F in os.listdir(Root):
 545             self._CACHE_.add(F)
 546             self._UPPER_CACHE_[F.upper()] = F
 547
 548     # =[] operator
 549     def __getitem__(self, Path):
 550         Path = Path[len(os.path.commonprefix([Path, self._Root])):]
 551         if not Path:
 552             return self._Root
 553         if Path and Path[0] == os.path.sep:
 554             Path = Path[1:]
 555         if Path in self._CACHE_:
 556             return os.path.join(self._Root, Path)
 557         UpperPath = Path.upper()
 558         if UpperPath in self._UPPER_CACHE_:
 559             return os.path.join(self._Root, self._UPPER_CACHE_[UpperPath])
 560
 561         IndexList = []
 562         LastSepIndex = -1
 563         SepIndex = Path.find(os.path.sep)
 564         while SepIndex > -1:
 565             Parent = UpperPath[:SepIndex]
 566             if Parent not in self._UPPER_CACHE_:
 567                 break
 568             LastSepIndex = SepIndex
 569             SepIndex = Path.find(os.path.sep, LastSepIndex + 1)
 570
 571         if LastSepIndex == -1:
 572             return None
 573
 574         Cwd = os.getcwd()
 575         os.chdir(self._Root)
 576         SepIndex = LastSepIndex
 577         while SepIndex > -1:
 578             Parent = Path[:SepIndex]
 579             ParentKey = UpperPath[:SepIndex]
 580             if ParentKey not in self._UPPER_CACHE_:
 581                 os.chdir(Cwd)
 582                 return None
 583
 584             if Parent in self._CACHE_:
 585                 ParentDir = Parent
 586             else:
 587                 ParentDir = self._UPPER_CACHE_[ParentKey]
 588             for F in os.listdir(ParentDir):
 589                 Dir = os.path.join(ParentDir, F)
 590                 self._CACHE_.add(Dir)
 591                 self._UPPER_CACHE_[Dir.upper()] = Dir
 592
 593             SepIndex = Path.find(os.path.sep, SepIndex + 1)
 594
 595         os.chdir(Cwd)
 596         if Path in self._CACHE_:
 597             return os.path.join(self._Root, Path)
 598         elif UpperPath in self._UPPER_CACHE_:
 599             return os.path.join(self._Root, self._UPPER_CACHE_[UpperPath])
 600         return None
 601
 602 def RealPath(File, Dir='', OverrideDir=''):
 603     NewFile = os.path.normpath(os.path.join(Dir, File))
 604     NewFile = GlobalData.gAllFiles[NewFile]
 605     if not NewFile and OverrideDir:
 606         NewFile = os.path.normpath(os.path.join(OverrideDir, File))
 607         NewFile = GlobalData.gAllFiles[NewFile]
 608     return NewFile
 609
 610 def RealPath2(File, Dir='', OverrideDir=''):
 611     NewFile = None
 612     if OverrideDir:
 613         NewFile = GlobalData.gAllFiles[os.path.normpath(os.path.join(OverrideDir, File))]
 614         if NewFile:
 615             if OverrideDir[-1] == os.path.sep:
 616                 return NewFile[len(OverrideDir):], NewFile[0:len(OverrideDir)]
 617             else:
 618                 return NewFile[len(OverrideDir) + 1:], NewFile[0:len(OverrideDir)]
 619     if GlobalData.gAllFiles:
 620         NewFile = GlobalData.gAllFiles[os.path.normpath(os.path.join(Dir, File))]
 621     if not NewFile:
 622         NewFile = os.path.normpath(os.path.join(Dir, File))
 623         if not os.path.exists(NewFile):
 624             return None, None
 625     if NewFile:
 626         if Dir:
 627             if Dir[-1] == os.path.sep:
 628                 return NewFile[len(Dir):], NewFile[0:len(Dir)]
 629             else:
 630                 return NewFile[len(Dir) + 1:], NewFile[0:len(Dir)]
 631         else:
 632             return NewFile, ''
 633
 634     return None, None
 635
 636 ## Get GUID value from given packages
 637 #
 638 #   @param      CName           The CName of the GUID
 639 #   @param      PackageList     List of packages looking-up in
 640 #   @param      Inffile         The driver file
 641 #
 642 #   @retval     GuidValue   if the CName is found in any given package
 643 #   @retval     None        if the CName is not found in all given packages
 644 #
 645 def GuidValue(CName, PackageList, Inffile = None):
 646     for P in PackageList:
 647         GuidKeys = P.Guids.keys()
 648         if Inffile and P._PrivateGuids:
 649             if not Inffile.startswith(P.MetaFile.Dir):
 650                 GuidKeys = [x for x in P.Guids if x not in P._PrivateGuids]
 651         if CName in GuidKeys:
 652             return P.Guids[CName]
 653     return None
 654
 655 ## Get Protocol value from given packages
 656 #
 657 #   @param      CName           The CName of the GUID
 658 #   @param      PackageList     List of packages looking-up in
 659 #   @param      Inffile         The driver file
 660 #
 661 #   @retval     GuidValue   if the CName is found in any given package
 662 #   @retval     None        if the CName is not found in all given packages
 663 #
 664 def ProtocolValue(CName, PackageList, Inffile = None):
 665     for P in PackageList:
 666         ProtocolKeys = P.Protocols.keys()
 667         if Inffile and P._PrivateProtocols:
 668             if not Inffile.startswith(P.MetaFile.Dir):
 669                 ProtocolKeys = [x for x in P.Protocols if x not in P._PrivateProtocols]
 670         if CName in ProtocolKeys:
 671             return P.Protocols[CName]
 672     return None
 673
 674 ## Get PPI value from given packages
 675 #
 676 #   @param      CName           The CName of the GUID
 677 #   @param      PackageList     List of packages looking-up in
 678 #   @param      Inffile         The driver file
 679 #
 680 #   @retval     GuidValue   if the CName is found in any given package
 681 #   @retval     None        if the CName is not found in all given packages
 682 #
 683 def PpiValue(CName, PackageList, Inffile = None):
 684     for P in PackageList:
 685         PpiKeys = P.Ppis.keys()
 686         if Inffile and P._PrivatePpis:
 687             if not Inffile.startswith(P.MetaFile.Dir):
 688                 PpiKeys = [x for x in P.Ppis if x not in P._PrivatePpis]
 689         if CName in PpiKeys:
 690             return P.Ppis[CName]
 691     return None
 692
 693 ## A string template class
 694 #
 695 #  This class implements a template for string replacement. A string template
 696 #  looks like following
 697 #
 698 #       ${BEGIN} other_string ${placeholder_name} other_string ${END}
 699 #
 700 #  The string between ${BEGIN} and ${END} will be repeated as many times as the
 701 #  length of "placeholder_name", which is a list passed through a dict. The
 702 #  "placeholder_name" is the key name of the dict. The ${BEGIN} and ${END} can
 703 #  be not used and, in this case, the "placeholder_name" must not a list and it
 704 #  will just be replaced once.
 705 #
 706 class TemplateString(object):
 707     _REPEAT_START_FLAG = "BEGIN"
 708     _REPEAT_END_FLAG = "END"
 709
 710     class Section(object):
 711         _LIST_TYPES = [type([]), type(set()), type((0,))]
 712
 713         def __init__(self, TemplateSection, PlaceHolderList):
 714             self._Template = TemplateSection
 715             self._PlaceHolderList = []
 716
 717             # Split the section into sub-sections according to the position of placeholders
 718             if PlaceHolderList:
 719                 self._SubSectionList = []
 720                 SubSectionStart = 0
 721                 #
 722                 # The placeholders passed in must be in the format of
 723                 #
 724                 #   PlaceHolderName, PlaceHolderStartPoint, PlaceHolderEndPoint
 725                 #
 726                 for PlaceHolder, Start, End in PlaceHolderList:
 727                     self._SubSectionList.append(TemplateSection[SubSectionStart:Start])
 728                     self._SubSectionList.append(TemplateSection[Start:End])
 729                     self._PlaceHolderList.append(PlaceHolder)
 730                     SubSectionStart = End
 731                 if SubSectionStart < len(TemplateSection):
 732                     self._SubSectionList.append(TemplateSection[SubSectionStart:])
 733             else:
 734                 self._SubSectionList = [TemplateSection]
 735
 736         def __str__(self):
 737             return self._Template + " : " + str(self._PlaceHolderList)
 738
 739         def Instantiate(self, PlaceHolderValues):
 740             RepeatTime = -1
 741             RepeatPlaceHolders = {}
 742             NonRepeatPlaceHolders = {}
 743
 744             for PlaceHolder in self._PlaceHolderList:
 745                 if PlaceHolder not in PlaceHolderValues:
 746                     continue
 747                 Value = PlaceHolderValues[PlaceHolder]
 748                 if type(Value) in self._LIST_TYPES:
 749                     if RepeatTime < 0:
 750                         RepeatTime = len(Value)
 751                     elif RepeatTime != len(Value):
 752                         EdkLogger.error(
 753                                     "TemplateString",
 754                                     PARAMETER_INVALID,
 755                                     "${%s} has different repeat time from others!" % PlaceHolder,
 756                                     ExtraData=str(self._Template)
 757                                     )
 758                     RepeatPlaceHolders["${%s}" % PlaceHolder] = Value
 759                 else:
 760                     NonRepeatPlaceHolders["${%s}" % PlaceHolder] = Value
 761
 762             if NonRepeatPlaceHolders:
 763                 StringList = []
 764                 for S in self._SubSectionList:
 765                     if S not in NonRepeatPlaceHolders:
 766                         StringList.append(S)
 767                     else:
 768                         StringList.append(str(NonRepeatPlaceHolders[S]))
 769             else:
 770                 StringList = self._SubSectionList
 771
 772             if RepeatPlaceHolders:
 773                 TempStringList = []
 774                 for Index in range(RepeatTime):
 775                     for S in StringList:
 776                         if S not in RepeatPlaceHolders:
 777                             TempStringList.append(S)
 778                         else:
 779                             TempStringList.append(str(RepeatPlaceHolders[S][Index]))
 780                 StringList = TempStringList
 781
 782             return "".join(StringList)
 783
 784     ## Constructor
 785     def __init__(self, Template=None):
 786         self.String = ''
 787         self.IsBinary = False
 788         self._Template = Template
 789         self._TemplateSectionList = self._Parse(Template)
 790
 791     ## str() operator
 792     #
 793     #   @retval     string  The string replaced
 794     #
 795     def __str__(self):
 796         return self.String
 797
 798     ## Split the template string into fragments per the ${BEGIN} and ${END} flags
 799     #
 800     #   @retval     list    A list of TemplateString.Section objects
 801     #
 802     def _Parse(self, Template):
 803         SectionStart = 0
 804         SearchFrom = 0
 805         MatchEnd = 0
 806         PlaceHolderList = []
 807         TemplateSectionList = []
 808         while Template:
 809             MatchObj = gPlaceholderPattern.search(Template, SearchFrom)
 810             if not MatchObj:
 811                 if MatchEnd <= len(Template):
 812                     TemplateSection = TemplateString.Section(Template[SectionStart:], PlaceHolderList)
 813                     TemplateSectionList.append(TemplateSection)
 814                 break
 815
 816             MatchString = MatchObj.group(1)
 817             MatchStart = MatchObj.start()
 818             MatchEnd = MatchObj.end()
 819
 820             if MatchString == self._REPEAT_START_FLAG:
 821                 if MatchStart > SectionStart:
 822                     TemplateSection = TemplateString.Section(Template[SectionStart:MatchStart], PlaceHolderList)
 823                     TemplateSectionList.append(TemplateSection)
 824                 SectionStart = MatchEnd
 825                 PlaceHolderList = []
 826             elif MatchString == self._REPEAT_END_FLAG:
 827                 TemplateSection = TemplateString.Section(Template[SectionStart:MatchStart], PlaceHolderList)
 828                 TemplateSectionList.append(TemplateSection)
 829                 SectionStart = MatchEnd
 830                 PlaceHolderList = []
 831             else:
 832                 PlaceHolderList.append((MatchString, MatchStart - SectionStart, MatchEnd - SectionStart))
 833             SearchFrom = MatchEnd
 834         return TemplateSectionList
 835
 836     ## Replace the string template with dictionary of placeholders and append it to previous one
 837     #
 838     #   @param      AppendString    The string template to append
 839     #   @param      Dictionary      The placeholder dictionaries
 840     #
 841     def Append(self, AppendString, Dictionary=None):
 842         if Dictionary:
 843             SectionList = self._Parse(AppendString)
 844             self.String += "".join(S.Instantiate(Dictionary) for S in SectionList)
 845         else:
 846             self.String += AppendString
 847
 848     ## Replace the string template with dictionary of placeholders
 849     #
 850     #   @param      Dictionary      The placeholder dictionaries
 851     #
 852     #   @retval     str             The string replaced with placeholder values
 853     #
 854     def Replace(self, Dictionary=None):
 855         return "".join(S.Instantiate(Dictionary) for S in self._TemplateSectionList)
 856
 857 ## Progress indicator class
 858 #
 859 #  This class makes use of thread to print progress on console.
 860 #
 861 class Progressor:
 862     # for avoiding deadloop
 863     _StopFlag = None
 864     _ProgressThread = None
 865     _CheckInterval = 0.25
 866
 867     ## Constructor
 868     #
 869     #   @param      OpenMessage     The string printed before progress charaters
 870     #   @param      CloseMessage    The string printed after progress charaters
 871     #   @param      ProgressChar    The charater used to indicate the progress
 872     #   @param      Interval        The interval in seconds between two progress charaters
 873     #
 874     def __init__(self, OpenMessage="", CloseMessage="", ProgressChar='.', Interval=1.0):
 875         self.PromptMessage = OpenMessage
 876         self.CodaMessage = CloseMessage
 877         self.ProgressChar = ProgressChar
 878         self.Interval = Interval
 879         if Progressor._StopFlag is None:
 880             Progressor._StopFlag = threading.Event()
 881
 882     ## Start to print progress charater
 883     #
 884     #   @param      OpenMessage     The string printed before progress charaters
 885     #
 886     def Start(self, OpenMessage=None):
 887         if OpenMessage is not None:
 888             self.PromptMessage = OpenMessage
 889         Progressor._StopFlag.clear()
 890         if Progressor._ProgressThread is None:
 891             Progressor._ProgressThread = threading.Thread(target=self._ProgressThreadEntry)
 892             Progressor._ProgressThread.setDaemon(False)
 893             Progressor._ProgressThread.start()
 894
 895     ## Stop printing progress charater
 896     #
 897     #   @param      CloseMessage    The string printed after progress charaters
 898     #
 899     def Stop(self, CloseMessage=None):
 900         OriginalCodaMessage = self.CodaMessage
 901         if CloseMessage is not None:
 902             self.CodaMessage = CloseMessage
 903         self.Abort()
 904         self.CodaMessage = OriginalCodaMessage
 905
 906     ## Thread entry method
 907     def _ProgressThreadEntry(self):
 908         sys.stdout.write(self.PromptMessage + " ")
 909         sys.stdout.flush()
 910         TimeUp = 0.0
 911         while not Progressor._StopFlag.isSet():
 912             if TimeUp <= 0.0:
 913                 sys.stdout.write(self.ProgressChar)
 914                 sys.stdout.flush()
 915                 TimeUp = self.Interval
 916             time.sleep(self._CheckInterval)
 917             TimeUp -= self._CheckInterval
 918         sys.stdout.write(" " + self.CodaMessage + "\n")
 919         sys.stdout.flush()
 920
 921     ## Abort the progress display
 922     @staticmethod
 923     def Abort():
 924         if Progressor._StopFlag is not None:
 925             Progressor._StopFlag.set()
 926         if Progressor._ProgressThread is not None:
 927             Progressor._ProgressThread.join()
 928             Progressor._ProgressThread = None
 929
 930 ## A dict which can access its keys and/or values orderly
 931 #
 932 #  The class implements a new kind of dict which its keys or values can be
 933 #  accessed in the order they are added into the dict. It guarantees the order
 934 #  by making use of an internal list to keep a copy of keys.
 935 #
 936 class sdict(IterableUserDict):
 937     ## Constructor
 938     def __init__(self):
 939         IterableUserDict.__init__(self)
 940         self._key_list = []
 941
 942     ## [] operator
 943     def __setitem__(self, key, value):
 944         if key not in self._key_list:
 945             self._key_list.append(key)
 946         IterableUserDict.__setitem__(self, key, value)
 947
 948     ## del operator
 949     def __delitem__(self, key):
 950         self._key_list.remove(key)
 951         IterableUserDict.__delitem__(self, key)
 952
 953     ## used in "for k in dict" loop to ensure the correct order
 954     def __iter__(self):
 955         return self.iterkeys()
 956
 957     ## len() support
 958     def __len__(self):
 959         return len(self._key_list)
 960
 961     ## "in" test support
 962     def __contains__(self, key):
 963         return key in self._key_list
 964
 965     ## indexof support
 966     def index(self, key):
 967         return self._key_list.index(key)
 968
 969     ## insert support
 970     def insert(self, key, newkey, newvalue, order):
 971         index = self._key_list.index(key)
 972         if order == 'BEFORE':
 973             self._key_list.insert(index, newkey)
 974             IterableUserDict.__setitem__(self, newkey, newvalue)
 975         elif order == 'AFTER':
 976             self._key_list.insert(index + 1, newkey)
 977             IterableUserDict.__setitem__(self, newkey, newvalue)
 978
 979     ## append support
 980     def append(self, sdict):
 981         for key in sdict:
 982             if key not in self._key_list:
 983                 self._key_list.append(key)
 984             IterableUserDict.__setitem__(self, key, sdict[key])
 985
 986     def has_key(self, key):
 987         return key in self._key_list
 988
 989     ## Empty the dict
 990     def clear(self):
 991         self._key_list = []
 992         IterableUserDict.clear(self)
 993
 994     ## Return a copy of keys
 995     def keys(self):
 996         keys = []
 997         for key in self._key_list:
 998             keys.append(key)
 999         return keys
1000
1001     ## Return a copy of values
1002     def values(self):
1003         values = []
1004         for key in self._key_list:
1005             values.append(self[key])
1006         return values
1007
1008     ## Return a copy of (key, value) list
1009     def items(self):
1010         items = []
1011         for key in self._key_list:
1012             items.append((key, self[key]))
1013         return items
1014
1015     ## Iteration support
1016     def iteritems(self):
1017         return iter(self.items())
1018
1019     ## Keys interation support
1020     def iterkeys(self):
1021         return iter(self.keys())
1022
1023     ## Values interation support
1024     def itervalues(self):
1025         return iter(self.values())
1026
1027     ## Return value related to a key, and remove the (key, value) from the dict
1028     def pop(self, key, *dv):
1029         value = None
1030         if key in self._key_list:
1031             value = self[key]
1032             self.__delitem__(key)
1033         elif len(dv) != 0 :
1034             value = kv[0]
1035         return value
1036
1037     ## Return (key, value) pair, and remove the (key, value) from the dict
1038     def popitem(self):
1039         key = self._key_list[-1]
1040         value = self[key]
1041         self.__delitem__(key)
1042         return key, value
1043
1044     def update(self, dict=None, **kwargs):
1045         if dict is not None:
1046             for k, v in dict.items():
1047                 self[k] = v
1048         if len(kwargs):
1049             for k, v in kwargs.items():
1050                 self[k] = v
1051
1052 ## Dictionary with restricted keys
1053 #
1054 class rdict(dict):
1055     ## Constructor
1056     def __init__(self, KeyList):
1057         for Key in KeyList:
1058             dict.__setitem__(self, Key, "")
1059
1060     ## []= operator
1061     def __setitem__(self, key, value):
1062         if key not in self:
1063             EdkLogger.error("RestrictedDict", ATTRIBUTE_SET_FAILURE, "Key [%s] is not allowed" % key,
1064                             ExtraData=", ".join(dict.keys(self)))
1065         dict.__setitem__(self, key, value)
1066
1067     ## =[] operator
1068     def __getitem__(self, key):
1069         if key not in self:
1070             return ""
1071         return dict.__getitem__(self, key)
1072
1073     ## del operator
1074     def __delitem__(self, key):
1075         EdkLogger.error("RestrictedDict", ATTRIBUTE_ACCESS_DENIED, ExtraData="del")
1076
1077     ## Empty the dict
1078     def clear(self):
1079         for Key in self:
1080             self.__setitem__(Key, "")
1081
1082     ## Return value related to a key, and remove the (key, value) from the dict
1083     def pop(self, key, *dv):
1084         EdkLogger.error("RestrictedDict", ATTRIBUTE_ACCESS_DENIED, ExtraData="pop")
1085
1086     ## Return (key, value) pair, and remove the (key, value) from the dict
1087     def popitem(self):
1088         EdkLogger.error("RestrictedDict", ATTRIBUTE_ACCESS_DENIED, ExtraData="popitem")
1089
1090 ## Dictionary using prioritized list as key
1091 #
1092 class tdict:
1093     _ListType = type([])
1094     _TupleType = type(())
1095     _Wildcard = 'COMMON'
1096     _ValidWildcardList = ['COMMON', 'DEFAULT', 'ALL', '*', 'PLATFORM']
1097
1098     def __init__(self, _Single_=False, _Level_=2):
1099         self._Level_ = _Level_
1100         self.data = {}
1101         self._Single_ = _Single_
1102
1103     # =[] operator
1104     def __getitem__(self, key):
1105         KeyType = type(key)
1106         RestKeys = None
1107         if KeyType == self._ListType or KeyType == self._TupleType:
1108             FirstKey = key[0]
1109             if len(key) > 1:
1110                 RestKeys = key[1:]
1111             elif self._Level_ > 1:
1112                 RestKeys = [self._Wildcard for i in range(0, self._Level_ - 1)]
1113         else:
1114             FirstKey = key
1115             if self._Level_ > 1:
1116                 RestKeys = [self._Wildcard for i in range(0, self._Level_ - 1)]
1117
1118         if FirstKey is None or str(FirstKey).upper() in self._ValidWildcardList:
1119             FirstKey = self._Wildcard
1120
1121         if self._Single_:
1122             return self._GetSingleValue(FirstKey, RestKeys)
1123         else:
1124             return self._GetAllValues(FirstKey, RestKeys)
1125
1126     def _GetSingleValue(self, FirstKey, RestKeys):
1127         Value = None
1128         #print "%s-%s" % (FirstKey, self._Level_) ,
1129         if self._Level_ > 1:
1130             if FirstKey == self._Wildcard:
1131                 if FirstKey in self.data:
1132                     Value = self.data[FirstKey][RestKeys]
1133                 if Value is None:
1134                     for Key in self.data:
1135                         Value = self.data[Key][RestKeys]
1136                         if Value is not None: break
1137             else:
1138                 if FirstKey in self.data:
1139                     Value = self.data[FirstKey][RestKeys]
1140                 if Value is None and self._Wildcard in self.data:
1141                     #print "Value=None"
1142                     Value = self.data[self._Wildcard][RestKeys]
1143         else:
1144             if FirstKey == self._Wildcard:
1145                 if FirstKey in self.data:
1146                     Value = self.data[FirstKey]
1147                 if Value is None:
1148                     for Key in self.data:
1149                         Value = self.data[Key]
1150                         if Value is not None: break
1151             else:
1152                 if FirstKey in self.data:
1153                     Value = self.data[FirstKey]
1154                 elif self._Wildcard in self.data:
1155                     Value = self.data[self._Wildcard]
1156         return Value
1157
1158     def _GetAllValues(self, FirstKey, RestKeys):
1159         Value = []
1160         if self._Level_ > 1:
1161             if FirstKey == self._Wildcard:
1162                 for Key in self.data:
1163                     Value += self.data[Key][RestKeys]
1164             else:
1165                 if FirstKey in self.data:
1166                     Value += self.data[FirstKey][RestKeys]
1167                 if self._Wildcard in self.data:
1168                     Value += self.data[self._Wildcard][RestKeys]
1169         else:
1170             if FirstKey == self._Wildcard:
1171                 for Key in self.data:
1172                     Value.append(self.data[Key])
1173             else:
1174                 if FirstKey in self.data:
1175                     Value.append(self.data[FirstKey])
1176                 if self._Wildcard in self.data:
1177                     Value.append(self.data[self._Wildcard])
1178         return Value
1179
1180     ## []= operator
1181     def __setitem__(self, key, value):
1182         KeyType = type(key)
1183         RestKeys = None
1184         if KeyType == self._ListType or KeyType == self._TupleType:
1185             FirstKey = key[0]
1186             if len(key) > 1:
1187                 RestKeys = key[1:]
1188             else:
1189                 RestKeys = [self._Wildcard for i in range(0, self._Level_ - 1)]
1190         else:
1191             FirstKey = key
1192             if self._Level_ > 1:
1193                 RestKeys = [self._Wildcard for i in range(0, self._Level_ - 1)]
1194
1195         if FirstKey in self._ValidWildcardList:
1196             FirstKey = self._Wildcard
1197
1198         if FirstKey not in self.data and self._Level_ > 0:
1199             self.data[FirstKey] = tdict(self._Single_, self._Level_ - 1)
1200
1201         if self._Level_ > 1:
1202             self.data[FirstKey][RestKeys] = value
1203         else:
1204             self.data[FirstKey] = value
1205
1206     def SetGreedyMode(self):
1207         self._Single_ = False
1208         if self._Level_ > 1:
1209             for Key in self.data:
1210                 self.data[Key].SetGreedyMode()
1211
1212     def SetSingleMode(self):
1213         self._Single_ = True
1214         if self._Level_ > 1:
1215             for Key in self.data:
1216                 self.data[Key].SetSingleMode()
1217
1218     def GetKeys(self, KeyIndex=0):
1219         assert KeyIndex >= 0
1220         if KeyIndex == 0:
1221             return set(self.data.keys())
1222         else:
1223             keys = set()
1224             for Key in self.data:
1225                 keys |= self.data[Key].GetKeys(KeyIndex - 1)
1226             return keys
1227
1228 def IsFieldValueAnArray (Value):
1229     Value = Value.strip()
1230     if Value.startswith(TAB_GUID) and Value.endswith(')'):
1231         return True
1232     if Value.startswith('L"') and Value.endswith('"')  and len(list(Value[2:-1])) > 1:
1233         return True
1234     if Value[0] == '"' and Value[-1] == '"' and len(list(Value[1:-1])) > 1:
1235         return True
1236     if Value[0] == '{' and Value[-1] == '}':
1237         return True
1238     if Value.startswith("L'") and Value.endswith("'") and len(list(Value[2:-1])) > 1:
1239         return True
1240     if Value[0] == "'" and Value[-1] == "'" and len(list(Value[1:-1])) > 1:
1241         return True
1242     return False
1243
1244 def AnalyzePcdExpression(Setting):
1245     Setting = Setting.strip()
1246     # There might be escaped quote in a string: \", \\\" , \', \\\'
1247     Data = Setting
1248     # There might be '|' in string and in ( ... | ... ), replace it with '-'
1249     NewStr = ''
1250     InSingleQuoteStr = False
1251     InDoubleQuoteStr = False
1252     Pair = 0
1253     for Index, ch in enumerate(Data):
1254         if ch == '"' and not InSingleQuoteStr:
1255             if Data[Index - 1] != '\\':
1256                 InDoubleQuoteStr = not InDoubleQuoteStr
1257         elif ch == "'" and not InDoubleQuoteStr:
1258             if Data[Index - 1] != '\\':
1259                 InSingleQuoteStr = not InSingleQuoteStr
1260         elif ch == '(' and not (InSingleQuoteStr or InDoubleQuoteStr):
1261             Pair += 1
1262         elif ch == ')' and not (InSingleQuoteStr or InDoubleQuoteStr):
1263             Pair -= 1
1264
1265         if (Pair > 0 or InSingleQuoteStr or InDoubleQuoteStr) and ch == TAB_VALUE_SPLIT:
1266             NewStr += '-'
1267         else:
1268             NewStr += ch
1269     FieldList = []
1270     StartPos = 0
1271     while True:
1272         Pos = NewStr.find(TAB_VALUE_SPLIT, StartPos)
1273         if Pos < 0:
1274             FieldList.append(Setting[StartPos:].strip())
1275             break
1276         FieldList.append(Setting[StartPos:Pos].strip())
1277         StartPos = Pos + 1
1278
1279     return FieldList
1280
1281 def ParseDevPathValue (Value):
1282     if '\\' in Value:
1283         Value.replace('\\', '/').replace(' ', '')
1284
1285     Cmd = 'DevicePath ' + '"' + Value + '"'
1286     try:
1287         p = subprocess.Popen(Cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
1288         out, err = p.communicate()
1289     except Exception as X:
1290         raise BadExpression("DevicePath: %s" % (str(X)) )
1291     finally:
1292         subprocess._cleanup()
1293         p.stdout.close()
1294         p.stderr.close()
1295     if err:
1296         raise BadExpression("DevicePath: %s" % str(err))
1297     Size = len(out.split())
1298     out = ','.join(out.split())
1299     return '{' + out + '}', Size
1300
1301 def ParseFieldValue (Value):
1302     if isinstance(Value, type(0)):
1303         return Value, (Value.bit_length() + 7) / 8
1304     if not isinstance(Value, type('')):
1305         raise BadExpression('Type %s is %s' %(Value, type(Value)))
1306     Value = Value.strip()
1307     if Value.startswith(TAB_UINT8) and Value.endswith(')'):
1308         Value, Size = ParseFieldValue(Value.split('(', 1)[1][:-1])
1309         if Size > 1:
1310             raise BadExpression('Value (%s) Size larger than %d' %(Value, Size))
1311         return Value, 1
1312     if Value.startswith(TAB_UINT16) and Value.endswith(')'):
1313         Value, Size = ParseFieldValue(Value.split('(', 1)[1][:-1])
1314         if Size > 2:
1315             raise BadExpression('Value (%s) Size larger than %d' %(Value, Size))
1316         return Value, 2
1317     if Value.startswith(TAB_UINT32) and Value.endswith(')'):
1318         Value, Size = ParseFieldValue(Value.split('(', 1)[1][:-1])
1319         if Size > 4:
1320             raise BadExpression('Value (%s) Size larger than %d' %(Value, Size))
1321         return Value, 4
1322     if Value.startswith(TAB_UINT64) and Value.endswith(')'):
1323         Value, Size = ParseFieldValue(Value.split('(', 1)[1][:-1])
1324         if Size > 8:
1325             raise BadExpression('Value (%s) Size larger than %d' % (Value, Size))
1326         return Value, 8
1327     if Value.startswith(TAB_GUID) and Value.endswith(')'):
1328         Value = Value.split('(', 1)[1][:-1].strip()
1329         if Value[0] == '{' and Value[-1] == '}':
1330             TmpValue = GuidStructureStringToGuidString(Value)
1331             if len(TmpValue) == 0:
1332                 raise BadExpression("Invalid GUID value string %s" % Value)
1333             Value = TmpValue
1334         if Value[0] == '"' and Value[-1] == '"':
1335             Value = Value[1:-1]
1336         try:
1337             Value = "'" + uuid.UUID(Value).get_bytes_le() + "'"
1338         except ValueError as Message:
1339             raise BadExpression(Message)
1340         Value, Size = ParseFieldValue(Value)
1341         return Value, 16
1342     if Value.startswith('L"') and Value.endswith('"'):
1343         # Unicode String
1344         # translate escape character
1345         Value = Value[1:]
1346         try:
1347             Value = eval(Value)
1348         except:
1349             Value = Value[1:-1]
1350         List = list(Value)
1351         List.reverse()
1352         Value = 0
1353         for Char in List:
1354             Value = (Value << 16) | ord(Char)
1355         return Value, (len(List) + 1) * 2
1356     if Value.startswith('"') and Value.endswith('"'):
1357         # ASCII String
1358         # translate escape character
1359         try:
1360             Value = eval(Value)
1361         except:
1362             Value = Value[1:-1]
1363         List = list(Value)
1364         List.reverse()
1365         Value = 0
1366         for Char in List:
1367             Value = (Value << 8) | ord(Char)
1368         return Value, len(List) + 1
1369     if Value.startswith("L'") and Value.endswith("'"):
1370         # Unicode Character Constant
1371         # translate escape character
1372         Value = Value[1:]
1373         try:
1374             Value = eval(Value)
1375         except:
1376             Value = Value[1:-1]
1377         List = list(Value)
1378         if len(List) == 0:
1379             raise BadExpression('Length %s is %s' % (Value, len(List)))
1380         List.reverse()
1381         Value = 0
1382         for Char in List:
1383             Value = (Value << 16) | ord(Char)
1384         return Value, len(List) * 2
1385     if Value.startswith("'") and Value.endswith("'"):
1386         # Character constant
1387         # translate escape character
1388         try:
1389             Value = eval(Value)
1390         except:
1391             Value = Value[1:-1]
1392         List = list(Value)
1393         if len(List) == 0:
1394             raise BadExpression('Length %s is %s' % (Value, len(List)))
1395         List.reverse()
1396         Value = 0
1397         for Char in List:
1398             Value = (Value << 8) | ord(Char)
1399         return Value, len(List)
1400     if Value.startswith('{') and Value.endswith('}'):
1401         # Byte array
1402         Value = Value[1:-1]
1403         List = [Item.strip() for Item in Value.split(',')]
1404         List.reverse()
1405         Value = 0
1406         RetSize = 0
1407         for Item in List:
1408             ItemValue, Size = ParseFieldValue(Item)
1409             RetSize += Size
1410             for I in range(Size):
1411                 Value = (Value << 8) | ((ItemValue >> 8 * I) & 0xff)
1412         return Value, RetSize
1413     if Value.startswith('DEVICE_PATH(') and Value.endswith(')'):
1414         Value = Value.replace("DEVICE_PATH(", '').rstrip(')')
1415         Value = Value.strip().strip('"')
1416         return ParseDevPathValue(Value)
1417     if Value.lower().startswith('0x'):
1418         Value = int(Value, 16)
1419         if Value == 0:
1420             return 0, 1
1421         return Value, (Value.bit_length() + 7) / 8
1422     if Value[0].isdigit():
1423         Value = int(Value, 10)
1424         if Value == 0:
1425             return 0, 1
1426         return Value, (Value.bit_length() + 7) / 8
1427     if Value.lower() == 'true':
1428         return 1, 1
1429     if Value.lower() == 'false':
1430         return 0, 1
1431     return Value, 1
1432
1433 ## AnalyzeDscPcd
1434 #
1435 #  Analyze DSC PCD value, since there is no data type info in DSC
1436 #  This fuction is used to match functions (AnalyzePcdData) used for retrieving PCD value from database
1437 #  1. Feature flag: TokenSpace.PcdCName|PcdValue
1438 #  2. Fix and Patch:TokenSpace.PcdCName|PcdValue[|MaxSize]
1439 #  3. Dynamic default:
1440 #     TokenSpace.PcdCName|PcdValue[|VOID*[|MaxSize]]
1441 #     TokenSpace.PcdCName|PcdValue
1442 #  4. Dynamic VPD:
1443 #     TokenSpace.PcdCName|VpdOffset[|VpdValue]
1444 #     TokenSpace.PcdCName|VpdOffset[|MaxSize[|VpdValue]]
1445 #  5. Dynamic HII:
1446 #     TokenSpace.PcdCName|HiiString|VaiableGuid|VariableOffset[|HiiValue]
1447 #  PCD value needs to be located in such kind of string, and the PCD value might be an expression in which
1448 #    there might have "|" operator, also in string value.
1449 #
1450 #  @param Setting: String contain information described above with "TokenSpace.PcdCName|" stripped
1451 #  @param PcdType: PCD type: feature, fixed, dynamic default VPD HII
1452 #  @param DataType: The datum type of PCD: VOID*, UNIT, BOOL
1453 #  @retval:
1454 #    ValueList: A List contain fields described above
1455 #    IsValid:   True if conforming EBNF, otherwise False
1456 #    Index:     The index where PcdValue is in ValueList
1457 #
1458 def AnalyzeDscPcd(Setting, PcdType, DataType=''):
1459     FieldList = AnalyzePcdExpression(Setting)
1460
1461     IsValid = True
1462     if PcdType in (MODEL_PCD_FIXED_AT_BUILD, MODEL_PCD_PATCHABLE_IN_MODULE, MODEL_PCD_FEATURE_FLAG):
1463         Value = FieldList[0]
1464         Size = ''
1465         if len(FieldList) > 1:
1466             if FieldList[1].upper().startswith("0X") or FieldList[1].isdigit():
1467                 Size = FieldList[1]
1468             else:
1469                 DataType = FieldList[1]
1470
1471         if len(FieldList) > 2:
1472             Size = FieldList[2]
1473         if DataType == "":
1474             IsValid = (len(FieldList) <= 1)
1475         else:
1476             IsValid = (len(FieldList) <= 3)
1477 #         Value, Size = ParseFieldValue(Value)
1478         if Size:
1479             try:
1480                 int(Size, 16) if Size.upper().startswith("0X") else int(Size)
1481             except:
1482                 IsValid = False
1483                 Size = -1
1484         return [str(Value), '', str(Size)], IsValid, 0
1485     elif PcdType in (MODEL_PCD_DYNAMIC_DEFAULT, MODEL_PCD_DYNAMIC_EX_DEFAULT):
1486         Value = FieldList[0]
1487         Size = Type = ''
1488         if len(FieldList) > 1:
1489             Type = FieldList[1]
1490         else:
1491             Type = DataType
1492         if len(FieldList) > 2:
1493             Size = FieldList[2]
1494         if DataType == "":
1495             IsValid = (len(FieldList) <= 1)
1496         else:
1497             IsValid = (len(FieldList) <= 3)
1498
1499         if Size:
1500             try:
1501                 int(Size, 16) if Size.upper().startswith("0X") else int(Size)
1502             except:
1503                 IsValid = False
1504                 Size = -1
1505         return [Value, Type, str(Size)], IsValid, 0
1506     elif PcdType in (MODEL_PCD_DYNAMIC_VPD, MODEL_PCD_DYNAMIC_EX_VPD):
1507         VpdOffset = FieldList[0]
1508         Value = Size = ''
1509         if not DataType == TAB_VOID:
1510             if len(FieldList) > 1:
1511                 Value = FieldList[1]
1512         else:
1513             if len(FieldList) > 1:
1514                 Size = FieldList[1]
1515             if len(FieldList) > 2:
1516                 Value = FieldList[2]
1517         if DataType == "":
1518             IsValid = (len(FieldList) <= 1)
1519         else:
1520             IsValid = (len(FieldList) <= 3)
1521         if Size:
1522             try:
1523                 int(Size, 16) if Size.upper().startswith("0X") else int(Size)
1524             except:
1525                 IsValid = False
1526                 Size = -1
1527         return [VpdOffset, str(Size), Value], IsValid, 2
1528     elif PcdType in (MODEL_PCD_DYNAMIC_HII, MODEL_PCD_DYNAMIC_EX_HII):
1529         HiiString = FieldList[0]
1530         Guid = Offset = Value = Attribute = ''
1531         if len(FieldList) > 1:
1532             Guid = FieldList[1]
1533         if len(FieldList) > 2:
1534             Offset = FieldList[2]
1535         if len(FieldList) > 3:
1536             Value = FieldList[3]
1537         if len(FieldList) > 4:
1538             Attribute = FieldList[4]
1539         IsValid = (3 <= len(FieldList) <= 5)
1540         return [HiiString, Guid, Offset, Value, Attribute], IsValid, 3
1541     return [], False, 0
1542
1543 ## AnalyzePcdData
1544 #
1545 #  Analyze the pcd Value, Datum type and TokenNumber.
1546 #  Used to avoid split issue while the value string contain "|" character
1547 #
1548 #  @param[in] Setting:  A String contain value/datum type/token number information;
1549 #
1550 #  @retval   ValueList: A List contain value, datum type and toke number.
1551 #
1552 def AnalyzePcdData(Setting):
1553     ValueList = ['', '', '']
1554
1555     ValueRe = re.compile(r'^\s*L?\".*\|.*\"')
1556     PtrValue = ValueRe.findall(Setting)
1557
1558     ValueUpdateFlag = False
1559
1560     if len(PtrValue) >= 1:
1561         Setting = re.sub(ValueRe, '', Setting)
1562         ValueUpdateFlag = True
1563
1564     TokenList = Setting.split(TAB_VALUE_SPLIT)
1565     ValueList[0:len(TokenList)] = TokenList
1566
1567     if ValueUpdateFlag:
1568         ValueList[0] = PtrValue[0]
1569
1570     return ValueList
1571
1572 ## check format of PCD value against its the datum type
1573 #
1574 # For PCD value setting
1575 #
1576 def CheckPcdDatum(Type, Value):
1577     if Type == TAB_VOID:
1578         ValueRe = re.compile(r'\s*L?\".*\"\s*$')
1579         if not (((Value.startswith('L"') or Value.startswith('"')) and Value.endswith('"'))
1580                 or (Value.startswith('{') and Value.endswith('}')) or (Value.startswith("L'") or Value.startswith("'") and Value.endswith("'"))
1581                ):
1582             return False, "Invalid value [%s] of type [%s]; must be in the form of {...} for array"\
1583                           ", \"...\" or \'...\' for string, L\"...\" or L\'...\' for unicode string" % (Value, Type)
1584         elif ValueRe.match(Value):
1585             # Check the chars in UnicodeString or CString is printable
1586             if Value.startswith("L"):
1587                 Value = Value[2:-1]
1588             else:
1589                 Value = Value[1:-1]
1590             Printset = set(string.printable)
1591             Printset.remove(TAB_PRINTCHAR_VT)
1592             Printset.add(TAB_PRINTCHAR_BS)
1593             Printset.add(TAB_PRINTCHAR_NUL)
1594             if not set(Value).issubset(Printset):
1595                 PrintList = sorted(Printset)
1596                 return False, "Invalid PCD string value of type [%s]; must be printable chars %s." % (Type, PrintList)
1597     elif Type == 'BOOLEAN':
1598         if Value not in ['TRUE', 'True', 'true', '0x1', '0x01', '1', 'FALSE', 'False', 'false', '0x0', '0x00', '0']:
1599             return False, "Invalid value [%s] of type [%s]; must be one of TRUE, True, true, 0x1, 0x01, 1"\
1600                           ", FALSE, False, false, 0x0, 0x00, 0" % (Value, Type)
1601     elif Type in [TAB_UINT8, TAB_UINT16, TAB_UINT32, TAB_UINT64]:
1602         try:
1603             Value = long(Value, 0)
1604         except:
1605             return False, "Invalid value [%s] of type [%s];"\
1606                           " must be a hexadecimal, decimal or octal in C language format." % (Value, Type)
1607     else:
1608         return True, "StructurePcd"
1609
1610     return True, ""
1611
1612 ## Split command line option string to list
1613 #
1614 # subprocess.Popen needs the args to be a sequence. Otherwise there's problem
1615 # in non-windows platform to launch command
1616 #
1617 def SplitOption(OptionString):
1618     OptionList = []
1619     LastChar = " "
1620     OptionStart = 0
1621     QuotationMark = ""
1622     for Index in range(0, len(OptionString)):
1623         CurrentChar = OptionString[Index]
1624         if CurrentChar in ['"', "'"]:
1625             if QuotationMark == CurrentChar:
1626                 QuotationMark = ""
1627             elif QuotationMark == "":
1628                 QuotationMark = CurrentChar
1629             continue
1630         elif QuotationMark:
1631             continue
1632
1633         if CurrentChar in ["/", "-"] and LastChar in [" ", "\t", "\r", "\n"]:
1634             if Index > OptionStart:
1635                 OptionList.append(OptionString[OptionStart:Index - 1])
1636             OptionStart = Index
1637         LastChar = CurrentChar
1638     OptionList.append(OptionString[OptionStart:])
1639     return OptionList
1640
1641 def CommonPath(PathList):
1642     P1 = min(PathList).split(os.path.sep)
1643     P2 = max(PathList).split(os.path.sep)
1644     for Index in xrange(min(len(P1), len(P2))):
1645         if P1[Index] != P2[Index]:
1646             return os.path.sep.join(P1[:Index])
1647     return os.path.sep.join(P1)
1648
1649 #
1650 # Convert string to C format array
1651 #
1652 def ConvertStringToByteArray(Value):
1653     Value = Value.strip()
1654     if not Value:
1655         return None
1656     if Value[0] == '{':
1657         if not Value.endswith('}'):
1658             return None
1659         Value = Value.replace(' ', '').replace('{', '').replace('}', '')
1660         ValFields = Value.split(',')
1661         try:
1662             for Index in range(len(ValFields)):
1663                 ValFields[Index] = str(int(ValFields[Index], 0))
1664         except ValueError:
1665             return None
1666         Value = '{' + ','.join(ValFields) + '}'
1667         return Value
1668
1669     Unicode = False
1670     if Value.startswith('L"'):
1671         if not Value.endswith('"'):
1672             return None
1673         Value = Value[1:]
1674         Unicode = True
1675     elif not Value.startswith('"') or not Value.endswith('"'):
1676         return None
1677
1678     Value = eval(Value)         # translate escape character
1679     NewValue = '{'
1680     for Index in range(0, len(Value)):
1681         if Unicode:
1682             NewValue = NewValue + str(ord(Value[Index]) % 0x10000) + ','
1683         else:
1684             NewValue = NewValue + str(ord(Value[Index]) % 0x100) + ','
1685     Value = NewValue + '0}'
1686     return Value
1687
1688 class PathClass(object):
1689     def __init__(self, File='', Root='', AlterRoot='', Type='', IsBinary=False,
1690                  Arch='COMMON', ToolChainFamily='', Target='', TagName='', ToolCode=''):
1691         self.Arch = Arch
1692         self.File = str(File)
1693         if os.path.isabs(self.File):
1694             self.Root = ''
1695             self.AlterRoot = ''
1696         else:
1697             self.Root = str(Root)
1698             self.AlterRoot = str(AlterRoot)
1699
1700         # Remove any '.' and '..' in path
1701         if self.Root:
1702             self.Root = mws.getWs(self.Root, self.File)
1703             self.Path = os.path.normpath(os.path.join(self.Root, self.File))
1704             self.Root = os.path.normpath(CommonPath([self.Root, self.Path]))
1705             # eliminate the side-effect of 'C:'
1706             if self.Root[-1] == ':':
1707                 self.Root += os.path.sep
1708             # file path should not start with path separator
1709             if self.Root[-1] == os.path.sep:
1710                 self.File = self.Path[len(self.Root):]
1711             else:
1712                 self.File = self.Path[len(self.Root) + 1:]
1713         else:
1714             self.Path = os.path.normpath(self.File)
1715
1716         self.SubDir, self.Name = os.path.split(self.File)
1717         self.BaseName, self.Ext = os.path.splitext(self.Name)
1718
1719         if self.Root:
1720             if self.SubDir:
1721                 self.Dir = os.path.join(self.Root, self.SubDir)
1722             else:
1723                 self.Dir = self.Root
1724         else:
1725             self.Dir = self.SubDir
1726
1727         if IsBinary:
1728             self.Type = Type
1729         else:
1730             self.Type = self.Ext.lower()
1731
1732         self.IsBinary = IsBinary
1733         self.Target = Target
1734         self.TagName = TagName
1735         self.ToolCode = ToolCode
1736         self.ToolChainFamily = ToolChainFamily
1737
1738         self._Key = None
1739
1740     ## Convert the object of this class to a string
1741     #
1742     #  Convert member Path of the class to a string
1743     #
1744     #  @retval string Formatted String
1745     #
1746     def __str__(self):
1747         return self.Path
1748
1749     ## Override __eq__ function
1750     #
1751     # Check whether PathClass are the same
1752     #
1753     # @retval False The two PathClass are different
1754     # @retval True  The two PathClass are the same
1755     #
1756     def __eq__(self, Other):
1757         if isinstance(Other, type(self)):
1758             return self.Path == Other.Path
1759         else:
1760             return self.Path == str(Other)
1761
1762     ## Override __cmp__ function
1763     #
1764     # Customize the comparsion operation of two PathClass
1765     #
1766     # @retval 0     The two PathClass are different
1767     # @retval -1    The first PathClass is less than the second PathClass
1768     # @retval 1     The first PathClass is Bigger than the second PathClass
1769     def __cmp__(self, Other):
1770         if isinstance(Other, type(self)):
1771             OtherKey = Other.Path
1772         else:
1773             OtherKey = str(Other)
1774
1775         SelfKey = self.Path
1776         if SelfKey == OtherKey:
1777             return 0
1778         elif SelfKey > OtherKey:
1779             return 1
1780         else:
1781             return -1
1782
1783     ## Override __hash__ function
1784     #
1785     # Use Path as key in hash table
1786     #
1787     # @retval string Key for hash table
1788     #
1789     def __hash__(self):
1790         return hash(self.Path)
1791
1792     def _GetFileKey(self):
1793         if self._Key is None:
1794             self._Key = self.Path.upper()   # + self.ToolChainFamily + self.TagName + self.ToolCode + self.Target
1795         return self._Key
1796
1797     def _GetTimeStamp(self):
1798         return os.stat(self.Path)[8]
1799
1800     def Validate(self, Type='', CaseSensitive=True):
1801         if GlobalData.gCaseInsensitive:
1802             CaseSensitive = False
1803         if Type and Type.lower() != self.Type:
1804             return FILE_TYPE_MISMATCH, '%s (expect %s but got %s)' % (self.File, Type, self.Type)
1805
1806         RealFile, RealRoot = RealPath2(self.File, self.Root, self.AlterRoot)
1807         if not RealRoot and not RealFile:
1808             RealFile = self.File
1809             if self.AlterRoot:
1810                 RealFile = os.path.join(self.AlterRoot, self.File)
1811             elif self.Root:
1812                 RealFile = os.path.join(self.Root, self.File)
1813             if len (mws.getPkgPath()) == 0:
1814                 return FILE_NOT_FOUND, os.path.join(self.AlterRoot, RealFile)
1815             else:
1816                 return FILE_NOT_FOUND, "%s is not found in packages path:\n\t%s" % (self.File, '\n\t'.join(mws.getPkgPath()))
1817
1818         ErrorCode = 0
1819         ErrorInfo = ''
1820         if RealRoot != self.Root or RealFile != self.File:
1821             if CaseSensitive and (RealFile != self.File or (RealRoot != self.Root and RealRoot != self.AlterRoot)):
1822                 ErrorCode = FILE_CASE_MISMATCH
1823                 ErrorInfo = self.File + '\n\t' + RealFile + " [in file system]"
1824
1825             self.SubDir, self.Name = os.path.split(RealFile)
1826             self.BaseName, self.Ext = os.path.splitext(self.Name)
1827             if self.SubDir:
1828                 self.Dir = os.path.join(RealRoot, self.SubDir)
1829             else:
1830                 self.Dir = RealRoot
1831             self.File = RealFile
1832             self.Root = RealRoot
1833             self.Path = os.path.join(RealRoot, RealFile)
1834         return ErrorCode, ErrorInfo
1835
1836     Key = property(_GetFileKey)
1837     TimeStamp = property(_GetTimeStamp)
1838
1839 ## Parse PE image to get the required PE informaion.
1840 #
1841 class PeImageClass():
1842     ## Constructor
1843     #
1844     #   @param  File FilePath of PeImage
1845     #
1846     def __init__(self, PeFile):
1847         self.FileName   = PeFile
1848         self.IsValid    = False
1849         self.Size       = 0
1850         self.EntryPoint = 0
1851         self.SectionAlignment  = 0
1852         self.SectionHeaderList = []
1853         self.ErrorInfo = ''
1854         try:
1855             PeObject = open(PeFile, 'rb')
1856         except:
1857             self.ErrorInfo = self.FileName + ' can not be found\n'
1858             return
1859         # Read DOS header
1860         ByteArray = array.array('B')
1861         ByteArray.fromfile(PeObject, 0x3E)
1862         ByteList = ByteArray.tolist()
1863         # DOS signature should be 'MZ'
1864         if self._ByteListToStr (ByteList[0x0:0x2]) != 'MZ':
1865             self.ErrorInfo = self.FileName + ' has no valid DOS signature MZ'
1866             return
1867
1868         # Read 4 byte PE Signature
1869         PeOffset = self._ByteListToInt(ByteList[0x3C:0x3E])
1870         PeObject.seek(PeOffset)
1871         ByteArray = array.array('B')
1872         ByteArray.fromfile(PeObject, 4)
1873         # PE signature should be 'PE\0\0'
1874         if ByteArray.tostring() != 'PE\0\0':
1875             self.ErrorInfo = self.FileName + ' has no valid PE signature PE00'
1876             return
1877
1878         # Read PE file header
1879         ByteArray = array.array('B')
1880         ByteArray.fromfile(PeObject, 0x14)
1881         ByteList = ByteArray.tolist()
1882         SecNumber = self._ByteListToInt(ByteList[0x2:0x4])
1883         if SecNumber == 0:
1884             self.ErrorInfo = self.FileName + ' has no section header'
1885             return
1886
1887         # Read PE optional header
1888         OptionalHeaderSize = self._ByteListToInt(ByteArray[0x10:0x12])
1889         ByteArray = array.array('B')
1890         ByteArray.fromfile(PeObject, OptionalHeaderSize)
1891         ByteList = ByteArray.tolist()
1892         self.EntryPoint       = self._ByteListToInt(ByteList[0x10:0x14])
1893         self.SectionAlignment = self._ByteListToInt(ByteList[0x20:0x24])
1894         self.Size             = self._ByteListToInt(ByteList[0x38:0x3C])
1895
1896         # Read each Section Header
1897         for Index in range(SecNumber):
1898             ByteArray = array.array('B')
1899             ByteArray.fromfile(PeObject, 0x28)
1900             ByteList = ByteArray.tolist()
1901             SecName  = self._ByteListToStr(ByteList[0:8])
1902             SecVirtualSize = self._ByteListToInt(ByteList[8:12])
1903             SecRawAddress  = self._ByteListToInt(ByteList[20:24])
1904             SecVirtualAddress = self._ByteListToInt(ByteList[12:16])
1905             self.SectionHeaderList.append((SecName, SecVirtualAddress, SecRawAddress, SecVirtualSize))
1906         self.IsValid = True
1907         PeObject.close()
1908
1909     def _ByteListToStr(self, ByteList):
1910         String = ''
1911         for index in range(len(ByteList)):
1912             if ByteList[index] == 0:
1913                 break
1914             String += chr(ByteList[index])
1915         return String
1916
1917     def _ByteListToInt(self, ByteList):
1918         Value = 0
1919         for index in range(len(ByteList) - 1, -1, -1):
1920             Value = (Value << 8) | int(ByteList[index])
1921         return Value
1922
1923 class DefaultStore():
1924     def __init__(self, DefaultStores ):
1925
1926         self.DefaultStores = DefaultStores
1927     def DefaultStoreID(self, DefaultStoreName):
1928         for key, value in self.DefaultStores.items():
1929             if value == DefaultStoreName:
1930                 return key
1931         return None
1932     def GetDefaultDefault(self):
1933         if not self.DefaultStores or "0" in self.DefaultStores:
1934             return "0", TAB_DEFAULT_STORES_DEFAULT
1935         else:
1936             minvalue = min(int(value_str) for value_str in self.DefaultStores)
1937             return (str(minvalue), self.DefaultStores[str(minvalue)])
1938     def GetMin(self, DefaultSIdList):
1939         if not DefaultSIdList:
1940             return TAB_DEFAULT_STORES_DEFAULT
1941         storeidset = {storeid for storeid, storename in self.DefaultStores.values() if storename in DefaultSIdList}
1942         if not storeidset:
1943             return ""
1944         minid = min(storeidset )
1945         for sid, name in self.DefaultStores.values():
1946             if sid == minid:
1947                 return name
1948 class SkuClass():
1949
1950     DEFAULT = 0
1951     SINGLE = 1
1952     MULTIPLE =2
1953
1954     def __init__(self,SkuIdentifier='', SkuIds=None):
1955         if SkuIds is None:
1956             SkuIds = {}
1957
1958         for SkuName in SkuIds:
1959             SkuId = SkuIds[SkuName][0]
1960             skuid_num = int(SkuId, 16) if SkuId.upper().startswith("0X") else int(SkuId)
1961             if skuid_num > 0xFFFFFFFFFFFFFFFF:
1962                 EdkLogger.error("build", PARAMETER_INVALID,
1963                             ExtraData = "SKU-ID [%s] value %s exceeds the max value of UINT64"
1964                                       % (SkuName, SkuId))
1965
1966         self.AvailableSkuIds = sdict()
1967         self.SkuIdSet = []
1968         self.SkuIdNumberSet = []
1969         self.SkuData = SkuIds
1970         self.__SkuInherit = {}
1971         self.__SkuIdentifier = SkuIdentifier
1972         if SkuIdentifier == '' or SkuIdentifier is None:
1973             self.SkuIdSet = ['DEFAULT']
1974             self.SkuIdNumberSet = ['0U']
1975         elif SkuIdentifier == 'ALL':
1976             self.SkuIdSet = SkuIds.keys()
1977             self.SkuIdNumberSet = [num[0].strip() + 'U' for num in SkuIds.values()]
1978         else:
1979             r = SkuIdentifier.split('|')
1980             self.SkuIdSet=[(r[k].strip()).upper() for k in range(len(r))]
1981             k = None
1982             try:
1983                 self.SkuIdNumberSet = [SkuIds[k][0].strip() + 'U' for k in self.SkuIdSet]
1984             except Exception:
1985                 EdkLogger.error("build", PARAMETER_INVALID,
1986                             ExtraData = "SKU-ID [%s] is not supported by the platform. [Valid SKU-ID: %s]"
1987                                       % (k, " | ".join(SkuIds.keys())))
1988         for each in self.SkuIdSet:
1989             if each in SkuIds:
1990                 self.AvailableSkuIds[each] = SkuIds[each][0]
1991             else:
1992                 EdkLogger.error("build", PARAMETER_INVALID,
1993                             ExtraData="SKU-ID [%s] is not supported by the platform. [Valid SKU-ID: %s]"
1994                                       % (each, " | ".join(SkuIds.keys())))
1995         if self.SkuUsageType != self.SINGLE:
1996             self.AvailableSkuIds.update({'DEFAULT':0, 'COMMON':0})
1997         if self.SkuIdSet:
1998             GlobalData.gSkuids = (self.SkuIdSet)
1999             if 'COMMON' in GlobalData.gSkuids:
2000                 GlobalData.gSkuids.remove('COMMON')
2001             if self.SkuUsageType == self.SINGLE:
2002                 if len(GlobalData.gSkuids) != 1:
2003                     if 'DEFAULT' in GlobalData.gSkuids:
2004                         GlobalData.gSkuids.remove('DEFAULT')
2005             if GlobalData.gSkuids:
2006                 GlobalData.gSkuids.sort()
2007
2008     def GetNextSkuId(self, skuname):
2009         if not self.__SkuInherit:
2010             self.__SkuInherit = {}
2011             for item in self.SkuData.values():
2012                 self.__SkuInherit[item[1]]=item[2] if item[2] else "DEFAULT"
2013         return self.__SkuInherit.get(skuname, "DEFAULT")
2014
2015     def GetSkuChain(self, sku):
2016         if sku == "DEFAULT":
2017             return ["DEFAULT"]
2018         skulist = [sku]
2019         nextsku = sku
2020         while True:
2021             nextsku = self.GetNextSkuId(nextsku)
2022             skulist.append(nextsku)
2023             if nextsku == "DEFAULT":
2024                 break
2025         skulist.reverse()
2026         return skulist
2027     def SkuOverrideOrder(self):
2028         skuorderset = []
2029         for skuname in self.SkuIdSet:
2030             skuorderset.append(self.GetSkuChain(skuname))
2031
2032         skuorder = []
2033         for index in range(max(len(item) for item in skuorderset)):
2034             for subset in skuorderset:
2035                 if index > len(subset)-1:
2036                     continue
2037                 if subset[index] in skuorder:
2038                     continue
2039                 skuorder.append(subset[index])
2040
2041         return skuorder
2042
2043     def __SkuUsageType(self):
2044
2045         if self.__SkuIdentifier.upper() == "ALL":
2046             return SkuClass.MULTIPLE
2047
2048         if len(self.SkuIdSet) == 1:
2049             if self.SkuIdSet[0] == 'DEFAULT':
2050                 return SkuClass.DEFAULT
2051             else:
2052                 return SkuClass.SINGLE
2053         elif len(self.SkuIdSet) == 2:
2054             if 'DEFAULT' in self.SkuIdSet:
2055                 return SkuClass.SINGLE
2056             else:
2057                 return SkuClass.MULTIPLE
2058         else:
2059             return SkuClass.MULTIPLE
2060     def DumpSkuIdArrary(self):
2061
2062         ArrayStrList = []
2063         if self.SkuUsageType == SkuClass.SINGLE:
2064             ArrayStr = "{0x0}"
2065         else:
2066             for skuname in self.AvailableSkuIds:
2067                 if skuname == "COMMON":
2068                     continue
2069                 while skuname != "DEFAULT":
2070                     ArrayStrList.append(hex(int(self.AvailableSkuIds[skuname])))
2071                     skuname = self.GetNextSkuId(skuname)
2072                 ArrayStrList.append("0x0")
2073             ArrayStr = "{" + ",".join(ArrayStrList) +  "}"
2074         return ArrayStr
2075     def __GetAvailableSkuIds(self):
2076         return self.AvailableSkuIds
2077
2078     def __GetSystemSkuID(self):
2079         if self.__SkuUsageType() == SkuClass.SINGLE:
2080             if len(self.SkuIdSet) == 1:
2081                 return self.SkuIdSet[0]
2082             else:
2083                 return self.SkuIdSet[0] if self.SkuIdSet[0] != 'DEFAULT' else self.SkuIdSet[1]
2084         else:
2085             return 'DEFAULT'
2086     def __GetAvailableSkuIdNumber(self):
2087         return self.SkuIdNumberSet
2088     SystemSkuId = property(__GetSystemSkuID)
2089     AvailableSkuIdSet = property(__GetAvailableSkuIds)
2090     SkuUsageType = property(__SkuUsageType)
2091     AvailableSkuIdNumSet = property(__GetAvailableSkuIdNumber)
2092
2093 #
2094 # Pack a registry format GUID
2095 #
2096 def PackRegistryFormatGuid(Guid):
2097     return PackGUID(Guid.split('-'))
2098
2099 ##  Get the integer value from string like "14U" or integer like 2
2100 #
2101 #   @param      Input   The object that may be either a integer value or a string
2102 #
2103 #   @retval     Value    The integer value that the input represents
2104 #
2105 def GetIntegerValue(Input):
2106     if type(Input) in (int, long):
2107         return Input
2108     String = Input
2109     if String.endswith("U"):
2110         String = String[:-1]
2111     if String.endswith("ULL"):
2112         String = String[:-3]
2113     if String.endswith("LL"):
2114         String = String[:-2]
2115
2116     if String.startswith("0x") or String.startswith("0X"):
2117         return int(String, 16)
2118     elif String == '':
2119         return 0
2120     else:
2121         return int(String)
2122
2123 #
2124 # Pack a GUID (registry format) list into a buffer and return it
2125 #
2126 def PackGUID(Guid):
2127     return pack(PACK_PATTERN_GUID,
2128                 int(Guid[0], 16),
2129                 int(Guid[1], 16),
2130                 int(Guid[2], 16),
2131                 int(Guid[3][-4:-2], 16),
2132                 int(Guid[3][-2:], 16),
2133                 int(Guid[4][-12:-10], 16),
2134                 int(Guid[4][-10:-8], 16),
2135                 int(Guid[4][-8:-6], 16),
2136                 int(Guid[4][-6:-4], 16),
2137                 int(Guid[4][-4:-2], 16),
2138                 int(Guid[4][-2:], 16)
2139                 )
2140
2141 #
2142 # Pack a GUID (byte) list into a buffer and return it
2143 #
2144 def PackByteFormatGUID(Guid):
2145     return pack(PACK_PATTERN_GUID,
2146                 Guid[0],
2147                 Guid[1],
2148                 Guid[2],
2149                 Guid[3],
2150                 Guid[4],
2151                 Guid[5],
2152                 Guid[6],
2153                 Guid[7],
2154                 Guid[8],
2155                 Guid[9],
2156                 Guid[10],
2157                 )
2158
2159 ##
2160 #
2161 # This acts like the main() function for the script, unless it is 'import'ed into another
2162 # script.
2163 #
2164 if __name__ == '__main__':
2165     pass
2166