BaseTools/Source/Python/Common/Misc.py

   1 ## @file
   2 # Common routines used by all tools
   3 #
   4 # Copyright (c) 2007 - 2018, Intel Corporation. All rights reserved.<BR>
   5 # This program and the accompanying materials
   6 # are licensed and made available under the terms and conditions of the BSD License
   7 # which accompanies this distribution.  The full text of the license may be found at
   8 # http://opensource.org/licenses/bsd-license.php
   9 #
  10 # THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS,
  11 # WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED.
  12 #
  13
  14 ##
  15 # Import Modules
  16 #
  17 import Common.LongFilePathOs as os
  18 import sys
  19 import string
  20 import threading
  21 import time
  22 import re
  23 import pickle
  24 import array
  25 import shutil
  26 from struct import pack
  27 from UserDict import IterableUserDict
  28 from UserList import UserList
  29
  30 from Common import EdkLogger as EdkLogger
  31 from Common import GlobalData as GlobalData
  32 from .DataType import *
  33 from .BuildToolError import *
  34 from CommonDataClass.DataClass import *
  35 from .Parsing import GetSplitValueList
  36 from Common.LongFilePathSupport import OpenLongFilePath as open
  37 from Common.MultipleWorkspace import MultipleWorkspace as mws
  38 import uuid
  39 from CommonDataClass.Exceptions import BadExpression
  40 from Common.caching import cached_property
  41 import subprocess
  42 ## Regular expression used to find out place holders in string template
  43 gPlaceholderPattern = re.compile("\$\{([^$()\s]+)\}", re.MULTILINE | re.UNICODE)
  44
  45 ## regular expressions for map file processing
  46 startPatternGeneral = re.compile("^Start[' ']+Length[' ']+Name[' ']+Class")
  47 addressPatternGeneral = re.compile("^Address[' ']+Publics by Value[' ']+Rva\+Base")
  48 valuePatternGcc = re.compile('^([\w_\.]+) +([\da-fA-Fx]+) +([\da-fA-Fx]+)$')
  49 pcdPatternGcc = re.compile('^([\da-fA-Fx]+) +([\da-fA-Fx]+)')
  50 secReGeneral = re.compile('^([\da-fA-F]+):([\da-fA-F]+) +([\da-fA-F]+)[Hh]? +([.\w\$]+) +(\w+)', re.UNICODE)
  51
  52 ## Dictionary used to store file time stamp for quick re-access
  53 gFileTimeStampCache = {}    # {file path : file time stamp}
  54
  55 ## Dictionary used to store dependencies of files
  56 gDependencyDatabase = {}    # arch : {file path : [dependent files list]}
  57
  58 #
  59 # If a module is built more than once with different PCDs or library classes
  60 # a temporary INF file with same content is created, the temporary file is removed
  61 # when build exits.
  62 #
  63 _TempInfs = []
  64
  65 def GetVariableOffset(mapfilepath, efifilepath, varnames):
  66     """ Parse map file to get variable offset in current EFI file
  67     @param mapfilepath    Map file absolution path
  68     @param efifilepath:   EFI binary file full path
  69     @param varnames       iteratable container whose elements are variable names to be searched
  70
  71     @return List whos elements are tuple with variable name and raw offset
  72     """
  73     lines = []
  74     try:
  75         f = open(mapfilepath, 'r')
  76         lines = f.readlines()
  77         f.close()
  78     except:
  79         return None
  80
  81     if len(lines) == 0: return None
  82     firstline = lines[0].strip()
  83     if (firstline.startswith("Archive member included ") and
  84         firstline.endswith(" file (symbol)")):
  85         return _parseForGCC(lines, efifilepath, varnames)
  86     if firstline.startswith("# Path:"):
  87         return _parseForXcode(lines, efifilepath, varnames)
  88     return _parseGeneral(lines, efifilepath, varnames)
  89
  90 def _parseForXcode(lines, efifilepath, varnames):
  91     status = 0
  92     ret = []
  93     for line in lines:
  94         line = line.strip()
  95         if status == 0 and line == "# Symbols:":
  96             status = 1
  97             continue
  98         if status == 1 and len(line) != 0:
  99             for varname in varnames:
 100                 if varname in line:
 101                     # cannot pregenerate this RegEx since it uses varname from varnames.
 102                     m = re.match('^([\da-fA-FxX]+)([\s\S]*)([_]*%s)$' % varname, line)
 103                     if m is not None:
 104                         ret.append((varname, m.group(1)))
 105     return ret
 106
 107 def _parseForGCC(lines, efifilepath, varnames):
 108     """ Parse map file generated by GCC linker """
 109     status = 0
 110     sections = []
 111     varoffset = []
 112     for index, line in enumerate(lines):
 113         line = line.strip()
 114         # status machine transection
 115         if status == 0 and line == "Memory Configuration":
 116             status = 1
 117             continue
 118         elif status == 1 and line == 'Linker script and memory map':
 119             status = 2
 120             continue
 121         elif status ==2 and line == 'START GROUP':
 122             status = 3
 123             continue
 124
 125         # status handler
 126         if status == 3:
 127             m = valuePatternGcc.match(line)
 128             if m is not None:
 129                 sections.append(m.groups(0))
 130             for varname in varnames:
 131                 Str = ''
 132                 m = re.match("^.data.(%s)" % varname, line)
 133                 if m is not None:
 134                     m = re.match(".data.(%s)$" % varname, line)
 135                     if m is not None:
 136                         Str = lines[index + 1]
 137                     else:
 138                         Str = line[len(".data.%s" % varname):]
 139                     if Str:
 140                         m = pcdPatternGcc.match(Str.strip())
 141                         if m is not None:
 142                             varoffset.append((varname, int(m.groups(0)[0], 16), int(sections[-1][1], 16), sections[-1][0]))
 143
 144     if not varoffset:
 145         return []
 146     # get section information from efi file
 147     efisecs = PeImageClass(efifilepath).SectionHeaderList
 148     if efisecs is None or len(efisecs) == 0:
 149         return []
 150     #redirection
 151     redirection = 0
 152     for efisec in efisecs:
 153         for section in sections:
 154             if section[0].strip() == efisec[0].strip() and section[0].strip() == '.text':
 155                 redirection = int(section[1], 16) - efisec[1]
 156
 157     ret = []
 158     for var in varoffset:
 159         for efisec in efisecs:
 160             if var[1] >= efisec[1] and var[1] < efisec[1]+efisec[3]:
 161                 ret.append((var[0], hex(efisec[2] + var[1] - efisec[1] - redirection)))
 162     return ret
 163
 164 def _parseGeneral(lines, efifilepath, varnames):
 165     status = 0    #0 - beginning of file; 1 - PE section definition; 2 - symbol table
 166     secs  = []    # key = section name
 167     varoffset = []
 168     symRe = re.compile('^([\da-fA-F]+):([\da-fA-F]+) +([\.:\\\\\w\?@\$]+) +([\da-fA-F]+)', re.UNICODE)
 169
 170     for line in lines:
 171         line = line.strip()
 172         if startPatternGeneral.match(line):
 173             status = 1
 174             continue
 175         if addressPatternGeneral.match(line):
 176             status = 2
 177             continue
 178         if line.startswith("entry point at"):
 179             status = 3
 180             continue
 181         if status == 1 and len(line) != 0:
 182             m =  secReGeneral.match(line)
 183             assert m is not None, "Fail to parse the section in map file , line is %s" % line
 184             sec_no, sec_start, sec_length, sec_name, sec_class = m.groups(0)
 185             secs.append([int(sec_no, 16), int(sec_start, 16), int(sec_length, 16), sec_name, sec_class])
 186         if status == 2 and len(line) != 0:
 187             for varname in varnames:
 188                 m = symRe.match(line)
 189                 assert m is not None, "Fail to parse the symbol in map file, line is %s" % line
 190                 sec_no, sym_offset, sym_name, vir_addr = m.groups(0)
 191                 sec_no     = int(sec_no,     16)
 192                 sym_offset = int(sym_offset, 16)
 193                 vir_addr   = int(vir_addr,   16)
 194                 # cannot pregenerate this RegEx since it uses varname from varnames.
 195                 m2 = re.match('^[_]*(%s)' % varname, sym_name)
 196                 if m2 is not None:
 197                     # fond a binary pcd entry in map file
 198                     for sec in secs:
 199                         if sec[0] == sec_no and (sym_offset >= sec[1] and sym_offset < sec[1] + sec[2]):
 200                             varoffset.append([varname, sec[3], sym_offset, vir_addr, sec_no])
 201
 202     if not varoffset: return []
 203
 204     # get section information from efi file
 205     efisecs = PeImageClass(efifilepath).SectionHeaderList
 206     if efisecs is None or len(efisecs) == 0:
 207         return []
 208
 209     ret = []
 210     for var in varoffset:
 211         index = 0
 212         for efisec in efisecs:
 213             index = index + 1
 214             if var[1].strip() == efisec[0].strip():
 215                 ret.append((var[0], hex(efisec[2] + var[2])))
 216             elif var[4] == index:
 217                 ret.append((var[0], hex(efisec[2] + var[2])))
 218
 219     return ret
 220
 221 ## Routine to process duplicated INF
 222 #
 223 #  This function is called by following two cases:
 224 #  Case 1 in DSC:
 225 #    [components.arch]
 226 #    Pkg/module/module.inf
 227 #    Pkg/module/module.inf {
 228 #      <Defines>
 229 #        FILE_GUID = 0D1B936F-68F3-4589-AFCC-FB8B7AEBC836
 230 #    }
 231 #  Case 2 in FDF:
 232 #    INF Pkg/module/module.inf
 233 #    INF FILE_GUID = 0D1B936F-68F3-4589-AFCC-FB8B7AEBC836 Pkg/module/module.inf
 234 #
 235 #  This function copies Pkg/module/module.inf to
 236 #  Conf/.cache/0D1B936F-68F3-4589-AFCC-FB8B7AEBC836module.inf
 237 #
 238 #  @param Path     Original PathClass object
 239 #  @param BaseName New file base name
 240 #
 241 #  @retval         return the new PathClass object
 242 #
 243 def ProcessDuplicatedInf(Path, BaseName, Workspace):
 244     Filename = os.path.split(Path.File)[1]
 245     if '.' in Filename:
 246         Filename = BaseName + Path.BaseName + Filename[Filename.rfind('.'):]
 247     else:
 248         Filename = BaseName + Path.BaseName
 249
 250     #
 251     # If -N is specified on command line, cache is disabled
 252     # The directory has to be created
 253     #
 254     DbDir = os.path.split(GlobalData.gDatabasePath)[0]
 255     if not os.path.exists(DbDir):
 256         os.makedirs(DbDir)
 257     #
 258     # A temporary INF is copied to database path which must have write permission
 259     # The temporary will be removed at the end of build
 260     # In case of name conflict, the file name is
 261     # FILE_GUIDBaseName (0D1B936F-68F3-4589-AFCC-FB8B7AEBC836module.inf)
 262     #
 263     TempFullPath = os.path.join(DbDir,
 264                                 Filename)
 265     RtPath = PathClass(Path.File, Workspace)
 266     #
 267     # Modify the full path to temporary path, keep other unchanged
 268     #
 269     # To build same module more than once, the module path with FILE_GUID overridden has
 270     # the file name FILE_GUIDmodule.inf, but the relative path (self.MetaFile.File) is the real path
 271     # in DSC which is used as relative path by C files and other files in INF.
 272     # A trick was used: all module paths are PathClass instances, after the initialization
 273     # of PathClass, the PathClass.Path is overridden by the temporary INF path.
 274     #
 275     # The reason for creating a temporary INF is:
 276     # Platform.Modules which is the base to create ModuleAutoGen objects is a dictionary,
 277     # the key is the full path of INF, the value is an object to save overridden library instances, PCDs.
 278     # A different key for the same module is needed to create different output directory,
 279     # retrieve overridden PCDs, library instances.
 280     #
 281     # The BaseName is the FILE_GUID which is also the output directory name.
 282     #
 283     #
 284     RtPath.Path = TempFullPath
 285     RtPath.BaseName = BaseName
 286     #
 287     # If file exists, compare contents
 288     #
 289     if os.path.exists(TempFullPath):
 290         with open(str(Path), 'rb') as f1, open(TempFullPath, 'rb') as f2:
 291             if f1.read() == f2.read():
 292                 return RtPath
 293     _TempInfs.append(TempFullPath)
 294     shutil.copy2(str(Path), TempFullPath)
 295     return RtPath
 296
 297 ## Remove temporary created INFs whose paths were saved in _TempInfs
 298 #
 299 def ClearDuplicatedInf():
 300     while _TempInfs:
 301         File = _TempInfs.pop()
 302         if os.path.exists(File):
 303             os.remove(File)
 304
 305 ## Convert GUID string in xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx style to C structure style
 306 #
 307 #   @param      Guid    The GUID string
 308 #
 309 #   @retval     string  The GUID string in C structure style
 310 #
 311 def GuidStringToGuidStructureString(Guid):
 312     GuidList = Guid.split('-')
 313     Result = '{'
 314     for Index in range(0, 3, 1):
 315         Result = Result + '0x' + GuidList[Index] + ', '
 316     Result = Result + '{0x' + GuidList[3][0:2] + ', 0x' + GuidList[3][2:4]
 317     for Index in range(0, 12, 2):
 318         Result = Result + ', 0x' + GuidList[4][Index:Index + 2]
 319     Result += '}}'
 320     return Result
 321
 322 ## Convert GUID structure in byte array to xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
 323 #
 324 #   @param      GuidValue   The GUID value in byte array
 325 #
 326 #   @retval     string      The GUID value in xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx format
 327 #
 328 def GuidStructureByteArrayToGuidString(GuidValue):
 329     guidValueString = GuidValue.lower().replace("{", "").replace("}", "").replace(" ", "").replace(";", "")
 330     guidValueList = guidValueString.split(",")
 331     if len(guidValueList) != 16:
 332         return ''
 333         #EdkLogger.error(None, None, "Invalid GUID value string %s" % GuidValue)
 334     try:
 335         return "%02x%02x%02x%02x-%02x%02x-%02x%02x-%02x%02x-%02x%02x%02x%02x%02x%02x" % (
 336                 int(guidValueList[3], 16),
 337                 int(guidValueList[2], 16),
 338                 int(guidValueList[1], 16),
 339                 int(guidValueList[0], 16),
 340                 int(guidValueList[5], 16),
 341                 int(guidValueList[4], 16),
 342                 int(guidValueList[7], 16),
 343                 int(guidValueList[6], 16),
 344                 int(guidValueList[8], 16),
 345                 int(guidValueList[9], 16),
 346                 int(guidValueList[10], 16),
 347                 int(guidValueList[11], 16),
 348                 int(guidValueList[12], 16),
 349                 int(guidValueList[13], 16),
 350                 int(guidValueList[14], 16),
 351                 int(guidValueList[15], 16)
 352                 )
 353     except:
 354         return ''
 355
 356 ## Convert GUID string in C structure style to xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
 357 #
 358 #   @param      GuidValue   The GUID value in C structure format
 359 #
 360 #   @retval     string      The GUID value in xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx format
 361 #
 362 def GuidStructureStringToGuidString(GuidValue):
 363     if not GlobalData.gGuidCFormatPattern.match(GuidValue):
 364         return ''
 365     guidValueString = GuidValue.lower().replace("{", "").replace("}", "").replace(" ", "").replace(";", "")
 366     guidValueList = guidValueString.split(",")
 367     if len(guidValueList) != 11:
 368         return ''
 369         #EdkLogger.error(None, None, "Invalid GUID value string %s" % GuidValue)
 370     try:
 371         return "%08x-%04x-%04x-%02x%02x-%02x%02x%02x%02x%02x%02x" % (
 372                 int(guidValueList[0], 16),
 373                 int(guidValueList[1], 16),
 374                 int(guidValueList[2], 16),
 375                 int(guidValueList[3], 16),
 376                 int(guidValueList[4], 16),
 377                 int(guidValueList[5], 16),
 378                 int(guidValueList[6], 16),
 379                 int(guidValueList[7], 16),
 380                 int(guidValueList[8], 16),
 381                 int(guidValueList[9], 16),
 382                 int(guidValueList[10], 16)
 383                 )
 384     except:
 385         return ''
 386
 387 ## Convert GUID string in C structure style to xxxxxxxx_xxxx_xxxx_xxxx_xxxxxxxxxxxx
 388 #
 389 #   @param      GuidValue   The GUID value in C structure format
 390 #
 391 #   @retval     string      The GUID value in xxxxxxxx_xxxx_xxxx_xxxx_xxxxxxxxxxxx format
 392 #
 393 def GuidStructureStringToGuidValueName(GuidValue):
 394     guidValueString = GuidValue.lower().replace("{", "").replace("}", "").replace(" ", "")
 395     guidValueList = guidValueString.split(",")
 396     if len(guidValueList) != 11:
 397         EdkLogger.error(None, FORMAT_INVALID, "Invalid GUID value string [%s]" % GuidValue)
 398     return "%08x_%04x_%04x_%02x%02x_%02x%02x%02x%02x%02x%02x" % (
 399             int(guidValueList[0], 16),
 400             int(guidValueList[1], 16),
 401             int(guidValueList[2], 16),
 402             int(guidValueList[3], 16),
 403             int(guidValueList[4], 16),
 404             int(guidValueList[5], 16),
 405             int(guidValueList[6], 16),
 406             int(guidValueList[7], 16),
 407             int(guidValueList[8], 16),
 408             int(guidValueList[9], 16),
 409             int(guidValueList[10], 16)
 410             )
 411
 412 ## Create directories
 413 #
 414 #   @param      Directory   The directory name
 415 #
 416 def CreateDirectory(Directory):
 417     if Directory is None or Directory.strip() == "":
 418         return True
 419     try:
 420         if not os.access(Directory, os.F_OK):
 421             os.makedirs(Directory)
 422     except:
 423         return False
 424     return True
 425
 426 ## Remove directories, including files and sub-directories in it
 427 #
 428 #   @param      Directory   The directory name
 429 #
 430 def RemoveDirectory(Directory, Recursively=False):
 431     if Directory is None or Directory.strip() == "" or not os.path.exists(Directory):
 432         return
 433     if Recursively:
 434         CurrentDirectory = os.getcwd()
 435         os.chdir(Directory)
 436         for File in os.listdir("."):
 437             if os.path.isdir(File):
 438                 RemoveDirectory(File, Recursively)
 439             else:
 440                 os.remove(File)
 441         os.chdir(CurrentDirectory)
 442     os.rmdir(Directory)
 443
 444 ## Store content in file
 445 #
 446 #  This method is used to save file only when its content is changed. This is
 447 #  quite useful for "make" system to decide what will be re-built and what won't.
 448 #
 449 #   @param      File            The path of file
 450 #   @param      Content         The new content of the file
 451 #   @param      IsBinaryFile    The flag indicating if the file is binary file or not
 452 #
 453 #   @retval     True            If the file content is changed and the file is renewed
 454 #   @retval     False           If the file content is the same
 455 #
 456 def SaveFileOnChange(File, Content, IsBinaryFile=True):
 457     if not IsBinaryFile:
 458         Content = Content.replace("\n", os.linesep)
 459
 460     if os.path.exists(File):
 461         try:
 462             if isinstance(Content, bytes):
 463                 with open(File, "rb") as f:
 464                     if Content == f.read():
 465                         return False
 466             else:
 467                 with open(File, "r") as f:
 468                     if Content == f.read():
 469                         return False
 470         except:
 471             EdkLogger.error(None, FILE_OPEN_FAILURE, ExtraData=File)
 472
 473     DirName = os.path.dirname(File)
 474     if not CreateDirectory(DirName):
 475         EdkLogger.error(None, FILE_CREATE_FAILURE, "Could not create directory %s" % DirName)
 476     else:
 477         if DirName == '':
 478             DirName = os.getcwd()
 479         if not os.access(DirName, os.W_OK):
 480             EdkLogger.error(None, PERMISSION_FAILURE, "Do not have write permission on directory %s" % DirName)
 481
 482     try:
 483         if GlobalData.gIsWindows:
 484             try:
 485                 from .PyUtility import SaveFileToDisk
 486                 if not SaveFileToDisk(File, Content):
 487                     EdkLogger.error(None, FILE_CREATE_FAILURE, ExtraData=File)
 488             except:
 489                 if isinstance(Content, bytes):
 490                     with open(File, "wb") as Fd:
 491                         Fd.write(Content)
 492                 else:
 493                     with open(File, "w") as Fd:
 494                         Fd.write(Content)
 495         else:
 496             if isinstance(Content, bytes):
 497                 with open(File, "wb") as Fd:
 498                     Fd.write(Content)
 499             else:
 500                 with open(File, "w") as Fd:
 501                     Fd.write(Content)
 502     except IOError as X:
 503         EdkLogger.error(None, FILE_CREATE_FAILURE, ExtraData='IOError %s' % X)
 504
 505     return True
 506
 507 ## Make a Python object persistent on file system
 508 #
 509 #   @param      Data    The object to be stored in file
 510 #   @param      File    The path of file to store the object
 511 #
 512 def DataDump(Data, File):
 513     Fd = None
 514     try:
 515         Fd = open(File, 'wb')
 516         pickle.dump(Data, Fd, pickle.HIGHEST_PROTOCOL)
 517     except:
 518         EdkLogger.error("", FILE_OPEN_FAILURE, ExtraData=File, RaiseError=False)
 519     finally:
 520         if Fd is not None:
 521             Fd.close()
 522
 523 ## Restore a Python object from a file
 524 #
 525 #   @param      File    The path of file stored the object
 526 #
 527 #   @retval     object  A python object
 528 #   @retval     None    If failure in file operation
 529 #
 530 def DataRestore(File):
 531     Data = None
 532     Fd = None
 533     try:
 534         Fd = open(File, 'rb')
 535         Data = pickle.load(Fd)
 536     except Exception as e:
 537         EdkLogger.verbose("Failed to load [%s]\n\t%s" % (File, str(e)))
 538         Data = None
 539     finally:
 540         if Fd is not None:
 541             Fd.close()
 542     return Data
 543
 544 ## Retrieve and cache the real path name in file system
 545 #
 546 #   @param      Root    The root directory of path relative to
 547 #
 548 #   @retval     str     The path string if the path exists
 549 #   @retval     None    If path doesn't exist
 550 #
 551 class DirCache:
 552     _CACHE_ = set()
 553     _UPPER_CACHE_ = {}
 554
 555     def __init__(self, Root):
 556         self._Root = Root
 557         for F in os.listdir(Root):
 558             self._CACHE_.add(F)
 559             self._UPPER_CACHE_[F.upper()] = F
 560
 561     # =[] operator
 562     def __getitem__(self, Path):
 563         Path = Path[len(os.path.commonprefix([Path, self._Root])):]
 564         if not Path:
 565             return self._Root
 566         if Path and Path[0] == os.path.sep:
 567             Path = Path[1:]
 568         if Path in self._CACHE_:
 569             return os.path.join(self._Root, Path)
 570         UpperPath = Path.upper()
 571         if UpperPath in self._UPPER_CACHE_:
 572             return os.path.join(self._Root, self._UPPER_CACHE_[UpperPath])
 573
 574         IndexList = []
 575         LastSepIndex = -1
 576         SepIndex = Path.find(os.path.sep)
 577         while SepIndex > -1:
 578             Parent = UpperPath[:SepIndex]
 579             if Parent not in self._UPPER_CACHE_:
 580                 break
 581             LastSepIndex = SepIndex
 582             SepIndex = Path.find(os.path.sep, LastSepIndex + 1)
 583
 584         if LastSepIndex == -1:
 585             return None
 586
 587         Cwd = os.getcwd()
 588         os.chdir(self._Root)
 589         SepIndex = LastSepIndex
 590         while SepIndex > -1:
 591             Parent = Path[:SepIndex]
 592             ParentKey = UpperPath[:SepIndex]
 593             if ParentKey not in self._UPPER_CACHE_:
 594                 os.chdir(Cwd)
 595                 return None
 596
 597             if Parent in self._CACHE_:
 598                 ParentDir = Parent
 599             else:
 600                 ParentDir = self._UPPER_CACHE_[ParentKey]
 601             for F in os.listdir(ParentDir):
 602                 Dir = os.path.join(ParentDir, F)
 603                 self._CACHE_.add(Dir)
 604                 self._UPPER_CACHE_[Dir.upper()] = Dir
 605
 606             SepIndex = Path.find(os.path.sep, SepIndex + 1)
 607
 608         os.chdir(Cwd)
 609         if Path in self._CACHE_:
 610             return os.path.join(self._Root, Path)
 611         elif UpperPath in self._UPPER_CACHE_:
 612             return os.path.join(self._Root, self._UPPER_CACHE_[UpperPath])
 613         return None
 614
 615 def RealPath(File, Dir='', OverrideDir=''):
 616     NewFile = os.path.normpath(os.path.join(Dir, File))
 617     NewFile = GlobalData.gAllFiles[NewFile]
 618     if not NewFile and OverrideDir:
 619         NewFile = os.path.normpath(os.path.join(OverrideDir, File))
 620         NewFile = GlobalData.gAllFiles[NewFile]
 621     return NewFile
 622
 623 def RealPath2(File, Dir='', OverrideDir=''):
 624     NewFile = None
 625     if OverrideDir:
 626         NewFile = GlobalData.gAllFiles[os.path.normpath(os.path.join(OverrideDir, File))]
 627         if NewFile:
 628             if OverrideDir[-1] == os.path.sep:
 629                 return NewFile[len(OverrideDir):], NewFile[0:len(OverrideDir)]
 630             else:
 631                 return NewFile[len(OverrideDir) + 1:], NewFile[0:len(OverrideDir)]
 632     if GlobalData.gAllFiles:
 633         NewFile = GlobalData.gAllFiles[os.path.normpath(os.path.join(Dir, File))]
 634     if not NewFile:
 635         NewFile = os.path.normpath(os.path.join(Dir, File))
 636         if not os.path.exists(NewFile):
 637             return None, None
 638     if NewFile:
 639         if Dir:
 640             if Dir[-1] == os.path.sep:
 641                 return NewFile[len(Dir):], NewFile[0:len(Dir)]
 642             else:
 643                 return NewFile[len(Dir) + 1:], NewFile[0:len(Dir)]
 644         else:
 645             return NewFile, ''
 646
 647     return None, None
 648
 649 ## Get GUID value from given packages
 650 #
 651 #   @param      CName           The CName of the GUID
 652 #   @param      PackageList     List of packages looking-up in
 653 #   @param      Inffile         The driver file
 654 #
 655 #   @retval     GuidValue   if the CName is found in any given package
 656 #   @retval     None        if the CName is not found in all given packages
 657 #
 658 def GuidValue(CName, PackageList, Inffile = None):
 659     for P in PackageList:
 660         GuidKeys = P.Guids.keys()
 661         if Inffile and P._PrivateGuids:
 662             if not Inffile.startswith(P.MetaFile.Dir):
 663                 GuidKeys = [x for x in P.Guids if x not in P._PrivateGuids]
 664         if CName in GuidKeys:
 665             return P.Guids[CName]
 666     return None
 667
 668 ## Get Protocol value from given packages
 669 #
 670 #   @param      CName           The CName of the GUID
 671 #   @param      PackageList     List of packages looking-up in
 672 #   @param      Inffile         The driver file
 673 #
 674 #   @retval     GuidValue   if the CName is found in any given package
 675 #   @retval     None        if the CName is not found in all given packages
 676 #
 677 def ProtocolValue(CName, PackageList, Inffile = None):
 678     for P in PackageList:
 679         ProtocolKeys = P.Protocols.keys()
 680         if Inffile and P._PrivateProtocols:
 681             if not Inffile.startswith(P.MetaFile.Dir):
 682                 ProtocolKeys = [x for x in P.Protocols if x not in P._PrivateProtocols]
 683         if CName in ProtocolKeys:
 684             return P.Protocols[CName]
 685     return None
 686
 687 ## Get PPI value from given packages
 688 #
 689 #   @param      CName           The CName of the GUID
 690 #   @param      PackageList     List of packages looking-up in
 691 #   @param      Inffile         The driver file
 692 #
 693 #   @retval     GuidValue   if the CName is found in any given package
 694 #   @retval     None        if the CName is not found in all given packages
 695 #
 696 def PpiValue(CName, PackageList, Inffile = None):
 697     for P in PackageList:
 698         PpiKeys = P.Ppis.keys()
 699         if Inffile and P._PrivatePpis:
 700             if not Inffile.startswith(P.MetaFile.Dir):
 701                 PpiKeys = [x for x in P.Ppis if x not in P._PrivatePpis]
 702         if CName in PpiKeys:
 703             return P.Ppis[CName]
 704     return None
 705
 706 ## A string template class
 707 #
 708 #  This class implements a template for string replacement. A string template
 709 #  looks like following
 710 #
 711 #       ${BEGIN} other_string ${placeholder_name} other_string ${END}
 712 #
 713 #  The string between ${BEGIN} and ${END} will be repeated as many times as the
 714 #  length of "placeholder_name", which is a list passed through a dict. The
 715 #  "placeholder_name" is the key name of the dict. The ${BEGIN} and ${END} can
 716 #  be not used and, in this case, the "placeholder_name" must not a list and it
 717 #  will just be replaced once.
 718 #
 719 class TemplateString(object):
 720     _REPEAT_START_FLAG = "BEGIN"
 721     _REPEAT_END_FLAG = "END"
 722
 723     class Section(object):
 724         _LIST_TYPES = [type([]), type(set()), type((0,))]
 725
 726         def __init__(self, TemplateSection, PlaceHolderList):
 727             self._Template = TemplateSection
 728             self._PlaceHolderList = []
 729
 730             # Split the section into sub-sections according to the position of placeholders
 731             if PlaceHolderList:
 732                 self._SubSectionList = []
 733                 SubSectionStart = 0
 734                 #
 735                 # The placeholders passed in must be in the format of
 736                 #
 737                 #   PlaceHolderName, PlaceHolderStartPoint, PlaceHolderEndPoint
 738                 #
 739                 for PlaceHolder, Start, End in PlaceHolderList:
 740                     self._SubSectionList.append(TemplateSection[SubSectionStart:Start])
 741                     self._SubSectionList.append(TemplateSection[Start:End])
 742                     self._PlaceHolderList.append(PlaceHolder)
 743                     SubSectionStart = End
 744                 if SubSectionStart < len(TemplateSection):
 745                     self._SubSectionList.append(TemplateSection[SubSectionStart:])
 746             else:
 747                 self._SubSectionList = [TemplateSection]
 748
 749         def __str__(self):
 750             return self._Template + " : " + str(self._PlaceHolderList)
 751
 752         def Instantiate(self, PlaceHolderValues):
 753             RepeatTime = -1
 754             RepeatPlaceHolders = {}
 755             NonRepeatPlaceHolders = {}
 756
 757             for PlaceHolder in self._PlaceHolderList:
 758                 if PlaceHolder not in PlaceHolderValues:
 759                     continue
 760                 Value = PlaceHolderValues[PlaceHolder]
 761                 if type(Value) in self._LIST_TYPES:
 762                     if RepeatTime < 0:
 763                         RepeatTime = len(Value)
 764                     elif RepeatTime != len(Value):
 765                         EdkLogger.error(
 766                                     "TemplateString",
 767                                     PARAMETER_INVALID,
 768                                     "${%s} has different repeat time from others!" % PlaceHolder,
 769                                     ExtraData=str(self._Template)
 770                                     )
 771                     RepeatPlaceHolders["${%s}" % PlaceHolder] = Value
 772                 else:
 773                     NonRepeatPlaceHolders["${%s}" % PlaceHolder] = Value
 774
 775             if NonRepeatPlaceHolders:
 776                 StringList = []
 777                 for S in self._SubSectionList:
 778                     if S not in NonRepeatPlaceHolders:
 779                         StringList.append(S)
 780                     else:
 781                         StringList.append(str(NonRepeatPlaceHolders[S]))
 782             else:
 783                 StringList = self._SubSectionList
 784
 785             if RepeatPlaceHolders:
 786                 TempStringList = []
 787                 for Index in range(RepeatTime):
 788                     for S in StringList:
 789                         if S not in RepeatPlaceHolders:
 790                             TempStringList.append(S)
 791                         else:
 792                             TempStringList.append(str(RepeatPlaceHolders[S][Index]))
 793                 StringList = TempStringList
 794
 795             return "".join(StringList)
 796
 797     ## Constructor
 798     def __init__(self, Template=None):
 799         self.String = ''
 800         self.IsBinary = False
 801         self._Template = Template
 802         self._TemplateSectionList = self._Parse(Template)
 803
 804     ## str() operator
 805     #
 806     #   @retval     string  The string replaced
 807     #
 808     def __str__(self):
 809         return self.String
 810
 811     ## Split the template string into fragments per the ${BEGIN} and ${END} flags
 812     #
 813     #   @retval     list    A list of TemplateString.Section objects
 814     #
 815     def _Parse(self, Template):
 816         SectionStart = 0
 817         SearchFrom = 0
 818         MatchEnd = 0
 819         PlaceHolderList = []
 820         TemplateSectionList = []
 821         while Template:
 822             MatchObj = gPlaceholderPattern.search(Template, SearchFrom)
 823             if not MatchObj:
 824                 if MatchEnd <= len(Template):
 825                     TemplateSection = TemplateString.Section(Template[SectionStart:], PlaceHolderList)
 826                     TemplateSectionList.append(TemplateSection)
 827                 break
 828
 829             MatchString = MatchObj.group(1)
 830             MatchStart = MatchObj.start()
 831             MatchEnd = MatchObj.end()
 832
 833             if MatchString == self._REPEAT_START_FLAG:
 834                 if MatchStart > SectionStart:
 835                     TemplateSection = TemplateString.Section(Template[SectionStart:MatchStart], PlaceHolderList)
 836                     TemplateSectionList.append(TemplateSection)
 837                 SectionStart = MatchEnd
 838                 PlaceHolderList = []
 839             elif MatchString == self._REPEAT_END_FLAG:
 840                 TemplateSection = TemplateString.Section(Template[SectionStart:MatchStart], PlaceHolderList)
 841                 TemplateSectionList.append(TemplateSection)
 842                 SectionStart = MatchEnd
 843                 PlaceHolderList = []
 844             else:
 845                 PlaceHolderList.append((MatchString, MatchStart - SectionStart, MatchEnd - SectionStart))
 846             SearchFrom = MatchEnd
 847         return TemplateSectionList
 848
 849     ## Replace the string template with dictionary of placeholders and append it to previous one
 850     #
 851     #   @param      AppendString    The string template to append
 852     #   @param      Dictionary      The placeholder dictionaries
 853     #
 854     def Append(self, AppendString, Dictionary=None):
 855         if Dictionary:
 856             SectionList = self._Parse(AppendString)
 857             self.String += "".join(S.Instantiate(Dictionary) for S in SectionList)
 858         else:
 859             self.String += AppendString
 860
 861     ## Replace the string template with dictionary of placeholders
 862     #
 863     #   @param      Dictionary      The placeholder dictionaries
 864     #
 865     #   @retval     str             The string replaced with placeholder values
 866     #
 867     def Replace(self, Dictionary=None):
 868         return "".join(S.Instantiate(Dictionary) for S in self._TemplateSectionList)
 869
 870 ## Progress indicator class
 871 #
 872 #  This class makes use of thread to print progress on console.
 873 #
 874 class Progressor:
 875     # for avoiding deadloop
 876     _StopFlag = None
 877     _ProgressThread = None
 878     _CheckInterval = 0.25
 879
 880     ## Constructor
 881     #
 882     #   @param      OpenMessage     The string printed before progress charaters
 883     #   @param      CloseMessage    The string printed after progress charaters
 884     #   @param      ProgressChar    The charater used to indicate the progress
 885     #   @param      Interval        The interval in seconds between two progress charaters
 886     #
 887     def __init__(self, OpenMessage="", CloseMessage="", ProgressChar='.', Interval=1.0):
 888         self.PromptMessage = OpenMessage
 889         self.CodaMessage = CloseMessage
 890         self.ProgressChar = ProgressChar
 891         self.Interval = Interval
 892         if Progressor._StopFlag is None:
 893             Progressor._StopFlag = threading.Event()
 894
 895     ## Start to print progress charater
 896     #
 897     #   @param      OpenMessage     The string printed before progress charaters
 898     #
 899     def Start(self, OpenMessage=None):
 900         if OpenMessage is not None:
 901             self.PromptMessage = OpenMessage
 902         Progressor._StopFlag.clear()
 903         if Progressor._ProgressThread is None:
 904             Progressor._ProgressThread = threading.Thread(target=self._ProgressThreadEntry)
 905             Progressor._ProgressThread.setDaemon(False)
 906             Progressor._ProgressThread.start()
 907
 908     ## Stop printing progress charater
 909     #
 910     #   @param      CloseMessage    The string printed after progress charaters
 911     #
 912     def Stop(self, CloseMessage=None):
 913         OriginalCodaMessage = self.CodaMessage
 914         if CloseMessage is not None:
 915             self.CodaMessage = CloseMessage
 916         self.Abort()
 917         self.CodaMessage = OriginalCodaMessage
 918
 919     ## Thread entry method
 920     def _ProgressThreadEntry(self):
 921         sys.stdout.write(self.PromptMessage + " ")
 922         sys.stdout.flush()
 923         TimeUp = 0.0
 924         while not Progressor._StopFlag.isSet():
 925             if TimeUp <= 0.0:
 926                 sys.stdout.write(self.ProgressChar)
 927                 sys.stdout.flush()
 928                 TimeUp = self.Interval
 929             time.sleep(self._CheckInterval)
 930             TimeUp -= self._CheckInterval
 931         sys.stdout.write(" " + self.CodaMessage + "\n")
 932         sys.stdout.flush()
 933
 934     ## Abort the progress display
 935     @staticmethod
 936     def Abort():
 937         if Progressor._StopFlag is not None:
 938             Progressor._StopFlag.set()
 939         if Progressor._ProgressThread is not None:
 940             Progressor._ProgressThread.join()
 941             Progressor._ProgressThread = None
 942
 943 ## A dict which can access its keys and/or values orderly
 944 #
 945 #  The class implements a new kind of dict which its keys or values can be
 946 #  accessed in the order they are added into the dict. It guarantees the order
 947 #  by making use of an internal list to keep a copy of keys.
 948 #
 949 class sdict(IterableUserDict):
 950     ## Constructor
 951     def __init__(self):
 952         IterableUserDict.__init__(self)
 953         self._key_list = []
 954
 955     ## [] operator
 956     def __setitem__(self, key, value):
 957         if key not in self._key_list:
 958             self._key_list.append(key)
 959         IterableUserDict.__setitem__(self, key, value)
 960
 961     ## del operator
 962     def __delitem__(self, key):
 963         self._key_list.remove(key)
 964         IterableUserDict.__delitem__(self, key)
 965
 966     ## used in "for k in dict" loop to ensure the correct order
 967     def __iter__(self):
 968         return self.iterkeys()
 969
 970     ## len() support
 971     def __len__(self):
 972         return len(self._key_list)
 973
 974     ## "in" test support
 975     def __contains__(self, key):
 976         return key in self._key_list
 977
 978     ## indexof support
 979     def index(self, key):
 980         return self._key_list.index(key)
 981
 982     ## insert support
 983     def insert(self, key, newkey, newvalue, order):
 984         index = self._key_list.index(key)
 985         if order == 'BEFORE':
 986             self._key_list.insert(index, newkey)
 987             IterableUserDict.__setitem__(self, newkey, newvalue)
 988         elif order == 'AFTER':
 989             self._key_list.insert(index + 1, newkey)
 990             IterableUserDict.__setitem__(self, newkey, newvalue)
 991
 992     ## append support
 993     def append(self, sdict):
 994         for key in sdict:
 995             if key not in self._key_list:
 996                 self._key_list.append(key)
 997             IterableUserDict.__setitem__(self, key, sdict[key])
 998
 999     def has_key(self, key):
1000         return key in self._key_list
1001
1002     ## Empty the dict
1003     def clear(self):
1004         self._key_list = []
1005         IterableUserDict.clear(self)
1006
1007     ## Return a copy of keys
1008     def keys(self):
1009         keys = []
1010         for key in self._key_list:
1011             keys.append(key)
1012         return keys
1013
1014     ## Return a copy of values
1015     def values(self):
1016         values = []
1017         for key in self._key_list:
1018             values.append(self[key])
1019         return values
1020
1021     ## Return a copy of (key, value) list
1022     def items(self):
1023         items = []
1024         for key in self._key_list:
1025             items.append((key, self[key]))
1026         return items
1027
1028     ## Iteration support
1029     def iteritems(self):
1030         return iter(self.items())
1031
1032     ## Keys interation support
1033     def iterkeys(self):
1034         return self.keys()
1035
1036     ## Values interation support
1037     def itervalues(self):
1038         return self.values()
1039
1040     ## Return value related to a key, and remove the (key, value) from the dict
1041     def pop(self, key, *dv):
1042         value = None
1043         if key in self._key_list:
1044             value = self[key]
1045             self.__delitem__(key)
1046         elif len(dv) != 0 :
1047             value = dv[0]
1048         return value
1049
1050     ## Return (key, value) pair, and remove the (key, value) from the dict
1051     def popitem(self):
1052         key = self._key_list[-1]
1053         value = self[key]
1054         self.__delitem__(key)
1055         return key, value
1056
1057     def update(self, dict=None, **kwargs):
1058         if dict is not None:
1059             for k, v in dict.items():
1060                 self[k] = v
1061         if len(kwargs):
1062             for k, v in kwargs.items():
1063                 self[k] = v
1064
1065 ## Dictionary with restricted keys
1066 #
1067 class rdict(dict):
1068     ## Constructor
1069     def __init__(self, KeyList):
1070         for Key in KeyList:
1071             dict.__setitem__(self, Key, "")
1072
1073     ## []= operator
1074     def __setitem__(self, key, value):
1075         if key not in self:
1076             EdkLogger.error("RestrictedDict", ATTRIBUTE_SET_FAILURE, "Key [%s] is not allowed" % key,
1077                             ExtraData=", ".join(dict.keys(self)))
1078         dict.__setitem__(self, key, value)
1079
1080     ## =[] operator
1081     def __getitem__(self, key):
1082         if key not in self:
1083             return ""
1084         return dict.__getitem__(self, key)
1085
1086     ## del operator
1087     def __delitem__(self, key):
1088         EdkLogger.error("RestrictedDict", ATTRIBUTE_ACCESS_DENIED, ExtraData="del")
1089
1090     ## Empty the dict
1091     def clear(self):
1092         for Key in self:
1093             self.__setitem__(Key, "")
1094
1095     ## Return value related to a key, and remove the (key, value) from the dict
1096     def pop(self, key, *dv):
1097         EdkLogger.error("RestrictedDict", ATTRIBUTE_ACCESS_DENIED, ExtraData="pop")
1098
1099     ## Return (key, value) pair, and remove the (key, value) from the dict
1100     def popitem(self):
1101         EdkLogger.error("RestrictedDict", ATTRIBUTE_ACCESS_DENIED, ExtraData="popitem")
1102
1103 ## Dictionary using prioritized list as key
1104 #
1105 class tdict:
1106     _ListType = type([])
1107     _TupleType = type(())
1108     _Wildcard = 'COMMON'
1109     _ValidWildcardList = ['COMMON', 'DEFAULT', 'ALL', '*', 'PLATFORM']
1110
1111     def __init__(self, _Single_=False, _Level_=2):
1112         self._Level_ = _Level_
1113         self.data = {}
1114         self._Single_ = _Single_
1115
1116     # =[] operator
1117     def __getitem__(self, key):
1118         KeyType = type(key)
1119         RestKeys = None
1120         if KeyType == self._ListType or KeyType == self._TupleType:
1121             FirstKey = key[0]
1122             if len(key) > 1:
1123                 RestKeys = key[1:]
1124             elif self._Level_ > 1:
1125                 RestKeys = [self._Wildcard for i in range(0, self._Level_ - 1)]
1126         else:
1127             FirstKey = key
1128             if self._Level_ > 1:
1129                 RestKeys = [self._Wildcard for i in range(0, self._Level_ - 1)]
1130
1131         if FirstKey is None or str(FirstKey).upper() in self._ValidWildcardList:
1132             FirstKey = self._Wildcard
1133
1134         if self._Single_:
1135             return self._GetSingleValue(FirstKey, RestKeys)
1136         else:
1137             return self._GetAllValues(FirstKey, RestKeys)
1138
1139     def _GetSingleValue(self, FirstKey, RestKeys):
1140         Value = None
1141         #print "%s-%s" % (FirstKey, self._Level_) ,
1142         if self._Level_ > 1:
1143             if FirstKey == self._Wildcard:
1144                 if FirstKey in self.data:
1145                     Value = self.data[FirstKey][RestKeys]
1146                 if Value is None:
1147                     for Key in self.data:
1148                         Value = self.data[Key][RestKeys]
1149                         if Value is not None: break
1150             else:
1151                 if FirstKey in self.data:
1152                     Value = self.data[FirstKey][RestKeys]
1153                 if Value is None and self._Wildcard in self.data:
1154                     #print "Value=None"
1155                     Value = self.data[self._Wildcard][RestKeys]
1156         else:
1157             if FirstKey == self._Wildcard:
1158                 if FirstKey in self.data:
1159                     Value = self.data[FirstKey]
1160                 if Value is None:
1161                     for Key in self.data:
1162                         Value = self.data[Key]
1163                         if Value is not None: break
1164             else:
1165                 if FirstKey in self.data:
1166                     Value = self.data[FirstKey]
1167                 elif self._Wildcard in self.data:
1168                     Value = self.data[self._Wildcard]
1169         return Value
1170
1171     def _GetAllValues(self, FirstKey, RestKeys):
1172         Value = []
1173         if self._Level_ > 1:
1174             if FirstKey == self._Wildcard:
1175                 for Key in self.data:
1176                     Value += self.data[Key][RestKeys]
1177             else:
1178                 if FirstKey in self.data:
1179                     Value += self.data[FirstKey][RestKeys]
1180                 if self._Wildcard in self.data:
1181                     Value += self.data[self._Wildcard][RestKeys]
1182         else:
1183             if FirstKey == self._Wildcard:
1184                 for Key in self.data:
1185                     Value.append(self.data[Key])
1186             else:
1187                 if FirstKey in self.data:
1188                     Value.append(self.data[FirstKey])
1189                 if self._Wildcard in self.data:
1190                     Value.append(self.data[self._Wildcard])
1191         return Value
1192
1193     ## []= operator
1194     def __setitem__(self, key, value):
1195         KeyType = type(key)
1196         RestKeys = None
1197         if KeyType == self._ListType or KeyType == self._TupleType:
1198             FirstKey = key[0]
1199             if len(key) > 1:
1200                 RestKeys = key[1:]
1201             else:
1202                 RestKeys = [self._Wildcard for i in range(0, self._Level_ - 1)]
1203         else:
1204             FirstKey = key
1205             if self._Level_ > 1:
1206                 RestKeys = [self._Wildcard for i in range(0, self._Level_ - 1)]
1207
1208         if FirstKey in self._ValidWildcardList:
1209             FirstKey = self._Wildcard
1210
1211         if FirstKey not in self.data and self._Level_ > 0:
1212             self.data[FirstKey] = tdict(self._Single_, self._Level_ - 1)
1213
1214         if self._Level_ > 1:
1215             self.data[FirstKey][RestKeys] = value
1216         else:
1217             self.data[FirstKey] = value
1218
1219     def SetGreedyMode(self):
1220         self._Single_ = False
1221         if self._Level_ > 1:
1222             for Key in self.data:
1223                 self.data[Key].SetGreedyMode()
1224
1225     def SetSingleMode(self):
1226         self._Single_ = True
1227         if self._Level_ > 1:
1228             for Key in self.data:
1229                 self.data[Key].SetSingleMode()
1230
1231     def GetKeys(self, KeyIndex=0):
1232         assert KeyIndex >= 0
1233         if KeyIndex == 0:
1234             return set(self.data.keys())
1235         else:
1236             keys = set()
1237             for Key in self.data:
1238                 keys |= self.data[Key].GetKeys(KeyIndex - 1)
1239             return keys
1240
1241 def IsFieldValueAnArray (Value):
1242     Value = Value.strip()
1243     if Value.startswith(TAB_GUID) and Value.endswith(')'):
1244         return True
1245     if Value.startswith('L"') and Value.endswith('"')  and len(list(Value[2:-1])) > 1:
1246         return True
1247     if Value[0] == '"' and Value[-1] == '"' and len(list(Value[1:-1])) > 1:
1248         return True
1249     if Value[0] == '{' and Value[-1] == '}':
1250         return True
1251     if Value.startswith("L'") and Value.endswith("'") and len(list(Value[2:-1])) > 1:
1252         return True
1253     if Value[0] == "'" and Value[-1] == "'" and len(list(Value[1:-1])) > 1:
1254         return True
1255     return False
1256
1257 def AnalyzePcdExpression(Setting):
1258     Setting = Setting.strip()
1259     # There might be escaped quote in a string: \", \\\" , \', \\\'
1260     Data = Setting
1261     # There might be '|' in string and in ( ... | ... ), replace it with '-'
1262     NewStr = ''
1263     InSingleQuoteStr = False
1264     InDoubleQuoteStr = False
1265     Pair = 0
1266     for Index, ch in enumerate(Data):
1267         if ch == '"' and not InSingleQuoteStr:
1268             if Data[Index - 1] != '\\':
1269                 InDoubleQuoteStr = not InDoubleQuoteStr
1270         elif ch == "'" and not InDoubleQuoteStr:
1271             if Data[Index - 1] != '\\':
1272                 InSingleQuoteStr = not InSingleQuoteStr
1273         elif ch == '(' and not (InSingleQuoteStr or InDoubleQuoteStr):
1274             Pair += 1
1275         elif ch == ')' and not (InSingleQuoteStr or InDoubleQuoteStr):
1276             Pair -= 1
1277
1278         if (Pair > 0 or InSingleQuoteStr or InDoubleQuoteStr) and ch == TAB_VALUE_SPLIT:
1279             NewStr += '-'
1280         else:
1281             NewStr += ch
1282     FieldList = []
1283     StartPos = 0
1284     while True:
1285         Pos = NewStr.find(TAB_VALUE_SPLIT, StartPos)
1286         if Pos < 0:
1287             FieldList.append(Setting[StartPos:].strip())
1288             break
1289         FieldList.append(Setting[StartPos:Pos].strip())
1290         StartPos = Pos + 1
1291
1292     return FieldList
1293
1294 def ParseDevPathValue (Value):
1295     if '\\' in Value:
1296         Value.replace('\\', '/').replace(' ', '')
1297
1298     Cmd = 'DevicePath ' + '"' + Value + '"'
1299     try:
1300         p = subprocess.Popen(Cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
1301         out, err = p.communicate()
1302     except Exception as X:
1303         raise BadExpression("DevicePath: %s" % (str(X)) )
1304     finally:
1305         subprocess._cleanup()
1306         p.stdout.close()
1307         p.stderr.close()
1308     if err:
1309         raise BadExpression("DevicePath: %s" % str(err))
1310     Size = len(out.split())
1311     out = ','.join(out.split())
1312     return '{' + out + '}', Size
1313
1314 def ParseFieldValue (Value):
1315     if isinstance(Value, type(0)):
1316         return Value, (Value.bit_length() + 7) / 8
1317     if not isinstance(Value, type('')):
1318         raise BadExpression('Type %s is %s' %(Value, type(Value)))
1319     Value = Value.strip()
1320     if Value.startswith(TAB_UINT8) and Value.endswith(')'):
1321         Value, Size = ParseFieldValue(Value.split('(', 1)[1][:-1])
1322         if Size > 1:
1323             raise BadExpression('Value (%s) Size larger than %d' %(Value, Size))
1324         return Value, 1
1325     if Value.startswith(TAB_UINT16) and Value.endswith(')'):
1326         Value, Size = ParseFieldValue(Value.split('(', 1)[1][:-1])
1327         if Size > 2:
1328             raise BadExpression('Value (%s) Size larger than %d' %(Value, Size))
1329         return Value, 2
1330     if Value.startswith(TAB_UINT32) and Value.endswith(')'):
1331         Value, Size = ParseFieldValue(Value.split('(', 1)[1][:-1])
1332         if Size > 4:
1333             raise BadExpression('Value (%s) Size larger than %d' %(Value, Size))
1334         return Value, 4
1335     if Value.startswith(TAB_UINT64) and Value.endswith(')'):
1336         Value, Size = ParseFieldValue(Value.split('(', 1)[1][:-1])
1337         if Size > 8:
1338             raise BadExpression('Value (%s) Size larger than %d' % (Value, Size))
1339         return Value, 8
1340     if Value.startswith(TAB_GUID) and Value.endswith(')'):
1341         Value = Value.split('(', 1)[1][:-1].strip()
1342         if Value[0] == '{' and Value[-1] == '}':
1343             TmpValue = GuidStructureStringToGuidString(Value)
1344             if not TmpValue:
1345                 raise BadExpression("Invalid GUID value string %s" % Value)
1346             Value = TmpValue
1347         if Value[0] == '"' and Value[-1] == '"':
1348             Value = Value[1:-1]
1349         try:
1350             Value = "'" + uuid.UUID(Value).get_bytes_le() + "'"
1351         except ValueError as Message:
1352             raise BadExpression(Message)
1353         Value, Size = ParseFieldValue(Value)
1354         return Value, 16
1355     if Value.startswith('L"') and Value.endswith('"'):
1356         # Unicode String
1357         # translate escape character
1358         Value = Value[1:]
1359         try:
1360             Value = eval(Value)
1361         except:
1362             Value = Value[1:-1]
1363         List = list(Value)
1364         List.reverse()
1365         Value = 0
1366         for Char in List:
1367             Value = (Value << 16) | ord(Char)
1368         return Value, (len(List) + 1) * 2
1369     if Value.startswith('"') and Value.endswith('"'):
1370         # ASCII String
1371         # translate escape character
1372         try:
1373             Value = eval(Value)
1374         except:
1375             Value = Value[1:-1]
1376         List = list(Value)
1377         List.reverse()
1378         Value = 0
1379         for Char in List:
1380             Value = (Value << 8) | ord(Char)
1381         return Value, len(List) + 1
1382     if Value.startswith("L'") and Value.endswith("'"):
1383         # Unicode Character Constant
1384         # translate escape character
1385         Value = Value[1:]
1386         try:
1387             Value = eval(Value)
1388         except:
1389             Value = Value[1:-1]
1390         List = list(Value)
1391         if len(List) == 0:
1392             raise BadExpression('Length %s is %s' % (Value, len(List)))
1393         List.reverse()
1394         Value = 0
1395         for Char in List:
1396             Value = (Value << 16) | ord(Char)
1397         return Value, len(List) * 2
1398     if Value.startswith("'") and Value.endswith("'"):
1399         # Character constant
1400         # translate escape character
1401         try:
1402             Value = eval(Value)
1403         except:
1404             Value = Value[1:-1]
1405         List = list(Value)
1406         if len(List) == 0:
1407             raise BadExpression('Length %s is %s' % (Value, len(List)))
1408         List.reverse()
1409         Value = 0
1410         for Char in List:
1411             Value = (Value << 8) | ord(Char)
1412         return Value, len(List)
1413     if Value.startswith('{') and Value.endswith('}'):
1414         # Byte array
1415         Value = Value[1:-1]
1416         List = [Item.strip() for Item in Value.split(',')]
1417         List.reverse()
1418         Value = 0
1419         RetSize = 0
1420         for Item in List:
1421             ItemValue, Size = ParseFieldValue(Item)
1422             RetSize += Size
1423             for I in range(Size):
1424                 Value = (Value << 8) | ((ItemValue >> 8 * I) & 0xff)
1425         return Value, RetSize
1426     if Value.startswith('DEVICE_PATH(') and Value.endswith(')'):
1427         Value = Value.replace("DEVICE_PATH(", '').rstrip(')')
1428         Value = Value.strip().strip('"')
1429         return ParseDevPathValue(Value)
1430     if Value.lower().startswith('0x'):
1431         try:
1432             Value = int(Value, 16)
1433         except:
1434             raise BadExpression("invalid hex value: %s" % Value)
1435         if Value == 0:
1436             return 0, 1
1437         return Value, (Value.bit_length() + 7) / 8
1438     if Value[0].isdigit():
1439         Value = int(Value, 10)
1440         if Value == 0:
1441             return 0, 1
1442         return Value, (Value.bit_length() + 7) / 8
1443     if Value.lower() == 'true':
1444         return 1, 1
1445     if Value.lower() == 'false':
1446         return 0, 1
1447     return Value, 1
1448
1449 ## AnalyzeDscPcd
1450 #
1451 #  Analyze DSC PCD value, since there is no data type info in DSC
1452 #  This function is used to match functions (AnalyzePcdData) used for retrieving PCD value from database
1453 #  1. Feature flag: TokenSpace.PcdCName|PcdValue
1454 #  2. Fix and Patch:TokenSpace.PcdCName|PcdValue[|VOID*[|MaxSize]]
1455 #  3. Dynamic default:
1456 #     TokenSpace.PcdCName|PcdValue[|VOID*[|MaxSize]]
1457 #     TokenSpace.PcdCName|PcdValue
1458 #  4. Dynamic VPD:
1459 #     TokenSpace.PcdCName|VpdOffset[|VpdValue]
1460 #     TokenSpace.PcdCName|VpdOffset[|MaxSize[|VpdValue]]
1461 #  5. Dynamic HII:
1462 #     TokenSpace.PcdCName|HiiString|VariableGuid|VariableOffset[|HiiValue]
1463 #  PCD value needs to be located in such kind of string, and the PCD value might be an expression in which
1464 #    there might have "|" operator, also in string value.
1465 #
1466 #  @param Setting: String contain information described above with "TokenSpace.PcdCName|" stripped
1467 #  @param PcdType: PCD type: feature, fixed, dynamic default VPD HII
1468 #  @param DataType: The datum type of PCD: VOID*, UNIT, BOOL
1469 #  @retval:
1470 #    ValueList: A List contain fields described above
1471 #    IsValid:   True if conforming EBNF, otherwise False
1472 #    Index:     The index where PcdValue is in ValueList
1473 #
1474 def AnalyzeDscPcd(Setting, PcdType, DataType=''):
1475     FieldList = AnalyzePcdExpression(Setting)
1476
1477     IsValid = True
1478     if PcdType in (MODEL_PCD_FIXED_AT_BUILD, MODEL_PCD_PATCHABLE_IN_MODULE, MODEL_PCD_DYNAMIC_DEFAULT, MODEL_PCD_DYNAMIC_EX_DEFAULT):
1479         Value = FieldList[0]
1480         Size = ''
1481         if len(FieldList) > 1 and FieldList[1]:
1482             DataType = FieldList[1]
1483             if FieldList[1] != TAB_VOID:
1484                 IsValid = False
1485         if len(FieldList) > 2:
1486             Size = FieldList[2]
1487         if IsValid:
1488             if DataType == "":
1489                 IsValid = (len(FieldList) <= 1)
1490             else:
1491                 IsValid = (len(FieldList) <= 3)
1492
1493         if Size:
1494             try:
1495                 int(Size, 16) if Size.upper().startswith("0X") else int(Size)
1496             except:
1497                 IsValid = False
1498                 Size = -1
1499         return [str(Value), DataType, str(Size)], IsValid, 0
1500     elif PcdType == MODEL_PCD_FEATURE_FLAG:
1501         Value = FieldList[0]
1502         Size = ''
1503         IsValid = (len(FieldList) <= 1)
1504         return [Value, DataType, str(Size)], IsValid, 0
1505     elif PcdType in (MODEL_PCD_DYNAMIC_VPD, MODEL_PCD_DYNAMIC_EX_VPD):
1506         VpdOffset = FieldList[0]
1507         Value = Size = ''
1508         if not DataType == TAB_VOID:
1509             if len(FieldList) > 1:
1510                 Value = FieldList[1]
1511         else:
1512             if len(FieldList) > 1:
1513                 Size = FieldList[1]
1514             if len(FieldList) > 2:
1515                 Value = FieldList[2]
1516         if DataType == "":
1517             IsValid = (len(FieldList) <= 1)
1518         else:
1519             IsValid = (len(FieldList) <= 3)
1520         if Size:
1521             try:
1522                 int(Size, 16) if Size.upper().startswith("0X") else int(Size)
1523             except:
1524                 IsValid = False
1525                 Size = -1
1526         return [VpdOffset, str(Size), Value], IsValid, 2
1527     elif PcdType in (MODEL_PCD_DYNAMIC_HII, MODEL_PCD_DYNAMIC_EX_HII):
1528         IsValid = (3 <= len(FieldList) <= 5)
1529         HiiString = FieldList[0]
1530         Guid = Offset = Value = Attribute = ''
1531         if len(FieldList) > 1:
1532             Guid = FieldList[1]
1533         if len(FieldList) > 2:
1534             Offset = FieldList[2]
1535         if len(FieldList) > 3:
1536             Value = FieldList[3]
1537             if not Value:
1538                 IsValid = False
1539         if len(FieldList) > 4:
1540             Attribute = FieldList[4]
1541         return [HiiString, Guid, Offset, Value, Attribute], IsValid, 3
1542     return [], False, 0
1543
1544 ## AnalyzePcdData
1545 #
1546 #  Analyze the pcd Value, Datum type and TokenNumber.
1547 #  Used to avoid split issue while the value string contain "|" character
1548 #
1549 #  @param[in] Setting:  A String contain value/datum type/token number information;
1550 #
1551 #  @retval   ValueList: A List contain value, datum type and toke number.
1552 #
1553 def AnalyzePcdData(Setting):
1554     ValueList = ['', '', '']
1555
1556     ValueRe = re.compile(r'^\s*L?\".*\|.*\"')
1557     PtrValue = ValueRe.findall(Setting)
1558
1559     ValueUpdateFlag = False
1560
1561     if len(PtrValue) >= 1:
1562         Setting = re.sub(ValueRe, '', Setting)
1563         ValueUpdateFlag = True
1564
1565     TokenList = Setting.split(TAB_VALUE_SPLIT)
1566     ValueList[0:len(TokenList)] = TokenList
1567
1568     if ValueUpdateFlag:
1569         ValueList[0] = PtrValue[0]
1570
1571     return ValueList
1572
1573 ## check format of PCD value against its the datum type
1574 #
1575 # For PCD value setting
1576 #
1577 def CheckPcdDatum(Type, Value):
1578     if Type == TAB_VOID:
1579         ValueRe = re.compile(r'\s*L?\".*\"\s*$')
1580         if not (((Value.startswith('L"') or Value.startswith('"')) and Value.endswith('"'))
1581                 or (Value.startswith('{') and Value.endswith('}')) or (Value.startswith("L'") or Value.startswith("'") and Value.endswith("'"))
1582                ):
1583             return False, "Invalid value [%s] of type [%s]; must be in the form of {...} for array"\
1584                           ", \"...\" or \'...\' for string, L\"...\" or L\'...\' for unicode string" % (Value, Type)
1585         elif ValueRe.match(Value):
1586             # Check the chars in UnicodeString or CString is printable
1587             if Value.startswith("L"):
1588                 Value = Value[2:-1]
1589             else:
1590                 Value = Value[1:-1]
1591             Printset = set(string.printable)
1592             Printset.remove(TAB_PRINTCHAR_VT)
1593             Printset.add(TAB_PRINTCHAR_BS)
1594             Printset.add(TAB_PRINTCHAR_NUL)
1595             if not set(Value).issubset(Printset):
1596                 PrintList = sorted(Printset)
1597                 return False, "Invalid PCD string value of type [%s]; must be printable chars %s." % (Type, PrintList)
1598     elif Type == 'BOOLEAN':
1599         if Value not in ['TRUE', 'True', 'true', '0x1', '0x01', '1', 'FALSE', 'False', 'false', '0x0', '0x00', '0']:
1600             return False, "Invalid value [%s] of type [%s]; must be one of TRUE, True, true, 0x1, 0x01, 1"\
1601                           ", FALSE, False, false, 0x0, 0x00, 0" % (Value, Type)
1602     elif Type in [TAB_UINT8, TAB_UINT16, TAB_UINT32, TAB_UINT64]:
1603         if Value and int(Value, 0) < 0:
1604             return False, "PCD can't be set to negative value[%s] for datum type [%s]" % (Value, Type)
1605         try:
1606             Value = int(Value, 0)
1607             if Value > MAX_VAL_TYPE[Type]:
1608                 return False, "Too large PCD value[%s] for datum type [%s]" % (Value, Type)
1609         except:
1610             return False, "Invalid value [%s] of type [%s];"\
1611                           " must be a hexadecimal, decimal or octal in C language format." % (Value, Type)
1612     else:
1613         return True, "StructurePcd"
1614
1615     return True, ""
1616
1617 ## Split command line option string to list
1618 #
1619 # subprocess.Popen needs the args to be a sequence. Otherwise there's problem
1620 # in non-windows platform to launch command
1621 #
1622 def SplitOption(OptionString):
1623     OptionList = []
1624     LastChar = " "
1625     OptionStart = 0
1626     QuotationMark = ""
1627     for Index in range(0, len(OptionString)):
1628         CurrentChar = OptionString[Index]
1629         if CurrentChar in ['"', "'"]:
1630             if QuotationMark == CurrentChar:
1631                 QuotationMark = ""
1632             elif QuotationMark == "":
1633                 QuotationMark = CurrentChar
1634             continue
1635         elif QuotationMark:
1636             continue
1637
1638         if CurrentChar in ["/", "-"] and LastChar in [" ", "\t", "\r", "\n"]:
1639             if Index > OptionStart:
1640                 OptionList.append(OptionString[OptionStart:Index - 1])
1641             OptionStart = Index
1642         LastChar = CurrentChar
1643     OptionList.append(OptionString[OptionStart:])
1644     return OptionList
1645
1646 def CommonPath(PathList):
1647     P1 = min(PathList).split(os.path.sep)
1648     P2 = max(PathList).split(os.path.sep)
1649     for Index in range(min(len(P1), len(P2))):
1650         if P1[Index] != P2[Index]:
1651             return os.path.sep.join(P1[:Index])
1652     return os.path.sep.join(P1)
1653
1654 #
1655 # Convert string to C format array
1656 #
1657 def ConvertStringToByteArray(Value):
1658     Value = Value.strip()
1659     if not Value:
1660         return None
1661     if Value[0] == '{':
1662         if not Value.endswith('}'):
1663             return None
1664         Value = Value.replace(' ', '').replace('{', '').replace('}', '')
1665         ValFields = Value.split(',')
1666         try:
1667             for Index in range(len(ValFields)):
1668                 ValFields[Index] = str(int(ValFields[Index], 0))
1669         except ValueError:
1670             return None
1671         Value = '{' + ','.join(ValFields) + '}'
1672         return Value
1673
1674     Unicode = False
1675     if Value.startswith('L"'):
1676         if not Value.endswith('"'):
1677             return None
1678         Value = Value[1:]
1679         Unicode = True
1680     elif not Value.startswith('"') or not Value.endswith('"'):
1681         return None
1682
1683     Value = eval(Value)         # translate escape character
1684     NewValue = '{'
1685     for Index in range(0, len(Value)):
1686         if Unicode:
1687             NewValue = NewValue + str(ord(Value[Index]) % 0x10000) + ','
1688         else:
1689             NewValue = NewValue + str(ord(Value[Index]) % 0x100) + ','
1690     Value = NewValue + '0}'
1691     return Value
1692
1693 class PathClass(object):
1694     def __init__(self, File='', Root='', AlterRoot='', Type='', IsBinary=False,
1695                  Arch='COMMON', ToolChainFamily='', Target='', TagName='', ToolCode=''):
1696         self.Arch = Arch
1697         self.File = str(File)
1698         if os.path.isabs(self.File):
1699             self.Root = ''
1700             self.AlterRoot = ''
1701         else:
1702             self.Root = str(Root)
1703             self.AlterRoot = str(AlterRoot)
1704
1705         # Remove any '.' and '..' in path
1706         if self.Root:
1707             self.Root = mws.getWs(self.Root, self.File)
1708             self.Path = os.path.normpath(os.path.join(self.Root, self.File))
1709             self.Root = os.path.normpath(CommonPath([self.Root, self.Path]))
1710             # eliminate the side-effect of 'C:'
1711             if self.Root[-1] == ':':
1712                 self.Root += os.path.sep
1713             # file path should not start with path separator
1714             if self.Root[-1] == os.path.sep:
1715                 self.File = self.Path[len(self.Root):]
1716             else:
1717                 self.File = self.Path[len(self.Root) + 1:]
1718         else:
1719             self.Path = os.path.normpath(self.File)
1720
1721         self.SubDir, self.Name = os.path.split(self.File)
1722         self.BaseName, self.Ext = os.path.splitext(self.Name)
1723
1724         if self.Root:
1725             if self.SubDir:
1726                 self.Dir = os.path.join(self.Root, self.SubDir)
1727             else:
1728                 self.Dir = self.Root
1729         else:
1730             self.Dir = self.SubDir
1731
1732         if IsBinary:
1733             self.Type = Type
1734         else:
1735             self.Type = self.Ext.lower()
1736
1737         self.IsBinary = IsBinary
1738         self.Target = Target
1739         self.TagName = TagName
1740         self.ToolCode = ToolCode
1741         self.ToolChainFamily = ToolChainFamily
1742
1743     ## Convert the object of this class to a string
1744     #
1745     #  Convert member Path of the class to a string
1746     #
1747     #  @retval string Formatted String
1748     #
1749     def __str__(self):
1750         return self.Path
1751
1752     ## Override __eq__ function
1753     #
1754     # Check whether PathClass are the same
1755     #
1756     # @retval False The two PathClass are different
1757     # @retval True  The two PathClass are the same
1758     #
1759     def __eq__(self, Other):
1760         if isinstance(Other, type(self)):
1761             return self.Path == Other.Path
1762         else:
1763             return self.Path == str(Other)
1764
1765     ## Override __cmp__ function
1766     #
1767     # Customize the comparsion operation of two PathClass
1768     #
1769     # @retval 0     The two PathClass are different
1770     # @retval -1    The first PathClass is less than the second PathClass
1771     # @retval 1     The first PathClass is Bigger than the second PathClass
1772     def __cmp__(self, Other):
1773         if isinstance(Other, type(self)):
1774             OtherKey = Other.Path
1775         else:
1776             OtherKey = str(Other)
1777
1778         SelfKey = self.Path
1779         if SelfKey == OtherKey:
1780             return 0
1781         elif SelfKey > OtherKey:
1782             return 1
1783         else:
1784             return -1
1785
1786     ## Override __hash__ function
1787     #
1788     # Use Path as key in hash table
1789     #
1790     # @retval string Key for hash table
1791     #
1792     def __hash__(self):
1793         return hash(self.Path)
1794
1795     @cached_property
1796     def Key(self):
1797         return self.Path.upper()
1798
1799     @property
1800     def TimeStamp(self):
1801         return os.stat(self.Path)[8]
1802
1803     def Validate(self, Type='', CaseSensitive=True):
1804         if GlobalData.gCaseInsensitive:
1805             CaseSensitive = False
1806         if Type and Type.lower() != self.Type:
1807             return FILE_TYPE_MISMATCH, '%s (expect %s but got %s)' % (self.File, Type, self.Type)
1808
1809         RealFile, RealRoot = RealPath2(self.File, self.Root, self.AlterRoot)
1810         if not RealRoot and not RealFile:
1811             RealFile = self.File
1812             if self.AlterRoot:
1813                 RealFile = os.path.join(self.AlterRoot, self.File)
1814             elif self.Root:
1815                 RealFile = os.path.join(self.Root, self.File)
1816             if len (mws.getPkgPath()) == 0:
1817                 return FILE_NOT_FOUND, os.path.join(self.AlterRoot, RealFile)
1818             else:
1819                 return FILE_NOT_FOUND, "%s is not found in packages path:\n\t%s" % (self.File, '\n\t'.join(mws.getPkgPath()))
1820
1821         ErrorCode = 0
1822         ErrorInfo = ''
1823         if RealRoot != self.Root or RealFile != self.File:
1824             if CaseSensitive and (RealFile != self.File or (RealRoot != self.Root and RealRoot != self.AlterRoot)):
1825                 ErrorCode = FILE_CASE_MISMATCH
1826                 ErrorInfo = self.File + '\n\t' + RealFile + " [in file system]"
1827
1828             self.SubDir, self.Name = os.path.split(RealFile)
1829             self.BaseName, self.Ext = os.path.splitext(self.Name)
1830             if self.SubDir:
1831                 self.Dir = os.path.join(RealRoot, self.SubDir)
1832             else:
1833                 self.Dir = RealRoot
1834             self.File = RealFile
1835             self.Root = RealRoot
1836             self.Path = os.path.join(RealRoot, RealFile)
1837         return ErrorCode, ErrorInfo
1838
1839 ## Parse PE image to get the required PE informaion.
1840 #
1841 class PeImageClass():
1842     ## Constructor
1843     #
1844     #   @param  File FilePath of PeImage
1845     #
1846     def __init__(self, PeFile):
1847         self.FileName   = PeFile
1848         self.IsValid    = False
1849         self.Size       = 0
1850         self.EntryPoint = 0
1851         self.SectionAlignment  = 0
1852         self.SectionHeaderList = []
1853         self.ErrorInfo = ''
1854         try:
1855             PeObject = open(PeFile, 'rb')
1856         except:
1857             self.ErrorInfo = self.FileName + ' can not be found\n'
1858             return
1859         # Read DOS header
1860         ByteArray = array.array('B')
1861         ByteArray.fromfile(PeObject, 0x3E)
1862         ByteList = ByteArray.tolist()
1863         # DOS signature should be 'MZ'
1864         if self._ByteListToStr (ByteList[0x0:0x2]) != 'MZ':
1865             self.ErrorInfo = self.FileName + ' has no valid DOS signature MZ'
1866             return
1867
1868         # Read 4 byte PE Signature
1869         PeOffset = self._ByteListToInt(ByteList[0x3C:0x3E])
1870         PeObject.seek(PeOffset)
1871         ByteArray = array.array('B')
1872         ByteArray.fromfile(PeObject, 4)
1873         # PE signature should be 'PE\0\0'
1874         if ByteArray.tostring() != 'PE\0\0':
1875             self.ErrorInfo = self.FileName + ' has no valid PE signature PE00'
1876             return
1877
1878         # Read PE file header
1879         ByteArray = array.array('B')
1880         ByteArray.fromfile(PeObject, 0x14)
1881         ByteList = ByteArray.tolist()
1882         SecNumber = self._ByteListToInt(ByteList[0x2:0x4])
1883         if SecNumber == 0:
1884             self.ErrorInfo = self.FileName + ' has no section header'
1885             return
1886
1887         # Read PE optional header
1888         OptionalHeaderSize = self._ByteListToInt(ByteArray[0x10:0x12])
1889         ByteArray = array.array('B')
1890         ByteArray.fromfile(PeObject, OptionalHeaderSize)
1891         ByteList = ByteArray.tolist()
1892         self.EntryPoint       = self._ByteListToInt(ByteList[0x10:0x14])
1893         self.SectionAlignment = self._ByteListToInt(ByteList[0x20:0x24])
1894         self.Size             = self._ByteListToInt(ByteList[0x38:0x3C])
1895
1896         # Read each Section Header
1897         for Index in range(SecNumber):
1898             ByteArray = array.array('B')
1899             ByteArray.fromfile(PeObject, 0x28)
1900             ByteList = ByteArray.tolist()
1901             SecName  = self._ByteListToStr(ByteList[0:8])
1902             SecVirtualSize = self._ByteListToInt(ByteList[8:12])
1903             SecRawAddress  = self._ByteListToInt(ByteList[20:24])
1904             SecVirtualAddress = self._ByteListToInt(ByteList[12:16])
1905             self.SectionHeaderList.append((SecName, SecVirtualAddress, SecRawAddress, SecVirtualSize))
1906         self.IsValid = True
1907         PeObject.close()
1908
1909     def _ByteListToStr(self, ByteList):
1910         String = ''
1911         for index in range(len(ByteList)):
1912             if ByteList[index] == 0:
1913                 break
1914             String += chr(ByteList[index])
1915         return String
1916
1917     def _ByteListToInt(self, ByteList):
1918         Value = 0
1919         for index in range(len(ByteList) - 1, -1, -1):
1920             Value = (Value << 8) | int(ByteList[index])
1921         return Value
1922
1923 class DefaultStore():
1924     def __init__(self, DefaultStores ):
1925
1926         self.DefaultStores = DefaultStores
1927     def DefaultStoreID(self, DefaultStoreName):
1928         for key, value in self.DefaultStores.items():
1929             if value == DefaultStoreName:
1930                 return key
1931         return None
1932     def GetDefaultDefault(self):
1933         if not self.DefaultStores or "0" in self.DefaultStores:
1934             return "0", TAB_DEFAULT_STORES_DEFAULT
1935         else:
1936             minvalue = min(int(value_str) for value_str in self.DefaultStores)
1937             return (str(minvalue), self.DefaultStores[str(minvalue)])
1938     def GetMin(self, DefaultSIdList):
1939         if not DefaultSIdList:
1940             return TAB_DEFAULT_STORES_DEFAULT
1941         storeidset = {storeid for storeid, storename in self.DefaultStores.values() if storename in DefaultSIdList}
1942         if not storeidset:
1943             return ""
1944         minid = min(storeidset )
1945         for sid, name in self.DefaultStores.values():
1946             if sid == minid:
1947                 return name
1948
1949 class SkuClass():
1950     DEFAULT = 0
1951     SINGLE = 1
1952     MULTIPLE =2
1953
1954     def __init__(self,SkuIdentifier='', SkuIds=None):
1955         if SkuIds is None:
1956             SkuIds = {}
1957
1958         for SkuName in SkuIds:
1959             SkuId = SkuIds[SkuName][0]
1960             skuid_num = int(SkuId, 16) if SkuId.upper().startswith("0X") else int(SkuId)
1961             if skuid_num > 0xFFFFFFFFFFFFFFFF:
1962                 EdkLogger.error("build", PARAMETER_INVALID,
1963                             ExtraData = "SKU-ID [%s] value %s exceeds the max value of UINT64"
1964                                       % (SkuName, SkuId))
1965
1966         self.AvailableSkuIds = sdict()
1967         self.SkuIdSet = []
1968         self.SkuIdNumberSet = []
1969         self.SkuData = SkuIds
1970         self._SkuInherit = {}
1971         self._SkuIdentifier = SkuIdentifier
1972         if SkuIdentifier == '' or SkuIdentifier is None:
1973             self.SkuIdSet = ['DEFAULT']
1974             self.SkuIdNumberSet = ['0U']
1975         elif SkuIdentifier == 'ALL':
1976             self.SkuIdSet = SkuIds.keys()
1977             self.SkuIdNumberSet = [num[0].strip() + 'U' for num in SkuIds.values()]
1978         else:
1979             r = SkuIdentifier.split('|')
1980             self.SkuIdSet=[(r[k].strip()).upper() for k in range(len(r))]
1981             k = None
1982             try:
1983                 self.SkuIdNumberSet = [SkuIds[k][0].strip() + 'U' for k in self.SkuIdSet]
1984             except Exception:
1985                 EdkLogger.error("build", PARAMETER_INVALID,
1986                             ExtraData = "SKU-ID [%s] is not supported by the platform. [Valid SKU-ID: %s]"
1987                                       % (k, " | ".join(SkuIds.keys())))
1988         for each in self.SkuIdSet:
1989             if each in SkuIds:
1990                 self.AvailableSkuIds[each] = SkuIds[each][0]
1991             else:
1992                 EdkLogger.error("build", PARAMETER_INVALID,
1993                             ExtraData="SKU-ID [%s] is not supported by the platform. [Valid SKU-ID: %s]"
1994                                       % (each, " | ".join(SkuIds.keys())))
1995         if self.SkuUsageType != SkuClass.SINGLE:
1996             self.AvailableSkuIds.update({'DEFAULT':0, 'COMMON':0})
1997         if self.SkuIdSet:
1998             GlobalData.gSkuids = (self.SkuIdSet)
1999             if 'COMMON' in GlobalData.gSkuids:
2000                 GlobalData.gSkuids.remove('COMMON')
2001             if self.SkuUsageType == self.SINGLE:
2002                 if len(GlobalData.gSkuids) != 1:
2003                     if 'DEFAULT' in GlobalData.gSkuids:
2004                         GlobalData.gSkuids.remove('DEFAULT')
2005             if GlobalData.gSkuids:
2006                 GlobalData.gSkuids.sort()
2007
2008     def GetNextSkuId(self, skuname):
2009         if not self._SkuInherit:
2010             self._SkuInherit = {}
2011             for item in self.SkuData.values():
2012                 self._SkuInherit[item[1]]=item[2] if item[2] else "DEFAULT"
2013         return self._SkuInherit.get(skuname, "DEFAULT")
2014
2015     def GetSkuChain(self, sku):
2016         if sku == "DEFAULT":
2017             return ["DEFAULT"]
2018         skulist = [sku]
2019         nextsku = sku
2020         while True:
2021             nextsku = self.GetNextSkuId(nextsku)
2022             skulist.append(nextsku)
2023             if nextsku == "DEFAULT":
2024                 break
2025         skulist.reverse()
2026         return skulist
2027     def SkuOverrideOrder(self):
2028         skuorderset = []
2029         for skuname in self.SkuIdSet:
2030             skuorderset.append(self.GetSkuChain(skuname))
2031
2032         skuorder = []
2033         for index in range(max(len(item) for item in skuorderset)):
2034             for subset in skuorderset:
2035                 if index > len(subset)-1:
2036                     continue
2037                 if subset[index] in skuorder:
2038                     continue
2039                 skuorder.append(subset[index])
2040
2041         return skuorder
2042
2043     @property
2044     def SkuUsageType(self):
2045         if self._SkuIdentifier.upper() == "ALL":
2046             return SkuClass.MULTIPLE
2047
2048         if len(self.SkuIdSet) == 1:
2049             if self.SkuIdSet[0] == 'DEFAULT':
2050                 return SkuClass.DEFAULT
2051             return SkuClass.SINGLE
2052         if len(self.SkuIdSet) == 2 and 'DEFAULT' in self.SkuIdSet:
2053             return SkuClass.SINGLE
2054         return SkuClass.MULTIPLE
2055
2056     def DumpSkuIdArrary(self):
2057         if self.SkuUsageType == SkuClass.SINGLE:
2058             return "{0x0}"
2059         ArrayStrList = []
2060         for skuname in self.AvailableSkuIds:
2061             if skuname == "COMMON":
2062                 continue
2063             while skuname != "DEFAULT":
2064                 ArrayStrList.append(hex(int(self.AvailableSkuIds[skuname])))
2065                 skuname = self.GetNextSkuId(skuname)
2066             ArrayStrList.append("0x0")
2067         return "{{{myList}}}".format(myList=",".join(ArrayStrList))
2068
2069     @property
2070     def AvailableSkuIdSet(self):
2071         return self.AvailableSkuIds
2072
2073     @property
2074     def SystemSkuId(self):
2075         if self.SkuUsageType == SkuClass.SINGLE:
2076             if len(self.SkuIdSet) == 1:
2077                 return self.SkuIdSet[0]
2078             else:
2079                 return self.SkuIdSet[0] if self.SkuIdSet[0] != 'DEFAULT' else self.SkuIdSet[1]
2080         else:
2081             return 'DEFAULT'
2082
2083 #
2084 # Pack a registry format GUID
2085 #
2086 def PackRegistryFormatGuid(Guid):
2087     return PackGUID(Guid.split('-'))
2088
2089 ##  Get the integer value from string like "14U" or integer like 2
2090 #
2091 #   @param      Input   The object that may be either a integer value or a string
2092 #
2093 #   @retval     Value    The integer value that the input represents
2094 #
2095 def GetIntegerValue(Input):
2096     if isinstance(Input, int):
2097         return Input
2098     String = Input
2099     if String.endswith("U"):
2100         String = String[:-1]
2101     if String.endswith("ULL"):
2102         String = String[:-3]
2103     if String.endswith("LL"):
2104         String = String[:-2]
2105
2106     if String.startswith("0x") or String.startswith("0X"):
2107         return int(String, 16)
2108     elif String == '':
2109         return 0
2110     else:
2111         return int(String)
2112
2113 #
2114 # Pack a GUID (registry format) list into a buffer and return it
2115 #
2116 def PackGUID(Guid):
2117     return pack(PACK_PATTERN_GUID,
2118                 int(Guid[0], 16),
2119                 int(Guid[1], 16),
2120                 int(Guid[2], 16),
2121                 int(Guid[3][-4:-2], 16),
2122                 int(Guid[3][-2:], 16),
2123                 int(Guid[4][-12:-10], 16),
2124                 int(Guid[4][-10:-8], 16),
2125                 int(Guid[4][-8:-6], 16),
2126                 int(Guid[4][-6:-4], 16),
2127                 int(Guid[4][-4:-2], 16),
2128                 int(Guid[4][-2:], 16)
2129                 )
2130
2131 #
2132 # Pack a GUID (byte) list into a buffer and return it
2133 #
2134 def PackByteFormatGUID(Guid):
2135     return pack(PACK_PATTERN_GUID,
2136                 Guid[0],
2137                 Guid[1],
2138                 Guid[2],
2139                 Guid[3],
2140                 Guid[4],
2141                 Guid[5],
2142                 Guid[6],
2143                 Guid[7],
2144                 Guid[8],
2145                 Guid[9],
2146                 Guid[10],
2147                 )
2148
2149 ##
2150 #
2151 # This acts like the main() function for the script, unless it is 'import'ed into another
2152 # script.
2153 #
2154 if __name__ == '__main__':
2155     pass
2156