BaseTools/Source/Python/Common/Misc.py

   1 ## @file
   2 # Common routines used by all tools
   3 #
   4 # Copyright (c) 2007 - 2018, Intel Corporation. All rights reserved.<BR>
   5 # This program and the accompanying materials
   6 # are licensed and made available under the terms and conditions of the BSD License
   7 # which accompanies this distribution.  The full text of the license may be found at
   8 # http://opensource.org/licenses/bsd-license.php
   9 #
  10 # THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS,
  11 # WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED.
  12 #
  13
  14 ##
  15 # Import Modules
  16 #
  17 from __future__ import absolute_import
  18 import Common.LongFilePathOs as os
  19 import sys
  20 import string
  21 import threading
  22 import time
  23 import re
  24 import pickle
  25 import array
  26 import shutil
  27 from random import sample
  28 from struct import pack
  29 from UserDict import IterableUserDict
  30 from UserList import UserList
  31
  32 from Common import EdkLogger as EdkLogger
  33 from Common import GlobalData as GlobalData
  34 from .DataType import *
  35 from .BuildToolError import *
  36 from CommonDataClass.DataClass import *
  37 from .Parsing import GetSplitValueList
  38 from Common.LongFilePathSupport import OpenLongFilePath as open
  39 from Common.MultipleWorkspace import MultipleWorkspace as mws
  40 import uuid
  41 from CommonDataClass.Exceptions import BadExpression
  42 from Common.caching import cached_property
  43 import subprocess
  44 ## Regular expression used to find out place holders in string template
  45 gPlaceholderPattern = re.compile("\$\{([^$()\s]+)\}", re.MULTILINE | re.UNICODE)
  46
  47 ## regular expressions for map file processing
  48 startPatternGeneral = re.compile("^Start[' ']+Length[' ']+Name[' ']+Class")
  49 addressPatternGeneral = re.compile("^Address[' ']+Publics by Value[' ']+Rva\+Base")
  50 valuePatternGcc = re.compile('^([\w_\.]+) +([\da-fA-Fx]+) +([\da-fA-Fx]+)$')
  51 pcdPatternGcc = re.compile('^([\da-fA-Fx]+) +([\da-fA-Fx]+)')
  52 secReGeneral = re.compile('^([\da-fA-F]+):([\da-fA-F]+) +([\da-fA-F]+)[Hh]? +([.\w\$]+) +(\w+)', re.UNICODE)
  53
  54 StructPattern = re.compile(r'[_a-zA-Z][0-9A-Za-z_]*$')
  55
  56 ## Dictionary used to store file time stamp for quick re-access
  57 gFileTimeStampCache = {}    # {file path : file time stamp}
  58
  59 ## Dictionary used to store dependencies of files
  60 gDependencyDatabase = {}    # arch : {file path : [dependent files list]}
  61
  62 #
  63 # If a module is built more than once with different PCDs or library classes
  64 # a temporary INF file with same content is created, the temporary file is removed
  65 # when build exits.
  66 #
  67 _TempInfs = []
  68
  69 def GetVariableOffset(mapfilepath, efifilepath, varnames):
  70     """ Parse map file to get variable offset in current EFI file
  71     @param mapfilepath    Map file absolution path
  72     @param efifilepath:   EFI binary file full path
  73     @param varnames       iteratable container whose elements are variable names to be searched
  74
  75     @return List whos elements are tuple with variable name and raw offset
  76     """
  77     lines = []
  78     try:
  79         f = open(mapfilepath, 'r')
  80         lines = f.readlines()
  81         f.close()
  82     except:
  83         return None
  84
  85     if len(lines) == 0: return None
  86     firstline = lines[0].strip()
  87     if (firstline.startswith("Archive member included ") and
  88         firstline.endswith(" file (symbol)")):
  89         return _parseForGCC(lines, efifilepath, varnames)
  90     if firstline.startswith("# Path:"):
  91         return _parseForXcode(lines, efifilepath, varnames)
  92     return _parseGeneral(lines, efifilepath, varnames)
  93
  94 def _parseForXcode(lines, efifilepath, varnames):
  95     status = 0
  96     ret = []
  97     for line in lines:
  98         line = line.strip()
  99         if status == 0 and line == "# Symbols:":
 100             status = 1
 101             continue
 102         if status == 1 and len(line) != 0:
 103             for varname in varnames:
 104                 if varname in line:
 105                     # cannot pregenerate this RegEx since it uses varname from varnames.
 106                     m = re.match('^([\da-fA-FxX]+)([\s\S]*)([_]*%s)$' % varname, line)
 107                     if m is not None:
 108                         ret.append((varname, m.group(1)))
 109     return ret
 110
 111 def _parseForGCC(lines, efifilepath, varnames):
 112     """ Parse map file generated by GCC linker """
 113     status = 0
 114     sections = []
 115     varoffset = []
 116     for index, line in enumerate(lines):
 117         line = line.strip()
 118         # status machine transection
 119         if status == 0 and line == "Memory Configuration":
 120             status = 1
 121             continue
 122         elif status == 1 and line == 'Linker script and memory map':
 123             status = 2
 124             continue
 125         elif status ==2 and line == 'START GROUP':
 126             status = 3
 127             continue
 128
 129         # status handler
 130         if status == 3:
 131             m = valuePatternGcc.match(line)
 132             if m is not None:
 133                 sections.append(m.groups(0))
 134             for varname in varnames:
 135                 Str = ''
 136                 m = re.match("^.data.(%s)" % varname, line)
 137                 if m is not None:
 138                     m = re.match(".data.(%s)$" % varname, line)
 139                     if m is not None:
 140                         Str = lines[index + 1]
 141                     else:
 142                         Str = line[len(".data.%s" % varname):]
 143                     if Str:
 144                         m = pcdPatternGcc.match(Str.strip())
 145                         if m is not None:
 146                             varoffset.append((varname, int(m.groups(0)[0], 16), int(sections[-1][1], 16), sections[-1][0]))
 147
 148     if not varoffset:
 149         return []
 150     # get section information from efi file
 151     efisecs = PeImageClass(efifilepath).SectionHeaderList
 152     if efisecs is None or len(efisecs) == 0:
 153         return []
 154     #redirection
 155     redirection = 0
 156     for efisec in efisecs:
 157         for section in sections:
 158             if section[0].strip() == efisec[0].strip() and section[0].strip() == '.text':
 159                 redirection = int(section[1], 16) - efisec[1]
 160
 161     ret = []
 162     for var in varoffset:
 163         for efisec in efisecs:
 164             if var[1] >= efisec[1] and var[1] < efisec[1]+efisec[3]:
 165                 ret.append((var[0], hex(efisec[2] + var[1] - efisec[1] - redirection)))
 166     return ret
 167
 168 def _parseGeneral(lines, efifilepath, varnames):
 169     status = 0    #0 - beginning of file; 1 - PE section definition; 2 - symbol table
 170     secs  = []    # key = section name
 171     varoffset = []
 172     symRe = re.compile('^([\da-fA-F]+):([\da-fA-F]+) +([\.:\\\\\w\?@\$]+) +([\da-fA-F]+)', re.UNICODE)
 173
 174     for line in lines:
 175         line = line.strip()
 176         if startPatternGeneral.match(line):
 177             status = 1
 178             continue
 179         if addressPatternGeneral.match(line):
 180             status = 2
 181             continue
 182         if line.startswith("entry point at"):
 183             status = 3
 184             continue
 185         if status == 1 and len(line) != 0:
 186             m =  secReGeneral.match(line)
 187             assert m is not None, "Fail to parse the section in map file , line is %s" % line
 188             sec_no, sec_start, sec_length, sec_name, sec_class = m.groups(0)
 189             secs.append([int(sec_no, 16), int(sec_start, 16), int(sec_length, 16), sec_name, sec_class])
 190         if status == 2 and len(line) != 0:
 191             for varname in varnames:
 192                 m = symRe.match(line)
 193                 assert m is not None, "Fail to parse the symbol in map file, line is %s" % line
 194                 sec_no, sym_offset, sym_name, vir_addr = m.groups(0)
 195                 sec_no     = int(sec_no,     16)
 196                 sym_offset = int(sym_offset, 16)
 197                 vir_addr   = int(vir_addr,   16)
 198                 # cannot pregenerate this RegEx since it uses varname from varnames.
 199                 m2 = re.match('^[_]*(%s)' % varname, sym_name)
 200                 if m2 is not None:
 201                     # fond a binary pcd entry in map file
 202                     for sec in secs:
 203                         if sec[0] == sec_no and (sym_offset >= sec[1] and sym_offset < sec[1] + sec[2]):
 204                             varoffset.append([varname, sec[3], sym_offset, vir_addr, sec_no])
 205
 206     if not varoffset: return []
 207
 208     # get section information from efi file
 209     efisecs = PeImageClass(efifilepath).SectionHeaderList
 210     if efisecs is None or len(efisecs) == 0:
 211         return []
 212
 213     ret = []
 214     for var in varoffset:
 215         index = 0
 216         for efisec in efisecs:
 217             index = index + 1
 218             if var[1].strip() == efisec[0].strip():
 219                 ret.append((var[0], hex(efisec[2] + var[2])))
 220             elif var[4] == index:
 221                 ret.append((var[0], hex(efisec[2] + var[2])))
 222
 223     return ret
 224
 225 ## Routine to process duplicated INF
 226 #
 227 #  This function is called by following two cases:
 228 #  Case 1 in DSC:
 229 #    [components.arch]
 230 #    Pkg/module/module.inf
 231 #    Pkg/module/module.inf {
 232 #      <Defines>
 233 #        FILE_GUID = 0D1B936F-68F3-4589-AFCC-FB8B7AEBC836
 234 #    }
 235 #  Case 2 in FDF:
 236 #    INF Pkg/module/module.inf
 237 #    INF FILE_GUID = 0D1B936F-68F3-4589-AFCC-FB8B7AEBC836 Pkg/module/module.inf
 238 #
 239 #  This function copies Pkg/module/module.inf to
 240 #  Conf/.cache/0D1B936F-68F3-4589-AFCC-FB8B7AEBC836module.inf
 241 #
 242 #  @param Path     Original PathClass object
 243 #  @param BaseName New file base name
 244 #
 245 #  @retval         return the new PathClass object
 246 #
 247 def ProcessDuplicatedInf(Path, BaseName, Workspace):
 248     Filename = os.path.split(Path.File)[1]
 249     if '.' in Filename:
 250         Filename = BaseName + Path.BaseName + Filename[Filename.rfind('.'):]
 251     else:
 252         Filename = BaseName + Path.BaseName
 253
 254     #
 255     # If -N is specified on command line, cache is disabled
 256     # The directory has to be created
 257     #
 258     DbDir = os.path.split(GlobalData.gDatabasePath)[0]
 259     if not os.path.exists(DbDir):
 260         os.makedirs(DbDir)
 261     #
 262     # A temporary INF is copied to database path which must have write permission
 263     # The temporary will be removed at the end of build
 264     # In case of name conflict, the file name is
 265     # FILE_GUIDBaseName (0D1B936F-68F3-4589-AFCC-FB8B7AEBC836module.inf)
 266     #
 267     TempFullPath = os.path.join(DbDir,
 268                                 Filename)
 269     RtPath = PathClass(Path.File, Workspace)
 270     #
 271     # Modify the full path to temporary path, keep other unchanged
 272     #
 273     # To build same module more than once, the module path with FILE_GUID overridden has
 274     # the file name FILE_GUIDmodule.inf, but the relative path (self.MetaFile.File) is the real path
 275     # in DSC which is used as relative path by C files and other files in INF.
 276     # A trick was used: all module paths are PathClass instances, after the initialization
 277     # of PathClass, the PathClass.Path is overridden by the temporary INF path.
 278     #
 279     # The reason for creating a temporary INF is:
 280     # Platform.Modules which is the base to create ModuleAutoGen objects is a dictionary,
 281     # the key is the full path of INF, the value is an object to save overridden library instances, PCDs.
 282     # A different key for the same module is needed to create different output directory,
 283     # retrieve overridden PCDs, library instances.
 284     #
 285     # The BaseName is the FILE_GUID which is also the output directory name.
 286     #
 287     #
 288     RtPath.Path = TempFullPath
 289     RtPath.BaseName = BaseName
 290     #
 291     # If file exists, compare contents
 292     #
 293     if os.path.exists(TempFullPath):
 294         with open(str(Path), 'rb') as f1, open(TempFullPath, 'rb') as f2:
 295             if f1.read() == f2.read():
 296                 return RtPath
 297     _TempInfs.append(TempFullPath)
 298     shutil.copy2(str(Path), TempFullPath)
 299     return RtPath
 300
 301 ## Remove temporary created INFs whose paths were saved in _TempInfs
 302 #
 303 def ClearDuplicatedInf():
 304     while _TempInfs:
 305         File = _TempInfs.pop()
 306         if os.path.exists(File):
 307             os.remove(File)
 308
 309 ## Convert GUID string in xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx style to C structure style
 310 #
 311 #   @param      Guid    The GUID string
 312 #
 313 #   @retval     string  The GUID string in C structure style
 314 #
 315 def GuidStringToGuidStructureString(Guid):
 316     GuidList = Guid.split('-')
 317     Result = '{'
 318     for Index in range(0, 3, 1):
 319         Result = Result + '0x' + GuidList[Index] + ', '
 320     Result = Result + '{0x' + GuidList[3][0:2] + ', 0x' + GuidList[3][2:4]
 321     for Index in range(0, 12, 2):
 322         Result = Result + ', 0x' + GuidList[4][Index:Index + 2]
 323     Result += '}}'
 324     return Result
 325
 326 ## Convert GUID structure in byte array to xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
 327 #
 328 #   @param      GuidValue   The GUID value in byte array
 329 #
 330 #   @retval     string      The GUID value in xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx format
 331 #
 332 def GuidStructureByteArrayToGuidString(GuidValue):
 333     guidValueString = GuidValue.lower().replace("{", "").replace("}", "").replace(" ", "").replace(";", "")
 334     guidValueList = guidValueString.split(",")
 335     if len(guidValueList) != 16:
 336         return ''
 337         #EdkLogger.error(None, None, "Invalid GUID value string %s" % GuidValue)
 338     try:
 339         return "%02x%02x%02x%02x-%02x%02x-%02x%02x-%02x%02x-%02x%02x%02x%02x%02x%02x" % (
 340                 int(guidValueList[3], 16),
 341                 int(guidValueList[2], 16),
 342                 int(guidValueList[1], 16),
 343                 int(guidValueList[0], 16),
 344                 int(guidValueList[5], 16),
 345                 int(guidValueList[4], 16),
 346                 int(guidValueList[7], 16),
 347                 int(guidValueList[6], 16),
 348                 int(guidValueList[8], 16),
 349                 int(guidValueList[9], 16),
 350                 int(guidValueList[10], 16),
 351                 int(guidValueList[11], 16),
 352                 int(guidValueList[12], 16),
 353                 int(guidValueList[13], 16),
 354                 int(guidValueList[14], 16),
 355                 int(guidValueList[15], 16)
 356                 )
 357     except:
 358         return ''
 359
 360 ## Convert GUID string in C structure style to xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
 361 #
 362 #   @param      GuidValue   The GUID value in C structure format
 363 #
 364 #   @retval     string      The GUID value in xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx format
 365 #
 366 def GuidStructureStringToGuidString(GuidValue):
 367     if not GlobalData.gGuidCFormatPattern.match(GuidValue):
 368         return ''
 369     guidValueString = GuidValue.lower().replace("{", "").replace("}", "").replace(" ", "").replace(";", "")
 370     guidValueList = guidValueString.split(",")
 371     if len(guidValueList) != 11:
 372         return ''
 373         #EdkLogger.error(None, None, "Invalid GUID value string %s" % GuidValue)
 374     try:
 375         return "%08x-%04x-%04x-%02x%02x-%02x%02x%02x%02x%02x%02x" % (
 376                 int(guidValueList[0], 16),
 377                 int(guidValueList[1], 16),
 378                 int(guidValueList[2], 16),
 379                 int(guidValueList[3], 16),
 380                 int(guidValueList[4], 16),
 381                 int(guidValueList[5], 16),
 382                 int(guidValueList[6], 16),
 383                 int(guidValueList[7], 16),
 384                 int(guidValueList[8], 16),
 385                 int(guidValueList[9], 16),
 386                 int(guidValueList[10], 16)
 387                 )
 388     except:
 389         return ''
 390
 391 ## Convert GUID string in C structure style to xxxxxxxx_xxxx_xxxx_xxxx_xxxxxxxxxxxx
 392 #
 393 #   @param      GuidValue   The GUID value in C structure format
 394 #
 395 #   @retval     string      The GUID value in xxxxxxxx_xxxx_xxxx_xxxx_xxxxxxxxxxxx format
 396 #
 397 def GuidStructureStringToGuidValueName(GuidValue):
 398     guidValueString = GuidValue.lower().replace("{", "").replace("}", "").replace(" ", "")
 399     guidValueList = guidValueString.split(",")
 400     if len(guidValueList) != 11:
 401         EdkLogger.error(None, FORMAT_INVALID, "Invalid GUID value string [%s]" % GuidValue)
 402     return "%08x_%04x_%04x_%02x%02x_%02x%02x%02x%02x%02x%02x" % (
 403             int(guidValueList[0], 16),
 404             int(guidValueList[1], 16),
 405             int(guidValueList[2], 16),
 406             int(guidValueList[3], 16),
 407             int(guidValueList[4], 16),
 408             int(guidValueList[5], 16),
 409             int(guidValueList[6], 16),
 410             int(guidValueList[7], 16),
 411             int(guidValueList[8], 16),
 412             int(guidValueList[9], 16),
 413             int(guidValueList[10], 16)
 414             )
 415
 416 ## Create directories
 417 #
 418 #   @param      Directory   The directory name
 419 #
 420 def CreateDirectory(Directory):
 421     if Directory is None or Directory.strip() == "":
 422         return True
 423     try:
 424         if not os.access(Directory, os.F_OK):
 425             os.makedirs(Directory)
 426     except:
 427         return False
 428     return True
 429
 430 ## Remove directories, including files and sub-directories in it
 431 #
 432 #   @param      Directory   The directory name
 433 #
 434 def RemoveDirectory(Directory, Recursively=False):
 435     if Directory is None or Directory.strip() == "" or not os.path.exists(Directory):
 436         return
 437     if Recursively:
 438         CurrentDirectory = os.getcwd()
 439         os.chdir(Directory)
 440         for File in os.listdir("."):
 441             if os.path.isdir(File):
 442                 RemoveDirectory(File, Recursively)
 443             else:
 444                 os.remove(File)
 445         os.chdir(CurrentDirectory)
 446     os.rmdir(Directory)
 447
 448 ## Store content in file
 449 #
 450 #  This method is used to save file only when its content is changed. This is
 451 #  quite useful for "make" system to decide what will be re-built and what won't.
 452 #
 453 #   @param      File            The path of file
 454 #   @param      Content         The new content of the file
 455 #   @param      IsBinaryFile    The flag indicating if the file is binary file or not
 456 #
 457 #   @retval     True            If the file content is changed and the file is renewed
 458 #   @retval     False           If the file content is the same
 459 #
 460 def SaveFileOnChange(File, Content, IsBinaryFile=True):
 461     if not IsBinaryFile:
 462         Content = Content.replace("\n", os.linesep)
 463
 464     if os.path.exists(File):
 465         try:
 466             if Content == open(File, "rb").read():
 467                 return False
 468         except:
 469             EdkLogger.error(None, FILE_OPEN_FAILURE, ExtraData=File)
 470
 471     DirName = os.path.dirname(File)
 472     if not CreateDirectory(DirName):
 473         EdkLogger.error(None, FILE_CREATE_FAILURE, "Could not create directory %s" % DirName)
 474     else:
 475         if DirName == '':
 476             DirName = os.getcwd()
 477         if not os.access(DirName, os.W_OK):
 478             EdkLogger.error(None, PERMISSION_FAILURE, "Do not have write permission on directory %s" % DirName)
 479
 480     try:
 481         Fd = open(File, "wb")
 482         Fd.write(Content)
 483         Fd.close()
 484     except IOError as X:
 485         EdkLogger.error(None, FILE_CREATE_FAILURE, ExtraData='IOError %s' % X)
 486
 487     return True
 488
 489 ## Make a Python object persistent on file system
 490 #
 491 #   @param      Data    The object to be stored in file
 492 #   @param      File    The path of file to store the object
 493 #
 494 def DataDump(Data, File):
 495     Fd = None
 496     try:
 497         Fd = open(File, 'wb')
 498         pickle.dump(Data, Fd, pickle.HIGHEST_PROTOCOL)
 499     except:
 500         EdkLogger.error("", FILE_OPEN_FAILURE, ExtraData=File, RaiseError=False)
 501     finally:
 502         if Fd is not None:
 503             Fd.close()
 504
 505 ## Restore a Python object from a file
 506 #
 507 #   @param      File    The path of file stored the object
 508 #
 509 #   @retval     object  A python object
 510 #   @retval     None    If failure in file operation
 511 #
 512 def DataRestore(File):
 513     Data = None
 514     Fd = None
 515     try:
 516         Fd = open(File, 'rb')
 517         Data = pickle.load(Fd)
 518     except Exception as e:
 519         EdkLogger.verbose("Failed to load [%s]\n\t%s" % (File, str(e)))
 520         Data = None
 521     finally:
 522         if Fd is not None:
 523             Fd.close()
 524     return Data
 525
 526 ## Retrieve and cache the real path name in file system
 527 #
 528 #   @param      Root    The root directory of path relative to
 529 #
 530 #   @retval     str     The path string if the path exists
 531 #   @retval     None    If path doesn't exist
 532 #
 533 class DirCache:
 534     _CACHE_ = set()
 535     _UPPER_CACHE_ = {}
 536
 537     def __init__(self, Root):
 538         self._Root = Root
 539         for F in os.listdir(Root):
 540             self._CACHE_.add(F)
 541             self._UPPER_CACHE_[F.upper()] = F
 542
 543     # =[] operator
 544     def __getitem__(self, Path):
 545         Path = Path[len(os.path.commonprefix([Path, self._Root])):]
 546         if not Path:
 547             return self._Root
 548         if Path and Path[0] == os.path.sep:
 549             Path = Path[1:]
 550         if Path in self._CACHE_:
 551             return os.path.join(self._Root, Path)
 552         UpperPath = Path.upper()
 553         if UpperPath in self._UPPER_CACHE_:
 554             return os.path.join(self._Root, self._UPPER_CACHE_[UpperPath])
 555
 556         IndexList = []
 557         LastSepIndex = -1
 558         SepIndex = Path.find(os.path.sep)
 559         while SepIndex > -1:
 560             Parent = UpperPath[:SepIndex]
 561             if Parent not in self._UPPER_CACHE_:
 562                 break
 563             LastSepIndex = SepIndex
 564             SepIndex = Path.find(os.path.sep, LastSepIndex + 1)
 565
 566         if LastSepIndex == -1:
 567             return None
 568
 569         Cwd = os.getcwd()
 570         os.chdir(self._Root)
 571         SepIndex = LastSepIndex
 572         while SepIndex > -1:
 573             Parent = Path[:SepIndex]
 574             ParentKey = UpperPath[:SepIndex]
 575             if ParentKey not in self._UPPER_CACHE_:
 576                 os.chdir(Cwd)
 577                 return None
 578
 579             if Parent in self._CACHE_:
 580                 ParentDir = Parent
 581             else:
 582                 ParentDir = self._UPPER_CACHE_[ParentKey]
 583             for F in os.listdir(ParentDir):
 584                 Dir = os.path.join(ParentDir, F)
 585                 self._CACHE_.add(Dir)
 586                 self._UPPER_CACHE_[Dir.upper()] = Dir
 587
 588             SepIndex = Path.find(os.path.sep, SepIndex + 1)
 589
 590         os.chdir(Cwd)
 591         if Path in self._CACHE_:
 592             return os.path.join(self._Root, Path)
 593         elif UpperPath in self._UPPER_CACHE_:
 594             return os.path.join(self._Root, self._UPPER_CACHE_[UpperPath])
 595         return None
 596
 597 def RealPath(File, Dir='', OverrideDir=''):
 598     NewFile = os.path.normpath(os.path.join(Dir, File))
 599     NewFile = GlobalData.gAllFiles[NewFile]
 600     if not NewFile and OverrideDir:
 601         NewFile = os.path.normpath(os.path.join(OverrideDir, File))
 602         NewFile = GlobalData.gAllFiles[NewFile]
 603     return NewFile
 604
 605 def RealPath2(File, Dir='', OverrideDir=''):
 606     NewFile = None
 607     if OverrideDir:
 608         NewFile = GlobalData.gAllFiles[os.path.normpath(os.path.join(OverrideDir, File))]
 609         if NewFile:
 610             if OverrideDir[-1] == os.path.sep:
 611                 return NewFile[len(OverrideDir):], NewFile[0:len(OverrideDir)]
 612             else:
 613                 return NewFile[len(OverrideDir) + 1:], NewFile[0:len(OverrideDir)]
 614     if GlobalData.gAllFiles:
 615         NewFile = GlobalData.gAllFiles[os.path.normpath(os.path.join(Dir, File))]
 616     if not NewFile:
 617         NewFile = os.path.normpath(os.path.join(Dir, File))
 618         if not os.path.exists(NewFile):
 619             return None, None
 620     if NewFile:
 621         if Dir:
 622             if Dir[-1] == os.path.sep:
 623                 return NewFile[len(Dir):], NewFile[0:len(Dir)]
 624             else:
 625                 return NewFile[len(Dir) + 1:], NewFile[0:len(Dir)]
 626         else:
 627             return NewFile, ''
 628
 629     return None, None
 630
 631 ## Get GUID value from given packages
 632 #
 633 #   @param      CName           The CName of the GUID
 634 #   @param      PackageList     List of packages looking-up in
 635 #   @param      Inffile         The driver file
 636 #
 637 #   @retval     GuidValue   if the CName is found in any given package
 638 #   @retval     None        if the CName is not found in all given packages
 639 #
 640 def GuidValue(CName, PackageList, Inffile = None):
 641     for P in PackageList:
 642         GuidKeys = P.Guids.keys()
 643         if Inffile and P._PrivateGuids:
 644             if not Inffile.startswith(P.MetaFile.Dir):
 645                 GuidKeys = [x for x in P.Guids if x not in P._PrivateGuids]
 646         if CName in GuidKeys:
 647             return P.Guids[CName]
 648     return None
 649
 650 ## Get Protocol value from given packages
 651 #
 652 #   @param      CName           The CName of the GUID
 653 #   @param      PackageList     List of packages looking-up in
 654 #   @param      Inffile         The driver file
 655 #
 656 #   @retval     GuidValue   if the CName is found in any given package
 657 #   @retval     None        if the CName is not found in all given packages
 658 #
 659 def ProtocolValue(CName, PackageList, Inffile = None):
 660     for P in PackageList:
 661         ProtocolKeys = P.Protocols.keys()
 662         if Inffile and P._PrivateProtocols:
 663             if not Inffile.startswith(P.MetaFile.Dir):
 664                 ProtocolKeys = [x for x in P.Protocols if x not in P._PrivateProtocols]
 665         if CName in ProtocolKeys:
 666             return P.Protocols[CName]
 667     return None
 668
 669 ## Get PPI value from given packages
 670 #
 671 #   @param      CName           The CName of the GUID
 672 #   @param      PackageList     List of packages looking-up in
 673 #   @param      Inffile         The driver file
 674 #
 675 #   @retval     GuidValue   if the CName is found in any given package
 676 #   @retval     None        if the CName is not found in all given packages
 677 #
 678 def PpiValue(CName, PackageList, Inffile = None):
 679     for P in PackageList:
 680         PpiKeys = P.Ppis.keys()
 681         if Inffile and P._PrivatePpis:
 682             if not Inffile.startswith(P.MetaFile.Dir):
 683                 PpiKeys = [x for x in P.Ppis if x not in P._PrivatePpis]
 684         if CName in PpiKeys:
 685             return P.Ppis[CName]
 686     return None
 687
 688 ## A string template class
 689 #
 690 #  This class implements a template for string replacement. A string template
 691 #  looks like following
 692 #
 693 #       ${BEGIN} other_string ${placeholder_name} other_string ${END}
 694 #
 695 #  The string between ${BEGIN} and ${END} will be repeated as many times as the
 696 #  length of "placeholder_name", which is a list passed through a dict. The
 697 #  "placeholder_name" is the key name of the dict. The ${BEGIN} and ${END} can
 698 #  be not used and, in this case, the "placeholder_name" must not a list and it
 699 #  will just be replaced once.
 700 #
 701 class TemplateString(object):
 702     _REPEAT_START_FLAG = "BEGIN"
 703     _REPEAT_END_FLAG = "END"
 704
 705     class Section(object):
 706         _LIST_TYPES = [type([]), type(set()), type((0,))]
 707
 708         def __init__(self, TemplateSection, PlaceHolderList):
 709             self._Template = TemplateSection
 710             self._PlaceHolderList = []
 711
 712             # Split the section into sub-sections according to the position of placeholders
 713             if PlaceHolderList:
 714                 self._SubSectionList = []
 715                 SubSectionStart = 0
 716                 #
 717                 # The placeholders passed in must be in the format of
 718                 #
 719                 #   PlaceHolderName, PlaceHolderStartPoint, PlaceHolderEndPoint
 720                 #
 721                 for PlaceHolder, Start, End in PlaceHolderList:
 722                     self._SubSectionList.append(TemplateSection[SubSectionStart:Start])
 723                     self._SubSectionList.append(TemplateSection[Start:End])
 724                     self._PlaceHolderList.append(PlaceHolder)
 725                     SubSectionStart = End
 726                 if SubSectionStart < len(TemplateSection):
 727                     self._SubSectionList.append(TemplateSection[SubSectionStart:])
 728             else:
 729                 self._SubSectionList = [TemplateSection]
 730
 731         def __str__(self):
 732             return self._Template + " : " + str(self._PlaceHolderList)
 733
 734         def Instantiate(self, PlaceHolderValues):
 735             RepeatTime = -1
 736             RepeatPlaceHolders = {}
 737             NonRepeatPlaceHolders = {}
 738
 739             for PlaceHolder in self._PlaceHolderList:
 740                 if PlaceHolder not in PlaceHolderValues:
 741                     continue
 742                 Value = PlaceHolderValues[PlaceHolder]
 743                 if type(Value) in self._LIST_TYPES:
 744                     if RepeatTime < 0:
 745                         RepeatTime = len(Value)
 746                     elif RepeatTime != len(Value):
 747                         EdkLogger.error(
 748                                     "TemplateString",
 749                                     PARAMETER_INVALID,
 750                                     "${%s} has different repeat time from others!" % PlaceHolder,
 751                                     ExtraData=str(self._Template)
 752                                     )
 753                     RepeatPlaceHolders["${%s}" % PlaceHolder] = Value
 754                 else:
 755                     NonRepeatPlaceHolders["${%s}" % PlaceHolder] = Value
 756
 757             if NonRepeatPlaceHolders:
 758                 StringList = []
 759                 for S in self._SubSectionList:
 760                     if S not in NonRepeatPlaceHolders:
 761                         StringList.append(S)
 762                     else:
 763                         StringList.append(str(NonRepeatPlaceHolders[S]))
 764             else:
 765                 StringList = self._SubSectionList
 766
 767             if RepeatPlaceHolders:
 768                 TempStringList = []
 769                 for Index in range(RepeatTime):
 770                     for S in StringList:
 771                         if S not in RepeatPlaceHolders:
 772                             TempStringList.append(S)
 773                         else:
 774                             TempStringList.append(str(RepeatPlaceHolders[S][Index]))
 775                 StringList = TempStringList
 776
 777             return "".join(StringList)
 778
 779     ## Constructor
 780     def __init__(self, Template=None):
 781         self.String = ''
 782         self.IsBinary = False
 783         self._Template = Template
 784         self._TemplateSectionList = self._Parse(Template)
 785
 786     ## str() operator
 787     #
 788     #   @retval     string  The string replaced
 789     #
 790     def __str__(self):
 791         return self.String
 792
 793     ## Split the template string into fragments per the ${BEGIN} and ${END} flags
 794     #
 795     #   @retval     list    A list of TemplateString.Section objects
 796     #
 797     def _Parse(self, Template):
 798         SectionStart = 0
 799         SearchFrom = 0
 800         MatchEnd = 0
 801         PlaceHolderList = []
 802         TemplateSectionList = []
 803         while Template:
 804             MatchObj = gPlaceholderPattern.search(Template, SearchFrom)
 805             if not MatchObj:
 806                 if MatchEnd <= len(Template):
 807                     TemplateSection = TemplateString.Section(Template[SectionStart:], PlaceHolderList)
 808                     TemplateSectionList.append(TemplateSection)
 809                 break
 810
 811             MatchString = MatchObj.group(1)
 812             MatchStart = MatchObj.start()
 813             MatchEnd = MatchObj.end()
 814
 815             if MatchString == self._REPEAT_START_FLAG:
 816                 if MatchStart > SectionStart:
 817                     TemplateSection = TemplateString.Section(Template[SectionStart:MatchStart], PlaceHolderList)
 818                     TemplateSectionList.append(TemplateSection)
 819                 SectionStart = MatchEnd
 820                 PlaceHolderList = []
 821             elif MatchString == self._REPEAT_END_FLAG:
 822                 TemplateSection = TemplateString.Section(Template[SectionStart:MatchStart], PlaceHolderList)
 823                 TemplateSectionList.append(TemplateSection)
 824                 SectionStart = MatchEnd
 825                 PlaceHolderList = []
 826             else:
 827                 PlaceHolderList.append((MatchString, MatchStart - SectionStart, MatchEnd - SectionStart))
 828             SearchFrom = MatchEnd
 829         return TemplateSectionList
 830
 831     ## Replace the string template with dictionary of placeholders and append it to previous one
 832     #
 833     #   @param      AppendString    The string template to append
 834     #   @param      Dictionary      The placeholder dictionaries
 835     #
 836     def Append(self, AppendString, Dictionary=None):
 837         if Dictionary:
 838             SectionList = self._Parse(AppendString)
 839             self.String += "".join(S.Instantiate(Dictionary) for S in SectionList)
 840         else:
 841             self.String += AppendString
 842
 843     ## Replace the string template with dictionary of placeholders
 844     #
 845     #   @param      Dictionary      The placeholder dictionaries
 846     #
 847     #   @retval     str             The string replaced with placeholder values
 848     #
 849     def Replace(self, Dictionary=None):
 850         return "".join(S.Instantiate(Dictionary) for S in self._TemplateSectionList)
 851
 852 ## Progress indicator class
 853 #
 854 #  This class makes use of thread to print progress on console.
 855 #
 856 class Progressor:
 857     # for avoiding deadloop
 858     _StopFlag = None
 859     _ProgressThread = None
 860     _CheckInterval = 0.25
 861
 862     ## Constructor
 863     #
 864     #   @param      OpenMessage     The string printed before progress charaters
 865     #   @param      CloseMessage    The string printed after progress charaters
 866     #   @param      ProgressChar    The charater used to indicate the progress
 867     #   @param      Interval        The interval in seconds between two progress charaters
 868     #
 869     def __init__(self, OpenMessage="", CloseMessage="", ProgressChar='.', Interval=1.0):
 870         self.PromptMessage = OpenMessage
 871         self.CodaMessage = CloseMessage
 872         self.ProgressChar = ProgressChar
 873         self.Interval = Interval
 874         if Progressor._StopFlag is None:
 875             Progressor._StopFlag = threading.Event()
 876
 877     ## Start to print progress charater
 878     #
 879     #   @param      OpenMessage     The string printed before progress charaters
 880     #
 881     def Start(self, OpenMessage=None):
 882         if OpenMessage is not None:
 883             self.PromptMessage = OpenMessage
 884         Progressor._StopFlag.clear()
 885         if Progressor._ProgressThread is None:
 886             Progressor._ProgressThread = threading.Thread(target=self._ProgressThreadEntry)
 887             Progressor._ProgressThread.setDaemon(False)
 888             Progressor._ProgressThread.start()
 889
 890     ## Stop printing progress charater
 891     #
 892     #   @param      CloseMessage    The string printed after progress charaters
 893     #
 894     def Stop(self, CloseMessage=None):
 895         OriginalCodaMessage = self.CodaMessage
 896         if CloseMessage is not None:
 897             self.CodaMessage = CloseMessage
 898         self.Abort()
 899         self.CodaMessage = OriginalCodaMessage
 900
 901     ## Thread entry method
 902     def _ProgressThreadEntry(self):
 903         sys.stdout.write(self.PromptMessage + " ")
 904         sys.stdout.flush()
 905         TimeUp = 0.0
 906         while not Progressor._StopFlag.isSet():
 907             if TimeUp <= 0.0:
 908                 sys.stdout.write(self.ProgressChar)
 909                 sys.stdout.flush()
 910                 TimeUp = self.Interval
 911             time.sleep(self._CheckInterval)
 912             TimeUp -= self._CheckInterval
 913         sys.stdout.write(" " + self.CodaMessage + "\n")
 914         sys.stdout.flush()
 915
 916     ## Abort the progress display
 917     @staticmethod
 918     def Abort():
 919         if Progressor._StopFlag is not None:
 920             Progressor._StopFlag.set()
 921         if Progressor._ProgressThread is not None:
 922             Progressor._ProgressThread.join()
 923             Progressor._ProgressThread = None
 924
 925 ## A dict which can access its keys and/or values orderly
 926 #
 927 #  The class implements a new kind of dict which its keys or values can be
 928 #  accessed in the order they are added into the dict. It guarantees the order
 929 #  by making use of an internal list to keep a copy of keys.
 930 #
 931 class sdict(IterableUserDict):
 932     ## Constructor
 933     def __init__(self):
 934         IterableUserDict.__init__(self)
 935         self._key_list = []
 936
 937     ## [] operator
 938     def __setitem__(self, key, value):
 939         if key not in self._key_list:
 940             self._key_list.append(key)
 941         IterableUserDict.__setitem__(self, key, value)
 942
 943     ## del operator
 944     def __delitem__(self, key):
 945         self._key_list.remove(key)
 946         IterableUserDict.__delitem__(self, key)
 947
 948     ## used in "for k in dict" loop to ensure the correct order
 949     def __iter__(self):
 950         return self.iterkeys()
 951
 952     ## len() support
 953     def __len__(self):
 954         return len(self._key_list)
 955
 956     ## "in" test support
 957     def __contains__(self, key):
 958         return key in self._key_list
 959
 960     ## indexof support
 961     def index(self, key):
 962         return self._key_list.index(key)
 963
 964     ## insert support
 965     def insert(self, key, newkey, newvalue, order):
 966         index = self._key_list.index(key)
 967         if order == 'BEFORE':
 968             self._key_list.insert(index, newkey)
 969             IterableUserDict.__setitem__(self, newkey, newvalue)
 970         elif order == 'AFTER':
 971             self._key_list.insert(index + 1, newkey)
 972             IterableUserDict.__setitem__(self, newkey, newvalue)
 973
 974     ## append support
 975     def append(self, sdict):
 976         for key in sdict:
 977             if key not in self._key_list:
 978                 self._key_list.append(key)
 979             IterableUserDict.__setitem__(self, key, sdict[key])
 980
 981     def has_key(self, key):
 982         return key in self._key_list
 983
 984     ## Empty the dict
 985     def clear(self):
 986         self._key_list = []
 987         IterableUserDict.clear(self)
 988
 989     ## Return a copy of keys
 990     def keys(self):
 991         keys = []
 992         for key in self._key_list:
 993             keys.append(key)
 994         return keys
 995
 996     ## Return a copy of values
 997     def values(self):
 998         values = []
 999         for key in self._key_list:
1000             values.append(self[key])
1001         return values
1002
1003     ## Return a copy of (key, value) list
1004     def items(self):
1005         items = []
1006         for key in self._key_list:
1007             items.append((key, self[key]))
1008         return items
1009
1010     ## Iteration support
1011     def iteritems(self):
1012         return iter(self.items())
1013
1014     ## Keys interation support
1015     def iterkeys(self):
1016         return iter(self.keys())
1017
1018     ## Values interation support
1019     def itervalues(self):
1020         return iter(self.values())
1021
1022     ## Return value related to a key, and remove the (key, value) from the dict
1023     def pop(self, key, *dv):
1024         value = None
1025         if key in self._key_list:
1026             value = self[key]
1027             self.__delitem__(key)
1028         elif len(dv) != 0 :
1029             value = kv[0]
1030         return value
1031
1032     ## Return (key, value) pair, and remove the (key, value) from the dict
1033     def popitem(self):
1034         key = self._key_list[-1]
1035         value = self[key]
1036         self.__delitem__(key)
1037         return key, value
1038
1039     def update(self, dict=None, **kwargs):
1040         if dict is not None:
1041             for k, v in dict.items():
1042                 self[k] = v
1043         if len(kwargs):
1044             for k, v in kwargs.items():
1045                 self[k] = v
1046
1047 ## Dictionary with restricted keys
1048 #
1049 class rdict(dict):
1050     ## Constructor
1051     def __init__(self, KeyList):
1052         for Key in KeyList:
1053             dict.__setitem__(self, Key, "")
1054
1055     ## []= operator
1056     def __setitem__(self, key, value):
1057         if key not in self:
1058             EdkLogger.error("RestrictedDict", ATTRIBUTE_SET_FAILURE, "Key [%s] is not allowed" % key,
1059                             ExtraData=", ".join(dict.keys(self)))
1060         dict.__setitem__(self, key, value)
1061
1062     ## =[] operator
1063     def __getitem__(self, key):
1064         if key not in self:
1065             return ""
1066         return dict.__getitem__(self, key)
1067
1068     ## del operator
1069     def __delitem__(self, key):
1070         EdkLogger.error("RestrictedDict", ATTRIBUTE_ACCESS_DENIED, ExtraData="del")
1071
1072     ## Empty the dict
1073     def clear(self):
1074         for Key in self:
1075             self.__setitem__(Key, "")
1076
1077     ## Return value related to a key, and remove the (key, value) from the dict
1078     def pop(self, key, *dv):
1079         EdkLogger.error("RestrictedDict", ATTRIBUTE_ACCESS_DENIED, ExtraData="pop")
1080
1081     ## Return (key, value) pair, and remove the (key, value) from the dict
1082     def popitem(self):
1083         EdkLogger.error("RestrictedDict", ATTRIBUTE_ACCESS_DENIED, ExtraData="popitem")
1084
1085 ## Dictionary using prioritized list as key
1086 #
1087 class tdict:
1088     _ListType = type([])
1089     _TupleType = type(())
1090     _Wildcard = 'COMMON'
1091     _ValidWildcardList = ['COMMON', 'DEFAULT', 'ALL', TAB_STAR, 'PLATFORM']
1092
1093     def __init__(self, _Single_=False, _Level_=2):
1094         self._Level_ = _Level_
1095         self.data = {}
1096         self._Single_ = _Single_
1097
1098     # =[] operator
1099     def __getitem__(self, key):
1100         KeyType = type(key)
1101         RestKeys = None
1102         if KeyType == self._ListType or KeyType == self._TupleType:
1103             FirstKey = key[0]
1104             if len(key) > 1:
1105                 RestKeys = key[1:]
1106             elif self._Level_ > 1:
1107                 RestKeys = [self._Wildcard for i in range(0, self._Level_ - 1)]
1108         else:
1109             FirstKey = key
1110             if self._Level_ > 1:
1111                 RestKeys = [self._Wildcard for i in range(0, self._Level_ - 1)]
1112
1113         if FirstKey is None or str(FirstKey).upper() in self._ValidWildcardList:
1114             FirstKey = self._Wildcard
1115
1116         if self._Single_:
1117             return self._GetSingleValue(FirstKey, RestKeys)
1118         else:
1119             return self._GetAllValues(FirstKey, RestKeys)
1120
1121     def _GetSingleValue(self, FirstKey, RestKeys):
1122         Value = None
1123         #print "%s-%s" % (FirstKey, self._Level_) ,
1124         if self._Level_ > 1:
1125             if FirstKey == self._Wildcard:
1126                 if FirstKey in self.data:
1127                     Value = self.data[FirstKey][RestKeys]
1128                 if Value is None:
1129                     for Key in self.data:
1130                         Value = self.data[Key][RestKeys]
1131                         if Value is not None: break
1132             else:
1133                 if FirstKey in self.data:
1134                     Value = self.data[FirstKey][RestKeys]
1135                 if Value is None and self._Wildcard in self.data:
1136                     #print "Value=None"
1137                     Value = self.data[self._Wildcard][RestKeys]
1138         else:
1139             if FirstKey == self._Wildcard:
1140                 if FirstKey in self.data:
1141                     Value = self.data[FirstKey]
1142                 if Value is None:
1143                     for Key in self.data:
1144                         Value = self.data[Key]
1145                         if Value is not None: break
1146             else:
1147                 if FirstKey in self.data:
1148                     Value = self.data[FirstKey]
1149                 elif self._Wildcard in self.data:
1150                     Value = self.data[self._Wildcard]
1151         return Value
1152
1153     def _GetAllValues(self, FirstKey, RestKeys):
1154         Value = []
1155         if self._Level_ > 1:
1156             if FirstKey == self._Wildcard:
1157                 for Key in self.data:
1158                     Value += self.data[Key][RestKeys]
1159             else:
1160                 if FirstKey in self.data:
1161                     Value += self.data[FirstKey][RestKeys]
1162                 if self._Wildcard in self.data:
1163                     Value += self.data[self._Wildcard][RestKeys]
1164         else:
1165             if FirstKey == self._Wildcard:
1166                 for Key in self.data:
1167                     Value.append(self.data[Key])
1168             else:
1169                 if FirstKey in self.data:
1170                     Value.append(self.data[FirstKey])
1171                 if self._Wildcard in self.data:
1172                     Value.append(self.data[self._Wildcard])
1173         return Value
1174
1175     ## []= operator
1176     def __setitem__(self, key, value):
1177         KeyType = type(key)
1178         RestKeys = None
1179         if KeyType == self._ListType or KeyType == self._TupleType:
1180             FirstKey = key[0]
1181             if len(key) > 1:
1182                 RestKeys = key[1:]
1183             else:
1184                 RestKeys = [self._Wildcard for i in range(0, self._Level_ - 1)]
1185         else:
1186             FirstKey = key
1187             if self._Level_ > 1:
1188                 RestKeys = [self._Wildcard for i in range(0, self._Level_ - 1)]
1189
1190         if FirstKey in self._ValidWildcardList:
1191             FirstKey = self._Wildcard
1192
1193         if FirstKey not in self.data and self._Level_ > 0:
1194             self.data[FirstKey] = tdict(self._Single_, self._Level_ - 1)
1195
1196         if self._Level_ > 1:
1197             self.data[FirstKey][RestKeys] = value
1198         else:
1199             self.data[FirstKey] = value
1200
1201     def SetGreedyMode(self):
1202         self._Single_ = False
1203         if self._Level_ > 1:
1204             for Key in self.data:
1205                 self.data[Key].SetGreedyMode()
1206
1207     def SetSingleMode(self):
1208         self._Single_ = True
1209         if self._Level_ > 1:
1210             for Key in self.data:
1211                 self.data[Key].SetSingleMode()
1212
1213     def GetKeys(self, KeyIndex=0):
1214         assert KeyIndex >= 0
1215         if KeyIndex == 0:
1216             return set(self.data.keys())
1217         else:
1218             keys = set()
1219             for Key in self.data:
1220                 keys |= self.data[Key].GetKeys(KeyIndex - 1)
1221             return keys
1222
1223 def IsFieldValueAnArray (Value):
1224     Value = Value.strip()
1225     if Value.startswith(TAB_GUID) and Value.endswith(')'):
1226         return True
1227     if Value.startswith('L"') and Value.endswith('"')  and len(list(Value[2:-1])) > 1:
1228         return True
1229     if Value[0] == '"' and Value[-1] == '"' and len(list(Value[1:-1])) > 1:
1230         return True
1231     if Value[0] == '{' and Value[-1] == '}':
1232         return True
1233     if Value.startswith("L'") and Value.endswith("'") and len(list(Value[2:-1])) > 1:
1234         return True
1235     if Value[0] == "'" and Value[-1] == "'" and len(list(Value[1:-1])) > 1:
1236         return True
1237     return False
1238
1239 def AnalyzePcdExpression(Setting):
1240     RanStr = ''.join(sample(string.ascii_letters + string.digits, 8))
1241     Setting = Setting.replace('\\\\', RanStr).strip()
1242     # There might be escaped quote in a string: \", \\\" , \', \\\'
1243     Data = Setting
1244     # There might be '|' in string and in ( ... | ... ), replace it with '-'
1245     NewStr = ''
1246     InSingleQuoteStr = False
1247     InDoubleQuoteStr = False
1248     Pair = 0
1249     for Index, ch in enumerate(Data):
1250         if ch == '"' and not InSingleQuoteStr:
1251             if Data[Index - 1] != '\\':
1252                 InDoubleQuoteStr = not InDoubleQuoteStr
1253         elif ch == "'" and not InDoubleQuoteStr:
1254             if Data[Index - 1] != '\\':
1255                 InSingleQuoteStr = not InSingleQuoteStr
1256         elif ch == '(' and not (InSingleQuoteStr or InDoubleQuoteStr):
1257             Pair += 1
1258         elif ch == ')' and not (InSingleQuoteStr or InDoubleQuoteStr):
1259             Pair -= 1
1260
1261         if (Pair > 0 or InSingleQuoteStr or InDoubleQuoteStr) and ch == TAB_VALUE_SPLIT:
1262             NewStr += '-'
1263         else:
1264             NewStr += ch
1265     FieldList = []
1266     StartPos = 0
1267     while True:
1268         Pos = NewStr.find(TAB_VALUE_SPLIT, StartPos)
1269         if Pos < 0:
1270             FieldList.append(Setting[StartPos:].strip())
1271             break
1272         FieldList.append(Setting[StartPos:Pos].strip())
1273         StartPos = Pos + 1
1274     for i, ch in enumerate(FieldList):
1275         if RanStr in ch:
1276             FieldList[i] = ch.replace(RanStr,'\\\\')
1277     return FieldList
1278
1279 def ParseDevPathValue (Value):
1280     if '\\' in Value:
1281         Value.replace('\\', '/').replace(' ', '')
1282
1283     Cmd = 'DevicePath ' + '"' + Value + '"'
1284     try:
1285         p = subprocess.Popen(Cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
1286         out, err = p.communicate()
1287     except Exception as X:
1288         raise BadExpression("DevicePath: %s" % (str(X)) )
1289     finally:
1290         subprocess._cleanup()
1291         p.stdout.close()
1292         p.stderr.close()
1293     if err:
1294         raise BadExpression("DevicePath: %s" % str(err))
1295     Size = len(out.split())
1296     out = ','.join(out.split())
1297     return '{' + out + '}', Size
1298
1299 def ParseFieldValue (Value):
1300     if "{CODE(" in Value:
1301         return Value, len(Value.split(","))
1302     if isinstance(Value, type(0)):
1303         return Value, (Value.bit_length() + 7) / 8
1304     if not isinstance(Value, type('')):
1305         raise BadExpression('Type %s is %s' %(Value, type(Value)))
1306     Value = Value.strip()
1307     if Value.startswith(TAB_UINT8) and Value.endswith(')'):
1308         Value, Size = ParseFieldValue(Value.split('(', 1)[1][:-1])
1309         if Size > 1:
1310             raise BadExpression('Value (%s) Size larger than %d' %(Value, Size))
1311         return Value, 1
1312     if Value.startswith(TAB_UINT16) and Value.endswith(')'):
1313         Value, Size = ParseFieldValue(Value.split('(', 1)[1][:-1])
1314         if Size > 2:
1315             raise BadExpression('Value (%s) Size larger than %d' %(Value, Size))
1316         return Value, 2
1317     if Value.startswith(TAB_UINT32) and Value.endswith(')'):
1318         Value, Size = ParseFieldValue(Value.split('(', 1)[1][:-1])
1319         if Size > 4:
1320             raise BadExpression('Value (%s) Size larger than %d' %(Value, Size))
1321         return Value, 4
1322     if Value.startswith(TAB_UINT64) and Value.endswith(')'):
1323         Value, Size = ParseFieldValue(Value.split('(', 1)[1][:-1])
1324         if Size > 8:
1325             raise BadExpression('Value (%s) Size larger than %d' % (Value, Size))
1326         return Value, 8
1327     if Value.startswith(TAB_GUID) and Value.endswith(')'):
1328         Value = Value.split('(', 1)[1][:-1].strip()
1329         if Value[0] == '{' and Value[-1] == '}':
1330             TmpValue = GuidStructureStringToGuidString(Value)
1331             if not TmpValue:
1332                 raise BadExpression("Invalid GUID value string %s" % Value)
1333             Value = TmpValue
1334         if Value[0] == '"' and Value[-1] == '"':
1335             Value = Value[1:-1]
1336         try:
1337             Value = "'" + uuid.UUID(Value).get_bytes_le() + "'"
1338         except ValueError as Message:
1339             raise BadExpression(Message)
1340         Value, Size = ParseFieldValue(Value)
1341         return Value, 16
1342     if Value.startswith('L"') and Value.endswith('"'):
1343         # Unicode String
1344         # translate escape character
1345         Value = Value[1:]
1346         try:
1347             Value = eval(Value)
1348         except:
1349             Value = Value[1:-1]
1350         List = list(Value)
1351         List.reverse()
1352         Value = 0
1353         for Char in List:
1354             Value = (Value << 16) | ord(Char)
1355         return Value, (len(List) + 1) * 2
1356     if Value.startswith('"') and Value.endswith('"'):
1357         # ASCII String
1358         # translate escape character
1359         try:
1360             Value = eval(Value)
1361         except:
1362             Value = Value[1:-1]
1363         List = list(Value)
1364         List.reverse()
1365         Value = 0
1366         for Char in List:
1367             Value = (Value << 8) | ord(Char)
1368         return Value, len(List) + 1
1369     if Value.startswith("L'") and Value.endswith("'"):
1370         # Unicode Character Constant
1371         # translate escape character
1372         Value = Value[1:]
1373         try:
1374             Value = eval(Value)
1375         except:
1376             Value = Value[1:-1]
1377         List = list(Value)
1378         if len(List) == 0:
1379             raise BadExpression('Length %s is %s' % (Value, len(List)))
1380         List.reverse()
1381         Value = 0
1382         for Char in List:
1383             Value = (Value << 16) | ord(Char)
1384         return Value, len(List) * 2
1385     if Value.startswith("'") and Value.endswith("'"):
1386         # Character constant
1387         # translate escape character
1388         try:
1389             Value = eval(Value)
1390         except:
1391             Value = Value[1:-1]
1392         List = list(Value)
1393         if len(List) == 0:
1394             raise BadExpression('Length %s is %s' % (Value, len(List)))
1395         List.reverse()
1396         Value = 0
1397         for Char in List:
1398             Value = (Value << 8) | ord(Char)
1399         return Value, len(List)
1400     if Value.startswith('{') and Value.endswith('}'):
1401         # Byte array
1402         Value = Value[1:-1]
1403         List = [Item.strip() for Item in Value.split(',')]
1404         List.reverse()
1405         Value = 0
1406         RetSize = 0
1407         for Item in List:
1408             ItemValue, Size = ParseFieldValue(Item)
1409             RetSize += Size
1410             for I in range(Size):
1411                 Value = (Value << 8) | ((ItemValue >> 8 * I) & 0xff)
1412         return Value, RetSize
1413     if Value.startswith('DEVICE_PATH(') and Value.endswith(')'):
1414         Value = Value.replace("DEVICE_PATH(", '').rstrip(')')
1415         Value = Value.strip().strip('"')
1416         return ParseDevPathValue(Value)
1417     if Value.lower().startswith('0x'):
1418         try:
1419             Value = int(Value, 16)
1420         except:
1421             raise BadExpression("invalid hex value: %s" % Value)
1422         if Value == 0:
1423             return 0, 1
1424         return Value, (Value.bit_length() + 7) / 8
1425     if Value[0].isdigit():
1426         Value = int(Value, 10)
1427         if Value == 0:
1428             return 0, 1
1429         return Value, (Value.bit_length() + 7) / 8
1430     if Value.lower() == 'true':
1431         return 1, 1
1432     if Value.lower() == 'false':
1433         return 0, 1
1434     return Value, 1
1435
1436 ## AnalyzeDscPcd
1437 #
1438 #  Analyze DSC PCD value, since there is no data type info in DSC
1439 #  This function is used to match functions (AnalyzePcdData) used for retrieving PCD value from database
1440 #  1. Feature flag: TokenSpace.PcdCName|PcdValue
1441 #  2. Fix and Patch:TokenSpace.PcdCName|PcdValue[|VOID*[|MaxSize]]
1442 #  3. Dynamic default:
1443 #     TokenSpace.PcdCName|PcdValue[|VOID*[|MaxSize]]
1444 #     TokenSpace.PcdCName|PcdValue
1445 #  4. Dynamic VPD:
1446 #     TokenSpace.PcdCName|VpdOffset[|VpdValue]
1447 #     TokenSpace.PcdCName|VpdOffset[|MaxSize[|VpdValue]]
1448 #  5. Dynamic HII:
1449 #     TokenSpace.PcdCName|HiiString|VariableGuid|VariableOffset[|HiiValue]
1450 #  PCD value needs to be located in such kind of string, and the PCD value might be an expression in which
1451 #    there might have "|" operator, also in string value.
1452 #
1453 #  @param Setting: String contain information described above with "TokenSpace.PcdCName|" stripped
1454 #  @param PcdType: PCD type: feature, fixed, dynamic default VPD HII
1455 #  @param DataType: The datum type of PCD: VOID*, UNIT, BOOL
1456 #  @retval:
1457 #    ValueList: A List contain fields described above
1458 #    IsValid:   True if conforming EBNF, otherwise False
1459 #    Index:     The index where PcdValue is in ValueList
1460 #
1461 def AnalyzeDscPcd(Setting, PcdType, DataType=''):
1462     FieldList = AnalyzePcdExpression(Setting)
1463
1464     IsValid = True
1465     if PcdType in (MODEL_PCD_FIXED_AT_BUILD, MODEL_PCD_PATCHABLE_IN_MODULE, MODEL_PCD_DYNAMIC_DEFAULT, MODEL_PCD_DYNAMIC_EX_DEFAULT):
1466         Value = FieldList[0]
1467         Size = ''
1468         if len(FieldList) > 1 and FieldList[1]:
1469             DataType = FieldList[1]
1470             if FieldList[1] != TAB_VOID and StructPattern.match(FieldList[1]) is None:
1471                 IsValid = False
1472         if len(FieldList) > 2:
1473             Size = FieldList[2]
1474         if IsValid:
1475             if DataType == "":
1476                 IsValid = (len(FieldList) <= 1)
1477             else:
1478                 IsValid = (len(FieldList) <= 3)
1479
1480         if Size:
1481             try:
1482                 int(Size, 16) if Size.upper().startswith("0X") else int(Size)
1483             except:
1484                 IsValid = False
1485                 Size = -1
1486         return [str(Value), DataType, str(Size)], IsValid, 0
1487     elif PcdType == MODEL_PCD_FEATURE_FLAG:
1488         Value = FieldList[0]
1489         Size = ''
1490         IsValid = (len(FieldList) <= 1)
1491         return [Value, DataType, str(Size)], IsValid, 0
1492     elif PcdType in (MODEL_PCD_DYNAMIC_VPD, MODEL_PCD_DYNAMIC_EX_VPD):
1493         VpdOffset = FieldList[0]
1494         Value = Size = ''
1495         if not DataType == TAB_VOID:
1496             if len(FieldList) > 1:
1497                 Value = FieldList[1]
1498         else:
1499             if len(FieldList) > 1:
1500                 Size = FieldList[1]
1501             if len(FieldList) > 2:
1502                 Value = FieldList[2]
1503         if DataType == "":
1504             IsValid = (len(FieldList) <= 1)
1505         else:
1506             IsValid = (len(FieldList) <= 3)
1507         if Size:
1508             try:
1509                 int(Size, 16) if Size.upper().startswith("0X") else int(Size)
1510             except:
1511                 IsValid = False
1512                 Size = -1
1513         return [VpdOffset, str(Size), Value], IsValid, 2
1514     elif PcdType in (MODEL_PCD_DYNAMIC_HII, MODEL_PCD_DYNAMIC_EX_HII):
1515         IsValid = (3 <= len(FieldList) <= 5)
1516         HiiString = FieldList[0]
1517         Guid = Offset = Value = Attribute = ''
1518         if len(FieldList) > 1:
1519             Guid = FieldList[1]
1520         if len(FieldList) > 2:
1521             Offset = FieldList[2]
1522         if len(FieldList) > 3:
1523             Value = FieldList[3]
1524             if not Value:
1525                 IsValid = False
1526         if len(FieldList) > 4:
1527             Attribute = FieldList[4]
1528         return [HiiString, Guid, Offset, Value, Attribute], IsValid, 3
1529     return [], False, 0
1530
1531 ## AnalyzePcdData
1532 #
1533 #  Analyze the pcd Value, Datum type and TokenNumber.
1534 #  Used to avoid split issue while the value string contain "|" character
1535 #
1536 #  @param[in] Setting:  A String contain value/datum type/token number information;
1537 #
1538 #  @retval   ValueList: A List contain value, datum type and toke number.
1539 #
1540 def AnalyzePcdData(Setting):
1541     ValueList = ['', '', '']
1542
1543     ValueRe = re.compile(r'^\s*L?\".*\|.*\"')
1544     PtrValue = ValueRe.findall(Setting)
1545
1546     ValueUpdateFlag = False
1547
1548     if len(PtrValue) >= 1:
1549         Setting = re.sub(ValueRe, '', Setting)
1550         ValueUpdateFlag = True
1551
1552     TokenList = Setting.split(TAB_VALUE_SPLIT)
1553     ValueList[0:len(TokenList)] = TokenList
1554
1555     if ValueUpdateFlag:
1556         ValueList[0] = PtrValue[0]
1557
1558     return ValueList
1559
1560 ## check format of PCD value against its the datum type
1561 #
1562 # For PCD value setting
1563 #
1564 def CheckPcdDatum(Type, Value):
1565     if Type == TAB_VOID:
1566         ValueRe = re.compile(r'\s*L?\".*\"\s*$')
1567         if not (((Value.startswith('L"') or Value.startswith('"')) and Value.endswith('"'))
1568                 or (Value.startswith('{') and Value.endswith('}')) or (Value.startswith("L'") or Value.startswith("'") and Value.endswith("'"))
1569                ):
1570             return False, "Invalid value [%s] of type [%s]; must be in the form of {...} for array"\
1571                           ", \"...\" or \'...\' for string, L\"...\" or L\'...\' for unicode string" % (Value, Type)
1572         elif ValueRe.match(Value):
1573             # Check the chars in UnicodeString or CString is printable
1574             if Value.startswith("L"):
1575                 Value = Value[2:-1]
1576             else:
1577                 Value = Value[1:-1]
1578             Printset = set(string.printable)
1579             Printset.remove(TAB_PRINTCHAR_VT)
1580             Printset.add(TAB_PRINTCHAR_BS)
1581             Printset.add(TAB_PRINTCHAR_NUL)
1582             if not set(Value).issubset(Printset):
1583                 PrintList = sorted(Printset)
1584                 return False, "Invalid PCD string value of type [%s]; must be printable chars %s." % (Type, PrintList)
1585     elif Type == 'BOOLEAN':
1586         if Value not in ['TRUE', 'True', 'true', '0x1', '0x01', '1', 'FALSE', 'False', 'false', '0x0', '0x00', '0']:
1587             return False, "Invalid value [%s] of type [%s]; must be one of TRUE, True, true, 0x1, 0x01, 1"\
1588                           ", FALSE, False, false, 0x0, 0x00, 0" % (Value, Type)
1589     elif Type in [TAB_UINT8, TAB_UINT16, TAB_UINT32, TAB_UINT64]:
1590         if Value and int(Value, 0) < 0:
1591             return False, "PCD can't be set to negative value[%s] for datum type [%s]" % (Value, Type)
1592         try:
1593             Value = long(Value, 0)
1594             if Value > MAX_VAL_TYPE[Type]:
1595                 return False, "Too large PCD value[%s] for datum type [%s]" % (Value, Type)
1596         except:
1597             return False, "Invalid value [%s] of type [%s];"\
1598                           " must be a hexadecimal, decimal or octal in C language format." % (Value, Type)
1599     else:
1600         return True, "StructurePcd"
1601
1602     return True, ""
1603
1604 ## Split command line option string to list
1605 #
1606 # subprocess.Popen needs the args to be a sequence. Otherwise there's problem
1607 # in non-windows platform to launch command
1608 #
1609 def SplitOption(OptionString):
1610     OptionList = []
1611     LastChar = " "
1612     OptionStart = 0
1613     QuotationMark = ""
1614     for Index in range(0, len(OptionString)):
1615         CurrentChar = OptionString[Index]
1616         if CurrentChar in ['"', "'"]:
1617             if QuotationMark == CurrentChar:
1618                 QuotationMark = ""
1619             elif QuotationMark == "":
1620                 QuotationMark = CurrentChar
1621             continue
1622         elif QuotationMark:
1623             continue
1624
1625         if CurrentChar in ["/", "-"] and LastChar in [" ", "\t", "\r", "\n"]:
1626             if Index > OptionStart:
1627                 OptionList.append(OptionString[OptionStart:Index - 1])
1628             OptionStart = Index
1629         LastChar = CurrentChar
1630     OptionList.append(OptionString[OptionStart:])
1631     return OptionList
1632
1633 def CommonPath(PathList):
1634     P1 = min(PathList).split(os.path.sep)
1635     P2 = max(PathList).split(os.path.sep)
1636     for Index in xrange(min(len(P1), len(P2))):
1637         if P1[Index] != P2[Index]:
1638             return os.path.sep.join(P1[:Index])
1639     return os.path.sep.join(P1)
1640
1641 #
1642 # Convert string to C format array
1643 #
1644 def ConvertStringToByteArray(Value):
1645     Value = Value.strip()
1646     if not Value:
1647         return None
1648     if Value[0] == '{':
1649         if not Value.endswith('}'):
1650             return None
1651         Value = Value.replace(' ', '').replace('{', '').replace('}', '')
1652         ValFields = Value.split(',')
1653         try:
1654             for Index in range(len(ValFields)):
1655                 ValFields[Index] = str(int(ValFields[Index], 0))
1656         except ValueError:
1657             return None
1658         Value = '{' + ','.join(ValFields) + '}'
1659         return Value
1660
1661     Unicode = False
1662     if Value.startswith('L"'):
1663         if not Value.endswith('"'):
1664             return None
1665         Value = Value[1:]
1666         Unicode = True
1667     elif not Value.startswith('"') or not Value.endswith('"'):
1668         return None
1669
1670     Value = eval(Value)         # translate escape character
1671     NewValue = '{'
1672     for Index in range(0, len(Value)):
1673         if Unicode:
1674             NewValue = NewValue + str(ord(Value[Index]) % 0x10000) + ','
1675         else:
1676             NewValue = NewValue + str(ord(Value[Index]) % 0x100) + ','
1677     Value = NewValue + '0}'
1678     return Value
1679
1680 class PathClass(object):
1681     def __init__(self, File='', Root='', AlterRoot='', Type='', IsBinary=False,
1682                  Arch='COMMON', ToolChainFamily='', Target='', TagName='', ToolCode=''):
1683         self.Arch = Arch
1684         self.File = str(File)
1685         if os.path.isabs(self.File):
1686             self.Root = ''
1687             self.AlterRoot = ''
1688         else:
1689             self.Root = str(Root)
1690             self.AlterRoot = str(AlterRoot)
1691
1692         # Remove any '.' and '..' in path
1693         if self.Root:
1694             self.Root = mws.getWs(self.Root, self.File)
1695             self.Path = os.path.normpath(os.path.join(self.Root, self.File))
1696             self.Root = os.path.normpath(CommonPath([self.Root, self.Path]))
1697             # eliminate the side-effect of 'C:'
1698             if self.Root[-1] == ':':
1699                 self.Root += os.path.sep
1700             # file path should not start with path separator
1701             if self.Root[-1] == os.path.sep:
1702                 self.File = self.Path[len(self.Root):]
1703             else:
1704                 self.File = self.Path[len(self.Root) + 1:]
1705         else:
1706             self.Path = os.path.normpath(self.File)
1707
1708         self.SubDir, self.Name = os.path.split(self.File)
1709         self.BaseName, self.Ext = os.path.splitext(self.Name)
1710
1711         if self.Root:
1712             if self.SubDir:
1713                 self.Dir = os.path.join(self.Root, self.SubDir)
1714             else:
1715                 self.Dir = self.Root
1716         else:
1717             self.Dir = self.SubDir
1718
1719         if IsBinary:
1720             self.Type = Type
1721         else:
1722             self.Type = self.Ext.lower()
1723
1724         self.IsBinary = IsBinary
1725         self.Target = Target
1726         self.TagName = TagName
1727         self.ToolCode = ToolCode
1728         self.ToolChainFamily = ToolChainFamily
1729
1730     ## Convert the object of this class to a string
1731     #
1732     #  Convert member Path of the class to a string
1733     #
1734     #  @retval string Formatted String
1735     #
1736     def __str__(self):
1737         return self.Path
1738
1739     ## Override __eq__ function
1740     #
1741     # Check whether PathClass are the same
1742     #
1743     # @retval False The two PathClass are different
1744     # @retval True  The two PathClass are the same
1745     #
1746     def __eq__(self, Other):
1747         if isinstance(Other, type(self)):
1748             return self.Path == Other.Path
1749         else:
1750             return self.Path == str(Other)
1751
1752     ## Override __cmp__ function
1753     #
1754     # Customize the comparsion operation of two PathClass
1755     #
1756     # @retval 0     The two PathClass are different
1757     # @retval -1    The first PathClass is less than the second PathClass
1758     # @retval 1     The first PathClass is Bigger than the second PathClass
1759     def __cmp__(self, Other):
1760         if isinstance(Other, type(self)):
1761             OtherKey = Other.Path
1762         else:
1763             OtherKey = str(Other)
1764
1765         SelfKey = self.Path
1766         if SelfKey == OtherKey:
1767             return 0
1768         elif SelfKey > OtherKey:
1769             return 1
1770         else:
1771             return -1
1772
1773     ## Override __hash__ function
1774     #
1775     # Use Path as key in hash table
1776     #
1777     # @retval string Key for hash table
1778     #
1779     def __hash__(self):
1780         return hash(self.Path)
1781
1782     @cached_property
1783     def Key(self):
1784         return self.Path.upper()
1785
1786     @property
1787     def TimeStamp(self):
1788         return os.stat(self.Path)[8]
1789
1790     def Validate(self, Type='', CaseSensitive=True):
1791         if GlobalData.gCaseInsensitive:
1792             CaseSensitive = False
1793         if Type and Type.lower() != self.Type:
1794             return FILE_TYPE_MISMATCH, '%s (expect %s but got %s)' % (self.File, Type, self.Type)
1795
1796         RealFile, RealRoot = RealPath2(self.File, self.Root, self.AlterRoot)
1797         if not RealRoot and not RealFile:
1798             RealFile = self.File
1799             if self.AlterRoot:
1800                 RealFile = os.path.join(self.AlterRoot, self.File)
1801             elif self.Root:
1802                 RealFile = os.path.join(self.Root, self.File)
1803             if len (mws.getPkgPath()) == 0:
1804                 return FILE_NOT_FOUND, os.path.join(self.AlterRoot, RealFile)
1805             else:
1806                 return FILE_NOT_FOUND, "%s is not found in packages path:\n\t%s" % (self.File, '\n\t'.join(mws.getPkgPath()))
1807
1808         ErrorCode = 0
1809         ErrorInfo = ''
1810         if RealRoot != self.Root or RealFile != self.File:
1811             if CaseSensitive and (RealFile != self.File or (RealRoot != self.Root and RealRoot != self.AlterRoot)):
1812                 ErrorCode = FILE_CASE_MISMATCH
1813                 ErrorInfo = self.File + '\n\t' + RealFile + " [in file system]"
1814
1815             self.SubDir, self.Name = os.path.split(RealFile)
1816             self.BaseName, self.Ext = os.path.splitext(self.Name)
1817             if self.SubDir:
1818                 self.Dir = os.path.join(RealRoot, self.SubDir)
1819             else:
1820                 self.Dir = RealRoot
1821             self.File = RealFile
1822             self.Root = RealRoot
1823             self.Path = os.path.join(RealRoot, RealFile)
1824         return ErrorCode, ErrorInfo
1825
1826 ## Parse PE image to get the required PE informaion.
1827 #
1828 class PeImageClass():
1829     ## Constructor
1830     #
1831     #   @param  File FilePath of PeImage
1832     #
1833     def __init__(self, PeFile):
1834         self.FileName   = PeFile
1835         self.IsValid    = False
1836         self.Size       = 0
1837         self.EntryPoint = 0
1838         self.SectionAlignment  = 0
1839         self.SectionHeaderList = []
1840         self.ErrorInfo = ''
1841         try:
1842             PeObject = open(PeFile, 'rb')
1843         except:
1844             self.ErrorInfo = self.FileName + ' can not be found\n'
1845             return
1846         # Read DOS header
1847         ByteArray = array.array('B')
1848         ByteArray.fromfile(PeObject, 0x3E)
1849         ByteList = ByteArray.tolist()
1850         # DOS signature should be 'MZ'
1851         if self._ByteListToStr (ByteList[0x0:0x2]) != 'MZ':
1852             self.ErrorInfo = self.FileName + ' has no valid DOS signature MZ'
1853             return
1854
1855         # Read 4 byte PE Signature
1856         PeOffset = self._ByteListToInt(ByteList[0x3C:0x3E])
1857         PeObject.seek(PeOffset)
1858         ByteArray = array.array('B')
1859         ByteArray.fromfile(PeObject, 4)
1860         # PE signature should be 'PE\0\0'
1861         if ByteArray.tostring() != 'PE\0\0':
1862             self.ErrorInfo = self.FileName + ' has no valid PE signature PE00'
1863             return
1864
1865         # Read PE file header
1866         ByteArray = array.array('B')
1867         ByteArray.fromfile(PeObject, 0x14)
1868         ByteList = ByteArray.tolist()
1869         SecNumber = self._ByteListToInt(ByteList[0x2:0x4])
1870         if SecNumber == 0:
1871             self.ErrorInfo = self.FileName + ' has no section header'
1872             return
1873
1874         # Read PE optional header
1875         OptionalHeaderSize = self._ByteListToInt(ByteArray[0x10:0x12])
1876         ByteArray = array.array('B')
1877         ByteArray.fromfile(PeObject, OptionalHeaderSize)
1878         ByteList = ByteArray.tolist()
1879         self.EntryPoint       = self._ByteListToInt(ByteList[0x10:0x14])
1880         self.SectionAlignment = self._ByteListToInt(ByteList[0x20:0x24])
1881         self.Size             = self._ByteListToInt(ByteList[0x38:0x3C])
1882
1883         # Read each Section Header
1884         for Index in range(SecNumber):
1885             ByteArray = array.array('B')
1886             ByteArray.fromfile(PeObject, 0x28)
1887             ByteList = ByteArray.tolist()
1888             SecName  = self._ByteListToStr(ByteList[0:8])
1889             SecVirtualSize = self._ByteListToInt(ByteList[8:12])
1890             SecRawAddress  = self._ByteListToInt(ByteList[20:24])
1891             SecVirtualAddress = self._ByteListToInt(ByteList[12:16])
1892             self.SectionHeaderList.append((SecName, SecVirtualAddress, SecRawAddress, SecVirtualSize))
1893         self.IsValid = True
1894         PeObject.close()
1895
1896     def _ByteListToStr(self, ByteList):
1897         String = ''
1898         for index in range(len(ByteList)):
1899             if ByteList[index] == 0:
1900                 break
1901             String += chr(ByteList[index])
1902         return String
1903
1904     def _ByteListToInt(self, ByteList):
1905         Value = 0
1906         for index in range(len(ByteList) - 1, -1, -1):
1907             Value = (Value << 8) | int(ByteList[index])
1908         return Value
1909
1910 class DefaultStore():
1911     def __init__(self, DefaultStores ):
1912
1913         self.DefaultStores = DefaultStores
1914     def DefaultStoreID(self, DefaultStoreName):
1915         for key, value in self.DefaultStores.items():
1916             if value == DefaultStoreName:
1917                 return key
1918         return None
1919     def GetDefaultDefault(self):
1920         if not self.DefaultStores or "0" in self.DefaultStores:
1921             return "0", TAB_DEFAULT_STORES_DEFAULT
1922         else:
1923             minvalue = min(int(value_str) for value_str in self.DefaultStores)
1924             return (str(minvalue), self.DefaultStores[str(minvalue)])
1925     def GetMin(self, DefaultSIdList):
1926         if not DefaultSIdList:
1927             return TAB_DEFAULT_STORES_DEFAULT
1928         storeidset = {storeid for storeid, storename in self.DefaultStores.values() if storename in DefaultSIdList}
1929         if not storeidset:
1930             return ""
1931         minid = min(storeidset )
1932         for sid, name in self.DefaultStores.values():
1933             if sid == minid:
1934                 return name
1935
1936 class SkuClass():
1937     DEFAULT = 0
1938     SINGLE = 1
1939     MULTIPLE =2
1940
1941     def __init__(self,SkuIdentifier='', SkuIds=None):
1942         if SkuIds is None:
1943             SkuIds = {}
1944
1945         for SkuName in SkuIds:
1946             SkuId = SkuIds[SkuName][0]
1947             skuid_num = int(SkuId, 16) if SkuId.upper().startswith("0X") else int(SkuId)
1948             if skuid_num > 0xFFFFFFFFFFFFFFFF:
1949                 EdkLogger.error("build", PARAMETER_INVALID,
1950                             ExtraData = "SKU-ID [%s] value %s exceeds the max value of UINT64"
1951                                       % (SkuName, SkuId))
1952
1953         self.AvailableSkuIds = sdict()
1954         self.SkuIdSet = []
1955         self.SkuIdNumberSet = []
1956         self.SkuData = SkuIds
1957         self._SkuInherit = {}
1958         self._SkuIdentifier = SkuIdentifier
1959         if SkuIdentifier == '' or SkuIdentifier is None:
1960             self.SkuIdSet = ['DEFAULT']
1961             self.SkuIdNumberSet = ['0U']
1962         elif SkuIdentifier == 'ALL':
1963             self.SkuIdSet = SkuIds.keys()
1964             self.SkuIdNumberSet = [num[0].strip() + 'U' for num in SkuIds.values()]
1965         else:
1966             r = SkuIdentifier.split('|')
1967             self.SkuIdSet=[(r[k].strip()).upper() for k in range(len(r))]
1968             k = None
1969             try:
1970                 self.SkuIdNumberSet = [SkuIds[k][0].strip() + 'U' for k in self.SkuIdSet]
1971             except Exception:
1972                 EdkLogger.error("build", PARAMETER_INVALID,
1973                             ExtraData = "SKU-ID [%s] is not supported by the platform. [Valid SKU-ID: %s]"
1974                                       % (k, " | ".join(SkuIds.keys())))
1975         for each in self.SkuIdSet:
1976             if each in SkuIds:
1977                 self.AvailableSkuIds[each] = SkuIds[each][0]
1978             else:
1979                 EdkLogger.error("build", PARAMETER_INVALID,
1980                             ExtraData="SKU-ID [%s] is not supported by the platform. [Valid SKU-ID: %s]"
1981                                       % (each, " | ".join(SkuIds.keys())))
1982         if self.SkuUsageType != SkuClass.SINGLE:
1983             self.AvailableSkuIds.update({'DEFAULT':0, 'COMMON':0})
1984         if self.SkuIdSet:
1985             GlobalData.gSkuids = (self.SkuIdSet)
1986             if 'COMMON' in GlobalData.gSkuids:
1987                 GlobalData.gSkuids.remove('COMMON')
1988             if self.SkuUsageType == self.SINGLE:
1989                 if len(GlobalData.gSkuids) != 1:
1990                     if 'DEFAULT' in GlobalData.gSkuids:
1991                         GlobalData.gSkuids.remove('DEFAULT')
1992             if GlobalData.gSkuids:
1993                 GlobalData.gSkuids.sort()
1994
1995     def GetNextSkuId(self, skuname):
1996         if not self._SkuInherit:
1997             self._SkuInherit = {}
1998             for item in self.SkuData.values():
1999                 self._SkuInherit[item[1]]=item[2] if item[2] else "DEFAULT"
2000         return self._SkuInherit.get(skuname, "DEFAULT")
2001
2002     def GetSkuChain(self, sku):
2003         if sku == "DEFAULT":
2004             return ["DEFAULT"]
2005         skulist = [sku]
2006         nextsku = sku
2007         while True:
2008             nextsku = self.GetNextSkuId(nextsku)
2009             skulist.append(nextsku)
2010             if nextsku == "DEFAULT":
2011                 break
2012         skulist.reverse()
2013         return skulist
2014     def SkuOverrideOrder(self):
2015         skuorderset = []
2016         for skuname in self.SkuIdSet:
2017             skuorderset.append(self.GetSkuChain(skuname))
2018
2019         skuorder = []
2020         for index in range(max(len(item) for item in skuorderset)):
2021             for subset in skuorderset:
2022                 if index > len(subset)-1:
2023                     continue
2024                 if subset[index] in skuorder:
2025                     continue
2026                 skuorder.append(subset[index])
2027
2028         return skuorder
2029
2030     @property
2031     def SkuUsageType(self):
2032         if self._SkuIdentifier.upper() == "ALL":
2033             return SkuClass.MULTIPLE
2034
2035         if len(self.SkuIdSet) == 1:
2036             if self.SkuIdSet[0] == 'DEFAULT':
2037                 return SkuClass.DEFAULT
2038             return SkuClass.SINGLE
2039         if len(self.SkuIdSet) == 2 and 'DEFAULT' in self.SkuIdSet:
2040             return SkuClass.SINGLE
2041         return SkuClass.MULTIPLE
2042
2043     def DumpSkuIdArrary(self):
2044         if self.SkuUsageType == SkuClass.SINGLE:
2045             return "{0x0}"
2046         ArrayStrList = []
2047         for skuname in self.AvailableSkuIds:
2048             if skuname == "COMMON":
2049                 continue
2050             while skuname != "DEFAULT":
2051                 ArrayStrList.append(hex(int(self.AvailableSkuIds[skuname])))
2052                 skuname = self.GetNextSkuId(skuname)
2053             ArrayStrList.append("0x0")
2054         return "{{{myList}}}".format(myList=",".join(ArrayStrList))
2055
2056     @property
2057     def AvailableSkuIdSet(self):
2058         return self.AvailableSkuIds
2059
2060     @property
2061     def SystemSkuId(self):
2062         if self.SkuUsageType == SkuClass.SINGLE:
2063             if len(self.SkuIdSet) == 1:
2064                 return self.SkuIdSet[0]
2065             else:
2066                 return self.SkuIdSet[0] if self.SkuIdSet[0] != 'DEFAULT' else self.SkuIdSet[1]
2067         else:
2068             return 'DEFAULT'
2069
2070 #
2071 # Pack a registry format GUID
2072 #
2073 def PackRegistryFormatGuid(Guid):
2074     return PackGUID(Guid.split('-'))
2075
2076 ##  Get the integer value from string like "14U" or integer like 2
2077 #
2078 #   @param      Input   The object that may be either a integer value or a string
2079 #
2080 #   @retval     Value    The integer value that the input represents
2081 #
2082 def GetIntegerValue(Input):
2083     if type(Input) in (int, long):
2084         return Input
2085     String = Input
2086     if String.endswith("U"):
2087         String = String[:-1]
2088     if String.endswith("ULL"):
2089         String = String[:-3]
2090     if String.endswith("LL"):
2091         String = String[:-2]
2092
2093     if String.startswith("0x") or String.startswith("0X"):
2094         return int(String, 16)
2095     elif String == '':
2096         return 0
2097     else:
2098         return int(String)
2099
2100 #
2101 # Pack a GUID (registry format) list into a buffer and return it
2102 #
2103 def PackGUID(Guid):
2104     return pack(PACK_PATTERN_GUID,
2105                 int(Guid[0], 16),
2106                 int(Guid[1], 16),
2107                 int(Guid[2], 16),
2108                 int(Guid[3][-4:-2], 16),
2109                 int(Guid[3][-2:], 16),
2110                 int(Guid[4][-12:-10], 16),
2111                 int(Guid[4][-10:-8], 16),
2112                 int(Guid[4][-8:-6], 16),
2113                 int(Guid[4][-6:-4], 16),
2114                 int(Guid[4][-4:-2], 16),
2115                 int(Guid[4][-2:], 16)
2116                 )
2117
2118 #
2119 # Pack a GUID (byte) list into a buffer and return it
2120 #
2121 def PackByteFormatGUID(Guid):
2122     return pack(PACK_PATTERN_GUID,
2123                 Guid[0],
2124                 Guid[1],
2125                 Guid[2],
2126                 Guid[3],
2127                 Guid[4],
2128                 Guid[5],
2129                 Guid[6],
2130                 Guid[7],
2131                 Guid[8],
2132                 Guid[9],
2133                 Guid[10],
2134                 )
2135
2136 ##
2137 #
2138 # This acts like the main() function for the script, unless it is 'import'ed into another
2139 # script.
2140 #
2141 if __name__ == '__main__':
2142     pass
2143