BaseTools/Source/Python/Common/Misc.py

   1 ## @file
   2 # Common routines used by all tools
   3 #
   4 # Copyright (c) 2007 - 2018, Intel Corporation. All rights reserved.<BR>
   5 # This program and the accompanying materials
   6 # are licensed and made available under the terms and conditions of the BSD License
   7 # which accompanies this distribution.  The full text of the license may be found at
   8 # http://opensource.org/licenses/bsd-license.php
   9 #
  10 # THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS,
  11 # WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED.
  12 #
  13
  14 ##
  15 # Import Modules
  16 #
  17 from __future__ import absolute_import
  18 import Common.LongFilePathOs as os
  19 import sys
  20 import string
  21 import threading
  22 import time
  23 import re
  24 import pickle
  25 import array
  26 import shutil
  27 from random import sample
  28 from struct import pack
  29 from UserDict import IterableUserDict
  30 from UserList import UserList
  31
  32 from Common import EdkLogger as EdkLogger
  33 from Common import GlobalData as GlobalData
  34 from .DataType import *
  35 from .BuildToolError import *
  36 from CommonDataClass.DataClass import *
  37 from .Parsing import GetSplitValueList
  38 from Common.LongFilePathSupport import OpenLongFilePath as open
  39 from Common.MultipleWorkspace import MultipleWorkspace as mws
  40 import uuid
  41 from CommonDataClass.Exceptions import BadExpression
  42 from Common.caching import cached_property
  43 import subprocess
  44 ## Regular expression used to find out place holders in string template
  45 gPlaceholderPattern = re.compile("\$\{([^$()\s]+)\}", re.MULTILINE | re.UNICODE)
  46
  47 ## regular expressions for map file processing
  48 startPatternGeneral = re.compile("^Start[' ']+Length[' ']+Name[' ']+Class")
  49 addressPatternGeneral = re.compile("^Address[' ']+Publics by Value[' ']+Rva\+Base")
  50 valuePatternGcc = re.compile('^([\w_\.]+) +([\da-fA-Fx]+) +([\da-fA-Fx]+)$')
  51 pcdPatternGcc = re.compile('^([\da-fA-Fx]+) +([\da-fA-Fx]+)')
  52 secReGeneral = re.compile('^([\da-fA-F]+):([\da-fA-F]+) +([\da-fA-F]+)[Hh]? +([.\w\$]+) +(\w+)', re.UNICODE)
  53
  54 StructPattern = re.compile(r'[_a-zA-Z][0-9A-Za-z_]*$')
  55
  56 ## Dictionary used to store file time stamp for quick re-access
  57 gFileTimeStampCache = {}    # {file path : file time stamp}
  58
  59 ## Dictionary used to store dependencies of files
  60 gDependencyDatabase = {}    # arch : {file path : [dependent files list]}
  61
  62 #
  63 # If a module is built more than once with different PCDs or library classes
  64 # a temporary INF file with same content is created, the temporary file is removed
  65 # when build exits.
  66 #
  67 _TempInfs = []
  68
  69 def GetVariableOffset(mapfilepath, efifilepath, varnames):
  70     """ Parse map file to get variable offset in current EFI file
  71     @param mapfilepath    Map file absolution path
  72     @param efifilepath:   EFI binary file full path
  73     @param varnames       iteratable container whose elements are variable names to be searched
  74
  75     @return List whos elements are tuple with variable name and raw offset
  76     """
  77     lines = []
  78     try:
  79         f = open(mapfilepath, 'r')
  80         lines = f.readlines()
  81         f.close()
  82     except:
  83         return None
  84
  85     if len(lines) == 0: return None
  86     firstline = lines[0].strip()
  87     if (firstline.startswith("Archive member included ") and
  88         firstline.endswith(" file (symbol)")):
  89         return _parseForGCC(lines, efifilepath, varnames)
  90     if firstline.startswith("# Path:"):
  91         return _parseForXcode(lines, efifilepath, varnames)
  92     return _parseGeneral(lines, efifilepath, varnames)
  93
  94 def _parseForXcode(lines, efifilepath, varnames):
  95     status = 0
  96     ret = []
  97     for line in lines:
  98         line = line.strip()
  99         if status == 0 and line == "# Symbols:":
 100             status = 1
 101             continue
 102         if status == 1 and len(line) != 0:
 103             for varname in varnames:
 104                 if varname in line:
 105                     # cannot pregenerate this RegEx since it uses varname from varnames.
 106                     m = re.match('^([\da-fA-FxX]+)([\s\S]*)([_]*%s)$' % varname, line)
 107                     if m is not None:
 108                         ret.append((varname, m.group(1)))
 109     return ret
 110
 111 def _parseForGCC(lines, efifilepath, varnames):
 112     """ Parse map file generated by GCC linker """
 113     status = 0
 114     sections = []
 115     varoffset = []
 116     for index, line in enumerate(lines):
 117         line = line.strip()
 118         # status machine transection
 119         if status == 0 and line == "Memory Configuration":
 120             status = 1
 121             continue
 122         elif status == 1 and line == 'Linker script and memory map':
 123             status = 2
 124             continue
 125         elif status ==2 and line == 'START GROUP':
 126             status = 3
 127             continue
 128
 129         # status handler
 130         if status == 3:
 131             m = valuePatternGcc.match(line)
 132             if m is not None:
 133                 sections.append(m.groups(0))
 134             for varname in varnames:
 135                 Str = ''
 136                 m = re.match("^.data.(%s)" % varname, line)
 137                 if m is not None:
 138                     m = re.match(".data.(%s)$" % varname, line)
 139                     if m is not None:
 140                         Str = lines[index + 1]
 141                     else:
 142                         Str = line[len(".data.%s" % varname):]
 143                     if Str:
 144                         m = pcdPatternGcc.match(Str.strip())
 145                         if m is not None:
 146                             varoffset.append((varname, int(m.groups(0)[0], 16), int(sections[-1][1], 16), sections[-1][0]))
 147
 148     if not varoffset:
 149         return []
 150     # get section information from efi file
 151     efisecs = PeImageClass(efifilepath).SectionHeaderList
 152     if efisecs is None or len(efisecs) == 0:
 153         return []
 154     #redirection
 155     redirection = 0
 156     for efisec in efisecs:
 157         for section in sections:
 158             if section[0].strip() == efisec[0].strip() and section[0].strip() == '.text':
 159                 redirection = int(section[1], 16) - efisec[1]
 160
 161     ret = []
 162     for var in varoffset:
 163         for efisec in efisecs:
 164             if var[1] >= efisec[1] and var[1] < efisec[1]+efisec[3]:
 165                 ret.append((var[0], hex(efisec[2] + var[1] - efisec[1] - redirection)))
 166     return ret
 167
 168 def _parseGeneral(lines, efifilepath, varnames):
 169     status = 0    #0 - beginning of file; 1 - PE section definition; 2 - symbol table
 170     secs  = []    # key = section name
 171     varoffset = []
 172     symRe = re.compile('^([\da-fA-F]+):([\da-fA-F]+) +([\.:\\\\\w\?@\$]+) +([\da-fA-F]+)', re.UNICODE)
 173
 174     for line in lines:
 175         line = line.strip()
 176         if startPatternGeneral.match(line):
 177             status = 1
 178             continue
 179         if addressPatternGeneral.match(line):
 180             status = 2
 181             continue
 182         if line.startswith("entry point at"):
 183             status = 3
 184             continue
 185         if status == 1 and len(line) != 0:
 186             m =  secReGeneral.match(line)
 187             assert m is not None, "Fail to parse the section in map file , line is %s" % line
 188             sec_no, sec_start, sec_length, sec_name, sec_class = m.groups(0)
 189             secs.append([int(sec_no, 16), int(sec_start, 16), int(sec_length, 16), sec_name, sec_class])
 190         if status == 2 and len(line) != 0:
 191             for varname in varnames:
 192                 m = symRe.match(line)
 193                 assert m is not None, "Fail to parse the symbol in map file, line is %s" % line
 194                 sec_no, sym_offset, sym_name, vir_addr = m.groups(0)
 195                 sec_no     = int(sec_no,     16)
 196                 sym_offset = int(sym_offset, 16)
 197                 vir_addr   = int(vir_addr,   16)
 198                 # cannot pregenerate this RegEx since it uses varname from varnames.
 199                 m2 = re.match('^[_]*(%s)' % varname, sym_name)
 200                 if m2 is not None:
 201                     # fond a binary pcd entry in map file
 202                     for sec in secs:
 203                         if sec[0] == sec_no and (sym_offset >= sec[1] and sym_offset < sec[1] + sec[2]):
 204                             varoffset.append([varname, sec[3], sym_offset, vir_addr, sec_no])
 205
 206     if not varoffset: return []
 207
 208     # get section information from efi file
 209     efisecs = PeImageClass(efifilepath).SectionHeaderList
 210     if efisecs is None or len(efisecs) == 0:
 211         return []
 212
 213     ret = []
 214     for var in varoffset:
 215         index = 0
 216         for efisec in efisecs:
 217             index = index + 1
 218             if var[1].strip() == efisec[0].strip():
 219                 ret.append((var[0], hex(efisec[2] + var[2])))
 220             elif var[4] == index:
 221                 ret.append((var[0], hex(efisec[2] + var[2])))
 222
 223     return ret
 224
 225 ## Routine to process duplicated INF
 226 #
 227 #  This function is called by following two cases:
 228 #  Case 1 in DSC:
 229 #    [components.arch]
 230 #    Pkg/module/module.inf
 231 #    Pkg/module/module.inf {
 232 #      <Defines>
 233 #        FILE_GUID = 0D1B936F-68F3-4589-AFCC-FB8B7AEBC836
 234 #    }
 235 #  Case 2 in FDF:
 236 #    INF Pkg/module/module.inf
 237 #    INF FILE_GUID = 0D1B936F-68F3-4589-AFCC-FB8B7AEBC836 Pkg/module/module.inf
 238 #
 239 #  This function copies Pkg/module/module.inf to
 240 #  Conf/.cache/0D1B936F-68F3-4589-AFCC-FB8B7AEBC836module.inf
 241 #
 242 #  @param Path     Original PathClass object
 243 #  @param BaseName New file base name
 244 #
 245 #  @retval         return the new PathClass object
 246 #
 247 def ProcessDuplicatedInf(Path, BaseName, Workspace):
 248     Filename = os.path.split(Path.File)[1]
 249     if '.' in Filename:
 250         Filename = BaseName + Path.BaseName + Filename[Filename.rfind('.'):]
 251     else:
 252         Filename = BaseName + Path.BaseName
 253
 254     #
 255     # If -N is specified on command line, cache is disabled
 256     # The directory has to be created
 257     #
 258     DbDir = os.path.split(GlobalData.gDatabasePath)[0]
 259     if not os.path.exists(DbDir):
 260         os.makedirs(DbDir)
 261     #
 262     # A temporary INF is copied to database path which must have write permission
 263     # The temporary will be removed at the end of build
 264     # In case of name conflict, the file name is
 265     # FILE_GUIDBaseName (0D1B936F-68F3-4589-AFCC-FB8B7AEBC836module.inf)
 266     #
 267     TempFullPath = os.path.join(DbDir,
 268                                 Filename)
 269     RtPath = PathClass(Path.File, Workspace)
 270     #
 271     # Modify the full path to temporary path, keep other unchanged
 272     #
 273     # To build same module more than once, the module path with FILE_GUID overridden has
 274     # the file name FILE_GUIDmodule.inf, but the relative path (self.MetaFile.File) is the real path
 275     # in DSC which is used as relative path by C files and other files in INF.
 276     # A trick was used: all module paths are PathClass instances, after the initialization
 277     # of PathClass, the PathClass.Path is overridden by the temporary INF path.
 278     #
 279     # The reason for creating a temporary INF is:
 280     # Platform.Modules which is the base to create ModuleAutoGen objects is a dictionary,
 281     # the key is the full path of INF, the value is an object to save overridden library instances, PCDs.
 282     # A different key for the same module is needed to create different output directory,
 283     # retrieve overridden PCDs, library instances.
 284     #
 285     # The BaseName is the FILE_GUID which is also the output directory name.
 286     #
 287     #
 288     RtPath.Path = TempFullPath
 289     RtPath.BaseName = BaseName
 290     #
 291     # If file exists, compare contents
 292     #
 293     if os.path.exists(TempFullPath):
 294         with open(str(Path), 'rb') as f1, open(TempFullPath, 'rb') as f2:
 295             if f1.read() == f2.read():
 296                 return RtPath
 297     _TempInfs.append(TempFullPath)
 298     shutil.copy2(str(Path), TempFullPath)
 299     return RtPath
 300
 301 ## Remove temporary created INFs whose paths were saved in _TempInfs
 302 #
 303 def ClearDuplicatedInf():
 304     while _TempInfs:
 305         File = _TempInfs.pop()
 306         if os.path.exists(File):
 307             os.remove(File)
 308
 309 ## Convert GUID string in xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx style to C structure style
 310 #
 311 #   @param      Guid    The GUID string
 312 #
 313 #   @retval     string  The GUID string in C structure style
 314 #
 315 def GuidStringToGuidStructureString(Guid):
 316     GuidList = Guid.split('-')
 317     Result = '{'
 318     for Index in range(0, 3, 1):
 319         Result = Result + '0x' + GuidList[Index] + ', '
 320     Result = Result + '{0x' + GuidList[3][0:2] + ', 0x' + GuidList[3][2:4]
 321     for Index in range(0, 12, 2):
 322         Result = Result + ', 0x' + GuidList[4][Index:Index + 2]
 323     Result += '}}'
 324     return Result
 325
 326 ## Convert GUID structure in byte array to xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
 327 #
 328 #   @param      GuidValue   The GUID value in byte array
 329 #
 330 #   @retval     string      The GUID value in xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx format
 331 #
 332 def GuidStructureByteArrayToGuidString(GuidValue):
 333     guidValueString = GuidValue.lower().replace("{", "").replace("}", "").replace(" ", "").replace(";", "")
 334     guidValueList = guidValueString.split(",")
 335     if len(guidValueList) != 16:
 336         return ''
 337         #EdkLogger.error(None, None, "Invalid GUID value string %s" % GuidValue)
 338     try:
 339         return "%02x%02x%02x%02x-%02x%02x-%02x%02x-%02x%02x-%02x%02x%02x%02x%02x%02x" % (
 340                 int(guidValueList[3], 16),
 341                 int(guidValueList[2], 16),
 342                 int(guidValueList[1], 16),
 343                 int(guidValueList[0], 16),
 344                 int(guidValueList[5], 16),
 345                 int(guidValueList[4], 16),
 346                 int(guidValueList[7], 16),
 347                 int(guidValueList[6], 16),
 348                 int(guidValueList[8], 16),
 349                 int(guidValueList[9], 16),
 350                 int(guidValueList[10], 16),
 351                 int(guidValueList[11], 16),
 352                 int(guidValueList[12], 16),
 353                 int(guidValueList[13], 16),
 354                 int(guidValueList[14], 16),
 355                 int(guidValueList[15], 16)
 356                 )
 357     except:
 358         return ''
 359
 360 ## Convert GUID string in C structure style to xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
 361 #
 362 #   @param      GuidValue   The GUID value in C structure format
 363 #
 364 #   @retval     string      The GUID value in xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx format
 365 #
 366 def GuidStructureStringToGuidString(GuidValue):
 367     if not GlobalData.gGuidCFormatPattern.match(GuidValue):
 368         return ''
 369     guidValueString = GuidValue.lower().replace("{", "").replace("}", "").replace(" ", "").replace(";", "")
 370     guidValueList = guidValueString.split(",")
 371     if len(guidValueList) != 11:
 372         return ''
 373         #EdkLogger.error(None, None, "Invalid GUID value string %s" % GuidValue)
 374     try:
 375         return "%08x-%04x-%04x-%02x%02x-%02x%02x%02x%02x%02x%02x" % (
 376                 int(guidValueList[0], 16),
 377                 int(guidValueList[1], 16),
 378                 int(guidValueList[2], 16),
 379                 int(guidValueList[3], 16),
 380                 int(guidValueList[4], 16),
 381                 int(guidValueList[5], 16),
 382                 int(guidValueList[6], 16),
 383                 int(guidValueList[7], 16),
 384                 int(guidValueList[8], 16),
 385                 int(guidValueList[9], 16),
 386                 int(guidValueList[10], 16)
 387                 )
 388     except:
 389         return ''
 390
 391 ## Convert GUID string in C structure style to xxxxxxxx_xxxx_xxxx_xxxx_xxxxxxxxxxxx
 392 #
 393 #   @param      GuidValue   The GUID value in C structure format
 394 #
 395 #   @retval     string      The GUID value in xxxxxxxx_xxxx_xxxx_xxxx_xxxxxxxxxxxx format
 396 #
 397 def GuidStructureStringToGuidValueName(GuidValue):
 398     guidValueString = GuidValue.lower().replace("{", "").replace("}", "").replace(" ", "")
 399     guidValueList = guidValueString.split(",")
 400     if len(guidValueList) != 11:
 401         EdkLogger.error(None, FORMAT_INVALID, "Invalid GUID value string [%s]" % GuidValue)
 402     return "%08x_%04x_%04x_%02x%02x_%02x%02x%02x%02x%02x%02x" % (
 403             int(guidValueList[0], 16),
 404             int(guidValueList[1], 16),
 405             int(guidValueList[2], 16),
 406             int(guidValueList[3], 16),
 407             int(guidValueList[4], 16),
 408             int(guidValueList[5], 16),
 409             int(guidValueList[6], 16),
 410             int(guidValueList[7], 16),
 411             int(guidValueList[8], 16),
 412             int(guidValueList[9], 16),
 413             int(guidValueList[10], 16)
 414             )
 415
 416 ## Create directories
 417 #
 418 #   @param      Directory   The directory name
 419 #
 420 def CreateDirectory(Directory):
 421     if Directory is None or Directory.strip() == "":
 422         return True
 423     try:
 424         if not os.access(Directory, os.F_OK):
 425             os.makedirs(Directory)
 426     except:
 427         return False
 428     return True
 429
 430 ## Remove directories, including files and sub-directories in it
 431 #
 432 #   @param      Directory   The directory name
 433 #
 434 def RemoveDirectory(Directory, Recursively=False):
 435     if Directory is None or Directory.strip() == "" or not os.path.exists(Directory):
 436         return
 437     if Recursively:
 438         CurrentDirectory = os.getcwd()
 439         os.chdir(Directory)
 440         for File in os.listdir("."):
 441             if os.path.isdir(File):
 442                 RemoveDirectory(File, Recursively)
 443             else:
 444                 os.remove(File)
 445         os.chdir(CurrentDirectory)
 446     os.rmdir(Directory)
 447
 448 ## Store content in file
 449 #
 450 #  This method is used to save file only when its content is changed. This is
 451 #  quite useful for "make" system to decide what will be re-built and what won't.
 452 #
 453 #   @param      File            The path of file
 454 #   @param      Content         The new content of the file
 455 #   @param      IsBinaryFile    The flag indicating if the file is binary file or not
 456 #
 457 #   @retval     True            If the file content is changed and the file is renewed
 458 #   @retval     False           If the file content is the same
 459 #
 460 def SaveFileOnChange(File, Content, IsBinaryFile=True):
 461     if not IsBinaryFile:
 462         Content = Content.replace("\n", os.linesep)
 463
 464     if os.path.exists(File):
 465         try:
 466             if Content == open(File, "rb").read():
 467                 return False
 468         except:
 469             EdkLogger.error(None, FILE_OPEN_FAILURE, ExtraData=File)
 470
 471     DirName = os.path.dirname(File)
 472     if not CreateDirectory(DirName):
 473         EdkLogger.error(None, FILE_CREATE_FAILURE, "Could not create directory %s" % DirName)
 474     else:
 475         if DirName == '':
 476             DirName = os.getcwd()
 477         if not os.access(DirName, os.W_OK):
 478             EdkLogger.error(None, PERMISSION_FAILURE, "Do not have write permission on directory %s" % DirName)
 479
 480     try:
 481         Fd = open(File, "wb")
 482         Fd.write(Content)
 483         Fd.close()
 484     except IOError as X:
 485         EdkLogger.error(None, FILE_CREATE_FAILURE, ExtraData='IOError %s' % X)
 486
 487     return True
 488
 489 ## Make a Python object persistent on file system
 490 #
 491 #   @param      Data    The object to be stored in file
 492 #   @param      File    The path of file to store the object
 493 #
 494 def DataDump(Data, File):
 495     Fd = None
 496     try:
 497         Fd = open(File, 'wb')
 498         pickle.dump(Data, Fd, pickle.HIGHEST_PROTOCOL)
 499     except:
 500         EdkLogger.error("", FILE_OPEN_FAILURE, ExtraData=File, RaiseError=False)
 501     finally:
 502         if Fd is not None:
 503             Fd.close()
 504
 505 ## Restore a Python object from a file
 506 #
 507 #   @param      File    The path of file stored the object
 508 #
 509 #   @retval     object  A python object
 510 #   @retval     None    If failure in file operation
 511 #
 512 def DataRestore(File):
 513     Data = None
 514     Fd = None
 515     try:
 516         Fd = open(File, 'rb')
 517         Data = pickle.load(Fd)
 518     except Exception as e:
 519         EdkLogger.verbose("Failed to load [%s]\n\t%s" % (File, str(e)))
 520         Data = None
 521     finally:
 522         if Fd is not None:
 523             Fd.close()
 524     return Data
 525
 526 ## Retrieve and cache the real path name in file system
 527 #
 528 #   @param      Root    The root directory of path relative to
 529 #
 530 #   @retval     str     The path string if the path exists
 531 #   @retval     None    If path doesn't exist
 532 #
 533 class DirCache:
 534     _CACHE_ = set()
 535     _UPPER_CACHE_ = {}
 536
 537     def __init__(self, Root):
 538         self._Root = Root
 539         for F in os.listdir(Root):
 540             self._CACHE_.add(F)
 541             self._UPPER_CACHE_[F.upper()] = F
 542
 543     # =[] operator
 544     def __getitem__(self, Path):
 545         Path = Path[len(os.path.commonprefix([Path, self._Root])):]
 546         if not Path:
 547             return self._Root
 548         if Path and Path[0] == os.path.sep:
 549             Path = Path[1:]
 550         if Path in self._CACHE_:
 551             return os.path.join(self._Root, Path)
 552         UpperPath = Path.upper()
 553         if UpperPath in self._UPPER_CACHE_:
 554             return os.path.join(self._Root, self._UPPER_CACHE_[UpperPath])
 555
 556         IndexList = []
 557         LastSepIndex = -1
 558         SepIndex = Path.find(os.path.sep)
 559         while SepIndex > -1:
 560             Parent = UpperPath[:SepIndex]
 561             if Parent not in self._UPPER_CACHE_:
 562                 break
 563             LastSepIndex = SepIndex
 564             SepIndex = Path.find(os.path.sep, LastSepIndex + 1)
 565
 566         if LastSepIndex == -1:
 567             return None
 568
 569         Cwd = os.getcwd()
 570         os.chdir(self._Root)
 571         SepIndex = LastSepIndex
 572         while SepIndex > -1:
 573             Parent = Path[:SepIndex]
 574             ParentKey = UpperPath[:SepIndex]
 575             if ParentKey not in self._UPPER_CACHE_:
 576                 os.chdir(Cwd)
 577                 return None
 578
 579             if Parent in self._CACHE_:
 580                 ParentDir = Parent
 581             else:
 582                 ParentDir = self._UPPER_CACHE_[ParentKey]
 583             for F in os.listdir(ParentDir):
 584                 Dir = os.path.join(ParentDir, F)
 585                 self._CACHE_.add(Dir)
 586                 self._UPPER_CACHE_[Dir.upper()] = Dir
 587
 588             SepIndex = Path.find(os.path.sep, SepIndex + 1)
 589
 590         os.chdir(Cwd)
 591         if Path in self._CACHE_:
 592             return os.path.join(self._Root, Path)
 593         elif UpperPath in self._UPPER_CACHE_:
 594             return os.path.join(self._Root, self._UPPER_CACHE_[UpperPath])
 595         return None
 596
 597 def RealPath(File, Dir='', OverrideDir=''):
 598     NewFile = os.path.normpath(os.path.join(Dir, File))
 599     NewFile = GlobalData.gAllFiles[NewFile]
 600     if not NewFile and OverrideDir:
 601         NewFile = os.path.normpath(os.path.join(OverrideDir, File))
 602         NewFile = GlobalData.gAllFiles[NewFile]
 603     return NewFile
 604
 605 def RealPath2(File, Dir='', OverrideDir=''):
 606     NewFile = None
 607     if OverrideDir:
 608         NewFile = GlobalData.gAllFiles[os.path.normpath(os.path.join(OverrideDir, File))]
 609         if NewFile:
 610             if OverrideDir[-1] == os.path.sep:
 611                 return NewFile[len(OverrideDir):], NewFile[0:len(OverrideDir)]
 612             else:
 613                 return NewFile[len(OverrideDir) + 1:], NewFile[0:len(OverrideDir)]
 614     if GlobalData.gAllFiles:
 615         NewFile = GlobalData.gAllFiles[os.path.normpath(os.path.join(Dir, File))]
 616     if not NewFile:
 617         NewFile = os.path.normpath(os.path.join(Dir, File))
 618         if not os.path.exists(NewFile):
 619             return None, None
 620     if NewFile:
 621         if Dir:
 622             if Dir[-1] == os.path.sep:
 623                 return NewFile[len(Dir):], NewFile[0:len(Dir)]
 624             else:
 625                 return NewFile[len(Dir) + 1:], NewFile[0:len(Dir)]
 626         else:
 627             return NewFile, ''
 628
 629     return None, None
 630
 631 ## Get GUID value from given packages
 632 #
 633 #   @param      CName           The CName of the GUID
 634 #   @param      PackageList     List of packages looking-up in
 635 #   @param      Inffile         The driver file
 636 #
 637 #   @retval     GuidValue   if the CName is found in any given package
 638 #   @retval     None        if the CName is not found in all given packages
 639 #
 640 def GuidValue(CName, PackageList, Inffile = None):
 641     for P in PackageList:
 642         GuidKeys = P.Guids.keys()
 643         if Inffile and P._PrivateGuids:
 644             if not Inffile.startswith(P.MetaFile.Dir):
 645                 GuidKeys = [x for x in P.Guids if x not in P._PrivateGuids]
 646         if CName in GuidKeys:
 647             return P.Guids[CName]
 648     return None
 649
 650 ## Get Protocol value from given packages
 651 #
 652 #   @param      CName           The CName of the GUID
 653 #   @param      PackageList     List of packages looking-up in
 654 #   @param      Inffile         The driver file
 655 #
 656 #   @retval     GuidValue   if the CName is found in any given package
 657 #   @retval     None        if the CName is not found in all given packages
 658 #
 659 def ProtocolValue(CName, PackageList, Inffile = None):
 660     for P in PackageList:
 661         ProtocolKeys = P.Protocols.keys()
 662         if Inffile and P._PrivateProtocols:
 663             if not Inffile.startswith(P.MetaFile.Dir):
 664                 ProtocolKeys = [x for x in P.Protocols if x not in P._PrivateProtocols]
 665         if CName in ProtocolKeys:
 666             return P.Protocols[CName]
 667     return None
 668
 669 ## Get PPI value from given packages
 670 #
 671 #   @param      CName           The CName of the GUID
 672 #   @param      PackageList     List of packages looking-up in
 673 #   @param      Inffile         The driver file
 674 #
 675 #   @retval     GuidValue   if the CName is found in any given package
 676 #   @retval     None        if the CName is not found in all given packages
 677 #
 678 def PpiValue(CName, PackageList, Inffile = None):
 679     for P in PackageList:
 680         PpiKeys = P.Ppis.keys()
 681         if Inffile and P._PrivatePpis:
 682             if not Inffile.startswith(P.MetaFile.Dir):
 683                 PpiKeys = [x for x in P.Ppis if x not in P._PrivatePpis]
 684         if CName in PpiKeys:
 685             return P.Ppis[CName]
 686     return None
 687
 688 ## A string template class
 689 #
 690 #  This class implements a template for string replacement. A string template
 691 #  looks like following
 692 #
 693 #       ${BEGIN} other_string ${placeholder_name} other_string ${END}
 694 #
 695 #  The string between ${BEGIN} and ${END} will be repeated as many times as the
 696 #  length of "placeholder_name", which is a list passed through a dict. The
 697 #  "placeholder_name" is the key name of the dict. The ${BEGIN} and ${END} can
 698 #  be not used and, in this case, the "placeholder_name" must not a list and it
 699 #  will just be replaced once.
 700 #
 701 class TemplateString(object):
 702     _REPEAT_START_FLAG = "BEGIN"
 703     _REPEAT_END_FLAG = "END"
 704
 705     class Section(object):
 706         _LIST_TYPES = [type([]), type(set()), type((0,))]
 707
 708         def __init__(self, TemplateSection, PlaceHolderList):
 709             self._Template = TemplateSection
 710             self._PlaceHolderList = []
 711
 712             # Split the section into sub-sections according to the position of placeholders
 713             if PlaceHolderList:
 714                 self._SubSectionList = []
 715                 SubSectionStart = 0
 716                 #
 717                 # The placeholders passed in must be in the format of
 718                 #
 719                 #   PlaceHolderName, PlaceHolderStartPoint, PlaceHolderEndPoint
 720                 #
 721                 for PlaceHolder, Start, End in PlaceHolderList:
 722                     self._SubSectionList.append(TemplateSection[SubSectionStart:Start])
 723                     self._SubSectionList.append(TemplateSection[Start:End])
 724                     self._PlaceHolderList.append(PlaceHolder)
 725                     SubSectionStart = End
 726                 if SubSectionStart < len(TemplateSection):
 727                     self._SubSectionList.append(TemplateSection[SubSectionStart:])
 728             else:
 729                 self._SubSectionList = [TemplateSection]
 730
 731         def __str__(self):
 732             return self._Template + " : " + str(self._PlaceHolderList)
 733
 734         def Instantiate(self, PlaceHolderValues):
 735             RepeatTime = -1
 736             RepeatPlaceHolders = {}
 737             NonRepeatPlaceHolders = {}
 738
 739             for PlaceHolder in self._PlaceHolderList:
 740                 if PlaceHolder not in PlaceHolderValues:
 741                     continue
 742                 Value = PlaceHolderValues[PlaceHolder]
 743                 if type(Value) in self._LIST_TYPES:
 744                     if RepeatTime < 0:
 745                         RepeatTime = len(Value)
 746                     elif RepeatTime != len(Value):
 747                         EdkLogger.error(
 748                                     "TemplateString",
 749                                     PARAMETER_INVALID,
 750                                     "${%s} has different repeat time from others!" % PlaceHolder,
 751                                     ExtraData=str(self._Template)
 752                                     )
 753                     RepeatPlaceHolders["${%s}" % PlaceHolder] = Value
 754                 else:
 755                     NonRepeatPlaceHolders["${%s}" % PlaceHolder] = Value
 756
 757             if NonRepeatPlaceHolders:
 758                 StringList = []
 759                 for S in self._SubSectionList:
 760                     if S not in NonRepeatPlaceHolders:
 761                         StringList.append(S)
 762                     else:
 763                         StringList.append(str(NonRepeatPlaceHolders[S]))
 764             else:
 765                 StringList = self._SubSectionList
 766
 767             if RepeatPlaceHolders:
 768                 TempStringList = []
 769                 for Index in range(RepeatTime):
 770                     for S in StringList:
 771                         if S not in RepeatPlaceHolders:
 772                             TempStringList.append(S)
 773                         else:
 774                             TempStringList.append(str(RepeatPlaceHolders[S][Index]))
 775                 StringList = TempStringList
 776
 777             return "".join(StringList)
 778
 779     ## Constructor
 780     def __init__(self, Template=None):
 781         self.String = ''
 782         self.IsBinary = False
 783         self._Template = Template
 784         self._TemplateSectionList = self._Parse(Template)
 785
 786     ## str() operator
 787     #
 788     #   @retval     string  The string replaced
 789     #
 790     def __str__(self):
 791         return self.String
 792
 793     ## Split the template string into fragments per the ${BEGIN} and ${END} flags
 794     #
 795     #   @retval     list    A list of TemplateString.Section objects
 796     #
 797     def _Parse(self, Template):
 798         SectionStart = 0
 799         SearchFrom = 0
 800         MatchEnd = 0
 801         PlaceHolderList = []
 802         TemplateSectionList = []
 803         while Template:
 804             MatchObj = gPlaceholderPattern.search(Template, SearchFrom)
 805             if not MatchObj:
 806                 if MatchEnd <= len(Template):
 807                     TemplateSection = TemplateString.Section(Template[SectionStart:], PlaceHolderList)
 808                     TemplateSectionList.append(TemplateSection)
 809                 break
 810
 811             MatchString = MatchObj.group(1)
 812             MatchStart = MatchObj.start()
 813             MatchEnd = MatchObj.end()
 814
 815             if MatchString == self._REPEAT_START_FLAG:
 816                 if MatchStart > SectionStart:
 817                     TemplateSection = TemplateString.Section(Template[SectionStart:MatchStart], PlaceHolderList)
 818                     TemplateSectionList.append(TemplateSection)
 819                 SectionStart = MatchEnd
 820                 PlaceHolderList = []
 821             elif MatchString == self._REPEAT_END_FLAG:
 822                 TemplateSection = TemplateString.Section(Template[SectionStart:MatchStart], PlaceHolderList)
 823                 TemplateSectionList.append(TemplateSection)
 824                 SectionStart = MatchEnd
 825                 PlaceHolderList = []
 826             else:
 827                 PlaceHolderList.append((MatchString, MatchStart - SectionStart, MatchEnd - SectionStart))
 828             SearchFrom = MatchEnd
 829         return TemplateSectionList
 830
 831     ## Replace the string template with dictionary of placeholders and append it to previous one
 832     #
 833     #   @param      AppendString    The string template to append
 834     #   @param      Dictionary      The placeholder dictionaries
 835     #
 836     def Append(self, AppendString, Dictionary=None):
 837         if Dictionary:
 838             SectionList = self._Parse(AppendString)
 839             self.String += "".join(S.Instantiate(Dictionary) for S in SectionList)
 840         else:
 841             self.String += AppendString
 842
 843     ## Replace the string template with dictionary of placeholders
 844     #
 845     #   @param      Dictionary      The placeholder dictionaries
 846     #
 847     #   @retval     str             The string replaced with placeholder values
 848     #
 849     def Replace(self, Dictionary=None):
 850         return "".join(S.Instantiate(Dictionary) for S in self._TemplateSectionList)
 851
 852 ## Progress indicator class
 853 #
 854 #  This class makes use of thread to print progress on console.
 855 #
 856 class Progressor:
 857     # for avoiding deadloop
 858     _StopFlag = None
 859     _ProgressThread = None
 860     _CheckInterval = 0.25
 861
 862     ## Constructor
 863     #
 864     #   @param      OpenMessage     The string printed before progress charaters
 865     #   @param      CloseMessage    The string printed after progress charaters
 866     #   @param      ProgressChar    The charater used to indicate the progress
 867     #   @param      Interval        The interval in seconds between two progress charaters
 868     #
 869     def __init__(self, OpenMessage="", CloseMessage="", ProgressChar='.', Interval=1.0):
 870         self.PromptMessage = OpenMessage
 871         self.CodaMessage = CloseMessage
 872         self.ProgressChar = ProgressChar
 873         self.Interval = Interval
 874         if Progressor._StopFlag is None:
 875             Progressor._StopFlag = threading.Event()
 876
 877     ## Start to print progress charater
 878     #
 879     #   @param      OpenMessage     The string printed before progress charaters
 880     #
 881     def Start(self, OpenMessage=None):
 882         if OpenMessage is not None:
 883             self.PromptMessage = OpenMessage
 884         Progressor._StopFlag.clear()
 885         if Progressor._ProgressThread is None:
 886             Progressor._ProgressThread = threading.Thread(target=self._ProgressThreadEntry)
 887             Progressor._ProgressThread.setDaemon(False)
 888             Progressor._ProgressThread.start()
 889
 890     ## Stop printing progress charater
 891     #
 892     #   @param      CloseMessage    The string printed after progress charaters
 893     #
 894     def Stop(self, CloseMessage=None):
 895         OriginalCodaMessage = self.CodaMessage
 896         if CloseMessage is not None:
 897             self.CodaMessage = CloseMessage
 898         self.Abort()
 899         self.CodaMessage = OriginalCodaMessage
 900
 901     ## Thread entry method
 902     def _ProgressThreadEntry(self):
 903         sys.stdout.write(self.PromptMessage + " ")
 904         sys.stdout.flush()
 905         TimeUp = 0.0
 906         while not Progressor._StopFlag.isSet():
 907             if TimeUp <= 0.0:
 908                 sys.stdout.write(self.ProgressChar)
 909                 sys.stdout.flush()
 910                 TimeUp = self.Interval
 911             time.sleep(self._CheckInterval)
 912             TimeUp -= self._CheckInterval
 913         sys.stdout.write(" " + self.CodaMessage + "\n")
 914         sys.stdout.flush()
 915
 916     ## Abort the progress display
 917     @staticmethod
 918     def Abort():
 919         if Progressor._StopFlag is not None:
 920             Progressor._StopFlag.set()
 921         if Progressor._ProgressThread is not None:
 922             Progressor._ProgressThread.join()
 923             Progressor._ProgressThread = None
 924
 925 ## A dict which can access its keys and/or values orderly
 926 #
 927 #  The class implements a new kind of dict which its keys or values can be
 928 #  accessed in the order they are added into the dict. It guarantees the order
 929 #  by making use of an internal list to keep a copy of keys.
 930 #
 931 class sdict(IterableUserDict):
 932     ## Constructor
 933     def __init__(self):
 934         IterableUserDict.__init__(self)
 935         self._key_list = []
 936
 937     ## [] operator
 938     def __setitem__(self, key, value):
 939         if key not in self._key_list:
 940             self._key_list.append(key)
 941         IterableUserDict.__setitem__(self, key, value)
 942
 943     ## del operator
 944     def __delitem__(self, key):
 945         self._key_list.remove(key)
 946         IterableUserDict.__delitem__(self, key)
 947
 948     ## used in "for k in dict" loop to ensure the correct order
 949     def __iter__(self):
 950         return self.iterkeys()
 951
 952     ## len() support
 953     def __len__(self):
 954         return len(self._key_list)
 955
 956     ## "in" test support
 957     def __contains__(self, key):
 958         return key in self._key_list
 959
 960     ## indexof support
 961     def index(self, key):
 962         return self._key_list.index(key)
 963
 964     ## insert support
 965     def insert(self, key, newkey, newvalue, order):
 966         index = self._key_list.index(key)
 967         if order == 'BEFORE':
 968             self._key_list.insert(index, newkey)
 969             IterableUserDict.__setitem__(self, newkey, newvalue)
 970         elif order == 'AFTER':
 971             self._key_list.insert(index + 1, newkey)
 972             IterableUserDict.__setitem__(self, newkey, newvalue)
 973
 974     ## append support
 975     def append(self, sdict):
 976         for key in sdict:
 977             if key not in self._key_list:
 978                 self._key_list.append(key)
 979             IterableUserDict.__setitem__(self, key, sdict[key])
 980
 981     def has_key(self, key):
 982         return key in self._key_list
 983
 984     ## Empty the dict
 985     def clear(self):
 986         self._key_list = []
 987         IterableUserDict.clear(self)
 988
 989     ## Return a copy of keys
 990     def keys(self):
 991         keys = []
 992         for key in self._key_list:
 993             keys.append(key)
 994         return keys
 995
 996     ## Return a copy of values
 997     def values(self):
 998         values = []
 999         for key in self._key_list:
1000             values.append(self[key])
1001         return values
1002
1003     ## Return a copy of (key, value) list
1004     def items(self):
1005         items = []
1006         for key in self._key_list:
1007             items.append((key, self[key]))
1008         return items
1009
1010     ## Iteration support
1011     def iteritems(self):
1012         return iter(self.items())
1013
1014     ## Keys interation support
1015     def iterkeys(self):
1016         return iter(self.keys())
1017
1018     ## Values interation support
1019     def itervalues(self):
1020         return iter(self.values())
1021
1022     ## Return value related to a key, and remove the (key, value) from the dict
1023     def pop(self, key, *dv):
1024         value = None
1025         if key in self._key_list:
1026             value = self[key]
1027             self.__delitem__(key)
1028         elif len(dv) != 0 :
1029             value = kv[0]
1030         return value
1031
1032     ## Return (key, value) pair, and remove the (key, value) from the dict
1033     def popitem(self):
1034         key = self._key_list[-1]
1035         value = self[key]
1036         self.__delitem__(key)
1037         return key, value
1038
1039     def update(self, dict=None, **kwargs):
1040         if dict is not None:
1041             for k, v in dict.items():
1042                 self[k] = v
1043         if len(kwargs):
1044             for k, v in kwargs.items():
1045                 self[k] = v
1046
1047 ## Dictionary with restricted keys
1048 #
1049 class rdict(dict):
1050     ## Constructor
1051     def __init__(self, KeyList):
1052         for Key in KeyList:
1053             dict.__setitem__(self, Key, "")
1054
1055     ## []= operator
1056     def __setitem__(self, key, value):
1057         if key not in self:
1058             EdkLogger.error("RestrictedDict", ATTRIBUTE_SET_FAILURE, "Key [%s] is not allowed" % key,
1059                             ExtraData=", ".join(dict.keys(self)))
1060         dict.__setitem__(self, key, value)
1061
1062     ## =[] operator
1063     def __getitem__(self, key):
1064         if key not in self:
1065             return ""
1066         return dict.__getitem__(self, key)
1067
1068     ## del operator
1069     def __delitem__(self, key):
1070         EdkLogger.error("RestrictedDict", ATTRIBUTE_ACCESS_DENIED, ExtraData="del")
1071
1072     ## Empty the dict
1073     def clear(self):
1074         for Key in self:
1075             self.__setitem__(Key, "")
1076
1077     ## Return value related to a key, and remove the (key, value) from the dict
1078     def pop(self, key, *dv):
1079         EdkLogger.error("RestrictedDict", ATTRIBUTE_ACCESS_DENIED, ExtraData="pop")
1080
1081     ## Return (key, value) pair, and remove the (key, value) from the dict
1082     def popitem(self):
1083         EdkLogger.error("RestrictedDict", ATTRIBUTE_ACCESS_DENIED, ExtraData="popitem")
1084
1085 ## Dictionary using prioritized list as key
1086 #
1087 class tdict:
1088     _ListType = type([])
1089     _TupleType = type(())
1090     _Wildcard = 'COMMON'
1091     _ValidWildcardList = ['COMMON', 'DEFAULT', 'ALL', TAB_STAR, 'PLATFORM']
1092
1093     def __init__(self, _Single_=False, _Level_=2):
1094         self._Level_ = _Level_
1095         self.data = {}
1096         self._Single_ = _Single_
1097
1098     # =[] operator
1099     def __getitem__(self, key):
1100         KeyType = type(key)
1101         RestKeys = None
1102         if KeyType == self._ListType or KeyType == self._TupleType:
1103             FirstKey = key[0]
1104             if len(key) > 1:
1105                 RestKeys = key[1:]
1106             elif self._Level_ > 1:
1107                 RestKeys = [self._Wildcard for i in range(0, self._Level_ - 1)]
1108         else:
1109             FirstKey = key
1110             if self._Level_ > 1:
1111                 RestKeys = [self._Wildcard for i in range(0, self._Level_ - 1)]
1112
1113         if FirstKey is None or str(FirstKey).upper() in self._ValidWildcardList:
1114             FirstKey = self._Wildcard
1115
1116         if self._Single_:
1117             return self._GetSingleValue(FirstKey, RestKeys)
1118         else:
1119             return self._GetAllValues(FirstKey, RestKeys)
1120
1121     def _GetSingleValue(self, FirstKey, RestKeys):
1122         Value = None
1123         #print "%s-%s" % (FirstKey, self._Level_) ,
1124         if self._Level_ > 1:
1125             if FirstKey == self._Wildcard:
1126                 if FirstKey in self.data:
1127                     Value = self.data[FirstKey][RestKeys]
1128                 if Value is None:
1129                     for Key in self.data:
1130                         Value = self.data[Key][RestKeys]
1131                         if Value is not None: break
1132             else:
1133                 if FirstKey in self.data:
1134                     Value = self.data[FirstKey][RestKeys]
1135                 if Value is None and self._Wildcard in self.data:
1136                     #print "Value=None"
1137                     Value = self.data[self._Wildcard][RestKeys]
1138         else:
1139             if FirstKey == self._Wildcard:
1140                 if FirstKey in self.data:
1141                     Value = self.data[FirstKey]
1142                 if Value is None:
1143                     for Key in self.data:
1144                         Value = self.data[Key]
1145                         if Value is not None: break
1146             else:
1147                 if FirstKey in self.data:
1148                     Value = self.data[FirstKey]
1149                 elif self._Wildcard in self.data:
1150                     Value = self.data[self._Wildcard]
1151         return Value
1152
1153     def _GetAllValues(self, FirstKey, RestKeys):
1154         Value = []
1155         if self._Level_ > 1:
1156             if FirstKey == self._Wildcard:
1157                 for Key in self.data:
1158                     Value += self.data[Key][RestKeys]
1159             else:
1160                 if FirstKey in self.data:
1161                     Value += self.data[FirstKey][RestKeys]
1162                 if self._Wildcard in self.data:
1163                     Value += self.data[self._Wildcard][RestKeys]
1164         else:
1165             if FirstKey == self._Wildcard:
1166                 for Key in self.data:
1167                     Value.append(self.data[Key])
1168             else:
1169                 if FirstKey in self.data:
1170                     Value.append(self.data[FirstKey])
1171                 if self._Wildcard in self.data:
1172                     Value.append(self.data[self._Wildcard])
1173         return Value
1174
1175     ## []= operator
1176     def __setitem__(self, key, value):
1177         KeyType = type(key)
1178         RestKeys = None
1179         if KeyType == self._ListType or KeyType == self._TupleType:
1180             FirstKey = key[0]
1181             if len(key) > 1:
1182                 RestKeys = key[1:]
1183             else:
1184                 RestKeys = [self._Wildcard for i in range(0, self._Level_ - 1)]
1185         else:
1186             FirstKey = key
1187             if self._Level_ > 1:
1188                 RestKeys = [self._Wildcard for i in range(0, self._Level_ - 1)]
1189
1190         if FirstKey in self._ValidWildcardList:
1191             FirstKey = self._Wildcard
1192
1193         if FirstKey not in self.data and self._Level_ > 0:
1194             self.data[FirstKey] = tdict(self._Single_, self._Level_ - 1)
1195
1196         if self._Level_ > 1:
1197             self.data[FirstKey][RestKeys] = value
1198         else:
1199             self.data[FirstKey] = value
1200
1201     def SetGreedyMode(self):
1202         self._Single_ = False
1203         if self._Level_ > 1:
1204             for Key in self.data:
1205                 self.data[Key].SetGreedyMode()
1206
1207     def SetSingleMode(self):
1208         self._Single_ = True
1209         if self._Level_ > 1:
1210             for Key in self.data:
1211                 self.data[Key].SetSingleMode()
1212
1213     def GetKeys(self, KeyIndex=0):
1214         assert KeyIndex >= 0
1215         if KeyIndex == 0:
1216             return set(self.data.keys())
1217         else:
1218             keys = set()
1219             for Key in self.data:
1220                 keys |= self.data[Key].GetKeys(KeyIndex - 1)
1221             return keys
1222
1223 def IsFieldValueAnArray (Value):
1224     Value = Value.strip()
1225     if Value.startswith(TAB_GUID) and Value.endswith(')'):
1226         return True
1227     if Value.startswith('L"') and Value.endswith('"')  and len(list(Value[2:-1])) > 1:
1228         return True
1229     if Value[0] == '"' and Value[-1] == '"' and len(list(Value[1:-1])) > 1:
1230         return True
1231     if Value[0] == '{' and Value[-1] == '}':
1232         return True
1233     if Value.startswith("L'") and Value.endswith("'") and len(list(Value[2:-1])) > 1:
1234         return True
1235     if Value[0] == "'" and Value[-1] == "'" and len(list(Value[1:-1])) > 1:
1236         return True
1237     return False
1238
1239 def AnalyzePcdExpression(Setting):
1240     RanStr = ''.join(sample(string.ascii_letters + string.digits, 8))
1241     Setting = Setting.replace('\\\\', RanStr).strip()
1242     # There might be escaped quote in a string: \", \\\" , \', \\\'
1243     Data = Setting
1244     # There might be '|' in string and in ( ... | ... ), replace it with '-'
1245     NewStr = ''
1246     InSingleQuoteStr = False
1247     InDoubleQuoteStr = False
1248     Pair = 0
1249     for Index, ch in enumerate(Data):
1250         if ch == '"' and not InSingleQuoteStr:
1251             if Data[Index - 1] != '\\':
1252                 InDoubleQuoteStr = not InDoubleQuoteStr
1253         elif ch == "'" and not InDoubleQuoteStr:
1254             if Data[Index - 1] != '\\':
1255                 InSingleQuoteStr = not InSingleQuoteStr
1256         elif ch == '(' and not (InSingleQuoteStr or InDoubleQuoteStr):
1257             Pair += 1
1258         elif ch == ')' and not (InSingleQuoteStr or InDoubleQuoteStr):
1259             Pair -= 1
1260
1261         if (Pair > 0 or InSingleQuoteStr or InDoubleQuoteStr) and ch == TAB_VALUE_SPLIT:
1262             NewStr += '-'
1263         else:
1264             NewStr += ch
1265     FieldList = []
1266     StartPos = 0
1267     while True:
1268         Pos = NewStr.find(TAB_VALUE_SPLIT, StartPos)
1269         if Pos < 0:
1270             FieldList.append(Setting[StartPos:].strip())
1271             break
1272         FieldList.append(Setting[StartPos:Pos].strip())
1273         StartPos = Pos + 1
1274     for i, ch in enumerate(FieldList):
1275         if RanStr in ch:
1276             FieldList[i] = ch.replace(RanStr,'\\\\')
1277     return FieldList
1278
1279 def ParseDevPathValue (Value):
1280     if '\\' in Value:
1281         Value.replace('\\', '/').replace(' ', '')
1282
1283     Cmd = 'DevicePath ' + '"' + Value + '"'
1284     try:
1285         p = subprocess.Popen(Cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
1286         out, err = p.communicate()
1287     except Exception as X:
1288         raise BadExpression("DevicePath: %s" % (str(X)) )
1289     finally:
1290         subprocess._cleanup()
1291         p.stdout.close()
1292         p.stderr.close()
1293     if err:
1294         raise BadExpression("DevicePath: %s" % str(err))
1295     Size = len(out.split())
1296     out = ','.join(out.split())
1297     return '{' + out + '}', Size
1298
1299 def ParseFieldValue (Value):
1300     if isinstance(Value, type(0)):
1301         return Value, (Value.bit_length() + 7) / 8
1302     if not isinstance(Value, type('')):
1303         raise BadExpression('Type %s is %s' %(Value, type(Value)))
1304     Value = Value.strip()
1305     if Value.startswith(TAB_UINT8) and Value.endswith(')'):
1306         Value, Size = ParseFieldValue(Value.split('(', 1)[1][:-1])
1307         if Size > 1:
1308             raise BadExpression('Value (%s) Size larger than %d' %(Value, Size))
1309         return Value, 1
1310     if Value.startswith(TAB_UINT16) and Value.endswith(')'):
1311         Value, Size = ParseFieldValue(Value.split('(', 1)[1][:-1])
1312         if Size > 2:
1313             raise BadExpression('Value (%s) Size larger than %d' %(Value, Size))
1314         return Value, 2
1315     if Value.startswith(TAB_UINT32) and Value.endswith(')'):
1316         Value, Size = ParseFieldValue(Value.split('(', 1)[1][:-1])
1317         if Size > 4:
1318             raise BadExpression('Value (%s) Size larger than %d' %(Value, Size))
1319         return Value, 4
1320     if Value.startswith(TAB_UINT64) and Value.endswith(')'):
1321         Value, Size = ParseFieldValue(Value.split('(', 1)[1][:-1])
1322         if Size > 8:
1323             raise BadExpression('Value (%s) Size larger than %d' % (Value, Size))
1324         return Value, 8
1325     if Value.startswith(TAB_GUID) and Value.endswith(')'):
1326         Value = Value.split('(', 1)[1][:-1].strip()
1327         if Value[0] == '{' and Value[-1] == '}':
1328             TmpValue = GuidStructureStringToGuidString(Value)
1329             if not TmpValue:
1330                 raise BadExpression("Invalid GUID value string %s" % Value)
1331             Value = TmpValue
1332         if Value[0] == '"' and Value[-1] == '"':
1333             Value = Value[1:-1]
1334         try:
1335             Value = "'" + uuid.UUID(Value).get_bytes_le() + "'"
1336         except ValueError as Message:
1337             raise BadExpression(Message)
1338         Value, Size = ParseFieldValue(Value)
1339         return Value, 16
1340     if Value.startswith('L"') and Value.endswith('"'):
1341         # Unicode String
1342         # translate escape character
1343         Value = Value[1:]
1344         try:
1345             Value = eval(Value)
1346         except:
1347             Value = Value[1:-1]
1348         List = list(Value)
1349         List.reverse()
1350         Value = 0
1351         for Char in List:
1352             Value = (Value << 16) | ord(Char)
1353         return Value, (len(List) + 1) * 2
1354     if Value.startswith('"') and Value.endswith('"'):
1355         # ASCII String
1356         # translate escape character
1357         try:
1358             Value = eval(Value)
1359         except:
1360             Value = Value[1:-1]
1361         List = list(Value)
1362         List.reverse()
1363         Value = 0
1364         for Char in List:
1365             Value = (Value << 8) | ord(Char)
1366         return Value, len(List) + 1
1367     if Value.startswith("L'") and Value.endswith("'"):
1368         # Unicode Character Constant
1369         # translate escape character
1370         Value = Value[1:]
1371         try:
1372             Value = eval(Value)
1373         except:
1374             Value = Value[1:-1]
1375         List = list(Value)
1376         if len(List) == 0:
1377             raise BadExpression('Length %s is %s' % (Value, len(List)))
1378         List.reverse()
1379         Value = 0
1380         for Char in List:
1381             Value = (Value << 16) | ord(Char)
1382         return Value, len(List) * 2
1383     if Value.startswith("'") and Value.endswith("'"):
1384         # Character constant
1385         # translate escape character
1386         try:
1387             Value = eval(Value)
1388         except:
1389             Value = Value[1:-1]
1390         List = list(Value)
1391         if len(List) == 0:
1392             raise BadExpression('Length %s is %s' % (Value, len(List)))
1393         List.reverse()
1394         Value = 0
1395         for Char in List:
1396             Value = (Value << 8) | ord(Char)
1397         return Value, len(List)
1398     if Value.startswith('{') and Value.endswith('}'):
1399         # Byte array
1400         Value = Value[1:-1]
1401         List = [Item.strip() for Item in Value.split(',')]
1402         List.reverse()
1403         Value = 0
1404         RetSize = 0
1405         for Item in List:
1406             ItemValue, Size = ParseFieldValue(Item)
1407             RetSize += Size
1408             for I in range(Size):
1409                 Value = (Value << 8) | ((ItemValue >> 8 * I) & 0xff)
1410         return Value, RetSize
1411     if Value.startswith('DEVICE_PATH(') and Value.endswith(')'):
1412         Value = Value.replace("DEVICE_PATH(", '').rstrip(')')
1413         Value = Value.strip().strip('"')
1414         return ParseDevPathValue(Value)
1415     if Value.lower().startswith('0x'):
1416         try:
1417             Value = int(Value, 16)
1418         except:
1419             raise BadExpression("invalid hex value: %s" % Value)
1420         if Value == 0:
1421             return 0, 1
1422         return Value, (Value.bit_length() + 7) / 8
1423     if Value[0].isdigit():
1424         Value = int(Value, 10)
1425         if Value == 0:
1426             return 0, 1
1427         return Value, (Value.bit_length() + 7) / 8
1428     if Value.lower() == 'true':
1429         return 1, 1
1430     if Value.lower() == 'false':
1431         return 0, 1
1432     return Value, 1
1433
1434 ## AnalyzeDscPcd
1435 #
1436 #  Analyze DSC PCD value, since there is no data type info in DSC
1437 #  This function is used to match functions (AnalyzePcdData) used for retrieving PCD value from database
1438 #  1. Feature flag: TokenSpace.PcdCName|PcdValue
1439 #  2. Fix and Patch:TokenSpace.PcdCName|PcdValue[|VOID*[|MaxSize]]
1440 #  3. Dynamic default:
1441 #     TokenSpace.PcdCName|PcdValue[|VOID*[|MaxSize]]
1442 #     TokenSpace.PcdCName|PcdValue
1443 #  4. Dynamic VPD:
1444 #     TokenSpace.PcdCName|VpdOffset[|VpdValue]
1445 #     TokenSpace.PcdCName|VpdOffset[|MaxSize[|VpdValue]]
1446 #  5. Dynamic HII:
1447 #     TokenSpace.PcdCName|HiiString|VariableGuid|VariableOffset[|HiiValue]
1448 #  PCD value needs to be located in such kind of string, and the PCD value might be an expression in which
1449 #    there might have "|" operator, also in string value.
1450 #
1451 #  @param Setting: String contain information described above with "TokenSpace.PcdCName|" stripped
1452 #  @param PcdType: PCD type: feature, fixed, dynamic default VPD HII
1453 #  @param DataType: The datum type of PCD: VOID*, UNIT, BOOL
1454 #  @retval:
1455 #    ValueList: A List contain fields described above
1456 #    IsValid:   True if conforming EBNF, otherwise False
1457 #    Index:     The index where PcdValue is in ValueList
1458 #
1459 def AnalyzeDscPcd(Setting, PcdType, DataType=''):
1460     FieldList = AnalyzePcdExpression(Setting)
1461
1462     IsValid = True
1463     if PcdType in (MODEL_PCD_FIXED_AT_BUILD, MODEL_PCD_PATCHABLE_IN_MODULE, MODEL_PCD_DYNAMIC_DEFAULT, MODEL_PCD_DYNAMIC_EX_DEFAULT):
1464         Value = FieldList[0]
1465         Size = ''
1466         if len(FieldList) > 1 and FieldList[1]:
1467             DataType = FieldList[1]
1468             if FieldList[1] != TAB_VOID and StructPattern.match(FieldList[1]) is None:
1469                 IsValid = False
1470         if len(FieldList) > 2:
1471             Size = FieldList[2]
1472         if IsValid:
1473             if DataType == "":
1474                 IsValid = (len(FieldList) <= 1)
1475             else:
1476                 IsValid = (len(FieldList) <= 3)
1477
1478         if Size:
1479             try:
1480                 int(Size, 16) if Size.upper().startswith("0X") else int(Size)
1481             except:
1482                 IsValid = False
1483                 Size = -1
1484         return [str(Value), DataType, str(Size)], IsValid, 0
1485     elif PcdType == MODEL_PCD_FEATURE_FLAG:
1486         Value = FieldList[0]
1487         Size = ''
1488         IsValid = (len(FieldList) <= 1)
1489         return [Value, DataType, str(Size)], IsValid, 0
1490     elif PcdType in (MODEL_PCD_DYNAMIC_VPD, MODEL_PCD_DYNAMIC_EX_VPD):
1491         VpdOffset = FieldList[0]
1492         Value = Size = ''
1493         if not DataType == TAB_VOID:
1494             if len(FieldList) > 1:
1495                 Value = FieldList[1]
1496         else:
1497             if len(FieldList) > 1:
1498                 Size = FieldList[1]
1499             if len(FieldList) > 2:
1500                 Value = FieldList[2]
1501         if DataType == "":
1502             IsValid = (len(FieldList) <= 1)
1503         else:
1504             IsValid = (len(FieldList) <= 3)
1505         if Size:
1506             try:
1507                 int(Size, 16) if Size.upper().startswith("0X") else int(Size)
1508             except:
1509                 IsValid = False
1510                 Size = -1
1511         return [VpdOffset, str(Size), Value], IsValid, 2
1512     elif PcdType in (MODEL_PCD_DYNAMIC_HII, MODEL_PCD_DYNAMIC_EX_HII):
1513         IsValid = (3 <= len(FieldList) <= 5)
1514         HiiString = FieldList[0]
1515         Guid = Offset = Value = Attribute = ''
1516         if len(FieldList) > 1:
1517             Guid = FieldList[1]
1518         if len(FieldList) > 2:
1519             Offset = FieldList[2]
1520         if len(FieldList) > 3:
1521             Value = FieldList[3]
1522             if not Value:
1523                 IsValid = False
1524         if len(FieldList) > 4:
1525             Attribute = FieldList[4]
1526         return [HiiString, Guid, Offset, Value, Attribute], IsValid, 3
1527     return [], False, 0
1528
1529 ## AnalyzePcdData
1530 #
1531 #  Analyze the pcd Value, Datum type and TokenNumber.
1532 #  Used to avoid split issue while the value string contain "|" character
1533 #
1534 #  @param[in] Setting:  A String contain value/datum type/token number information;
1535 #
1536 #  @retval   ValueList: A List contain value, datum type and toke number.
1537 #
1538 def AnalyzePcdData(Setting):
1539     ValueList = ['', '', '']
1540
1541     ValueRe = re.compile(r'^\s*L?\".*\|.*\"')
1542     PtrValue = ValueRe.findall(Setting)
1543
1544     ValueUpdateFlag = False
1545
1546     if len(PtrValue) >= 1:
1547         Setting = re.sub(ValueRe, '', Setting)
1548         ValueUpdateFlag = True
1549
1550     TokenList = Setting.split(TAB_VALUE_SPLIT)
1551     ValueList[0:len(TokenList)] = TokenList
1552
1553     if ValueUpdateFlag:
1554         ValueList[0] = PtrValue[0]
1555
1556     return ValueList
1557
1558 ## check format of PCD value against its the datum type
1559 #
1560 # For PCD value setting
1561 #
1562 def CheckPcdDatum(Type, Value):
1563     if Type == TAB_VOID:
1564         ValueRe = re.compile(r'\s*L?\".*\"\s*$')
1565         if not (((Value.startswith('L"') or Value.startswith('"')) and Value.endswith('"'))
1566                 or (Value.startswith('{') and Value.endswith('}')) or (Value.startswith("L'") or Value.startswith("'") and Value.endswith("'"))
1567                ):
1568             return False, "Invalid value [%s] of type [%s]; must be in the form of {...} for array"\
1569                           ", \"...\" or \'...\' for string, L\"...\" or L\'...\' for unicode string" % (Value, Type)
1570         elif ValueRe.match(Value):
1571             # Check the chars in UnicodeString or CString is printable
1572             if Value.startswith("L"):
1573                 Value = Value[2:-1]
1574             else:
1575                 Value = Value[1:-1]
1576             Printset = set(string.printable)
1577             Printset.remove(TAB_PRINTCHAR_VT)
1578             Printset.add(TAB_PRINTCHAR_BS)
1579             Printset.add(TAB_PRINTCHAR_NUL)
1580             if not set(Value).issubset(Printset):
1581                 PrintList = sorted(Printset)
1582                 return False, "Invalid PCD string value of type [%s]; must be printable chars %s." % (Type, PrintList)
1583     elif Type == 'BOOLEAN':
1584         if Value not in ['TRUE', 'True', 'true', '0x1', '0x01', '1', 'FALSE', 'False', 'false', '0x0', '0x00', '0']:
1585             return False, "Invalid value [%s] of type [%s]; must be one of TRUE, True, true, 0x1, 0x01, 1"\
1586                           ", FALSE, False, false, 0x0, 0x00, 0" % (Value, Type)
1587     elif Type in [TAB_UINT8, TAB_UINT16, TAB_UINT32, TAB_UINT64]:
1588         if Value and int(Value, 0) < 0:
1589             return False, "PCD can't be set to negative value[%s] for datum type [%s]" % (Value, Type)
1590         try:
1591             Value = long(Value, 0)
1592             if Value > MAX_VAL_TYPE[Type]:
1593                 return False, "Too large PCD value[%s] for datum type [%s]" % (Value, Type)
1594         except:
1595             return False, "Invalid value [%s] of type [%s];"\
1596                           " must be a hexadecimal, decimal or octal in C language format." % (Value, Type)
1597     else:
1598         return True, "StructurePcd"
1599
1600     return True, ""
1601
1602 ## Split command line option string to list
1603 #
1604 # subprocess.Popen needs the args to be a sequence. Otherwise there's problem
1605 # in non-windows platform to launch command
1606 #
1607 def SplitOption(OptionString):
1608     OptionList = []
1609     LastChar = " "
1610     OptionStart = 0
1611     QuotationMark = ""
1612     for Index in range(0, len(OptionString)):
1613         CurrentChar = OptionString[Index]
1614         if CurrentChar in ['"', "'"]:
1615             if QuotationMark == CurrentChar:
1616                 QuotationMark = ""
1617             elif QuotationMark == "":
1618                 QuotationMark = CurrentChar
1619             continue
1620         elif QuotationMark:
1621             continue
1622
1623         if CurrentChar in ["/", "-"] and LastChar in [" ", "\t", "\r", "\n"]:
1624             if Index > OptionStart:
1625                 OptionList.append(OptionString[OptionStart:Index - 1])
1626             OptionStart = Index
1627         LastChar = CurrentChar
1628     OptionList.append(OptionString[OptionStart:])
1629     return OptionList
1630
1631 def CommonPath(PathList):
1632     P1 = min(PathList).split(os.path.sep)
1633     P2 = max(PathList).split(os.path.sep)
1634     for Index in xrange(min(len(P1), len(P2))):
1635         if P1[Index] != P2[Index]:
1636             return os.path.sep.join(P1[:Index])
1637     return os.path.sep.join(P1)
1638
1639 #
1640 # Convert string to C format array
1641 #
1642 def ConvertStringToByteArray(Value):
1643     Value = Value.strip()
1644     if not Value:
1645         return None
1646     if Value[0] == '{':
1647         if not Value.endswith('}'):
1648             return None
1649         Value = Value.replace(' ', '').replace('{', '').replace('}', '')
1650         ValFields = Value.split(',')
1651         try:
1652             for Index in range(len(ValFields)):
1653                 ValFields[Index] = str(int(ValFields[Index], 0))
1654         except ValueError:
1655             return None
1656         Value = '{' + ','.join(ValFields) + '}'
1657         return Value
1658
1659     Unicode = False
1660     if Value.startswith('L"'):
1661         if not Value.endswith('"'):
1662             return None
1663         Value = Value[1:]
1664         Unicode = True
1665     elif not Value.startswith('"') or not Value.endswith('"'):
1666         return None
1667
1668     Value = eval(Value)         # translate escape character
1669     NewValue = '{'
1670     for Index in range(0, len(Value)):
1671         if Unicode:
1672             NewValue = NewValue + str(ord(Value[Index]) % 0x10000) + ','
1673         else:
1674             NewValue = NewValue + str(ord(Value[Index]) % 0x100) + ','
1675     Value = NewValue + '0}'
1676     return Value
1677
1678 class PathClass(object):
1679     def __init__(self, File='', Root='', AlterRoot='', Type='', IsBinary=False,
1680                  Arch='COMMON', ToolChainFamily='', Target='', TagName='', ToolCode=''):
1681         self.Arch = Arch
1682         self.File = str(File)
1683         if os.path.isabs(self.File):
1684             self.Root = ''
1685             self.AlterRoot = ''
1686         else:
1687             self.Root = str(Root)
1688             self.AlterRoot = str(AlterRoot)
1689
1690         # Remove any '.' and '..' in path
1691         if self.Root:
1692             self.Root = mws.getWs(self.Root, self.File)
1693             self.Path = os.path.normpath(os.path.join(self.Root, self.File))
1694             self.Root = os.path.normpath(CommonPath([self.Root, self.Path]))
1695             # eliminate the side-effect of 'C:'
1696             if self.Root[-1] == ':':
1697                 self.Root += os.path.sep
1698             # file path should not start with path separator
1699             if self.Root[-1] == os.path.sep:
1700                 self.File = self.Path[len(self.Root):]
1701             else:
1702                 self.File = self.Path[len(self.Root) + 1:]
1703         else:
1704             self.Path = os.path.normpath(self.File)
1705
1706         self.SubDir, self.Name = os.path.split(self.File)
1707         self.BaseName, self.Ext = os.path.splitext(self.Name)
1708
1709         if self.Root:
1710             if self.SubDir:
1711                 self.Dir = os.path.join(self.Root, self.SubDir)
1712             else:
1713                 self.Dir = self.Root
1714         else:
1715             self.Dir = self.SubDir
1716
1717         if IsBinary:
1718             self.Type = Type
1719         else:
1720             self.Type = self.Ext.lower()
1721
1722         self.IsBinary = IsBinary
1723         self.Target = Target
1724         self.TagName = TagName
1725         self.ToolCode = ToolCode
1726         self.ToolChainFamily = ToolChainFamily
1727
1728     ## Convert the object of this class to a string
1729     #
1730     #  Convert member Path of the class to a string
1731     #
1732     #  @retval string Formatted String
1733     #
1734     def __str__(self):
1735         return self.Path
1736
1737     ## Override __eq__ function
1738     #
1739     # Check whether PathClass are the same
1740     #
1741     # @retval False The two PathClass are different
1742     # @retval True  The two PathClass are the same
1743     #
1744     def __eq__(self, Other):
1745         if isinstance(Other, type(self)):
1746             return self.Path == Other.Path
1747         else:
1748             return self.Path == str(Other)
1749
1750     ## Override __cmp__ function
1751     #
1752     # Customize the comparsion operation of two PathClass
1753     #
1754     # @retval 0     The two PathClass are different
1755     # @retval -1    The first PathClass is less than the second PathClass
1756     # @retval 1     The first PathClass is Bigger than the second PathClass
1757     def __cmp__(self, Other):
1758         if isinstance(Other, type(self)):
1759             OtherKey = Other.Path
1760         else:
1761             OtherKey = str(Other)
1762
1763         SelfKey = self.Path
1764         if SelfKey == OtherKey:
1765             return 0
1766         elif SelfKey > OtherKey:
1767             return 1
1768         else:
1769             return -1
1770
1771     ## Override __hash__ function
1772     #
1773     # Use Path as key in hash table
1774     #
1775     # @retval string Key for hash table
1776     #
1777     def __hash__(self):
1778         return hash(self.Path)
1779
1780     @cached_property
1781     def Key(self):
1782         return self.Path.upper()
1783
1784     @property
1785     def TimeStamp(self):
1786         return os.stat(self.Path)[8]
1787
1788     def Validate(self, Type='', CaseSensitive=True):
1789         if GlobalData.gCaseInsensitive:
1790             CaseSensitive = False
1791         if Type and Type.lower() != self.Type:
1792             return FILE_TYPE_MISMATCH, '%s (expect %s but got %s)' % (self.File, Type, self.Type)
1793
1794         RealFile, RealRoot = RealPath2(self.File, self.Root, self.AlterRoot)
1795         if not RealRoot and not RealFile:
1796             RealFile = self.File
1797             if self.AlterRoot:
1798                 RealFile = os.path.join(self.AlterRoot, self.File)
1799             elif self.Root:
1800                 RealFile = os.path.join(self.Root, self.File)
1801             if len (mws.getPkgPath()) == 0:
1802                 return FILE_NOT_FOUND, os.path.join(self.AlterRoot, RealFile)
1803             else:
1804                 return FILE_NOT_FOUND, "%s is not found in packages path:\n\t%s" % (self.File, '\n\t'.join(mws.getPkgPath()))
1805
1806         ErrorCode = 0
1807         ErrorInfo = ''
1808         if RealRoot != self.Root or RealFile != self.File:
1809             if CaseSensitive and (RealFile != self.File or (RealRoot != self.Root and RealRoot != self.AlterRoot)):
1810                 ErrorCode = FILE_CASE_MISMATCH
1811                 ErrorInfo = self.File + '\n\t' + RealFile + " [in file system]"
1812
1813             self.SubDir, self.Name = os.path.split(RealFile)
1814             self.BaseName, self.Ext = os.path.splitext(self.Name)
1815             if self.SubDir:
1816                 self.Dir = os.path.join(RealRoot, self.SubDir)
1817             else:
1818                 self.Dir = RealRoot
1819             self.File = RealFile
1820             self.Root = RealRoot
1821             self.Path = os.path.join(RealRoot, RealFile)
1822         return ErrorCode, ErrorInfo
1823
1824 ## Parse PE image to get the required PE informaion.
1825 #
1826 class PeImageClass():
1827     ## Constructor
1828     #
1829     #   @param  File FilePath of PeImage
1830     #
1831     def __init__(self, PeFile):
1832         self.FileName   = PeFile
1833         self.IsValid    = False
1834         self.Size       = 0
1835         self.EntryPoint = 0
1836         self.SectionAlignment  = 0
1837         self.SectionHeaderList = []
1838         self.ErrorInfo = ''
1839         try:
1840             PeObject = open(PeFile, 'rb')
1841         except:
1842             self.ErrorInfo = self.FileName + ' can not be found\n'
1843             return
1844         # Read DOS header
1845         ByteArray = array.array('B')
1846         ByteArray.fromfile(PeObject, 0x3E)
1847         ByteList = ByteArray.tolist()
1848         # DOS signature should be 'MZ'
1849         if self._ByteListToStr (ByteList[0x0:0x2]) != 'MZ':
1850             self.ErrorInfo = self.FileName + ' has no valid DOS signature MZ'
1851             return
1852
1853         # Read 4 byte PE Signature
1854         PeOffset = self._ByteListToInt(ByteList[0x3C:0x3E])
1855         PeObject.seek(PeOffset)
1856         ByteArray = array.array('B')
1857         ByteArray.fromfile(PeObject, 4)
1858         # PE signature should be 'PE\0\0'
1859         if ByteArray.tostring() != 'PE\0\0':
1860             self.ErrorInfo = self.FileName + ' has no valid PE signature PE00'
1861             return
1862
1863         # Read PE file header
1864         ByteArray = array.array('B')
1865         ByteArray.fromfile(PeObject, 0x14)
1866         ByteList = ByteArray.tolist()
1867         SecNumber = self._ByteListToInt(ByteList[0x2:0x4])
1868         if SecNumber == 0:
1869             self.ErrorInfo = self.FileName + ' has no section header'
1870             return
1871
1872         # Read PE optional header
1873         OptionalHeaderSize = self._ByteListToInt(ByteArray[0x10:0x12])
1874         ByteArray = array.array('B')
1875         ByteArray.fromfile(PeObject, OptionalHeaderSize)
1876         ByteList = ByteArray.tolist()
1877         self.EntryPoint       = self._ByteListToInt(ByteList[0x10:0x14])
1878         self.SectionAlignment = self._ByteListToInt(ByteList[0x20:0x24])
1879         self.Size             = self._ByteListToInt(ByteList[0x38:0x3C])
1880
1881         # Read each Section Header
1882         for Index in range(SecNumber):
1883             ByteArray = array.array('B')
1884             ByteArray.fromfile(PeObject, 0x28)
1885             ByteList = ByteArray.tolist()
1886             SecName  = self._ByteListToStr(ByteList[0:8])
1887             SecVirtualSize = self._ByteListToInt(ByteList[8:12])
1888             SecRawAddress  = self._ByteListToInt(ByteList[20:24])
1889             SecVirtualAddress = self._ByteListToInt(ByteList[12:16])
1890             self.SectionHeaderList.append((SecName, SecVirtualAddress, SecRawAddress, SecVirtualSize))
1891         self.IsValid = True
1892         PeObject.close()
1893
1894     def _ByteListToStr(self, ByteList):
1895         String = ''
1896         for index in range(len(ByteList)):
1897             if ByteList[index] == 0:
1898                 break
1899             String += chr(ByteList[index])
1900         return String
1901
1902     def _ByteListToInt(self, ByteList):
1903         Value = 0
1904         for index in range(len(ByteList) - 1, -1, -1):
1905             Value = (Value << 8) | int(ByteList[index])
1906         return Value
1907
1908 class DefaultStore():
1909     def __init__(self, DefaultStores ):
1910
1911         self.DefaultStores = DefaultStores
1912     def DefaultStoreID(self, DefaultStoreName):
1913         for key, value in self.DefaultStores.items():
1914             if value == DefaultStoreName:
1915                 return key
1916         return None
1917     def GetDefaultDefault(self):
1918         if not self.DefaultStores or "0" in self.DefaultStores:
1919             return "0", TAB_DEFAULT_STORES_DEFAULT
1920         else:
1921             minvalue = min(int(value_str) for value_str in self.DefaultStores)
1922             return (str(minvalue), self.DefaultStores[str(minvalue)])
1923     def GetMin(self, DefaultSIdList):
1924         if not DefaultSIdList:
1925             return TAB_DEFAULT_STORES_DEFAULT
1926         storeidset = {storeid for storeid, storename in self.DefaultStores.values() if storename in DefaultSIdList}
1927         if not storeidset:
1928             return ""
1929         minid = min(storeidset )
1930         for sid, name in self.DefaultStores.values():
1931             if sid == minid:
1932                 return name
1933
1934 class SkuClass():
1935     DEFAULT = 0
1936     SINGLE = 1
1937     MULTIPLE =2
1938
1939     def __init__(self,SkuIdentifier='', SkuIds=None):
1940         if SkuIds is None:
1941             SkuIds = {}
1942
1943         for SkuName in SkuIds:
1944             SkuId = SkuIds[SkuName][0]
1945             skuid_num = int(SkuId, 16) if SkuId.upper().startswith("0X") else int(SkuId)
1946             if skuid_num > 0xFFFFFFFFFFFFFFFF:
1947                 EdkLogger.error("build", PARAMETER_INVALID,
1948                             ExtraData = "SKU-ID [%s] value %s exceeds the max value of UINT64"
1949                                       % (SkuName, SkuId))
1950
1951         self.AvailableSkuIds = sdict()
1952         self.SkuIdSet = []
1953         self.SkuIdNumberSet = []
1954         self.SkuData = SkuIds
1955         self._SkuInherit = {}
1956         self._SkuIdentifier = SkuIdentifier
1957         if SkuIdentifier == '' or SkuIdentifier is None:
1958             self.SkuIdSet = ['DEFAULT']
1959             self.SkuIdNumberSet = ['0U']
1960         elif SkuIdentifier == 'ALL':
1961             self.SkuIdSet = SkuIds.keys()
1962             self.SkuIdNumberSet = [num[0].strip() + 'U' for num in SkuIds.values()]
1963         else:
1964             r = SkuIdentifier.split('|')
1965             self.SkuIdSet=[(r[k].strip()).upper() for k in range(len(r))]
1966             k = None
1967             try:
1968                 self.SkuIdNumberSet = [SkuIds[k][0].strip() + 'U' for k in self.SkuIdSet]
1969             except Exception:
1970                 EdkLogger.error("build", PARAMETER_INVALID,
1971                             ExtraData = "SKU-ID [%s] is not supported by the platform. [Valid SKU-ID: %s]"
1972                                       % (k, " | ".join(SkuIds.keys())))
1973         for each in self.SkuIdSet:
1974             if each in SkuIds:
1975                 self.AvailableSkuIds[each] = SkuIds[each][0]
1976             else:
1977                 EdkLogger.error("build", PARAMETER_INVALID,
1978                             ExtraData="SKU-ID [%s] is not supported by the platform. [Valid SKU-ID: %s]"
1979                                       % (each, " | ".join(SkuIds.keys())))
1980         if self.SkuUsageType != SkuClass.SINGLE:
1981             self.AvailableSkuIds.update({'DEFAULT':0, 'COMMON':0})
1982         if self.SkuIdSet:
1983             GlobalData.gSkuids = (self.SkuIdSet)
1984             if 'COMMON' in GlobalData.gSkuids:
1985                 GlobalData.gSkuids.remove('COMMON')
1986             if self.SkuUsageType == self.SINGLE:
1987                 if len(GlobalData.gSkuids) != 1:
1988                     if 'DEFAULT' in GlobalData.gSkuids:
1989                         GlobalData.gSkuids.remove('DEFAULT')
1990             if GlobalData.gSkuids:
1991                 GlobalData.gSkuids.sort()
1992
1993     def GetNextSkuId(self, skuname):
1994         if not self._SkuInherit:
1995             self._SkuInherit = {}
1996             for item in self.SkuData.values():
1997                 self._SkuInherit[item[1]]=item[2] if item[2] else "DEFAULT"
1998         return self._SkuInherit.get(skuname, "DEFAULT")
1999
2000     def GetSkuChain(self, sku):
2001         if sku == "DEFAULT":
2002             return ["DEFAULT"]
2003         skulist = [sku]
2004         nextsku = sku
2005         while True:
2006             nextsku = self.GetNextSkuId(nextsku)
2007             skulist.append(nextsku)
2008             if nextsku == "DEFAULT":
2009                 break
2010         skulist.reverse()
2011         return skulist
2012     def SkuOverrideOrder(self):
2013         skuorderset = []
2014         for skuname in self.SkuIdSet:
2015             skuorderset.append(self.GetSkuChain(skuname))
2016
2017         skuorder = []
2018         for index in range(max(len(item) for item in skuorderset)):
2019             for subset in skuorderset:
2020                 if index > len(subset)-1:
2021                     continue
2022                 if subset[index] in skuorder:
2023                     continue
2024                 skuorder.append(subset[index])
2025
2026         return skuorder
2027
2028     @property
2029     def SkuUsageType(self):
2030         if self._SkuIdentifier.upper() == "ALL":
2031             return SkuClass.MULTIPLE
2032
2033         if len(self.SkuIdSet) == 1:
2034             if self.SkuIdSet[0] == 'DEFAULT':
2035                 return SkuClass.DEFAULT
2036             return SkuClass.SINGLE
2037         if len(self.SkuIdSet) == 2 and 'DEFAULT' in self.SkuIdSet:
2038             return SkuClass.SINGLE
2039         return SkuClass.MULTIPLE
2040
2041     def DumpSkuIdArrary(self):
2042         if self.SkuUsageType == SkuClass.SINGLE:
2043             return "{0x0}"
2044         ArrayStrList = []
2045         for skuname in self.AvailableSkuIds:
2046             if skuname == "COMMON":
2047                 continue
2048             while skuname != "DEFAULT":
2049                 ArrayStrList.append(hex(int(self.AvailableSkuIds[skuname])))
2050                 skuname = self.GetNextSkuId(skuname)
2051             ArrayStrList.append("0x0")
2052         return "{{{myList}}}".format(myList=",".join(ArrayStrList))
2053
2054     @property
2055     def AvailableSkuIdSet(self):
2056         return self.AvailableSkuIds
2057
2058     @property
2059     def SystemSkuId(self):
2060         if self.SkuUsageType == SkuClass.SINGLE:
2061             if len(self.SkuIdSet) == 1:
2062                 return self.SkuIdSet[0]
2063             else:
2064                 return self.SkuIdSet[0] if self.SkuIdSet[0] != 'DEFAULT' else self.SkuIdSet[1]
2065         else:
2066             return 'DEFAULT'
2067
2068 #
2069 # Pack a registry format GUID
2070 #
2071 def PackRegistryFormatGuid(Guid):
2072     return PackGUID(Guid.split('-'))
2073
2074 ##  Get the integer value from string like "14U" or integer like 2
2075 #
2076 #   @param      Input   The object that may be either a integer value or a string
2077 #
2078 #   @retval     Value    The integer value that the input represents
2079 #
2080 def GetIntegerValue(Input):
2081     if type(Input) in (int, long):
2082         return Input
2083     String = Input
2084     if String.endswith("U"):
2085         String = String[:-1]
2086     if String.endswith("ULL"):
2087         String = String[:-3]
2088     if String.endswith("LL"):
2089         String = String[:-2]
2090
2091     if String.startswith("0x") or String.startswith("0X"):
2092         return int(String, 16)
2093     elif String == '':
2094         return 0
2095     else:
2096         return int(String)
2097
2098 #
2099 # Pack a GUID (registry format) list into a buffer and return it
2100 #
2101 def PackGUID(Guid):
2102     return pack(PACK_PATTERN_GUID,
2103                 int(Guid[0], 16),
2104                 int(Guid[1], 16),
2105                 int(Guid[2], 16),
2106                 int(Guid[3][-4:-2], 16),
2107                 int(Guid[3][-2:], 16),
2108                 int(Guid[4][-12:-10], 16),
2109                 int(Guid[4][-10:-8], 16),
2110                 int(Guid[4][-8:-6], 16),
2111                 int(Guid[4][-6:-4], 16),
2112                 int(Guid[4][-4:-2], 16),
2113                 int(Guid[4][-2:], 16)
2114                 )
2115
2116 #
2117 # Pack a GUID (byte) list into a buffer and return it
2118 #
2119 def PackByteFormatGUID(Guid):
2120     return pack(PACK_PATTERN_GUID,
2121                 Guid[0],
2122                 Guid[1],
2123                 Guid[2],
2124                 Guid[3],
2125                 Guid[4],
2126                 Guid[5],
2127                 Guid[6],
2128                 Guid[7],
2129                 Guid[8],
2130                 Guid[9],
2131                 Guid[10],
2132                 )
2133
2134 ##
2135 #
2136 # This acts like the main() function for the script, unless it is 'import'ed into another
2137 # script.
2138 #
2139 if __name__ == '__main__':
2140     pass
2141