BaseTools/Source/Python/Common/Misc.py

   1 ## @file
   2 # Common routines used by all tools
   3 #
   4 # Copyright (c) 2007 - 2018, Intel Corporation. All rights reserved.<BR>
   5 # This program and the accompanying materials
   6 # are licensed and made available under the terms and conditions of the BSD License
   7 # which accompanies this distribution.  The full text of the license may be found at
   8 # http://opensource.org/licenses/bsd-license.php
   9 #
  10 # THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS,
  11 # WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED.
  12 #
  13
  14 ##
  15 # Import Modules
  16 #
  17 from __future__ import absolute_import
  18 import Common.LongFilePathOs as os
  19 import sys
  20 import string
  21 import threading
  22 import time
  23 import re
  24 import pickle
  25 import array
  26 import shutil
  27 from struct import pack
  28 from UserDict import IterableUserDict
  29 from UserList import UserList
  30
  31 from Common import EdkLogger as EdkLogger
  32 from Common import GlobalData as GlobalData
  33 from .DataType import *
  34 from .BuildToolError import *
  35 from CommonDataClass.DataClass import *
  36 from .Parsing import GetSplitValueList
  37 from Common.LongFilePathSupport import OpenLongFilePath as open
  38 from Common.MultipleWorkspace import MultipleWorkspace as mws
  39 import uuid
  40 from CommonDataClass.Exceptions import BadExpression
  41 import subprocess
  42 ## Regular expression used to find out place holders in string template
  43 gPlaceholderPattern = re.compile("\$\{([^$()\s]+)\}", re.MULTILINE | re.UNICODE)
  44
  45 ## regular expressions for map file processing
  46 startPatternGeneral = re.compile("^Start[' ']+Length[' ']+Name[' ']+Class")
  47 addressPatternGeneral = re.compile("^Address[' ']+Publics by Value[' ']+Rva\+Base")
  48 valuePatternGcc = re.compile('^([\w_\.]+) +([\da-fA-Fx]+) +([\da-fA-Fx]+)$')
  49 pcdPatternGcc = re.compile('^([\da-fA-Fx]+) +([\da-fA-Fx]+)')
  50 secReGeneral = re.compile('^([\da-fA-F]+):([\da-fA-F]+) +([\da-fA-F]+)[Hh]? +([.\w\$]+) +(\w+)', re.UNICODE)
  51
  52 ## Dictionary used to store file time stamp for quick re-access
  53 gFileTimeStampCache = {}    # {file path : file time stamp}
  54
  55 ## Dictionary used to store dependencies of files
  56 gDependencyDatabase = {}    # arch : {file path : [dependent files list]}
  57
  58 #
  59 # If a module is built more than once with different PCDs or library classes
  60 # a temporary INF file with same content is created, the temporary file is removed
  61 # when build exits.
  62 #
  63 _TempInfs = []
  64
  65 def GetVariableOffset(mapfilepath, efifilepath, varnames):
  66     """ Parse map file to get variable offset in current EFI file
  67     @param mapfilepath    Map file absolution path
  68     @param efifilepath:   EFI binary file full path
  69     @param varnames       iteratable container whose elements are variable names to be searched
  70
  71     @return List whos elements are tuple with variable name and raw offset
  72     """
  73     lines = []
  74     try:
  75         f = open(mapfilepath, 'r')
  76         lines = f.readlines()
  77         f.close()
  78     except:
  79         return None
  80
  81     if len(lines) == 0: return None
  82     firstline = lines[0].strip()
  83     if (firstline.startswith("Archive member included ") and
  84         firstline.endswith(" file (symbol)")):
  85         return _parseForGCC(lines, efifilepath, varnames)
  86     if firstline.startswith("# Path:"):
  87         return _parseForXcode(lines, efifilepath, varnames)
  88     return _parseGeneral(lines, efifilepath, varnames)
  89
  90 def _parseForXcode(lines, efifilepath, varnames):
  91     status = 0
  92     ret = []
  93     for line in lines:
  94         line = line.strip()
  95         if status == 0 and line == "# Symbols:":
  96             status = 1
  97             continue
  98         if status == 1 and len(line) != 0:
  99             for varname in varnames:
 100                 if varname in line:
 101                     # cannot pregenerate this RegEx since it uses varname from varnames.
 102                     m = re.match('^([\da-fA-FxX]+)([\s\S]*)([_]*%s)$' % varname, line)
 103                     if m is not None:
 104                         ret.append((varname, m.group(1)))
 105     return ret
 106
 107 def _parseForGCC(lines, efifilepath, varnames):
 108     """ Parse map file generated by GCC linker """
 109     status = 0
 110     sections = []
 111     varoffset = []
 112     for index, line in enumerate(lines):
 113         line = line.strip()
 114         # status machine transection
 115         if status == 0 and line == "Memory Configuration":
 116             status = 1
 117             continue
 118         elif status == 1 and line == 'Linker script and memory map':
 119             status = 2
 120             continue
 121         elif status ==2 and line == 'START GROUP':
 122             status = 3
 123             continue
 124
 125         # status handler
 126         if status == 3:
 127             m = valuePatternGcc.match(line)
 128             if m is not None:
 129                 sections.append(m.groups(0))
 130             for varname in varnames:
 131                 Str = ''
 132                 m = re.match("^.data.(%s)" % varname, line)
 133                 if m is not None:
 134                     m = re.match(".data.(%s)$" % varname, line)
 135                     if m is not None:
 136                         Str = lines[index + 1]
 137                     else:
 138                         Str = line[len(".data.%s" % varname):]
 139                     if Str:
 140                         m = pcdPatternGcc.match(Str.strip())
 141                         if m is not None:
 142                             varoffset.append((varname, int(m.groups(0)[0], 16), int(sections[-1][1], 16), sections[-1][0]))
 143
 144     if not varoffset:
 145         return []
 146     # get section information from efi file
 147     efisecs = PeImageClass(efifilepath).SectionHeaderList
 148     if efisecs is None or len(efisecs) == 0:
 149         return []
 150     #redirection
 151     redirection = 0
 152     for efisec in efisecs:
 153         for section in sections:
 154             if section[0].strip() == efisec[0].strip() and section[0].strip() == '.text':
 155                 redirection = int(section[1], 16) - efisec[1]
 156
 157     ret = []
 158     for var in varoffset:
 159         for efisec in efisecs:
 160             if var[1] >= efisec[1] and var[1] < efisec[1]+efisec[3]:
 161                 ret.append((var[0], hex(efisec[2] + var[1] - efisec[1] - redirection)))
 162     return ret
 163
 164 def _parseGeneral(lines, efifilepath, varnames):
 165     status = 0    #0 - beginning of file; 1 - PE section definition; 2 - symbol table
 166     secs  = []    # key = section name
 167     varoffset = []
 168     symRe = re.compile('^([\da-fA-F]+):([\da-fA-F]+) +([\.:\\\\\w\?@\$]+) +([\da-fA-F]+)', re.UNICODE)
 169
 170     for line in lines:
 171         line = line.strip()
 172         if startPatternGeneral.match(line):
 173             status = 1
 174             continue
 175         if addressPatternGeneral.match(line):
 176             status = 2
 177             continue
 178         if line.startswith("entry point at"):
 179             status = 3
 180             continue
 181         if status == 1 and len(line) != 0:
 182             m =  secReGeneral.match(line)
 183             assert m is not None, "Fail to parse the section in map file , line is %s" % line
 184             sec_no, sec_start, sec_length, sec_name, sec_class = m.groups(0)
 185             secs.append([int(sec_no, 16), int(sec_start, 16), int(sec_length, 16), sec_name, sec_class])
 186         if status == 2 and len(line) != 0:
 187             for varname in varnames:
 188                 m = symRe.match(line)
 189                 assert m is not None, "Fail to parse the symbol in map file, line is %s" % line
 190                 sec_no, sym_offset, sym_name, vir_addr = m.groups(0)
 191                 sec_no     = int(sec_no,     16)
 192                 sym_offset = int(sym_offset, 16)
 193                 vir_addr   = int(vir_addr,   16)
 194                 # cannot pregenerate this RegEx since it uses varname from varnames.
 195                 m2 = re.match('^[_]*(%s)' % varname, sym_name)
 196                 if m2 is not None:
 197                     # fond a binary pcd entry in map file
 198                     for sec in secs:
 199                         if sec[0] == sec_no and (sym_offset >= sec[1] and sym_offset < sec[1] + sec[2]):
 200                             varoffset.append([varname, sec[3], sym_offset, vir_addr, sec_no])
 201
 202     if not varoffset: return []
 203
 204     # get section information from efi file
 205     efisecs = PeImageClass(efifilepath).SectionHeaderList
 206     if efisecs is None or len(efisecs) == 0:
 207         return []
 208
 209     ret = []
 210     for var in varoffset:
 211         index = 0
 212         for efisec in efisecs:
 213             index = index + 1
 214             if var[1].strip() == efisec[0].strip():
 215                 ret.append((var[0], hex(efisec[2] + var[2])))
 216             elif var[4] == index:
 217                 ret.append((var[0], hex(efisec[2] + var[2])))
 218
 219     return ret
 220
 221 ## Routine to process duplicated INF
 222 #
 223 #  This function is called by following two cases:
 224 #  Case 1 in DSC:
 225 #    [components.arch]
 226 #    Pkg/module/module.inf
 227 #    Pkg/module/module.inf {
 228 #      <Defines>
 229 #        FILE_GUID = 0D1B936F-68F3-4589-AFCC-FB8B7AEBC836
 230 #    }
 231 #  Case 2 in FDF:
 232 #    INF Pkg/module/module.inf
 233 #    INF FILE_GUID = 0D1B936F-68F3-4589-AFCC-FB8B7AEBC836 Pkg/module/module.inf
 234 #
 235 #  This function copies Pkg/module/module.inf to
 236 #  Conf/.cache/0D1B936F-68F3-4589-AFCC-FB8B7AEBC836module.inf
 237 #
 238 #  @param Path     Original PathClass object
 239 #  @param BaseName New file base name
 240 #
 241 #  @retval         return the new PathClass object
 242 #
 243 def ProcessDuplicatedInf(Path, BaseName, Workspace):
 244     Filename = os.path.split(Path.File)[1]
 245     if '.' in Filename:
 246         Filename = BaseName + Path.BaseName + Filename[Filename.rfind('.'):]
 247     else:
 248         Filename = BaseName + Path.BaseName
 249
 250     #
 251     # If -N is specified on command line, cache is disabled
 252     # The directory has to be created
 253     #
 254     DbDir = os.path.split(GlobalData.gDatabasePath)[0]
 255     if not os.path.exists(DbDir):
 256         os.makedirs(DbDir)
 257     #
 258     # A temporary INF is copied to database path which must have write permission
 259     # The temporary will be removed at the end of build
 260     # In case of name conflict, the file name is
 261     # FILE_GUIDBaseName (0D1B936F-68F3-4589-AFCC-FB8B7AEBC836module.inf)
 262     #
 263     TempFullPath = os.path.join(DbDir,
 264                                 Filename)
 265     RtPath = PathClass(Path.File, Workspace)
 266     #
 267     # Modify the full path to temporary path, keep other unchanged
 268     #
 269     # To build same module more than once, the module path with FILE_GUID overridden has
 270     # the file name FILE_GUIDmodule.inf, but the relative path (self.MetaFile.File) is the real path
 271     # in DSC which is used as relative path by C files and other files in INF.
 272     # A trick was used: all module paths are PathClass instances, after the initialization
 273     # of PathClass, the PathClass.Path is overridden by the temporary INF path.
 274     #
 275     # The reason for creating a temporary INF is:
 276     # Platform.Modules which is the base to create ModuleAutoGen objects is a dictionary,
 277     # the key is the full path of INF, the value is an object to save overridden library instances, PCDs.
 278     # A different key for the same module is needed to create different output directory,
 279     # retrieve overridden PCDs, library instances.
 280     #
 281     # The BaseName is the FILE_GUID which is also the output directory name.
 282     #
 283     #
 284     RtPath.Path = TempFullPath
 285     RtPath.BaseName = BaseName
 286     #
 287     # If file exists, compare contents
 288     #
 289     if os.path.exists(TempFullPath):
 290         with open(str(Path), 'rb') as f1, open(TempFullPath, 'rb') as f2:
 291             if f1.read() == f2.read():
 292                 return RtPath
 293     _TempInfs.append(TempFullPath)
 294     shutil.copy2(str(Path), TempFullPath)
 295     return RtPath
 296
 297 ## Remove temporary created INFs whose paths were saved in _TempInfs
 298 #
 299 def ClearDuplicatedInf():
 300     while _TempInfs:
 301         File = _TempInfs.pop()
 302         if os.path.exists(File):
 303             os.remove(File)
 304
 305 ## Convert GUID string in xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx style to C structure style
 306 #
 307 #   @param      Guid    The GUID string
 308 #
 309 #   @retval     string  The GUID string in C structure style
 310 #
 311 def GuidStringToGuidStructureString(Guid):
 312     GuidList = Guid.split('-')
 313     Result = '{'
 314     for Index in range(0, 3, 1):
 315         Result = Result + '0x' + GuidList[Index] + ', '
 316     Result = Result + '{0x' + GuidList[3][0:2] + ', 0x' + GuidList[3][2:4]
 317     for Index in range(0, 12, 2):
 318         Result = Result + ', 0x' + GuidList[4][Index:Index + 2]
 319     Result += '}}'
 320     return Result
 321
 322 ## Convert GUID structure in byte array to xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
 323 #
 324 #   @param      GuidValue   The GUID value in byte array
 325 #
 326 #   @retval     string      The GUID value in xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx format
 327 #
 328 def GuidStructureByteArrayToGuidString(GuidValue):
 329     guidValueString = GuidValue.lower().replace("{", "").replace("}", "").replace(" ", "").replace(";", "")
 330     guidValueList = guidValueString.split(",")
 331     if len(guidValueList) != 16:
 332         return ''
 333         #EdkLogger.error(None, None, "Invalid GUID value string %s" % GuidValue)
 334     try:
 335         return "%02x%02x%02x%02x-%02x%02x-%02x%02x-%02x%02x-%02x%02x%02x%02x%02x%02x" % (
 336                 int(guidValueList[3], 16),
 337                 int(guidValueList[2], 16),
 338                 int(guidValueList[1], 16),
 339                 int(guidValueList[0], 16),
 340                 int(guidValueList[5], 16),
 341                 int(guidValueList[4], 16),
 342                 int(guidValueList[7], 16),
 343                 int(guidValueList[6], 16),
 344                 int(guidValueList[8], 16),
 345                 int(guidValueList[9], 16),
 346                 int(guidValueList[10], 16),
 347                 int(guidValueList[11], 16),
 348                 int(guidValueList[12], 16),
 349                 int(guidValueList[13], 16),
 350                 int(guidValueList[14], 16),
 351                 int(guidValueList[15], 16)
 352                 )
 353     except:
 354         return ''
 355
 356 ## Convert GUID string in C structure style to xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
 357 #
 358 #   @param      GuidValue   The GUID value in C structure format
 359 #
 360 #   @retval     string      The GUID value in xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx format
 361 #
 362 def GuidStructureStringToGuidString(GuidValue):
 363     if not GlobalData.gGuidCFormatPattern.match(GuidValue):
 364         return ''
 365     guidValueString = GuidValue.lower().replace("{", "").replace("}", "").replace(" ", "").replace(";", "")
 366     guidValueList = guidValueString.split(",")
 367     if len(guidValueList) != 11:
 368         return ''
 369         #EdkLogger.error(None, None, "Invalid GUID value string %s" % GuidValue)
 370     try:
 371         return "%08x-%04x-%04x-%02x%02x-%02x%02x%02x%02x%02x%02x" % (
 372                 int(guidValueList[0], 16),
 373                 int(guidValueList[1], 16),
 374                 int(guidValueList[2], 16),
 375                 int(guidValueList[3], 16),
 376                 int(guidValueList[4], 16),
 377                 int(guidValueList[5], 16),
 378                 int(guidValueList[6], 16),
 379                 int(guidValueList[7], 16),
 380                 int(guidValueList[8], 16),
 381                 int(guidValueList[9], 16),
 382                 int(guidValueList[10], 16)
 383                 )
 384     except:
 385         return ''
 386
 387 ## Convert GUID string in C structure style to xxxxxxxx_xxxx_xxxx_xxxx_xxxxxxxxxxxx
 388 #
 389 #   @param      GuidValue   The GUID value in C structure format
 390 #
 391 #   @retval     string      The GUID value in xxxxxxxx_xxxx_xxxx_xxxx_xxxxxxxxxxxx format
 392 #
 393 def GuidStructureStringToGuidValueName(GuidValue):
 394     guidValueString = GuidValue.lower().replace("{", "").replace("}", "").replace(" ", "")
 395     guidValueList = guidValueString.split(",")
 396     if len(guidValueList) != 11:
 397         EdkLogger.error(None, FORMAT_INVALID, "Invalid GUID value string [%s]" % GuidValue)
 398     return "%08x_%04x_%04x_%02x%02x_%02x%02x%02x%02x%02x%02x" % (
 399             int(guidValueList[0], 16),
 400             int(guidValueList[1], 16),
 401             int(guidValueList[2], 16),
 402             int(guidValueList[3], 16),
 403             int(guidValueList[4], 16),
 404             int(guidValueList[5], 16),
 405             int(guidValueList[6], 16),
 406             int(guidValueList[7], 16),
 407             int(guidValueList[8], 16),
 408             int(guidValueList[9], 16),
 409             int(guidValueList[10], 16)
 410             )
 411
 412 ## Create directories
 413 #
 414 #   @param      Directory   The directory name
 415 #
 416 def CreateDirectory(Directory):
 417     if Directory is None or Directory.strip() == "":
 418         return True
 419     try:
 420         if not os.access(Directory, os.F_OK):
 421             os.makedirs(Directory)
 422     except:
 423         return False
 424     return True
 425
 426 ## Remove directories, including files and sub-directories in it
 427 #
 428 #   @param      Directory   The directory name
 429 #
 430 def RemoveDirectory(Directory, Recursively=False):
 431     if Directory is None or Directory.strip() == "" or not os.path.exists(Directory):
 432         return
 433     if Recursively:
 434         CurrentDirectory = os.getcwd()
 435         os.chdir(Directory)
 436         for File in os.listdir("."):
 437             if os.path.isdir(File):
 438                 RemoveDirectory(File, Recursively)
 439             else:
 440                 os.remove(File)
 441         os.chdir(CurrentDirectory)
 442     os.rmdir(Directory)
 443
 444 ## Store content in file
 445 #
 446 #  This method is used to save file only when its content is changed. This is
 447 #  quite useful for "make" system to decide what will be re-built and what won't.
 448 #
 449 #   @param      File            The path of file
 450 #   @param      Content         The new content of the file
 451 #   @param      IsBinaryFile    The flag indicating if the file is binary file or not
 452 #
 453 #   @retval     True            If the file content is changed and the file is renewed
 454 #   @retval     False           If the file content is the same
 455 #
 456 def SaveFileOnChange(File, Content, IsBinaryFile=True):
 457     if not IsBinaryFile:
 458         Content = Content.replace("\n", os.linesep)
 459
 460     if os.path.exists(File):
 461         try:
 462             if Content == open(File, "rb").read():
 463                 return False
 464         except:
 465             EdkLogger.error(None, FILE_OPEN_FAILURE, ExtraData=File)
 466
 467     DirName = os.path.dirname(File)
 468     if not CreateDirectory(DirName):
 469         EdkLogger.error(None, FILE_CREATE_FAILURE, "Could not create directory %s" % DirName)
 470     else:
 471         if DirName == '':
 472             DirName = os.getcwd()
 473         if not os.access(DirName, os.W_OK):
 474             EdkLogger.error(None, PERMISSION_FAILURE, "Do not have write permission on directory %s" % DirName)
 475
 476     try:
 477         if GlobalData.gIsWindows:
 478             try:
 479                 from .PyUtility import SaveFileToDisk
 480                 if not SaveFileToDisk(File, Content):
 481                     EdkLogger.error(None, FILE_CREATE_FAILURE, ExtraData=File)
 482             except:
 483                 Fd = open(File, "wb")
 484                 Fd.write(Content)
 485                 Fd.close()
 486         else:
 487             Fd = open(File, "wb")
 488             Fd.write(Content)
 489             Fd.close()
 490     except IOError as X:
 491         EdkLogger.error(None, FILE_CREATE_FAILURE, ExtraData='IOError %s' % X)
 492
 493     return True
 494
 495 ## Make a Python object persistent on file system
 496 #
 497 #   @param      Data    The object to be stored in file
 498 #   @param      File    The path of file to store the object
 499 #
 500 def DataDump(Data, File):
 501     Fd = None
 502     try:
 503         Fd = open(File, 'wb')
 504         pickle.dump(Data, Fd, pickle.HIGHEST_PROTOCOL)
 505     except:
 506         EdkLogger.error("", FILE_OPEN_FAILURE, ExtraData=File, RaiseError=False)
 507     finally:
 508         if Fd is not None:
 509             Fd.close()
 510
 511 ## Restore a Python object from a file
 512 #
 513 #   @param      File    The path of file stored the object
 514 #
 515 #   @retval     object  A python object
 516 #   @retval     None    If failure in file operation
 517 #
 518 def DataRestore(File):
 519     Data = None
 520     Fd = None
 521     try:
 522         Fd = open(File, 'rb')
 523         Data = pickle.load(Fd)
 524     except Exception as e:
 525         EdkLogger.verbose("Failed to load [%s]\n\t%s" % (File, str(e)))
 526         Data = None
 527     finally:
 528         if Fd is not None:
 529             Fd.close()
 530     return Data
 531
 532 ## Retrieve and cache the real path name in file system
 533 #
 534 #   @param      Root    The root directory of path relative to
 535 #
 536 #   @retval     str     The path string if the path exists
 537 #   @retval     None    If path doesn't exist
 538 #
 539 class DirCache:
 540     _CACHE_ = set()
 541     _UPPER_CACHE_ = {}
 542
 543     def __init__(self, Root):
 544         self._Root = Root
 545         for F in os.listdir(Root):
 546             self._CACHE_.add(F)
 547             self._UPPER_CACHE_[F.upper()] = F
 548
 549     # =[] operator
 550     def __getitem__(self, Path):
 551         Path = Path[len(os.path.commonprefix([Path, self._Root])):]
 552         if not Path:
 553             return self._Root
 554         if Path and Path[0] == os.path.sep:
 555             Path = Path[1:]
 556         if Path in self._CACHE_:
 557             return os.path.join(self._Root, Path)
 558         UpperPath = Path.upper()
 559         if UpperPath in self._UPPER_CACHE_:
 560             return os.path.join(self._Root, self._UPPER_CACHE_[UpperPath])
 561
 562         IndexList = []
 563         LastSepIndex = -1
 564         SepIndex = Path.find(os.path.sep)
 565         while SepIndex > -1:
 566             Parent = UpperPath[:SepIndex]
 567             if Parent not in self._UPPER_CACHE_:
 568                 break
 569             LastSepIndex = SepIndex
 570             SepIndex = Path.find(os.path.sep, LastSepIndex + 1)
 571
 572         if LastSepIndex == -1:
 573             return None
 574
 575         Cwd = os.getcwd()
 576         os.chdir(self._Root)
 577         SepIndex = LastSepIndex
 578         while SepIndex > -1:
 579             Parent = Path[:SepIndex]
 580             ParentKey = UpperPath[:SepIndex]
 581             if ParentKey not in self._UPPER_CACHE_:
 582                 os.chdir(Cwd)
 583                 return None
 584
 585             if Parent in self._CACHE_:
 586                 ParentDir = Parent
 587             else:
 588                 ParentDir = self._UPPER_CACHE_[ParentKey]
 589             for F in os.listdir(ParentDir):
 590                 Dir = os.path.join(ParentDir, F)
 591                 self._CACHE_.add(Dir)
 592                 self._UPPER_CACHE_[Dir.upper()] = Dir
 593
 594             SepIndex = Path.find(os.path.sep, SepIndex + 1)
 595
 596         os.chdir(Cwd)
 597         if Path in self._CACHE_:
 598             return os.path.join(self._Root, Path)
 599         elif UpperPath in self._UPPER_CACHE_:
 600             return os.path.join(self._Root, self._UPPER_CACHE_[UpperPath])
 601         return None
 602
 603 def RealPath(File, Dir='', OverrideDir=''):
 604     NewFile = os.path.normpath(os.path.join(Dir, File))
 605     NewFile = GlobalData.gAllFiles[NewFile]
 606     if not NewFile and OverrideDir:
 607         NewFile = os.path.normpath(os.path.join(OverrideDir, File))
 608         NewFile = GlobalData.gAllFiles[NewFile]
 609     return NewFile
 610
 611 def RealPath2(File, Dir='', OverrideDir=''):
 612     NewFile = None
 613     if OverrideDir:
 614         NewFile = GlobalData.gAllFiles[os.path.normpath(os.path.join(OverrideDir, File))]
 615         if NewFile:
 616             if OverrideDir[-1] == os.path.sep:
 617                 return NewFile[len(OverrideDir):], NewFile[0:len(OverrideDir)]
 618             else:
 619                 return NewFile[len(OverrideDir) + 1:], NewFile[0:len(OverrideDir)]
 620     if GlobalData.gAllFiles:
 621         NewFile = GlobalData.gAllFiles[os.path.normpath(os.path.join(Dir, File))]
 622     if not NewFile:
 623         NewFile = os.path.normpath(os.path.join(Dir, File))
 624         if not os.path.exists(NewFile):
 625             return None, None
 626     if NewFile:
 627         if Dir:
 628             if Dir[-1] == os.path.sep:
 629                 return NewFile[len(Dir):], NewFile[0:len(Dir)]
 630             else:
 631                 return NewFile[len(Dir) + 1:], NewFile[0:len(Dir)]
 632         else:
 633             return NewFile, ''
 634
 635     return None, None
 636
 637 ## Get GUID value from given packages
 638 #
 639 #   @param      CName           The CName of the GUID
 640 #   @param      PackageList     List of packages looking-up in
 641 #   @param      Inffile         The driver file
 642 #
 643 #   @retval     GuidValue   if the CName is found in any given package
 644 #   @retval     None        if the CName is not found in all given packages
 645 #
 646 def GuidValue(CName, PackageList, Inffile = None):
 647     for P in PackageList:
 648         GuidKeys = P.Guids.keys()
 649         if Inffile and P._PrivateGuids:
 650             if not Inffile.startswith(P.MetaFile.Dir):
 651                 GuidKeys = [x for x in P.Guids if x not in P._PrivateGuids]
 652         if CName in GuidKeys:
 653             return P.Guids[CName]
 654     return None
 655
 656 ## Get Protocol value from given packages
 657 #
 658 #   @param      CName           The CName of the GUID
 659 #   @param      PackageList     List of packages looking-up in
 660 #   @param      Inffile         The driver file
 661 #
 662 #   @retval     GuidValue   if the CName is found in any given package
 663 #   @retval     None        if the CName is not found in all given packages
 664 #
 665 def ProtocolValue(CName, PackageList, Inffile = None):
 666     for P in PackageList:
 667         ProtocolKeys = P.Protocols.keys()
 668         if Inffile and P._PrivateProtocols:
 669             if not Inffile.startswith(P.MetaFile.Dir):
 670                 ProtocolKeys = [x for x in P.Protocols if x not in P._PrivateProtocols]
 671         if CName in ProtocolKeys:
 672             return P.Protocols[CName]
 673     return None
 674
 675 ## Get PPI value from given packages
 676 #
 677 #   @param      CName           The CName of the GUID
 678 #   @param      PackageList     List of packages looking-up in
 679 #   @param      Inffile         The driver file
 680 #
 681 #   @retval     GuidValue   if the CName is found in any given package
 682 #   @retval     None        if the CName is not found in all given packages
 683 #
 684 def PpiValue(CName, PackageList, Inffile = None):
 685     for P in PackageList:
 686         PpiKeys = P.Ppis.keys()
 687         if Inffile and P._PrivatePpis:
 688             if not Inffile.startswith(P.MetaFile.Dir):
 689                 PpiKeys = [x for x in P.Ppis if x not in P._PrivatePpis]
 690         if CName in PpiKeys:
 691             return P.Ppis[CName]
 692     return None
 693
 694 ## A string template class
 695 #
 696 #  This class implements a template for string replacement. A string template
 697 #  looks like following
 698 #
 699 #       ${BEGIN} other_string ${placeholder_name} other_string ${END}
 700 #
 701 #  The string between ${BEGIN} and ${END} will be repeated as many times as the
 702 #  length of "placeholder_name", which is a list passed through a dict. The
 703 #  "placeholder_name" is the key name of the dict. The ${BEGIN} and ${END} can
 704 #  be not used and, in this case, the "placeholder_name" must not a list and it
 705 #  will just be replaced once.
 706 #
 707 class TemplateString(object):
 708     _REPEAT_START_FLAG = "BEGIN"
 709     _REPEAT_END_FLAG = "END"
 710
 711     class Section(object):
 712         _LIST_TYPES = [type([]), type(set()), type((0,))]
 713
 714         def __init__(self, TemplateSection, PlaceHolderList):
 715             self._Template = TemplateSection
 716             self._PlaceHolderList = []
 717
 718             # Split the section into sub-sections according to the position of placeholders
 719             if PlaceHolderList:
 720                 self._SubSectionList = []
 721                 SubSectionStart = 0
 722                 #
 723                 # The placeholders passed in must be in the format of
 724                 #
 725                 #   PlaceHolderName, PlaceHolderStartPoint, PlaceHolderEndPoint
 726                 #
 727                 for PlaceHolder, Start, End in PlaceHolderList:
 728                     self._SubSectionList.append(TemplateSection[SubSectionStart:Start])
 729                     self._SubSectionList.append(TemplateSection[Start:End])
 730                     self._PlaceHolderList.append(PlaceHolder)
 731                     SubSectionStart = End
 732                 if SubSectionStart < len(TemplateSection):
 733                     self._SubSectionList.append(TemplateSection[SubSectionStart:])
 734             else:
 735                 self._SubSectionList = [TemplateSection]
 736
 737         def __str__(self):
 738             return self._Template + " : " + str(self._PlaceHolderList)
 739
 740         def Instantiate(self, PlaceHolderValues):
 741             RepeatTime = -1
 742             RepeatPlaceHolders = {}
 743             NonRepeatPlaceHolders = {}
 744
 745             for PlaceHolder in self._PlaceHolderList:
 746                 if PlaceHolder not in PlaceHolderValues:
 747                     continue
 748                 Value = PlaceHolderValues[PlaceHolder]
 749                 if type(Value) in self._LIST_TYPES:
 750                     if RepeatTime < 0:
 751                         RepeatTime = len(Value)
 752                     elif RepeatTime != len(Value):
 753                         EdkLogger.error(
 754                                     "TemplateString",
 755                                     PARAMETER_INVALID,
 756                                     "${%s} has different repeat time from others!" % PlaceHolder,
 757                                     ExtraData=str(self._Template)
 758                                     )
 759                     RepeatPlaceHolders["${%s}" % PlaceHolder] = Value
 760                 else:
 761                     NonRepeatPlaceHolders["${%s}" % PlaceHolder] = Value
 762
 763             if NonRepeatPlaceHolders:
 764                 StringList = []
 765                 for S in self._SubSectionList:
 766                     if S not in NonRepeatPlaceHolders:
 767                         StringList.append(S)
 768                     else:
 769                         StringList.append(str(NonRepeatPlaceHolders[S]))
 770             else:
 771                 StringList = self._SubSectionList
 772
 773             if RepeatPlaceHolders:
 774                 TempStringList = []
 775                 for Index in range(RepeatTime):
 776                     for S in StringList:
 777                         if S not in RepeatPlaceHolders:
 778                             TempStringList.append(S)
 779                         else:
 780                             TempStringList.append(str(RepeatPlaceHolders[S][Index]))
 781                 StringList = TempStringList
 782
 783             return "".join(StringList)
 784
 785     ## Constructor
 786     def __init__(self, Template=None):
 787         self.String = ''
 788         self.IsBinary = False
 789         self._Template = Template
 790         self._TemplateSectionList = self._Parse(Template)
 791
 792     ## str() operator
 793     #
 794     #   @retval     string  The string replaced
 795     #
 796     def __str__(self):
 797         return self.String
 798
 799     ## Split the template string into fragments per the ${BEGIN} and ${END} flags
 800     #
 801     #   @retval     list    A list of TemplateString.Section objects
 802     #
 803     def _Parse(self, Template):
 804         SectionStart = 0
 805         SearchFrom = 0
 806         MatchEnd = 0
 807         PlaceHolderList = []
 808         TemplateSectionList = []
 809         while Template:
 810             MatchObj = gPlaceholderPattern.search(Template, SearchFrom)
 811             if not MatchObj:
 812                 if MatchEnd <= len(Template):
 813                     TemplateSection = TemplateString.Section(Template[SectionStart:], PlaceHolderList)
 814                     TemplateSectionList.append(TemplateSection)
 815                 break
 816
 817             MatchString = MatchObj.group(1)
 818             MatchStart = MatchObj.start()
 819             MatchEnd = MatchObj.end()
 820
 821             if MatchString == self._REPEAT_START_FLAG:
 822                 if MatchStart > SectionStart:
 823                     TemplateSection = TemplateString.Section(Template[SectionStart:MatchStart], PlaceHolderList)
 824                     TemplateSectionList.append(TemplateSection)
 825                 SectionStart = MatchEnd
 826                 PlaceHolderList = []
 827             elif MatchString == self._REPEAT_END_FLAG:
 828                 TemplateSection = TemplateString.Section(Template[SectionStart:MatchStart], PlaceHolderList)
 829                 TemplateSectionList.append(TemplateSection)
 830                 SectionStart = MatchEnd
 831                 PlaceHolderList = []
 832             else:
 833                 PlaceHolderList.append((MatchString, MatchStart - SectionStart, MatchEnd - SectionStart))
 834             SearchFrom = MatchEnd
 835         return TemplateSectionList
 836
 837     ## Replace the string template with dictionary of placeholders and append it to previous one
 838     #
 839     #   @param      AppendString    The string template to append
 840     #   @param      Dictionary      The placeholder dictionaries
 841     #
 842     def Append(self, AppendString, Dictionary=None):
 843         if Dictionary:
 844             SectionList = self._Parse(AppendString)
 845             self.String += "".join(S.Instantiate(Dictionary) for S in SectionList)
 846         else:
 847             self.String += AppendString
 848
 849     ## Replace the string template with dictionary of placeholders
 850     #
 851     #   @param      Dictionary      The placeholder dictionaries
 852     #
 853     #   @retval     str             The string replaced with placeholder values
 854     #
 855     def Replace(self, Dictionary=None):
 856         return "".join(S.Instantiate(Dictionary) for S in self._TemplateSectionList)
 857
 858 ## Progress indicator class
 859 #
 860 #  This class makes use of thread to print progress on console.
 861 #
 862 class Progressor:
 863     # for avoiding deadloop
 864     _StopFlag = None
 865     _ProgressThread = None
 866     _CheckInterval = 0.25
 867
 868     ## Constructor
 869     #
 870     #   @param      OpenMessage     The string printed before progress charaters
 871     #   @param      CloseMessage    The string printed after progress charaters
 872     #   @param      ProgressChar    The charater used to indicate the progress
 873     #   @param      Interval        The interval in seconds between two progress charaters
 874     #
 875     def __init__(self, OpenMessage="", CloseMessage="", ProgressChar='.', Interval=1.0):
 876         self.PromptMessage = OpenMessage
 877         self.CodaMessage = CloseMessage
 878         self.ProgressChar = ProgressChar
 879         self.Interval = Interval
 880         if Progressor._StopFlag is None:
 881             Progressor._StopFlag = threading.Event()
 882
 883     ## Start to print progress charater
 884     #
 885     #   @param      OpenMessage     The string printed before progress charaters
 886     #
 887     def Start(self, OpenMessage=None):
 888         if OpenMessage is not None:
 889             self.PromptMessage = OpenMessage
 890         Progressor._StopFlag.clear()
 891         if Progressor._ProgressThread is None:
 892             Progressor._ProgressThread = threading.Thread(target=self._ProgressThreadEntry)
 893             Progressor._ProgressThread.setDaemon(False)
 894             Progressor._ProgressThread.start()
 895
 896     ## Stop printing progress charater
 897     #
 898     #   @param      CloseMessage    The string printed after progress charaters
 899     #
 900     def Stop(self, CloseMessage=None):
 901         OriginalCodaMessage = self.CodaMessage
 902         if CloseMessage is not None:
 903             self.CodaMessage = CloseMessage
 904         self.Abort()
 905         self.CodaMessage = OriginalCodaMessage
 906
 907     ## Thread entry method
 908     def _ProgressThreadEntry(self):
 909         sys.stdout.write(self.PromptMessage + " ")
 910         sys.stdout.flush()
 911         TimeUp = 0.0
 912         while not Progressor._StopFlag.isSet():
 913             if TimeUp <= 0.0:
 914                 sys.stdout.write(self.ProgressChar)
 915                 sys.stdout.flush()
 916                 TimeUp = self.Interval
 917             time.sleep(self._CheckInterval)
 918             TimeUp -= self._CheckInterval
 919         sys.stdout.write(" " + self.CodaMessage + "\n")
 920         sys.stdout.flush()
 921
 922     ## Abort the progress display
 923     @staticmethod
 924     def Abort():
 925         if Progressor._StopFlag is not None:
 926             Progressor._StopFlag.set()
 927         if Progressor._ProgressThread is not None:
 928             Progressor._ProgressThread.join()
 929             Progressor._ProgressThread = None
 930
 931 ## A dict which can access its keys and/or values orderly
 932 #
 933 #  The class implements a new kind of dict which its keys or values can be
 934 #  accessed in the order they are added into the dict. It guarantees the order
 935 #  by making use of an internal list to keep a copy of keys.
 936 #
 937 class sdict(IterableUserDict):
 938     ## Constructor
 939     def __init__(self):
 940         IterableUserDict.__init__(self)
 941         self._key_list = []
 942
 943     ## [] operator
 944     def __setitem__(self, key, value):
 945         if key not in self._key_list:
 946             self._key_list.append(key)
 947         IterableUserDict.__setitem__(self, key, value)
 948
 949     ## del operator
 950     def __delitem__(self, key):
 951         self._key_list.remove(key)
 952         IterableUserDict.__delitem__(self, key)
 953
 954     ## used in "for k in dict" loop to ensure the correct order
 955     def __iter__(self):
 956         return self.iterkeys()
 957
 958     ## len() support
 959     def __len__(self):
 960         return len(self._key_list)
 961
 962     ## "in" test support
 963     def __contains__(self, key):
 964         return key in self._key_list
 965
 966     ## indexof support
 967     def index(self, key):
 968         return self._key_list.index(key)
 969
 970     ## insert support
 971     def insert(self, key, newkey, newvalue, order):
 972         index = self._key_list.index(key)
 973         if order == 'BEFORE':
 974             self._key_list.insert(index, newkey)
 975             IterableUserDict.__setitem__(self, newkey, newvalue)
 976         elif order == 'AFTER':
 977             self._key_list.insert(index + 1, newkey)
 978             IterableUserDict.__setitem__(self, newkey, newvalue)
 979
 980     ## append support
 981     def append(self, sdict):
 982         for key in sdict:
 983             if key not in self._key_list:
 984                 self._key_list.append(key)
 985             IterableUserDict.__setitem__(self, key, sdict[key])
 986
 987     def has_key(self, key):
 988         return key in self._key_list
 989
 990     ## Empty the dict
 991     def clear(self):
 992         self._key_list = []
 993         IterableUserDict.clear(self)
 994
 995     ## Return a copy of keys
 996     def keys(self):
 997         keys = []
 998         for key in self._key_list:
 999             keys.append(key)
1000         return keys
1001
1002     ## Return a copy of values
1003     def values(self):
1004         values = []
1005         for key in self._key_list:
1006             values.append(self[key])
1007         return values
1008
1009     ## Return a copy of (key, value) list
1010     def items(self):
1011         items = []
1012         for key in self._key_list:
1013             items.append((key, self[key]))
1014         return items
1015
1016     ## Iteration support
1017     def iteritems(self):
1018         return iter(self.items())
1019
1020     ## Keys interation support
1021     def iterkeys(self):
1022         return iter(self.keys())
1023
1024     ## Values interation support
1025     def itervalues(self):
1026         return iter(self.values())
1027
1028     ## Return value related to a key, and remove the (key, value) from the dict
1029     def pop(self, key, *dv):
1030         value = None
1031         if key in self._key_list:
1032             value = self[key]
1033             self.__delitem__(key)
1034         elif len(dv) != 0 :
1035             value = kv[0]
1036         return value
1037
1038     ## Return (key, value) pair, and remove the (key, value) from the dict
1039     def popitem(self):
1040         key = self._key_list[-1]
1041         value = self[key]
1042         self.__delitem__(key)
1043         return key, value
1044
1045     def update(self, dict=None, **kwargs):
1046         if dict is not None:
1047             for k, v in dict.items():
1048                 self[k] = v
1049         if len(kwargs):
1050             for k, v in kwargs.items():
1051                 self[k] = v
1052
1053 ## Dictionary with restricted keys
1054 #
1055 class rdict(dict):
1056     ## Constructor
1057     def __init__(self, KeyList):
1058         for Key in KeyList:
1059             dict.__setitem__(self, Key, "")
1060
1061     ## []= operator
1062     def __setitem__(self, key, value):
1063         if key not in self:
1064             EdkLogger.error("RestrictedDict", ATTRIBUTE_SET_FAILURE, "Key [%s] is not allowed" % key,
1065                             ExtraData=", ".join(dict.keys(self)))
1066         dict.__setitem__(self, key, value)
1067
1068     ## =[] operator
1069     def __getitem__(self, key):
1070         if key not in self:
1071             return ""
1072         return dict.__getitem__(self, key)
1073
1074     ## del operator
1075     def __delitem__(self, key):
1076         EdkLogger.error("RestrictedDict", ATTRIBUTE_ACCESS_DENIED, ExtraData="del")
1077
1078     ## Empty the dict
1079     def clear(self):
1080         for Key in self:
1081             self.__setitem__(Key, "")
1082
1083     ## Return value related to a key, and remove the (key, value) from the dict
1084     def pop(self, key, *dv):
1085         EdkLogger.error("RestrictedDict", ATTRIBUTE_ACCESS_DENIED, ExtraData="pop")
1086
1087     ## Return (key, value) pair, and remove the (key, value) from the dict
1088     def popitem(self):
1089         EdkLogger.error("RestrictedDict", ATTRIBUTE_ACCESS_DENIED, ExtraData="popitem")
1090
1091 ## Dictionary using prioritized list as key
1092 #
1093 class tdict:
1094     _ListType = type([])
1095     _TupleType = type(())
1096     _Wildcard = 'COMMON'
1097     _ValidWildcardList = ['COMMON', 'DEFAULT', 'ALL', '*', 'PLATFORM']
1098
1099     def __init__(self, _Single_=False, _Level_=2):
1100         self._Level_ = _Level_
1101         self.data = {}
1102         self._Single_ = _Single_
1103
1104     # =[] operator
1105     def __getitem__(self, key):
1106         KeyType = type(key)
1107         RestKeys = None
1108         if KeyType == self._ListType or KeyType == self._TupleType:
1109             FirstKey = key[0]
1110             if len(key) > 1:
1111                 RestKeys = key[1:]
1112             elif self._Level_ > 1:
1113                 RestKeys = [self._Wildcard for i in range(0, self._Level_ - 1)]
1114         else:
1115             FirstKey = key
1116             if self._Level_ > 1:
1117                 RestKeys = [self._Wildcard for i in range(0, self._Level_ - 1)]
1118
1119         if FirstKey is None or str(FirstKey).upper() in self._ValidWildcardList:
1120             FirstKey = self._Wildcard
1121
1122         if self._Single_:
1123             return self._GetSingleValue(FirstKey, RestKeys)
1124         else:
1125             return self._GetAllValues(FirstKey, RestKeys)
1126
1127     def _GetSingleValue(self, FirstKey, RestKeys):
1128         Value = None
1129         #print "%s-%s" % (FirstKey, self._Level_) ,
1130         if self._Level_ > 1:
1131             if FirstKey == self._Wildcard:
1132                 if FirstKey in self.data:
1133                     Value = self.data[FirstKey][RestKeys]
1134                 if Value is None:
1135                     for Key in self.data:
1136                         Value = self.data[Key][RestKeys]
1137                         if Value is not None: break
1138             else:
1139                 if FirstKey in self.data:
1140                     Value = self.data[FirstKey][RestKeys]
1141                 if Value is None and self._Wildcard in self.data:
1142                     #print "Value=None"
1143                     Value = self.data[self._Wildcard][RestKeys]
1144         else:
1145             if FirstKey == self._Wildcard:
1146                 if FirstKey in self.data:
1147                     Value = self.data[FirstKey]
1148                 if Value is None:
1149                     for Key in self.data:
1150                         Value = self.data[Key]
1151                         if Value is not None: break
1152             else:
1153                 if FirstKey in self.data:
1154                     Value = self.data[FirstKey]
1155                 elif self._Wildcard in self.data:
1156                     Value = self.data[self._Wildcard]
1157         return Value
1158
1159     def _GetAllValues(self, FirstKey, RestKeys):
1160         Value = []
1161         if self._Level_ > 1:
1162             if FirstKey == self._Wildcard:
1163                 for Key in self.data:
1164                     Value += self.data[Key][RestKeys]
1165             else:
1166                 if FirstKey in self.data:
1167                     Value += self.data[FirstKey][RestKeys]
1168                 if self._Wildcard in self.data:
1169                     Value += self.data[self._Wildcard][RestKeys]
1170         else:
1171             if FirstKey == self._Wildcard:
1172                 for Key in self.data:
1173                     Value.append(self.data[Key])
1174             else:
1175                 if FirstKey in self.data:
1176                     Value.append(self.data[FirstKey])
1177                 if self._Wildcard in self.data:
1178                     Value.append(self.data[self._Wildcard])
1179         return Value
1180
1181     ## []= operator
1182     def __setitem__(self, key, value):
1183         KeyType = type(key)
1184         RestKeys = None
1185         if KeyType == self._ListType or KeyType == self._TupleType:
1186             FirstKey = key[0]
1187             if len(key) > 1:
1188                 RestKeys = key[1:]
1189             else:
1190                 RestKeys = [self._Wildcard for i in range(0, self._Level_ - 1)]
1191         else:
1192             FirstKey = key
1193             if self._Level_ > 1:
1194                 RestKeys = [self._Wildcard for i in range(0, self._Level_ - 1)]
1195
1196         if FirstKey in self._ValidWildcardList:
1197             FirstKey = self._Wildcard
1198
1199         if FirstKey not in self.data and self._Level_ > 0:
1200             self.data[FirstKey] = tdict(self._Single_, self._Level_ - 1)
1201
1202         if self._Level_ > 1:
1203             self.data[FirstKey][RestKeys] = value
1204         else:
1205             self.data[FirstKey] = value
1206
1207     def SetGreedyMode(self):
1208         self._Single_ = False
1209         if self._Level_ > 1:
1210             for Key in self.data:
1211                 self.data[Key].SetGreedyMode()
1212
1213     def SetSingleMode(self):
1214         self._Single_ = True
1215         if self._Level_ > 1:
1216             for Key in self.data:
1217                 self.data[Key].SetSingleMode()
1218
1219     def GetKeys(self, KeyIndex=0):
1220         assert KeyIndex >= 0
1221         if KeyIndex == 0:
1222             return set(self.data.keys())
1223         else:
1224             keys = set()
1225             for Key in self.data:
1226                 keys |= self.data[Key].GetKeys(KeyIndex - 1)
1227             return keys
1228
1229 def IsFieldValueAnArray (Value):
1230     Value = Value.strip()
1231     if Value.startswith(TAB_GUID) and Value.endswith(')'):
1232         return True
1233     if Value.startswith('L"') and Value.endswith('"')  and len(list(Value[2:-1])) > 1:
1234         return True
1235     if Value[0] == '"' and Value[-1] == '"' and len(list(Value[1:-1])) > 1:
1236         return True
1237     if Value[0] == '{' and Value[-1] == '}':
1238         return True
1239     if Value.startswith("L'") and Value.endswith("'") and len(list(Value[2:-1])) > 1:
1240         return True
1241     if Value[0] == "'" and Value[-1] == "'" and len(list(Value[1:-1])) > 1:
1242         return True
1243     return False
1244
1245 def AnalyzePcdExpression(Setting):
1246     Setting = Setting.strip()
1247     # There might be escaped quote in a string: \", \\\" , \', \\\'
1248     Data = Setting
1249     # There might be '|' in string and in ( ... | ... ), replace it with '-'
1250     NewStr = ''
1251     InSingleQuoteStr = False
1252     InDoubleQuoteStr = False
1253     Pair = 0
1254     for Index, ch in enumerate(Data):
1255         if ch == '"' and not InSingleQuoteStr:
1256             if Data[Index - 1] != '\\':
1257                 InDoubleQuoteStr = not InDoubleQuoteStr
1258         elif ch == "'" and not InDoubleQuoteStr:
1259             if Data[Index - 1] != '\\':
1260                 InSingleQuoteStr = not InSingleQuoteStr
1261         elif ch == '(' and not (InSingleQuoteStr or InDoubleQuoteStr):
1262             Pair += 1
1263         elif ch == ')' and not (InSingleQuoteStr or InDoubleQuoteStr):
1264             Pair -= 1
1265
1266         if (Pair > 0 or InSingleQuoteStr or InDoubleQuoteStr) and ch == TAB_VALUE_SPLIT:
1267             NewStr += '-'
1268         else:
1269             NewStr += ch
1270     FieldList = []
1271     StartPos = 0
1272     while True:
1273         Pos = NewStr.find(TAB_VALUE_SPLIT, StartPos)
1274         if Pos < 0:
1275             FieldList.append(Setting[StartPos:].strip())
1276             break
1277         FieldList.append(Setting[StartPos:Pos].strip())
1278         StartPos = Pos + 1
1279
1280     return FieldList
1281
1282 def ParseDevPathValue (Value):
1283     if '\\' in Value:
1284         Value.replace('\\', '/').replace(' ', '')
1285
1286     Cmd = 'DevicePath ' + '"' + Value + '"'
1287     try:
1288         p = subprocess.Popen(Cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
1289         out, err = p.communicate()
1290     except Exception as X:
1291         raise BadExpression("DevicePath: %s" % (str(X)) )
1292     finally:
1293         subprocess._cleanup()
1294         p.stdout.close()
1295         p.stderr.close()
1296     if err:
1297         raise BadExpression("DevicePath: %s" % str(err))
1298     Size = len(out.split())
1299     out = ','.join(out.split())
1300     return '{' + out + '}', Size
1301
1302 def ParseFieldValue (Value):
1303     if isinstance(Value, type(0)):
1304         return Value, (Value.bit_length() + 7) / 8
1305     if not isinstance(Value, type('')):
1306         raise BadExpression('Type %s is %s' %(Value, type(Value)))
1307     Value = Value.strip()
1308     if Value.startswith(TAB_UINT8) and Value.endswith(')'):
1309         Value, Size = ParseFieldValue(Value.split('(', 1)[1][:-1])
1310         if Size > 1:
1311             raise BadExpression('Value (%s) Size larger than %d' %(Value, Size))
1312         return Value, 1
1313     if Value.startswith(TAB_UINT16) and Value.endswith(')'):
1314         Value, Size = ParseFieldValue(Value.split('(', 1)[1][:-1])
1315         if Size > 2:
1316             raise BadExpression('Value (%s) Size larger than %d' %(Value, Size))
1317         return Value, 2
1318     if Value.startswith(TAB_UINT32) and Value.endswith(')'):
1319         Value, Size = ParseFieldValue(Value.split('(', 1)[1][:-1])
1320         if Size > 4:
1321             raise BadExpression('Value (%s) Size larger than %d' %(Value, Size))
1322         return Value, 4
1323     if Value.startswith(TAB_UINT64) and Value.endswith(')'):
1324         Value, Size = ParseFieldValue(Value.split('(', 1)[1][:-1])
1325         if Size > 8:
1326             raise BadExpression('Value (%s) Size larger than %d' % (Value, Size))
1327         return Value, 8
1328     if Value.startswith(TAB_GUID) and Value.endswith(')'):
1329         Value = Value.split('(', 1)[1][:-1].strip()
1330         if Value[0] == '{' and Value[-1] == '}':
1331             TmpValue = GuidStructureStringToGuidString(Value)
1332             if not TmpValue:
1333                 raise BadExpression("Invalid GUID value string %s" % Value)
1334             Value = TmpValue
1335         if Value[0] == '"' and Value[-1] == '"':
1336             Value = Value[1:-1]
1337         try:
1338             Value = "'" + uuid.UUID(Value).get_bytes_le() + "'"
1339         except ValueError as Message:
1340             raise BadExpression(Message)
1341         Value, Size = ParseFieldValue(Value)
1342         return Value, 16
1343     if Value.startswith('L"') and Value.endswith('"'):
1344         # Unicode String
1345         # translate escape character
1346         Value = Value[1:]
1347         try:
1348             Value = eval(Value)
1349         except:
1350             Value = Value[1:-1]
1351         List = list(Value)
1352         List.reverse()
1353         Value = 0
1354         for Char in List:
1355             Value = (Value << 16) | ord(Char)
1356         return Value, (len(List) + 1) * 2
1357     if Value.startswith('"') and Value.endswith('"'):
1358         # ASCII String
1359         # translate escape character
1360         try:
1361             Value = eval(Value)
1362         except:
1363             Value = Value[1:-1]
1364         List = list(Value)
1365         List.reverse()
1366         Value = 0
1367         for Char in List:
1368             Value = (Value << 8) | ord(Char)
1369         return Value, len(List) + 1
1370     if Value.startswith("L'") and Value.endswith("'"):
1371         # Unicode Character Constant
1372         # translate escape character
1373         Value = Value[1:]
1374         try:
1375             Value = eval(Value)
1376         except:
1377             Value = Value[1:-1]
1378         List = list(Value)
1379         if len(List) == 0:
1380             raise BadExpression('Length %s is %s' % (Value, len(List)))
1381         List.reverse()
1382         Value = 0
1383         for Char in List:
1384             Value = (Value << 16) | ord(Char)
1385         return Value, len(List) * 2
1386     if Value.startswith("'") and Value.endswith("'"):
1387         # Character constant
1388         # translate escape character
1389         try:
1390             Value = eval(Value)
1391         except:
1392             Value = Value[1:-1]
1393         List = list(Value)
1394         if len(List) == 0:
1395             raise BadExpression('Length %s is %s' % (Value, len(List)))
1396         List.reverse()
1397         Value = 0
1398         for Char in List:
1399             Value = (Value << 8) | ord(Char)
1400         return Value, len(List)
1401     if Value.startswith('{') and Value.endswith('}'):
1402         # Byte array
1403         Value = Value[1:-1]
1404         List = [Item.strip() for Item in Value.split(',')]
1405         List.reverse()
1406         Value = 0
1407         RetSize = 0
1408         for Item in List:
1409             ItemValue, Size = ParseFieldValue(Item)
1410             RetSize += Size
1411             for I in range(Size):
1412                 Value = (Value << 8) | ((ItemValue >> 8 * I) & 0xff)
1413         return Value, RetSize
1414     if Value.startswith('DEVICE_PATH(') and Value.endswith(')'):
1415         Value = Value.replace("DEVICE_PATH(", '').rstrip(')')
1416         Value = Value.strip().strip('"')
1417         return ParseDevPathValue(Value)
1418     if Value.lower().startswith('0x'):
1419         try:
1420             Value = int(Value, 16)
1421         except:
1422             raise BadExpression("invalid hex value: %s" % Value)
1423         if Value == 0:
1424             return 0, 1
1425         return Value, (Value.bit_length() + 7) / 8
1426     if Value[0].isdigit():
1427         Value = int(Value, 10)
1428         if Value == 0:
1429             return 0, 1
1430         return Value, (Value.bit_length() + 7) / 8
1431     if Value.lower() == 'true':
1432         return 1, 1
1433     if Value.lower() == 'false':
1434         return 0, 1
1435     return Value, 1
1436
1437 ## AnalyzeDscPcd
1438 #
1439 #  Analyze DSC PCD value, since there is no data type info in DSC
1440 #  This function is used to match functions (AnalyzePcdData) used for retrieving PCD value from database
1441 #  1. Feature flag: TokenSpace.PcdCName|PcdValue
1442 #  2. Fix and Patch:TokenSpace.PcdCName|PcdValue[|VOID*[|MaxSize]]
1443 #  3. Dynamic default:
1444 #     TokenSpace.PcdCName|PcdValue[|VOID*[|MaxSize]]
1445 #     TokenSpace.PcdCName|PcdValue
1446 #  4. Dynamic VPD:
1447 #     TokenSpace.PcdCName|VpdOffset[|VpdValue]
1448 #     TokenSpace.PcdCName|VpdOffset[|MaxSize[|VpdValue]]
1449 #  5. Dynamic HII:
1450 #     TokenSpace.PcdCName|HiiString|VariableGuid|VariableOffset[|HiiValue]
1451 #  PCD value needs to be located in such kind of string, and the PCD value might be an expression in which
1452 #    there might have "|" operator, also in string value.
1453 #
1454 #  @param Setting: String contain information described above with "TokenSpace.PcdCName|" stripped
1455 #  @param PcdType: PCD type: feature, fixed, dynamic default VPD HII
1456 #  @param DataType: The datum type of PCD: VOID*, UNIT, BOOL
1457 #  @retval:
1458 #    ValueList: A List contain fields described above
1459 #    IsValid:   True if conforming EBNF, otherwise False
1460 #    Index:     The index where PcdValue is in ValueList
1461 #
1462 def AnalyzeDscPcd(Setting, PcdType, DataType=''):
1463     FieldList = AnalyzePcdExpression(Setting)
1464
1465     IsValid = True
1466     if PcdType in (MODEL_PCD_FIXED_AT_BUILD, MODEL_PCD_PATCHABLE_IN_MODULE, MODEL_PCD_DYNAMIC_DEFAULT, MODEL_PCD_DYNAMIC_EX_DEFAULT):
1467         Value = FieldList[0]
1468         Size = ''
1469         if len(FieldList) > 1 and FieldList[1]:
1470             DataType = FieldList[1]
1471             if FieldList[1] != TAB_VOID:
1472                 IsValid = False
1473         if len(FieldList) > 2:
1474             Size = FieldList[2]
1475         if IsValid:
1476             if DataType == "":
1477                 IsValid = (len(FieldList) <= 1)
1478             else:
1479                 IsValid = (len(FieldList) <= 3)
1480
1481         if Size:
1482             try:
1483                 int(Size, 16) if Size.upper().startswith("0X") else int(Size)
1484             except:
1485                 IsValid = False
1486                 Size = -1
1487         return [str(Value), DataType, str(Size)], IsValid, 0
1488     elif PcdType == MODEL_PCD_FEATURE_FLAG:
1489         Value = FieldList[0]
1490         Size = ''
1491         IsValid = (len(FieldList) <= 1)
1492         return [Value, DataType, str(Size)], IsValid, 0
1493     elif PcdType in (MODEL_PCD_DYNAMIC_VPD, MODEL_PCD_DYNAMIC_EX_VPD):
1494         VpdOffset = FieldList[0]
1495         Value = Size = ''
1496         if not DataType == TAB_VOID:
1497             if len(FieldList) > 1:
1498                 Value = FieldList[1]
1499         else:
1500             if len(FieldList) > 1:
1501                 Size = FieldList[1]
1502             if len(FieldList) > 2:
1503                 Value = FieldList[2]
1504         if DataType == "":
1505             IsValid = (len(FieldList) <= 1)
1506         else:
1507             IsValid = (len(FieldList) <= 3)
1508         if Size:
1509             try:
1510                 int(Size, 16) if Size.upper().startswith("0X") else int(Size)
1511             except:
1512                 IsValid = False
1513                 Size = -1
1514         return [VpdOffset, str(Size), Value], IsValid, 2
1515     elif PcdType in (MODEL_PCD_DYNAMIC_HII, MODEL_PCD_DYNAMIC_EX_HII):
1516         IsValid = (3 <= len(FieldList) <= 5)
1517         HiiString = FieldList[0]
1518         Guid = Offset = Value = Attribute = ''
1519         if len(FieldList) > 1:
1520             Guid = FieldList[1]
1521         if len(FieldList) > 2:
1522             Offset = FieldList[2]
1523         if len(FieldList) > 3:
1524             Value = FieldList[3]
1525             if not Value:
1526                 IsValid = False
1527         if len(FieldList) > 4:
1528             Attribute = FieldList[4]
1529         return [HiiString, Guid, Offset, Value, Attribute], IsValid, 3
1530     return [], False, 0
1531
1532 ## AnalyzePcdData
1533 #
1534 #  Analyze the pcd Value, Datum type and TokenNumber.
1535 #  Used to avoid split issue while the value string contain "|" character
1536 #
1537 #  @param[in] Setting:  A String contain value/datum type/token number information;
1538 #
1539 #  @retval   ValueList: A List contain value, datum type and toke number.
1540 #
1541 def AnalyzePcdData(Setting):
1542     ValueList = ['', '', '']
1543
1544     ValueRe = re.compile(r'^\s*L?\".*\|.*\"')
1545     PtrValue = ValueRe.findall(Setting)
1546
1547     ValueUpdateFlag = False
1548
1549     if len(PtrValue) >= 1:
1550         Setting = re.sub(ValueRe, '', Setting)
1551         ValueUpdateFlag = True
1552
1553     TokenList = Setting.split(TAB_VALUE_SPLIT)
1554     ValueList[0:len(TokenList)] = TokenList
1555
1556     if ValueUpdateFlag:
1557         ValueList[0] = PtrValue[0]
1558
1559     return ValueList
1560
1561 ## check format of PCD value against its the datum type
1562 #
1563 # For PCD value setting
1564 #
1565 def CheckPcdDatum(Type, Value):
1566     if Type == TAB_VOID:
1567         ValueRe = re.compile(r'\s*L?\".*\"\s*$')
1568         if not (((Value.startswith('L"') or Value.startswith('"')) and Value.endswith('"'))
1569                 or (Value.startswith('{') and Value.endswith('}')) or (Value.startswith("L'") or Value.startswith("'") and Value.endswith("'"))
1570                ):
1571             return False, "Invalid value [%s] of type [%s]; must be in the form of {...} for array"\
1572                           ", \"...\" or \'...\' for string, L\"...\" or L\'...\' for unicode string" % (Value, Type)
1573         elif ValueRe.match(Value):
1574             # Check the chars in UnicodeString or CString is printable
1575             if Value.startswith("L"):
1576                 Value = Value[2:-1]
1577             else:
1578                 Value = Value[1:-1]
1579             Printset = set(string.printable)
1580             Printset.remove(TAB_PRINTCHAR_VT)
1581             Printset.add(TAB_PRINTCHAR_BS)
1582             Printset.add(TAB_PRINTCHAR_NUL)
1583             if not set(Value).issubset(Printset):
1584                 PrintList = sorted(Printset)
1585                 return False, "Invalid PCD string value of type [%s]; must be printable chars %s." % (Type, PrintList)
1586     elif Type == 'BOOLEAN':
1587         if Value not in ['TRUE', 'True', 'true', '0x1', '0x01', '1', 'FALSE', 'False', 'false', '0x0', '0x00', '0']:
1588             return False, "Invalid value [%s] of type [%s]; must be one of TRUE, True, true, 0x1, 0x01, 1"\
1589                           ", FALSE, False, false, 0x0, 0x00, 0" % (Value, Type)
1590     elif Type in [TAB_UINT8, TAB_UINT16, TAB_UINT32, TAB_UINT64]:
1591         try:
1592             Value = long(Value, 0)
1593         except:
1594             return False, "Invalid value [%s] of type [%s];"\
1595                           " must be a hexadecimal, decimal or octal in C language format." % (Value, Type)
1596     else:
1597         return True, "StructurePcd"
1598
1599     return True, ""
1600
1601 ## Split command line option string to list
1602 #
1603 # subprocess.Popen needs the args to be a sequence. Otherwise there's problem
1604 # in non-windows platform to launch command
1605 #
1606 def SplitOption(OptionString):
1607     OptionList = []
1608     LastChar = " "
1609     OptionStart = 0
1610     QuotationMark = ""
1611     for Index in range(0, len(OptionString)):
1612         CurrentChar = OptionString[Index]
1613         if CurrentChar in ['"', "'"]:
1614             if QuotationMark == CurrentChar:
1615                 QuotationMark = ""
1616             elif QuotationMark == "":
1617                 QuotationMark = CurrentChar
1618             continue
1619         elif QuotationMark:
1620             continue
1621
1622         if CurrentChar in ["/", "-"] and LastChar in [" ", "\t", "\r", "\n"]:
1623             if Index > OptionStart:
1624                 OptionList.append(OptionString[OptionStart:Index - 1])
1625             OptionStart = Index
1626         LastChar = CurrentChar
1627     OptionList.append(OptionString[OptionStart:])
1628     return OptionList
1629
1630 def CommonPath(PathList):
1631     P1 = min(PathList).split(os.path.sep)
1632     P2 = max(PathList).split(os.path.sep)
1633     for Index in xrange(min(len(P1), len(P2))):
1634         if P1[Index] != P2[Index]:
1635             return os.path.sep.join(P1[:Index])
1636     return os.path.sep.join(P1)
1637
1638 #
1639 # Convert string to C format array
1640 #
1641 def ConvertStringToByteArray(Value):
1642     Value = Value.strip()
1643     if not Value:
1644         return None
1645     if Value[0] == '{':
1646         if not Value.endswith('}'):
1647             return None
1648         Value = Value.replace(' ', '').replace('{', '').replace('}', '')
1649         ValFields = Value.split(',')
1650         try:
1651             for Index in range(len(ValFields)):
1652                 ValFields[Index] = str(int(ValFields[Index], 0))
1653         except ValueError:
1654             return None
1655         Value = '{' + ','.join(ValFields) + '}'
1656         return Value
1657
1658     Unicode = False
1659     if Value.startswith('L"'):
1660         if not Value.endswith('"'):
1661             return None
1662         Value = Value[1:]
1663         Unicode = True
1664     elif not Value.startswith('"') or not Value.endswith('"'):
1665         return None
1666
1667     Value = eval(Value)         # translate escape character
1668     NewValue = '{'
1669     for Index in range(0, len(Value)):
1670         if Unicode:
1671             NewValue = NewValue + str(ord(Value[Index]) % 0x10000) + ','
1672         else:
1673             NewValue = NewValue + str(ord(Value[Index]) % 0x100) + ','
1674     Value = NewValue + '0}'
1675     return Value
1676
1677 class PathClass(object):
1678     def __init__(self, File='', Root='', AlterRoot='', Type='', IsBinary=False,
1679                  Arch='COMMON', ToolChainFamily='', Target='', TagName='', ToolCode=''):
1680         self.Arch = Arch
1681         self.File = str(File)
1682         if os.path.isabs(self.File):
1683             self.Root = ''
1684             self.AlterRoot = ''
1685         else:
1686             self.Root = str(Root)
1687             self.AlterRoot = str(AlterRoot)
1688
1689         # Remove any '.' and '..' in path
1690         if self.Root:
1691             self.Root = mws.getWs(self.Root, self.File)
1692             self.Path = os.path.normpath(os.path.join(self.Root, self.File))
1693             self.Root = os.path.normpath(CommonPath([self.Root, self.Path]))
1694             # eliminate the side-effect of 'C:'
1695             if self.Root[-1] == ':':
1696                 self.Root += os.path.sep
1697             # file path should not start with path separator
1698             if self.Root[-1] == os.path.sep:
1699                 self.File = self.Path[len(self.Root):]
1700             else:
1701                 self.File = self.Path[len(self.Root) + 1:]
1702         else:
1703             self.Path = os.path.normpath(self.File)
1704
1705         self.SubDir, self.Name = os.path.split(self.File)
1706         self.BaseName, self.Ext = os.path.splitext(self.Name)
1707
1708         if self.Root:
1709             if self.SubDir:
1710                 self.Dir = os.path.join(self.Root, self.SubDir)
1711             else:
1712                 self.Dir = self.Root
1713         else:
1714             self.Dir = self.SubDir
1715
1716         if IsBinary:
1717             self.Type = Type
1718         else:
1719             self.Type = self.Ext.lower()
1720
1721         self.IsBinary = IsBinary
1722         self.Target = Target
1723         self.TagName = TagName
1724         self.ToolCode = ToolCode
1725         self.ToolChainFamily = ToolChainFamily
1726
1727         self._Key = None
1728
1729     ## Convert the object of this class to a string
1730     #
1731     #  Convert member Path of the class to a string
1732     #
1733     #  @retval string Formatted String
1734     #
1735     def __str__(self):
1736         return self.Path
1737
1738     ## Override __eq__ function
1739     #
1740     # Check whether PathClass are the same
1741     #
1742     # @retval False The two PathClass are different
1743     # @retval True  The two PathClass are the same
1744     #
1745     def __eq__(self, Other):
1746         if isinstance(Other, type(self)):
1747             return self.Path == Other.Path
1748         else:
1749             return self.Path == str(Other)
1750
1751     ## Override __cmp__ function
1752     #
1753     # Customize the comparsion operation of two PathClass
1754     #
1755     # @retval 0     The two PathClass are different
1756     # @retval -1    The first PathClass is less than the second PathClass
1757     # @retval 1     The first PathClass is Bigger than the second PathClass
1758     def __cmp__(self, Other):
1759         if isinstance(Other, type(self)):
1760             OtherKey = Other.Path
1761         else:
1762             OtherKey = str(Other)
1763
1764         SelfKey = self.Path
1765         if SelfKey == OtherKey:
1766             return 0
1767         elif SelfKey > OtherKey:
1768             return 1
1769         else:
1770             return -1
1771
1772     ## Override __hash__ function
1773     #
1774     # Use Path as key in hash table
1775     #
1776     # @retval string Key for hash table
1777     #
1778     def __hash__(self):
1779         return hash(self.Path)
1780
1781     def _GetFileKey(self):
1782         if self._Key is None:
1783             self._Key = self.Path.upper()   # + self.ToolChainFamily + self.TagName + self.ToolCode + self.Target
1784         return self._Key
1785
1786     def _GetTimeStamp(self):
1787         return os.stat(self.Path)[8]
1788
1789     def Validate(self, Type='', CaseSensitive=True):
1790         if GlobalData.gCaseInsensitive:
1791             CaseSensitive = False
1792         if Type and Type.lower() != self.Type:
1793             return FILE_TYPE_MISMATCH, '%s (expect %s but got %s)' % (self.File, Type, self.Type)
1794
1795         RealFile, RealRoot = RealPath2(self.File, self.Root, self.AlterRoot)
1796         if not RealRoot and not RealFile:
1797             RealFile = self.File
1798             if self.AlterRoot:
1799                 RealFile = os.path.join(self.AlterRoot, self.File)
1800             elif self.Root:
1801                 RealFile = os.path.join(self.Root, self.File)
1802             if len (mws.getPkgPath()) == 0:
1803                 return FILE_NOT_FOUND, os.path.join(self.AlterRoot, RealFile)
1804             else:
1805                 return FILE_NOT_FOUND, "%s is not found in packages path:\n\t%s" % (self.File, '\n\t'.join(mws.getPkgPath()))
1806
1807         ErrorCode = 0
1808         ErrorInfo = ''
1809         if RealRoot != self.Root or RealFile != self.File:
1810             if CaseSensitive and (RealFile != self.File or (RealRoot != self.Root and RealRoot != self.AlterRoot)):
1811                 ErrorCode = FILE_CASE_MISMATCH
1812                 ErrorInfo = self.File + '\n\t' + RealFile + " [in file system]"
1813
1814             self.SubDir, self.Name = os.path.split(RealFile)
1815             self.BaseName, self.Ext = os.path.splitext(self.Name)
1816             if self.SubDir:
1817                 self.Dir = os.path.join(RealRoot, self.SubDir)
1818             else:
1819                 self.Dir = RealRoot
1820             self.File = RealFile
1821             self.Root = RealRoot
1822             self.Path = os.path.join(RealRoot, RealFile)
1823         return ErrorCode, ErrorInfo
1824
1825     Key = property(_GetFileKey)
1826     TimeStamp = property(_GetTimeStamp)
1827
1828 ## Parse PE image to get the required PE informaion.
1829 #
1830 class PeImageClass():
1831     ## Constructor
1832     #
1833     #   @param  File FilePath of PeImage
1834     #
1835     def __init__(self, PeFile):
1836         self.FileName   = PeFile
1837         self.IsValid    = False
1838         self.Size       = 0
1839         self.EntryPoint = 0
1840         self.SectionAlignment  = 0
1841         self.SectionHeaderList = []
1842         self.ErrorInfo = ''
1843         try:
1844             PeObject = open(PeFile, 'rb')
1845         except:
1846             self.ErrorInfo = self.FileName + ' can not be found\n'
1847             return
1848         # Read DOS header
1849         ByteArray = array.array('B')
1850         ByteArray.fromfile(PeObject, 0x3E)
1851         ByteList = ByteArray.tolist()
1852         # DOS signature should be 'MZ'
1853         if self._ByteListToStr (ByteList[0x0:0x2]) != 'MZ':
1854             self.ErrorInfo = self.FileName + ' has no valid DOS signature MZ'
1855             return
1856
1857         # Read 4 byte PE Signature
1858         PeOffset = self._ByteListToInt(ByteList[0x3C:0x3E])
1859         PeObject.seek(PeOffset)
1860         ByteArray = array.array('B')
1861         ByteArray.fromfile(PeObject, 4)
1862         # PE signature should be 'PE\0\0'
1863         if ByteArray.tostring() != 'PE\0\0':
1864             self.ErrorInfo = self.FileName + ' has no valid PE signature PE00'
1865             return
1866
1867         # Read PE file header
1868         ByteArray = array.array('B')
1869         ByteArray.fromfile(PeObject, 0x14)
1870         ByteList = ByteArray.tolist()
1871         SecNumber = self._ByteListToInt(ByteList[0x2:0x4])
1872         if SecNumber == 0:
1873             self.ErrorInfo = self.FileName + ' has no section header'
1874             return
1875
1876         # Read PE optional header
1877         OptionalHeaderSize = self._ByteListToInt(ByteArray[0x10:0x12])
1878         ByteArray = array.array('B')
1879         ByteArray.fromfile(PeObject, OptionalHeaderSize)
1880         ByteList = ByteArray.tolist()
1881         self.EntryPoint       = self._ByteListToInt(ByteList[0x10:0x14])
1882         self.SectionAlignment = self._ByteListToInt(ByteList[0x20:0x24])
1883         self.Size             = self._ByteListToInt(ByteList[0x38:0x3C])
1884
1885         # Read each Section Header
1886         for Index in range(SecNumber):
1887             ByteArray = array.array('B')
1888             ByteArray.fromfile(PeObject, 0x28)
1889             ByteList = ByteArray.tolist()
1890             SecName  = self._ByteListToStr(ByteList[0:8])
1891             SecVirtualSize = self._ByteListToInt(ByteList[8:12])
1892             SecRawAddress  = self._ByteListToInt(ByteList[20:24])
1893             SecVirtualAddress = self._ByteListToInt(ByteList[12:16])
1894             self.SectionHeaderList.append((SecName, SecVirtualAddress, SecRawAddress, SecVirtualSize))
1895         self.IsValid = True
1896         PeObject.close()
1897
1898     def _ByteListToStr(self, ByteList):
1899         String = ''
1900         for index in range(len(ByteList)):
1901             if ByteList[index] == 0:
1902                 break
1903             String += chr(ByteList[index])
1904         return String
1905
1906     def _ByteListToInt(self, ByteList):
1907         Value = 0
1908         for index in range(len(ByteList) - 1, -1, -1):
1909             Value = (Value << 8) | int(ByteList[index])
1910         return Value
1911
1912 class DefaultStore():
1913     def __init__(self, DefaultStores ):
1914
1915         self.DefaultStores = DefaultStores
1916     def DefaultStoreID(self, DefaultStoreName):
1917         for key, value in self.DefaultStores.items():
1918             if value == DefaultStoreName:
1919                 return key
1920         return None
1921     def GetDefaultDefault(self):
1922         if not self.DefaultStores or "0" in self.DefaultStores:
1923             return "0", TAB_DEFAULT_STORES_DEFAULT
1924         else:
1925             minvalue = min(int(value_str) for value_str in self.DefaultStores)
1926             return (str(minvalue), self.DefaultStores[str(minvalue)])
1927     def GetMin(self, DefaultSIdList):
1928         if not DefaultSIdList:
1929             return TAB_DEFAULT_STORES_DEFAULT
1930         storeidset = {storeid for storeid, storename in self.DefaultStores.values() if storename in DefaultSIdList}
1931         if not storeidset:
1932             return ""
1933         minid = min(storeidset )
1934         for sid, name in self.DefaultStores.values():
1935             if sid == minid:
1936                 return name
1937 class SkuClass():
1938
1939     DEFAULT = 0
1940     SINGLE = 1
1941     MULTIPLE =2
1942
1943     def __init__(self,SkuIdentifier='', SkuIds=None):
1944         if SkuIds is None:
1945             SkuIds = {}
1946
1947         for SkuName in SkuIds:
1948             SkuId = SkuIds[SkuName][0]
1949             skuid_num = int(SkuId, 16) if SkuId.upper().startswith("0X") else int(SkuId)
1950             if skuid_num > 0xFFFFFFFFFFFFFFFF:
1951                 EdkLogger.error("build", PARAMETER_INVALID,
1952                             ExtraData = "SKU-ID [%s] value %s exceeds the max value of UINT64"
1953                                       % (SkuName, SkuId))
1954
1955         self.AvailableSkuIds = sdict()
1956         self.SkuIdSet = []
1957         self.SkuIdNumberSet = []
1958         self.SkuData = SkuIds
1959         self.__SkuInherit = {}
1960         self.__SkuIdentifier = SkuIdentifier
1961         if SkuIdentifier == '' or SkuIdentifier is None:
1962             self.SkuIdSet = ['DEFAULT']
1963             self.SkuIdNumberSet = ['0U']
1964         elif SkuIdentifier == 'ALL':
1965             self.SkuIdSet = SkuIds.keys()
1966             self.SkuIdNumberSet = [num[0].strip() + 'U' for num in SkuIds.values()]
1967         else:
1968             r = SkuIdentifier.split('|')
1969             self.SkuIdSet=[(r[k].strip()).upper() for k in range(len(r))]
1970             k = None
1971             try:
1972                 self.SkuIdNumberSet = [SkuIds[k][0].strip() + 'U' for k in self.SkuIdSet]
1973             except Exception:
1974                 EdkLogger.error("build", PARAMETER_INVALID,
1975                             ExtraData = "SKU-ID [%s] is not supported by the platform. [Valid SKU-ID: %s]"
1976                                       % (k, " | ".join(SkuIds.keys())))
1977         for each in self.SkuIdSet:
1978             if each in SkuIds:
1979                 self.AvailableSkuIds[each] = SkuIds[each][0]
1980             else:
1981                 EdkLogger.error("build", PARAMETER_INVALID,
1982                             ExtraData="SKU-ID [%s] is not supported by the platform. [Valid SKU-ID: %s]"
1983                                       % (each, " | ".join(SkuIds.keys())))
1984         if self.SkuUsageType != self.SINGLE:
1985             self.AvailableSkuIds.update({'DEFAULT':0, 'COMMON':0})
1986         if self.SkuIdSet:
1987             GlobalData.gSkuids = (self.SkuIdSet)
1988             if 'COMMON' in GlobalData.gSkuids:
1989                 GlobalData.gSkuids.remove('COMMON')
1990             if self.SkuUsageType == self.SINGLE:
1991                 if len(GlobalData.gSkuids) != 1:
1992                     if 'DEFAULT' in GlobalData.gSkuids:
1993                         GlobalData.gSkuids.remove('DEFAULT')
1994             if GlobalData.gSkuids:
1995                 GlobalData.gSkuids.sort()
1996
1997     def GetNextSkuId(self, skuname):
1998         if not self.__SkuInherit:
1999             self.__SkuInherit = {}
2000             for item in self.SkuData.values():
2001                 self.__SkuInherit[item[1]]=item[2] if item[2] else "DEFAULT"
2002         return self.__SkuInherit.get(skuname, "DEFAULT")
2003
2004     def GetSkuChain(self, sku):
2005         if sku == "DEFAULT":
2006             return ["DEFAULT"]
2007         skulist = [sku]
2008         nextsku = sku
2009         while True:
2010             nextsku = self.GetNextSkuId(nextsku)
2011             skulist.append(nextsku)
2012             if nextsku == "DEFAULT":
2013                 break
2014         skulist.reverse()
2015         return skulist
2016     def SkuOverrideOrder(self):
2017         skuorderset = []
2018         for skuname in self.SkuIdSet:
2019             skuorderset.append(self.GetSkuChain(skuname))
2020
2021         skuorder = []
2022         for index in range(max(len(item) for item in skuorderset)):
2023             for subset in skuorderset:
2024                 if index > len(subset)-1:
2025                     continue
2026                 if subset[index] in skuorder:
2027                     continue
2028                 skuorder.append(subset[index])
2029
2030         return skuorder
2031
2032     def __SkuUsageType(self):
2033
2034         if self.__SkuIdentifier.upper() == "ALL":
2035             return SkuClass.MULTIPLE
2036
2037         if len(self.SkuIdSet) == 1:
2038             if self.SkuIdSet[0] == 'DEFAULT':
2039                 return SkuClass.DEFAULT
2040             else:
2041                 return SkuClass.SINGLE
2042         elif len(self.SkuIdSet) == 2:
2043             if 'DEFAULT' in self.SkuIdSet:
2044                 return SkuClass.SINGLE
2045             else:
2046                 return SkuClass.MULTIPLE
2047         else:
2048             return SkuClass.MULTIPLE
2049     def DumpSkuIdArrary(self):
2050
2051         ArrayStrList = []
2052         if self.SkuUsageType == SkuClass.SINGLE:
2053             ArrayStr = "{0x0}"
2054         else:
2055             for skuname in self.AvailableSkuIds:
2056                 if skuname == "COMMON":
2057                     continue
2058                 while skuname != "DEFAULT":
2059                     ArrayStrList.append(hex(int(self.AvailableSkuIds[skuname])))
2060                     skuname = self.GetNextSkuId(skuname)
2061                 ArrayStrList.append("0x0")
2062             ArrayStr = "{" + ",".join(ArrayStrList) +  "}"
2063         return ArrayStr
2064     def __GetAvailableSkuIds(self):
2065         return self.AvailableSkuIds
2066
2067     def __GetSystemSkuID(self):
2068         if self.__SkuUsageType() == SkuClass.SINGLE:
2069             if len(self.SkuIdSet) == 1:
2070                 return self.SkuIdSet[0]
2071             else:
2072                 return self.SkuIdSet[0] if self.SkuIdSet[0] != 'DEFAULT' else self.SkuIdSet[1]
2073         else:
2074             return 'DEFAULT'
2075     def __GetAvailableSkuIdNumber(self):
2076         return self.SkuIdNumberSet
2077     SystemSkuId = property(__GetSystemSkuID)
2078     AvailableSkuIdSet = property(__GetAvailableSkuIds)
2079     SkuUsageType = property(__SkuUsageType)
2080     AvailableSkuIdNumSet = property(__GetAvailableSkuIdNumber)
2081
2082 #
2083 # Pack a registry format GUID
2084 #
2085 def PackRegistryFormatGuid(Guid):
2086     return PackGUID(Guid.split('-'))
2087
2088 ##  Get the integer value from string like "14U" or integer like 2
2089 #
2090 #   @param      Input   The object that may be either a integer value or a string
2091 #
2092 #   @retval     Value    The integer value that the input represents
2093 #
2094 def GetIntegerValue(Input):
2095     if type(Input) in (int, long):
2096         return Input
2097     String = Input
2098     if String.endswith("U"):
2099         String = String[:-1]
2100     if String.endswith("ULL"):
2101         String = String[:-3]
2102     if String.endswith("LL"):
2103         String = String[:-2]
2104
2105     if String.startswith("0x") or String.startswith("0X"):
2106         return int(String, 16)
2107     elif String == '':
2108         return 0
2109     else:
2110         return int(String)
2111
2112 #
2113 # Pack a GUID (registry format) list into a buffer and return it
2114 #
2115 def PackGUID(Guid):
2116     return pack(PACK_PATTERN_GUID,
2117                 int(Guid[0], 16),
2118                 int(Guid[1], 16),
2119                 int(Guid[2], 16),
2120                 int(Guid[3][-4:-2], 16),
2121                 int(Guid[3][-2:], 16),
2122                 int(Guid[4][-12:-10], 16),
2123                 int(Guid[4][-10:-8], 16),
2124                 int(Guid[4][-8:-6], 16),
2125                 int(Guid[4][-6:-4], 16),
2126                 int(Guid[4][-4:-2], 16),
2127                 int(Guid[4][-2:], 16)
2128                 )
2129
2130 #
2131 # Pack a GUID (byte) list into a buffer and return it
2132 #
2133 def PackByteFormatGUID(Guid):
2134     return pack(PACK_PATTERN_GUID,
2135                 Guid[0],
2136                 Guid[1],
2137                 Guid[2],
2138                 Guid[3],
2139                 Guid[4],
2140                 Guid[5],
2141                 Guid[6],
2142                 Guid[7],
2143                 Guid[8],
2144                 Guid[9],
2145                 Guid[10],
2146                 )
2147
2148 ##
2149 #
2150 # This acts like the main() function for the script, unless it is 'import'ed into another
2151 # script.
2152 #
2153 if __name__ == '__main__':
2154     pass
2155