BaseTools/Source/Python/Common/Misc.py

   1 ## @file
   2 # Common routines used by all tools
   3 #
   4 # Copyright (c) 2007 - 2018, Intel Corporation. All rights reserved.<BR>
   5 # This program and the accompanying materials
   6 # are licensed and made available under the terms and conditions of the BSD License
   7 # which accompanies this distribution.  The full text of the license may be found at
   8 # http://opensource.org/licenses/bsd-license.php
   9 #
  10 # THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS,
  11 # WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED.
  12 #
  13
  14 ##
  15 # Import Modules
  16 #
  17 import Common.LongFilePathOs as os
  18 import sys
  19 import string
  20 import threading
  21 import time
  22 import re
  23 import pickle
  24 import array
  25 import shutil
  26 from struct import pack
  27 from UserDict import IterableUserDict
  28 from UserList import UserList
  29
  30 from Common import EdkLogger as EdkLogger
  31 from Common import GlobalData as GlobalData
  32 from .DataType import *
  33 from .BuildToolError import *
  34 from CommonDataClass.DataClass import *
  35 from .Parsing import GetSplitValueList
  36 from Common.LongFilePathSupport import OpenLongFilePath as open
  37 from Common.MultipleWorkspace import MultipleWorkspace as mws
  38 import uuid
  39 from CommonDataClass.Exceptions import BadExpression
  40 from Common.caching import cached_property
  41 import subprocess
  42 ## Regular expression used to find out place holders in string template
  43 gPlaceholderPattern = re.compile("\$\{([^$()\s]+)\}", re.MULTILINE | re.UNICODE)
  44
  45 ## regular expressions for map file processing
  46 startPatternGeneral = re.compile("^Start[' ']+Length[' ']+Name[' ']+Class")
  47 addressPatternGeneral = re.compile("^Address[' ']+Publics by Value[' ']+Rva\+Base")
  48 valuePatternGcc = re.compile('^([\w_\.]+) +([\da-fA-Fx]+) +([\da-fA-Fx]+)$')
  49 pcdPatternGcc = re.compile('^([\da-fA-Fx]+) +([\da-fA-Fx]+)')
  50 secReGeneral = re.compile('^([\da-fA-F]+):([\da-fA-F]+) +([\da-fA-F]+)[Hh]? +([.\w\$]+) +(\w+)', re.UNICODE)
  51
  52 ## Dictionary used to store file time stamp for quick re-access
  53 gFileTimeStampCache = {}    # {file path : file time stamp}
  54
  55 ## Dictionary used to store dependencies of files
  56 gDependencyDatabase = {}    # arch : {file path : [dependent files list]}
  57
  58 #
  59 # If a module is built more than once with different PCDs or library classes
  60 # a temporary INF file with same content is created, the temporary file is removed
  61 # when build exits.
  62 #
  63 _TempInfs = []
  64
  65 def GetVariableOffset(mapfilepath, efifilepath, varnames):
  66     """ Parse map file to get variable offset in current EFI file
  67     @param mapfilepath    Map file absolution path
  68     @param efifilepath:   EFI binary file full path
  69     @param varnames       iteratable container whose elements are variable names to be searched
  70
  71     @return List whos elements are tuple with variable name and raw offset
  72     """
  73     lines = []
  74     try:
  75         f = open(mapfilepath, 'r')
  76         lines = f.readlines()
  77         f.close()
  78     except:
  79         return None
  80
  81     if len(lines) == 0: return None
  82     firstline = lines[0].strip()
  83     if (firstline.startswith("Archive member included ") and
  84         firstline.endswith(" file (symbol)")):
  85         return _parseForGCC(lines, efifilepath, varnames)
  86     if firstline.startswith("# Path:"):
  87         return _parseForXcode(lines, efifilepath, varnames)
  88     return _parseGeneral(lines, efifilepath, varnames)
  89
  90 def _parseForXcode(lines, efifilepath, varnames):
  91     status = 0
  92     ret = []
  93     for line in lines:
  94         line = line.strip()
  95         if status == 0 and line == "# Symbols:":
  96             status = 1
  97             continue
  98         if status == 1 and len(line) != 0:
  99             for varname in varnames:
 100                 if varname in line:
 101                     # cannot pregenerate this RegEx since it uses varname from varnames.
 102                     m = re.match('^([\da-fA-FxX]+)([\s\S]*)([_]*%s)$' % varname, line)
 103                     if m is not None:
 104                         ret.append((varname, m.group(1)))
 105     return ret
 106
 107 def _parseForGCC(lines, efifilepath, varnames):
 108     """ Parse map file generated by GCC linker """
 109     status = 0
 110     sections = []
 111     varoffset = []
 112     for index, line in enumerate(lines):
 113         line = line.strip()
 114         # status machine transection
 115         if status == 0 and line == "Memory Configuration":
 116             status = 1
 117             continue
 118         elif status == 1 and line == 'Linker script and memory map':
 119             status = 2
 120             continue
 121         elif status ==2 and line == 'START GROUP':
 122             status = 3
 123             continue
 124
 125         # status handler
 126         if status == 3:
 127             m = valuePatternGcc.match(line)
 128             if m is not None:
 129                 sections.append(m.groups(0))
 130             for varname in varnames:
 131                 Str = ''
 132                 m = re.match("^.data.(%s)" % varname, line)
 133                 if m is not None:
 134                     m = re.match(".data.(%s)$" % varname, line)
 135                     if m is not None:
 136                         Str = lines[index + 1]
 137                     else:
 138                         Str = line[len(".data.%s" % varname):]
 139                     if Str:
 140                         m = pcdPatternGcc.match(Str.strip())
 141                         if m is not None:
 142                             varoffset.append((varname, int(m.groups(0)[0], 16), int(sections[-1][1], 16), sections[-1][0]))
 143
 144     if not varoffset:
 145         return []
 146     # get section information from efi file
 147     efisecs = PeImageClass(efifilepath).SectionHeaderList
 148     if efisecs is None or len(efisecs) == 0:
 149         return []
 150     #redirection
 151     redirection = 0
 152     for efisec in efisecs:
 153         for section in sections:
 154             if section[0].strip() == efisec[0].strip() and section[0].strip() == '.text':
 155                 redirection = int(section[1], 16) - efisec[1]
 156
 157     ret = []
 158     for var in varoffset:
 159         for efisec in efisecs:
 160             if var[1] >= efisec[1] and var[1] < efisec[1]+efisec[3]:
 161                 ret.append((var[0], hex(efisec[2] + var[1] - efisec[1] - redirection)))
 162     return ret
 163
 164 def _parseGeneral(lines, efifilepath, varnames):
 165     status = 0    #0 - beginning of file; 1 - PE section definition; 2 - symbol table
 166     secs  = []    # key = section name
 167     varoffset = []
 168     symRe = re.compile('^([\da-fA-F]+):([\da-fA-F]+) +([\.:\\\\\w\?@\$]+) +([\da-fA-F]+)', re.UNICODE)
 169
 170     for line in lines:
 171         line = line.strip()
 172         if startPatternGeneral.match(line):
 173             status = 1
 174             continue
 175         if addressPatternGeneral.match(line):
 176             status = 2
 177             continue
 178         if line.startswith("entry point at"):
 179             status = 3
 180             continue
 181         if status == 1 and len(line) != 0:
 182             m =  secReGeneral.match(line)
 183             assert m is not None, "Fail to parse the section in map file , line is %s" % line
 184             sec_no, sec_start, sec_length, sec_name, sec_class = m.groups(0)
 185             secs.append([int(sec_no, 16), int(sec_start, 16), int(sec_length, 16), sec_name, sec_class])
 186         if status == 2 and len(line) != 0:
 187             for varname in varnames:
 188                 m = symRe.match(line)
 189                 assert m is not None, "Fail to parse the symbol in map file, line is %s" % line
 190                 sec_no, sym_offset, sym_name, vir_addr = m.groups(0)
 191                 sec_no     = int(sec_no,     16)
 192                 sym_offset = int(sym_offset, 16)
 193                 vir_addr   = int(vir_addr,   16)
 194                 # cannot pregenerate this RegEx since it uses varname from varnames.
 195                 m2 = re.match('^[_]*(%s)' % varname, sym_name)
 196                 if m2 is not None:
 197                     # fond a binary pcd entry in map file
 198                     for sec in secs:
 199                         if sec[0] == sec_no and (sym_offset >= sec[1] and sym_offset < sec[1] + sec[2]):
 200                             varoffset.append([varname, sec[3], sym_offset, vir_addr, sec_no])
 201
 202     if not varoffset: return []
 203
 204     # get section information from efi file
 205     efisecs = PeImageClass(efifilepath).SectionHeaderList
 206     if efisecs is None or len(efisecs) == 0:
 207         return []
 208
 209     ret = []
 210     for var in varoffset:
 211         index = 0
 212         for efisec in efisecs:
 213             index = index + 1
 214             if var[1].strip() == efisec[0].strip():
 215                 ret.append((var[0], hex(efisec[2] + var[2])))
 216             elif var[4] == index:
 217                 ret.append((var[0], hex(efisec[2] + var[2])))
 218
 219     return ret
 220
 221 ## Routine to process duplicated INF
 222 #
 223 #  This function is called by following two cases:
 224 #  Case 1 in DSC:
 225 #    [components.arch]
 226 #    Pkg/module/module.inf
 227 #    Pkg/module/module.inf {
 228 #      <Defines>
 229 #        FILE_GUID = 0D1B936F-68F3-4589-AFCC-FB8B7AEBC836
 230 #    }
 231 #  Case 2 in FDF:
 232 #    INF Pkg/module/module.inf
 233 #    INF FILE_GUID = 0D1B936F-68F3-4589-AFCC-FB8B7AEBC836 Pkg/module/module.inf
 234 #
 235 #  This function copies Pkg/module/module.inf to
 236 #  Conf/.cache/0D1B936F-68F3-4589-AFCC-FB8B7AEBC836module.inf
 237 #
 238 #  @param Path     Original PathClass object
 239 #  @param BaseName New file base name
 240 #
 241 #  @retval         return the new PathClass object
 242 #
 243 def ProcessDuplicatedInf(Path, BaseName, Workspace):
 244     Filename = os.path.split(Path.File)[1]
 245     if '.' in Filename:
 246         Filename = BaseName + Path.BaseName + Filename[Filename.rfind('.'):]
 247     else:
 248         Filename = BaseName + Path.BaseName
 249
 250     #
 251     # If -N is specified on command line, cache is disabled
 252     # The directory has to be created
 253     #
 254     DbDir = os.path.split(GlobalData.gDatabasePath)[0]
 255     if not os.path.exists(DbDir):
 256         os.makedirs(DbDir)
 257     #
 258     # A temporary INF is copied to database path which must have write permission
 259     # The temporary will be removed at the end of build
 260     # In case of name conflict, the file name is
 261     # FILE_GUIDBaseName (0D1B936F-68F3-4589-AFCC-FB8B7AEBC836module.inf)
 262     #
 263     TempFullPath = os.path.join(DbDir,
 264                                 Filename)
 265     RtPath = PathClass(Path.File, Workspace)
 266     #
 267     # Modify the full path to temporary path, keep other unchanged
 268     #
 269     # To build same module more than once, the module path with FILE_GUID overridden has
 270     # the file name FILE_GUIDmodule.inf, but the relative path (self.MetaFile.File) is the real path
 271     # in DSC which is used as relative path by C files and other files in INF.
 272     # A trick was used: all module paths are PathClass instances, after the initialization
 273     # of PathClass, the PathClass.Path is overridden by the temporary INF path.
 274     #
 275     # The reason for creating a temporary INF is:
 276     # Platform.Modules which is the base to create ModuleAutoGen objects is a dictionary,
 277     # the key is the full path of INF, the value is an object to save overridden library instances, PCDs.
 278     # A different key for the same module is needed to create different output directory,
 279     # retrieve overridden PCDs, library instances.
 280     #
 281     # The BaseName is the FILE_GUID which is also the output directory name.
 282     #
 283     #
 284     RtPath.Path = TempFullPath
 285     RtPath.BaseName = BaseName
 286     #
 287     # If file exists, compare contents
 288     #
 289     if os.path.exists(TempFullPath):
 290         with open(str(Path), 'rb') as f1, open(TempFullPath, 'rb') as f2:
 291             if f1.read() == f2.read():
 292                 return RtPath
 293     _TempInfs.append(TempFullPath)
 294     shutil.copy2(str(Path), TempFullPath)
 295     return RtPath
 296
 297 ## Remove temporary created INFs whose paths were saved in _TempInfs
 298 #
 299 def ClearDuplicatedInf():
 300     while _TempInfs:
 301         File = _TempInfs.pop()
 302         if os.path.exists(File):
 303             os.remove(File)
 304
 305 ## Convert GUID string in xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx style to C structure style
 306 #
 307 #   @param      Guid    The GUID string
 308 #
 309 #   @retval     string  The GUID string in C structure style
 310 #
 311 def GuidStringToGuidStructureString(Guid):
 312     GuidList = Guid.split('-')
 313     Result = '{'
 314     for Index in range(0, 3, 1):
 315         Result = Result + '0x' + GuidList[Index] + ', '
 316     Result = Result + '{0x' + GuidList[3][0:2] + ', 0x' + GuidList[3][2:4]
 317     for Index in range(0, 12, 2):
 318         Result = Result + ', 0x' + GuidList[4][Index:Index + 2]
 319     Result += '}}'
 320     return Result
 321
 322 ## Convert GUID structure in byte array to xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
 323 #
 324 #   @param      GuidValue   The GUID value in byte array
 325 #
 326 #   @retval     string      The GUID value in xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx format
 327 #
 328 def GuidStructureByteArrayToGuidString(GuidValue):
 329     guidValueString = GuidValue.lower().replace("{", "").replace("}", "").replace(" ", "").replace(";", "")
 330     guidValueList = guidValueString.split(",")
 331     if len(guidValueList) != 16:
 332         return ''
 333         #EdkLogger.error(None, None, "Invalid GUID value string %s" % GuidValue)
 334     try:
 335         return "%02x%02x%02x%02x-%02x%02x-%02x%02x-%02x%02x-%02x%02x%02x%02x%02x%02x" % (
 336                 int(guidValueList[3], 16),
 337                 int(guidValueList[2], 16),
 338                 int(guidValueList[1], 16),
 339                 int(guidValueList[0], 16),
 340                 int(guidValueList[5], 16),
 341                 int(guidValueList[4], 16),
 342                 int(guidValueList[7], 16),
 343                 int(guidValueList[6], 16),
 344                 int(guidValueList[8], 16),
 345                 int(guidValueList[9], 16),
 346                 int(guidValueList[10], 16),
 347                 int(guidValueList[11], 16),
 348                 int(guidValueList[12], 16),
 349                 int(guidValueList[13], 16),
 350                 int(guidValueList[14], 16),
 351                 int(guidValueList[15], 16)
 352                 )
 353     except:
 354         return ''
 355
 356 ## Convert GUID string in C structure style to xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
 357 #
 358 #   @param      GuidValue   The GUID value in C structure format
 359 #
 360 #   @retval     string      The GUID value in xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx format
 361 #
 362 def GuidStructureStringToGuidString(GuidValue):
 363     if not GlobalData.gGuidCFormatPattern.match(GuidValue):
 364         return ''
 365     guidValueString = GuidValue.lower().replace("{", "").replace("}", "").replace(" ", "").replace(";", "")
 366     guidValueList = guidValueString.split(",")
 367     if len(guidValueList) != 11:
 368         return ''
 369         #EdkLogger.error(None, None, "Invalid GUID value string %s" % GuidValue)
 370     try:
 371         return "%08x-%04x-%04x-%02x%02x-%02x%02x%02x%02x%02x%02x" % (
 372                 int(guidValueList[0], 16),
 373                 int(guidValueList[1], 16),
 374                 int(guidValueList[2], 16),
 375                 int(guidValueList[3], 16),
 376                 int(guidValueList[4], 16),
 377                 int(guidValueList[5], 16),
 378                 int(guidValueList[6], 16),
 379                 int(guidValueList[7], 16),
 380                 int(guidValueList[8], 16),
 381                 int(guidValueList[9], 16),
 382                 int(guidValueList[10], 16)
 383                 )
 384     except:
 385         return ''
 386
 387 ## Convert GUID string in C structure style to xxxxxxxx_xxxx_xxxx_xxxx_xxxxxxxxxxxx
 388 #
 389 #   @param      GuidValue   The GUID value in C structure format
 390 #
 391 #   @retval     string      The GUID value in xxxxxxxx_xxxx_xxxx_xxxx_xxxxxxxxxxxx format
 392 #
 393 def GuidStructureStringToGuidValueName(GuidValue):
 394     guidValueString = GuidValue.lower().replace("{", "").replace("}", "").replace(" ", "")
 395     guidValueList = guidValueString.split(",")
 396     if len(guidValueList) != 11:
 397         EdkLogger.error(None, FORMAT_INVALID, "Invalid GUID value string [%s]" % GuidValue)
 398     return "%08x_%04x_%04x_%02x%02x_%02x%02x%02x%02x%02x%02x" % (
 399             int(guidValueList[0], 16),
 400             int(guidValueList[1], 16),
 401             int(guidValueList[2], 16),
 402             int(guidValueList[3], 16),
 403             int(guidValueList[4], 16),
 404             int(guidValueList[5], 16),
 405             int(guidValueList[6], 16),
 406             int(guidValueList[7], 16),
 407             int(guidValueList[8], 16),
 408             int(guidValueList[9], 16),
 409             int(guidValueList[10], 16)
 410             )
 411
 412 ## Create directories
 413 #
 414 #   @param      Directory   The directory name
 415 #
 416 def CreateDirectory(Directory):
 417     if Directory is None or Directory.strip() == "":
 418         return True
 419     try:
 420         if not os.access(Directory, os.F_OK):
 421             os.makedirs(Directory)
 422     except:
 423         return False
 424     return True
 425
 426 ## Remove directories, including files and sub-directories in it
 427 #
 428 #   @param      Directory   The directory name
 429 #
 430 def RemoveDirectory(Directory, Recursively=False):
 431     if Directory is None or Directory.strip() == "" or not os.path.exists(Directory):
 432         return
 433     if Recursively:
 434         CurrentDirectory = os.getcwd()
 435         os.chdir(Directory)
 436         for File in os.listdir("."):
 437             if os.path.isdir(File):
 438                 RemoveDirectory(File, Recursively)
 439             else:
 440                 os.remove(File)
 441         os.chdir(CurrentDirectory)
 442     os.rmdir(Directory)
 443
 444 ## Store content in file
 445 #
 446 #  This method is used to save file only when its content is changed. This is
 447 #  quite useful for "make" system to decide what will be re-built and what won't.
 448 #
 449 #   @param      File            The path of file
 450 #   @param      Content         The new content of the file
 451 #   @param      IsBinaryFile    The flag indicating if the file is binary file or not
 452 #
 453 #   @retval     True            If the file content is changed and the file is renewed
 454 #   @retval     False           If the file content is the same
 455 #
 456 def SaveFileOnChange(File, Content, IsBinaryFile=True):
 457     if not IsBinaryFile:
 458         Content = Content.replace("\n", os.linesep)
 459
 460     if os.path.exists(File):
 461         try:
 462             if Content == open(File, "rb").read():
 463                 return False
 464         except:
 465             EdkLogger.error(None, FILE_OPEN_FAILURE, ExtraData=File)
 466
 467     DirName = os.path.dirname(File)
 468     if not CreateDirectory(DirName):
 469         EdkLogger.error(None, FILE_CREATE_FAILURE, "Could not create directory %s" % DirName)
 470     else:
 471         if DirName == '':
 472             DirName = os.getcwd()
 473         if not os.access(DirName, os.W_OK):
 474             EdkLogger.error(None, PERMISSION_FAILURE, "Do not have write permission on directory %s" % DirName)
 475
 476     try:
 477         if GlobalData.gIsWindows:
 478             try:
 479                 from .PyUtility import SaveFileToDisk
 480                 if not SaveFileToDisk(File, Content):
 481                     EdkLogger.error(None, FILE_CREATE_FAILURE, ExtraData=File)
 482             except:
 483                 Fd = open(File, "wb")
 484                 Fd.write(Content)
 485                 Fd.close()
 486         else:
 487             Fd = open(File, "wb")
 488             Fd.write(Content)
 489             Fd.close()
 490     except IOError as X:
 491         EdkLogger.error(None, FILE_CREATE_FAILURE, ExtraData='IOError %s' % X)
 492
 493     return True
 494
 495 ## Make a Python object persistent on file system
 496 #
 497 #   @param      Data    The object to be stored in file
 498 #   @param      File    The path of file to store the object
 499 #
 500 def DataDump(Data, File):
 501     Fd = None
 502     try:
 503         Fd = open(File, 'wb')
 504         pickle.dump(Data, Fd, pickle.HIGHEST_PROTOCOL)
 505     except:
 506         EdkLogger.error("", FILE_OPEN_FAILURE, ExtraData=File, RaiseError=False)
 507     finally:
 508         if Fd is not None:
 509             Fd.close()
 510
 511 ## Restore a Python object from a file
 512 #
 513 #   @param      File    The path of file stored the object
 514 #
 515 #   @retval     object  A python object
 516 #   @retval     None    If failure in file operation
 517 #
 518 def DataRestore(File):
 519     Data = None
 520     Fd = None
 521     try:
 522         Fd = open(File, 'rb')
 523         Data = pickle.load(Fd)
 524     except Exception as e:
 525         EdkLogger.verbose("Failed to load [%s]\n\t%s" % (File, str(e)))
 526         Data = None
 527     finally:
 528         if Fd is not None:
 529             Fd.close()
 530     return Data
 531
 532 ## Retrieve and cache the real path name in file system
 533 #
 534 #   @param      Root    The root directory of path relative to
 535 #
 536 #   @retval     str     The path string if the path exists
 537 #   @retval     None    If path doesn't exist
 538 #
 539 class DirCache:
 540     _CACHE_ = set()
 541     _UPPER_CACHE_ = {}
 542
 543     def __init__(self, Root):
 544         self._Root = Root
 545         for F in os.listdir(Root):
 546             self._CACHE_.add(F)
 547             self._UPPER_CACHE_[F.upper()] = F
 548
 549     # =[] operator
 550     def __getitem__(self, Path):
 551         Path = Path[len(os.path.commonprefix([Path, self._Root])):]
 552         if not Path:
 553             return self._Root
 554         if Path and Path[0] == os.path.sep:
 555             Path = Path[1:]
 556         if Path in self._CACHE_:
 557             return os.path.join(self._Root, Path)
 558         UpperPath = Path.upper()
 559         if UpperPath in self._UPPER_CACHE_:
 560             return os.path.join(self._Root, self._UPPER_CACHE_[UpperPath])
 561
 562         IndexList = []
 563         LastSepIndex = -1
 564         SepIndex = Path.find(os.path.sep)
 565         while SepIndex > -1:
 566             Parent = UpperPath[:SepIndex]
 567             if Parent not in self._UPPER_CACHE_:
 568                 break
 569             LastSepIndex = SepIndex
 570             SepIndex = Path.find(os.path.sep, LastSepIndex + 1)
 571
 572         if LastSepIndex == -1:
 573             return None
 574
 575         Cwd = os.getcwd()
 576         os.chdir(self._Root)
 577         SepIndex = LastSepIndex
 578         while SepIndex > -1:
 579             Parent = Path[:SepIndex]
 580             ParentKey = UpperPath[:SepIndex]
 581             if ParentKey not in self._UPPER_CACHE_:
 582                 os.chdir(Cwd)
 583                 return None
 584
 585             if Parent in self._CACHE_:
 586                 ParentDir = Parent
 587             else:
 588                 ParentDir = self._UPPER_CACHE_[ParentKey]
 589             for F in os.listdir(ParentDir):
 590                 Dir = os.path.join(ParentDir, F)
 591                 self._CACHE_.add(Dir)
 592                 self._UPPER_CACHE_[Dir.upper()] = Dir
 593
 594             SepIndex = Path.find(os.path.sep, SepIndex + 1)
 595
 596         os.chdir(Cwd)
 597         if Path in self._CACHE_:
 598             return os.path.join(self._Root, Path)
 599         elif UpperPath in self._UPPER_CACHE_:
 600             return os.path.join(self._Root, self._UPPER_CACHE_[UpperPath])
 601         return None
 602
 603 def RealPath(File, Dir='', OverrideDir=''):
 604     NewFile = os.path.normpath(os.path.join(Dir, File))
 605     NewFile = GlobalData.gAllFiles[NewFile]
 606     if not NewFile and OverrideDir:
 607         NewFile = os.path.normpath(os.path.join(OverrideDir, File))
 608         NewFile = GlobalData.gAllFiles[NewFile]
 609     return NewFile
 610
 611 def RealPath2(File, Dir='', OverrideDir=''):
 612     NewFile = None
 613     if OverrideDir:
 614         NewFile = GlobalData.gAllFiles[os.path.normpath(os.path.join(OverrideDir, File))]
 615         if NewFile:
 616             if OverrideDir[-1] == os.path.sep:
 617                 return NewFile[len(OverrideDir):], NewFile[0:len(OverrideDir)]
 618             else:
 619                 return NewFile[len(OverrideDir) + 1:], NewFile[0:len(OverrideDir)]
 620     if GlobalData.gAllFiles:
 621         NewFile = GlobalData.gAllFiles[os.path.normpath(os.path.join(Dir, File))]
 622     if not NewFile:
 623         NewFile = os.path.normpath(os.path.join(Dir, File))
 624         if not os.path.exists(NewFile):
 625             return None, None
 626     if NewFile:
 627         if Dir:
 628             if Dir[-1] == os.path.sep:
 629                 return NewFile[len(Dir):], NewFile[0:len(Dir)]
 630             else:
 631                 return NewFile[len(Dir) + 1:], NewFile[0:len(Dir)]
 632         else:
 633             return NewFile, ''
 634
 635     return None, None
 636
 637 ## Get GUID value from given packages
 638 #
 639 #   @param      CName           The CName of the GUID
 640 #   @param      PackageList     List of packages looking-up in
 641 #   @param      Inffile         The driver file
 642 #
 643 #   @retval     GuidValue   if the CName is found in any given package
 644 #   @retval     None        if the CName is not found in all given packages
 645 #
 646 def GuidValue(CName, PackageList, Inffile = None):
 647     for P in PackageList:
 648         GuidKeys = P.Guids.keys()
 649         if Inffile and P._PrivateGuids:
 650             if not Inffile.startswith(P.MetaFile.Dir):
 651                 GuidKeys = [x for x in P.Guids if x not in P._PrivateGuids]
 652         if CName in GuidKeys:
 653             return P.Guids[CName]
 654     return None
 655
 656 ## Get Protocol value from given packages
 657 #
 658 #   @param      CName           The CName of the GUID
 659 #   @param      PackageList     List of packages looking-up in
 660 #   @param      Inffile         The driver file
 661 #
 662 #   @retval     GuidValue   if the CName is found in any given package
 663 #   @retval     None        if the CName is not found in all given packages
 664 #
 665 def ProtocolValue(CName, PackageList, Inffile = None):
 666     for P in PackageList:
 667         ProtocolKeys = P.Protocols.keys()
 668         if Inffile and P._PrivateProtocols:
 669             if not Inffile.startswith(P.MetaFile.Dir):
 670                 ProtocolKeys = [x for x in P.Protocols if x not in P._PrivateProtocols]
 671         if CName in ProtocolKeys:
 672             return P.Protocols[CName]
 673     return None
 674
 675 ## Get PPI value from given packages
 676 #
 677 #   @param      CName           The CName of the GUID
 678 #   @param      PackageList     List of packages looking-up in
 679 #   @param      Inffile         The driver file
 680 #
 681 #   @retval     GuidValue   if the CName is found in any given package
 682 #   @retval     None        if the CName is not found in all given packages
 683 #
 684 def PpiValue(CName, PackageList, Inffile = None):
 685     for P in PackageList:
 686         PpiKeys = P.Ppis.keys()
 687         if Inffile and P._PrivatePpis:
 688             if not Inffile.startswith(P.MetaFile.Dir):
 689                 PpiKeys = [x for x in P.Ppis if x not in P._PrivatePpis]
 690         if CName in PpiKeys:
 691             return P.Ppis[CName]
 692     return None
 693
 694 ## A string template class
 695 #
 696 #  This class implements a template for string replacement. A string template
 697 #  looks like following
 698 #
 699 #       ${BEGIN} other_string ${placeholder_name} other_string ${END}
 700 #
 701 #  The string between ${BEGIN} and ${END} will be repeated as many times as the
 702 #  length of "placeholder_name", which is a list passed through a dict. The
 703 #  "placeholder_name" is the key name of the dict. The ${BEGIN} and ${END} can
 704 #  be not used and, in this case, the "placeholder_name" must not a list and it
 705 #  will just be replaced once.
 706 #
 707 class TemplateString(object):
 708     _REPEAT_START_FLAG = "BEGIN"
 709     _REPEAT_END_FLAG = "END"
 710
 711     class Section(object):
 712         _LIST_TYPES = [type([]), type(set()), type((0,))]
 713
 714         def __init__(self, TemplateSection, PlaceHolderList):
 715             self._Template = TemplateSection
 716             self._PlaceHolderList = []
 717
 718             # Split the section into sub-sections according to the position of placeholders
 719             if PlaceHolderList:
 720                 self._SubSectionList = []
 721                 SubSectionStart = 0
 722                 #
 723                 # The placeholders passed in must be in the format of
 724                 #
 725                 #   PlaceHolderName, PlaceHolderStartPoint, PlaceHolderEndPoint
 726                 #
 727                 for PlaceHolder, Start, End in PlaceHolderList:
 728                     self._SubSectionList.append(TemplateSection[SubSectionStart:Start])
 729                     self._SubSectionList.append(TemplateSection[Start:End])
 730                     self._PlaceHolderList.append(PlaceHolder)
 731                     SubSectionStart = End
 732                 if SubSectionStart < len(TemplateSection):
 733                     self._SubSectionList.append(TemplateSection[SubSectionStart:])
 734             else:
 735                 self._SubSectionList = [TemplateSection]
 736
 737         def __str__(self):
 738             return self._Template + " : " + str(self._PlaceHolderList)
 739
 740         def Instantiate(self, PlaceHolderValues):
 741             RepeatTime = -1
 742             RepeatPlaceHolders = {}
 743             NonRepeatPlaceHolders = {}
 744
 745             for PlaceHolder in self._PlaceHolderList:
 746                 if PlaceHolder not in PlaceHolderValues:
 747                     continue
 748                 Value = PlaceHolderValues[PlaceHolder]
 749                 if type(Value) in self._LIST_TYPES:
 750                     if RepeatTime < 0:
 751                         RepeatTime = len(Value)
 752                     elif RepeatTime != len(Value):
 753                         EdkLogger.error(
 754                                     "TemplateString",
 755                                     PARAMETER_INVALID,
 756                                     "${%s} has different repeat time from others!" % PlaceHolder,
 757                                     ExtraData=str(self._Template)
 758                                     )
 759                     RepeatPlaceHolders["${%s}" % PlaceHolder] = Value
 760                 else:
 761                     NonRepeatPlaceHolders["${%s}" % PlaceHolder] = Value
 762
 763             if NonRepeatPlaceHolders:
 764                 StringList = []
 765                 for S in self._SubSectionList:
 766                     if S not in NonRepeatPlaceHolders:
 767                         StringList.append(S)
 768                     else:
 769                         StringList.append(str(NonRepeatPlaceHolders[S]))
 770             else:
 771                 StringList = self._SubSectionList
 772
 773             if RepeatPlaceHolders:
 774                 TempStringList = []
 775                 for Index in range(RepeatTime):
 776                     for S in StringList:
 777                         if S not in RepeatPlaceHolders:
 778                             TempStringList.append(S)
 779                         else:
 780                             TempStringList.append(str(RepeatPlaceHolders[S][Index]))
 781                 StringList = TempStringList
 782
 783             return "".join(StringList)
 784
 785     ## Constructor
 786     def __init__(self, Template=None):
 787         self.String = ''
 788         self.IsBinary = False
 789         self._Template = Template
 790         self._TemplateSectionList = self._Parse(Template)
 791
 792     ## str() operator
 793     #
 794     #   @retval     string  The string replaced
 795     #
 796     def __str__(self):
 797         return self.String
 798
 799     ## Split the template string into fragments per the ${BEGIN} and ${END} flags
 800     #
 801     #   @retval     list    A list of TemplateString.Section objects
 802     #
 803     def _Parse(self, Template):
 804         SectionStart = 0
 805         SearchFrom = 0
 806         MatchEnd = 0
 807         PlaceHolderList = []
 808         TemplateSectionList = []
 809         while Template:
 810             MatchObj = gPlaceholderPattern.search(Template, SearchFrom)
 811             if not MatchObj:
 812                 if MatchEnd <= len(Template):
 813                     TemplateSection = TemplateString.Section(Template[SectionStart:], PlaceHolderList)
 814                     TemplateSectionList.append(TemplateSection)
 815                 break
 816
 817             MatchString = MatchObj.group(1)
 818             MatchStart = MatchObj.start()
 819             MatchEnd = MatchObj.end()
 820
 821             if MatchString == self._REPEAT_START_FLAG:
 822                 if MatchStart > SectionStart:
 823                     TemplateSection = TemplateString.Section(Template[SectionStart:MatchStart], PlaceHolderList)
 824                     TemplateSectionList.append(TemplateSection)
 825                 SectionStart = MatchEnd
 826                 PlaceHolderList = []
 827             elif MatchString == self._REPEAT_END_FLAG:
 828                 TemplateSection = TemplateString.Section(Template[SectionStart:MatchStart], PlaceHolderList)
 829                 TemplateSectionList.append(TemplateSection)
 830                 SectionStart = MatchEnd
 831                 PlaceHolderList = []
 832             else:
 833                 PlaceHolderList.append((MatchString, MatchStart - SectionStart, MatchEnd - SectionStart))
 834             SearchFrom = MatchEnd
 835         return TemplateSectionList
 836
 837     ## Replace the string template with dictionary of placeholders and append it to previous one
 838     #
 839     #   @param      AppendString    The string template to append
 840     #   @param      Dictionary      The placeholder dictionaries
 841     #
 842     def Append(self, AppendString, Dictionary=None):
 843         if Dictionary:
 844             SectionList = self._Parse(AppendString)
 845             self.String += "".join(S.Instantiate(Dictionary) for S in SectionList)
 846         else:
 847             self.String += AppendString
 848
 849     ## Replace the string template with dictionary of placeholders
 850     #
 851     #   @param      Dictionary      The placeholder dictionaries
 852     #
 853     #   @retval     str             The string replaced with placeholder values
 854     #
 855     def Replace(self, Dictionary=None):
 856         return "".join(S.Instantiate(Dictionary) for S in self._TemplateSectionList)
 857
 858 ## Progress indicator class
 859 #
 860 #  This class makes use of thread to print progress on console.
 861 #
 862 class Progressor:
 863     # for avoiding deadloop
 864     _StopFlag = None
 865     _ProgressThread = None
 866     _CheckInterval = 0.25
 867
 868     ## Constructor
 869     #
 870     #   @param      OpenMessage     The string printed before progress charaters
 871     #   @param      CloseMessage    The string printed after progress charaters
 872     #   @param      ProgressChar    The charater used to indicate the progress
 873     #   @param      Interval        The interval in seconds between two progress charaters
 874     #
 875     def __init__(self, OpenMessage="", CloseMessage="", ProgressChar='.', Interval=1.0):
 876         self.PromptMessage = OpenMessage
 877         self.CodaMessage = CloseMessage
 878         self.ProgressChar = ProgressChar
 879         self.Interval = Interval
 880         if Progressor._StopFlag is None:
 881             Progressor._StopFlag = threading.Event()
 882
 883     ## Start to print progress charater
 884     #
 885     #   @param      OpenMessage     The string printed before progress charaters
 886     #
 887     def Start(self, OpenMessage=None):
 888         if OpenMessage is not None:
 889             self.PromptMessage = OpenMessage
 890         Progressor._StopFlag.clear()
 891         if Progressor._ProgressThread is None:
 892             Progressor._ProgressThread = threading.Thread(target=self._ProgressThreadEntry)
 893             Progressor._ProgressThread.setDaemon(False)
 894             Progressor._ProgressThread.start()
 895
 896     ## Stop printing progress charater
 897     #
 898     #   @param      CloseMessage    The string printed after progress charaters
 899     #
 900     def Stop(self, CloseMessage=None):
 901         OriginalCodaMessage = self.CodaMessage
 902         if CloseMessage is not None:
 903             self.CodaMessage = CloseMessage
 904         self.Abort()
 905         self.CodaMessage = OriginalCodaMessage
 906
 907     ## Thread entry method
 908     def _ProgressThreadEntry(self):
 909         sys.stdout.write(self.PromptMessage + " ")
 910         sys.stdout.flush()
 911         TimeUp = 0.0
 912         while not Progressor._StopFlag.isSet():
 913             if TimeUp <= 0.0:
 914                 sys.stdout.write(self.ProgressChar)
 915                 sys.stdout.flush()
 916                 TimeUp = self.Interval
 917             time.sleep(self._CheckInterval)
 918             TimeUp -= self._CheckInterval
 919         sys.stdout.write(" " + self.CodaMessage + "\n")
 920         sys.stdout.flush()
 921
 922     ## Abort the progress display
 923     @staticmethod
 924     def Abort():
 925         if Progressor._StopFlag is not None:
 926             Progressor._StopFlag.set()
 927         if Progressor._ProgressThread is not None:
 928             Progressor._ProgressThread.join()
 929             Progressor._ProgressThread = None
 930
 931 ## A dict which can access its keys and/or values orderly
 932 #
 933 #  The class implements a new kind of dict which its keys or values can be
 934 #  accessed in the order they are added into the dict. It guarantees the order
 935 #  by making use of an internal list to keep a copy of keys.
 936 #
 937 class sdict(IterableUserDict):
 938     ## Constructor
 939     def __init__(self):
 940         IterableUserDict.__init__(self)
 941         self._key_list = []
 942
 943     ## [] operator
 944     def __setitem__(self, key, value):
 945         if key not in self._key_list:
 946             self._key_list.append(key)
 947         IterableUserDict.__setitem__(self, key, value)
 948
 949     ## del operator
 950     def __delitem__(self, key):
 951         self._key_list.remove(key)
 952         IterableUserDict.__delitem__(self, key)
 953
 954     ## used in "for k in dict" loop to ensure the correct order
 955     def __iter__(self):
 956         return self.iterkeys()
 957
 958     ## len() support
 959     def __len__(self):
 960         return len(self._key_list)
 961
 962     ## "in" test support
 963     def __contains__(self, key):
 964         return key in self._key_list
 965
 966     ## indexof support
 967     def index(self, key):
 968         return self._key_list.index(key)
 969
 970     ## insert support
 971     def insert(self, key, newkey, newvalue, order):
 972         index = self._key_list.index(key)
 973         if order == 'BEFORE':
 974             self._key_list.insert(index, newkey)
 975             IterableUserDict.__setitem__(self, newkey, newvalue)
 976         elif order == 'AFTER':
 977             self._key_list.insert(index + 1, newkey)
 978             IterableUserDict.__setitem__(self, newkey, newvalue)
 979
 980     ## append support
 981     def append(self, sdict):
 982         for key in sdict:
 983             if key not in self._key_list:
 984                 self._key_list.append(key)
 985             IterableUserDict.__setitem__(self, key, sdict[key])
 986
 987     def has_key(self, key):
 988         return key in self._key_list
 989
 990     ## Empty the dict
 991     def clear(self):
 992         self._key_list = []
 993         IterableUserDict.clear(self)
 994
 995     ## Return a copy of keys
 996     def keys(self):
 997         keys = []
 998         for key in self._key_list:
 999             keys.append(key)
1000         return keys
1001
1002     ## Return a copy of values
1003     def values(self):
1004         values = []
1005         for key in self._key_list:
1006             values.append(self[key])
1007         return values
1008
1009     ## Return a copy of (key, value) list
1010     def items(self):
1011         items = []
1012         for key in self._key_list:
1013             items.append((key, self[key]))
1014         return items
1015
1016     ## Iteration support
1017     def iteritems(self):
1018         return iter(self.items())
1019
1020     ## Keys interation support
1021     def iterkeys(self):
1022         return iter(self.keys())
1023
1024     ## Values interation support
1025     def itervalues(self):
1026         return iter(self.values())
1027
1028     ## Return value related to a key, and remove the (key, value) from the dict
1029     def pop(self, key, *dv):
1030         value = None
1031         if key in self._key_list:
1032             value = self[key]
1033             self.__delitem__(key)
1034         elif len(dv) != 0 :
1035             value = kv[0]
1036         return value
1037
1038     ## Return (key, value) pair, and remove the (key, value) from the dict
1039     def popitem(self):
1040         key = self._key_list[-1]
1041         value = self[key]
1042         self.__delitem__(key)
1043         return key, value
1044
1045     def update(self, dict=None, **kwargs):
1046         if dict is not None:
1047             for k, v in dict.items():
1048                 self[k] = v
1049         if len(kwargs):
1050             for k, v in kwargs.items():
1051                 self[k] = v
1052
1053 ## Dictionary with restricted keys
1054 #
1055 class rdict(dict):
1056     ## Constructor
1057     def __init__(self, KeyList):
1058         for Key in KeyList:
1059             dict.__setitem__(self, Key, "")
1060
1061     ## []= operator
1062     def __setitem__(self, key, value):
1063         if key not in self:
1064             EdkLogger.error("RestrictedDict", ATTRIBUTE_SET_FAILURE, "Key [%s] is not allowed" % key,
1065                             ExtraData=", ".join(dict.keys(self)))
1066         dict.__setitem__(self, key, value)
1067
1068     ## =[] operator
1069     def __getitem__(self, key):
1070         if key not in self:
1071             return ""
1072         return dict.__getitem__(self, key)
1073
1074     ## del operator
1075     def __delitem__(self, key):
1076         EdkLogger.error("RestrictedDict", ATTRIBUTE_ACCESS_DENIED, ExtraData="del")
1077
1078     ## Empty the dict
1079     def clear(self):
1080         for Key in self:
1081             self.__setitem__(Key, "")
1082
1083     ## Return value related to a key, and remove the (key, value) from the dict
1084     def pop(self, key, *dv):
1085         EdkLogger.error("RestrictedDict", ATTRIBUTE_ACCESS_DENIED, ExtraData="pop")
1086
1087     ## Return (key, value) pair, and remove the (key, value) from the dict
1088     def popitem(self):
1089         EdkLogger.error("RestrictedDict", ATTRIBUTE_ACCESS_DENIED, ExtraData="popitem")
1090
1091 ## Dictionary using prioritized list as key
1092 #
1093 class tdict:
1094     _ListType = type([])
1095     _TupleType = type(())
1096     _Wildcard = 'COMMON'
1097     _ValidWildcardList = ['COMMON', 'DEFAULT', 'ALL', '*', 'PLATFORM']
1098
1099     def __init__(self, _Single_=False, _Level_=2):
1100         self._Level_ = _Level_
1101         self.data = {}
1102         self._Single_ = _Single_
1103
1104     # =[] operator
1105     def __getitem__(self, key):
1106         KeyType = type(key)
1107         RestKeys = None
1108         if KeyType == self._ListType or KeyType == self._TupleType:
1109             FirstKey = key[0]
1110             if len(key) > 1:
1111                 RestKeys = key[1:]
1112             elif self._Level_ > 1:
1113                 RestKeys = [self._Wildcard for i in range(0, self._Level_ - 1)]
1114         else:
1115             FirstKey = key
1116             if self._Level_ > 1:
1117                 RestKeys = [self._Wildcard for i in range(0, self._Level_ - 1)]
1118
1119         if FirstKey is None or str(FirstKey).upper() in self._ValidWildcardList:
1120             FirstKey = self._Wildcard
1121
1122         if self._Single_:
1123             return self._GetSingleValue(FirstKey, RestKeys)
1124         else:
1125             return self._GetAllValues(FirstKey, RestKeys)
1126
1127     def _GetSingleValue(self, FirstKey, RestKeys):
1128         Value = None
1129         #print "%s-%s" % (FirstKey, self._Level_) ,
1130         if self._Level_ > 1:
1131             if FirstKey == self._Wildcard:
1132                 if FirstKey in self.data:
1133                     Value = self.data[FirstKey][RestKeys]
1134                 if Value is None:
1135                     for Key in self.data:
1136                         Value = self.data[Key][RestKeys]
1137                         if Value is not None: break
1138             else:
1139                 if FirstKey in self.data:
1140                     Value = self.data[FirstKey][RestKeys]
1141                 if Value is None and self._Wildcard in self.data:
1142                     #print "Value=None"
1143                     Value = self.data[self._Wildcard][RestKeys]
1144         else:
1145             if FirstKey == self._Wildcard:
1146                 if FirstKey in self.data:
1147                     Value = self.data[FirstKey]
1148                 if Value is None:
1149                     for Key in self.data:
1150                         Value = self.data[Key]
1151                         if Value is not None: break
1152             else:
1153                 if FirstKey in self.data:
1154                     Value = self.data[FirstKey]
1155                 elif self._Wildcard in self.data:
1156                     Value = self.data[self._Wildcard]
1157         return Value
1158
1159     def _GetAllValues(self, FirstKey, RestKeys):
1160         Value = []
1161         if self._Level_ > 1:
1162             if FirstKey == self._Wildcard:
1163                 for Key in self.data:
1164                     Value += self.data[Key][RestKeys]
1165             else:
1166                 if FirstKey in self.data:
1167                     Value += self.data[FirstKey][RestKeys]
1168                 if self._Wildcard in self.data:
1169                     Value += self.data[self._Wildcard][RestKeys]
1170         else:
1171             if FirstKey == self._Wildcard:
1172                 for Key in self.data:
1173                     Value.append(self.data[Key])
1174             else:
1175                 if FirstKey in self.data:
1176                     Value.append(self.data[FirstKey])
1177                 if self._Wildcard in self.data:
1178                     Value.append(self.data[self._Wildcard])
1179         return Value
1180
1181     ## []= operator
1182     def __setitem__(self, key, value):
1183         KeyType = type(key)
1184         RestKeys = None
1185         if KeyType == self._ListType or KeyType == self._TupleType:
1186             FirstKey = key[0]
1187             if len(key) > 1:
1188                 RestKeys = key[1:]
1189             else:
1190                 RestKeys = [self._Wildcard for i in range(0, self._Level_ - 1)]
1191         else:
1192             FirstKey = key
1193             if self._Level_ > 1:
1194                 RestKeys = [self._Wildcard for i in range(0, self._Level_ - 1)]
1195
1196         if FirstKey in self._ValidWildcardList:
1197             FirstKey = self._Wildcard
1198
1199         if FirstKey not in self.data and self._Level_ > 0:
1200             self.data[FirstKey] = tdict(self._Single_, self._Level_ - 1)
1201
1202         if self._Level_ > 1:
1203             self.data[FirstKey][RestKeys] = value
1204         else:
1205             self.data[FirstKey] = value
1206
1207     def SetGreedyMode(self):
1208         self._Single_ = False
1209         if self._Level_ > 1:
1210             for Key in self.data:
1211                 self.data[Key].SetGreedyMode()
1212
1213     def SetSingleMode(self):
1214         self._Single_ = True
1215         if self._Level_ > 1:
1216             for Key in self.data:
1217                 self.data[Key].SetSingleMode()
1218
1219     def GetKeys(self, KeyIndex=0):
1220         assert KeyIndex >= 0
1221         if KeyIndex == 0:
1222             return set(self.data.keys())
1223         else:
1224             keys = set()
1225             for Key in self.data:
1226                 keys |= self.data[Key].GetKeys(KeyIndex - 1)
1227             return keys
1228
1229 def IsFieldValueAnArray (Value):
1230     Value = Value.strip()
1231     if Value.startswith(TAB_GUID) and Value.endswith(')'):
1232         return True
1233     if Value.startswith('L"') and Value.endswith('"')  and len(list(Value[2:-1])) > 1:
1234         return True
1235     if Value[0] == '"' and Value[-1] == '"' and len(list(Value[1:-1])) > 1:
1236         return True
1237     if Value[0] == '{' and Value[-1] == '}':
1238         return True
1239     if Value.startswith("L'") and Value.endswith("'") and len(list(Value[2:-1])) > 1:
1240         return True
1241     if Value[0] == "'" and Value[-1] == "'" and len(list(Value[1:-1])) > 1:
1242         return True
1243     return False
1244
1245 def AnalyzePcdExpression(Setting):
1246     Setting = Setting.strip()
1247     # There might be escaped quote in a string: \", \\\" , \', \\\'
1248     Data = Setting
1249     # There might be '|' in string and in ( ... | ... ), replace it with '-'
1250     NewStr = ''
1251     InSingleQuoteStr = False
1252     InDoubleQuoteStr = False
1253     Pair = 0
1254     for Index, ch in enumerate(Data):
1255         if ch == '"' and not InSingleQuoteStr:
1256             if Data[Index - 1] != '\\':
1257                 InDoubleQuoteStr = not InDoubleQuoteStr
1258         elif ch == "'" and not InDoubleQuoteStr:
1259             if Data[Index - 1] != '\\':
1260                 InSingleQuoteStr = not InSingleQuoteStr
1261         elif ch == '(' and not (InSingleQuoteStr or InDoubleQuoteStr):
1262             Pair += 1
1263         elif ch == ')' and not (InSingleQuoteStr or InDoubleQuoteStr):
1264             Pair -= 1
1265
1266         if (Pair > 0 or InSingleQuoteStr or InDoubleQuoteStr) and ch == TAB_VALUE_SPLIT:
1267             NewStr += '-'
1268         else:
1269             NewStr += ch
1270     FieldList = []
1271     StartPos = 0
1272     while True:
1273         Pos = NewStr.find(TAB_VALUE_SPLIT, StartPos)
1274         if Pos < 0:
1275             FieldList.append(Setting[StartPos:].strip())
1276             break
1277         FieldList.append(Setting[StartPos:Pos].strip())
1278         StartPos = Pos + 1
1279
1280     return FieldList
1281
1282 def ParseDevPathValue (Value):
1283     if '\\' in Value:
1284         Value.replace('\\', '/').replace(' ', '')
1285
1286     Cmd = 'DevicePath ' + '"' + Value + '"'
1287     try:
1288         p = subprocess.Popen(Cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
1289         out, err = p.communicate()
1290     except Exception as X:
1291         raise BadExpression("DevicePath: %s" % (str(X)) )
1292     finally:
1293         subprocess._cleanup()
1294         p.stdout.close()
1295         p.stderr.close()
1296     if err:
1297         raise BadExpression("DevicePath: %s" % str(err))
1298     Size = len(out.split())
1299     out = ','.join(out.split())
1300     return '{' + out + '}', Size
1301
1302 def ParseFieldValue (Value):
1303     if isinstance(Value, type(0)):
1304         return Value, (Value.bit_length() + 7) / 8
1305     if not isinstance(Value, type('')):
1306         raise BadExpression('Type %s is %s' %(Value, type(Value)))
1307     Value = Value.strip()
1308     if Value.startswith(TAB_UINT8) and Value.endswith(')'):
1309         Value, Size = ParseFieldValue(Value.split('(', 1)[1][:-1])
1310         if Size > 1:
1311             raise BadExpression('Value (%s) Size larger than %d' %(Value, Size))
1312         return Value, 1
1313     if Value.startswith(TAB_UINT16) and Value.endswith(')'):
1314         Value, Size = ParseFieldValue(Value.split('(', 1)[1][:-1])
1315         if Size > 2:
1316             raise BadExpression('Value (%s) Size larger than %d' %(Value, Size))
1317         return Value, 2
1318     if Value.startswith(TAB_UINT32) and Value.endswith(')'):
1319         Value, Size = ParseFieldValue(Value.split('(', 1)[1][:-1])
1320         if Size > 4:
1321             raise BadExpression('Value (%s) Size larger than %d' %(Value, Size))
1322         return Value, 4
1323     if Value.startswith(TAB_UINT64) and Value.endswith(')'):
1324         Value, Size = ParseFieldValue(Value.split('(', 1)[1][:-1])
1325         if Size > 8:
1326             raise BadExpression('Value (%s) Size larger than %d' % (Value, Size))
1327         return Value, 8
1328     if Value.startswith(TAB_GUID) and Value.endswith(')'):
1329         Value = Value.split('(', 1)[1][:-1].strip()
1330         if Value[0] == '{' and Value[-1] == '}':
1331             TmpValue = GuidStructureStringToGuidString(Value)
1332             if not TmpValue:
1333                 raise BadExpression("Invalid GUID value string %s" % Value)
1334             Value = TmpValue
1335         if Value[0] == '"' and Value[-1] == '"':
1336             Value = Value[1:-1]
1337         try:
1338             Value = "'" + uuid.UUID(Value).get_bytes_le() + "'"
1339         except ValueError as Message:
1340             raise BadExpression(Message)
1341         Value, Size = ParseFieldValue(Value)
1342         return Value, 16
1343     if Value.startswith('L"') and Value.endswith('"'):
1344         # Unicode String
1345         # translate escape character
1346         Value = Value[1:]
1347         try:
1348             Value = eval(Value)
1349         except:
1350             Value = Value[1:-1]
1351         List = list(Value)
1352         List.reverse()
1353         Value = 0
1354         for Char in List:
1355             Value = (Value << 16) | ord(Char)
1356         return Value, (len(List) + 1) * 2
1357     if Value.startswith('"') and Value.endswith('"'):
1358         # ASCII String
1359         # translate escape character
1360         try:
1361             Value = eval(Value)
1362         except:
1363             Value = Value[1:-1]
1364         List = list(Value)
1365         List.reverse()
1366         Value = 0
1367         for Char in List:
1368             Value = (Value << 8) | ord(Char)
1369         return Value, len(List) + 1
1370     if Value.startswith("L'") and Value.endswith("'"):
1371         # Unicode Character Constant
1372         # translate escape character
1373         Value = Value[1:]
1374         try:
1375             Value = eval(Value)
1376         except:
1377             Value = Value[1:-1]
1378         List = list(Value)
1379         if len(List) == 0:
1380             raise BadExpression('Length %s is %s' % (Value, len(List)))
1381         List.reverse()
1382         Value = 0
1383         for Char in List:
1384             Value = (Value << 16) | ord(Char)
1385         return Value, len(List) * 2
1386     if Value.startswith("'") and Value.endswith("'"):
1387         # Character constant
1388         # translate escape character
1389         try:
1390             Value = eval(Value)
1391         except:
1392             Value = Value[1:-1]
1393         List = list(Value)
1394         if len(List) == 0:
1395             raise BadExpression('Length %s is %s' % (Value, len(List)))
1396         List.reverse()
1397         Value = 0
1398         for Char in List:
1399             Value = (Value << 8) | ord(Char)
1400         return Value, len(List)
1401     if Value.startswith('{') and Value.endswith('}'):
1402         # Byte array
1403         Value = Value[1:-1]
1404         List = [Item.strip() for Item in Value.split(',')]
1405         List.reverse()
1406         Value = 0
1407         RetSize = 0
1408         for Item in List:
1409             ItemValue, Size = ParseFieldValue(Item)
1410             RetSize += Size
1411             for I in range(Size):
1412                 Value = (Value << 8) | ((ItemValue >> 8 * I) & 0xff)
1413         return Value, RetSize
1414     if Value.startswith('DEVICE_PATH(') and Value.endswith(')'):
1415         Value = Value.replace("DEVICE_PATH(", '').rstrip(')')
1416         Value = Value.strip().strip('"')
1417         return ParseDevPathValue(Value)
1418     if Value.lower().startswith('0x'):
1419         try:
1420             Value = int(Value, 16)
1421         except:
1422             raise BadExpression("invalid hex value: %s" % Value)
1423         if Value == 0:
1424             return 0, 1
1425         return Value, (Value.bit_length() + 7) / 8
1426     if Value[0].isdigit():
1427         Value = int(Value, 10)
1428         if Value == 0:
1429             return 0, 1
1430         return Value, (Value.bit_length() + 7) / 8
1431     if Value.lower() == 'true':
1432         return 1, 1
1433     if Value.lower() == 'false':
1434         return 0, 1
1435     return Value, 1
1436
1437 ## AnalyzeDscPcd
1438 #
1439 #  Analyze DSC PCD value, since there is no data type info in DSC
1440 #  This function is used to match functions (AnalyzePcdData) used for retrieving PCD value from database
1441 #  1. Feature flag: TokenSpace.PcdCName|PcdValue
1442 #  2. Fix and Patch:TokenSpace.PcdCName|PcdValue[|VOID*[|MaxSize]]
1443 #  3. Dynamic default:
1444 #     TokenSpace.PcdCName|PcdValue[|VOID*[|MaxSize]]
1445 #     TokenSpace.PcdCName|PcdValue
1446 #  4. Dynamic VPD:
1447 #     TokenSpace.PcdCName|VpdOffset[|VpdValue]
1448 #     TokenSpace.PcdCName|VpdOffset[|MaxSize[|VpdValue]]
1449 #  5. Dynamic HII:
1450 #     TokenSpace.PcdCName|HiiString|VariableGuid|VariableOffset[|HiiValue]
1451 #  PCD value needs to be located in such kind of string, and the PCD value might be an expression in which
1452 #    there might have "|" operator, also in string value.
1453 #
1454 #  @param Setting: String contain information described above with "TokenSpace.PcdCName|" stripped
1455 #  @param PcdType: PCD type: feature, fixed, dynamic default VPD HII
1456 #  @param DataType: The datum type of PCD: VOID*, UNIT, BOOL
1457 #  @retval:
1458 #    ValueList: A List contain fields described above
1459 #    IsValid:   True if conforming EBNF, otherwise False
1460 #    Index:     The index where PcdValue is in ValueList
1461 #
1462 def AnalyzeDscPcd(Setting, PcdType, DataType=''):
1463     FieldList = AnalyzePcdExpression(Setting)
1464
1465     IsValid = True
1466     if PcdType in (MODEL_PCD_FIXED_AT_BUILD, MODEL_PCD_PATCHABLE_IN_MODULE, MODEL_PCD_DYNAMIC_DEFAULT, MODEL_PCD_DYNAMIC_EX_DEFAULT):
1467         Value = FieldList[0]
1468         Size = ''
1469         if len(FieldList) > 1 and FieldList[1]:
1470             DataType = FieldList[1]
1471             if FieldList[1] != TAB_VOID:
1472                 IsValid = False
1473         if len(FieldList) > 2:
1474             Size = FieldList[2]
1475         if IsValid:
1476             if DataType == "":
1477                 IsValid = (len(FieldList) <= 1)
1478             else:
1479                 IsValid = (len(FieldList) <= 3)
1480
1481         if Size:
1482             try:
1483                 int(Size, 16) if Size.upper().startswith("0X") else int(Size)
1484             except:
1485                 IsValid = False
1486                 Size = -1
1487         return [str(Value), DataType, str(Size)], IsValid, 0
1488     elif PcdType == MODEL_PCD_FEATURE_FLAG:
1489         Value = FieldList[0]
1490         Size = ''
1491         IsValid = (len(FieldList) <= 1)
1492         return [Value, DataType, str(Size)], IsValid, 0
1493     elif PcdType in (MODEL_PCD_DYNAMIC_VPD, MODEL_PCD_DYNAMIC_EX_VPD):
1494         VpdOffset = FieldList[0]
1495         Value = Size = ''
1496         if not DataType == TAB_VOID:
1497             if len(FieldList) > 1:
1498                 Value = FieldList[1]
1499         else:
1500             if len(FieldList) > 1:
1501                 Size = FieldList[1]
1502             if len(FieldList) > 2:
1503                 Value = FieldList[2]
1504         if DataType == "":
1505             IsValid = (len(FieldList) <= 1)
1506         else:
1507             IsValid = (len(FieldList) <= 3)
1508         if Size:
1509             try:
1510                 int(Size, 16) if Size.upper().startswith("0X") else int(Size)
1511             except:
1512                 IsValid = False
1513                 Size = -1
1514         return [VpdOffset, str(Size), Value], IsValid, 2
1515     elif PcdType in (MODEL_PCD_DYNAMIC_HII, MODEL_PCD_DYNAMIC_EX_HII):
1516         IsValid = (3 <= len(FieldList) <= 5)
1517         HiiString = FieldList[0]
1518         Guid = Offset = Value = Attribute = ''
1519         if len(FieldList) > 1:
1520             Guid = FieldList[1]
1521         if len(FieldList) > 2:
1522             Offset = FieldList[2]
1523         if len(FieldList) > 3:
1524             Value = FieldList[3]
1525             if not Value:
1526                 IsValid = False
1527         if len(FieldList) > 4:
1528             Attribute = FieldList[4]
1529         return [HiiString, Guid, Offset, Value, Attribute], IsValid, 3
1530     return [], False, 0
1531
1532 ## AnalyzePcdData
1533 #
1534 #  Analyze the pcd Value, Datum type and TokenNumber.
1535 #  Used to avoid split issue while the value string contain "|" character
1536 #
1537 #  @param[in] Setting:  A String contain value/datum type/token number information;
1538 #
1539 #  @retval   ValueList: A List contain value, datum type and toke number.
1540 #
1541 def AnalyzePcdData(Setting):
1542     ValueList = ['', '', '']
1543
1544     ValueRe = re.compile(r'^\s*L?\".*\|.*\"')
1545     PtrValue = ValueRe.findall(Setting)
1546
1547     ValueUpdateFlag = False
1548
1549     if len(PtrValue) >= 1:
1550         Setting = re.sub(ValueRe, '', Setting)
1551         ValueUpdateFlag = True
1552
1553     TokenList = Setting.split(TAB_VALUE_SPLIT)
1554     ValueList[0:len(TokenList)] = TokenList
1555
1556     if ValueUpdateFlag:
1557         ValueList[0] = PtrValue[0]
1558
1559     return ValueList
1560
1561 ## check format of PCD value against its the datum type
1562 #
1563 # For PCD value setting
1564 #
1565 def CheckPcdDatum(Type, Value):
1566     if Type == TAB_VOID:
1567         ValueRe = re.compile(r'\s*L?\".*\"\s*$')
1568         if not (((Value.startswith('L"') or Value.startswith('"')) and Value.endswith('"'))
1569                 or (Value.startswith('{') and Value.endswith('}')) or (Value.startswith("L'") or Value.startswith("'") and Value.endswith("'"))
1570                ):
1571             return False, "Invalid value [%s] of type [%s]; must be in the form of {...} for array"\
1572                           ", \"...\" or \'...\' for string, L\"...\" or L\'...\' for unicode string" % (Value, Type)
1573         elif ValueRe.match(Value):
1574             # Check the chars in UnicodeString or CString is printable
1575             if Value.startswith("L"):
1576                 Value = Value[2:-1]
1577             else:
1578                 Value = Value[1:-1]
1579             Printset = set(string.printable)
1580             Printset.remove(TAB_PRINTCHAR_VT)
1581             Printset.add(TAB_PRINTCHAR_BS)
1582             Printset.add(TAB_PRINTCHAR_NUL)
1583             if not set(Value).issubset(Printset):
1584                 PrintList = sorted(Printset)
1585                 return False, "Invalid PCD string value of type [%s]; must be printable chars %s." % (Type, PrintList)
1586     elif Type == 'BOOLEAN':
1587         if Value not in ['TRUE', 'True', 'true', '0x1', '0x01', '1', 'FALSE', 'False', 'false', '0x0', '0x00', '0']:
1588             return False, "Invalid value [%s] of type [%s]; must be one of TRUE, True, true, 0x1, 0x01, 1"\
1589                           ", FALSE, False, false, 0x0, 0x00, 0" % (Value, Type)
1590     elif Type in [TAB_UINT8, TAB_UINT16, TAB_UINT32, TAB_UINT64]:
1591         if Value and int(Value, 0) < 0:
1592             return False, "PCD can't be set to negative value[%s] for datum type [%s]" % (Value, Type)
1593         try:
1594             Value = long(Value, 0)
1595             if Value > MAX_VAL_TYPE[Type]:
1596                 return False, "Too large PCD value[%s] for datum type [%s]" % (Value, Type)
1597         except:
1598             return False, "Invalid value [%s] of type [%s];"\
1599                           " must be a hexadecimal, decimal or octal in C language format." % (Value, Type)
1600     else:
1601         return True, "StructurePcd"
1602
1603     return True, ""
1604
1605 ## Split command line option string to list
1606 #
1607 # subprocess.Popen needs the args to be a sequence. Otherwise there's problem
1608 # in non-windows platform to launch command
1609 #
1610 def SplitOption(OptionString):
1611     OptionList = []
1612     LastChar = " "
1613     OptionStart = 0
1614     QuotationMark = ""
1615     for Index in range(0, len(OptionString)):
1616         CurrentChar = OptionString[Index]
1617         if CurrentChar in ['"', "'"]:
1618             if QuotationMark == CurrentChar:
1619                 QuotationMark = ""
1620             elif QuotationMark == "":
1621                 QuotationMark = CurrentChar
1622             continue
1623         elif QuotationMark:
1624             continue
1625
1626         if CurrentChar in ["/", "-"] and LastChar in [" ", "\t", "\r", "\n"]:
1627             if Index > OptionStart:
1628                 OptionList.append(OptionString[OptionStart:Index - 1])
1629             OptionStart = Index
1630         LastChar = CurrentChar
1631     OptionList.append(OptionString[OptionStart:])
1632     return OptionList
1633
1634 def CommonPath(PathList):
1635     P1 = min(PathList).split(os.path.sep)
1636     P2 = max(PathList).split(os.path.sep)
1637     for Index in xrange(min(len(P1), len(P2))):
1638         if P1[Index] != P2[Index]:
1639             return os.path.sep.join(P1[:Index])
1640     return os.path.sep.join(P1)
1641
1642 #
1643 # Convert string to C format array
1644 #
1645 def ConvertStringToByteArray(Value):
1646     Value = Value.strip()
1647     if not Value:
1648         return None
1649     if Value[0] == '{':
1650         if not Value.endswith('}'):
1651             return None
1652         Value = Value.replace(' ', '').replace('{', '').replace('}', '')
1653         ValFields = Value.split(',')
1654         try:
1655             for Index in range(len(ValFields)):
1656                 ValFields[Index] = str(int(ValFields[Index], 0))
1657         except ValueError:
1658             return None
1659         Value = '{' + ','.join(ValFields) + '}'
1660         return Value
1661
1662     Unicode = False
1663     if Value.startswith('L"'):
1664         if not Value.endswith('"'):
1665             return None
1666         Value = Value[1:]
1667         Unicode = True
1668     elif not Value.startswith('"') or not Value.endswith('"'):
1669         return None
1670
1671     Value = eval(Value)         # translate escape character
1672     NewValue = '{'
1673     for Index in range(0, len(Value)):
1674         if Unicode:
1675             NewValue = NewValue + str(ord(Value[Index]) % 0x10000) + ','
1676         else:
1677             NewValue = NewValue + str(ord(Value[Index]) % 0x100) + ','
1678     Value = NewValue + '0}'
1679     return Value
1680
1681 class PathClass(object):
1682     def __init__(self, File='', Root='', AlterRoot='', Type='', IsBinary=False,
1683                  Arch='COMMON', ToolChainFamily='', Target='', TagName='', ToolCode=''):
1684         self.Arch = Arch
1685         self.File = str(File)
1686         if os.path.isabs(self.File):
1687             self.Root = ''
1688             self.AlterRoot = ''
1689         else:
1690             self.Root = str(Root)
1691             self.AlterRoot = str(AlterRoot)
1692
1693         # Remove any '.' and '..' in path
1694         if self.Root:
1695             self.Root = mws.getWs(self.Root, self.File)
1696             self.Path = os.path.normpath(os.path.join(self.Root, self.File))
1697             self.Root = os.path.normpath(CommonPath([self.Root, self.Path]))
1698             # eliminate the side-effect of 'C:'
1699             if self.Root[-1] == ':':
1700                 self.Root += os.path.sep
1701             # file path should not start with path separator
1702             if self.Root[-1] == os.path.sep:
1703                 self.File = self.Path[len(self.Root):]
1704             else:
1705                 self.File = self.Path[len(self.Root) + 1:]
1706         else:
1707             self.Path = os.path.normpath(self.File)
1708
1709         self.SubDir, self.Name = os.path.split(self.File)
1710         self.BaseName, self.Ext = os.path.splitext(self.Name)
1711
1712         if self.Root:
1713             if self.SubDir:
1714                 self.Dir = os.path.join(self.Root, self.SubDir)
1715             else:
1716                 self.Dir = self.Root
1717         else:
1718             self.Dir = self.SubDir
1719
1720         if IsBinary:
1721             self.Type = Type
1722         else:
1723             self.Type = self.Ext.lower()
1724
1725         self.IsBinary = IsBinary
1726         self.Target = Target
1727         self.TagName = TagName
1728         self.ToolCode = ToolCode
1729         self.ToolChainFamily = ToolChainFamily
1730
1731     ## Convert the object of this class to a string
1732     #
1733     #  Convert member Path of the class to a string
1734     #
1735     #  @retval string Formatted String
1736     #
1737     def __str__(self):
1738         return self.Path
1739
1740     ## Override __eq__ function
1741     #
1742     # Check whether PathClass are the same
1743     #
1744     # @retval False The two PathClass are different
1745     # @retval True  The two PathClass are the same
1746     #
1747     def __eq__(self, Other):
1748         if isinstance(Other, type(self)):
1749             return self.Path == Other.Path
1750         else:
1751             return self.Path == str(Other)
1752
1753     ## Override __cmp__ function
1754     #
1755     # Customize the comparsion operation of two PathClass
1756     #
1757     # @retval 0     The two PathClass are different
1758     # @retval -1    The first PathClass is less than the second PathClass
1759     # @retval 1     The first PathClass is Bigger than the second PathClass
1760     def __cmp__(self, Other):
1761         if isinstance(Other, type(self)):
1762             OtherKey = Other.Path
1763         else:
1764             OtherKey = str(Other)
1765
1766         SelfKey = self.Path
1767         if SelfKey == OtherKey:
1768             return 0
1769         elif SelfKey > OtherKey:
1770             return 1
1771         else:
1772             return -1
1773
1774     ## Override __hash__ function
1775     #
1776     # Use Path as key in hash table
1777     #
1778     # @retval string Key for hash table
1779     #
1780     def __hash__(self):
1781         return hash(self.Path)
1782
1783     @cached_property
1784     def Key(self):
1785         return self.Path.upper()
1786
1787     @property
1788     def TimeStamp(self):
1789         return os.stat(self.Path)[8]
1790
1791     def Validate(self, Type='', CaseSensitive=True):
1792         if GlobalData.gCaseInsensitive:
1793             CaseSensitive = False
1794         if Type and Type.lower() != self.Type:
1795             return FILE_TYPE_MISMATCH, '%s (expect %s but got %s)' % (self.File, Type, self.Type)
1796
1797         RealFile, RealRoot = RealPath2(self.File, self.Root, self.AlterRoot)
1798         if not RealRoot and not RealFile:
1799             RealFile = self.File
1800             if self.AlterRoot:
1801                 RealFile = os.path.join(self.AlterRoot, self.File)
1802             elif self.Root:
1803                 RealFile = os.path.join(self.Root, self.File)
1804             if len (mws.getPkgPath()) == 0:
1805                 return FILE_NOT_FOUND, os.path.join(self.AlterRoot, RealFile)
1806             else:
1807                 return FILE_NOT_FOUND, "%s is not found in packages path:\n\t%s" % (self.File, '\n\t'.join(mws.getPkgPath()))
1808
1809         ErrorCode = 0
1810         ErrorInfo = ''
1811         if RealRoot != self.Root or RealFile != self.File:
1812             if CaseSensitive and (RealFile != self.File or (RealRoot != self.Root and RealRoot != self.AlterRoot)):
1813                 ErrorCode = FILE_CASE_MISMATCH
1814                 ErrorInfo = self.File + '\n\t' + RealFile + " [in file system]"
1815
1816             self.SubDir, self.Name = os.path.split(RealFile)
1817             self.BaseName, self.Ext = os.path.splitext(self.Name)
1818             if self.SubDir:
1819                 self.Dir = os.path.join(RealRoot, self.SubDir)
1820             else:
1821                 self.Dir = RealRoot
1822             self.File = RealFile
1823             self.Root = RealRoot
1824             self.Path = os.path.join(RealRoot, RealFile)
1825         return ErrorCode, ErrorInfo
1826
1827 ## Parse PE image to get the required PE informaion.
1828 #
1829 class PeImageClass():
1830     ## Constructor
1831     #
1832     #   @param  File FilePath of PeImage
1833     #
1834     def __init__(self, PeFile):
1835         self.FileName   = PeFile
1836         self.IsValid    = False
1837         self.Size       = 0
1838         self.EntryPoint = 0
1839         self.SectionAlignment  = 0
1840         self.SectionHeaderList = []
1841         self.ErrorInfo = ''
1842         try:
1843             PeObject = open(PeFile, 'rb')
1844         except:
1845             self.ErrorInfo = self.FileName + ' can not be found\n'
1846             return
1847         # Read DOS header
1848         ByteArray = array.array('B')
1849         ByteArray.fromfile(PeObject, 0x3E)
1850         ByteList = ByteArray.tolist()
1851         # DOS signature should be 'MZ'
1852         if self._ByteListToStr (ByteList[0x0:0x2]) != 'MZ':
1853             self.ErrorInfo = self.FileName + ' has no valid DOS signature MZ'
1854             return
1855
1856         # Read 4 byte PE Signature
1857         PeOffset = self._ByteListToInt(ByteList[0x3C:0x3E])
1858         PeObject.seek(PeOffset)
1859         ByteArray = array.array('B')
1860         ByteArray.fromfile(PeObject, 4)
1861         # PE signature should be 'PE\0\0'
1862         if ByteArray.tostring() != 'PE\0\0':
1863             self.ErrorInfo = self.FileName + ' has no valid PE signature PE00'
1864             return
1865
1866         # Read PE file header
1867         ByteArray = array.array('B')
1868         ByteArray.fromfile(PeObject, 0x14)
1869         ByteList = ByteArray.tolist()
1870         SecNumber = self._ByteListToInt(ByteList[0x2:0x4])
1871         if SecNumber == 0:
1872             self.ErrorInfo = self.FileName + ' has no section header'
1873             return
1874
1875         # Read PE optional header
1876         OptionalHeaderSize = self._ByteListToInt(ByteArray[0x10:0x12])
1877         ByteArray = array.array('B')
1878         ByteArray.fromfile(PeObject, OptionalHeaderSize)
1879         ByteList = ByteArray.tolist()
1880         self.EntryPoint       = self._ByteListToInt(ByteList[0x10:0x14])
1881         self.SectionAlignment = self._ByteListToInt(ByteList[0x20:0x24])
1882         self.Size             = self._ByteListToInt(ByteList[0x38:0x3C])
1883
1884         # Read each Section Header
1885         for Index in range(SecNumber):
1886             ByteArray = array.array('B')
1887             ByteArray.fromfile(PeObject, 0x28)
1888             ByteList = ByteArray.tolist()
1889             SecName  = self._ByteListToStr(ByteList[0:8])
1890             SecVirtualSize = self._ByteListToInt(ByteList[8:12])
1891             SecRawAddress  = self._ByteListToInt(ByteList[20:24])
1892             SecVirtualAddress = self._ByteListToInt(ByteList[12:16])
1893             self.SectionHeaderList.append((SecName, SecVirtualAddress, SecRawAddress, SecVirtualSize))
1894         self.IsValid = True
1895         PeObject.close()
1896
1897     def _ByteListToStr(self, ByteList):
1898         String = ''
1899         for index in range(len(ByteList)):
1900             if ByteList[index] == 0:
1901                 break
1902             String += chr(ByteList[index])
1903         return String
1904
1905     def _ByteListToInt(self, ByteList):
1906         Value = 0
1907         for index in range(len(ByteList) - 1, -1, -1):
1908             Value = (Value << 8) | int(ByteList[index])
1909         return Value
1910
1911 class DefaultStore():
1912     def __init__(self, DefaultStores ):
1913
1914         self.DefaultStores = DefaultStores
1915     def DefaultStoreID(self, DefaultStoreName):
1916         for key, value in self.DefaultStores.items():
1917             if value == DefaultStoreName:
1918                 return key
1919         return None
1920     def GetDefaultDefault(self):
1921         if not self.DefaultStores or "0" in self.DefaultStores:
1922             return "0", TAB_DEFAULT_STORES_DEFAULT
1923         else:
1924             minvalue = min(int(value_str) for value_str in self.DefaultStores)
1925             return (str(minvalue), self.DefaultStores[str(minvalue)])
1926     def GetMin(self, DefaultSIdList):
1927         if not DefaultSIdList:
1928             return TAB_DEFAULT_STORES_DEFAULT
1929         storeidset = {storeid for storeid, storename in self.DefaultStores.values() if storename in DefaultSIdList}
1930         if not storeidset:
1931             return ""
1932         minid = min(storeidset )
1933         for sid, name in self.DefaultStores.values():
1934             if sid == minid:
1935                 return name
1936
1937 class SkuClass():
1938     DEFAULT = 0
1939     SINGLE = 1
1940     MULTIPLE =2
1941
1942     def __init__(self,SkuIdentifier='', SkuIds=None):
1943         if SkuIds is None:
1944             SkuIds = {}
1945
1946         for SkuName in SkuIds:
1947             SkuId = SkuIds[SkuName][0]
1948             skuid_num = int(SkuId, 16) if SkuId.upper().startswith("0X") else int(SkuId)
1949             if skuid_num > 0xFFFFFFFFFFFFFFFF:
1950                 EdkLogger.error("build", PARAMETER_INVALID,
1951                             ExtraData = "SKU-ID [%s] value %s exceeds the max value of UINT64"
1952                                       % (SkuName, SkuId))
1953
1954         self.AvailableSkuIds = sdict()
1955         self.SkuIdSet = []
1956         self.SkuIdNumberSet = []
1957         self.SkuData = SkuIds
1958         self._SkuInherit = {}
1959         self._SkuIdentifier = SkuIdentifier
1960         if SkuIdentifier == '' or SkuIdentifier is None:
1961             self.SkuIdSet = ['DEFAULT']
1962             self.SkuIdNumberSet = ['0U']
1963         elif SkuIdentifier == 'ALL':
1964             self.SkuIdSet = SkuIds.keys()
1965             self.SkuIdNumberSet = [num[0].strip() + 'U' for num in SkuIds.values()]
1966         else:
1967             r = SkuIdentifier.split('|')
1968             self.SkuIdSet=[(r[k].strip()).upper() for k in range(len(r))]
1969             k = None
1970             try:
1971                 self.SkuIdNumberSet = [SkuIds[k][0].strip() + 'U' for k in self.SkuIdSet]
1972             except Exception:
1973                 EdkLogger.error("build", PARAMETER_INVALID,
1974                             ExtraData = "SKU-ID [%s] is not supported by the platform. [Valid SKU-ID: %s]"
1975                                       % (k, " | ".join(SkuIds.keys())))
1976         for each in self.SkuIdSet:
1977             if each in SkuIds:
1978                 self.AvailableSkuIds[each] = SkuIds[each][0]
1979             else:
1980                 EdkLogger.error("build", PARAMETER_INVALID,
1981                             ExtraData="SKU-ID [%s] is not supported by the platform. [Valid SKU-ID: %s]"
1982                                       % (each, " | ".join(SkuIds.keys())))
1983         if self.SkuUsageType != SkuClass.SINGLE:
1984             self.AvailableSkuIds.update({'DEFAULT':0, 'COMMON':0})
1985         if self.SkuIdSet:
1986             GlobalData.gSkuids = (self.SkuIdSet)
1987             if 'COMMON' in GlobalData.gSkuids:
1988                 GlobalData.gSkuids.remove('COMMON')
1989             if self.SkuUsageType == self.SINGLE:
1990                 if len(GlobalData.gSkuids) != 1:
1991                     if 'DEFAULT' in GlobalData.gSkuids:
1992                         GlobalData.gSkuids.remove('DEFAULT')
1993             if GlobalData.gSkuids:
1994                 GlobalData.gSkuids.sort()
1995
1996     def GetNextSkuId(self, skuname):
1997         if not self._SkuInherit:
1998             self._SkuInherit = {}
1999             for item in self.SkuData.values():
2000                 self._SkuInherit[item[1]]=item[2] if item[2] else "DEFAULT"
2001         return self._SkuInherit.get(skuname, "DEFAULT")
2002
2003     def GetSkuChain(self, sku):
2004         if sku == "DEFAULT":
2005             return ["DEFAULT"]
2006         skulist = [sku]
2007         nextsku = sku
2008         while True:
2009             nextsku = self.GetNextSkuId(nextsku)
2010             skulist.append(nextsku)
2011             if nextsku == "DEFAULT":
2012                 break
2013         skulist.reverse()
2014         return skulist
2015     def SkuOverrideOrder(self):
2016         skuorderset = []
2017         for skuname in self.SkuIdSet:
2018             skuorderset.append(self.GetSkuChain(skuname))
2019
2020         skuorder = []
2021         for index in range(max(len(item) for item in skuorderset)):
2022             for subset in skuorderset:
2023                 if index > len(subset)-1:
2024                     continue
2025                 if subset[index] in skuorder:
2026                     continue
2027                 skuorder.append(subset[index])
2028
2029         return skuorder
2030
2031     @property
2032     def SkuUsageType(self):
2033         if self._SkuIdentifier.upper() == "ALL":
2034             return SkuClass.MULTIPLE
2035
2036         if len(self.SkuIdSet) == 1:
2037             if self.SkuIdSet[0] == 'DEFAULT':
2038                 return SkuClass.DEFAULT
2039             return SkuClass.SINGLE
2040         if len(self.SkuIdSet) == 2 and 'DEFAULT' in self.SkuIdSet:
2041             return SkuClass.SINGLE
2042         return SkuClass.MULTIPLE
2043
2044     def DumpSkuIdArrary(self):
2045         if self.SkuUsageType == SkuClass.SINGLE:
2046             return "{0x0}"
2047         ArrayStrList = []
2048         for skuname in self.AvailableSkuIds:
2049             if skuname == "COMMON":
2050                 continue
2051             while skuname != "DEFAULT":
2052                 ArrayStrList.append(hex(int(self.AvailableSkuIds[skuname])))
2053                 skuname = self.GetNextSkuId(skuname)
2054             ArrayStrList.append("0x0")
2055         return "{{{myList}}}".format(myList=",".join(ArrayStrList))
2056
2057     @property
2058     def AvailableSkuIdSet(self):
2059         return self.AvailableSkuIds
2060
2061     @property
2062     def SystemSkuId(self):
2063         if self.SkuUsageType == SkuClass.SINGLE:
2064             if len(self.SkuIdSet) == 1:
2065                 return self.SkuIdSet[0]
2066             else:
2067                 return self.SkuIdSet[0] if self.SkuIdSet[0] != 'DEFAULT' else self.SkuIdSet[1]
2068         else:
2069             return 'DEFAULT'
2070
2071 #
2072 # Pack a registry format GUID
2073 #
2074 def PackRegistryFormatGuid(Guid):
2075     return PackGUID(Guid.split('-'))
2076
2077 ##  Get the integer value from string like "14U" or integer like 2
2078 #
2079 #   @param      Input   The object that may be either a integer value or a string
2080 #
2081 #   @retval     Value    The integer value that the input represents
2082 #
2083 def GetIntegerValue(Input):
2084     if type(Input) in (int, long):
2085         return Input
2086     String = Input
2087     if String.endswith("U"):
2088         String = String[:-1]
2089     if String.endswith("ULL"):
2090         String = String[:-3]
2091     if String.endswith("LL"):
2092         String = String[:-2]
2093
2094     if String.startswith("0x") or String.startswith("0X"):
2095         return int(String, 16)
2096     elif String == '':
2097         return 0
2098     else:
2099         return int(String)
2100
2101 #
2102 # Pack a GUID (registry format) list into a buffer and return it
2103 #
2104 def PackGUID(Guid):
2105     return pack(PACK_PATTERN_GUID,
2106                 int(Guid[0], 16),
2107                 int(Guid[1], 16),
2108                 int(Guid[2], 16),
2109                 int(Guid[3][-4:-2], 16),
2110                 int(Guid[3][-2:], 16),
2111                 int(Guid[4][-12:-10], 16),
2112                 int(Guid[4][-10:-8], 16),
2113                 int(Guid[4][-8:-6], 16),
2114                 int(Guid[4][-6:-4], 16),
2115                 int(Guid[4][-4:-2], 16),
2116                 int(Guid[4][-2:], 16)
2117                 )
2118
2119 #
2120 # Pack a GUID (byte) list into a buffer and return it
2121 #
2122 def PackByteFormatGUID(Guid):
2123     return pack(PACK_PATTERN_GUID,
2124                 Guid[0],
2125                 Guid[1],
2126                 Guid[2],
2127                 Guid[3],
2128                 Guid[4],
2129                 Guid[5],
2130                 Guid[6],
2131                 Guid[7],
2132                 Guid[8],
2133                 Guid[9],
2134                 Guid[10],
2135                 )
2136
2137 ##
2138 #
2139 # This acts like the main() function for the script, unless it is 'import'ed into another
2140 # script.
2141 #
2142 if __name__ == '__main__':
2143     pass
2144