BaseTools/Source/Python/Common/Misc.py

   1 ## @file
   2 # Common routines used by all tools
   3 #
   4 # Copyright (c) 2007 - 2018, Intel Corporation. All rights reserved.<BR>
   5 # This program and the accompanying materials
   6 # are licensed and made available under the terms and conditions of the BSD License
   7 # which accompanies this distribution.  The full text of the license may be found at
   8 # http://opensource.org/licenses/bsd-license.php
   9 #
  10 # THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS,
  11 # WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED.
  12 #
  13
  14 ##
  15 # Import Modules
  16 #
  17 import Common.LongFilePathOs as os
  18 import sys
  19 import string
  20 import thread
  21 import threading
  22 import time
  23 import re
  24 import cPickle
  25 import array
  26 import shutil
  27 from struct import pack
  28 from UserDict import IterableUserDict
  29 from UserList import UserList
  30
  31 from Common import EdkLogger as EdkLogger
  32 from Common import GlobalData as GlobalData
  33 from DataType import *
  34 from BuildToolError import *
  35 from CommonDataClass.DataClass import *
  36 from Parsing import GetSplitValueList
  37 from Common.LongFilePathSupport import OpenLongFilePath as open
  38 from Common.MultipleWorkspace import MultipleWorkspace as mws
  39 import uuid
  40 from CommonDataClass.Exceptions import BadExpression
  41 import subprocess
  42 ## Regular expression used to find out place holders in string template
  43 gPlaceholderPattern = re.compile("\$\{([^$()\s]+)\}", re.MULTILINE | re.UNICODE)
  44
  45 ## regular expressions for map file processing
  46 startPatternGeneral = re.compile("^Start[' ']+Length[' ']+Name[' ']+Class")
  47 addressPatternGeneral = re.compile("^Address[' ']+Publics by Value[' ']+Rva\+Base")
  48 valuePatternGcc = re.compile('^([\w_\.]+) +([\da-fA-Fx]+) +([\da-fA-Fx]+)$')
  49 pcdPatternGcc = re.compile('^([\da-fA-Fx]+) +([\da-fA-Fx]+)')
  50 secReGeneral = re.compile('^([\da-fA-F]+):([\da-fA-F]+) +([\da-fA-F]+)[Hh]? +([.\w\$]+) +(\w+)', re.UNICODE)
  51
  52 ## Dictionary used to store file time stamp for quick re-access
  53 gFileTimeStampCache = {}    # {file path : file time stamp}
  54
  55 ## Dictionary used to store dependencies of files
  56 gDependencyDatabase = {}    # arch : {file path : [dependent files list]}
  57
  58 def GetVariableOffset(mapfilepath, efifilepath, varnames):
  59     """ Parse map file to get variable offset in current EFI file
  60     @param mapfilepath    Map file absolution path
  61     @param efifilepath:   EFI binary file full path
  62     @param varnames       iteratable container whose elements are variable names to be searched
  63
  64     @return List whos elements are tuple with variable name and raw offset
  65     """
  66     lines = []
  67     try:
  68         f = open(mapfilepath, 'r')
  69         lines = f.readlines()
  70         f.close()
  71     except:
  72         return None
  73
  74     if len(lines) == 0: return None
  75     firstline = lines[0].strip()
  76     if (firstline.startswith("Archive member included ") and
  77         firstline.endswith(" file (symbol)")):
  78         return _parseForGCC(lines, efifilepath, varnames)
  79     if firstline.startswith("# Path:"):
  80         return _parseForXcode(lines, efifilepath, varnames)
  81     return _parseGeneral(lines, efifilepath, varnames)
  82
  83 def _parseForXcode(lines, efifilepath, varnames):
  84     status = 0
  85     ret = []
  86     for line in lines:
  87         line = line.strip()
  88         if status == 0 and line == "# Symbols:":
  89             status = 1
  90             continue
  91         if status == 1 and len(line) != 0:
  92             for varname in varnames:
  93                 if varname in line:
  94                     # cannot pregenerate this RegEx since it uses varname from varnames.
  95                     m = re.match('^([\da-fA-FxX]+)([\s\S]*)([_]*%s)$' % varname, line)
  96                     if m is not None:
  97                         ret.append((varname, m.group(1)))
  98     return ret
  99
 100 def _parseForGCC(lines, efifilepath, varnames):
 101     """ Parse map file generated by GCC linker """
 102     status = 0
 103     sections = []
 104     varoffset = []
 105     for index, line in enumerate(lines):
 106         line = line.strip()
 107         # status machine transection
 108         if status == 0 and line == "Memory Configuration":
 109             status = 1
 110             continue
 111         elif status == 1 and line == 'Linker script and memory map':
 112             status = 2
 113             continue
 114         elif status ==2 and line == 'START GROUP':
 115             status = 3
 116             continue
 117
 118         # status handler
 119         if status == 3:
 120             m = valuePatternGcc.match(line)
 121             if m is not None:
 122                 sections.append(m.groups(0))
 123             for varname in varnames:
 124                 Str = ''
 125                 m = re.match("^.data.(%s)" % varname, line)
 126                 if m is not None:
 127                     m = re.match(".data.(%s)$" % varname, line)
 128                     if m is not None:
 129                         Str = lines[index + 1]
 130                     else:
 131                         Str = line[len(".data.%s" % varname):]
 132                     if Str:
 133                         m = pcdPatternGcc.match(Str.strip())
 134                         if m is not None:
 135                             varoffset.append((varname, int(m.groups(0)[0], 16) , int(sections[-1][1], 16), sections[-1][0]))
 136
 137     if not varoffset:
 138         return []
 139     # get section information from efi file
 140     efisecs = PeImageClass(efifilepath).SectionHeaderList
 141     if efisecs is None or len(efisecs) == 0:
 142         return []
 143     #redirection
 144     redirection = 0
 145     for efisec in efisecs:
 146         for section in sections:
 147             if section[0].strip() == efisec[0].strip() and section[0].strip() == '.text':
 148                 redirection = int(section[1], 16) - efisec[1]
 149
 150     ret = []
 151     for var in varoffset:
 152         for efisec in efisecs:
 153             if var[1] >= efisec[1] and var[1] < efisec[1]+efisec[3]:
 154                 ret.append((var[0], hex(efisec[2] + var[1] - efisec[1] - redirection)))
 155     return ret
 156
 157 def _parseGeneral(lines, efifilepath, varnames):
 158     status = 0    #0 - beginning of file; 1 - PE section definition; 2 - symbol table
 159     secs  = []    # key = section name
 160     varoffset = []
 161     symRe = re.compile('^([\da-fA-F]+):([\da-fA-F]+) +([\.:\\\\\w\?@\$]+) +([\da-fA-F]+)', re.UNICODE)
 162
 163     for line in lines:
 164         line = line.strip()
 165         if startPatternGeneral.match(line):
 166             status = 1
 167             continue
 168         if addressPatternGeneral.match(line):
 169             status = 2
 170             continue
 171         if line.startswith("entry point at"):
 172             status = 3
 173             continue
 174         if status == 1 and len(line) != 0:
 175             m =  secReGeneral.match(line)
 176             assert m is not None, "Fail to parse the section in map file , line is %s" % line
 177             sec_no, sec_start, sec_length, sec_name, sec_class = m.groups(0)
 178             secs.append([int(sec_no, 16), int(sec_start, 16), int(sec_length, 16), sec_name, sec_class])
 179         if status == 2 and len(line) != 0:
 180             for varname in varnames:
 181                 m = symRe.match(line)
 182                 assert m is not None, "Fail to parse the symbol in map file, line is %s" % line
 183                 sec_no, sym_offset, sym_name, vir_addr = m.groups(0)
 184                 sec_no     = int(sec_no,     16)
 185                 sym_offset = int(sym_offset, 16)
 186                 vir_addr   = int(vir_addr,   16)
 187                 # cannot pregenerate this RegEx since it uses varname from varnames.
 188                 m2 = re.match('^[_]*(%s)' % varname, sym_name)
 189                 if m2 is not None:
 190                     # fond a binary pcd entry in map file
 191                     for sec in secs:
 192                         if sec[0] == sec_no and (sym_offset >= sec[1] and sym_offset < sec[1] + sec[2]):
 193                             varoffset.append([varname, sec[3], sym_offset, vir_addr, sec_no])
 194
 195     if not varoffset: return []
 196
 197     # get section information from efi file
 198     efisecs = PeImageClass(efifilepath).SectionHeaderList
 199     if efisecs is None or len(efisecs) == 0:
 200         return []
 201
 202     ret = []
 203     for var in varoffset:
 204         index = 0
 205         for efisec in efisecs:
 206             index = index + 1
 207             if var[1].strip() == efisec[0].strip():
 208                 ret.append((var[0], hex(efisec[2] + var[2])))
 209             elif var[4] == index:
 210                 ret.append((var[0], hex(efisec[2] + var[2])))
 211
 212     return ret
 213
 214 ## Routine to process duplicated INF
 215 #
 216 #  This function is called by following two cases:
 217 #  Case 1 in DSC:
 218 #    [components.arch]
 219 #    Pkg/module/module.inf
 220 #    Pkg/module/module.inf {
 221 #      <Defines>
 222 #        FILE_GUID = 0D1B936F-68F3-4589-AFCC-FB8B7AEBC836
 223 #    }
 224 #  Case 2 in FDF:
 225 #    INF Pkg/module/module.inf
 226 #    INF FILE_GUID = 0D1B936F-68F3-4589-AFCC-FB8B7AEBC836 Pkg/module/module.inf
 227 #
 228 #  This function copies Pkg/module/module.inf to
 229 #  Conf/.cache/0D1B936F-68F3-4589-AFCC-FB8B7AEBC836module.inf
 230 #
 231 #  @param Path     Original PathClass object
 232 #  @param BaseName New file base name
 233 #
 234 #  @retval         return the new PathClass object
 235 #
 236 def ProcessDuplicatedInf(Path, BaseName, Workspace):
 237     Filename = os.path.split(Path.File)[1]
 238     if '.' in Filename:
 239         Filename = BaseName + Path.BaseName + Filename[Filename.rfind('.'):]
 240     else:
 241         Filename = BaseName + Path.BaseName
 242
 243     #
 244     # If -N is specified on command line, cache is disabled
 245     # The directory has to be created
 246     #
 247     DbDir = os.path.split(GlobalData.gDatabasePath)[0]
 248     if not os.path.exists(DbDir):
 249         os.makedirs(DbDir)
 250     #
 251     # A temporary INF is copied to database path which must have write permission
 252     # The temporary will be removed at the end of build
 253     # In case of name conflict, the file name is
 254     # FILE_GUIDBaseName (0D1B936F-68F3-4589-AFCC-FB8B7AEBC836module.inf)
 255     #
 256     TempFullPath = os.path.join(DbDir,
 257                                 Filename)
 258     RtPath = PathClass(Path.File, Workspace)
 259     #
 260     # Modify the full path to temporary path, keep other unchanged
 261     #
 262     # To build same module more than once, the module path with FILE_GUID overridden has
 263     # the file name FILE_GUIDmodule.inf, but the relative path (self.MetaFile.File) is the real path
 264     # in DSC which is used as relative path by C files and other files in INF.
 265     # A trick was used: all module paths are PathClass instances, after the initialization
 266     # of PathClass, the PathClass.Path is overridden by the temporary INF path.
 267     #
 268     # The reason for creating a temporary INF is:
 269     # Platform.Modules which is the base to create ModuleAutoGen objects is a dictionary,
 270     # the key is the full path of INF, the value is an object to save overridden library instances, PCDs.
 271     # A different key for the same module is needed to create different output directory,
 272     # retrieve overridden PCDs, library instances.
 273     #
 274     # The BaseName is the FILE_GUID which is also the output directory name.
 275     #
 276     #
 277     RtPath.Path = TempFullPath
 278     RtPath.BaseName = BaseName
 279     #
 280     # If file exists, compare contents
 281     #
 282     if os.path.exists(TempFullPath):
 283         with open(str(Path), 'rb') as f1: Src = f1.read()
 284         with open(TempFullPath, 'rb') as f2: Dst = f2.read()
 285         if Src == Dst:
 286             return RtPath
 287     GlobalData.gTempInfs.append(TempFullPath)
 288     shutil.copy2(str(Path), TempFullPath)
 289     return RtPath
 290
 291 ## Remove temporary created INFs whose paths were saved in gTempInfs
 292 #
 293 def ClearDuplicatedInf():
 294     for File in GlobalData.gTempInfs:
 295         if os.path.exists(File):
 296             os.remove(File)
 297
 298 ## Convert GUID string in xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx style to C structure style
 299 #
 300 #   @param      Guid    The GUID string
 301 #
 302 #   @retval     string  The GUID string in C structure style
 303 #
 304 def GuidStringToGuidStructureString(Guid):
 305     GuidList = Guid.split('-')
 306     Result = '{'
 307     for Index in range(0, 3, 1):
 308         Result = Result + '0x' + GuidList[Index] + ', '
 309     Result = Result + '{0x' + GuidList[3][0:2] + ', 0x' + GuidList[3][2:4]
 310     for Index in range(0, 12, 2):
 311         Result = Result + ', 0x' + GuidList[4][Index:Index + 2]
 312     Result += '}}'
 313     return Result
 314
 315 ## Convert GUID structure in byte array to xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
 316 #
 317 #   @param      GuidValue   The GUID value in byte array
 318 #
 319 #   @retval     string      The GUID value in xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx format
 320 #
 321 def GuidStructureByteArrayToGuidString(GuidValue):
 322     guidValueString = GuidValue.lower().replace("{", "").replace("}", "").replace(" ", "").replace(";", "")
 323     guidValueList = guidValueString.split(",")
 324     if len(guidValueList) != 16:
 325         return ''
 326         #EdkLogger.error(None, None, "Invalid GUID value string %s" % GuidValue)
 327     try:
 328         return "%02x%02x%02x%02x-%02x%02x-%02x%02x-%02x%02x-%02x%02x%02x%02x%02x%02x" % (
 329                 int(guidValueList[3], 16),
 330                 int(guidValueList[2], 16),
 331                 int(guidValueList[1], 16),
 332                 int(guidValueList[0], 16),
 333                 int(guidValueList[5], 16),
 334                 int(guidValueList[4], 16),
 335                 int(guidValueList[7], 16),
 336                 int(guidValueList[6], 16),
 337                 int(guidValueList[8], 16),
 338                 int(guidValueList[9], 16),
 339                 int(guidValueList[10], 16),
 340                 int(guidValueList[11], 16),
 341                 int(guidValueList[12], 16),
 342                 int(guidValueList[13], 16),
 343                 int(guidValueList[14], 16),
 344                 int(guidValueList[15], 16)
 345                 )
 346     except:
 347         return ''
 348
 349 ## Convert GUID string in C structure style to xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
 350 #
 351 #   @param      GuidValue   The GUID value in C structure format
 352 #
 353 #   @retval     string      The GUID value in xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx format
 354 #
 355 def GuidStructureStringToGuidString(GuidValue):
 356     guidValueString = GuidValue.lower().replace("{", "").replace("}", "").replace(" ", "").replace(";", "")
 357     guidValueList = guidValueString.split(",")
 358     if len(guidValueList) != 11:
 359         return ''
 360         #EdkLogger.error(None, None, "Invalid GUID value string %s" % GuidValue)
 361     try:
 362         return "%08x-%04x-%04x-%02x%02x-%02x%02x%02x%02x%02x%02x" % (
 363                 int(guidValueList[0], 16),
 364                 int(guidValueList[1], 16),
 365                 int(guidValueList[2], 16),
 366                 int(guidValueList[3], 16),
 367                 int(guidValueList[4], 16),
 368                 int(guidValueList[5], 16),
 369                 int(guidValueList[6], 16),
 370                 int(guidValueList[7], 16),
 371                 int(guidValueList[8], 16),
 372                 int(guidValueList[9], 16),
 373                 int(guidValueList[10], 16)
 374                 )
 375     except:
 376         return ''
 377
 378 ## Convert GUID string in C structure style to xxxxxxxx_xxxx_xxxx_xxxx_xxxxxxxxxxxx
 379 #
 380 #   @param      GuidValue   The GUID value in C structure format
 381 #
 382 #   @retval     string      The GUID value in xxxxxxxx_xxxx_xxxx_xxxx_xxxxxxxxxxxx format
 383 #
 384 def GuidStructureStringToGuidValueName(GuidValue):
 385     guidValueString = GuidValue.lower().replace("{", "").replace("}", "").replace(" ", "")
 386     guidValueList = guidValueString.split(",")
 387     if len(guidValueList) != 11:
 388         EdkLogger.error(None, FORMAT_INVALID, "Invalid GUID value string [%s]" % GuidValue)
 389     return "%08x_%04x_%04x_%02x%02x_%02x%02x%02x%02x%02x%02x" % (
 390             int(guidValueList[0], 16),
 391             int(guidValueList[1], 16),
 392             int(guidValueList[2], 16),
 393             int(guidValueList[3], 16),
 394             int(guidValueList[4], 16),
 395             int(guidValueList[5], 16),
 396             int(guidValueList[6], 16),
 397             int(guidValueList[7], 16),
 398             int(guidValueList[8], 16),
 399             int(guidValueList[9], 16),
 400             int(guidValueList[10], 16)
 401             )
 402
 403 ## Create directories
 404 #
 405 #   @param      Directory   The directory name
 406 #
 407 def CreateDirectory(Directory):
 408     if Directory is None or Directory.strip() == "":
 409         return True
 410     try:
 411         if not os.access(Directory, os.F_OK):
 412             os.makedirs(Directory)
 413     except:
 414         return False
 415     return True
 416
 417 ## Remove directories, including files and sub-directories in it
 418 #
 419 #   @param      Directory   The directory name
 420 #
 421 def RemoveDirectory(Directory, Recursively=False):
 422     if Directory is None or Directory.strip() == "" or not os.path.exists(Directory):
 423         return
 424     if Recursively:
 425         CurrentDirectory = os.getcwd()
 426         os.chdir(Directory)
 427         for File in os.listdir("."):
 428             if os.path.isdir(File):
 429                 RemoveDirectory(File, Recursively)
 430             else:
 431                 os.remove(File)
 432         os.chdir(CurrentDirectory)
 433     os.rmdir(Directory)
 434
 435 ## Store content in file
 436 #
 437 #  This method is used to save file only when its content is changed. This is
 438 #  quite useful for "make" system to decide what will be re-built and what won't.
 439 #
 440 #   @param      File            The path of file
 441 #   @param      Content         The new content of the file
 442 #   @param      IsBinaryFile    The flag indicating if the file is binary file or not
 443 #
 444 #   @retval     True            If the file content is changed and the file is renewed
 445 #   @retval     False           If the file content is the same
 446 #
 447 def SaveFileOnChange(File, Content, IsBinaryFile=True):
 448     if not IsBinaryFile:
 449         Content = Content.replace("\n", os.linesep)
 450
 451     if os.path.exists(File):
 452         try:
 453             if Content == open(File, "rb").read():
 454                 return False
 455         except:
 456             EdkLogger.error(None, FILE_OPEN_FAILURE, ExtraData=File)
 457
 458     DirName = os.path.dirname(File)
 459     if not CreateDirectory(DirName):
 460         EdkLogger.error(None, FILE_CREATE_FAILURE, "Could not create directory %s" % DirName)
 461     else:
 462         if DirName == '':
 463             DirName = os.getcwd()
 464         if not os.access(DirName, os.W_OK):
 465             EdkLogger.error(None, PERMISSION_FAILURE, "Do not have write permission on directory %s" % DirName)
 466
 467     try:
 468         if GlobalData.gIsWindows:
 469             try:
 470                 from PyUtility import SaveFileToDisk
 471                 if not SaveFileToDisk(File, Content):
 472                     EdkLogger.error(None, FILE_CREATE_FAILURE, ExtraData=File)
 473             except:
 474                 Fd = open(File, "wb")
 475                 Fd.write(Content)
 476                 Fd.close()
 477         else:
 478             Fd = open(File, "wb")
 479             Fd.write(Content)
 480             Fd.close()
 481     except IOError as X:
 482         EdkLogger.error(None, FILE_CREATE_FAILURE, ExtraData='IOError %s' % X)
 483
 484     return True
 485
 486 ## Make a Python object persistent on file system
 487 #
 488 #   @param      Data    The object to be stored in file
 489 #   @param      File    The path of file to store the object
 490 #
 491 def DataDump(Data, File):
 492     Fd = None
 493     try:
 494         Fd = open(File, 'wb')
 495         cPickle.dump(Data, Fd, cPickle.HIGHEST_PROTOCOL)
 496     except:
 497         EdkLogger.error("", FILE_OPEN_FAILURE, ExtraData=File, RaiseError=False)
 498     finally:
 499         if Fd is not None:
 500             Fd.close()
 501
 502 ## Restore a Python object from a file
 503 #
 504 #   @param      File    The path of file stored the object
 505 #
 506 #   @retval     object  A python object
 507 #   @retval     None    If failure in file operation
 508 #
 509 def DataRestore(File):
 510     Data = None
 511     Fd = None
 512     try:
 513         Fd = open(File, 'rb')
 514         Data = cPickle.load(Fd)
 515     except Exception as e:
 516         EdkLogger.verbose("Failed to load [%s]\n\t%s" % (File, str(e)))
 517         Data = None
 518     finally:
 519         if Fd is not None:
 520             Fd.close()
 521     return Data
 522
 523 ## Retrieve and cache the real path name in file system
 524 #
 525 #   @param      Root    The root directory of path relative to
 526 #
 527 #   @retval     str     The path string if the path exists
 528 #   @retval     None    If path doesn't exist
 529 #
 530 class DirCache:
 531     _CACHE_ = set()
 532     _UPPER_CACHE_ = {}
 533
 534     def __init__(self, Root):
 535         self._Root = Root
 536         for F in os.listdir(Root):
 537             self._CACHE_.add(F)
 538             self._UPPER_CACHE_[F.upper()] = F
 539
 540     # =[] operator
 541     def __getitem__(self, Path):
 542         Path = Path[len(os.path.commonprefix([Path, self._Root])):]
 543         if not Path:
 544             return self._Root
 545         if Path and Path[0] == os.path.sep:
 546             Path = Path[1:]
 547         if Path in self._CACHE_:
 548             return os.path.join(self._Root, Path)
 549         UpperPath = Path.upper()
 550         if UpperPath in self._UPPER_CACHE_:
 551             return os.path.join(self._Root, self._UPPER_CACHE_[UpperPath])
 552
 553         IndexList = []
 554         LastSepIndex = -1
 555         SepIndex = Path.find(os.path.sep)
 556         while SepIndex > -1:
 557             Parent = UpperPath[:SepIndex]
 558             if Parent not in self._UPPER_CACHE_:
 559                 break
 560             LastSepIndex = SepIndex
 561             SepIndex = Path.find(os.path.sep, LastSepIndex + 1)
 562
 563         if LastSepIndex == -1:
 564             return None
 565
 566         Cwd = os.getcwd()
 567         os.chdir(self._Root)
 568         SepIndex = LastSepIndex
 569         while SepIndex > -1:
 570             Parent = Path[:SepIndex]
 571             ParentKey = UpperPath[:SepIndex]
 572             if ParentKey not in self._UPPER_CACHE_:
 573                 os.chdir(Cwd)
 574                 return None
 575
 576             if Parent in self._CACHE_:
 577                 ParentDir = Parent
 578             else:
 579                 ParentDir = self._UPPER_CACHE_[ParentKey]
 580             for F in os.listdir(ParentDir):
 581                 Dir = os.path.join(ParentDir, F)
 582                 self._CACHE_.add(Dir)
 583                 self._UPPER_CACHE_[Dir.upper()] = Dir
 584
 585             SepIndex = Path.find(os.path.sep, SepIndex + 1)
 586
 587         os.chdir(Cwd)
 588         if Path in self._CACHE_:
 589             return os.path.join(self._Root, Path)
 590         elif UpperPath in self._UPPER_CACHE_:
 591             return os.path.join(self._Root, self._UPPER_CACHE_[UpperPath])
 592         return None
 593
 594 def RealPath(File, Dir='', OverrideDir=''):
 595     NewFile = os.path.normpath(os.path.join(Dir, File))
 596     NewFile = GlobalData.gAllFiles[NewFile]
 597     if not NewFile and OverrideDir:
 598         NewFile = os.path.normpath(os.path.join(OverrideDir, File))
 599         NewFile = GlobalData.gAllFiles[NewFile]
 600     return NewFile
 601
 602 def RealPath2(File, Dir='', OverrideDir=''):
 603     NewFile = None
 604     if OverrideDir:
 605         NewFile = GlobalData.gAllFiles[os.path.normpath(os.path.join(OverrideDir, File))]
 606         if NewFile:
 607             if OverrideDir[-1] == os.path.sep:
 608                 return NewFile[len(OverrideDir):], NewFile[0:len(OverrideDir)]
 609             else:
 610                 return NewFile[len(OverrideDir) + 1:], NewFile[0:len(OverrideDir)]
 611     if GlobalData.gAllFiles:
 612         NewFile = GlobalData.gAllFiles[os.path.normpath(os.path.join(Dir, File))]
 613     if not NewFile:
 614         NewFile = os.path.normpath(os.path.join(Dir, File))
 615         if not os.path.exists(NewFile):
 616             return None, None
 617     if NewFile:
 618         if Dir:
 619             if Dir[-1] == os.path.sep:
 620                 return NewFile[len(Dir):], NewFile[0:len(Dir)]
 621             else:
 622                 return NewFile[len(Dir) + 1:], NewFile[0:len(Dir)]
 623         else:
 624             return NewFile, ''
 625
 626     return None, None
 627
 628 ## Get GUID value from given packages
 629 #
 630 #   @param      CName           The CName of the GUID
 631 #   @param      PackageList     List of packages looking-up in
 632 #   @param      Inffile         The driver file
 633 #
 634 #   @retval     GuidValue   if the CName is found in any given package
 635 #   @retval     None        if the CName is not found in all given packages
 636 #
 637 def GuidValue(CName, PackageList, Inffile = None):
 638     for P in PackageList:
 639         GuidKeys = P.Guids.keys()
 640         if Inffile and P._PrivateGuids:
 641             if not Inffile.startswith(P.MetaFile.Dir):
 642                 GuidKeys = [x for x in P.Guids if x not in P._PrivateGuids]
 643         if CName in GuidKeys:
 644             return P.Guids[CName]
 645     return None
 646
 647 ## Get Protocol value from given packages
 648 #
 649 #   @param      CName           The CName of the GUID
 650 #   @param      PackageList     List of packages looking-up in
 651 #   @param      Inffile         The driver file
 652 #
 653 #   @retval     GuidValue   if the CName is found in any given package
 654 #   @retval     None        if the CName is not found in all given packages
 655 #
 656 def ProtocolValue(CName, PackageList, Inffile = None):
 657     for P in PackageList:
 658         ProtocolKeys = P.Protocols.keys()
 659         if Inffile and P._PrivateProtocols:
 660             if not Inffile.startswith(P.MetaFile.Dir):
 661                 ProtocolKeys = [x for x in P.Protocols if x not in P._PrivateProtocols]
 662         if CName in ProtocolKeys:
 663             return P.Protocols[CName]
 664     return None
 665
 666 ## Get PPI value from given packages
 667 #
 668 #   @param      CName           The CName of the GUID
 669 #   @param      PackageList     List of packages looking-up in
 670 #   @param      Inffile         The driver file
 671 #
 672 #   @retval     GuidValue   if the CName is found in any given package
 673 #   @retval     None        if the CName is not found in all given packages
 674 #
 675 def PpiValue(CName, PackageList, Inffile = None):
 676     for P in PackageList:
 677         PpiKeys = P.Ppis.keys()
 678         if Inffile and P._PrivatePpis:
 679             if not Inffile.startswith(P.MetaFile.Dir):
 680                 PpiKeys = [x for x in P.Ppis if x not in P._PrivatePpis]
 681         if CName in PpiKeys:
 682             return P.Ppis[CName]
 683     return None
 684
 685 ## A string template class
 686 #
 687 #  This class implements a template for string replacement. A string template
 688 #  looks like following
 689 #
 690 #       ${BEGIN} other_string ${placeholder_name} other_string ${END}
 691 #
 692 #  The string between ${BEGIN} and ${END} will be repeated as many times as the
 693 #  length of "placeholder_name", which is a list passed through a dict. The
 694 #  "placeholder_name" is the key name of the dict. The ${BEGIN} and ${END} can
 695 #  be not used and, in this case, the "placeholder_name" must not a list and it
 696 #  will just be replaced once.
 697 #
 698 class TemplateString(object):
 699     _REPEAT_START_FLAG = "BEGIN"
 700     _REPEAT_END_FLAG = "END"
 701
 702     class Section(object):
 703         _LIST_TYPES = [type([]), type(set()), type((0,))]
 704
 705         def __init__(self, TemplateSection, PlaceHolderList):
 706             self._Template = TemplateSection
 707             self._PlaceHolderList = []
 708
 709             # Split the section into sub-sections according to the position of placeholders
 710             if PlaceHolderList:
 711                 self._SubSectionList = []
 712                 SubSectionStart = 0
 713                 #
 714                 # The placeholders passed in must be in the format of
 715                 #
 716                 #   PlaceHolderName, PlaceHolderStartPoint, PlaceHolderEndPoint
 717                 #
 718                 for PlaceHolder, Start, End in PlaceHolderList:
 719                     self._SubSectionList.append(TemplateSection[SubSectionStart:Start])
 720                     self._SubSectionList.append(TemplateSection[Start:End])
 721                     self._PlaceHolderList.append(PlaceHolder)
 722                     SubSectionStart = End
 723                 if SubSectionStart < len(TemplateSection):
 724                     self._SubSectionList.append(TemplateSection[SubSectionStart:])
 725             else:
 726                 self._SubSectionList = [TemplateSection]
 727
 728         def __str__(self):
 729             return self._Template + " : " + str(self._PlaceHolderList)
 730
 731         def Instantiate(self, PlaceHolderValues):
 732             RepeatTime = -1
 733             RepeatPlaceHolders = {}
 734             NonRepeatPlaceHolders = {}
 735
 736             for PlaceHolder in self._PlaceHolderList:
 737                 if PlaceHolder not in PlaceHolderValues:
 738                     continue
 739                 Value = PlaceHolderValues[PlaceHolder]
 740                 if type(Value) in self._LIST_TYPES:
 741                     if RepeatTime < 0:
 742                         RepeatTime = len(Value)
 743                     elif RepeatTime != len(Value):
 744                         EdkLogger.error(
 745                                     "TemplateString",
 746                                     PARAMETER_INVALID,
 747                                     "${%s} has different repeat time from others!" % PlaceHolder,
 748                                     ExtraData=str(self._Template)
 749                                     )
 750                     RepeatPlaceHolders["${%s}" % PlaceHolder] = Value
 751                 else:
 752                     NonRepeatPlaceHolders["${%s}" % PlaceHolder] = Value
 753
 754             if NonRepeatPlaceHolders:
 755                 StringList = []
 756                 for S in self._SubSectionList:
 757                     if S not in NonRepeatPlaceHolders:
 758                         StringList.append(S)
 759                     else:
 760                         StringList.append(str(NonRepeatPlaceHolders[S]))
 761             else:
 762                 StringList = self._SubSectionList
 763
 764             if RepeatPlaceHolders:
 765                 TempStringList = []
 766                 for Index in range(RepeatTime):
 767                     for S in StringList:
 768                         if S not in RepeatPlaceHolders:
 769                             TempStringList.append(S)
 770                         else:
 771                             TempStringList.append(str(RepeatPlaceHolders[S][Index]))
 772                 StringList = TempStringList
 773
 774             return "".join(StringList)
 775
 776     ## Constructor
 777     def __init__(self, Template=None):
 778         self.String = ''
 779         self.IsBinary = False
 780         self._Template = Template
 781         self._TemplateSectionList = self._Parse(Template)
 782
 783     ## str() operator
 784     #
 785     #   @retval     string  The string replaced
 786     #
 787     def __str__(self):
 788         return self.String
 789
 790     ## Split the template string into fragments per the ${BEGIN} and ${END} flags
 791     #
 792     #   @retval     list    A list of TemplateString.Section objects
 793     #
 794     def _Parse(self, Template):
 795         SectionStart = 0
 796         SearchFrom = 0
 797         MatchEnd = 0
 798         PlaceHolderList = []
 799         TemplateSectionList = []
 800         while Template:
 801             MatchObj = gPlaceholderPattern.search(Template, SearchFrom)
 802             if not MatchObj:
 803                 if MatchEnd <= len(Template):
 804                     TemplateSection = TemplateString.Section(Template[SectionStart:], PlaceHolderList)
 805                     TemplateSectionList.append(TemplateSection)
 806                 break
 807
 808             MatchString = MatchObj.group(1)
 809             MatchStart = MatchObj.start()
 810             MatchEnd = MatchObj.end()
 811
 812             if MatchString == self._REPEAT_START_FLAG:
 813                 if MatchStart > SectionStart:
 814                     TemplateSection = TemplateString.Section(Template[SectionStart:MatchStart], PlaceHolderList)
 815                     TemplateSectionList.append(TemplateSection)
 816                 SectionStart = MatchEnd
 817                 PlaceHolderList = []
 818             elif MatchString == self._REPEAT_END_FLAG:
 819                 TemplateSection = TemplateString.Section(Template[SectionStart:MatchStart], PlaceHolderList)
 820                 TemplateSectionList.append(TemplateSection)
 821                 SectionStart = MatchEnd
 822                 PlaceHolderList = []
 823             else:
 824                 PlaceHolderList.append((MatchString, MatchStart - SectionStart, MatchEnd - SectionStart))
 825             SearchFrom = MatchEnd
 826         return TemplateSectionList
 827
 828     ## Replace the string template with dictionary of placeholders and append it to previous one
 829     #
 830     #   @param      AppendString    The string template to append
 831     #   @param      Dictionary      The placeholder dictionaries
 832     #
 833     def Append(self, AppendString, Dictionary=None):
 834         if Dictionary:
 835             SectionList = self._Parse(AppendString)
 836             self.String += "".join(S.Instantiate(Dictionary) for S in SectionList)
 837         else:
 838             self.String += AppendString
 839
 840     ## Replace the string template with dictionary of placeholders
 841     #
 842     #   @param      Dictionary      The placeholder dictionaries
 843     #
 844     #   @retval     str             The string replaced with placeholder values
 845     #
 846     def Replace(self, Dictionary=None):
 847         return "".join(S.Instantiate(Dictionary) for S in self._TemplateSectionList)
 848
 849 ## Progress indicator class
 850 #
 851 #  This class makes use of thread to print progress on console.
 852 #
 853 class Progressor:
 854     # for avoiding deadloop
 855     _StopFlag = None
 856     _ProgressThread = None
 857     _CheckInterval = 0.25
 858
 859     ## Constructor
 860     #
 861     #   @param      OpenMessage     The string printed before progress charaters
 862     #   @param      CloseMessage    The string printed after progress charaters
 863     #   @param      ProgressChar    The charater used to indicate the progress
 864     #   @param      Interval        The interval in seconds between two progress charaters
 865     #
 866     def __init__(self, OpenMessage="", CloseMessage="", ProgressChar='.', Interval=1.0):
 867         self.PromptMessage = OpenMessage
 868         self.CodaMessage = CloseMessage
 869         self.ProgressChar = ProgressChar
 870         self.Interval = Interval
 871         if Progressor._StopFlag is None:
 872             Progressor._StopFlag = threading.Event()
 873
 874     ## Start to print progress charater
 875     #
 876     #   @param      OpenMessage     The string printed before progress charaters
 877     #
 878     def Start(self, OpenMessage=None):
 879         if OpenMessage is not None:
 880             self.PromptMessage = OpenMessage
 881         Progressor._StopFlag.clear()
 882         if Progressor._ProgressThread is None:
 883             Progressor._ProgressThread = threading.Thread(target=self._ProgressThreadEntry)
 884             Progressor._ProgressThread.setDaemon(False)
 885             Progressor._ProgressThread.start()
 886
 887     ## Stop printing progress charater
 888     #
 889     #   @param      CloseMessage    The string printed after progress charaters
 890     #
 891     def Stop(self, CloseMessage=None):
 892         OriginalCodaMessage = self.CodaMessage
 893         if CloseMessage is not None:
 894             self.CodaMessage = CloseMessage
 895         self.Abort()
 896         self.CodaMessage = OriginalCodaMessage
 897
 898     ## Thread entry method
 899     def _ProgressThreadEntry(self):
 900         sys.stdout.write(self.PromptMessage + " ")
 901         sys.stdout.flush()
 902         TimeUp = 0.0
 903         while not Progressor._StopFlag.isSet():
 904             if TimeUp <= 0.0:
 905                 sys.stdout.write(self.ProgressChar)
 906                 sys.stdout.flush()
 907                 TimeUp = self.Interval
 908             time.sleep(self._CheckInterval)
 909             TimeUp -= self._CheckInterval
 910         sys.stdout.write(" " + self.CodaMessage + "\n")
 911         sys.stdout.flush()
 912
 913     ## Abort the progress display
 914     @staticmethod
 915     def Abort():
 916         if Progressor._StopFlag is not None:
 917             Progressor._StopFlag.set()
 918         if Progressor._ProgressThread is not None:
 919             Progressor._ProgressThread.join()
 920             Progressor._ProgressThread = None
 921
 922 ## A dict which can access its keys and/or values orderly
 923 #
 924 #  The class implements a new kind of dict which its keys or values can be
 925 #  accessed in the order they are added into the dict. It guarantees the order
 926 #  by making use of an internal list to keep a copy of keys.
 927 #
 928 class sdict(IterableUserDict):
 929     ## Constructor
 930     def __init__(self):
 931         IterableUserDict.__init__(self)
 932         self._key_list = []
 933
 934     ## [] operator
 935     def __setitem__(self, key, value):
 936         if key not in self._key_list:
 937             self._key_list.append(key)
 938         IterableUserDict.__setitem__(self, key, value)
 939
 940     ## del operator
 941     def __delitem__(self, key):
 942         self._key_list.remove(key)
 943         IterableUserDict.__delitem__(self, key)
 944
 945     ## used in "for k in dict" loop to ensure the correct order
 946     def __iter__(self):
 947         return self.iterkeys()
 948
 949     ## len() support
 950     def __len__(self):
 951         return len(self._key_list)
 952
 953     ## "in" test support
 954     def __contains__(self, key):
 955         return key in self._key_list
 956
 957     ## indexof support
 958     def index(self, key):
 959         return self._key_list.index(key)
 960
 961     ## insert support
 962     def insert(self, key, newkey, newvalue, order):
 963         index = self._key_list.index(key)
 964         if order == 'BEFORE':
 965             self._key_list.insert(index, newkey)
 966             IterableUserDict.__setitem__(self, newkey, newvalue)
 967         elif order == 'AFTER':
 968             self._key_list.insert(index + 1, newkey)
 969             IterableUserDict.__setitem__(self, newkey, newvalue)
 970
 971     ## append support
 972     def append(self, sdict):
 973         for key in sdict:
 974             if key not in self._key_list:
 975                 self._key_list.append(key)
 976             IterableUserDict.__setitem__(self, key, sdict[key])
 977
 978     def has_key(self, key):
 979         return key in self._key_list
 980
 981     ## Empty the dict
 982     def clear(self):
 983         self._key_list = []
 984         IterableUserDict.clear(self)
 985
 986     ## Return a copy of keys
 987     def keys(self):
 988         keys = []
 989         for key in self._key_list:
 990             keys.append(key)
 991         return keys
 992
 993     ## Return a copy of values
 994     def values(self):
 995         values = []
 996         for key in self._key_list:
 997             values.append(self[key])
 998         return values
 999
1000     ## Return a copy of (key, value) list
1001     def items(self):
1002         items = []
1003         for key in self._key_list:
1004             items.append((key, self[key]))
1005         return items
1006
1007     ## Iteration support
1008     def iteritems(self):
1009         return iter(self.items())
1010
1011     ## Keys interation support
1012     def iterkeys(self):
1013         return iter(self.keys())
1014
1015     ## Values interation support
1016     def itervalues(self):
1017         return iter(self.values())
1018
1019     ## Return value related to a key, and remove the (key, value) from the dict
1020     def pop(self, key, *dv):
1021         value = None
1022         if key in self._key_list:
1023             value = self[key]
1024             self.__delitem__(key)
1025         elif len(dv) != 0 :
1026             value = kv[0]
1027         return value
1028
1029     ## Return (key, value) pair, and remove the (key, value) from the dict
1030     def popitem(self):
1031         key = self._key_list[-1]
1032         value = self[key]
1033         self.__delitem__(key)
1034         return key, value
1035
1036     def update(self, dict=None, **kwargs):
1037         if dict is not None:
1038             for k, v in dict.items():
1039                 self[k] = v
1040         if len(kwargs):
1041             for k, v in kwargs.items():
1042                 self[k] = v
1043
1044 ## Dictionary with restricted keys
1045 #
1046 class rdict(dict):
1047     ## Constructor
1048     def __init__(self, KeyList):
1049         for Key in KeyList:
1050             dict.__setitem__(self, Key, "")
1051
1052     ## []= operator
1053     def __setitem__(self, key, value):
1054         if key not in self:
1055             EdkLogger.error("RestrictedDict", ATTRIBUTE_SET_FAILURE, "Key [%s] is not allowed" % key,
1056                             ExtraData=", ".join(dict.keys(self)))
1057         dict.__setitem__(self, key, value)
1058
1059     ## =[] operator
1060     def __getitem__(self, key):
1061         if key not in self:
1062             return ""
1063         return dict.__getitem__(self, key)
1064
1065     ## del operator
1066     def __delitem__(self, key):
1067         EdkLogger.error("RestrictedDict", ATTRIBUTE_ACCESS_DENIED, ExtraData="del")
1068
1069     ## Empty the dict
1070     def clear(self):
1071         for Key in self:
1072             self.__setitem__(Key, "")
1073
1074     ## Return value related to a key, and remove the (key, value) from the dict
1075     def pop(self, key, *dv):
1076         EdkLogger.error("RestrictedDict", ATTRIBUTE_ACCESS_DENIED, ExtraData="pop")
1077
1078     ## Return (key, value) pair, and remove the (key, value) from the dict
1079     def popitem(self):
1080         EdkLogger.error("RestrictedDict", ATTRIBUTE_ACCESS_DENIED, ExtraData="popitem")
1081
1082 ## Dictionary using prioritized list as key
1083 #
1084 class tdict:
1085     _ListType = type([])
1086     _TupleType = type(())
1087     _Wildcard = 'COMMON'
1088     _ValidWildcardList = ['COMMON', 'DEFAULT', 'ALL', '*', 'PLATFORM']
1089
1090     def __init__(self, _Single_=False, _Level_=2):
1091         self._Level_ = _Level_
1092         self.data = {}
1093         self._Single_ = _Single_
1094
1095     # =[] operator
1096     def __getitem__(self, key):
1097         KeyType = type(key)
1098         RestKeys = None
1099         if KeyType == self._ListType or KeyType == self._TupleType:
1100             FirstKey = key[0]
1101             if len(key) > 1:
1102                 RestKeys = key[1:]
1103             elif self._Level_ > 1:
1104                 RestKeys = [self._Wildcard for i in range(0, self._Level_ - 1)]
1105         else:
1106             FirstKey = key
1107             if self._Level_ > 1:
1108                 RestKeys = [self._Wildcard for i in range(0, self._Level_ - 1)]
1109
1110         if FirstKey is None or str(FirstKey).upper() in self._ValidWildcardList:
1111             FirstKey = self._Wildcard
1112
1113         if self._Single_:
1114             return self._GetSingleValue(FirstKey, RestKeys)
1115         else:
1116             return self._GetAllValues(FirstKey, RestKeys)
1117
1118     def _GetSingleValue(self, FirstKey, RestKeys):
1119         Value = None
1120         #print "%s-%s" % (FirstKey, self._Level_) ,
1121         if self._Level_ > 1:
1122             if FirstKey == self._Wildcard:
1123                 if FirstKey in self.data:
1124                     Value = self.data[FirstKey][RestKeys]
1125                 if Value is None:
1126                     for Key in self.data:
1127                         Value = self.data[Key][RestKeys]
1128                         if Value is not None: break
1129             else:
1130                 if FirstKey in self.data:
1131                     Value = self.data[FirstKey][RestKeys]
1132                 if Value is None and self._Wildcard in self.data:
1133                     #print "Value=None"
1134                     Value = self.data[self._Wildcard][RestKeys]
1135         else:
1136             if FirstKey == self._Wildcard:
1137                 if FirstKey in self.data:
1138                     Value = self.data[FirstKey]
1139                 if Value is None:
1140                     for Key in self.data:
1141                         Value = self.data[Key]
1142                         if Value is not None: break
1143             else:
1144                 if FirstKey in self.data:
1145                     Value = self.data[FirstKey]
1146                 elif self._Wildcard in self.data:
1147                     Value = self.data[self._Wildcard]
1148         return Value
1149
1150     def _GetAllValues(self, FirstKey, RestKeys):
1151         Value = []
1152         if self._Level_ > 1:
1153             if FirstKey == self._Wildcard:
1154                 for Key in self.data:
1155                     Value += self.data[Key][RestKeys]
1156             else:
1157                 if FirstKey in self.data:
1158                     Value += self.data[FirstKey][RestKeys]
1159                 if self._Wildcard in self.data:
1160                     Value += self.data[self._Wildcard][RestKeys]
1161         else:
1162             if FirstKey == self._Wildcard:
1163                 for Key in self.data:
1164                     Value.append(self.data[Key])
1165             else:
1166                 if FirstKey in self.data:
1167                     Value.append(self.data[FirstKey])
1168                 if self._Wildcard in self.data:
1169                     Value.append(self.data[self._Wildcard])
1170         return Value
1171
1172     ## []= operator
1173     def __setitem__(self, key, value):
1174         KeyType = type(key)
1175         RestKeys = None
1176         if KeyType == self._ListType or KeyType == self._TupleType:
1177             FirstKey = key[0]
1178             if len(key) > 1:
1179                 RestKeys = key[1:]
1180             else:
1181                 RestKeys = [self._Wildcard for i in range(0, self._Level_ - 1)]
1182         else:
1183             FirstKey = key
1184             if self._Level_ > 1:
1185                 RestKeys = [self._Wildcard for i in range(0, self._Level_ - 1)]
1186
1187         if FirstKey in self._ValidWildcardList:
1188             FirstKey = self._Wildcard
1189
1190         if FirstKey not in self.data and self._Level_ > 0:
1191             self.data[FirstKey] = tdict(self._Single_, self._Level_ - 1)
1192
1193         if self._Level_ > 1:
1194             self.data[FirstKey][RestKeys] = value
1195         else:
1196             self.data[FirstKey] = value
1197
1198     def SetGreedyMode(self):
1199         self._Single_ = False
1200         if self._Level_ > 1:
1201             for Key in self.data:
1202                 self.data[Key].SetGreedyMode()
1203
1204     def SetSingleMode(self):
1205         self._Single_ = True
1206         if self._Level_ > 1:
1207             for Key in self.data:
1208                 self.data[Key].SetSingleMode()
1209
1210     def GetKeys(self, KeyIndex=0):
1211         assert KeyIndex >= 0
1212         if KeyIndex == 0:
1213             return set(self.data.keys())
1214         else:
1215             keys = set()
1216             for Key in self.data:
1217                 keys |= self.data[Key].GetKeys(KeyIndex - 1)
1218             return keys
1219
1220 def IsFieldValueAnArray (Value):
1221     Value = Value.strip()
1222     if Value.startswith(TAB_GUID) and Value.endswith(')'):
1223         return True
1224     if Value.startswith('L"') and Value.endswith('"')  and len(list(Value[2:-1])) > 1:
1225         return True
1226     if Value[0] == '"' and Value[-1] == '"' and len(list(Value[1:-1])) > 1:
1227         return True
1228     if Value[0] == '{' and Value[-1] == '}':
1229         return True
1230     if Value.startswith("L'") and Value.endswith("'") and len(list(Value[2:-1])) > 1:
1231         return True
1232     if Value[0] == "'" and Value[-1] == "'" and len(list(Value[1:-1])) > 1:
1233         return True
1234     return False
1235
1236 def AnalyzePcdExpression(Setting):
1237     Setting = Setting.strip()
1238     # There might be escaped quote in a string: \", \\\" , \', \\\'
1239     Data = Setting
1240     # There might be '|' in string and in ( ... | ... ), replace it with '-'
1241     NewStr = ''
1242     InSingleQuoteStr = False
1243     InDoubleQuoteStr = False
1244     Pair = 0
1245     for Index, ch in enumerate(Data):
1246         if ch == '"' and not InSingleQuoteStr:
1247             if Data[Index - 1] != '\\':
1248                 InDoubleQuoteStr = not InDoubleQuoteStr
1249         elif ch == "'" and not InDoubleQuoteStr:
1250             if Data[Index - 1] != '\\':
1251                 InSingleQuoteStr = not InSingleQuoteStr
1252         elif ch == '(' and not (InSingleQuoteStr or InDoubleQuoteStr):
1253             Pair += 1
1254         elif ch == ')' and not (InSingleQuoteStr or InDoubleQuoteStr):
1255             Pair -= 1
1256
1257         if (Pair > 0 or InSingleQuoteStr or InDoubleQuoteStr) and ch == TAB_VALUE_SPLIT:
1258             NewStr += '-'
1259         else:
1260             NewStr += ch
1261     FieldList = []
1262     StartPos = 0
1263     while True:
1264         Pos = NewStr.find(TAB_VALUE_SPLIT, StartPos)
1265         if Pos < 0:
1266             FieldList.append(Setting[StartPos:].strip())
1267             break
1268         FieldList.append(Setting[StartPos:Pos].strip())
1269         StartPos = Pos + 1
1270
1271     return FieldList
1272
1273 def ParseDevPathValue (Value):
1274     if '\\' in Value:
1275         Value.replace('\\', '/').replace(' ', '')
1276
1277     Cmd = 'DevicePath ' + '"' + Value + '"'
1278     try:
1279         p = subprocess.Popen(Cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
1280         out, err = p.communicate()
1281     except Exception as X:
1282         raise BadExpression("DevicePath: %s" % (str(X)) )
1283     finally:
1284         subprocess._cleanup()
1285         p.stdout.close()
1286         p.stderr.close()
1287     if err:
1288         raise BadExpression("DevicePath: %s" % str(err))
1289     Size = len(out.split())
1290     out = ','.join(out.split())
1291     return '{' + out + '}', Size
1292
1293 def ParseFieldValue (Value):
1294     if type(Value) == type(0):
1295         return Value, (Value.bit_length() + 7) / 8
1296     if type(Value) <> type(''):
1297         raise BadExpression('Type %s is %s' %(Value, type(Value)))
1298     Value = Value.strip()
1299     if Value.startswith(TAB_UINT8) and Value.endswith(')'):
1300         Value, Size = ParseFieldValue(Value.split('(', 1)[1][:-1])
1301         if Size > 1:
1302             raise BadExpression('Value (%s) Size larger than %d' %(Value, Size))
1303         return Value, 1
1304     if Value.startswith(TAB_UINT16) and Value.endswith(')'):
1305         Value, Size = ParseFieldValue(Value.split('(', 1)[1][:-1])
1306         if Size > 2:
1307             raise BadExpression('Value (%s) Size larger than %d' %(Value, Size))
1308         return Value, 2
1309     if Value.startswith(TAB_UINT32) and Value.endswith(')'):
1310         Value, Size = ParseFieldValue(Value.split('(', 1)[1][:-1])
1311         if Size > 4:
1312             raise BadExpression('Value (%s) Size larger than %d' %(Value, Size))
1313         return Value, 4
1314     if Value.startswith(TAB_UINT64) and Value.endswith(')'):
1315         Value, Size = ParseFieldValue(Value.split('(', 1)[1][:-1])
1316         if Size > 8:
1317             raise BadExpression('Value (%s) Size larger than %d' % (Value, Size))
1318         return Value, 8
1319     if Value.startswith(TAB_GUID) and Value.endswith(')'):
1320         Value = Value.split('(', 1)[1][:-1].strip()
1321         if Value[0] == '{' and Value[-1] == '}':
1322             TmpValue = GuidStructureStringToGuidString(Value)
1323             if len(TmpValue) == 0:
1324                 raise BadExpression("Invalid GUID value string %s" % Value)
1325             Value = TmpValue
1326         if Value[0] == '"' and Value[-1] == '"':
1327             Value = Value[1:-1]
1328         try:
1329             Value = "'" + uuid.UUID(Value).get_bytes_le() + "'"
1330         except ValueError as Message:
1331             raise BadExpression(Message)
1332         Value, Size = ParseFieldValue(Value)
1333         return Value, 16
1334     if Value.startswith('L"') and Value.endswith('"'):
1335         # Unicode String
1336         # translate escape character
1337         Value = Value[1:]
1338         try:
1339             Value = eval(Value)
1340         except:
1341             Value = Value[1:-1]
1342         List = list(Value)
1343         List.reverse()
1344         Value = 0
1345         for Char in List:
1346             Value = (Value << 16) | ord(Char)
1347         return Value, (len(List) + 1) * 2
1348     if Value.startswith('"') and Value.endswith('"'):
1349         # ASCII String
1350         # translate escape character
1351         try:
1352             Value = eval(Value)
1353         except:
1354             Value = Value[1:-1]
1355         List = list(Value)
1356         List.reverse()
1357         Value = 0
1358         for Char in List:
1359             Value = (Value << 8) | ord(Char)
1360         return Value, len(List) + 1
1361     if Value.startswith("L'") and Value.endswith("'"):
1362         # Unicode Character Constant
1363         # translate escape character
1364         Value = Value[1:]
1365         try:
1366             Value = eval(Value)
1367         except:
1368             Value = Value[1:-1]
1369         List = list(Value)
1370         if len(List) == 0:
1371             raise BadExpression('Length %s is %s' % (Value, len(List)))
1372         List.reverse()
1373         Value = 0
1374         for Char in List:
1375             Value = (Value << 16) | ord(Char)
1376         return Value, len(List) * 2
1377     if Value.startswith("'") and Value.endswith("'"):
1378         # Character constant
1379         # translate escape character
1380         try:
1381             Value = eval(Value)
1382         except:
1383             Value = Value[1:-1]
1384         List = list(Value)
1385         if len(List) == 0:
1386             raise BadExpression('Length %s is %s' % (Value, len(List)))
1387         List.reverse()
1388         Value = 0
1389         for Char in List:
1390             Value = (Value << 8) | ord(Char)
1391         return Value, len(List)
1392     if Value.startswith('{') and Value.endswith('}'):
1393         # Byte array
1394         Value = Value[1:-1]
1395         List = [Item.strip() for Item in Value.split(',')]
1396         List.reverse()
1397         Value = 0
1398         RetSize = 0
1399         for Item in List:
1400             ItemValue, Size = ParseFieldValue(Item)
1401             RetSize += Size
1402             for I in range(Size):
1403                 Value = (Value << 8) | ((ItemValue >> 8 * I) & 0xff)
1404         return Value, RetSize
1405     if Value.startswith('DEVICE_PATH(') and Value.endswith(')'):
1406         Value = Value.replace("DEVICE_PATH(", '').rstrip(')')
1407         Value = Value.strip().strip('"')
1408         return ParseDevPathValue(Value)
1409     if Value.lower().startswith('0x'):
1410         Value = int(Value, 16)
1411         if Value == 0:
1412             return 0, 1
1413         return Value, (Value.bit_length() + 7) / 8
1414     if Value[0].isdigit():
1415         Value = int(Value, 10)
1416         if Value == 0:
1417             return 0, 1
1418         return Value, (Value.bit_length() + 7) / 8
1419     if Value.lower() == 'true':
1420         return 1, 1
1421     if Value.lower() == 'false':
1422         return 0, 1
1423     return Value, 1
1424
1425 ## AnalyzeDscPcd
1426 #
1427 #  Analyze DSC PCD value, since there is no data type info in DSC
1428 #  This fuction is used to match functions (AnalyzePcdData) used for retrieving PCD value from database
1429 #  1. Feature flag: TokenSpace.PcdCName|PcdValue
1430 #  2. Fix and Patch:TokenSpace.PcdCName|PcdValue[|MaxSize]
1431 #  3. Dynamic default:
1432 #     TokenSpace.PcdCName|PcdValue[|VOID*[|MaxSize]]
1433 #     TokenSpace.PcdCName|PcdValue
1434 #  4. Dynamic VPD:
1435 #     TokenSpace.PcdCName|VpdOffset[|VpdValue]
1436 #     TokenSpace.PcdCName|VpdOffset[|MaxSize[|VpdValue]]
1437 #  5. Dynamic HII:
1438 #     TokenSpace.PcdCName|HiiString|VaiableGuid|VariableOffset[|HiiValue]
1439 #  PCD value needs to be located in such kind of string, and the PCD value might be an expression in which
1440 #    there might have "|" operator, also in string value.
1441 #
1442 #  @param Setting: String contain information described above with "TokenSpace.PcdCName|" stripped
1443 #  @param PcdType: PCD type: feature, fixed, dynamic default VPD HII
1444 #  @param DataType: The datum type of PCD: VOID*, UNIT, BOOL
1445 #  @retval:
1446 #    ValueList: A List contain fields described above
1447 #    IsValid:   True if conforming EBNF, otherwise False
1448 #    Index:     The index where PcdValue is in ValueList
1449 #
1450 def AnalyzeDscPcd(Setting, PcdType, DataType=''):
1451     FieldList = AnalyzePcdExpression(Setting)
1452
1453     IsValid = True
1454     if PcdType in (MODEL_PCD_FIXED_AT_BUILD, MODEL_PCD_PATCHABLE_IN_MODULE, MODEL_PCD_FEATURE_FLAG):
1455         Value = FieldList[0]
1456         Size = ''
1457         if len(FieldList) > 1:
1458             if FieldList[1].upper().startswith("0X") or FieldList[1].isdigit():
1459                 Size = FieldList[1]
1460             else:
1461                 DataType = FieldList[1]
1462
1463         if len(FieldList) > 2:
1464             Size = FieldList[2]
1465         if DataType == "":
1466             IsValid = (len(FieldList) <= 1)
1467         else:
1468             IsValid = (len(FieldList) <= 3)
1469 #         Value, Size = ParseFieldValue(Value)
1470         if Size:
1471             try:
1472                 int(Size,16) if Size.upper().startswith("0X") else int(Size)
1473             except:
1474                 IsValid = False
1475                 Size = -1
1476         return [str(Value), '', str(Size)], IsValid, 0
1477     elif PcdType in (MODEL_PCD_DYNAMIC_DEFAULT, MODEL_PCD_DYNAMIC_EX_DEFAULT):
1478         Value = FieldList[0]
1479         Size = Type = ''
1480         if len(FieldList) > 1:
1481             Type = FieldList[1]
1482         else:
1483             Type = DataType
1484         if len(FieldList) > 2:
1485             Size = FieldList[2]
1486         if DataType == "":
1487             IsValid = (len(FieldList) <= 1)
1488         else:
1489             IsValid = (len(FieldList) <= 3)
1490
1491         if Size:
1492             try:
1493                 int(Size,16) if Size.upper().startswith("0X") else int(Size)
1494             except:
1495                 IsValid = False
1496                 Size = -1
1497         return [Value, Type, str(Size)], IsValid, 0
1498     elif PcdType in (MODEL_PCD_DYNAMIC_VPD, MODEL_PCD_DYNAMIC_EX_VPD):
1499         VpdOffset = FieldList[0]
1500         Value = Size = ''
1501         if not DataType == TAB_VOID:
1502             if len(FieldList) > 1:
1503                 Value = FieldList[1]
1504         else:
1505             if len(FieldList) > 1:
1506                 Size = FieldList[1]
1507             if len(FieldList) > 2:
1508                 Value = FieldList[2]
1509         if DataType == "":
1510             IsValid = (len(FieldList) <= 1)
1511         else:
1512             IsValid = (len(FieldList) <= 3)
1513         if Size:
1514             try:
1515                 int(Size,16) if Size.upper().startswith("0X") else int(Size)
1516             except:
1517                 IsValid = False
1518                 Size = -1
1519         return [VpdOffset, str(Size), Value], IsValid, 2
1520     elif PcdType in (MODEL_PCD_DYNAMIC_HII, MODEL_PCD_DYNAMIC_EX_HII):
1521         HiiString = FieldList[0]
1522         Guid = Offset = Value = Attribute = ''
1523         if len(FieldList) > 1:
1524             Guid = FieldList[1]
1525         if len(FieldList) > 2:
1526             Offset = FieldList[2]
1527         if len(FieldList) > 3:
1528             Value = FieldList[3]
1529         if len(FieldList) > 4:
1530             Attribute = FieldList[4]
1531         IsValid = (3 <= len(FieldList) <= 5)
1532         return [HiiString, Guid, Offset, Value, Attribute], IsValid, 3
1533     return [], False, 0
1534
1535 ## AnalyzePcdData
1536 #
1537 #  Analyze the pcd Value, Datum type and TokenNumber.
1538 #  Used to avoid split issue while the value string contain "|" character
1539 #
1540 #  @param[in] Setting:  A String contain value/datum type/token number information;
1541 #
1542 #  @retval   ValueList: A List contain value, datum type and toke number.
1543 #
1544 def AnalyzePcdData(Setting):
1545     ValueList = ['', '', '']
1546
1547     ValueRe = re.compile(r'^\s*L?\".*\|.*\"')
1548     PtrValue = ValueRe.findall(Setting)
1549
1550     ValueUpdateFlag = False
1551
1552     if len(PtrValue) >= 1:
1553         Setting = re.sub(ValueRe, '', Setting)
1554         ValueUpdateFlag = True
1555
1556     TokenList = Setting.split(TAB_VALUE_SPLIT)
1557     ValueList[0:len(TokenList)] = TokenList
1558
1559     if ValueUpdateFlag:
1560         ValueList[0] = PtrValue[0]
1561
1562     return ValueList
1563
1564 ## check format of PCD value against its the datum type
1565 #
1566 # For PCD value setting
1567 #
1568 def CheckPcdDatum(Type, Value):
1569     if Type == TAB_VOID:
1570         ValueRe = re.compile(r'\s*L?\".*\"\s*$')
1571         if not (((Value.startswith('L"') or Value.startswith('"')) and Value.endswith('"'))
1572                 or (Value.startswith('{') and Value.endswith('}')) or (Value.startswith("L'") or Value.startswith("'") and Value.endswith("'"))
1573                ):
1574             return False, "Invalid value [%s] of type [%s]; must be in the form of {...} for array"\
1575                           ", \"...\" or \'...\' for string, L\"...\" or L\'...\' for unicode string" % (Value, Type)
1576         elif ValueRe.match(Value):
1577             # Check the chars in UnicodeString or CString is printable
1578             if Value.startswith("L"):
1579                 Value = Value[2:-1]
1580             else:
1581                 Value = Value[1:-1]
1582             Printset = set(string.printable)
1583             Printset.remove(TAB_PRINTCHAR_VT)
1584             Printset.add(TAB_PRINTCHAR_BS)
1585             Printset.add(TAB_PRINTCHAR_NUL)
1586             if not set(Value).issubset(Printset):
1587                 PrintList = list(Printset)
1588                 PrintList.sort()
1589                 return False, "Invalid PCD string value of type [%s]; must be printable chars %s." % (Type, PrintList)
1590     elif Type == 'BOOLEAN':
1591         if Value not in ['TRUE', 'True', 'true', '0x1', '0x01', '1', 'FALSE', 'False', 'false', '0x0', '0x00', '0']:
1592             return False, "Invalid value [%s] of type [%s]; must be one of TRUE, True, true, 0x1, 0x01, 1"\
1593                           ", FALSE, False, false, 0x0, 0x00, 0" % (Value, Type)
1594     elif Type in [TAB_UINT8, TAB_UINT16, TAB_UINT32, TAB_UINT64]:
1595         try:
1596             Value = long(Value, 0)
1597         except:
1598             return False, "Invalid value [%s] of type [%s];"\
1599                           " must be a hexadecimal, decimal or octal in C language format." % (Value, Type)
1600     else:
1601         return True, "StructurePcd"
1602
1603     return True, ""
1604
1605 ## Split command line option string to list
1606 #
1607 # subprocess.Popen needs the args to be a sequence. Otherwise there's problem
1608 # in non-windows platform to launch command
1609 #
1610 def SplitOption(OptionString):
1611     OptionList = []
1612     LastChar = " "
1613     OptionStart = 0
1614     QuotationMark = ""
1615     for Index in range(0, len(OptionString)):
1616         CurrentChar = OptionString[Index]
1617         if CurrentChar in ['"', "'"]:
1618             if QuotationMark == CurrentChar:
1619                 QuotationMark = ""
1620             elif QuotationMark == "":
1621                 QuotationMark = CurrentChar
1622             continue
1623         elif QuotationMark:
1624             continue
1625
1626         if CurrentChar in ["/", "-"] and LastChar in [" ", "\t", "\r", "\n"]:
1627             if Index > OptionStart:
1628                 OptionList.append(OptionString[OptionStart:Index - 1])
1629             OptionStart = Index
1630         LastChar = CurrentChar
1631     OptionList.append(OptionString[OptionStart:])
1632     return OptionList
1633
1634 def CommonPath(PathList):
1635     P1 = min(PathList).split(os.path.sep)
1636     P2 = max(PathList).split(os.path.sep)
1637     for Index in xrange(min(len(P1), len(P2))):
1638         if P1[Index] != P2[Index]:
1639             return os.path.sep.join(P1[:Index])
1640     return os.path.sep.join(P1)
1641
1642 #
1643 # Convert string to C format array
1644 #
1645 def ConvertStringToByteArray(Value):
1646     Value = Value.strip()
1647     if not Value:
1648         return None
1649     if Value[0] == '{':
1650         if not Value.endswith('}'):
1651             return None
1652         Value = Value.replace(' ', '').replace('{', '').replace('}', '')
1653         ValFields = Value.split(',')
1654         try:
1655             for Index in range(len(ValFields)):
1656                 ValFields[Index] = str(int(ValFields[Index], 0))
1657         except ValueError:
1658             return None
1659         Value = '{' + ','.join(ValFields) + '}'
1660         return Value
1661
1662     Unicode = False
1663     if Value.startswith('L"'):
1664         if not Value.endswith('"'):
1665             return None
1666         Value = Value[1:]
1667         Unicode = True
1668     elif not Value.startswith('"') or not Value.endswith('"'):
1669         return None
1670
1671     Value = eval(Value)         # translate escape character
1672     NewValue = '{'
1673     for Index in range(0,len(Value)):
1674         if Unicode:
1675             NewValue = NewValue + str(ord(Value[Index]) % 0x10000) + ','
1676         else:
1677             NewValue = NewValue + str(ord(Value[Index]) % 0x100) + ','
1678     Value = NewValue + '0}'
1679     return Value
1680
1681 class PathClass(object):
1682     def __init__(self, File='', Root='', AlterRoot='', Type='', IsBinary=False,
1683                  Arch='COMMON', ToolChainFamily='', Target='', TagName='', ToolCode=''):
1684         self.Arch = Arch
1685         self.File = str(File)
1686         if os.path.isabs(self.File):
1687             self.Root = ''
1688             self.AlterRoot = ''
1689         else:
1690             self.Root = str(Root)
1691             self.AlterRoot = str(AlterRoot)
1692
1693         # Remove any '.' and '..' in path
1694         if self.Root:
1695             self.Root = mws.getWs(self.Root, self.File)
1696             self.Path = os.path.normpath(os.path.join(self.Root, self.File))
1697             self.Root = os.path.normpath(CommonPath([self.Root, self.Path]))
1698             # eliminate the side-effect of 'C:'
1699             if self.Root[-1] == ':':
1700                 self.Root += os.path.sep
1701             # file path should not start with path separator
1702             if self.Root[-1] == os.path.sep:
1703                 self.File = self.Path[len(self.Root):]
1704             else:
1705                 self.File = self.Path[len(self.Root) + 1:]
1706         else:
1707             self.Path = os.path.normpath(self.File)
1708
1709         self.SubDir, self.Name = os.path.split(self.File)
1710         self.BaseName, self.Ext = os.path.splitext(self.Name)
1711
1712         if self.Root:
1713             if self.SubDir:
1714                 self.Dir = os.path.join(self.Root, self.SubDir)
1715             else:
1716                 self.Dir = self.Root
1717         else:
1718             self.Dir = self.SubDir
1719
1720         if IsBinary:
1721             self.Type = Type
1722         else:
1723             self.Type = self.Ext.lower()
1724
1725         self.IsBinary = IsBinary
1726         self.Target = Target
1727         self.TagName = TagName
1728         self.ToolCode = ToolCode
1729         self.ToolChainFamily = ToolChainFamily
1730
1731         self._Key = None
1732
1733     ## Convert the object of this class to a string
1734     #
1735     #  Convert member Path of the class to a string
1736     #
1737     #  @retval string Formatted String
1738     #
1739     def __str__(self):
1740         return self.Path
1741
1742     ## Override __eq__ function
1743     #
1744     # Check whether PathClass are the same
1745     #
1746     # @retval False The two PathClass are different
1747     # @retval True  The two PathClass are the same
1748     #
1749     def __eq__(self, Other):
1750         if type(Other) == type(self):
1751             return self.Path == Other.Path
1752         else:
1753             return self.Path == str(Other)
1754
1755     ## Override __cmp__ function
1756     #
1757     # Customize the comparsion operation of two PathClass
1758     #
1759     # @retval 0     The two PathClass are different
1760     # @retval -1    The first PathClass is less than the second PathClass
1761     # @retval 1     The first PathClass is Bigger than the second PathClass
1762     def __cmp__(self, Other):
1763         if type(Other) == type(self):
1764             OtherKey = Other.Path
1765         else:
1766             OtherKey = str(Other)
1767
1768         SelfKey = self.Path
1769         if SelfKey == OtherKey:
1770             return 0
1771         elif SelfKey > OtherKey:
1772             return 1
1773         else:
1774             return -1
1775
1776     ## Override __hash__ function
1777     #
1778     # Use Path as key in hash table
1779     #
1780     # @retval string Key for hash table
1781     #
1782     def __hash__(self):
1783         return hash(self.Path)
1784
1785     def _GetFileKey(self):
1786         if self._Key is None:
1787             self._Key = self.Path.upper()   # + self.ToolChainFamily + self.TagName + self.ToolCode + self.Target
1788         return self._Key
1789
1790     def _GetTimeStamp(self):
1791         return os.stat(self.Path)[8]
1792
1793     def Validate(self, Type='', CaseSensitive=True):
1794         if GlobalData.gCaseInsensitive:
1795             CaseSensitive = False
1796         if Type and Type.lower() != self.Type:
1797             return FILE_TYPE_MISMATCH, '%s (expect %s but got %s)' % (self.File, Type, self.Type)
1798
1799         RealFile, RealRoot = RealPath2(self.File, self.Root, self.AlterRoot)
1800         if not RealRoot and not RealFile:
1801             RealFile = self.File
1802             if self.AlterRoot:
1803                 RealFile = os.path.join(self.AlterRoot, self.File)
1804             elif self.Root:
1805                 RealFile = os.path.join(self.Root, self.File)
1806             if len (mws.getPkgPath()) == 0:
1807                 return FILE_NOT_FOUND, os.path.join(self.AlterRoot, RealFile)
1808             else:
1809                 return FILE_NOT_FOUND, "%s is not found in packages path:\n\t%s" % (self.File, '\n\t'.join(mws.getPkgPath()))
1810
1811         ErrorCode = 0
1812         ErrorInfo = ''
1813         if RealRoot != self.Root or RealFile != self.File:
1814             if CaseSensitive and (RealFile != self.File or (RealRoot != self.Root and RealRoot != self.AlterRoot)):
1815                 ErrorCode = FILE_CASE_MISMATCH
1816                 ErrorInfo = self.File + '\n\t' + RealFile + " [in file system]"
1817
1818             self.SubDir, self.Name = os.path.split(RealFile)
1819             self.BaseName, self.Ext = os.path.splitext(self.Name)
1820             if self.SubDir:
1821                 self.Dir = os.path.join(RealRoot, self.SubDir)
1822             else:
1823                 self.Dir = RealRoot
1824             self.File = RealFile
1825             self.Root = RealRoot
1826             self.Path = os.path.join(RealRoot, RealFile)
1827         return ErrorCode, ErrorInfo
1828
1829     Key = property(_GetFileKey)
1830     TimeStamp = property(_GetTimeStamp)
1831
1832 ## Parse PE image to get the required PE informaion.
1833 #
1834 class PeImageClass():
1835     ## Constructor
1836     #
1837     #   @param  File FilePath of PeImage
1838     #
1839     def __init__(self, PeFile):
1840         self.FileName   = PeFile
1841         self.IsValid    = False
1842         self.Size       = 0
1843         self.EntryPoint = 0
1844         self.SectionAlignment  = 0
1845         self.SectionHeaderList = []
1846         self.ErrorInfo = ''
1847         try:
1848             PeObject = open(PeFile, 'rb')
1849         except:
1850             self.ErrorInfo = self.FileName + ' can not be found\n'
1851             return
1852         # Read DOS header
1853         ByteArray = array.array('B')
1854         ByteArray.fromfile(PeObject, 0x3E)
1855         ByteList = ByteArray.tolist()
1856         # DOS signature should be 'MZ'
1857         if self._ByteListToStr (ByteList[0x0:0x2]) != 'MZ':
1858             self.ErrorInfo = self.FileName + ' has no valid DOS signature MZ'
1859             return
1860
1861         # Read 4 byte PE Signature
1862         PeOffset = self._ByteListToInt(ByteList[0x3C:0x3E])
1863         PeObject.seek(PeOffset)
1864         ByteArray = array.array('B')
1865         ByteArray.fromfile(PeObject, 4)
1866         # PE signature should be 'PE\0\0'
1867         if ByteArray.tostring() != 'PE\0\0':
1868             self.ErrorInfo = self.FileName + ' has no valid PE signature PE00'
1869             return
1870
1871         # Read PE file header
1872         ByteArray = array.array('B')
1873         ByteArray.fromfile(PeObject, 0x14)
1874         ByteList = ByteArray.tolist()
1875         SecNumber = self._ByteListToInt(ByteList[0x2:0x4])
1876         if SecNumber == 0:
1877             self.ErrorInfo = self.FileName + ' has no section header'
1878             return
1879
1880         # Read PE optional header
1881         OptionalHeaderSize = self._ByteListToInt(ByteArray[0x10:0x12])
1882         ByteArray = array.array('B')
1883         ByteArray.fromfile(PeObject, OptionalHeaderSize)
1884         ByteList = ByteArray.tolist()
1885         self.EntryPoint       = self._ByteListToInt(ByteList[0x10:0x14])
1886         self.SectionAlignment = self._ByteListToInt(ByteList[0x20:0x24])
1887         self.Size             = self._ByteListToInt(ByteList[0x38:0x3C])
1888
1889         # Read each Section Header
1890         for Index in range(SecNumber):
1891             ByteArray = array.array('B')
1892             ByteArray.fromfile(PeObject, 0x28)
1893             ByteList = ByteArray.tolist()
1894             SecName  = self._ByteListToStr(ByteList[0:8])
1895             SecVirtualSize = self._ByteListToInt(ByteList[8:12])
1896             SecRawAddress  = self._ByteListToInt(ByteList[20:24])
1897             SecVirtualAddress = self._ByteListToInt(ByteList[12:16])
1898             self.SectionHeaderList.append((SecName, SecVirtualAddress, SecRawAddress, SecVirtualSize))
1899         self.IsValid = True
1900         PeObject.close()
1901
1902     def _ByteListToStr(self, ByteList):
1903         String = ''
1904         for index in range(len(ByteList)):
1905             if ByteList[index] == 0:
1906                 break
1907             String += chr(ByteList[index])
1908         return String
1909
1910     def _ByteListToInt(self, ByteList):
1911         Value = 0
1912         for index in range(len(ByteList) - 1, -1, -1):
1913             Value = (Value << 8) | int(ByteList[index])
1914         return Value
1915
1916 class DefaultStore():
1917     def __init__(self,DefaultStores ):
1918
1919         self.DefaultStores = DefaultStores
1920     def DefaultStoreID(self,DefaultStoreName):
1921         for key,value in self.DefaultStores.items():
1922             if value == DefaultStoreName:
1923                 return key
1924         return None
1925     def GetDefaultDefault(self):
1926         if not self.DefaultStores or "0" in self.DefaultStores:
1927             return "0",TAB_DEFAULT_STORES_DEFAULT
1928         else:
1929             minvalue = min(int(value_str) for value_str in self.DefaultStores)
1930             return (str(minvalue), self.DefaultStores[str(minvalue)])
1931     def GetMin(self,DefaultSIdList):
1932         if not DefaultSIdList:
1933             return TAB_DEFAULT_STORES_DEFAULT
1934         storeidset = {storeid for storeid, storename in self.DefaultStores.values() if storename in DefaultSIdList}
1935         if not storeidset:
1936             return ""
1937         minid = min(storeidset )
1938         for sid,name in self.DefaultStores.values():
1939             if sid == minid:
1940                 return name
1941 class SkuClass():
1942
1943     DEFAULT = 0
1944     SINGLE = 1
1945     MULTIPLE =2
1946
1947     def __init__(self,SkuIdentifier='', SkuIds=None):
1948         if SkuIds is None:
1949             SkuIds = {}
1950
1951         for SkuName in SkuIds:
1952             SkuId = SkuIds[SkuName][0]
1953             skuid_num = int(SkuId,16) if SkuId.upper().startswith("0X") else int(SkuId)
1954             if skuid_num > 0xFFFFFFFFFFFFFFFF:
1955                 EdkLogger.error("build", PARAMETER_INVALID,
1956                             ExtraData = "SKU-ID [%s] value %s exceeds the max value of UINT64"
1957                                       % (SkuName, SkuId))
1958
1959         self.AvailableSkuIds = sdict()
1960         self.SkuIdSet = []
1961         self.SkuIdNumberSet = []
1962         self.SkuData = SkuIds
1963         self.__SkuInherit = {}
1964         self.__SkuIdentifier = SkuIdentifier
1965         if SkuIdentifier == '' or SkuIdentifier is None:
1966             self.SkuIdSet = ['DEFAULT']
1967             self.SkuIdNumberSet = ['0U']
1968         elif SkuIdentifier == 'ALL':
1969             self.SkuIdSet = SkuIds.keys()
1970             self.SkuIdNumberSet = [num[0].strip() + 'U' for num in SkuIds.values()]
1971         else:
1972             r = SkuIdentifier.split('|')
1973             self.SkuIdSet=[(r[k].strip()).upper() for k in range(len(r))]
1974             k = None
1975             try:
1976                 self.SkuIdNumberSet = [SkuIds[k][0].strip() + 'U' for k in self.SkuIdSet]
1977             except Exception:
1978                 EdkLogger.error("build", PARAMETER_INVALID,
1979                             ExtraData = "SKU-ID [%s] is not supported by the platform. [Valid SKU-ID: %s]"
1980                                       % (k, " | ".join(SkuIds.keys())))
1981         for each in self.SkuIdSet:
1982             if each in SkuIds:
1983                 self.AvailableSkuIds[each] = SkuIds[each][0]
1984             else:
1985                 EdkLogger.error("build", PARAMETER_INVALID,
1986                             ExtraData="SKU-ID [%s] is not supported by the platform. [Valid SKU-ID: %s]"
1987                                       % (each, " | ".join(SkuIds.keys())))
1988         if self.SkuUsageType != self.SINGLE:
1989             self.AvailableSkuIds.update({'DEFAULT':0, 'COMMON':0})
1990         if self.SkuIdSet:
1991             GlobalData.gSkuids = (self.SkuIdSet)
1992             if 'COMMON' in GlobalData.gSkuids:
1993                 GlobalData.gSkuids.remove('COMMON')
1994             if self.SkuUsageType == self.SINGLE:
1995                 if len(GlobalData.gSkuids) != 1:
1996                     if 'DEFAULT' in GlobalData.gSkuids:
1997                         GlobalData.gSkuids.remove('DEFAULT')
1998             if GlobalData.gSkuids:
1999                 GlobalData.gSkuids.sort()
2000
2001     def GetNextSkuId(self, skuname):
2002         if not self.__SkuInherit:
2003             self.__SkuInherit = {}
2004             for item in self.SkuData.values():
2005                 self.__SkuInherit[item[1]]=item[2] if item[2] else "DEFAULT"
2006         return self.__SkuInherit.get(skuname,"DEFAULT")
2007
2008     def GetSkuChain(self,sku):
2009         if sku == "DEFAULT":
2010             return ["DEFAULT"]
2011         skulist = [sku]
2012         nextsku = sku
2013         while 1:
2014             nextsku = self.GetNextSkuId(nextsku)
2015             skulist.append(nextsku)
2016             if nextsku == "DEFAULT":
2017                 break
2018         skulist.reverse()
2019         return skulist
2020     def SkuOverrideOrder(self):
2021         skuorderset = []
2022         for skuname in self.SkuIdSet:
2023             skuorderset.append(self.GetSkuChain(skuname))
2024
2025         skuorder = []
2026         for index in range(max(len(item) for item in skuorderset)):
2027             for subset in skuorderset:
2028                 if index > len(subset)-1:
2029                     continue
2030                 if subset[index] in skuorder:
2031                     continue
2032                 skuorder.append(subset[index])
2033
2034         return skuorder
2035
2036     def __SkuUsageType(self):
2037
2038         if self.__SkuIdentifier.upper() == "ALL":
2039             return SkuClass.MULTIPLE
2040
2041         if len(self.SkuIdSet) == 1:
2042             if self.SkuIdSet[0] == 'DEFAULT':
2043                 return SkuClass.DEFAULT
2044             else:
2045                 return SkuClass.SINGLE
2046         elif len(self.SkuIdSet) == 2:
2047             if 'DEFAULT' in self.SkuIdSet:
2048                 return SkuClass.SINGLE
2049             else:
2050                 return SkuClass.MULTIPLE
2051         else:
2052             return SkuClass.MULTIPLE
2053     def DumpSkuIdArrary(self):
2054
2055         ArrayStrList = []
2056         if self.SkuUsageType == SkuClass.SINGLE:
2057             ArrayStr = "{0x0}"
2058         else:
2059             for skuname in self.AvailableSkuIds:
2060                 if skuname == "COMMON":
2061                     continue
2062                 while skuname != "DEFAULT":
2063                     ArrayStrList.append(hex(int(self.AvailableSkuIds[skuname])))
2064                     skuname = self.GetNextSkuId(skuname)
2065                 ArrayStrList.append("0x0")
2066             ArrayStr = "{" + ",".join(ArrayStrList) +  "}"
2067         return ArrayStr
2068     def __GetAvailableSkuIds(self):
2069         return self.AvailableSkuIds
2070
2071     def __GetSystemSkuID(self):
2072         if self.__SkuUsageType() == SkuClass.SINGLE:
2073             if len(self.SkuIdSet) == 1:
2074                 return self.SkuIdSet[0]
2075             else:
2076                 return self.SkuIdSet[0] if self.SkuIdSet[0] != 'DEFAULT' else self.SkuIdSet[1]
2077         else:
2078             return 'DEFAULT'
2079     def __GetAvailableSkuIdNumber(self):
2080         return self.SkuIdNumberSet
2081     SystemSkuId = property(__GetSystemSkuID)
2082     AvailableSkuIdSet = property(__GetAvailableSkuIds)
2083     SkuUsageType = property(__SkuUsageType)
2084     AvailableSkuIdNumSet = property(__GetAvailableSkuIdNumber)
2085
2086 #
2087 # Pack a registry format GUID
2088 #
2089 def PackRegistryFormatGuid(Guid):
2090     return PackGUID(Guid.split('-'))
2091
2092 ##  Get the integer value from string like "14U" or integer like 2
2093 #
2094 #   @param      Input   The object that may be either a integer value or a string
2095 #
2096 #   @retval     Value    The integer value that the input represents
2097 #
2098 def GetIntegerValue(Input):
2099     if type(Input) in (int, long):
2100         return Input
2101     String = Input
2102     if String.endswith("U"):
2103         String = String[:-1]
2104     if String.endswith("ULL"):
2105         String = String[:-3]
2106     if String.endswith("LL"):
2107         String = String[:-2]
2108
2109     if String.startswith("0x") or String.startswith("0X"):
2110         return int(String, 16)
2111     elif String == '':
2112         return 0
2113     else:
2114         return int(String)
2115
2116 #
2117 # Pack a GUID (registry format) list into a buffer and return it
2118 #
2119 def PackGUID(Guid):
2120     return pack(PACK_PATTERN_GUID,
2121                 int(Guid[0], 16),
2122                 int(Guid[1], 16),
2123                 int(Guid[2], 16),
2124                 int(Guid[3][-4:-2], 16),
2125                 int(Guid[3][-2:], 16),
2126                 int(Guid[4][-12:-10], 16),
2127                 int(Guid[4][-10:-8], 16),
2128                 int(Guid[4][-8:-6], 16),
2129                 int(Guid[4][-6:-4], 16),
2130                 int(Guid[4][-4:-2], 16),
2131                 int(Guid[4][-2:], 16)
2132                 )
2133
2134 #
2135 # Pack a GUID (byte) list into a buffer and return it
2136 #
2137 def PackByteFormatGUID(Guid):
2138     return pack(PACK_PATTERN_GUID,
2139                 Guid[0],
2140                 Guid[1],
2141                 Guid[2],
2142                 Guid[3],
2143                 Guid[4],
2144                 Guid[5],
2145                 Guid[6],
2146                 Guid[7],
2147                 Guid[8],
2148                 Guid[9],
2149                 Guid[10],
2150                 )
2151
2152 ##
2153 #
2154 # This acts like the main() function for the script, unless it is 'import'ed into another
2155 # script.
2156 #
2157 if __name__ == '__main__':
2158     pass
2159