BaseTools/Source/Python/Common/Misc.py

   1 ## @file
   2 # Common routines used by all tools
   3 #
   4 # Copyright (c) 2007 - 2018, Intel Corporation. All rights reserved.<BR>
   5 # This program and the accompanying materials
   6 # are licensed and made available under the terms and conditions of the BSD License
   7 # which accompanies this distribution.  The full text of the license may be found at
   8 # http://opensource.org/licenses/bsd-license.php
   9 #
  10 # THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS,
  11 # WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED.
  12 #
  13
  14 ##
  15 # Import Modules
  16 #
  17 import Common.LongFilePathOs as os
  18 import sys
  19 import string
  20 import thread
  21 import threading
  22 import time
  23 import re
  24 import cPickle
  25 import array
  26 import shutil
  27 from struct import pack
  28 from UserDict import IterableUserDict
  29 from UserList import UserList
  30
  31 from Common import EdkLogger as EdkLogger
  32 from Common import GlobalData as GlobalData
  33 from DataType import *
  34 from BuildToolError import *
  35 from CommonDataClass.DataClass import *
  36 from Parsing import GetSplitValueList
  37 from Common.LongFilePathSupport import OpenLongFilePath as open
  38 from Common.MultipleWorkspace import MultipleWorkspace as mws
  39 import uuid
  40 from CommonDataClass.Exceptions import BadExpression
  41 import subprocess
  42 ## Regular expression used to find out place holders in string template
  43 gPlaceholderPattern = re.compile("\$\{([^$()\s]+)\}", re.MULTILINE | re.UNICODE)
  44
  45 ## regular expressions for map file processing
  46 startPatternGeneral = re.compile("^Start[' ']+Length[' ']+Name[' ']+Class")
  47 addressPatternGeneral = re.compile("^Address[' ']+Publics by Value[' ']+Rva\+Base")
  48 valuePatternGcc = re.compile('^([\w_\.]+) +([\da-fA-Fx]+) +([\da-fA-Fx]+)$')
  49 pcdPatternGcc = re.compile('^([\da-fA-Fx]+) +([\da-fA-Fx]+)')
  50 secReGeneral = re.compile('^([\da-fA-F]+):([\da-fA-F]+) +([\da-fA-F]+)[Hh]? +([.\w\$]+) +(\w+)', re.UNICODE)
  51
  52 ## Dictionary used to store file time stamp for quick re-access
  53 gFileTimeStampCache = {}    # {file path : file time stamp}
  54
  55 ## Dictionary used to store dependencies of files
  56 gDependencyDatabase = {}    # arch : {file path : [dependent files list]}
  57
  58 def GetVariableOffset(mapfilepath, efifilepath, varnames):
  59     """ Parse map file to get variable offset in current EFI file
  60     @param mapfilepath    Map file absolution path
  61     @param efifilepath:   EFI binary file full path
  62     @param varnames       iteratable container whose elements are variable names to be searched
  63
  64     @return List whos elements are tuple with variable name and raw offset
  65     """
  66     lines = []
  67     try:
  68         f = open(mapfilepath, 'r')
  69         lines = f.readlines()
  70         f.close()
  71     except:
  72         return None
  73
  74     if len(lines) == 0: return None
  75     firstline = lines[0].strip()
  76     if (firstline.startswith("Archive member included ") and
  77         firstline.endswith(" file (symbol)")):
  78         return _parseForGCC(lines, efifilepath, varnames)
  79     if firstline.startswith("# Path:"):
  80         return _parseForXcode(lines, efifilepath, varnames)
  81     return _parseGeneral(lines, efifilepath, varnames)
  82
  83 def _parseForXcode(lines, efifilepath, varnames):
  84     status = 0
  85     ret = []
  86     for line in lines:
  87         line = line.strip()
  88         if status == 0 and line == "# Symbols:":
  89             status = 1
  90             continue
  91         if status == 1 and len(line) != 0:
  92             for varname in varnames:
  93                 if varname in line:
  94                     # cannot pregenerate this RegEx since it uses varname from varnames.
  95                     m = re.match('^([\da-fA-FxX]+)([\s\S]*)([_]*%s)$' % varname, line)
  96                     if m is not None:
  97                         ret.append((varname, m.group(1)))
  98     return ret
  99
 100 def _parseForGCC(lines, efifilepath, varnames):
 101     """ Parse map file generated by GCC linker """
 102     status = 0
 103     sections = []
 104     varoffset = []
 105     for index, line in enumerate(lines):
 106         line = line.strip()
 107         # status machine transection
 108         if status == 0 and line == "Memory Configuration":
 109             status = 1
 110             continue
 111         elif status == 1 and line == 'Linker script and memory map':
 112             status = 2
 113             continue
 114         elif status ==2 and line == 'START GROUP':
 115             status = 3
 116             continue
 117
 118         # status handler
 119         if status == 3:
 120             m = valuePatternGcc.match(line)
 121             if m is not None:
 122                 sections.append(m.groups(0))
 123             for varname in varnames:
 124                 Str = ''
 125                 m = re.match("^.data.(%s)" % varname, line)
 126                 if m is not None:
 127                     m = re.match(".data.(%s)$" % varname, line)
 128                     if m is not None:
 129                         Str = lines[index + 1]
 130                     else:
 131                         Str = line[len(".data.%s" % varname):]
 132                     if Str:
 133                         m = pcdPatternGcc.match(Str.strip())
 134                         if m is not None:
 135                             varoffset.append((varname, int(m.groups(0)[0], 16), int(sections[-1][1], 16), sections[-1][0]))
 136
 137     if not varoffset:
 138         return []
 139     # get section information from efi file
 140     efisecs = PeImageClass(efifilepath).SectionHeaderList
 141     if efisecs is None or len(efisecs) == 0:
 142         return []
 143     #redirection
 144     redirection = 0
 145     for efisec in efisecs:
 146         for section in sections:
 147             if section[0].strip() == efisec[0].strip() and section[0].strip() == '.text':
 148                 redirection = int(section[1], 16) - efisec[1]
 149
 150     ret = []
 151     for var in varoffset:
 152         for efisec in efisecs:
 153             if var[1] >= efisec[1] and var[1] < efisec[1]+efisec[3]:
 154                 ret.append((var[0], hex(efisec[2] + var[1] - efisec[1] - redirection)))
 155     return ret
 156
 157 def _parseGeneral(lines, efifilepath, varnames):
 158     status = 0    #0 - beginning of file; 1 - PE section definition; 2 - symbol table
 159     secs  = []    # key = section name
 160     varoffset = []
 161     symRe = re.compile('^([\da-fA-F]+):([\da-fA-F]+) +([\.:\\\\\w\?@\$]+) +([\da-fA-F]+)', re.UNICODE)
 162
 163     for line in lines:
 164         line = line.strip()
 165         if startPatternGeneral.match(line):
 166             status = 1
 167             continue
 168         if addressPatternGeneral.match(line):
 169             status = 2
 170             continue
 171         if line.startswith("entry point at"):
 172             status = 3
 173             continue
 174         if status == 1 and len(line) != 0:
 175             m =  secReGeneral.match(line)
 176             assert m is not None, "Fail to parse the section in map file , line is %s" % line
 177             sec_no, sec_start, sec_length, sec_name, sec_class = m.groups(0)
 178             secs.append([int(sec_no, 16), int(sec_start, 16), int(sec_length, 16), sec_name, sec_class])
 179         if status == 2 and len(line) != 0:
 180             for varname in varnames:
 181                 m = symRe.match(line)
 182                 assert m is not None, "Fail to parse the symbol in map file, line is %s" % line
 183                 sec_no, sym_offset, sym_name, vir_addr = m.groups(0)
 184                 sec_no     = int(sec_no,     16)
 185                 sym_offset = int(sym_offset, 16)
 186                 vir_addr   = int(vir_addr,   16)
 187                 # cannot pregenerate this RegEx since it uses varname from varnames.
 188                 m2 = re.match('^[_]*(%s)' % varname, sym_name)
 189                 if m2 is not None:
 190                     # fond a binary pcd entry in map file
 191                     for sec in secs:
 192                         if sec[0] == sec_no and (sym_offset >= sec[1] and sym_offset < sec[1] + sec[2]):
 193                             varoffset.append([varname, sec[3], sym_offset, vir_addr, sec_no])
 194
 195     if not varoffset: return []
 196
 197     # get section information from efi file
 198     efisecs = PeImageClass(efifilepath).SectionHeaderList
 199     if efisecs is None or len(efisecs) == 0:
 200         return []
 201
 202     ret = []
 203     for var in varoffset:
 204         index = 0
 205         for efisec in efisecs:
 206             index = index + 1
 207             if var[1].strip() == efisec[0].strip():
 208                 ret.append((var[0], hex(efisec[2] + var[2])))
 209             elif var[4] == index:
 210                 ret.append((var[0], hex(efisec[2] + var[2])))
 211
 212     return ret
 213
 214 ## Routine to process duplicated INF
 215 #
 216 #  This function is called by following two cases:
 217 #  Case 1 in DSC:
 218 #    [components.arch]
 219 #    Pkg/module/module.inf
 220 #    Pkg/module/module.inf {
 221 #      <Defines>
 222 #        FILE_GUID = 0D1B936F-68F3-4589-AFCC-FB8B7AEBC836
 223 #    }
 224 #  Case 2 in FDF:
 225 #    INF Pkg/module/module.inf
 226 #    INF FILE_GUID = 0D1B936F-68F3-4589-AFCC-FB8B7AEBC836 Pkg/module/module.inf
 227 #
 228 #  This function copies Pkg/module/module.inf to
 229 #  Conf/.cache/0D1B936F-68F3-4589-AFCC-FB8B7AEBC836module.inf
 230 #
 231 #  @param Path     Original PathClass object
 232 #  @param BaseName New file base name
 233 #
 234 #  @retval         return the new PathClass object
 235 #
 236 def ProcessDuplicatedInf(Path, BaseName, Workspace):
 237     Filename = os.path.split(Path.File)[1]
 238     if '.' in Filename:
 239         Filename = BaseName + Path.BaseName + Filename[Filename.rfind('.'):]
 240     else:
 241         Filename = BaseName + Path.BaseName
 242
 243     #
 244     # If -N is specified on command line, cache is disabled
 245     # The directory has to be created
 246     #
 247     DbDir = os.path.split(GlobalData.gDatabasePath)[0]
 248     if not os.path.exists(DbDir):
 249         os.makedirs(DbDir)
 250     #
 251     # A temporary INF is copied to database path which must have write permission
 252     # The temporary will be removed at the end of build
 253     # In case of name conflict, the file name is
 254     # FILE_GUIDBaseName (0D1B936F-68F3-4589-AFCC-FB8B7AEBC836module.inf)
 255     #
 256     TempFullPath = os.path.join(DbDir,
 257                                 Filename)
 258     RtPath = PathClass(Path.File, Workspace)
 259     #
 260     # Modify the full path to temporary path, keep other unchanged
 261     #
 262     # To build same module more than once, the module path with FILE_GUID overridden has
 263     # the file name FILE_GUIDmodule.inf, but the relative path (self.MetaFile.File) is the real path
 264     # in DSC which is used as relative path by C files and other files in INF.
 265     # A trick was used: all module paths are PathClass instances, after the initialization
 266     # of PathClass, the PathClass.Path is overridden by the temporary INF path.
 267     #
 268     # The reason for creating a temporary INF is:
 269     # Platform.Modules which is the base to create ModuleAutoGen objects is a dictionary,
 270     # the key is the full path of INF, the value is an object to save overridden library instances, PCDs.
 271     # A different key for the same module is needed to create different output directory,
 272     # retrieve overridden PCDs, library instances.
 273     #
 274     # The BaseName is the FILE_GUID which is also the output directory name.
 275     #
 276     #
 277     RtPath.Path = TempFullPath
 278     RtPath.BaseName = BaseName
 279     #
 280     # If file exists, compare contents
 281     #
 282     if os.path.exists(TempFullPath):
 283         with open(str(Path), 'rb') as f1: Src = f1.read()
 284         with open(TempFullPath, 'rb') as f2: Dst = f2.read()
 285         if Src == Dst:
 286             return RtPath
 287     GlobalData.gTempInfs.append(TempFullPath)
 288     shutil.copy2(str(Path), TempFullPath)
 289     return RtPath
 290
 291 ## Remove temporary created INFs whose paths were saved in gTempInfs
 292 #
 293 def ClearDuplicatedInf():
 294     for File in GlobalData.gTempInfs:
 295         if os.path.exists(File):
 296             os.remove(File)
 297
 298 ## Convert GUID string in xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx style to C structure style
 299 #
 300 #   @param      Guid    The GUID string
 301 #
 302 #   @retval     string  The GUID string in C structure style
 303 #
 304 def GuidStringToGuidStructureString(Guid):
 305     GuidList = Guid.split('-')
 306     Result = '{'
 307     for Index in range(0, 3, 1):
 308         Result = Result + '0x' + GuidList[Index] + ', '
 309     Result = Result + '{0x' + GuidList[3][0:2] + ', 0x' + GuidList[3][2:4]
 310     for Index in range(0, 12, 2):
 311         Result = Result + ', 0x' + GuidList[4][Index:Index + 2]
 312     Result += '}}'
 313     return Result
 314
 315 ## Convert GUID structure in byte array to xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
 316 #
 317 #   @param      GuidValue   The GUID value in byte array
 318 #
 319 #   @retval     string      The GUID value in xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx format
 320 #
 321 def GuidStructureByteArrayToGuidString(GuidValue):
 322     guidValueString = GuidValue.lower().replace("{", "").replace("}", "").replace(" ", "").replace(";", "")
 323     guidValueList = guidValueString.split(",")
 324     if len(guidValueList) != 16:
 325         return ''
 326         #EdkLogger.error(None, None, "Invalid GUID value string %s" % GuidValue)
 327     try:
 328         return "%02x%02x%02x%02x-%02x%02x-%02x%02x-%02x%02x-%02x%02x%02x%02x%02x%02x" % (
 329                 int(guidValueList[3], 16),
 330                 int(guidValueList[2], 16),
 331                 int(guidValueList[1], 16),
 332                 int(guidValueList[0], 16),
 333                 int(guidValueList[5], 16),
 334                 int(guidValueList[4], 16),
 335                 int(guidValueList[7], 16),
 336                 int(guidValueList[6], 16),
 337                 int(guidValueList[8], 16),
 338                 int(guidValueList[9], 16),
 339                 int(guidValueList[10], 16),
 340                 int(guidValueList[11], 16),
 341                 int(guidValueList[12], 16),
 342                 int(guidValueList[13], 16),
 343                 int(guidValueList[14], 16),
 344                 int(guidValueList[15], 16)
 345                 )
 346     except:
 347         return ''
 348
 349 ## Convert GUID string in C structure style to xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
 350 #
 351 #   @param      GuidValue   The GUID value in C structure format
 352 #
 353 #   @retval     string      The GUID value in xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx format
 354 #
 355 def GuidStructureStringToGuidString(GuidValue):
 356     guidValueString = GuidValue.lower().replace("{", "").replace("}", "").replace(" ", "").replace(";", "")
 357     guidValueList = guidValueString.split(",")
 358     if len(guidValueList) != 11:
 359         return ''
 360         #EdkLogger.error(None, None, "Invalid GUID value string %s" % GuidValue)
 361     try:
 362         return "%08x-%04x-%04x-%02x%02x-%02x%02x%02x%02x%02x%02x" % (
 363                 int(guidValueList[0], 16),
 364                 int(guidValueList[1], 16),
 365                 int(guidValueList[2], 16),
 366                 int(guidValueList[3], 16),
 367                 int(guidValueList[4], 16),
 368                 int(guidValueList[5], 16),
 369                 int(guidValueList[6], 16),
 370                 int(guidValueList[7], 16),
 371                 int(guidValueList[8], 16),
 372                 int(guidValueList[9], 16),
 373                 int(guidValueList[10], 16)
 374                 )
 375     except:
 376         return ''
 377
 378 ## Convert GUID string in C structure style to xxxxxxxx_xxxx_xxxx_xxxx_xxxxxxxxxxxx
 379 #
 380 #   @param      GuidValue   The GUID value in C structure format
 381 #
 382 #   @retval     string      The GUID value in xxxxxxxx_xxxx_xxxx_xxxx_xxxxxxxxxxxx format
 383 #
 384 def GuidStructureStringToGuidValueName(GuidValue):
 385     guidValueString = GuidValue.lower().replace("{", "").replace("}", "").replace(" ", "")
 386     guidValueList = guidValueString.split(",")
 387     if len(guidValueList) != 11:
 388         EdkLogger.error(None, FORMAT_INVALID, "Invalid GUID value string [%s]" % GuidValue)
 389     return "%08x_%04x_%04x_%02x%02x_%02x%02x%02x%02x%02x%02x" % (
 390             int(guidValueList[0], 16),
 391             int(guidValueList[1], 16),
 392             int(guidValueList[2], 16),
 393             int(guidValueList[3], 16),
 394             int(guidValueList[4], 16),
 395             int(guidValueList[5], 16),
 396             int(guidValueList[6], 16),
 397             int(guidValueList[7], 16),
 398             int(guidValueList[8], 16),
 399             int(guidValueList[9], 16),
 400             int(guidValueList[10], 16)
 401             )
 402
 403 ## Create directories
 404 #
 405 #   @param      Directory   The directory name
 406 #
 407 def CreateDirectory(Directory):
 408     if Directory is None or Directory.strip() == "":
 409         return True
 410     try:
 411         if not os.access(Directory, os.F_OK):
 412             os.makedirs(Directory)
 413     except:
 414         return False
 415     return True
 416
 417 ## Remove directories, including files and sub-directories in it
 418 #
 419 #   @param      Directory   The directory name
 420 #
 421 def RemoveDirectory(Directory, Recursively=False):
 422     if Directory is None or Directory.strip() == "" or not os.path.exists(Directory):
 423         return
 424     if Recursively:
 425         CurrentDirectory = os.getcwd()
 426         os.chdir(Directory)
 427         for File in os.listdir("."):
 428             if os.path.isdir(File):
 429                 RemoveDirectory(File, Recursively)
 430             else:
 431                 os.remove(File)
 432         os.chdir(CurrentDirectory)
 433     os.rmdir(Directory)
 434
 435 ## Store content in file
 436 #
 437 #  This method is used to save file only when its content is changed. This is
 438 #  quite useful for "make" system to decide what will be re-built and what won't.
 439 #
 440 #   @param      File            The path of file
 441 #   @param      Content         The new content of the file
 442 #   @param      IsBinaryFile    The flag indicating if the file is binary file or not
 443 #
 444 #   @retval     True            If the file content is changed and the file is renewed
 445 #   @retval     False           If the file content is the same
 446 #
 447 def SaveFileOnChange(File, Content, IsBinaryFile=True):
 448     if not IsBinaryFile:
 449         Content = Content.replace("\n", os.linesep)
 450
 451     if os.path.exists(File):
 452         try:
 453             if Content == open(File, "rb").read():
 454                 return False
 455         except:
 456             EdkLogger.error(None, FILE_OPEN_FAILURE, ExtraData=File)
 457
 458     DirName = os.path.dirname(File)
 459     if not CreateDirectory(DirName):
 460         EdkLogger.error(None, FILE_CREATE_FAILURE, "Could not create directory %s" % DirName)
 461     else:
 462         if DirName == '':
 463             DirName = os.getcwd()
 464         if not os.access(DirName, os.W_OK):
 465             EdkLogger.error(None, PERMISSION_FAILURE, "Do not have write permission on directory %s" % DirName)
 466
 467     try:
 468         if GlobalData.gIsWindows:
 469             try:
 470                 from PyUtility import SaveFileToDisk
 471                 if not SaveFileToDisk(File, Content):
 472                     EdkLogger.error(None, FILE_CREATE_FAILURE, ExtraData=File)
 473             except:
 474                 Fd = open(File, "wb")
 475                 Fd.write(Content)
 476                 Fd.close()
 477         else:
 478             Fd = open(File, "wb")
 479             Fd.write(Content)
 480             Fd.close()
 481     except IOError as X:
 482         EdkLogger.error(None, FILE_CREATE_FAILURE, ExtraData='IOError %s' % X)
 483
 484     return True
 485
 486 ## Make a Python object persistent on file system
 487 #
 488 #   @param      Data    The object to be stored in file
 489 #   @param      File    The path of file to store the object
 490 #
 491 def DataDump(Data, File):
 492     Fd = None
 493     try:
 494         Fd = open(File, 'wb')
 495         cPickle.dump(Data, Fd, cPickle.HIGHEST_PROTOCOL)
 496     except:
 497         EdkLogger.error("", FILE_OPEN_FAILURE, ExtraData=File, RaiseError=False)
 498     finally:
 499         if Fd is not None:
 500             Fd.close()
 501
 502 ## Restore a Python object from a file
 503 #
 504 #   @param      File    The path of file stored the object
 505 #
 506 #   @retval     object  A python object
 507 #   @retval     None    If failure in file operation
 508 #
 509 def DataRestore(File):
 510     Data = None
 511     Fd = None
 512     try:
 513         Fd = open(File, 'rb')
 514         Data = cPickle.load(Fd)
 515     except Exception as e:
 516         EdkLogger.verbose("Failed to load [%s]\n\t%s" % (File, str(e)))
 517         Data = None
 518     finally:
 519         if Fd is not None:
 520             Fd.close()
 521     return Data
 522
 523 ## Retrieve and cache the real path name in file system
 524 #
 525 #   @param      Root    The root directory of path relative to
 526 #
 527 #   @retval     str     The path string if the path exists
 528 #   @retval     None    If path doesn't exist
 529 #
 530 class DirCache:
 531     _CACHE_ = set()
 532     _UPPER_CACHE_ = {}
 533
 534     def __init__(self, Root):
 535         self._Root = Root
 536         for F in os.listdir(Root):
 537             self._CACHE_.add(F)
 538             self._UPPER_CACHE_[F.upper()] = F
 539
 540     # =[] operator
 541     def __getitem__(self, Path):
 542         Path = Path[len(os.path.commonprefix([Path, self._Root])):]
 543         if not Path:
 544             return self._Root
 545         if Path and Path[0] == os.path.sep:
 546             Path = Path[1:]
 547         if Path in self._CACHE_:
 548             return os.path.join(self._Root, Path)
 549         UpperPath = Path.upper()
 550         if UpperPath in self._UPPER_CACHE_:
 551             return os.path.join(self._Root, self._UPPER_CACHE_[UpperPath])
 552
 553         IndexList = []
 554         LastSepIndex = -1
 555         SepIndex = Path.find(os.path.sep)
 556         while SepIndex > -1:
 557             Parent = UpperPath[:SepIndex]
 558             if Parent not in self._UPPER_CACHE_:
 559                 break
 560             LastSepIndex = SepIndex
 561             SepIndex = Path.find(os.path.sep, LastSepIndex + 1)
 562
 563         if LastSepIndex == -1:
 564             return None
 565
 566         Cwd = os.getcwd()
 567         os.chdir(self._Root)
 568         SepIndex = LastSepIndex
 569         while SepIndex > -1:
 570             Parent = Path[:SepIndex]
 571             ParentKey = UpperPath[:SepIndex]
 572             if ParentKey not in self._UPPER_CACHE_:
 573                 os.chdir(Cwd)
 574                 return None
 575
 576             if Parent in self._CACHE_:
 577                 ParentDir = Parent
 578             else:
 579                 ParentDir = self._UPPER_CACHE_[ParentKey]
 580             for F in os.listdir(ParentDir):
 581                 Dir = os.path.join(ParentDir, F)
 582                 self._CACHE_.add(Dir)
 583                 self._UPPER_CACHE_[Dir.upper()] = Dir
 584
 585             SepIndex = Path.find(os.path.sep, SepIndex + 1)
 586
 587         os.chdir(Cwd)
 588         if Path in self._CACHE_:
 589             return os.path.join(self._Root, Path)
 590         elif UpperPath in self._UPPER_CACHE_:
 591             return os.path.join(self._Root, self._UPPER_CACHE_[UpperPath])
 592         return None
 593
 594 def RealPath(File, Dir='', OverrideDir=''):
 595     NewFile = os.path.normpath(os.path.join(Dir, File))
 596     NewFile = GlobalData.gAllFiles[NewFile]
 597     if not NewFile and OverrideDir:
 598         NewFile = os.path.normpath(os.path.join(OverrideDir, File))
 599         NewFile = GlobalData.gAllFiles[NewFile]
 600     return NewFile
 601
 602 def RealPath2(File, Dir='', OverrideDir=''):
 603     NewFile = None
 604     if OverrideDir:
 605         NewFile = GlobalData.gAllFiles[os.path.normpath(os.path.join(OverrideDir, File))]
 606         if NewFile:
 607             if OverrideDir[-1] == os.path.sep:
 608                 return NewFile[len(OverrideDir):], NewFile[0:len(OverrideDir)]
 609             else:
 610                 return NewFile[len(OverrideDir) + 1:], NewFile[0:len(OverrideDir)]
 611     if GlobalData.gAllFiles:
 612         NewFile = GlobalData.gAllFiles[os.path.normpath(os.path.join(Dir, File))]
 613     if not NewFile:
 614         NewFile = os.path.normpath(os.path.join(Dir, File))
 615         if not os.path.exists(NewFile):
 616             return None, None
 617     if NewFile:
 618         if Dir:
 619             if Dir[-1] == os.path.sep:
 620                 return NewFile[len(Dir):], NewFile[0:len(Dir)]
 621             else:
 622                 return NewFile[len(Dir) + 1:], NewFile[0:len(Dir)]
 623         else:
 624             return NewFile, ''
 625
 626     return None, None
 627
 628 ## Get GUID value from given packages
 629 #
 630 #   @param      CName           The CName of the GUID
 631 #   @param      PackageList     List of packages looking-up in
 632 #   @param      Inffile         The driver file
 633 #
 634 #   @retval     GuidValue   if the CName is found in any given package
 635 #   @retval     None        if the CName is not found in all given packages
 636 #
 637 def GuidValue(CName, PackageList, Inffile = None):
 638     for P in PackageList:
 639         GuidKeys = P.Guids.keys()
 640         if Inffile and P._PrivateGuids:
 641             if not Inffile.startswith(P.MetaFile.Dir):
 642                 GuidKeys = [x for x in P.Guids if x not in P._PrivateGuids]
 643         if CName in GuidKeys:
 644             return P.Guids[CName]
 645     return None
 646
 647 ## Get Protocol value from given packages
 648 #
 649 #   @param      CName           The CName of the GUID
 650 #   @param      PackageList     List of packages looking-up in
 651 #   @param      Inffile         The driver file
 652 #
 653 #   @retval     GuidValue   if the CName is found in any given package
 654 #   @retval     None        if the CName is not found in all given packages
 655 #
 656 def ProtocolValue(CName, PackageList, Inffile = None):
 657     for P in PackageList:
 658         ProtocolKeys = P.Protocols.keys()
 659         if Inffile and P._PrivateProtocols:
 660             if not Inffile.startswith(P.MetaFile.Dir):
 661                 ProtocolKeys = [x for x in P.Protocols if x not in P._PrivateProtocols]
 662         if CName in ProtocolKeys:
 663             return P.Protocols[CName]
 664     return None
 665
 666 ## Get PPI value from given packages
 667 #
 668 #   @param      CName           The CName of the GUID
 669 #   @param      PackageList     List of packages looking-up in
 670 #   @param      Inffile         The driver file
 671 #
 672 #   @retval     GuidValue   if the CName is found in any given package
 673 #   @retval     None        if the CName is not found in all given packages
 674 #
 675 def PpiValue(CName, PackageList, Inffile = None):
 676     for P in PackageList:
 677         PpiKeys = P.Ppis.keys()
 678         if Inffile and P._PrivatePpis:
 679             if not Inffile.startswith(P.MetaFile.Dir):
 680                 PpiKeys = [x for x in P.Ppis if x not in P._PrivatePpis]
 681         if CName in PpiKeys:
 682             return P.Ppis[CName]
 683     return None
 684
 685 ## A string template class
 686 #
 687 #  This class implements a template for string replacement. A string template
 688 #  looks like following
 689 #
 690 #       ${BEGIN} other_string ${placeholder_name} other_string ${END}
 691 #
 692 #  The string between ${BEGIN} and ${END} will be repeated as many times as the
 693 #  length of "placeholder_name", which is a list passed through a dict. The
 694 #  "placeholder_name" is the key name of the dict. The ${BEGIN} and ${END} can
 695 #  be not used and, in this case, the "placeholder_name" must not a list and it
 696 #  will just be replaced once.
 697 #
 698 class TemplateString(object):
 699     _REPEAT_START_FLAG = "BEGIN"
 700     _REPEAT_END_FLAG = "END"
 701
 702     class Section(object):
 703         _LIST_TYPES = [type([]), type(set()), type((0,))]
 704
 705         def __init__(self, TemplateSection, PlaceHolderList):
 706             self._Template = TemplateSection
 707             self._PlaceHolderList = []
 708
 709             # Split the section into sub-sections according to the position of placeholders
 710             if PlaceHolderList:
 711                 self._SubSectionList = []
 712                 SubSectionStart = 0
 713                 #
 714                 # The placeholders passed in must be in the format of
 715                 #
 716                 #   PlaceHolderName, PlaceHolderStartPoint, PlaceHolderEndPoint
 717                 #
 718                 for PlaceHolder, Start, End in PlaceHolderList:
 719                     self._SubSectionList.append(TemplateSection[SubSectionStart:Start])
 720                     self._SubSectionList.append(TemplateSection[Start:End])
 721                     self._PlaceHolderList.append(PlaceHolder)
 722                     SubSectionStart = End
 723                 if SubSectionStart < len(TemplateSection):
 724                     self._SubSectionList.append(TemplateSection[SubSectionStart:])
 725             else:
 726                 self._SubSectionList = [TemplateSection]
 727
 728         def __str__(self):
 729             return self._Template + " : " + str(self._PlaceHolderList)
 730
 731         def Instantiate(self, PlaceHolderValues):
 732             RepeatTime = -1
 733             RepeatPlaceHolders = {}
 734             NonRepeatPlaceHolders = {}
 735
 736             for PlaceHolder in self._PlaceHolderList:
 737                 if PlaceHolder not in PlaceHolderValues:
 738                     continue
 739                 Value = PlaceHolderValues[PlaceHolder]
 740                 if type(Value) in self._LIST_TYPES:
 741                     if RepeatTime < 0:
 742                         RepeatTime = len(Value)
 743                     elif RepeatTime != len(Value):
 744                         EdkLogger.error(
 745                                     "TemplateString",
 746                                     PARAMETER_INVALID,
 747                                     "${%s} has different repeat time from others!" % PlaceHolder,
 748                                     ExtraData=str(self._Template)
 749                                     )
 750                     RepeatPlaceHolders["${%s}" % PlaceHolder] = Value
 751                 else:
 752                     NonRepeatPlaceHolders["${%s}" % PlaceHolder] = Value
 753
 754             if NonRepeatPlaceHolders:
 755                 StringList = []
 756                 for S in self._SubSectionList:
 757                     if S not in NonRepeatPlaceHolders:
 758                         StringList.append(S)
 759                     else:
 760                         StringList.append(str(NonRepeatPlaceHolders[S]))
 761             else:
 762                 StringList = self._SubSectionList
 763
 764             if RepeatPlaceHolders:
 765                 TempStringList = []
 766                 for Index in range(RepeatTime):
 767                     for S in StringList:
 768                         if S not in RepeatPlaceHolders:
 769                             TempStringList.append(S)
 770                         else:
 771                             TempStringList.append(str(RepeatPlaceHolders[S][Index]))
 772                 StringList = TempStringList
 773
 774             return "".join(StringList)
 775
 776     ## Constructor
 777     def __init__(self, Template=None):
 778         self.String = ''
 779         self.IsBinary = False
 780         self._Template = Template
 781         self._TemplateSectionList = self._Parse(Template)
 782
 783     ## str() operator
 784     #
 785     #   @retval     string  The string replaced
 786     #
 787     def __str__(self):
 788         return self.String
 789
 790     ## Split the template string into fragments per the ${BEGIN} and ${END} flags
 791     #
 792     #   @retval     list    A list of TemplateString.Section objects
 793     #
 794     def _Parse(self, Template):
 795         SectionStart = 0
 796         SearchFrom = 0
 797         MatchEnd = 0
 798         PlaceHolderList = []
 799         TemplateSectionList = []
 800         while Template:
 801             MatchObj = gPlaceholderPattern.search(Template, SearchFrom)
 802             if not MatchObj:
 803                 if MatchEnd <= len(Template):
 804                     TemplateSection = TemplateString.Section(Template[SectionStart:], PlaceHolderList)
 805                     TemplateSectionList.append(TemplateSection)
 806                 break
 807
 808             MatchString = MatchObj.group(1)
 809             MatchStart = MatchObj.start()
 810             MatchEnd = MatchObj.end()
 811
 812             if MatchString == self._REPEAT_START_FLAG:
 813                 if MatchStart > SectionStart:
 814                     TemplateSection = TemplateString.Section(Template[SectionStart:MatchStart], PlaceHolderList)
 815                     TemplateSectionList.append(TemplateSection)
 816                 SectionStart = MatchEnd
 817                 PlaceHolderList = []
 818             elif MatchString == self._REPEAT_END_FLAG:
 819                 TemplateSection = TemplateString.Section(Template[SectionStart:MatchStart], PlaceHolderList)
 820                 TemplateSectionList.append(TemplateSection)
 821                 SectionStart = MatchEnd
 822                 PlaceHolderList = []
 823             else:
 824                 PlaceHolderList.append((MatchString, MatchStart - SectionStart, MatchEnd - SectionStart))
 825             SearchFrom = MatchEnd
 826         return TemplateSectionList
 827
 828     ## Replace the string template with dictionary of placeholders and append it to previous one
 829     #
 830     #   @param      AppendString    The string template to append
 831     #   @param      Dictionary      The placeholder dictionaries
 832     #
 833     def Append(self, AppendString, Dictionary=None):
 834         if Dictionary:
 835             SectionList = self._Parse(AppendString)
 836             self.String += "".join(S.Instantiate(Dictionary) for S in SectionList)
 837         else:
 838             self.String += AppendString
 839
 840     ## Replace the string template with dictionary of placeholders
 841     #
 842     #   @param      Dictionary      The placeholder dictionaries
 843     #
 844     #   @retval     str             The string replaced with placeholder values
 845     #
 846     def Replace(self, Dictionary=None):
 847         return "".join(S.Instantiate(Dictionary) for S in self._TemplateSectionList)
 848
 849 ## Progress indicator class
 850 #
 851 #  This class makes use of thread to print progress on console.
 852 #
 853 class Progressor:
 854     # for avoiding deadloop
 855     _StopFlag = None
 856     _ProgressThread = None
 857     _CheckInterval = 0.25
 858
 859     ## Constructor
 860     #
 861     #   @param      OpenMessage     The string printed before progress charaters
 862     #   @param      CloseMessage    The string printed after progress charaters
 863     #   @param      ProgressChar    The charater used to indicate the progress
 864     #   @param      Interval        The interval in seconds between two progress charaters
 865     #
 866     def __init__(self, OpenMessage="", CloseMessage="", ProgressChar='.', Interval=1.0):
 867         self.PromptMessage = OpenMessage
 868         self.CodaMessage = CloseMessage
 869         self.ProgressChar = ProgressChar
 870         self.Interval = Interval
 871         if Progressor._StopFlag is None:
 872             Progressor._StopFlag = threading.Event()
 873
 874     ## Start to print progress charater
 875     #
 876     #   @param      OpenMessage     The string printed before progress charaters
 877     #
 878     def Start(self, OpenMessage=None):
 879         if OpenMessage is not None:
 880             self.PromptMessage = OpenMessage
 881         Progressor._StopFlag.clear()
 882         if Progressor._ProgressThread is None:
 883             Progressor._ProgressThread = threading.Thread(target=self._ProgressThreadEntry)
 884             Progressor._ProgressThread.setDaemon(False)
 885             Progressor._ProgressThread.start()
 886
 887     ## Stop printing progress charater
 888     #
 889     #   @param      CloseMessage    The string printed after progress charaters
 890     #
 891     def Stop(self, CloseMessage=None):
 892         OriginalCodaMessage = self.CodaMessage
 893         if CloseMessage is not None:
 894             self.CodaMessage = CloseMessage
 895         self.Abort()
 896         self.CodaMessage = OriginalCodaMessage
 897
 898     ## Thread entry method
 899     def _ProgressThreadEntry(self):
 900         sys.stdout.write(self.PromptMessage + " ")
 901         sys.stdout.flush()
 902         TimeUp = 0.0
 903         while not Progressor._StopFlag.isSet():
 904             if TimeUp <= 0.0:
 905                 sys.stdout.write(self.ProgressChar)
 906                 sys.stdout.flush()
 907                 TimeUp = self.Interval
 908             time.sleep(self._CheckInterval)
 909             TimeUp -= self._CheckInterval
 910         sys.stdout.write(" " + self.CodaMessage + "\n")
 911         sys.stdout.flush()
 912
 913     ## Abort the progress display
 914     @staticmethod
 915     def Abort():
 916         if Progressor._StopFlag is not None:
 917             Progressor._StopFlag.set()
 918         if Progressor._ProgressThread is not None:
 919             Progressor._ProgressThread.join()
 920             Progressor._ProgressThread = None
 921
 922 ## A dict which can access its keys and/or values orderly
 923 #
 924 #  The class implements a new kind of dict which its keys or values can be
 925 #  accessed in the order they are added into the dict. It guarantees the order
 926 #  by making use of an internal list to keep a copy of keys.
 927 #
 928 class sdict(IterableUserDict):
 929     ## Constructor
 930     def __init__(self):
 931         IterableUserDict.__init__(self)
 932         self._key_list = []
 933
 934     ## [] operator
 935     def __setitem__(self, key, value):
 936         if key not in self._key_list:
 937             self._key_list.append(key)
 938         IterableUserDict.__setitem__(self, key, value)
 939
 940     ## del operator
 941     def __delitem__(self, key):
 942         self._key_list.remove(key)
 943         IterableUserDict.__delitem__(self, key)
 944
 945     ## used in "for k in dict" loop to ensure the correct order
 946     def __iter__(self):
 947         return self.iterkeys()
 948
 949     ## len() support
 950     def __len__(self):
 951         return len(self._key_list)
 952
 953     ## "in" test support
 954     def __contains__(self, key):
 955         return key in self._key_list
 956
 957     ## indexof support
 958     def index(self, key):
 959         return self._key_list.index(key)
 960
 961     ## insert support
 962     def insert(self, key, newkey, newvalue, order):
 963         index = self._key_list.index(key)
 964         if order == 'BEFORE':
 965             self._key_list.insert(index, newkey)
 966             IterableUserDict.__setitem__(self, newkey, newvalue)
 967         elif order == 'AFTER':
 968             self._key_list.insert(index + 1, newkey)
 969             IterableUserDict.__setitem__(self, newkey, newvalue)
 970
 971     ## append support
 972     def append(self, sdict):
 973         for key in sdict:
 974             if key not in self._key_list:
 975                 self._key_list.append(key)
 976             IterableUserDict.__setitem__(self, key, sdict[key])
 977
 978     def has_key(self, key):
 979         return key in self._key_list
 980
 981     ## Empty the dict
 982     def clear(self):
 983         self._key_list = []
 984         IterableUserDict.clear(self)
 985
 986     ## Return a copy of keys
 987     def keys(self):
 988         keys = []
 989         for key in self._key_list:
 990             keys.append(key)
 991         return keys
 992
 993     ## Return a copy of values
 994     def values(self):
 995         values = []
 996         for key in self._key_list:
 997             values.append(self[key])
 998         return values
 999
1000     ## Return a copy of (key, value) list
1001     def items(self):
1002         items = []
1003         for key in self._key_list:
1004             items.append((key, self[key]))
1005         return items
1006
1007     ## Iteration support
1008     def iteritems(self):
1009         return iter(self.items())
1010
1011     ## Keys interation support
1012     def iterkeys(self):
1013         return iter(self.keys())
1014
1015     ## Values interation support
1016     def itervalues(self):
1017         return iter(self.values())
1018
1019     ## Return value related to a key, and remove the (key, value) from the dict
1020     def pop(self, key, *dv):
1021         value = None
1022         if key in self._key_list:
1023             value = self[key]
1024             self.__delitem__(key)
1025         elif len(dv) != 0 :
1026             value = kv[0]
1027         return value
1028
1029     ## Return (key, value) pair, and remove the (key, value) from the dict
1030     def popitem(self):
1031         key = self._key_list[-1]
1032         value = self[key]
1033         self.__delitem__(key)
1034         return key, value
1035
1036     def update(self, dict=None, **kwargs):
1037         if dict is not None:
1038             for k, v in dict.items():
1039                 self[k] = v
1040         if len(kwargs):
1041             for k, v in kwargs.items():
1042                 self[k] = v
1043
1044 ## Dictionary with restricted keys
1045 #
1046 class rdict(dict):
1047     ## Constructor
1048     def __init__(self, KeyList):
1049         for Key in KeyList:
1050             dict.__setitem__(self, Key, "")
1051
1052     ## []= operator
1053     def __setitem__(self, key, value):
1054         if key not in self:
1055             EdkLogger.error("RestrictedDict", ATTRIBUTE_SET_FAILURE, "Key [%s] is not allowed" % key,
1056                             ExtraData=", ".join(dict.keys(self)))
1057         dict.__setitem__(self, key, value)
1058
1059     ## =[] operator
1060     def __getitem__(self, key):
1061         if key not in self:
1062             return ""
1063         return dict.__getitem__(self, key)
1064
1065     ## del operator
1066     def __delitem__(self, key):
1067         EdkLogger.error("RestrictedDict", ATTRIBUTE_ACCESS_DENIED, ExtraData="del")
1068
1069     ## Empty the dict
1070     def clear(self):
1071         for Key in self:
1072             self.__setitem__(Key, "")
1073
1074     ## Return value related to a key, and remove the (key, value) from the dict
1075     def pop(self, key, *dv):
1076         EdkLogger.error("RestrictedDict", ATTRIBUTE_ACCESS_DENIED, ExtraData="pop")
1077
1078     ## Return (key, value) pair, and remove the (key, value) from the dict
1079     def popitem(self):
1080         EdkLogger.error("RestrictedDict", ATTRIBUTE_ACCESS_DENIED, ExtraData="popitem")
1081
1082 ## Dictionary using prioritized list as key
1083 #
1084 class tdict:
1085     _ListType = type([])
1086     _TupleType = type(())
1087     _Wildcard = 'COMMON'
1088     _ValidWildcardList = ['COMMON', 'DEFAULT', 'ALL', '*', 'PLATFORM']
1089
1090     def __init__(self, _Single_=False, _Level_=2):
1091         self._Level_ = _Level_
1092         self.data = {}
1093         self._Single_ = _Single_
1094
1095     # =[] operator
1096     def __getitem__(self, key):
1097         KeyType = type(key)
1098         RestKeys = None
1099         if KeyType == self._ListType or KeyType == self._TupleType:
1100             FirstKey = key[0]
1101             if len(key) > 1:
1102                 RestKeys = key[1:]
1103             elif self._Level_ > 1:
1104                 RestKeys = [self._Wildcard for i in range(0, self._Level_ - 1)]
1105         else:
1106             FirstKey = key
1107             if self._Level_ > 1:
1108                 RestKeys = [self._Wildcard for i in range(0, self._Level_ - 1)]
1109
1110         if FirstKey is None or str(FirstKey).upper() in self._ValidWildcardList:
1111             FirstKey = self._Wildcard
1112
1113         if self._Single_:
1114             return self._GetSingleValue(FirstKey, RestKeys)
1115         else:
1116             return self._GetAllValues(FirstKey, RestKeys)
1117
1118     def _GetSingleValue(self, FirstKey, RestKeys):
1119         Value = None
1120         #print "%s-%s" % (FirstKey, self._Level_) ,
1121         if self._Level_ > 1:
1122             if FirstKey == self._Wildcard:
1123                 if FirstKey in self.data:
1124                     Value = self.data[FirstKey][RestKeys]
1125                 if Value is None:
1126                     for Key in self.data:
1127                         Value = self.data[Key][RestKeys]
1128                         if Value is not None: break
1129             else:
1130                 if FirstKey in self.data:
1131                     Value = self.data[FirstKey][RestKeys]
1132                 if Value is None and self._Wildcard in self.data:
1133                     #print "Value=None"
1134                     Value = self.data[self._Wildcard][RestKeys]
1135         else:
1136             if FirstKey == self._Wildcard:
1137                 if FirstKey in self.data:
1138                     Value = self.data[FirstKey]
1139                 if Value is None:
1140                     for Key in self.data:
1141                         Value = self.data[Key]
1142                         if Value is not None: break
1143             else:
1144                 if FirstKey in self.data:
1145                     Value = self.data[FirstKey]
1146                 elif self._Wildcard in self.data:
1147                     Value = self.data[self._Wildcard]
1148         return Value
1149
1150     def _GetAllValues(self, FirstKey, RestKeys):
1151         Value = []
1152         if self._Level_ > 1:
1153             if FirstKey == self._Wildcard:
1154                 for Key in self.data:
1155                     Value += self.data[Key][RestKeys]
1156             else:
1157                 if FirstKey in self.data:
1158                     Value += self.data[FirstKey][RestKeys]
1159                 if self._Wildcard in self.data:
1160                     Value += self.data[self._Wildcard][RestKeys]
1161         else:
1162             if FirstKey == self._Wildcard:
1163                 for Key in self.data:
1164                     Value.append(self.data[Key])
1165             else:
1166                 if FirstKey in self.data:
1167                     Value.append(self.data[FirstKey])
1168                 if self._Wildcard in self.data:
1169                     Value.append(self.data[self._Wildcard])
1170         return Value
1171
1172     ## []= operator
1173     def __setitem__(self, key, value):
1174         KeyType = type(key)
1175         RestKeys = None
1176         if KeyType == self._ListType or KeyType == self._TupleType:
1177             FirstKey = key[0]
1178             if len(key) > 1:
1179                 RestKeys = key[1:]
1180             else:
1181                 RestKeys = [self._Wildcard for i in range(0, self._Level_ - 1)]
1182         else:
1183             FirstKey = key
1184             if self._Level_ > 1:
1185                 RestKeys = [self._Wildcard for i in range(0, self._Level_ - 1)]
1186
1187         if FirstKey in self._ValidWildcardList:
1188             FirstKey = self._Wildcard
1189
1190         if FirstKey not in self.data and self._Level_ > 0:
1191             self.data[FirstKey] = tdict(self._Single_, self._Level_ - 1)
1192
1193         if self._Level_ > 1:
1194             self.data[FirstKey][RestKeys] = value
1195         else:
1196             self.data[FirstKey] = value
1197
1198     def SetGreedyMode(self):
1199         self._Single_ = False
1200         if self._Level_ > 1:
1201             for Key in self.data:
1202                 self.data[Key].SetGreedyMode()
1203
1204     def SetSingleMode(self):
1205         self._Single_ = True
1206         if self._Level_ > 1:
1207             for Key in self.data:
1208                 self.data[Key].SetSingleMode()
1209
1210     def GetKeys(self, KeyIndex=0):
1211         assert KeyIndex >= 0
1212         if KeyIndex == 0:
1213             return set(self.data.keys())
1214         else:
1215             keys = set()
1216             for Key in self.data:
1217                 keys |= self.data[Key].GetKeys(KeyIndex - 1)
1218             return keys
1219
1220 def IsFieldValueAnArray (Value):
1221     Value = Value.strip()
1222     if Value.startswith(TAB_GUID) and Value.endswith(')'):
1223         return True
1224     if Value.startswith('L"') and Value.endswith('"')  and len(list(Value[2:-1])) > 1:
1225         return True
1226     if Value[0] == '"' and Value[-1] == '"' and len(list(Value[1:-1])) > 1:
1227         return True
1228     if Value[0] == '{' and Value[-1] == '}':
1229         return True
1230     if Value.startswith("L'") and Value.endswith("'") and len(list(Value[2:-1])) > 1:
1231         return True
1232     if Value[0] == "'" and Value[-1] == "'" and len(list(Value[1:-1])) > 1:
1233         return True
1234     return False
1235
1236 def AnalyzePcdExpression(Setting):
1237     Setting = Setting.strip()
1238     # There might be escaped quote in a string: \", \\\" , \', \\\'
1239     Data = Setting
1240     # There might be '|' in string and in ( ... | ... ), replace it with '-'
1241     NewStr = ''
1242     InSingleQuoteStr = False
1243     InDoubleQuoteStr = False
1244     Pair = 0
1245     for Index, ch in enumerate(Data):
1246         if ch == '"' and not InSingleQuoteStr:
1247             if Data[Index - 1] != '\\':
1248                 InDoubleQuoteStr = not InDoubleQuoteStr
1249         elif ch == "'" and not InDoubleQuoteStr:
1250             if Data[Index - 1] != '\\':
1251                 InSingleQuoteStr = not InSingleQuoteStr
1252         elif ch == '(' and not (InSingleQuoteStr or InDoubleQuoteStr):
1253             Pair += 1
1254         elif ch == ')' and not (InSingleQuoteStr or InDoubleQuoteStr):
1255             Pair -= 1
1256
1257         if (Pair > 0 or InSingleQuoteStr or InDoubleQuoteStr) and ch == TAB_VALUE_SPLIT:
1258             NewStr += '-'
1259         else:
1260             NewStr += ch
1261     FieldList = []
1262     StartPos = 0
1263     while True:
1264         Pos = NewStr.find(TAB_VALUE_SPLIT, StartPos)
1265         if Pos < 0:
1266             FieldList.append(Setting[StartPos:].strip())
1267             break
1268         FieldList.append(Setting[StartPos:Pos].strip())
1269         StartPos = Pos + 1
1270
1271     return FieldList
1272
1273 def ParseDevPathValue (Value):
1274     if '\\' in Value:
1275         Value.replace('\\', '/').replace(' ', '')
1276
1277     Cmd = 'DevicePath ' + '"' + Value + '"'
1278     try:
1279         p = subprocess.Popen(Cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
1280         out, err = p.communicate()
1281     except Exception as X:
1282         raise BadExpression("DevicePath: %s" % (str(X)) )
1283     finally:
1284         subprocess._cleanup()
1285         p.stdout.close()
1286         p.stderr.close()
1287     if err:
1288         raise BadExpression("DevicePath: %s" % str(err))
1289     Size = len(out.split())
1290     out = ','.join(out.split())
1291     return '{' + out + '}', Size
1292
1293 def ParseFieldValue (Value):
1294     if isinstance(Value, type(0)):
1295         return Value, (Value.bit_length() + 7) / 8
1296     if not isinstance(Value, type('')):
1297         raise BadExpression('Type %s is %s' %(Value, type(Value)))
1298     Value = Value.strip()
1299     if Value.startswith(TAB_UINT8) and Value.endswith(')'):
1300         Value, Size = ParseFieldValue(Value.split('(', 1)[1][:-1])
1301         if Size > 1:
1302             raise BadExpression('Value (%s) Size larger than %d' %(Value, Size))
1303         return Value, 1
1304     if Value.startswith(TAB_UINT16) and Value.endswith(')'):
1305         Value, Size = ParseFieldValue(Value.split('(', 1)[1][:-1])
1306         if Size > 2:
1307             raise BadExpression('Value (%s) Size larger than %d' %(Value, Size))
1308         return Value, 2
1309     if Value.startswith(TAB_UINT32) and Value.endswith(')'):
1310         Value, Size = ParseFieldValue(Value.split('(', 1)[1][:-1])
1311         if Size > 4:
1312             raise BadExpression('Value (%s) Size larger than %d' %(Value, Size))
1313         return Value, 4
1314     if Value.startswith(TAB_UINT64) and Value.endswith(')'):
1315         Value, Size = ParseFieldValue(Value.split('(', 1)[1][:-1])
1316         if Size > 8:
1317             raise BadExpression('Value (%s) Size larger than %d' % (Value, Size))
1318         return Value, 8
1319     if Value.startswith(TAB_GUID) and Value.endswith(')'):
1320         Value = Value.split('(', 1)[1][:-1].strip()
1321         if Value[0] == '{' and Value[-1] == '}':
1322             TmpValue = GuidStructureStringToGuidString(Value)
1323             if len(TmpValue) == 0:
1324                 raise BadExpression("Invalid GUID value string %s" % Value)
1325             Value = TmpValue
1326         if Value[0] == '"' and Value[-1] == '"':
1327             Value = Value[1:-1]
1328         try:
1329             Value = "'" + uuid.UUID(Value).get_bytes_le() + "'"
1330         except ValueError as Message:
1331             raise BadExpression(Message)
1332         Value, Size = ParseFieldValue(Value)
1333         return Value, 16
1334     if Value.startswith('L"') and Value.endswith('"'):
1335         # Unicode String
1336         # translate escape character
1337         Value = Value[1:]
1338         try:
1339             Value = eval(Value)
1340         except:
1341             Value = Value[1:-1]
1342         List = list(Value)
1343         List.reverse()
1344         Value = 0
1345         for Char in List:
1346             Value = (Value << 16) | ord(Char)
1347         return Value, (len(List) + 1) * 2
1348     if Value.startswith('"') and Value.endswith('"'):
1349         # ASCII String
1350         # translate escape character
1351         try:
1352             Value = eval(Value)
1353         except:
1354             Value = Value[1:-1]
1355         List = list(Value)
1356         List.reverse()
1357         Value = 0
1358         for Char in List:
1359             Value = (Value << 8) | ord(Char)
1360         return Value, len(List) + 1
1361     if Value.startswith("L'") and Value.endswith("'"):
1362         # Unicode Character Constant
1363         # translate escape character
1364         Value = Value[1:]
1365         try:
1366             Value = eval(Value)
1367         except:
1368             Value = Value[1:-1]
1369         List = list(Value)
1370         if len(List) == 0:
1371             raise BadExpression('Length %s is %s' % (Value, len(List)))
1372         List.reverse()
1373         Value = 0
1374         for Char in List:
1375             Value = (Value << 16) | ord(Char)
1376         return Value, len(List) * 2
1377     if Value.startswith("'") and Value.endswith("'"):
1378         # Character constant
1379         # translate escape character
1380         try:
1381             Value = eval(Value)
1382         except:
1383             Value = Value[1:-1]
1384         List = list(Value)
1385         if len(List) == 0:
1386             raise BadExpression('Length %s is %s' % (Value, len(List)))
1387         List.reverse()
1388         Value = 0
1389         for Char in List:
1390             Value = (Value << 8) | ord(Char)
1391         return Value, len(List)
1392     if Value.startswith('{') and Value.endswith('}'):
1393         # Byte array
1394         Value = Value[1:-1]
1395         List = [Item.strip() for Item in Value.split(',')]
1396         List.reverse()
1397         Value = 0
1398         RetSize = 0
1399         for Item in List:
1400             ItemValue, Size = ParseFieldValue(Item)
1401             RetSize += Size
1402             for I in range(Size):
1403                 Value = (Value << 8) | ((ItemValue >> 8 * I) & 0xff)
1404         return Value, RetSize
1405     if Value.startswith('DEVICE_PATH(') and Value.endswith(')'):
1406         Value = Value.replace("DEVICE_PATH(", '').rstrip(')')
1407         Value = Value.strip().strip('"')
1408         return ParseDevPathValue(Value)
1409     if Value.lower().startswith('0x'):
1410         Value = int(Value, 16)
1411         if Value == 0:
1412             return 0, 1
1413         return Value, (Value.bit_length() + 7) / 8
1414     if Value[0].isdigit():
1415         Value = int(Value, 10)
1416         if Value == 0:
1417             return 0, 1
1418         return Value, (Value.bit_length() + 7) / 8
1419     if Value.lower() == 'true':
1420         return 1, 1
1421     if Value.lower() == 'false':
1422         return 0, 1
1423     return Value, 1
1424
1425 ## AnalyzeDscPcd
1426 #
1427 #  Analyze DSC PCD value, since there is no data type info in DSC
1428 #  This fuction is used to match functions (AnalyzePcdData) used for retrieving PCD value from database
1429 #  1. Feature flag: TokenSpace.PcdCName|PcdValue
1430 #  2. Fix and Patch:TokenSpace.PcdCName|PcdValue[|MaxSize]
1431 #  3. Dynamic default:
1432 #     TokenSpace.PcdCName|PcdValue[|VOID*[|MaxSize]]
1433 #     TokenSpace.PcdCName|PcdValue
1434 #  4. Dynamic VPD:
1435 #     TokenSpace.PcdCName|VpdOffset[|VpdValue]
1436 #     TokenSpace.PcdCName|VpdOffset[|MaxSize[|VpdValue]]
1437 #  5. Dynamic HII:
1438 #     TokenSpace.PcdCName|HiiString|VaiableGuid|VariableOffset[|HiiValue]
1439 #  PCD value needs to be located in such kind of string, and the PCD value might be an expression in which
1440 #    there might have "|" operator, also in string value.
1441 #
1442 #  @param Setting: String contain information described above with "TokenSpace.PcdCName|" stripped
1443 #  @param PcdType: PCD type: feature, fixed, dynamic default VPD HII
1444 #  @param DataType: The datum type of PCD: VOID*, UNIT, BOOL
1445 #  @retval:
1446 #    ValueList: A List contain fields described above
1447 #    IsValid:   True if conforming EBNF, otherwise False
1448 #    Index:     The index where PcdValue is in ValueList
1449 #
1450 def AnalyzeDscPcd(Setting, PcdType, DataType=''):
1451     FieldList = AnalyzePcdExpression(Setting)
1452
1453     IsValid = True
1454     if PcdType in (MODEL_PCD_FIXED_AT_BUILD, MODEL_PCD_PATCHABLE_IN_MODULE, MODEL_PCD_FEATURE_FLAG):
1455         Value = FieldList[0]
1456         Size = ''
1457         if len(FieldList) > 1:
1458             if FieldList[1].upper().startswith("0X") or FieldList[1].isdigit():
1459                 Size = FieldList[1]
1460             else:
1461                 DataType = FieldList[1]
1462
1463         if len(FieldList) > 2:
1464             Size = FieldList[2]
1465         if DataType == "":
1466             IsValid = (len(FieldList) <= 1)
1467         else:
1468             IsValid = (len(FieldList) <= 3)
1469 #         Value, Size = ParseFieldValue(Value)
1470         if Size:
1471             try:
1472                 int(Size, 16) if Size.upper().startswith("0X") else int(Size)
1473             except:
1474                 IsValid = False
1475                 Size = -1
1476         return [str(Value), '', str(Size)], IsValid, 0
1477     elif PcdType in (MODEL_PCD_DYNAMIC_DEFAULT, MODEL_PCD_DYNAMIC_EX_DEFAULT):
1478         Value = FieldList[0]
1479         Size = Type = ''
1480         if len(FieldList) > 1:
1481             Type = FieldList[1]
1482         else:
1483             Type = DataType
1484         if len(FieldList) > 2:
1485             Size = FieldList[2]
1486         if DataType == "":
1487             IsValid = (len(FieldList) <= 1)
1488         else:
1489             IsValid = (len(FieldList) <= 3)
1490
1491         if Size:
1492             try:
1493                 int(Size, 16) if Size.upper().startswith("0X") else int(Size)
1494             except:
1495                 IsValid = False
1496                 Size = -1
1497         return [Value, Type, str(Size)], IsValid, 0
1498     elif PcdType in (MODEL_PCD_DYNAMIC_VPD, MODEL_PCD_DYNAMIC_EX_VPD):
1499         VpdOffset = FieldList[0]
1500         Value = Size = ''
1501         if not DataType == TAB_VOID:
1502             if len(FieldList) > 1:
1503                 Value = FieldList[1]
1504         else:
1505             if len(FieldList) > 1:
1506                 Size = FieldList[1]
1507             if len(FieldList) > 2:
1508                 Value = FieldList[2]
1509         if DataType == "":
1510             IsValid = (len(FieldList) <= 1)
1511         else:
1512             IsValid = (len(FieldList) <= 3)
1513         if Size:
1514             try:
1515                 int(Size, 16) if Size.upper().startswith("0X") else int(Size)
1516             except:
1517                 IsValid = False
1518                 Size = -1
1519         return [VpdOffset, str(Size), Value], IsValid, 2
1520     elif PcdType in (MODEL_PCD_DYNAMIC_HII, MODEL_PCD_DYNAMIC_EX_HII):
1521         HiiString = FieldList[0]
1522         Guid = Offset = Value = Attribute = ''
1523         if len(FieldList) > 1:
1524             Guid = FieldList[1]
1525         if len(FieldList) > 2:
1526             Offset = FieldList[2]
1527         if len(FieldList) > 3:
1528             Value = FieldList[3]
1529         if len(FieldList) > 4:
1530             Attribute = FieldList[4]
1531         IsValid = (3 <= len(FieldList) <= 5)
1532         return [HiiString, Guid, Offset, Value, Attribute], IsValid, 3
1533     return [], False, 0
1534
1535 ## AnalyzePcdData
1536 #
1537 #  Analyze the pcd Value, Datum type and TokenNumber.
1538 #  Used to avoid split issue while the value string contain "|" character
1539 #
1540 #  @param[in] Setting:  A String contain value/datum type/token number information;
1541 #
1542 #  @retval   ValueList: A List contain value, datum type and toke number.
1543 #
1544 def AnalyzePcdData(Setting):
1545     ValueList = ['', '', '']
1546
1547     ValueRe = re.compile(r'^\s*L?\".*\|.*\"')
1548     PtrValue = ValueRe.findall(Setting)
1549
1550     ValueUpdateFlag = False
1551
1552     if len(PtrValue) >= 1:
1553         Setting = re.sub(ValueRe, '', Setting)
1554         ValueUpdateFlag = True
1555
1556     TokenList = Setting.split(TAB_VALUE_SPLIT)
1557     ValueList[0:len(TokenList)] = TokenList
1558
1559     if ValueUpdateFlag:
1560         ValueList[0] = PtrValue[0]
1561
1562     return ValueList
1563
1564 ## check format of PCD value against its the datum type
1565 #
1566 # For PCD value setting
1567 #
1568 def CheckPcdDatum(Type, Value):
1569     if Type == TAB_VOID:
1570         ValueRe = re.compile(r'\s*L?\".*\"\s*$')
1571         if not (((Value.startswith('L"') or Value.startswith('"')) and Value.endswith('"'))
1572                 or (Value.startswith('{') and Value.endswith('}')) or (Value.startswith("L'") or Value.startswith("'") and Value.endswith("'"))
1573                ):
1574             return False, "Invalid value [%s] of type [%s]; must be in the form of {...} for array"\
1575                           ", \"...\" or \'...\' for string, L\"...\" or L\'...\' for unicode string" % (Value, Type)
1576         elif ValueRe.match(Value):
1577             # Check the chars in UnicodeString or CString is printable
1578             if Value.startswith("L"):
1579                 Value = Value[2:-1]
1580             else:
1581                 Value = Value[1:-1]
1582             Printset = set(string.printable)
1583             Printset.remove(TAB_PRINTCHAR_VT)
1584             Printset.add(TAB_PRINTCHAR_BS)
1585             Printset.add(TAB_PRINTCHAR_NUL)
1586             if not set(Value).issubset(Printset):
1587                 PrintList = sorted(Printset)
1588                 return False, "Invalid PCD string value of type [%s]; must be printable chars %s." % (Type, PrintList)
1589     elif Type == 'BOOLEAN':
1590         if Value not in ['TRUE', 'True', 'true', '0x1', '0x01', '1', 'FALSE', 'False', 'false', '0x0', '0x00', '0']:
1591             return False, "Invalid value [%s] of type [%s]; must be one of TRUE, True, true, 0x1, 0x01, 1"\
1592                           ", FALSE, False, false, 0x0, 0x00, 0" % (Value, Type)
1593     elif Type in [TAB_UINT8, TAB_UINT16, TAB_UINT32, TAB_UINT64]:
1594         try:
1595             Value = long(Value, 0)
1596         except:
1597             return False, "Invalid value [%s] of type [%s];"\
1598                           " must be a hexadecimal, decimal or octal in C language format." % (Value, Type)
1599     else:
1600         return True, "StructurePcd"
1601
1602     return True, ""
1603
1604 ## Split command line option string to list
1605 #
1606 # subprocess.Popen needs the args to be a sequence. Otherwise there's problem
1607 # in non-windows platform to launch command
1608 #
1609 def SplitOption(OptionString):
1610     OptionList = []
1611     LastChar = " "
1612     OptionStart = 0
1613     QuotationMark = ""
1614     for Index in range(0, len(OptionString)):
1615         CurrentChar = OptionString[Index]
1616         if CurrentChar in ['"', "'"]:
1617             if QuotationMark == CurrentChar:
1618                 QuotationMark = ""
1619             elif QuotationMark == "":
1620                 QuotationMark = CurrentChar
1621             continue
1622         elif QuotationMark:
1623             continue
1624
1625         if CurrentChar in ["/", "-"] and LastChar in [" ", "\t", "\r", "\n"]:
1626             if Index > OptionStart:
1627                 OptionList.append(OptionString[OptionStart:Index - 1])
1628             OptionStart = Index
1629         LastChar = CurrentChar
1630     OptionList.append(OptionString[OptionStart:])
1631     return OptionList
1632
1633 def CommonPath(PathList):
1634     P1 = min(PathList).split(os.path.sep)
1635     P2 = max(PathList).split(os.path.sep)
1636     for Index in xrange(min(len(P1), len(P2))):
1637         if P1[Index] != P2[Index]:
1638             return os.path.sep.join(P1[:Index])
1639     return os.path.sep.join(P1)
1640
1641 #
1642 # Convert string to C format array
1643 #
1644 def ConvertStringToByteArray(Value):
1645     Value = Value.strip()
1646     if not Value:
1647         return None
1648     if Value[0] == '{':
1649         if not Value.endswith('}'):
1650             return None
1651         Value = Value.replace(' ', '').replace('{', '').replace('}', '')
1652         ValFields = Value.split(',')
1653         try:
1654             for Index in range(len(ValFields)):
1655                 ValFields[Index] = str(int(ValFields[Index], 0))
1656         except ValueError:
1657             return None
1658         Value = '{' + ','.join(ValFields) + '}'
1659         return Value
1660
1661     Unicode = False
1662     if Value.startswith('L"'):
1663         if not Value.endswith('"'):
1664             return None
1665         Value = Value[1:]
1666         Unicode = True
1667     elif not Value.startswith('"') or not Value.endswith('"'):
1668         return None
1669
1670     Value = eval(Value)         # translate escape character
1671     NewValue = '{'
1672     for Index in range(0, len(Value)):
1673         if Unicode:
1674             NewValue = NewValue + str(ord(Value[Index]) % 0x10000) + ','
1675         else:
1676             NewValue = NewValue + str(ord(Value[Index]) % 0x100) + ','
1677     Value = NewValue + '0}'
1678     return Value
1679
1680 class PathClass(object):
1681     def __init__(self, File='', Root='', AlterRoot='', Type='', IsBinary=False,
1682                  Arch='COMMON', ToolChainFamily='', Target='', TagName='', ToolCode=''):
1683         self.Arch = Arch
1684         self.File = str(File)
1685         if os.path.isabs(self.File):
1686             self.Root = ''
1687             self.AlterRoot = ''
1688         else:
1689             self.Root = str(Root)
1690             self.AlterRoot = str(AlterRoot)
1691
1692         # Remove any '.' and '..' in path
1693         if self.Root:
1694             self.Root = mws.getWs(self.Root, self.File)
1695             self.Path = os.path.normpath(os.path.join(self.Root, self.File))
1696             self.Root = os.path.normpath(CommonPath([self.Root, self.Path]))
1697             # eliminate the side-effect of 'C:'
1698             if self.Root[-1] == ':':
1699                 self.Root += os.path.sep
1700             # file path should not start with path separator
1701             if self.Root[-1] == os.path.sep:
1702                 self.File = self.Path[len(self.Root):]
1703             else:
1704                 self.File = self.Path[len(self.Root) + 1:]
1705         else:
1706             self.Path = os.path.normpath(self.File)
1707
1708         self.SubDir, self.Name = os.path.split(self.File)
1709         self.BaseName, self.Ext = os.path.splitext(self.Name)
1710
1711         if self.Root:
1712             if self.SubDir:
1713                 self.Dir = os.path.join(self.Root, self.SubDir)
1714             else:
1715                 self.Dir = self.Root
1716         else:
1717             self.Dir = self.SubDir
1718
1719         if IsBinary:
1720             self.Type = Type
1721         else:
1722             self.Type = self.Ext.lower()
1723
1724         self.IsBinary = IsBinary
1725         self.Target = Target
1726         self.TagName = TagName
1727         self.ToolCode = ToolCode
1728         self.ToolChainFamily = ToolChainFamily
1729
1730         self._Key = None
1731
1732     ## Convert the object of this class to a string
1733     #
1734     #  Convert member Path of the class to a string
1735     #
1736     #  @retval string Formatted String
1737     #
1738     def __str__(self):
1739         return self.Path
1740
1741     ## Override __eq__ function
1742     #
1743     # Check whether PathClass are the same
1744     #
1745     # @retval False The two PathClass are different
1746     # @retval True  The two PathClass are the same
1747     #
1748     def __eq__(self, Other):
1749         if isinstance(Other, type(self)):
1750             return self.Path == Other.Path
1751         else:
1752             return self.Path == str(Other)
1753
1754     ## Override __cmp__ function
1755     #
1756     # Customize the comparsion operation of two PathClass
1757     #
1758     # @retval 0     The two PathClass are different
1759     # @retval -1    The first PathClass is less than the second PathClass
1760     # @retval 1     The first PathClass is Bigger than the second PathClass
1761     def __cmp__(self, Other):
1762         if isinstance(Other, type(self)):
1763             OtherKey = Other.Path
1764         else:
1765             OtherKey = str(Other)
1766
1767         SelfKey = self.Path
1768         if SelfKey == OtherKey:
1769             return 0
1770         elif SelfKey > OtherKey:
1771             return 1
1772         else:
1773             return -1
1774
1775     ## Override __hash__ function
1776     #
1777     # Use Path as key in hash table
1778     #
1779     # @retval string Key for hash table
1780     #
1781     def __hash__(self):
1782         return hash(self.Path)
1783
1784     def _GetFileKey(self):
1785         if self._Key is None:
1786             self._Key = self.Path.upper()   # + self.ToolChainFamily + self.TagName + self.ToolCode + self.Target
1787         return self._Key
1788
1789     def _GetTimeStamp(self):
1790         return os.stat(self.Path)[8]
1791
1792     def Validate(self, Type='', CaseSensitive=True):
1793         if GlobalData.gCaseInsensitive:
1794             CaseSensitive = False
1795         if Type and Type.lower() != self.Type:
1796             return FILE_TYPE_MISMATCH, '%s (expect %s but got %s)' % (self.File, Type, self.Type)
1797
1798         RealFile, RealRoot = RealPath2(self.File, self.Root, self.AlterRoot)
1799         if not RealRoot and not RealFile:
1800             RealFile = self.File
1801             if self.AlterRoot:
1802                 RealFile = os.path.join(self.AlterRoot, self.File)
1803             elif self.Root:
1804                 RealFile = os.path.join(self.Root, self.File)
1805             if len (mws.getPkgPath()) == 0:
1806                 return FILE_NOT_FOUND, os.path.join(self.AlterRoot, RealFile)
1807             else:
1808                 return FILE_NOT_FOUND, "%s is not found in packages path:\n\t%s" % (self.File, '\n\t'.join(mws.getPkgPath()))
1809
1810         ErrorCode = 0
1811         ErrorInfo = ''
1812         if RealRoot != self.Root or RealFile != self.File:
1813             if CaseSensitive and (RealFile != self.File or (RealRoot != self.Root and RealRoot != self.AlterRoot)):
1814                 ErrorCode = FILE_CASE_MISMATCH
1815                 ErrorInfo = self.File + '\n\t' + RealFile + " [in file system]"
1816
1817             self.SubDir, self.Name = os.path.split(RealFile)
1818             self.BaseName, self.Ext = os.path.splitext(self.Name)
1819             if self.SubDir:
1820                 self.Dir = os.path.join(RealRoot, self.SubDir)
1821             else:
1822                 self.Dir = RealRoot
1823             self.File = RealFile
1824             self.Root = RealRoot
1825             self.Path = os.path.join(RealRoot, RealFile)
1826         return ErrorCode, ErrorInfo
1827
1828     Key = property(_GetFileKey)
1829     TimeStamp = property(_GetTimeStamp)
1830
1831 ## Parse PE image to get the required PE informaion.
1832 #
1833 class PeImageClass():
1834     ## Constructor
1835     #
1836     #   @param  File FilePath of PeImage
1837     #
1838     def __init__(self, PeFile):
1839         self.FileName   = PeFile
1840         self.IsValid    = False
1841         self.Size       = 0
1842         self.EntryPoint = 0
1843         self.SectionAlignment  = 0
1844         self.SectionHeaderList = []
1845         self.ErrorInfo = ''
1846         try:
1847             PeObject = open(PeFile, 'rb')
1848         except:
1849             self.ErrorInfo = self.FileName + ' can not be found\n'
1850             return
1851         # Read DOS header
1852         ByteArray = array.array('B')
1853         ByteArray.fromfile(PeObject, 0x3E)
1854         ByteList = ByteArray.tolist()
1855         # DOS signature should be 'MZ'
1856         if self._ByteListToStr (ByteList[0x0:0x2]) != 'MZ':
1857             self.ErrorInfo = self.FileName + ' has no valid DOS signature MZ'
1858             return
1859
1860         # Read 4 byte PE Signature
1861         PeOffset = self._ByteListToInt(ByteList[0x3C:0x3E])
1862         PeObject.seek(PeOffset)
1863         ByteArray = array.array('B')
1864         ByteArray.fromfile(PeObject, 4)
1865         # PE signature should be 'PE\0\0'
1866         if ByteArray.tostring() != 'PE\0\0':
1867             self.ErrorInfo = self.FileName + ' has no valid PE signature PE00'
1868             return
1869
1870         # Read PE file header
1871         ByteArray = array.array('B')
1872         ByteArray.fromfile(PeObject, 0x14)
1873         ByteList = ByteArray.tolist()
1874         SecNumber = self._ByteListToInt(ByteList[0x2:0x4])
1875         if SecNumber == 0:
1876             self.ErrorInfo = self.FileName + ' has no section header'
1877             return
1878
1879         # Read PE optional header
1880         OptionalHeaderSize = self._ByteListToInt(ByteArray[0x10:0x12])
1881         ByteArray = array.array('B')
1882         ByteArray.fromfile(PeObject, OptionalHeaderSize)
1883         ByteList = ByteArray.tolist()
1884         self.EntryPoint       = self._ByteListToInt(ByteList[0x10:0x14])
1885         self.SectionAlignment = self._ByteListToInt(ByteList[0x20:0x24])
1886         self.Size             = self._ByteListToInt(ByteList[0x38:0x3C])
1887
1888         # Read each Section Header
1889         for Index in range(SecNumber):
1890             ByteArray = array.array('B')
1891             ByteArray.fromfile(PeObject, 0x28)
1892             ByteList = ByteArray.tolist()
1893             SecName  = self._ByteListToStr(ByteList[0:8])
1894             SecVirtualSize = self._ByteListToInt(ByteList[8:12])
1895             SecRawAddress  = self._ByteListToInt(ByteList[20:24])
1896             SecVirtualAddress = self._ByteListToInt(ByteList[12:16])
1897             self.SectionHeaderList.append((SecName, SecVirtualAddress, SecRawAddress, SecVirtualSize))
1898         self.IsValid = True
1899         PeObject.close()
1900
1901     def _ByteListToStr(self, ByteList):
1902         String = ''
1903         for index in range(len(ByteList)):
1904             if ByteList[index] == 0:
1905                 break
1906             String += chr(ByteList[index])
1907         return String
1908
1909     def _ByteListToInt(self, ByteList):
1910         Value = 0
1911         for index in range(len(ByteList) - 1, -1, -1):
1912             Value = (Value << 8) | int(ByteList[index])
1913         return Value
1914
1915 class DefaultStore():
1916     def __init__(self, DefaultStores ):
1917
1918         self.DefaultStores = DefaultStores
1919     def DefaultStoreID(self, DefaultStoreName):
1920         for key, value in self.DefaultStores.items():
1921             if value == DefaultStoreName:
1922                 return key
1923         return None
1924     def GetDefaultDefault(self):
1925         if not self.DefaultStores or "0" in self.DefaultStores:
1926             return "0", TAB_DEFAULT_STORES_DEFAULT
1927         else:
1928             minvalue = min(int(value_str) for value_str in self.DefaultStores)
1929             return (str(minvalue), self.DefaultStores[str(minvalue)])
1930     def GetMin(self, DefaultSIdList):
1931         if not DefaultSIdList:
1932             return TAB_DEFAULT_STORES_DEFAULT
1933         storeidset = {storeid for storeid, storename in self.DefaultStores.values() if storename in DefaultSIdList}
1934         if not storeidset:
1935             return ""
1936         minid = min(storeidset )
1937         for sid, name in self.DefaultStores.values():
1938             if sid == minid:
1939                 return name
1940 class SkuClass():
1941
1942     DEFAULT = 0
1943     SINGLE = 1
1944     MULTIPLE =2
1945
1946     def __init__(self,SkuIdentifier='', SkuIds=None):
1947         if SkuIds is None:
1948             SkuIds = {}
1949
1950         for SkuName in SkuIds:
1951             SkuId = SkuIds[SkuName][0]
1952             skuid_num = int(SkuId, 16) if SkuId.upper().startswith("0X") else int(SkuId)
1953             if skuid_num > 0xFFFFFFFFFFFFFFFF:
1954                 EdkLogger.error("build", PARAMETER_INVALID,
1955                             ExtraData = "SKU-ID [%s] value %s exceeds the max value of UINT64"
1956                                       % (SkuName, SkuId))
1957
1958         self.AvailableSkuIds = sdict()
1959         self.SkuIdSet = []
1960         self.SkuIdNumberSet = []
1961         self.SkuData = SkuIds
1962         self.__SkuInherit = {}
1963         self.__SkuIdentifier = SkuIdentifier
1964         if SkuIdentifier == '' or SkuIdentifier is None:
1965             self.SkuIdSet = ['DEFAULT']
1966             self.SkuIdNumberSet = ['0U']
1967         elif SkuIdentifier == 'ALL':
1968             self.SkuIdSet = SkuIds.keys()
1969             self.SkuIdNumberSet = [num[0].strip() + 'U' for num in SkuIds.values()]
1970         else:
1971             r = SkuIdentifier.split('|')
1972             self.SkuIdSet=[(r[k].strip()).upper() for k in range(len(r))]
1973             k = None
1974             try:
1975                 self.SkuIdNumberSet = [SkuIds[k][0].strip() + 'U' for k in self.SkuIdSet]
1976             except Exception:
1977                 EdkLogger.error("build", PARAMETER_INVALID,
1978                             ExtraData = "SKU-ID [%s] is not supported by the platform. [Valid SKU-ID: %s]"
1979                                       % (k, " | ".join(SkuIds.keys())))
1980         for each in self.SkuIdSet:
1981             if each in SkuIds:
1982                 self.AvailableSkuIds[each] = SkuIds[each][0]
1983             else:
1984                 EdkLogger.error("build", PARAMETER_INVALID,
1985                             ExtraData="SKU-ID [%s] is not supported by the platform. [Valid SKU-ID: %s]"
1986                                       % (each, " | ".join(SkuIds.keys())))
1987         if self.SkuUsageType != self.SINGLE:
1988             self.AvailableSkuIds.update({'DEFAULT':0, 'COMMON':0})
1989         if self.SkuIdSet:
1990             GlobalData.gSkuids = (self.SkuIdSet)
1991             if 'COMMON' in GlobalData.gSkuids:
1992                 GlobalData.gSkuids.remove('COMMON')
1993             if self.SkuUsageType == self.SINGLE:
1994                 if len(GlobalData.gSkuids) != 1:
1995                     if 'DEFAULT' in GlobalData.gSkuids:
1996                         GlobalData.gSkuids.remove('DEFAULT')
1997             if GlobalData.gSkuids:
1998                 GlobalData.gSkuids.sort()
1999
2000     def GetNextSkuId(self, skuname):
2001         if not self.__SkuInherit:
2002             self.__SkuInherit = {}
2003             for item in self.SkuData.values():
2004                 self.__SkuInherit[item[1]]=item[2] if item[2] else "DEFAULT"
2005         return self.__SkuInherit.get(skuname, "DEFAULT")
2006
2007     def GetSkuChain(self, sku):
2008         if sku == "DEFAULT":
2009             return ["DEFAULT"]
2010         skulist = [sku]
2011         nextsku = sku
2012         while True:
2013             nextsku = self.GetNextSkuId(nextsku)
2014             skulist.append(nextsku)
2015             if nextsku == "DEFAULT":
2016                 break
2017         skulist.reverse()
2018         return skulist
2019     def SkuOverrideOrder(self):
2020         skuorderset = []
2021         for skuname in self.SkuIdSet:
2022             skuorderset.append(self.GetSkuChain(skuname))
2023
2024         skuorder = []
2025         for index in range(max(len(item) for item in skuorderset)):
2026             for subset in skuorderset:
2027                 if index > len(subset)-1:
2028                     continue
2029                 if subset[index] in skuorder:
2030                     continue
2031                 skuorder.append(subset[index])
2032
2033         return skuorder
2034
2035     def __SkuUsageType(self):
2036
2037         if self.__SkuIdentifier.upper() == "ALL":
2038             return SkuClass.MULTIPLE
2039
2040         if len(self.SkuIdSet) == 1:
2041             if self.SkuIdSet[0] == 'DEFAULT':
2042                 return SkuClass.DEFAULT
2043             else:
2044                 return SkuClass.SINGLE
2045         elif len(self.SkuIdSet) == 2:
2046             if 'DEFAULT' in self.SkuIdSet:
2047                 return SkuClass.SINGLE
2048             else:
2049                 return SkuClass.MULTIPLE
2050         else:
2051             return SkuClass.MULTIPLE
2052     def DumpSkuIdArrary(self):
2053
2054         ArrayStrList = []
2055         if self.SkuUsageType == SkuClass.SINGLE:
2056             ArrayStr = "{0x0}"
2057         else:
2058             for skuname in self.AvailableSkuIds:
2059                 if skuname == "COMMON":
2060                     continue
2061                 while skuname != "DEFAULT":
2062                     ArrayStrList.append(hex(int(self.AvailableSkuIds[skuname])))
2063                     skuname = self.GetNextSkuId(skuname)
2064                 ArrayStrList.append("0x0")
2065             ArrayStr = "{" + ",".join(ArrayStrList) +  "}"
2066         return ArrayStr
2067     def __GetAvailableSkuIds(self):
2068         return self.AvailableSkuIds
2069
2070     def __GetSystemSkuID(self):
2071         if self.__SkuUsageType() == SkuClass.SINGLE:
2072             if len(self.SkuIdSet) == 1:
2073                 return self.SkuIdSet[0]
2074             else:
2075                 return self.SkuIdSet[0] if self.SkuIdSet[0] != 'DEFAULT' else self.SkuIdSet[1]
2076         else:
2077             return 'DEFAULT'
2078     def __GetAvailableSkuIdNumber(self):
2079         return self.SkuIdNumberSet
2080     SystemSkuId = property(__GetSystemSkuID)
2081     AvailableSkuIdSet = property(__GetAvailableSkuIds)
2082     SkuUsageType = property(__SkuUsageType)
2083     AvailableSkuIdNumSet = property(__GetAvailableSkuIdNumber)
2084
2085 #
2086 # Pack a registry format GUID
2087 #
2088 def PackRegistryFormatGuid(Guid):
2089     return PackGUID(Guid.split('-'))
2090
2091 ##  Get the integer value from string like "14U" or integer like 2
2092 #
2093 #   @param      Input   The object that may be either a integer value or a string
2094 #
2095 #   @retval     Value    The integer value that the input represents
2096 #
2097 def GetIntegerValue(Input):
2098     if type(Input) in (int, long):
2099         return Input
2100     String = Input
2101     if String.endswith("U"):
2102         String = String[:-1]
2103     if String.endswith("ULL"):
2104         String = String[:-3]
2105     if String.endswith("LL"):
2106         String = String[:-2]
2107
2108     if String.startswith("0x") or String.startswith("0X"):
2109         return int(String, 16)
2110     elif String == '':
2111         return 0
2112     else:
2113         return int(String)
2114
2115 #
2116 # Pack a GUID (registry format) list into a buffer and return it
2117 #
2118 def PackGUID(Guid):
2119     return pack(PACK_PATTERN_GUID,
2120                 int(Guid[0], 16),
2121                 int(Guid[1], 16),
2122                 int(Guid[2], 16),
2123                 int(Guid[3][-4:-2], 16),
2124                 int(Guid[3][-2:], 16),
2125                 int(Guid[4][-12:-10], 16),
2126                 int(Guid[4][-10:-8], 16),
2127                 int(Guid[4][-8:-6], 16),
2128                 int(Guid[4][-6:-4], 16),
2129                 int(Guid[4][-4:-2], 16),
2130                 int(Guid[4][-2:], 16)
2131                 )
2132
2133 #
2134 # Pack a GUID (byte) list into a buffer and return it
2135 #
2136 def PackByteFormatGUID(Guid):
2137     return pack(PACK_PATTERN_GUID,
2138                 Guid[0],
2139                 Guid[1],
2140                 Guid[2],
2141                 Guid[3],
2142                 Guid[4],
2143                 Guid[5],
2144                 Guid[6],
2145                 Guid[7],
2146                 Guid[8],
2147                 Guid[9],
2148                 Guid[10],
2149                 )
2150
2151 ##
2152 #
2153 # This acts like the main() function for the script, unless it is 'import'ed into another
2154 # script.
2155 #
2156 if __name__ == '__main__':
2157     pass
2158