BaseTools/Source/Python/Common/Misc.py

   1 ## @file
   2 # Common routines used by all tools
   3 #
   4 # Copyright (c) 2007 - 2018, Intel Corporation. All rights reserved.<BR>
   5 # This program and the accompanying materials
   6 # are licensed and made available under the terms and conditions of the BSD License
   7 # which accompanies this distribution.  The full text of the license may be found at
   8 # http://opensource.org/licenses/bsd-license.php
   9 #
  10 # THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS,
  11 # WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED.
  12 #
  13
  14 ##
  15 # Import Modules
  16 #
  17 from __future__ import absolute_import
  18 import Common.LongFilePathOs as os
  19 import sys
  20 import string
  21 import threading
  22 import time
  23 import re
  24 import pickle
  25 import array
  26 import shutil
  27 from random import sample
  28 from struct import pack
  29 from UserDict import IterableUserDict
  30 from UserList import UserList
  31
  32 from Common import EdkLogger as EdkLogger
  33 from Common import GlobalData as GlobalData
  34 from .DataType import *
  35 from .BuildToolError import *
  36 from CommonDataClass.DataClass import *
  37 from .Parsing import GetSplitValueList
  38 from Common.LongFilePathSupport import OpenLongFilePath as open
  39 from Common.MultipleWorkspace import MultipleWorkspace as mws
  40 import uuid
  41 from CommonDataClass.Exceptions import BadExpression
  42 from Common.caching import cached_property
  43 import subprocess
  44 ## Regular expression used to find out place holders in string template
  45 gPlaceholderPattern = re.compile("\$\{([^$()\s]+)\}", re.MULTILINE | re.UNICODE)
  46
  47 ## regular expressions for map file processing
  48 startPatternGeneral = re.compile("^Start[' ']+Length[' ']+Name[' ']+Class")
  49 addressPatternGeneral = re.compile("^Address[' ']+Publics by Value[' ']+Rva\+Base")
  50 valuePatternGcc = re.compile('^([\w_\.]+) +([\da-fA-Fx]+) +([\da-fA-Fx]+)$')
  51 pcdPatternGcc = re.compile('^([\da-fA-Fx]+) +([\da-fA-Fx]+)')
  52 secReGeneral = re.compile('^([\da-fA-F]+):([\da-fA-F]+) +([\da-fA-F]+)[Hh]? +([.\w\$]+) +(\w+)', re.UNICODE)
  53
  54 StructPattern = re.compile(r'[_a-zA-Z][0-9A-Za-z_]*$')
  55
  56 ## Dictionary used to store file time stamp for quick re-access
  57 gFileTimeStampCache = {}    # {file path : file time stamp}
  58
  59 ## Dictionary used to store dependencies of files
  60 gDependencyDatabase = {}    # arch : {file path : [dependent files list]}
  61
  62 #
  63 # If a module is built more than once with different PCDs or library classes
  64 # a temporary INF file with same content is created, the temporary file is removed
  65 # when build exits.
  66 #
  67 _TempInfs = []
  68
  69 def GetVariableOffset(mapfilepath, efifilepath, varnames):
  70     """ Parse map file to get variable offset in current EFI file
  71     @param mapfilepath    Map file absolution path
  72     @param efifilepath:   EFI binary file full path
  73     @param varnames       iteratable container whose elements are variable names to be searched
  74
  75     @return List whos elements are tuple with variable name and raw offset
  76     """
  77     lines = []
  78     try:
  79         f = open(mapfilepath, 'r')
  80         lines = f.readlines()
  81         f.close()
  82     except:
  83         return None
  84
  85     if len(lines) == 0: return None
  86     firstline = lines[0].strip()
  87     if (firstline.startswith("Archive member included ") and
  88         firstline.endswith(" file (symbol)")):
  89         return _parseForGCC(lines, efifilepath, varnames)
  90     if firstline.startswith("# Path:"):
  91         return _parseForXcode(lines, efifilepath, varnames)
  92     return _parseGeneral(lines, efifilepath, varnames)
  93
  94 def _parseForXcode(lines, efifilepath, varnames):
  95     status = 0
  96     ret = []
  97     for line in lines:
  98         line = line.strip()
  99         if status == 0 and line == "# Symbols:":
 100             status = 1
 101             continue
 102         if status == 1 and len(line) != 0:
 103             for varname in varnames:
 104                 if varname in line:
 105                     # cannot pregenerate this RegEx since it uses varname from varnames.
 106                     m = re.match('^([\da-fA-FxX]+)([\s\S]*)([_]*%s)$' % varname, line)
 107                     if m is not None:
 108                         ret.append((varname, m.group(1)))
 109     return ret
 110
 111 def _parseForGCC(lines, efifilepath, varnames):
 112     """ Parse map file generated by GCC linker """
 113     status = 0
 114     sections = []
 115     varoffset = []
 116     for index, line in enumerate(lines):
 117         line = line.strip()
 118         # status machine transection
 119         if status == 0 and line == "Memory Configuration":
 120             status = 1
 121             continue
 122         elif status == 1 and line == 'Linker script and memory map':
 123             status = 2
 124             continue
 125         elif status ==2 and line == 'START GROUP':
 126             status = 3
 127             continue
 128
 129         # status handler
 130         if status == 3:
 131             m = valuePatternGcc.match(line)
 132             if m is not None:
 133                 sections.append(m.groups(0))
 134             for varname in varnames:
 135                 Str = ''
 136                 m = re.match("^.data.(%s)" % varname, line)
 137                 if m is not None:
 138                     m = re.match(".data.(%s)$" % varname, line)
 139                     if m is not None:
 140                         Str = lines[index + 1]
 141                     else:
 142                         Str = line[len(".data.%s" % varname):]
 143                     if Str:
 144                         m = pcdPatternGcc.match(Str.strip())
 145                         if m is not None:
 146                             varoffset.append((varname, int(m.groups(0)[0], 16), int(sections[-1][1], 16), sections[-1][0]))
 147
 148     if not varoffset:
 149         return []
 150     # get section information from efi file
 151     efisecs = PeImageClass(efifilepath).SectionHeaderList
 152     if efisecs is None or len(efisecs) == 0:
 153         return []
 154     #redirection
 155     redirection = 0
 156     for efisec in efisecs:
 157         for section in sections:
 158             if section[0].strip() == efisec[0].strip() and section[0].strip() == '.text':
 159                 redirection = int(section[1], 16) - efisec[1]
 160
 161     ret = []
 162     for var in varoffset:
 163         for efisec in efisecs:
 164             if var[1] >= efisec[1] and var[1] < efisec[1]+efisec[3]:
 165                 ret.append((var[0], hex(efisec[2] + var[1] - efisec[1] - redirection)))
 166     return ret
 167
 168 def _parseGeneral(lines, efifilepath, varnames):
 169     status = 0    #0 - beginning of file; 1 - PE section definition; 2 - symbol table
 170     secs  = []    # key = section name
 171     varoffset = []
 172     symRe = re.compile('^([\da-fA-F]+):([\da-fA-F]+) +([\.:\\\\\w\?@\$]+) +([\da-fA-F]+)', re.UNICODE)
 173
 174     for line in lines:
 175         line = line.strip()
 176         if startPatternGeneral.match(line):
 177             status = 1
 178             continue
 179         if addressPatternGeneral.match(line):
 180             status = 2
 181             continue
 182         if line.startswith("entry point at"):
 183             status = 3
 184             continue
 185         if status == 1 and len(line) != 0:
 186             m =  secReGeneral.match(line)
 187             assert m is not None, "Fail to parse the section in map file , line is %s" % line
 188             sec_no, sec_start, sec_length, sec_name, sec_class = m.groups(0)
 189             secs.append([int(sec_no, 16), int(sec_start, 16), int(sec_length, 16), sec_name, sec_class])
 190         if status == 2 and len(line) != 0:
 191             for varname in varnames:
 192                 m = symRe.match(line)
 193                 assert m is not None, "Fail to parse the symbol in map file, line is %s" % line
 194                 sec_no, sym_offset, sym_name, vir_addr = m.groups(0)
 195                 sec_no     = int(sec_no,     16)
 196                 sym_offset = int(sym_offset, 16)
 197                 vir_addr   = int(vir_addr,   16)
 198                 # cannot pregenerate this RegEx since it uses varname from varnames.
 199                 m2 = re.match('^[_]*(%s)' % varname, sym_name)
 200                 if m2 is not None:
 201                     # fond a binary pcd entry in map file
 202                     for sec in secs:
 203                         if sec[0] == sec_no and (sym_offset >= sec[1] and sym_offset < sec[1] + sec[2]):
 204                             varoffset.append([varname, sec[3], sym_offset, vir_addr, sec_no])
 205
 206     if not varoffset: return []
 207
 208     # get section information from efi file
 209     efisecs = PeImageClass(efifilepath).SectionHeaderList
 210     if efisecs is None or len(efisecs) == 0:
 211         return []
 212
 213     ret = []
 214     for var in varoffset:
 215         index = 0
 216         for efisec in efisecs:
 217             index = index + 1
 218             if var[1].strip() == efisec[0].strip():
 219                 ret.append((var[0], hex(efisec[2] + var[2])))
 220             elif var[4] == index:
 221                 ret.append((var[0], hex(efisec[2] + var[2])))
 222
 223     return ret
 224
 225 ## Routine to process duplicated INF
 226 #
 227 #  This function is called by following two cases:
 228 #  Case 1 in DSC:
 229 #    [components.arch]
 230 #    Pkg/module/module.inf
 231 #    Pkg/module/module.inf {
 232 #      <Defines>
 233 #        FILE_GUID = 0D1B936F-68F3-4589-AFCC-FB8B7AEBC836
 234 #    }
 235 #  Case 2 in FDF:
 236 #    INF Pkg/module/module.inf
 237 #    INF FILE_GUID = 0D1B936F-68F3-4589-AFCC-FB8B7AEBC836 Pkg/module/module.inf
 238 #
 239 #  This function copies Pkg/module/module.inf to
 240 #  Conf/.cache/0D1B936F-68F3-4589-AFCC-FB8B7AEBC836module.inf
 241 #
 242 #  @param Path     Original PathClass object
 243 #  @param BaseName New file base name
 244 #
 245 #  @retval         return the new PathClass object
 246 #
 247 def ProcessDuplicatedInf(Path, BaseName, Workspace):
 248     Filename = os.path.split(Path.File)[1]
 249     if '.' in Filename:
 250         Filename = BaseName + Path.BaseName + Filename[Filename.rfind('.'):]
 251     else:
 252         Filename = BaseName + Path.BaseName
 253
 254     #
 255     # If -N is specified on command line, cache is disabled
 256     # The directory has to be created
 257     #
 258     DbDir = os.path.split(GlobalData.gDatabasePath)[0]
 259     if not os.path.exists(DbDir):
 260         os.makedirs(DbDir)
 261     #
 262     # A temporary INF is copied to database path which must have write permission
 263     # The temporary will be removed at the end of build
 264     # In case of name conflict, the file name is
 265     # FILE_GUIDBaseName (0D1B936F-68F3-4589-AFCC-FB8B7AEBC836module.inf)
 266     #
 267     TempFullPath = os.path.join(DbDir,
 268                                 Filename)
 269     RtPath = PathClass(Path.File, Workspace)
 270     #
 271     # Modify the full path to temporary path, keep other unchanged
 272     #
 273     # To build same module more than once, the module path with FILE_GUID overridden has
 274     # the file name FILE_GUIDmodule.inf, but the relative path (self.MetaFile.File) is the real path
 275     # in DSC which is used as relative path by C files and other files in INF.
 276     # A trick was used: all module paths are PathClass instances, after the initialization
 277     # of PathClass, the PathClass.Path is overridden by the temporary INF path.
 278     #
 279     # The reason for creating a temporary INF is:
 280     # Platform.Modules which is the base to create ModuleAutoGen objects is a dictionary,
 281     # the key is the full path of INF, the value is an object to save overridden library instances, PCDs.
 282     # A different key for the same module is needed to create different output directory,
 283     # retrieve overridden PCDs, library instances.
 284     #
 285     # The BaseName is the FILE_GUID which is also the output directory name.
 286     #
 287     #
 288     RtPath.Path = TempFullPath
 289     RtPath.BaseName = BaseName
 290     #
 291     # If file exists, compare contents
 292     #
 293     if os.path.exists(TempFullPath):
 294         with open(str(Path), 'rb') as f1, open(TempFullPath, 'rb') as f2:
 295             if f1.read() == f2.read():
 296                 return RtPath
 297     _TempInfs.append(TempFullPath)
 298     shutil.copy2(str(Path), TempFullPath)
 299     return RtPath
 300
 301 ## Remove temporary created INFs whose paths were saved in _TempInfs
 302 #
 303 def ClearDuplicatedInf():
 304     while _TempInfs:
 305         File = _TempInfs.pop()
 306         if os.path.exists(File):
 307             os.remove(File)
 308
 309 ## Convert GUID string in xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx style to C structure style
 310 #
 311 #   @param      Guid    The GUID string
 312 #
 313 #   @retval     string  The GUID string in C structure style
 314 #
 315 def GuidStringToGuidStructureString(Guid):
 316     GuidList = Guid.split('-')
 317     Result = '{'
 318     for Index in range(0, 3, 1):
 319         Result = Result + '0x' + GuidList[Index] + ', '
 320     Result = Result + '{0x' + GuidList[3][0:2] + ', 0x' + GuidList[3][2:4]
 321     for Index in range(0, 12, 2):
 322         Result = Result + ', 0x' + GuidList[4][Index:Index + 2]
 323     Result += '}}'
 324     return Result
 325
 326 ## Convert GUID structure in byte array to xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
 327 #
 328 #   @param      GuidValue   The GUID value in byte array
 329 #
 330 #   @retval     string      The GUID value in xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx format
 331 #
 332 def GuidStructureByteArrayToGuidString(GuidValue):
 333     guidValueString = GuidValue.lower().replace("{", "").replace("}", "").replace(" ", "").replace(";", "")
 334     guidValueList = guidValueString.split(",")
 335     if len(guidValueList) != 16:
 336         return ''
 337         #EdkLogger.error(None, None, "Invalid GUID value string %s" % GuidValue)
 338     try:
 339         return "%02x%02x%02x%02x-%02x%02x-%02x%02x-%02x%02x-%02x%02x%02x%02x%02x%02x" % (
 340                 int(guidValueList[3], 16),
 341                 int(guidValueList[2], 16),
 342                 int(guidValueList[1], 16),
 343                 int(guidValueList[0], 16),
 344                 int(guidValueList[5], 16),
 345                 int(guidValueList[4], 16),
 346                 int(guidValueList[7], 16),
 347                 int(guidValueList[6], 16),
 348                 int(guidValueList[8], 16),
 349                 int(guidValueList[9], 16),
 350                 int(guidValueList[10], 16),
 351                 int(guidValueList[11], 16),
 352                 int(guidValueList[12], 16),
 353                 int(guidValueList[13], 16),
 354                 int(guidValueList[14], 16),
 355                 int(guidValueList[15], 16)
 356                 )
 357     except:
 358         return ''
 359
 360 ## Convert GUID string in C structure style to xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
 361 #
 362 #   @param      GuidValue   The GUID value in C structure format
 363 #
 364 #   @retval     string      The GUID value in xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx format
 365 #
 366 def GuidStructureStringToGuidString(GuidValue):
 367     if not GlobalData.gGuidCFormatPattern.match(GuidValue):
 368         return ''
 369     guidValueString = GuidValue.lower().replace("{", "").replace("}", "").replace(" ", "").replace(";", "")
 370     guidValueList = guidValueString.split(",")
 371     if len(guidValueList) != 11:
 372         return ''
 373         #EdkLogger.error(None, None, "Invalid GUID value string %s" % GuidValue)
 374     try:
 375         return "%08x-%04x-%04x-%02x%02x-%02x%02x%02x%02x%02x%02x" % (
 376                 int(guidValueList[0], 16),
 377                 int(guidValueList[1], 16),
 378                 int(guidValueList[2], 16),
 379                 int(guidValueList[3], 16),
 380                 int(guidValueList[4], 16),
 381                 int(guidValueList[5], 16),
 382                 int(guidValueList[6], 16),
 383                 int(guidValueList[7], 16),
 384                 int(guidValueList[8], 16),
 385                 int(guidValueList[9], 16),
 386                 int(guidValueList[10], 16)
 387                 )
 388     except:
 389         return ''
 390
 391 ## Convert GUID string in C structure style to xxxxxxxx_xxxx_xxxx_xxxx_xxxxxxxxxxxx
 392 #
 393 #   @param      GuidValue   The GUID value in C structure format
 394 #
 395 #   @retval     string      The GUID value in xxxxxxxx_xxxx_xxxx_xxxx_xxxxxxxxxxxx format
 396 #
 397 def GuidStructureStringToGuidValueName(GuidValue):
 398     guidValueString = GuidValue.lower().replace("{", "").replace("}", "").replace(" ", "")
 399     guidValueList = guidValueString.split(",")
 400     if len(guidValueList) != 11:
 401         EdkLogger.error(None, FORMAT_INVALID, "Invalid GUID value string [%s]" % GuidValue)
 402     return "%08x_%04x_%04x_%02x%02x_%02x%02x%02x%02x%02x%02x" % (
 403             int(guidValueList[0], 16),
 404             int(guidValueList[1], 16),
 405             int(guidValueList[2], 16),
 406             int(guidValueList[3], 16),
 407             int(guidValueList[4], 16),
 408             int(guidValueList[5], 16),
 409             int(guidValueList[6], 16),
 410             int(guidValueList[7], 16),
 411             int(guidValueList[8], 16),
 412             int(guidValueList[9], 16),
 413             int(guidValueList[10], 16)
 414             )
 415
 416 ## Create directories
 417 #
 418 #   @param      Directory   The directory name
 419 #
 420 def CreateDirectory(Directory):
 421     if Directory is None or Directory.strip() == "":
 422         return True
 423     try:
 424         if not os.access(Directory, os.F_OK):
 425             os.makedirs(Directory)
 426     except:
 427         return False
 428     return True
 429
 430 ## Remove directories, including files and sub-directories in it
 431 #
 432 #   @param      Directory   The directory name
 433 #
 434 def RemoveDirectory(Directory, Recursively=False):
 435     if Directory is None or Directory.strip() == "" or not os.path.exists(Directory):
 436         return
 437     if Recursively:
 438         CurrentDirectory = os.getcwd()
 439         os.chdir(Directory)
 440         for File in os.listdir("."):
 441             if os.path.isdir(File):
 442                 RemoveDirectory(File, Recursively)
 443             else:
 444                 os.remove(File)
 445         os.chdir(CurrentDirectory)
 446     os.rmdir(Directory)
 447
 448 ## Store content in file
 449 #
 450 #  This method is used to save file only when its content is changed. This is
 451 #  quite useful for "make" system to decide what will be re-built and what won't.
 452 #
 453 #   @param      File            The path of file
 454 #   @param      Content         The new content of the file
 455 #   @param      IsBinaryFile    The flag indicating if the file is binary file or not
 456 #
 457 #   @retval     True            If the file content is changed and the file is renewed
 458 #   @retval     False           If the file content is the same
 459 #
 460 def SaveFileOnChange(File, Content, IsBinaryFile=True):
 461     if not IsBinaryFile:
 462         Content = Content.replace("\n", os.linesep)
 463
 464     if os.path.exists(File):
 465         try:
 466             if Content == open(File, "rb").read():
 467                 return False
 468         except:
 469             EdkLogger.error(None, FILE_OPEN_FAILURE, ExtraData=File)
 470
 471     DirName = os.path.dirname(File)
 472     if not CreateDirectory(DirName):
 473         EdkLogger.error(None, FILE_CREATE_FAILURE, "Could not create directory %s" % DirName)
 474     else:
 475         if DirName == '':
 476             DirName = os.getcwd()
 477         if not os.access(DirName, os.W_OK):
 478             EdkLogger.error(None, PERMISSION_FAILURE, "Do not have write permission on directory %s" % DirName)
 479
 480     try:
 481         Fd = open(File, "wb")
 482         Fd.write(Content)
 483         Fd.close()
 484     except IOError as X:
 485         EdkLogger.error(None, FILE_CREATE_FAILURE, ExtraData='IOError %s' % X)
 486
 487     return True
 488
 489 ## Make a Python object persistent on file system
 490 #
 491 #   @param      Data    The object to be stored in file
 492 #   @param      File    The path of file to store the object
 493 #
 494 def DataDump(Data, File):
 495     Fd = None
 496     try:
 497         Fd = open(File, 'wb')
 498         pickle.dump(Data, Fd, pickle.HIGHEST_PROTOCOL)
 499     except:
 500         EdkLogger.error("", FILE_OPEN_FAILURE, ExtraData=File, RaiseError=False)
 501     finally:
 502         if Fd is not None:
 503             Fd.close()
 504
 505 ## Restore a Python object from a file
 506 #
 507 #   @param      File    The path of file stored the object
 508 #
 509 #   @retval     object  A python object
 510 #   @retval     None    If failure in file operation
 511 #
 512 def DataRestore(File):
 513     Data = None
 514     Fd = None
 515     try:
 516         Fd = open(File, 'rb')
 517         Data = pickle.load(Fd)
 518     except Exception as e:
 519         EdkLogger.verbose("Failed to load [%s]\n\t%s" % (File, str(e)))
 520         Data = None
 521     finally:
 522         if Fd is not None:
 523             Fd.close()
 524     return Data
 525
 526 ## Retrieve and cache the real path name in file system
 527 #
 528 #   @param      Root    The root directory of path relative to
 529 #
 530 #   @retval     str     The path string if the path exists
 531 #   @retval     None    If path doesn't exist
 532 #
 533 class DirCache:
 534     _CACHE_ = set()
 535     _UPPER_CACHE_ = {}
 536
 537     def __init__(self, Root):
 538         self._Root = Root
 539         for F in os.listdir(Root):
 540             self._CACHE_.add(F)
 541             self._UPPER_CACHE_[F.upper()] = F
 542
 543     # =[] operator
 544     def __getitem__(self, Path):
 545         Path = Path[len(os.path.commonprefix([Path, self._Root])):]
 546         if not Path:
 547             return self._Root
 548         if Path and Path[0] == os.path.sep:
 549             Path = Path[1:]
 550         if Path in self._CACHE_:
 551             return os.path.join(self._Root, Path)
 552         UpperPath = Path.upper()
 553         if UpperPath in self._UPPER_CACHE_:
 554             return os.path.join(self._Root, self._UPPER_CACHE_[UpperPath])
 555
 556         IndexList = []
 557         LastSepIndex = -1
 558         SepIndex = Path.find(os.path.sep)
 559         while SepIndex > -1:
 560             Parent = UpperPath[:SepIndex]
 561             if Parent not in self._UPPER_CACHE_:
 562                 break
 563             LastSepIndex = SepIndex
 564             SepIndex = Path.find(os.path.sep, LastSepIndex + 1)
 565
 566         if LastSepIndex == -1:
 567             return None
 568
 569         Cwd = os.getcwd()
 570         os.chdir(self._Root)
 571         SepIndex = LastSepIndex
 572         while SepIndex > -1:
 573             Parent = Path[:SepIndex]
 574             ParentKey = UpperPath[:SepIndex]
 575             if ParentKey not in self._UPPER_CACHE_:
 576                 os.chdir(Cwd)
 577                 return None
 578
 579             if Parent in self._CACHE_:
 580                 ParentDir = Parent
 581             else:
 582                 ParentDir = self._UPPER_CACHE_[ParentKey]
 583             for F in os.listdir(ParentDir):
 584                 Dir = os.path.join(ParentDir, F)
 585                 self._CACHE_.add(Dir)
 586                 self._UPPER_CACHE_[Dir.upper()] = Dir
 587
 588             SepIndex = Path.find(os.path.sep, SepIndex + 1)
 589
 590         os.chdir(Cwd)
 591         if Path in self._CACHE_:
 592             return os.path.join(self._Root, Path)
 593         elif UpperPath in self._UPPER_CACHE_:
 594             return os.path.join(self._Root, self._UPPER_CACHE_[UpperPath])
 595         return None
 596
 597 def RealPath(File, Dir='', OverrideDir=''):
 598     NewFile = os.path.normpath(os.path.join(Dir, File))
 599     NewFile = GlobalData.gAllFiles[NewFile]
 600     if not NewFile and OverrideDir:
 601         NewFile = os.path.normpath(os.path.join(OverrideDir, File))
 602         NewFile = GlobalData.gAllFiles[NewFile]
 603     return NewFile
 604
 605 def RealPath2(File, Dir='', OverrideDir=''):
 606     NewFile = None
 607     if OverrideDir:
 608         NewFile = GlobalData.gAllFiles[os.path.normpath(os.path.join(OverrideDir, File))]
 609         if NewFile:
 610             if OverrideDir[-1] == os.path.sep:
 611                 return NewFile[len(OverrideDir):], NewFile[0:len(OverrideDir)]
 612             else:
 613                 return NewFile[len(OverrideDir) + 1:], NewFile[0:len(OverrideDir)]
 614     if GlobalData.gAllFiles:
 615         NewFile = GlobalData.gAllFiles[os.path.normpath(os.path.join(Dir, File))]
 616     if not NewFile:
 617         NewFile = os.path.normpath(os.path.join(Dir, File))
 618         if not os.path.exists(NewFile):
 619             return None, None
 620     if NewFile:
 621         if Dir:
 622             if Dir[-1] == os.path.sep:
 623                 return NewFile[len(Dir):], NewFile[0:len(Dir)]
 624             else:
 625                 return NewFile[len(Dir) + 1:], NewFile[0:len(Dir)]
 626         else:
 627             return NewFile, ''
 628
 629     return None, None
 630
 631 ## Get GUID value from given packages
 632 #
 633 #   @param      CName           The CName of the GUID
 634 #   @param      PackageList     List of packages looking-up in
 635 #   @param      Inffile         The driver file
 636 #
 637 #   @retval     GuidValue   if the CName is found in any given package
 638 #   @retval     None        if the CName is not found in all given packages
 639 #
 640 def GuidValue(CName, PackageList, Inffile = None):
 641     for P in PackageList:
 642         GuidKeys = P.Guids.keys()
 643         if Inffile and P._PrivateGuids:
 644             if not Inffile.startswith(P.MetaFile.Dir):
 645                 GuidKeys = [x for x in P.Guids if x not in P._PrivateGuids]
 646         if CName in GuidKeys:
 647             return P.Guids[CName]
 648     return None
 649
 650 ## Get Protocol value from given packages
 651 #
 652 #   @param      CName           The CName of the GUID
 653 #   @param      PackageList     List of packages looking-up in
 654 #   @param      Inffile         The driver file
 655 #
 656 #   @retval     GuidValue   if the CName is found in any given package
 657 #   @retval     None        if the CName is not found in all given packages
 658 #
 659 def ProtocolValue(CName, PackageList, Inffile = None):
 660     for P in PackageList:
 661         ProtocolKeys = P.Protocols.keys()
 662         if Inffile and P._PrivateProtocols:
 663             if not Inffile.startswith(P.MetaFile.Dir):
 664                 ProtocolKeys = [x for x in P.Protocols if x not in P._PrivateProtocols]
 665         if CName in ProtocolKeys:
 666             return P.Protocols[CName]
 667     return None
 668
 669 ## Get PPI value from given packages
 670 #
 671 #   @param      CName           The CName of the GUID
 672 #   @param      PackageList     List of packages looking-up in
 673 #   @param      Inffile         The driver file
 674 #
 675 #   @retval     GuidValue   if the CName is found in any given package
 676 #   @retval     None        if the CName is not found in all given packages
 677 #
 678 def PpiValue(CName, PackageList, Inffile = None):
 679     for P in PackageList:
 680         PpiKeys = P.Ppis.keys()
 681         if Inffile and P._PrivatePpis:
 682             if not Inffile.startswith(P.MetaFile.Dir):
 683                 PpiKeys = [x for x in P.Ppis if x not in P._PrivatePpis]
 684         if CName in PpiKeys:
 685             return P.Ppis[CName]
 686     return None
 687
 688 ## A string template class
 689 #
 690 #  This class implements a template for string replacement. A string template
 691 #  looks like following
 692 #
 693 #       ${BEGIN} other_string ${placeholder_name} other_string ${END}
 694 #
 695 #  The string between ${BEGIN} and ${END} will be repeated as many times as the
 696 #  length of "placeholder_name", which is a list passed through a dict. The
 697 #  "placeholder_name" is the key name of the dict. The ${BEGIN} and ${END} can
 698 #  be not used and, in this case, the "placeholder_name" must not a list and it
 699 #  will just be replaced once.
 700 #
 701 class TemplateString(object):
 702     _REPEAT_START_FLAG = "BEGIN"
 703     _REPEAT_END_FLAG = "END"
 704
 705     class Section(object):
 706         _LIST_TYPES = [type([]), type(set()), type((0,))]
 707
 708         def __init__(self, TemplateSection, PlaceHolderList):
 709             self._Template = TemplateSection
 710             self._PlaceHolderList = []
 711
 712             # Split the section into sub-sections according to the position of placeholders
 713             if PlaceHolderList:
 714                 self._SubSectionList = []
 715                 SubSectionStart = 0
 716                 #
 717                 # The placeholders passed in must be in the format of
 718                 #
 719                 #   PlaceHolderName, PlaceHolderStartPoint, PlaceHolderEndPoint
 720                 #
 721                 for PlaceHolder, Start, End in PlaceHolderList:
 722                     self._SubSectionList.append(TemplateSection[SubSectionStart:Start])
 723                     self._SubSectionList.append(TemplateSection[Start:End])
 724                     self._PlaceHolderList.append(PlaceHolder)
 725                     SubSectionStart = End
 726                 if SubSectionStart < len(TemplateSection):
 727                     self._SubSectionList.append(TemplateSection[SubSectionStart:])
 728             else:
 729                 self._SubSectionList = [TemplateSection]
 730
 731         def __str__(self):
 732             return self._Template + " : " + str(self._PlaceHolderList)
 733
 734         def Instantiate(self, PlaceHolderValues):
 735             RepeatTime = -1
 736             RepeatPlaceHolders = {}
 737             NonRepeatPlaceHolders = {}
 738
 739             for PlaceHolder in self._PlaceHolderList:
 740                 if PlaceHolder not in PlaceHolderValues:
 741                     continue
 742                 Value = PlaceHolderValues[PlaceHolder]
 743                 if type(Value) in self._LIST_TYPES:
 744                     if RepeatTime < 0:
 745                         RepeatTime = len(Value)
 746                     elif RepeatTime != len(Value):
 747                         EdkLogger.error(
 748                                     "TemplateString",
 749                                     PARAMETER_INVALID,
 750                                     "${%s} has different repeat time from others!" % PlaceHolder,
 751                                     ExtraData=str(self._Template)
 752                                     )
 753                     RepeatPlaceHolders["${%s}" % PlaceHolder] = Value
 754                 else:
 755                     NonRepeatPlaceHolders["${%s}" % PlaceHolder] = Value
 756
 757             if NonRepeatPlaceHolders:
 758                 StringList = []
 759                 for S in self._SubSectionList:
 760                     if S not in NonRepeatPlaceHolders:
 761                         StringList.append(S)
 762                     else:
 763                         StringList.append(str(NonRepeatPlaceHolders[S]))
 764             else:
 765                 StringList = self._SubSectionList
 766
 767             if RepeatPlaceHolders:
 768                 TempStringList = []
 769                 for Index in range(RepeatTime):
 770                     for S in StringList:
 771                         if S not in RepeatPlaceHolders:
 772                             TempStringList.append(S)
 773                         else:
 774                             TempStringList.append(str(RepeatPlaceHolders[S][Index]))
 775                 StringList = TempStringList
 776
 777             return "".join(StringList)
 778
 779     ## Constructor
 780     def __init__(self, Template=None):
 781         self.String = []
 782         self.IsBinary = False
 783         self._Template = Template
 784         self._TemplateSectionList = self._Parse(Template)
 785
 786     ## str() operator
 787     #
 788     #   @retval     string  The string replaced
 789     #
 790     def __str__(self):
 791         return "".join(self.String)
 792
 793     ## Split the template string into fragments per the ${BEGIN} and ${END} flags
 794     #
 795     #   @retval     list    A list of TemplateString.Section objects
 796     #
 797     def _Parse(self, Template):
 798         SectionStart = 0
 799         SearchFrom = 0
 800         MatchEnd = 0
 801         PlaceHolderList = []
 802         TemplateSectionList = []
 803         while Template:
 804             MatchObj = gPlaceholderPattern.search(Template, SearchFrom)
 805             if not MatchObj:
 806                 if MatchEnd <= len(Template):
 807                     TemplateSection = TemplateString.Section(Template[SectionStart:], PlaceHolderList)
 808                     TemplateSectionList.append(TemplateSection)
 809                 break
 810
 811             MatchString = MatchObj.group(1)
 812             MatchStart = MatchObj.start()
 813             MatchEnd = MatchObj.end()
 814
 815             if MatchString == self._REPEAT_START_FLAG:
 816                 if MatchStart > SectionStart:
 817                     TemplateSection = TemplateString.Section(Template[SectionStart:MatchStart], PlaceHolderList)
 818                     TemplateSectionList.append(TemplateSection)
 819                 SectionStart = MatchEnd
 820                 PlaceHolderList = []
 821             elif MatchString == self._REPEAT_END_FLAG:
 822                 TemplateSection = TemplateString.Section(Template[SectionStart:MatchStart], PlaceHolderList)
 823                 TemplateSectionList.append(TemplateSection)
 824                 SectionStart = MatchEnd
 825                 PlaceHolderList = []
 826             else:
 827                 PlaceHolderList.append((MatchString, MatchStart - SectionStart, MatchEnd - SectionStart))
 828             SearchFrom = MatchEnd
 829         return TemplateSectionList
 830
 831     ## Replace the string template with dictionary of placeholders and append it to previous one
 832     #
 833     #   @param      AppendString    The string template to append
 834     #   @param      Dictionary      The placeholder dictionaries
 835     #
 836     def Append(self, AppendString, Dictionary=None):
 837         if Dictionary:
 838             SectionList = self._Parse(AppendString)
 839             self.String.append( "".join(S.Instantiate(Dictionary) for S in SectionList))
 840         else:
 841             if isinstance(AppendString,list):
 842                 self.String.extend(AppendString)
 843             else:
 844                 self.String.append(AppendString)
 845
 846     ## Replace the string template with dictionary of placeholders
 847     #
 848     #   @param      Dictionary      The placeholder dictionaries
 849     #
 850     #   @retval     str             The string replaced with placeholder values
 851     #
 852     def Replace(self, Dictionary=None):
 853         return "".join(S.Instantiate(Dictionary) for S in self._TemplateSectionList)
 854
 855 ## Progress indicator class
 856 #
 857 #  This class makes use of thread to print progress on console.
 858 #
 859 class Progressor:
 860     # for avoiding deadloop
 861     _StopFlag = None
 862     _ProgressThread = None
 863     _CheckInterval = 0.25
 864
 865     ## Constructor
 866     #
 867     #   @param      OpenMessage     The string printed before progress charaters
 868     #   @param      CloseMessage    The string printed after progress charaters
 869     #   @param      ProgressChar    The charater used to indicate the progress
 870     #   @param      Interval        The interval in seconds between two progress charaters
 871     #
 872     def __init__(self, OpenMessage="", CloseMessage="", ProgressChar='.', Interval=1.0):
 873         self.PromptMessage = OpenMessage
 874         self.CodaMessage = CloseMessage
 875         self.ProgressChar = ProgressChar
 876         self.Interval = Interval
 877         if Progressor._StopFlag is None:
 878             Progressor._StopFlag = threading.Event()
 879
 880     ## Start to print progress charater
 881     #
 882     #   @param      OpenMessage     The string printed before progress charaters
 883     #
 884     def Start(self, OpenMessage=None):
 885         if OpenMessage is not None:
 886             self.PromptMessage = OpenMessage
 887         Progressor._StopFlag.clear()
 888         if Progressor._ProgressThread is None:
 889             Progressor._ProgressThread = threading.Thread(target=self._ProgressThreadEntry)
 890             Progressor._ProgressThread.setDaemon(False)
 891             Progressor._ProgressThread.start()
 892
 893     ## Stop printing progress charater
 894     #
 895     #   @param      CloseMessage    The string printed after progress charaters
 896     #
 897     def Stop(self, CloseMessage=None):
 898         OriginalCodaMessage = self.CodaMessage
 899         if CloseMessage is not None:
 900             self.CodaMessage = CloseMessage
 901         self.Abort()
 902         self.CodaMessage = OriginalCodaMessage
 903
 904     ## Thread entry method
 905     def _ProgressThreadEntry(self):
 906         sys.stdout.write(self.PromptMessage + " ")
 907         sys.stdout.flush()
 908         TimeUp = 0.0
 909         while not Progressor._StopFlag.isSet():
 910             if TimeUp <= 0.0:
 911                 sys.stdout.write(self.ProgressChar)
 912                 sys.stdout.flush()
 913                 TimeUp = self.Interval
 914             time.sleep(self._CheckInterval)
 915             TimeUp -= self._CheckInterval
 916         sys.stdout.write(" " + self.CodaMessage + "\n")
 917         sys.stdout.flush()
 918
 919     ## Abort the progress display
 920     @staticmethod
 921     def Abort():
 922         if Progressor._StopFlag is not None:
 923             Progressor._StopFlag.set()
 924         if Progressor._ProgressThread is not None:
 925             Progressor._ProgressThread.join()
 926             Progressor._ProgressThread = None
 927
 928 ## A dict which can access its keys and/or values orderly
 929 #
 930 #  The class implements a new kind of dict which its keys or values can be
 931 #  accessed in the order they are added into the dict. It guarantees the order
 932 #  by making use of an internal list to keep a copy of keys.
 933 #
 934 class sdict(IterableUserDict):
 935     ## Constructor
 936     def __init__(self):
 937         IterableUserDict.__init__(self)
 938         self._key_list = []
 939
 940     ## [] operator
 941     def __setitem__(self, key, value):
 942         if key not in self._key_list:
 943             self._key_list.append(key)
 944         IterableUserDict.__setitem__(self, key, value)
 945
 946     ## del operator
 947     def __delitem__(self, key):
 948         self._key_list.remove(key)
 949         IterableUserDict.__delitem__(self, key)
 950
 951     ## used in "for k in dict" loop to ensure the correct order
 952     def __iter__(self):
 953         return self.iterkeys()
 954
 955     ## len() support
 956     def __len__(self):
 957         return len(self._key_list)
 958
 959     ## "in" test support
 960     def __contains__(self, key):
 961         return key in self._key_list
 962
 963     ## indexof support
 964     def index(self, key):
 965         return self._key_list.index(key)
 966
 967     ## insert support
 968     def insert(self, key, newkey, newvalue, order):
 969         index = self._key_list.index(key)
 970         if order == 'BEFORE':
 971             self._key_list.insert(index, newkey)
 972             IterableUserDict.__setitem__(self, newkey, newvalue)
 973         elif order == 'AFTER':
 974             self._key_list.insert(index + 1, newkey)
 975             IterableUserDict.__setitem__(self, newkey, newvalue)
 976
 977     ## append support
 978     def append(self, sdict):
 979         for key in sdict:
 980             if key not in self._key_list:
 981                 self._key_list.append(key)
 982             IterableUserDict.__setitem__(self, key, sdict[key])
 983
 984     def has_key(self, key):
 985         return key in self._key_list
 986
 987     ## Empty the dict
 988     def clear(self):
 989         self._key_list = []
 990         IterableUserDict.clear(self)
 991
 992     ## Return a copy of keys
 993     def keys(self):
 994         keys = []
 995         for key in self._key_list:
 996             keys.append(key)
 997         return keys
 998
 999     ## Return a copy of values
1000     def values(self):
1001         values = []
1002         for key in self._key_list:
1003             values.append(self[key])
1004         return values
1005
1006     ## Return a copy of (key, value) list
1007     def items(self):
1008         items = []
1009         for key in self._key_list:
1010             items.append((key, self[key]))
1011         return items
1012
1013     ## Iteration support
1014     def iteritems(self):
1015         return iter(self.items())
1016
1017     ## Keys interation support
1018     def iterkeys(self):
1019         return iter(self.keys())
1020
1021     ## Values interation support
1022     def itervalues(self):
1023         return iter(self.values())
1024
1025     ## Return value related to a key, and remove the (key, value) from the dict
1026     def pop(self, key, *dv):
1027         value = None
1028         if key in self._key_list:
1029             value = self[key]
1030             self.__delitem__(key)
1031         elif len(dv) != 0 :
1032             value = kv[0]
1033         return value
1034
1035     ## Return (key, value) pair, and remove the (key, value) from the dict
1036     def popitem(self):
1037         key = self._key_list[-1]
1038         value = self[key]
1039         self.__delitem__(key)
1040         return key, value
1041
1042     def update(self, dict=None, **kwargs):
1043         if dict is not None:
1044             for k, v in dict.items():
1045                 self[k] = v
1046         if len(kwargs):
1047             for k, v in kwargs.items():
1048                 self[k] = v
1049
1050 ## Dictionary with restricted keys
1051 #
1052 class rdict(dict):
1053     ## Constructor
1054     def __init__(self, KeyList):
1055         for Key in KeyList:
1056             dict.__setitem__(self, Key, "")
1057
1058     ## []= operator
1059     def __setitem__(self, key, value):
1060         if key not in self:
1061             EdkLogger.error("RestrictedDict", ATTRIBUTE_SET_FAILURE, "Key [%s] is not allowed" % key,
1062                             ExtraData=", ".join(dict.keys(self)))
1063         dict.__setitem__(self, key, value)
1064
1065     ## =[] operator
1066     def __getitem__(self, key):
1067         if key not in self:
1068             return ""
1069         return dict.__getitem__(self, key)
1070
1071     ## del operator
1072     def __delitem__(self, key):
1073         EdkLogger.error("RestrictedDict", ATTRIBUTE_ACCESS_DENIED, ExtraData="del")
1074
1075     ## Empty the dict
1076     def clear(self):
1077         for Key in self:
1078             self.__setitem__(Key, "")
1079
1080     ## Return value related to a key, and remove the (key, value) from the dict
1081     def pop(self, key, *dv):
1082         EdkLogger.error("RestrictedDict", ATTRIBUTE_ACCESS_DENIED, ExtraData="pop")
1083
1084     ## Return (key, value) pair, and remove the (key, value) from the dict
1085     def popitem(self):
1086         EdkLogger.error("RestrictedDict", ATTRIBUTE_ACCESS_DENIED, ExtraData="popitem")
1087
1088 ## Dictionary using prioritized list as key
1089 #
1090 class tdict:
1091     _ListType = type([])
1092     _TupleType = type(())
1093     _Wildcard = 'COMMON'
1094     _ValidWildcardList = ['COMMON', 'DEFAULT', 'ALL', TAB_STAR, 'PLATFORM']
1095
1096     def __init__(self, _Single_=False, _Level_=2):
1097         self._Level_ = _Level_
1098         self.data = {}
1099         self._Single_ = _Single_
1100
1101     # =[] operator
1102     def __getitem__(self, key):
1103         KeyType = type(key)
1104         RestKeys = None
1105         if KeyType == self._ListType or KeyType == self._TupleType:
1106             FirstKey = key[0]
1107             if len(key) > 1:
1108                 RestKeys = key[1:]
1109             elif self._Level_ > 1:
1110                 RestKeys = [self._Wildcard for i in range(0, self._Level_ - 1)]
1111         else:
1112             FirstKey = key
1113             if self._Level_ > 1:
1114                 RestKeys = [self._Wildcard for i in range(0, self._Level_ - 1)]
1115
1116         if FirstKey is None or str(FirstKey).upper() in self._ValidWildcardList:
1117             FirstKey = self._Wildcard
1118
1119         if self._Single_:
1120             return self._GetSingleValue(FirstKey, RestKeys)
1121         else:
1122             return self._GetAllValues(FirstKey, RestKeys)
1123
1124     def _GetSingleValue(self, FirstKey, RestKeys):
1125         Value = None
1126         #print "%s-%s" % (FirstKey, self._Level_) ,
1127         if self._Level_ > 1:
1128             if FirstKey == self._Wildcard:
1129                 if FirstKey in self.data:
1130                     Value = self.data[FirstKey][RestKeys]
1131                 if Value is None:
1132                     for Key in self.data:
1133                         Value = self.data[Key][RestKeys]
1134                         if Value is not None: break
1135             else:
1136                 if FirstKey in self.data:
1137                     Value = self.data[FirstKey][RestKeys]
1138                 if Value is None and self._Wildcard in self.data:
1139                     #print "Value=None"
1140                     Value = self.data[self._Wildcard][RestKeys]
1141         else:
1142             if FirstKey == self._Wildcard:
1143                 if FirstKey in self.data:
1144                     Value = self.data[FirstKey]
1145                 if Value is None:
1146                     for Key in self.data:
1147                         Value = self.data[Key]
1148                         if Value is not None: break
1149             else:
1150                 if FirstKey in self.data:
1151                     Value = self.data[FirstKey]
1152                 elif self._Wildcard in self.data:
1153                     Value = self.data[self._Wildcard]
1154         return Value
1155
1156     def _GetAllValues(self, FirstKey, RestKeys):
1157         Value = []
1158         if self._Level_ > 1:
1159             if FirstKey == self._Wildcard:
1160                 for Key in self.data:
1161                     Value += self.data[Key][RestKeys]
1162             else:
1163                 if FirstKey in self.data:
1164                     Value += self.data[FirstKey][RestKeys]
1165                 if self._Wildcard in self.data:
1166                     Value += self.data[self._Wildcard][RestKeys]
1167         else:
1168             if FirstKey == self._Wildcard:
1169                 for Key in self.data:
1170                     Value.append(self.data[Key])
1171             else:
1172                 if FirstKey in self.data:
1173                     Value.append(self.data[FirstKey])
1174                 if self._Wildcard in self.data:
1175                     Value.append(self.data[self._Wildcard])
1176         return Value
1177
1178     ## []= operator
1179     def __setitem__(self, key, value):
1180         KeyType = type(key)
1181         RestKeys = None
1182         if KeyType == self._ListType or KeyType == self._TupleType:
1183             FirstKey = key[0]
1184             if len(key) > 1:
1185                 RestKeys = key[1:]
1186             else:
1187                 RestKeys = [self._Wildcard for i in range(0, self._Level_ - 1)]
1188         else:
1189             FirstKey = key
1190             if self._Level_ > 1:
1191                 RestKeys = [self._Wildcard for i in range(0, self._Level_ - 1)]
1192
1193         if FirstKey in self._ValidWildcardList:
1194             FirstKey = self._Wildcard
1195
1196         if FirstKey not in self.data and self._Level_ > 0:
1197             self.data[FirstKey] = tdict(self._Single_, self._Level_ - 1)
1198
1199         if self._Level_ > 1:
1200             self.data[FirstKey][RestKeys] = value
1201         else:
1202             self.data[FirstKey] = value
1203
1204     def SetGreedyMode(self):
1205         self._Single_ = False
1206         if self._Level_ > 1:
1207             for Key in self.data:
1208                 self.data[Key].SetGreedyMode()
1209
1210     def SetSingleMode(self):
1211         self._Single_ = True
1212         if self._Level_ > 1:
1213             for Key in self.data:
1214                 self.data[Key].SetSingleMode()
1215
1216     def GetKeys(self, KeyIndex=0):
1217         assert KeyIndex >= 0
1218         if KeyIndex == 0:
1219             return set(self.data.keys())
1220         else:
1221             keys = set()
1222             for Key in self.data:
1223                 keys |= self.data[Key].GetKeys(KeyIndex - 1)
1224             return keys
1225
1226 def IsFieldValueAnArray (Value):
1227     Value = Value.strip()
1228     if Value.startswith(TAB_GUID) and Value.endswith(')'):
1229         return True
1230     if Value.startswith('L"') and Value.endswith('"')  and len(list(Value[2:-1])) > 1:
1231         return True
1232     if Value[0] == '"' and Value[-1] == '"' and len(list(Value[1:-1])) > 1:
1233         return True
1234     if Value[0] == '{' and Value[-1] == '}':
1235         return True
1236     if Value.startswith("L'") and Value.endswith("'") and len(list(Value[2:-1])) > 1:
1237         return True
1238     if Value[0] == "'" and Value[-1] == "'" and len(list(Value[1:-1])) > 1:
1239         return True
1240     return False
1241
1242 def AnalyzePcdExpression(Setting):
1243     RanStr = ''.join(sample(string.ascii_letters + string.digits, 8))
1244     Setting = Setting.replace('\\\\', RanStr).strip()
1245     # There might be escaped quote in a string: \", \\\" , \', \\\'
1246     Data = Setting
1247     # There might be '|' in string and in ( ... | ... ), replace it with '-'
1248     NewStr = ''
1249     InSingleQuoteStr = False
1250     InDoubleQuoteStr = False
1251     Pair = 0
1252     for Index, ch in enumerate(Data):
1253         if ch == '"' and not InSingleQuoteStr:
1254             if Data[Index - 1] != '\\':
1255                 InDoubleQuoteStr = not InDoubleQuoteStr
1256         elif ch == "'" and not InDoubleQuoteStr:
1257             if Data[Index - 1] != '\\':
1258                 InSingleQuoteStr = not InSingleQuoteStr
1259         elif ch == '(' and not (InSingleQuoteStr or InDoubleQuoteStr):
1260             Pair += 1
1261         elif ch == ')' and not (InSingleQuoteStr or InDoubleQuoteStr):
1262             Pair -= 1
1263
1264         if (Pair > 0 or InSingleQuoteStr or InDoubleQuoteStr) and ch == TAB_VALUE_SPLIT:
1265             NewStr += '-'
1266         else:
1267             NewStr += ch
1268     FieldList = []
1269     StartPos = 0
1270     while True:
1271         Pos = NewStr.find(TAB_VALUE_SPLIT, StartPos)
1272         if Pos < 0:
1273             FieldList.append(Setting[StartPos:].strip())
1274             break
1275         FieldList.append(Setting[StartPos:Pos].strip())
1276         StartPos = Pos + 1
1277     for i, ch in enumerate(FieldList):
1278         if RanStr in ch:
1279             FieldList[i] = ch.replace(RanStr,'\\\\')
1280     return FieldList
1281
1282 def ParseDevPathValue (Value):
1283     if '\\' in Value:
1284         Value.replace('\\', '/').replace(' ', '')
1285
1286     Cmd = 'DevicePath ' + '"' + Value + '"'
1287     try:
1288         p = subprocess.Popen(Cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
1289         out, err = p.communicate()
1290     except Exception as X:
1291         raise BadExpression("DevicePath: %s" % (str(X)) )
1292     finally:
1293         subprocess._cleanup()
1294         p.stdout.close()
1295         p.stderr.close()
1296     if err:
1297         raise BadExpression("DevicePath: %s" % str(err))
1298     Size = len(out.split())
1299     out = ','.join(out.split())
1300     return '{' + out + '}', Size
1301
1302 def ParseFieldValue (Value):
1303     if "{CODE(" in Value:
1304         return Value, len(Value.split(","))
1305     if isinstance(Value, type(0)):
1306         return Value, (Value.bit_length() + 7) / 8
1307     if not isinstance(Value, type('')):
1308         raise BadExpression('Type %s is %s' %(Value, type(Value)))
1309     Value = Value.strip()
1310     if Value.startswith(TAB_UINT8) and Value.endswith(')'):
1311         Value, Size = ParseFieldValue(Value.split('(', 1)[1][:-1])
1312         if Size > 1:
1313             raise BadExpression('Value (%s) Size larger than %d' %(Value, Size))
1314         return Value, 1
1315     if Value.startswith(TAB_UINT16) and Value.endswith(')'):
1316         Value, Size = ParseFieldValue(Value.split('(', 1)[1][:-1])
1317         if Size > 2:
1318             raise BadExpression('Value (%s) Size larger than %d' %(Value, Size))
1319         return Value, 2
1320     if Value.startswith(TAB_UINT32) and Value.endswith(')'):
1321         Value, Size = ParseFieldValue(Value.split('(', 1)[1][:-1])
1322         if Size > 4:
1323             raise BadExpression('Value (%s) Size larger than %d' %(Value, Size))
1324         return Value, 4
1325     if Value.startswith(TAB_UINT64) and Value.endswith(')'):
1326         Value, Size = ParseFieldValue(Value.split('(', 1)[1][:-1])
1327         if Size > 8:
1328             raise BadExpression('Value (%s) Size larger than %d' % (Value, Size))
1329         return Value, 8
1330     if Value.startswith(TAB_GUID) and Value.endswith(')'):
1331         Value = Value.split('(', 1)[1][:-1].strip()
1332         if Value[0] == '{' and Value[-1] == '}':
1333             TmpValue = GuidStructureStringToGuidString(Value)
1334             if not TmpValue:
1335                 raise BadExpression("Invalid GUID value string %s" % Value)
1336             Value = TmpValue
1337         if Value[0] == '"' and Value[-1] == '"':
1338             Value = Value[1:-1]
1339         try:
1340             Value = "'" + uuid.UUID(Value).get_bytes_le() + "'"
1341         except ValueError as Message:
1342             raise BadExpression(Message)
1343         Value, Size = ParseFieldValue(Value)
1344         return Value, 16
1345     if Value.startswith('L"') and Value.endswith('"'):
1346         # Unicode String
1347         # translate escape character
1348         Value = Value[1:]
1349         try:
1350             Value = eval(Value)
1351         except:
1352             Value = Value[1:-1]
1353         List = list(Value)
1354         List.reverse()
1355         Value = 0
1356         for Char in List:
1357             Value = (Value << 16) | ord(Char)
1358         return Value, (len(List) + 1) * 2
1359     if Value.startswith('"') and Value.endswith('"'):
1360         # ASCII String
1361         # translate escape character
1362         try:
1363             Value = eval(Value)
1364         except:
1365             Value = Value[1:-1]
1366         List = list(Value)
1367         List.reverse()
1368         Value = 0
1369         for Char in List:
1370             Value = (Value << 8) | ord(Char)
1371         return Value, len(List) + 1
1372     if Value.startswith("L'") and Value.endswith("'"):
1373         # Unicode Character Constant
1374         # translate escape character
1375         Value = Value[1:]
1376         try:
1377             Value = eval(Value)
1378         except:
1379             Value = Value[1:-1]
1380         List = list(Value)
1381         if len(List) == 0:
1382             raise BadExpression('Length %s is %s' % (Value, len(List)))
1383         List.reverse()
1384         Value = 0
1385         for Char in List:
1386             Value = (Value << 16) | ord(Char)
1387         return Value, len(List) * 2
1388     if Value.startswith("'") and Value.endswith("'"):
1389         # Character constant
1390         # translate escape character
1391         try:
1392             Value = eval(Value)
1393         except:
1394             Value = Value[1:-1]
1395         List = list(Value)
1396         if len(List) == 0:
1397             raise BadExpression('Length %s is %s' % (Value, len(List)))
1398         List.reverse()
1399         Value = 0
1400         for Char in List:
1401             Value = (Value << 8) | ord(Char)
1402         return Value, len(List)
1403     if Value.startswith('{') and Value.endswith('}'):
1404         # Byte array
1405         Value = Value[1:-1]
1406         List = [Item.strip() for Item in Value.split(',')]
1407         List.reverse()
1408         Value = 0
1409         RetSize = 0
1410         for Item in List:
1411             ItemValue, Size = ParseFieldValue(Item)
1412             RetSize += Size
1413             for I in range(Size):
1414                 Value = (Value << 8) | ((ItemValue >> 8 * I) & 0xff)
1415         return Value, RetSize
1416     if Value.startswith('DEVICE_PATH(') and Value.endswith(')'):
1417         Value = Value.replace("DEVICE_PATH(", '').rstrip(')')
1418         Value = Value.strip().strip('"')
1419         return ParseDevPathValue(Value)
1420     if Value.lower().startswith('0x'):
1421         try:
1422             Value = int(Value, 16)
1423         except:
1424             raise BadExpression("invalid hex value: %s" % Value)
1425         if Value == 0:
1426             return 0, 1
1427         return Value, (Value.bit_length() + 7) / 8
1428     if Value[0].isdigit():
1429         Value = int(Value, 10)
1430         if Value == 0:
1431             return 0, 1
1432         return Value, (Value.bit_length() + 7) / 8
1433     if Value.lower() == 'true':
1434         return 1, 1
1435     if Value.lower() == 'false':
1436         return 0, 1
1437     return Value, 1
1438
1439 ## AnalyzeDscPcd
1440 #
1441 #  Analyze DSC PCD value, since there is no data type info in DSC
1442 #  This function is used to match functions (AnalyzePcdData) used for retrieving PCD value from database
1443 #  1. Feature flag: TokenSpace.PcdCName|PcdValue
1444 #  2. Fix and Patch:TokenSpace.PcdCName|PcdValue[|VOID*[|MaxSize]]
1445 #  3. Dynamic default:
1446 #     TokenSpace.PcdCName|PcdValue[|VOID*[|MaxSize]]
1447 #     TokenSpace.PcdCName|PcdValue
1448 #  4. Dynamic VPD:
1449 #     TokenSpace.PcdCName|VpdOffset[|VpdValue]
1450 #     TokenSpace.PcdCName|VpdOffset[|MaxSize[|VpdValue]]
1451 #  5. Dynamic HII:
1452 #     TokenSpace.PcdCName|HiiString|VariableGuid|VariableOffset[|HiiValue]
1453 #  PCD value needs to be located in such kind of string, and the PCD value might be an expression in which
1454 #    there might have "|" operator, also in string value.
1455 #
1456 #  @param Setting: String contain information described above with "TokenSpace.PcdCName|" stripped
1457 #  @param PcdType: PCD type: feature, fixed, dynamic default VPD HII
1458 #  @param DataType: The datum type of PCD: VOID*, UNIT, BOOL
1459 #  @retval:
1460 #    ValueList: A List contain fields described above
1461 #    IsValid:   True if conforming EBNF, otherwise False
1462 #    Index:     The index where PcdValue is in ValueList
1463 #
1464 def AnalyzeDscPcd(Setting, PcdType, DataType=''):
1465     FieldList = AnalyzePcdExpression(Setting)
1466
1467     IsValid = True
1468     if PcdType in (MODEL_PCD_FIXED_AT_BUILD, MODEL_PCD_PATCHABLE_IN_MODULE, MODEL_PCD_DYNAMIC_DEFAULT, MODEL_PCD_DYNAMIC_EX_DEFAULT):
1469         Value = FieldList[0]
1470         Size = ''
1471         if len(FieldList) > 1 and FieldList[1]:
1472             DataType = FieldList[1]
1473             if FieldList[1] != TAB_VOID and StructPattern.match(FieldList[1]) is None:
1474                 IsValid = False
1475         if len(FieldList) > 2:
1476             Size = FieldList[2]
1477         if IsValid:
1478             if DataType == "":
1479                 IsValid = (len(FieldList) <= 1)
1480             else:
1481                 IsValid = (len(FieldList) <= 3)
1482
1483         if Size:
1484             try:
1485                 int(Size, 16) if Size.upper().startswith("0X") else int(Size)
1486             except:
1487                 IsValid = False
1488                 Size = -1
1489         return [str(Value), DataType, str(Size)], IsValid, 0
1490     elif PcdType == MODEL_PCD_FEATURE_FLAG:
1491         Value = FieldList[0]
1492         Size = ''
1493         IsValid = (len(FieldList) <= 1)
1494         return [Value, DataType, str(Size)], IsValid, 0
1495     elif PcdType in (MODEL_PCD_DYNAMIC_VPD, MODEL_PCD_DYNAMIC_EX_VPD):
1496         VpdOffset = FieldList[0]
1497         Value = Size = ''
1498         if not DataType == TAB_VOID:
1499             if len(FieldList) > 1:
1500                 Value = FieldList[1]
1501         else:
1502             if len(FieldList) > 1:
1503                 Size = FieldList[1]
1504             if len(FieldList) > 2:
1505                 Value = FieldList[2]
1506         if DataType == "":
1507             IsValid = (len(FieldList) <= 1)
1508         else:
1509             IsValid = (len(FieldList) <= 3)
1510         if Size:
1511             try:
1512                 int(Size, 16) if Size.upper().startswith("0X") else int(Size)
1513             except:
1514                 IsValid = False
1515                 Size = -1
1516         return [VpdOffset, str(Size), Value], IsValid, 2
1517     elif PcdType in (MODEL_PCD_DYNAMIC_HII, MODEL_PCD_DYNAMIC_EX_HII):
1518         IsValid = (3 <= len(FieldList) <= 5)
1519         HiiString = FieldList[0]
1520         Guid = Offset = Value = Attribute = ''
1521         if len(FieldList) > 1:
1522             Guid = FieldList[1]
1523         if len(FieldList) > 2:
1524             Offset = FieldList[2]
1525         if len(FieldList) > 3:
1526             Value = FieldList[3]
1527             if not Value:
1528                 IsValid = False
1529         if len(FieldList) > 4:
1530             Attribute = FieldList[4]
1531         return [HiiString, Guid, Offset, Value, Attribute], IsValid, 3
1532     return [], False, 0
1533
1534 ## AnalyzePcdData
1535 #
1536 #  Analyze the pcd Value, Datum type and TokenNumber.
1537 #  Used to avoid split issue while the value string contain "|" character
1538 #
1539 #  @param[in] Setting:  A String contain value/datum type/token number information;
1540 #
1541 #  @retval   ValueList: A List contain value, datum type and toke number.
1542 #
1543 def AnalyzePcdData(Setting):
1544     ValueList = ['', '', '']
1545
1546     ValueRe = re.compile(r'^\s*L?\".*\|.*\"')
1547     PtrValue = ValueRe.findall(Setting)
1548
1549     ValueUpdateFlag = False
1550
1551     if len(PtrValue) >= 1:
1552         Setting = re.sub(ValueRe, '', Setting)
1553         ValueUpdateFlag = True
1554
1555     TokenList = Setting.split(TAB_VALUE_SPLIT)
1556     ValueList[0:len(TokenList)] = TokenList
1557
1558     if ValueUpdateFlag:
1559         ValueList[0] = PtrValue[0]
1560
1561     return ValueList
1562
1563 ## check format of PCD value against its the datum type
1564 #
1565 # For PCD value setting
1566 #
1567 def CheckPcdDatum(Type, Value):
1568     if Type == TAB_VOID:
1569         ValueRe = re.compile(r'\s*L?\".*\"\s*$')
1570         if not (((Value.startswith('L"') or Value.startswith('"')) and Value.endswith('"'))
1571                 or (Value.startswith('{') and Value.endswith('}')) or (Value.startswith("L'") or Value.startswith("'") and Value.endswith("'"))
1572                ):
1573             return False, "Invalid value [%s] of type [%s]; must be in the form of {...} for array"\
1574                           ", \"...\" or \'...\' for string, L\"...\" or L\'...\' for unicode string" % (Value, Type)
1575         elif ValueRe.match(Value):
1576             # Check the chars in UnicodeString or CString is printable
1577             if Value.startswith("L"):
1578                 Value = Value[2:-1]
1579             else:
1580                 Value = Value[1:-1]
1581             Printset = set(string.printable)
1582             Printset.remove(TAB_PRINTCHAR_VT)
1583             Printset.add(TAB_PRINTCHAR_BS)
1584             Printset.add(TAB_PRINTCHAR_NUL)
1585             if not set(Value).issubset(Printset):
1586                 PrintList = sorted(Printset)
1587                 return False, "Invalid PCD string value of type [%s]; must be printable chars %s." % (Type, PrintList)
1588     elif Type == 'BOOLEAN':
1589         if Value not in ['TRUE', 'True', 'true', '0x1', '0x01', '1', 'FALSE', 'False', 'false', '0x0', '0x00', '0']:
1590             return False, "Invalid value [%s] of type [%s]; must be one of TRUE, True, true, 0x1, 0x01, 1"\
1591                           ", FALSE, False, false, 0x0, 0x00, 0" % (Value, Type)
1592     elif Type in [TAB_UINT8, TAB_UINT16, TAB_UINT32, TAB_UINT64]:
1593         if Value and int(Value, 0) < 0:
1594             return False, "PCD can't be set to negative value[%s] for datum type [%s]" % (Value, Type)
1595         try:
1596             Value = long(Value, 0)
1597             if Value > MAX_VAL_TYPE[Type]:
1598                 return False, "Too large PCD value[%s] for datum type [%s]" % (Value, Type)
1599         except:
1600             return False, "Invalid value [%s] of type [%s];"\
1601                           " must be a hexadecimal, decimal or octal in C language format." % (Value, Type)
1602     else:
1603         return True, "StructurePcd"
1604
1605     return True, ""
1606
1607 ## Split command line option string to list
1608 #
1609 # subprocess.Popen needs the args to be a sequence. Otherwise there's problem
1610 # in non-windows platform to launch command
1611 #
1612 def SplitOption(OptionString):
1613     OptionList = []
1614     LastChar = " "
1615     OptionStart = 0
1616     QuotationMark = ""
1617     for Index in range(0, len(OptionString)):
1618         CurrentChar = OptionString[Index]
1619         if CurrentChar in ['"', "'"]:
1620             if QuotationMark == CurrentChar:
1621                 QuotationMark = ""
1622             elif QuotationMark == "":
1623                 QuotationMark = CurrentChar
1624             continue
1625         elif QuotationMark:
1626             continue
1627
1628         if CurrentChar in ["/", "-"] and LastChar in [" ", "\t", "\r", "\n"]:
1629             if Index > OptionStart:
1630                 OptionList.append(OptionString[OptionStart:Index - 1])
1631             OptionStart = Index
1632         LastChar = CurrentChar
1633     OptionList.append(OptionString[OptionStart:])
1634     return OptionList
1635
1636 def CommonPath(PathList):
1637     P1 = min(PathList).split(os.path.sep)
1638     P2 = max(PathList).split(os.path.sep)
1639     for Index in xrange(min(len(P1), len(P2))):
1640         if P1[Index] != P2[Index]:
1641             return os.path.sep.join(P1[:Index])
1642     return os.path.sep.join(P1)
1643
1644 #
1645 # Convert string to C format array
1646 #
1647 def ConvertStringToByteArray(Value):
1648     Value = Value.strip()
1649     if not Value:
1650         return None
1651     if Value[0] == '{':
1652         if not Value.endswith('}'):
1653             return None
1654         Value = Value.replace(' ', '').replace('{', '').replace('}', '')
1655         ValFields = Value.split(',')
1656         try:
1657             for Index in range(len(ValFields)):
1658                 ValFields[Index] = str(int(ValFields[Index], 0))
1659         except ValueError:
1660             return None
1661         Value = '{' + ','.join(ValFields) + '}'
1662         return Value
1663
1664     Unicode = False
1665     if Value.startswith('L"'):
1666         if not Value.endswith('"'):
1667             return None
1668         Value = Value[1:]
1669         Unicode = True
1670     elif not Value.startswith('"') or not Value.endswith('"'):
1671         return None
1672
1673     Value = eval(Value)         # translate escape character
1674     NewValue = '{'
1675     for Index in range(0, len(Value)):
1676         if Unicode:
1677             NewValue = NewValue + str(ord(Value[Index]) % 0x10000) + ','
1678         else:
1679             NewValue = NewValue + str(ord(Value[Index]) % 0x100) + ','
1680     Value = NewValue + '0}'
1681     return Value
1682
1683 class PathClass(object):
1684     def __init__(self, File='', Root='', AlterRoot='', Type='', IsBinary=False,
1685                  Arch='COMMON', ToolChainFamily='', Target='', TagName='', ToolCode=''):
1686         self.Arch = Arch
1687         self.File = str(File)
1688         if os.path.isabs(self.File):
1689             self.Root = ''
1690             self.AlterRoot = ''
1691         else:
1692             self.Root = str(Root)
1693             self.AlterRoot = str(AlterRoot)
1694
1695         # Remove any '.' and '..' in path
1696         if self.Root:
1697             self.Root = mws.getWs(self.Root, self.File)
1698             self.Path = os.path.normpath(os.path.join(self.Root, self.File))
1699             self.Root = os.path.normpath(CommonPath([self.Root, self.Path]))
1700             # eliminate the side-effect of 'C:'
1701             if self.Root[-1] == ':':
1702                 self.Root += os.path.sep
1703             # file path should not start with path separator
1704             if self.Root[-1] == os.path.sep:
1705                 self.File = self.Path[len(self.Root):]
1706             else:
1707                 self.File = self.Path[len(self.Root) + 1:]
1708         else:
1709             self.Path = os.path.normpath(self.File)
1710
1711         self.SubDir, self.Name = os.path.split(self.File)
1712         self.BaseName, self.Ext = os.path.splitext(self.Name)
1713
1714         if self.Root:
1715             if self.SubDir:
1716                 self.Dir = os.path.join(self.Root, self.SubDir)
1717             else:
1718                 self.Dir = self.Root
1719         else:
1720             self.Dir = self.SubDir
1721
1722         if IsBinary:
1723             self.Type = Type
1724         else:
1725             self.Type = self.Ext.lower()
1726
1727         self.IsBinary = IsBinary
1728         self.Target = Target
1729         self.TagName = TagName
1730         self.ToolCode = ToolCode
1731         self.ToolChainFamily = ToolChainFamily
1732
1733     ## Convert the object of this class to a string
1734     #
1735     #  Convert member Path of the class to a string
1736     #
1737     #  @retval string Formatted String
1738     #
1739     def __str__(self):
1740         return self.Path
1741
1742     ## Override __eq__ function
1743     #
1744     # Check whether PathClass are the same
1745     #
1746     # @retval False The two PathClass are different
1747     # @retval True  The two PathClass are the same
1748     #
1749     def __eq__(self, Other):
1750         return self.Path == str(Other)
1751
1752     ## Override __cmp__ function
1753     #
1754     # Customize the comparsion operation of two PathClass
1755     #
1756     # @retval 0     The two PathClass are different
1757     # @retval -1    The first PathClass is less than the second PathClass
1758     # @retval 1     The first PathClass is Bigger than the second PathClass
1759     def __cmp__(self, Other):
1760         OtherKey = str(Other)
1761
1762         SelfKey = self.Path
1763         if SelfKey == OtherKey:
1764             return 0
1765         elif SelfKey > OtherKey:
1766             return 1
1767         else:
1768             return -1
1769
1770     ## Override __hash__ function
1771     #
1772     # Use Path as key in hash table
1773     #
1774     # @retval string Key for hash table
1775     #
1776     def __hash__(self):
1777         return hash(self.Path)
1778
1779     @cached_property
1780     def Key(self):
1781         return self.Path.upper()
1782
1783     @property
1784     def TimeStamp(self):
1785         return os.stat(self.Path)[8]
1786
1787     def Validate(self, Type='', CaseSensitive=True):
1788         if GlobalData.gCaseInsensitive:
1789             CaseSensitive = False
1790         if Type and Type.lower() != self.Type:
1791             return FILE_TYPE_MISMATCH, '%s (expect %s but got %s)' % (self.File, Type, self.Type)
1792
1793         RealFile, RealRoot = RealPath2(self.File, self.Root, self.AlterRoot)
1794         if not RealRoot and not RealFile:
1795             RealFile = self.File
1796             if self.AlterRoot:
1797                 RealFile = os.path.join(self.AlterRoot, self.File)
1798             elif self.Root:
1799                 RealFile = os.path.join(self.Root, self.File)
1800             if len (mws.getPkgPath()) == 0:
1801                 return FILE_NOT_FOUND, os.path.join(self.AlterRoot, RealFile)
1802             else:
1803                 return FILE_NOT_FOUND, "%s is not found in packages path:\n\t%s" % (self.File, '\n\t'.join(mws.getPkgPath()))
1804
1805         ErrorCode = 0
1806         ErrorInfo = ''
1807         if RealRoot != self.Root or RealFile != self.File:
1808             if CaseSensitive and (RealFile != self.File or (RealRoot != self.Root and RealRoot != self.AlterRoot)):
1809                 ErrorCode = FILE_CASE_MISMATCH
1810                 ErrorInfo = self.File + '\n\t' + RealFile + " [in file system]"
1811
1812             self.SubDir, self.Name = os.path.split(RealFile)
1813             self.BaseName, self.Ext = os.path.splitext(self.Name)
1814             if self.SubDir:
1815                 self.Dir = os.path.join(RealRoot, self.SubDir)
1816             else:
1817                 self.Dir = RealRoot
1818             self.File = RealFile
1819             self.Root = RealRoot
1820             self.Path = os.path.join(RealRoot, RealFile)
1821         return ErrorCode, ErrorInfo
1822
1823 ## Parse PE image to get the required PE informaion.
1824 #
1825 class PeImageClass():
1826     ## Constructor
1827     #
1828     #   @param  File FilePath of PeImage
1829     #
1830     def __init__(self, PeFile):
1831         self.FileName   = PeFile
1832         self.IsValid    = False
1833         self.Size       = 0
1834         self.EntryPoint = 0
1835         self.SectionAlignment  = 0
1836         self.SectionHeaderList = []
1837         self.ErrorInfo = ''
1838         try:
1839             PeObject = open(PeFile, 'rb')
1840         except:
1841             self.ErrorInfo = self.FileName + ' can not be found\n'
1842             return
1843         # Read DOS header
1844         ByteArray = array.array('B')
1845         ByteArray.fromfile(PeObject, 0x3E)
1846         ByteList = ByteArray.tolist()
1847         # DOS signature should be 'MZ'
1848         if self._ByteListToStr (ByteList[0x0:0x2]) != 'MZ':
1849             self.ErrorInfo = self.FileName + ' has no valid DOS signature MZ'
1850             return
1851
1852         # Read 4 byte PE Signature
1853         PeOffset = self._ByteListToInt(ByteList[0x3C:0x3E])
1854         PeObject.seek(PeOffset)
1855         ByteArray = array.array('B')
1856         ByteArray.fromfile(PeObject, 4)
1857         # PE signature should be 'PE\0\0'
1858         if ByteArray.tostring() != 'PE\0\0':
1859             self.ErrorInfo = self.FileName + ' has no valid PE signature PE00'
1860             return
1861
1862         # Read PE file header
1863         ByteArray = array.array('B')
1864         ByteArray.fromfile(PeObject, 0x14)
1865         ByteList = ByteArray.tolist()
1866         SecNumber = self._ByteListToInt(ByteList[0x2:0x4])
1867         if SecNumber == 0:
1868             self.ErrorInfo = self.FileName + ' has no section header'
1869             return
1870
1871         # Read PE optional header
1872         OptionalHeaderSize = self._ByteListToInt(ByteArray[0x10:0x12])
1873         ByteArray = array.array('B')
1874         ByteArray.fromfile(PeObject, OptionalHeaderSize)
1875         ByteList = ByteArray.tolist()
1876         self.EntryPoint       = self._ByteListToInt(ByteList[0x10:0x14])
1877         self.SectionAlignment = self._ByteListToInt(ByteList[0x20:0x24])
1878         self.Size             = self._ByteListToInt(ByteList[0x38:0x3C])
1879
1880         # Read each Section Header
1881         for Index in range(SecNumber):
1882             ByteArray = array.array('B')
1883             ByteArray.fromfile(PeObject, 0x28)
1884             ByteList = ByteArray.tolist()
1885             SecName  = self._ByteListToStr(ByteList[0:8])
1886             SecVirtualSize = self._ByteListToInt(ByteList[8:12])
1887             SecRawAddress  = self._ByteListToInt(ByteList[20:24])
1888             SecVirtualAddress = self._ByteListToInt(ByteList[12:16])
1889             self.SectionHeaderList.append((SecName, SecVirtualAddress, SecRawAddress, SecVirtualSize))
1890         self.IsValid = True
1891         PeObject.close()
1892
1893     def _ByteListToStr(self, ByteList):
1894         String = ''
1895         for index in range(len(ByteList)):
1896             if ByteList[index] == 0:
1897                 break
1898             String += chr(ByteList[index])
1899         return String
1900
1901     def _ByteListToInt(self, ByteList):
1902         Value = 0
1903         for index in range(len(ByteList) - 1, -1, -1):
1904             Value = (Value << 8) | int(ByteList[index])
1905         return Value
1906
1907 class DefaultStore():
1908     def __init__(self, DefaultStores ):
1909
1910         self.DefaultStores = DefaultStores
1911     def DefaultStoreID(self, DefaultStoreName):
1912         for key, value in self.DefaultStores.items():
1913             if value == DefaultStoreName:
1914                 return key
1915         return None
1916     def GetDefaultDefault(self):
1917         if not self.DefaultStores or "0" in self.DefaultStores:
1918             return "0", TAB_DEFAULT_STORES_DEFAULT
1919         else:
1920             minvalue = min(int(value_str) for value_str in self.DefaultStores)
1921             return (str(minvalue), self.DefaultStores[str(minvalue)])
1922     def GetMin(self, DefaultSIdList):
1923         if not DefaultSIdList:
1924             return TAB_DEFAULT_STORES_DEFAULT
1925         storeidset = {storeid for storeid, storename in self.DefaultStores.values() if storename in DefaultSIdList}
1926         if not storeidset:
1927             return ""
1928         minid = min(storeidset )
1929         for sid, name in self.DefaultStores.values():
1930             if sid == minid:
1931                 return name
1932
1933 class SkuClass():
1934     DEFAULT = 0
1935     SINGLE = 1
1936     MULTIPLE =2
1937
1938     def __init__(self,SkuIdentifier='', SkuIds=None):
1939         if SkuIds is None:
1940             SkuIds = {}
1941
1942         for SkuName in SkuIds:
1943             SkuId = SkuIds[SkuName][0]
1944             skuid_num = int(SkuId, 16) if SkuId.upper().startswith("0X") else int(SkuId)
1945             if skuid_num > 0xFFFFFFFFFFFFFFFF:
1946                 EdkLogger.error("build", PARAMETER_INVALID,
1947                             ExtraData = "SKU-ID [%s] value %s exceeds the max value of UINT64"
1948                                       % (SkuName, SkuId))
1949
1950         self.AvailableSkuIds = sdict()
1951         self.SkuIdSet = []
1952         self.SkuIdNumberSet = []
1953         self.SkuData = SkuIds
1954         self._SkuInherit = {}
1955         self._SkuIdentifier = SkuIdentifier
1956         if SkuIdentifier == '' or SkuIdentifier is None:
1957             self.SkuIdSet = ['DEFAULT']
1958             self.SkuIdNumberSet = ['0U']
1959         elif SkuIdentifier == 'ALL':
1960             self.SkuIdSet = SkuIds.keys()
1961             self.SkuIdNumberSet = [num[0].strip() + 'U' for num in SkuIds.values()]
1962         else:
1963             r = SkuIdentifier.split('|')
1964             self.SkuIdSet=[(r[k].strip()).upper() for k in range(len(r))]
1965             k = None
1966             try:
1967                 self.SkuIdNumberSet = [SkuIds[k][0].strip() + 'U' for k in self.SkuIdSet]
1968             except Exception:
1969                 EdkLogger.error("build", PARAMETER_INVALID,
1970                             ExtraData = "SKU-ID [%s] is not supported by the platform. [Valid SKU-ID: %s]"
1971                                       % (k, " | ".join(SkuIds.keys())))
1972         for each in self.SkuIdSet:
1973             if each in SkuIds:
1974                 self.AvailableSkuIds[each] = SkuIds[each][0]
1975             else:
1976                 EdkLogger.error("build", PARAMETER_INVALID,
1977                             ExtraData="SKU-ID [%s] is not supported by the platform. [Valid SKU-ID: %s]"
1978                                       % (each, " | ".join(SkuIds.keys())))
1979         if self.SkuUsageType != SkuClass.SINGLE:
1980             self.AvailableSkuIds.update({'DEFAULT':0, 'COMMON':0})
1981         if self.SkuIdSet:
1982             GlobalData.gSkuids = (self.SkuIdSet)
1983             if 'COMMON' in GlobalData.gSkuids:
1984                 GlobalData.gSkuids.remove('COMMON')
1985             if self.SkuUsageType == self.SINGLE:
1986                 if len(GlobalData.gSkuids) != 1:
1987                     if 'DEFAULT' in GlobalData.gSkuids:
1988                         GlobalData.gSkuids.remove('DEFAULT')
1989             if GlobalData.gSkuids:
1990                 GlobalData.gSkuids.sort()
1991
1992     def GetNextSkuId(self, skuname):
1993         if not self._SkuInherit:
1994             self._SkuInherit = {}
1995             for item in self.SkuData.values():
1996                 self._SkuInherit[item[1]]=item[2] if item[2] else "DEFAULT"
1997         return self._SkuInherit.get(skuname, "DEFAULT")
1998
1999     def GetSkuChain(self, sku):
2000         if sku == "DEFAULT":
2001             return ["DEFAULT"]
2002         skulist = [sku]
2003         nextsku = sku
2004         while True:
2005             nextsku = self.GetNextSkuId(nextsku)
2006             skulist.append(nextsku)
2007             if nextsku == "DEFAULT":
2008                 break
2009         skulist.reverse()
2010         return skulist
2011     def SkuOverrideOrder(self):
2012         skuorderset = []
2013         for skuname in self.SkuIdSet:
2014             skuorderset.append(self.GetSkuChain(skuname))
2015
2016         skuorder = []
2017         for index in range(max(len(item) for item in skuorderset)):
2018             for subset in skuorderset:
2019                 if index > len(subset)-1:
2020                     continue
2021                 if subset[index] in skuorder:
2022                     continue
2023                 skuorder.append(subset[index])
2024
2025         return skuorder
2026
2027     @property
2028     def SkuUsageType(self):
2029         if self._SkuIdentifier.upper() == "ALL":
2030             return SkuClass.MULTIPLE
2031
2032         if len(self.SkuIdSet) == 1:
2033             if self.SkuIdSet[0] == 'DEFAULT':
2034                 return SkuClass.DEFAULT
2035             return SkuClass.SINGLE
2036         if len(self.SkuIdSet) == 2 and 'DEFAULT' in self.SkuIdSet:
2037             return SkuClass.SINGLE
2038         return SkuClass.MULTIPLE
2039
2040     def DumpSkuIdArrary(self):
2041         if self.SkuUsageType == SkuClass.SINGLE:
2042             return "{0x0}"
2043         ArrayStrList = []
2044         for skuname in self.AvailableSkuIds:
2045             if skuname == "COMMON":
2046                 continue
2047             while skuname != "DEFAULT":
2048                 ArrayStrList.append(hex(int(self.AvailableSkuIds[skuname])))
2049                 skuname = self.GetNextSkuId(skuname)
2050             ArrayStrList.append("0x0")
2051         return "{{{myList}}}".format(myList=",".join(ArrayStrList))
2052
2053     @property
2054     def AvailableSkuIdSet(self):
2055         return self.AvailableSkuIds
2056
2057     @property
2058     def SystemSkuId(self):
2059         if self.SkuUsageType == SkuClass.SINGLE:
2060             if len(self.SkuIdSet) == 1:
2061                 return self.SkuIdSet[0]
2062             else:
2063                 return self.SkuIdSet[0] if self.SkuIdSet[0] != 'DEFAULT' else self.SkuIdSet[1]
2064         else:
2065             return 'DEFAULT'
2066
2067 #
2068 # Pack a registry format GUID
2069 #
2070 def PackRegistryFormatGuid(Guid):
2071     return PackGUID(Guid.split('-'))
2072
2073 ##  Get the integer value from string like "14U" or integer like 2
2074 #
2075 #   @param      Input   The object that may be either a integer value or a string
2076 #
2077 #   @retval     Value    The integer value that the input represents
2078 #
2079 def GetIntegerValue(Input):
2080     if type(Input) in (int, long):
2081         return Input
2082     String = Input
2083     if String.endswith("U"):
2084         String = String[:-1]
2085     if String.endswith("ULL"):
2086         String = String[:-3]
2087     if String.endswith("LL"):
2088         String = String[:-2]
2089
2090     if String.startswith("0x") or String.startswith("0X"):
2091         return int(String, 16)
2092     elif String == '':
2093         return 0
2094     else:
2095         return int(String)
2096
2097 #
2098 # Pack a GUID (registry format) list into a buffer and return it
2099 #
2100 def PackGUID(Guid):
2101     return pack(PACK_PATTERN_GUID,
2102                 int(Guid[0], 16),
2103                 int(Guid[1], 16),
2104                 int(Guid[2], 16),
2105                 int(Guid[3][-4:-2], 16),
2106                 int(Guid[3][-2:], 16),
2107                 int(Guid[4][-12:-10], 16),
2108                 int(Guid[4][-10:-8], 16),
2109                 int(Guid[4][-8:-6], 16),
2110                 int(Guid[4][-6:-4], 16),
2111                 int(Guid[4][-4:-2], 16),
2112                 int(Guid[4][-2:], 16)
2113                 )
2114
2115 #
2116 # Pack a GUID (byte) list into a buffer and return it
2117 #
2118 def PackByteFormatGUID(Guid):
2119     return pack(PACK_PATTERN_GUID,
2120                 Guid[0],
2121                 Guid[1],
2122                 Guid[2],
2123                 Guid[3],
2124                 Guid[4],
2125                 Guid[5],
2126                 Guid[6],
2127                 Guid[7],
2128                 Guid[8],
2129                 Guid[9],
2130                 Guid[10],
2131                 )
2132
2133 ##
2134 #
2135 # This acts like the main() function for the script, unless it is 'import'ed into another
2136 # script.
2137 #
2138 if __name__ == '__main__':
2139     pass
2140