BaseTools/Source/Python/Common/Misc.py

   1 ## @file
   2 # Common routines used by all tools
   3 #
   4 # Copyright (c) 2007 - 2018, Intel Corporation. All rights reserved.<BR>
   5 # This program and the accompanying materials
   6 # are licensed and made available under the terms and conditions of the BSD License
   7 # which accompanies this distribution.  The full text of the license may be found at
   8 # http://opensource.org/licenses/bsd-license.php
   9 #
  10 # THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS,
  11 # WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED.
  12 #
  13
  14 ##
  15 # Import Modules
  16 #
  17 from __future__ import absolute_import
  18 import Common.LongFilePathOs as os
  19 import sys
  20 import string
  21 import threading
  22 import time
  23 import re
  24 import pickle
  25 import array
  26 import shutil
  27 from random import sample
  28 from struct import pack
  29 from UserDict import IterableUserDict
  30 from UserList import UserList
  31
  32 from Common import EdkLogger as EdkLogger
  33 from Common import GlobalData as GlobalData
  34 from .DataType import *
  35 from .BuildToolError import *
  36 from CommonDataClass.DataClass import *
  37 from .Parsing import GetSplitValueList
  38 from Common.LongFilePathSupport import OpenLongFilePath as open
  39 from Common.MultipleWorkspace import MultipleWorkspace as mws
  40 import uuid
  41 from CommonDataClass.Exceptions import BadExpression
  42 from Common.caching import cached_property
  43 import subprocess
  44 from collections import OrderedDict
  45 ## Regular expression used to find out place holders in string template
  46 gPlaceholderPattern = re.compile("\$\{([^$()\s]+)\}", re.MULTILINE | re.UNICODE)
  47
  48 ## regular expressions for map file processing
  49 startPatternGeneral = re.compile("^Start[' ']+Length[' ']+Name[' ']+Class")
  50 addressPatternGeneral = re.compile("^Address[' ']+Publics by Value[' ']+Rva\+Base")
  51 valuePatternGcc = re.compile('^([\w_\.]+) +([\da-fA-Fx]+) +([\da-fA-Fx]+)$')
  52 pcdPatternGcc = re.compile('^([\da-fA-Fx]+) +([\da-fA-Fx]+)')
  53 secReGeneral = re.compile('^([\da-fA-F]+):([\da-fA-F]+) +([\da-fA-F]+)[Hh]? +([.\w\$]+) +(\w+)', re.UNICODE)
  54
  55 StructPattern = re.compile(r'[_a-zA-Z][0-9A-Za-z_]*$')
  56
  57 ## Dictionary used to store file time stamp for quick re-access
  58 gFileTimeStampCache = {}    # {file path : file time stamp}
  59
  60 ## Dictionary used to store dependencies of files
  61 gDependencyDatabase = {}    # arch : {file path : [dependent files list]}
  62
  63 #
  64 # If a module is built more than once with different PCDs or library classes
  65 # a temporary INF file with same content is created, the temporary file is removed
  66 # when build exits.
  67 #
  68 _TempInfs = []
  69
  70 def GetVariableOffset(mapfilepath, efifilepath, varnames):
  71     """ Parse map file to get variable offset in current EFI file
  72     @param mapfilepath    Map file absolution path
  73     @param efifilepath:   EFI binary file full path
  74     @param varnames       iteratable container whose elements are variable names to be searched
  75
  76     @return List whos elements are tuple with variable name and raw offset
  77     """
  78     lines = []
  79     try:
  80         f = open(mapfilepath, 'r')
  81         lines = f.readlines()
  82         f.close()
  83     except:
  84         return None
  85
  86     if len(lines) == 0: return None
  87     firstline = lines[0].strip()
  88     if (firstline.startswith("Archive member included ") and
  89         firstline.endswith(" file (symbol)")):
  90         return _parseForGCC(lines, efifilepath, varnames)
  91     if firstline.startswith("# Path:"):
  92         return _parseForXcode(lines, efifilepath, varnames)
  93     return _parseGeneral(lines, efifilepath, varnames)
  94
  95 def _parseForXcode(lines, efifilepath, varnames):
  96     status = 0
  97     ret = []
  98     for line in lines:
  99         line = line.strip()
 100         if status == 0 and line == "# Symbols:":
 101             status = 1
 102             continue
 103         if status == 1 and len(line) != 0:
 104             for varname in varnames:
 105                 if varname in line:
 106                     # cannot pregenerate this RegEx since it uses varname from varnames.
 107                     m = re.match('^([\da-fA-FxX]+)([\s\S]*)([_]*%s)$' % varname, line)
 108                     if m is not None:
 109                         ret.append((varname, m.group(1)))
 110     return ret
 111
 112 def _parseForGCC(lines, efifilepath, varnames):
 113     """ Parse map file generated by GCC linker """
 114     status = 0
 115     sections = []
 116     varoffset = []
 117     for index, line in enumerate(lines):
 118         line = line.strip()
 119         # status machine transection
 120         if status == 0 and line == "Memory Configuration":
 121             status = 1
 122             continue
 123         elif status == 1 and line == 'Linker script and memory map':
 124             status = 2
 125             continue
 126         elif status ==2 and line == 'START GROUP':
 127             status = 3
 128             continue
 129
 130         # status handler
 131         if status == 3:
 132             m = valuePatternGcc.match(line)
 133             if m is not None:
 134                 sections.append(m.groups(0))
 135             for varname in varnames:
 136                 Str = ''
 137                 m = re.match("^.data.(%s)" % varname, line)
 138                 if m is not None:
 139                     m = re.match(".data.(%s)$" % varname, line)
 140                     if m is not None:
 141                         Str = lines[index + 1]
 142                     else:
 143                         Str = line[len(".data.%s" % varname):]
 144                     if Str:
 145                         m = pcdPatternGcc.match(Str.strip())
 146                         if m is not None:
 147                             varoffset.append((varname, int(m.groups(0)[0], 16), int(sections[-1][1], 16), sections[-1][0]))
 148
 149     if not varoffset:
 150         return []
 151     # get section information from efi file
 152     efisecs = PeImageClass(efifilepath).SectionHeaderList
 153     if efisecs is None or len(efisecs) == 0:
 154         return []
 155     #redirection
 156     redirection = 0
 157     for efisec in efisecs:
 158         for section in sections:
 159             if section[0].strip() == efisec[0].strip() and section[0].strip() == '.text':
 160                 redirection = int(section[1], 16) - efisec[1]
 161
 162     ret = []
 163     for var in varoffset:
 164         for efisec in efisecs:
 165             if var[1] >= efisec[1] and var[1] < efisec[1]+efisec[3]:
 166                 ret.append((var[0], hex(efisec[2] + var[1] - efisec[1] - redirection)))
 167     return ret
 168
 169 def _parseGeneral(lines, efifilepath, varnames):
 170     status = 0    #0 - beginning of file; 1 - PE section definition; 2 - symbol table
 171     secs  = []    # key = section name
 172     varoffset = []
 173     symRe = re.compile('^([\da-fA-F]+):([\da-fA-F]+) +([\.:\\\\\w\?@\$]+) +([\da-fA-F]+)', re.UNICODE)
 174
 175     for line in lines:
 176         line = line.strip()
 177         if startPatternGeneral.match(line):
 178             status = 1
 179             continue
 180         if addressPatternGeneral.match(line):
 181             status = 2
 182             continue
 183         if line.startswith("entry point at"):
 184             status = 3
 185             continue
 186         if status == 1 and len(line) != 0:
 187             m =  secReGeneral.match(line)
 188             assert m is not None, "Fail to parse the section in map file , line is %s" % line
 189             sec_no, sec_start, sec_length, sec_name, sec_class = m.groups(0)
 190             secs.append([int(sec_no, 16), int(sec_start, 16), int(sec_length, 16), sec_name, sec_class])
 191         if status == 2 and len(line) != 0:
 192             for varname in varnames:
 193                 m = symRe.match(line)
 194                 assert m is not None, "Fail to parse the symbol in map file, line is %s" % line
 195                 sec_no, sym_offset, sym_name, vir_addr = m.groups(0)
 196                 sec_no     = int(sec_no,     16)
 197                 sym_offset = int(sym_offset, 16)
 198                 vir_addr   = int(vir_addr,   16)
 199                 # cannot pregenerate this RegEx since it uses varname from varnames.
 200                 m2 = re.match('^[_]*(%s)' % varname, sym_name)
 201                 if m2 is not None:
 202                     # fond a binary pcd entry in map file
 203                     for sec in secs:
 204                         if sec[0] == sec_no and (sym_offset >= sec[1] and sym_offset < sec[1] + sec[2]):
 205                             varoffset.append([varname, sec[3], sym_offset, vir_addr, sec_no])
 206
 207     if not varoffset: return []
 208
 209     # get section information from efi file
 210     efisecs = PeImageClass(efifilepath).SectionHeaderList
 211     if efisecs is None or len(efisecs) == 0:
 212         return []
 213
 214     ret = []
 215     for var in varoffset:
 216         index = 0
 217         for efisec in efisecs:
 218             index = index + 1
 219             if var[1].strip() == efisec[0].strip():
 220                 ret.append((var[0], hex(efisec[2] + var[2])))
 221             elif var[4] == index:
 222                 ret.append((var[0], hex(efisec[2] + var[2])))
 223
 224     return ret
 225
 226 ## Routine to process duplicated INF
 227 #
 228 #  This function is called by following two cases:
 229 #  Case 1 in DSC:
 230 #    [components.arch]
 231 #    Pkg/module/module.inf
 232 #    Pkg/module/module.inf {
 233 #      <Defines>
 234 #        FILE_GUID = 0D1B936F-68F3-4589-AFCC-FB8B7AEBC836
 235 #    }
 236 #  Case 2 in FDF:
 237 #    INF Pkg/module/module.inf
 238 #    INF FILE_GUID = 0D1B936F-68F3-4589-AFCC-FB8B7AEBC836 Pkg/module/module.inf
 239 #
 240 #  This function copies Pkg/module/module.inf to
 241 #  Conf/.cache/0D1B936F-68F3-4589-AFCC-FB8B7AEBC836module.inf
 242 #
 243 #  @param Path     Original PathClass object
 244 #  @param BaseName New file base name
 245 #
 246 #  @retval         return the new PathClass object
 247 #
 248 def ProcessDuplicatedInf(Path, BaseName, Workspace):
 249     Filename = os.path.split(Path.File)[1]
 250     if '.' in Filename:
 251         Filename = BaseName + Path.BaseName + Filename[Filename.rfind('.'):]
 252     else:
 253         Filename = BaseName + Path.BaseName
 254
 255     #
 256     # If -N is specified on command line, cache is disabled
 257     # The directory has to be created
 258     #
 259     DbDir = os.path.split(GlobalData.gDatabasePath)[0]
 260     if not os.path.exists(DbDir):
 261         os.makedirs(DbDir)
 262     #
 263     # A temporary INF is copied to database path which must have write permission
 264     # The temporary will be removed at the end of build
 265     # In case of name conflict, the file name is
 266     # FILE_GUIDBaseName (0D1B936F-68F3-4589-AFCC-FB8B7AEBC836module.inf)
 267     #
 268     TempFullPath = os.path.join(DbDir,
 269                                 Filename)
 270     RtPath = PathClass(Path.File, Workspace)
 271     #
 272     # Modify the full path to temporary path, keep other unchanged
 273     #
 274     # To build same module more than once, the module path with FILE_GUID overridden has
 275     # the file name FILE_GUIDmodule.inf, but the relative path (self.MetaFile.File) is the real path
 276     # in DSC which is used as relative path by C files and other files in INF.
 277     # A trick was used: all module paths are PathClass instances, after the initialization
 278     # of PathClass, the PathClass.Path is overridden by the temporary INF path.
 279     #
 280     # The reason for creating a temporary INF is:
 281     # Platform.Modules which is the base to create ModuleAutoGen objects is a dictionary,
 282     # the key is the full path of INF, the value is an object to save overridden library instances, PCDs.
 283     # A different key for the same module is needed to create different output directory,
 284     # retrieve overridden PCDs, library instances.
 285     #
 286     # The BaseName is the FILE_GUID which is also the output directory name.
 287     #
 288     #
 289     RtPath.Path = TempFullPath
 290     RtPath.BaseName = BaseName
 291     #
 292     # If file exists, compare contents
 293     #
 294     if os.path.exists(TempFullPath):
 295         with open(str(Path), 'rb') as f1, open(TempFullPath, 'rb') as f2:
 296             if f1.read() == f2.read():
 297                 return RtPath
 298     _TempInfs.append(TempFullPath)
 299     shutil.copy2(str(Path), TempFullPath)
 300     return RtPath
 301
 302 ## Remove temporary created INFs whose paths were saved in _TempInfs
 303 #
 304 def ClearDuplicatedInf():
 305     while _TempInfs:
 306         File = _TempInfs.pop()
 307         if os.path.exists(File):
 308             os.remove(File)
 309
 310 ## Convert GUID string in xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx style to C structure style
 311 #
 312 #   @param      Guid    The GUID string
 313 #
 314 #   @retval     string  The GUID string in C structure style
 315 #
 316 def GuidStringToGuidStructureString(Guid):
 317     GuidList = Guid.split('-')
 318     Result = '{'
 319     for Index in range(0, 3, 1):
 320         Result = Result + '0x' + GuidList[Index] + ', '
 321     Result = Result + '{0x' + GuidList[3][0:2] + ', 0x' + GuidList[3][2:4]
 322     for Index in range(0, 12, 2):
 323         Result = Result + ', 0x' + GuidList[4][Index:Index + 2]
 324     Result += '}}'
 325     return Result
 326
 327 ## Convert GUID structure in byte array to xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
 328 #
 329 #   @param      GuidValue   The GUID value in byte array
 330 #
 331 #   @retval     string      The GUID value in xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx format
 332 #
 333 def GuidStructureByteArrayToGuidString(GuidValue):
 334     guidValueString = GuidValue.lower().replace("{", "").replace("}", "").replace(" ", "").replace(";", "")
 335     guidValueList = guidValueString.split(",")
 336     if len(guidValueList) != 16:
 337         return ''
 338         #EdkLogger.error(None, None, "Invalid GUID value string %s" % GuidValue)
 339     try:
 340         return "%02x%02x%02x%02x-%02x%02x-%02x%02x-%02x%02x-%02x%02x%02x%02x%02x%02x" % (
 341                 int(guidValueList[3], 16),
 342                 int(guidValueList[2], 16),
 343                 int(guidValueList[1], 16),
 344                 int(guidValueList[0], 16),
 345                 int(guidValueList[5], 16),
 346                 int(guidValueList[4], 16),
 347                 int(guidValueList[7], 16),
 348                 int(guidValueList[6], 16),
 349                 int(guidValueList[8], 16),
 350                 int(guidValueList[9], 16),
 351                 int(guidValueList[10], 16),
 352                 int(guidValueList[11], 16),
 353                 int(guidValueList[12], 16),
 354                 int(guidValueList[13], 16),
 355                 int(guidValueList[14], 16),
 356                 int(guidValueList[15], 16)
 357                 )
 358     except:
 359         return ''
 360
 361 ## Convert GUID string in C structure style to xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
 362 #
 363 #   @param      GuidValue   The GUID value in C structure format
 364 #
 365 #   @retval     string      The GUID value in xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx format
 366 #
 367 def GuidStructureStringToGuidString(GuidValue):
 368     if not GlobalData.gGuidCFormatPattern.match(GuidValue):
 369         return ''
 370     guidValueString = GuidValue.lower().replace("{", "").replace("}", "").replace(" ", "").replace(";", "")
 371     guidValueList = guidValueString.split(",")
 372     if len(guidValueList) != 11:
 373         return ''
 374         #EdkLogger.error(None, None, "Invalid GUID value string %s" % GuidValue)
 375     try:
 376         return "%08x-%04x-%04x-%02x%02x-%02x%02x%02x%02x%02x%02x" % (
 377                 int(guidValueList[0], 16),
 378                 int(guidValueList[1], 16),
 379                 int(guidValueList[2], 16),
 380                 int(guidValueList[3], 16),
 381                 int(guidValueList[4], 16),
 382                 int(guidValueList[5], 16),
 383                 int(guidValueList[6], 16),
 384                 int(guidValueList[7], 16),
 385                 int(guidValueList[8], 16),
 386                 int(guidValueList[9], 16),
 387                 int(guidValueList[10], 16)
 388                 )
 389     except:
 390         return ''
 391
 392 ## Convert GUID string in C structure style to xxxxxxxx_xxxx_xxxx_xxxx_xxxxxxxxxxxx
 393 #
 394 #   @param      GuidValue   The GUID value in C structure format
 395 #
 396 #   @retval     string      The GUID value in xxxxxxxx_xxxx_xxxx_xxxx_xxxxxxxxxxxx format
 397 #
 398 def GuidStructureStringToGuidValueName(GuidValue):
 399     guidValueString = GuidValue.lower().replace("{", "").replace("}", "").replace(" ", "")
 400     guidValueList = guidValueString.split(",")
 401     if len(guidValueList) != 11:
 402         EdkLogger.error(None, FORMAT_INVALID, "Invalid GUID value string [%s]" % GuidValue)
 403     return "%08x_%04x_%04x_%02x%02x_%02x%02x%02x%02x%02x%02x" % (
 404             int(guidValueList[0], 16),
 405             int(guidValueList[1], 16),
 406             int(guidValueList[2], 16),
 407             int(guidValueList[3], 16),
 408             int(guidValueList[4], 16),
 409             int(guidValueList[5], 16),
 410             int(guidValueList[6], 16),
 411             int(guidValueList[7], 16),
 412             int(guidValueList[8], 16),
 413             int(guidValueList[9], 16),
 414             int(guidValueList[10], 16)
 415             )
 416
 417 ## Create directories
 418 #
 419 #   @param      Directory   The directory name
 420 #
 421 def CreateDirectory(Directory):
 422     if Directory is None or Directory.strip() == "":
 423         return True
 424     try:
 425         if not os.access(Directory, os.F_OK):
 426             os.makedirs(Directory)
 427     except:
 428         return False
 429     return True
 430
 431 ## Remove directories, including files and sub-directories in it
 432 #
 433 #   @param      Directory   The directory name
 434 #
 435 def RemoveDirectory(Directory, Recursively=False):
 436     if Directory is None or Directory.strip() == "" or not os.path.exists(Directory):
 437         return
 438     if Recursively:
 439         CurrentDirectory = os.getcwd()
 440         os.chdir(Directory)
 441         for File in os.listdir("."):
 442             if os.path.isdir(File):
 443                 RemoveDirectory(File, Recursively)
 444             else:
 445                 os.remove(File)
 446         os.chdir(CurrentDirectory)
 447     os.rmdir(Directory)
 448
 449 ## Store content in file
 450 #
 451 #  This method is used to save file only when its content is changed. This is
 452 #  quite useful for "make" system to decide what will be re-built and what won't.
 453 #
 454 #   @param      File            The path of file
 455 #   @param      Content         The new content of the file
 456 #   @param      IsBinaryFile    The flag indicating if the file is binary file or not
 457 #
 458 #   @retval     True            If the file content is changed and the file is renewed
 459 #   @retval     False           If the file content is the same
 460 #
 461 def SaveFileOnChange(File, Content, IsBinaryFile=True):
 462     if not IsBinaryFile:
 463         Content = Content.replace("\n", os.linesep)
 464
 465     if os.path.exists(File):
 466         try:
 467             if Content == open(File, "rb").read():
 468                 return False
 469         except:
 470             EdkLogger.error(None, FILE_OPEN_FAILURE, ExtraData=File)
 471
 472     DirName = os.path.dirname(File)
 473     if not CreateDirectory(DirName):
 474         EdkLogger.error(None, FILE_CREATE_FAILURE, "Could not create directory %s" % DirName)
 475     else:
 476         if DirName == '':
 477             DirName = os.getcwd()
 478         if not os.access(DirName, os.W_OK):
 479             EdkLogger.error(None, PERMISSION_FAILURE, "Do not have write permission on directory %s" % DirName)
 480
 481     try:
 482         Fd = open(File, "wb")
 483         Fd.write(Content)
 484         Fd.close()
 485     except IOError as X:
 486         EdkLogger.error(None, FILE_CREATE_FAILURE, ExtraData='IOError %s' % X)
 487
 488     return True
 489
 490 ## Make a Python object persistent on file system
 491 #
 492 #   @param      Data    The object to be stored in file
 493 #   @param      File    The path of file to store the object
 494 #
 495 def DataDump(Data, File):
 496     Fd = None
 497     try:
 498         Fd = open(File, 'wb')
 499         pickle.dump(Data, Fd, pickle.HIGHEST_PROTOCOL)
 500     except:
 501         EdkLogger.error("", FILE_OPEN_FAILURE, ExtraData=File, RaiseError=False)
 502     finally:
 503         if Fd is not None:
 504             Fd.close()
 505
 506 ## Restore a Python object from a file
 507 #
 508 #   @param      File    The path of file stored the object
 509 #
 510 #   @retval     object  A python object
 511 #   @retval     None    If failure in file operation
 512 #
 513 def DataRestore(File):
 514     Data = None
 515     Fd = None
 516     try:
 517         Fd = open(File, 'rb')
 518         Data = pickle.load(Fd)
 519     except Exception as e:
 520         EdkLogger.verbose("Failed to load [%s]\n\t%s" % (File, str(e)))
 521         Data = None
 522     finally:
 523         if Fd is not None:
 524             Fd.close()
 525     return Data
 526
 527 ## Retrieve and cache the real path name in file system
 528 #
 529 #   @param      Root    The root directory of path relative to
 530 #
 531 #   @retval     str     The path string if the path exists
 532 #   @retval     None    If path doesn't exist
 533 #
 534 class DirCache:
 535     _CACHE_ = set()
 536     _UPPER_CACHE_ = {}
 537
 538     def __init__(self, Root):
 539         self._Root = Root
 540         for F in os.listdir(Root):
 541             self._CACHE_.add(F)
 542             self._UPPER_CACHE_[F.upper()] = F
 543
 544     # =[] operator
 545     def __getitem__(self, Path):
 546         Path = Path[len(os.path.commonprefix([Path, self._Root])):]
 547         if not Path:
 548             return self._Root
 549         if Path and Path[0] == os.path.sep:
 550             Path = Path[1:]
 551         if Path in self._CACHE_:
 552             return os.path.join(self._Root, Path)
 553         UpperPath = Path.upper()
 554         if UpperPath in self._UPPER_CACHE_:
 555             return os.path.join(self._Root, self._UPPER_CACHE_[UpperPath])
 556
 557         IndexList = []
 558         LastSepIndex = -1
 559         SepIndex = Path.find(os.path.sep)
 560         while SepIndex > -1:
 561             Parent = UpperPath[:SepIndex]
 562             if Parent not in self._UPPER_CACHE_:
 563                 break
 564             LastSepIndex = SepIndex
 565             SepIndex = Path.find(os.path.sep, LastSepIndex + 1)
 566
 567         if LastSepIndex == -1:
 568             return None
 569
 570         Cwd = os.getcwd()
 571         os.chdir(self._Root)
 572         SepIndex = LastSepIndex
 573         while SepIndex > -1:
 574             Parent = Path[:SepIndex]
 575             ParentKey = UpperPath[:SepIndex]
 576             if ParentKey not in self._UPPER_CACHE_:
 577                 os.chdir(Cwd)
 578                 return None
 579
 580             if Parent in self._CACHE_:
 581                 ParentDir = Parent
 582             else:
 583                 ParentDir = self._UPPER_CACHE_[ParentKey]
 584             for F in os.listdir(ParentDir):
 585                 Dir = os.path.join(ParentDir, F)
 586                 self._CACHE_.add(Dir)
 587                 self._UPPER_CACHE_[Dir.upper()] = Dir
 588
 589             SepIndex = Path.find(os.path.sep, SepIndex + 1)
 590
 591         os.chdir(Cwd)
 592         if Path in self._CACHE_:
 593             return os.path.join(self._Root, Path)
 594         elif UpperPath in self._UPPER_CACHE_:
 595             return os.path.join(self._Root, self._UPPER_CACHE_[UpperPath])
 596         return None
 597
 598 def RealPath(File, Dir='', OverrideDir=''):
 599     NewFile = os.path.normpath(os.path.join(Dir, File))
 600     NewFile = GlobalData.gAllFiles[NewFile]
 601     if not NewFile and OverrideDir:
 602         NewFile = os.path.normpath(os.path.join(OverrideDir, File))
 603         NewFile = GlobalData.gAllFiles[NewFile]
 604     return NewFile
 605
 606 def RealPath2(File, Dir='', OverrideDir=''):
 607     NewFile = None
 608     if OverrideDir:
 609         NewFile = GlobalData.gAllFiles[os.path.normpath(os.path.join(OverrideDir, File))]
 610         if NewFile:
 611             if OverrideDir[-1] == os.path.sep:
 612                 return NewFile[len(OverrideDir):], NewFile[0:len(OverrideDir)]
 613             else:
 614                 return NewFile[len(OverrideDir) + 1:], NewFile[0:len(OverrideDir)]
 615     if GlobalData.gAllFiles:
 616         NewFile = GlobalData.gAllFiles[os.path.normpath(os.path.join(Dir, File))]
 617     if not NewFile:
 618         NewFile = os.path.normpath(os.path.join(Dir, File))
 619         if not os.path.exists(NewFile):
 620             return None, None
 621     if NewFile:
 622         if Dir:
 623             if Dir[-1] == os.path.sep:
 624                 return NewFile[len(Dir):], NewFile[0:len(Dir)]
 625             else:
 626                 return NewFile[len(Dir) + 1:], NewFile[0:len(Dir)]
 627         else:
 628             return NewFile, ''
 629
 630     return None, None
 631
 632 ## Get GUID value from given packages
 633 #
 634 #   @param      CName           The CName of the GUID
 635 #   @param      PackageList     List of packages looking-up in
 636 #   @param      Inffile         The driver file
 637 #
 638 #   @retval     GuidValue   if the CName is found in any given package
 639 #   @retval     None        if the CName is not found in all given packages
 640 #
 641 def GuidValue(CName, PackageList, Inffile = None):
 642     for P in PackageList:
 643         GuidKeys = P.Guids.keys()
 644         if Inffile and P._PrivateGuids:
 645             if not Inffile.startswith(P.MetaFile.Dir):
 646                 GuidKeys = [x for x in P.Guids if x not in P._PrivateGuids]
 647         if CName in GuidKeys:
 648             return P.Guids[CName]
 649     return None
 650
 651 ## Get Protocol value from given packages
 652 #
 653 #   @param      CName           The CName of the GUID
 654 #   @param      PackageList     List of packages looking-up in
 655 #   @param      Inffile         The driver file
 656 #
 657 #   @retval     GuidValue   if the CName is found in any given package
 658 #   @retval     None        if the CName is not found in all given packages
 659 #
 660 def ProtocolValue(CName, PackageList, Inffile = None):
 661     for P in PackageList:
 662         ProtocolKeys = P.Protocols.keys()
 663         if Inffile and P._PrivateProtocols:
 664             if not Inffile.startswith(P.MetaFile.Dir):
 665                 ProtocolKeys = [x for x in P.Protocols if x not in P._PrivateProtocols]
 666         if CName in ProtocolKeys:
 667             return P.Protocols[CName]
 668     return None
 669
 670 ## Get PPI value from given packages
 671 #
 672 #   @param      CName           The CName of the GUID
 673 #   @param      PackageList     List of packages looking-up in
 674 #   @param      Inffile         The driver file
 675 #
 676 #   @retval     GuidValue   if the CName is found in any given package
 677 #   @retval     None        if the CName is not found in all given packages
 678 #
 679 def PpiValue(CName, PackageList, Inffile = None):
 680     for P in PackageList:
 681         PpiKeys = P.Ppis.keys()
 682         if Inffile and P._PrivatePpis:
 683             if not Inffile.startswith(P.MetaFile.Dir):
 684                 PpiKeys = [x for x in P.Ppis if x not in P._PrivatePpis]
 685         if CName in PpiKeys:
 686             return P.Ppis[CName]
 687     return None
 688
 689 ## A string template class
 690 #
 691 #  This class implements a template for string replacement. A string template
 692 #  looks like following
 693 #
 694 #       ${BEGIN} other_string ${placeholder_name} other_string ${END}
 695 #
 696 #  The string between ${BEGIN} and ${END} will be repeated as many times as the
 697 #  length of "placeholder_name", which is a list passed through a dict. The
 698 #  "placeholder_name" is the key name of the dict. The ${BEGIN} and ${END} can
 699 #  be not used and, in this case, the "placeholder_name" must not a list and it
 700 #  will just be replaced once.
 701 #
 702 class TemplateString(object):
 703     _REPEAT_START_FLAG = "BEGIN"
 704     _REPEAT_END_FLAG = "END"
 705
 706     class Section(object):
 707         _LIST_TYPES = [type([]), type(set()), type((0,))]
 708
 709         def __init__(self, TemplateSection, PlaceHolderList):
 710             self._Template = TemplateSection
 711             self._PlaceHolderList = []
 712
 713             # Split the section into sub-sections according to the position of placeholders
 714             if PlaceHolderList:
 715                 self._SubSectionList = []
 716                 SubSectionStart = 0
 717                 #
 718                 # The placeholders passed in must be in the format of
 719                 #
 720                 #   PlaceHolderName, PlaceHolderStartPoint, PlaceHolderEndPoint
 721                 #
 722                 for PlaceHolder, Start, End in PlaceHolderList:
 723                     self._SubSectionList.append(TemplateSection[SubSectionStart:Start])
 724                     self._SubSectionList.append(TemplateSection[Start:End])
 725                     self._PlaceHolderList.append(PlaceHolder)
 726                     SubSectionStart = End
 727                 if SubSectionStart < len(TemplateSection):
 728                     self._SubSectionList.append(TemplateSection[SubSectionStart:])
 729             else:
 730                 self._SubSectionList = [TemplateSection]
 731
 732         def __str__(self):
 733             return self._Template + " : " + str(self._PlaceHolderList)
 734
 735         def Instantiate(self, PlaceHolderValues):
 736             RepeatTime = -1
 737             RepeatPlaceHolders = {}
 738             NonRepeatPlaceHolders = {}
 739
 740             for PlaceHolder in self._PlaceHolderList:
 741                 if PlaceHolder not in PlaceHolderValues:
 742                     continue
 743                 Value = PlaceHolderValues[PlaceHolder]
 744                 if type(Value) in self._LIST_TYPES:
 745                     if RepeatTime < 0:
 746                         RepeatTime = len(Value)
 747                     elif RepeatTime != len(Value):
 748                         EdkLogger.error(
 749                                     "TemplateString",
 750                                     PARAMETER_INVALID,
 751                                     "${%s} has different repeat time from others!" % PlaceHolder,
 752                                     ExtraData=str(self._Template)
 753                                     )
 754                     RepeatPlaceHolders["${%s}" % PlaceHolder] = Value
 755                 else:
 756                     NonRepeatPlaceHolders["${%s}" % PlaceHolder] = Value
 757
 758             if NonRepeatPlaceHolders:
 759                 StringList = []
 760                 for S in self._SubSectionList:
 761                     if S not in NonRepeatPlaceHolders:
 762                         StringList.append(S)
 763                     else:
 764                         StringList.append(str(NonRepeatPlaceHolders[S]))
 765             else:
 766                 StringList = self._SubSectionList
 767
 768             if RepeatPlaceHolders:
 769                 TempStringList = []
 770                 for Index in range(RepeatTime):
 771                     for S in StringList:
 772                         if S not in RepeatPlaceHolders:
 773                             TempStringList.append(S)
 774                         else:
 775                             TempStringList.append(str(RepeatPlaceHolders[S][Index]))
 776                 StringList = TempStringList
 777
 778             return "".join(StringList)
 779
 780     ## Constructor
 781     def __init__(self, Template=None):
 782         self.String = []
 783         self.IsBinary = False
 784         self._Template = Template
 785         self._TemplateSectionList = self._Parse(Template)
 786
 787     ## str() operator
 788     #
 789     #   @retval     string  The string replaced
 790     #
 791     def __str__(self):
 792         return "".join(self.String)
 793
 794     ## Split the template string into fragments per the ${BEGIN} and ${END} flags
 795     #
 796     #   @retval     list    A list of TemplateString.Section objects
 797     #
 798     def _Parse(self, Template):
 799         SectionStart = 0
 800         SearchFrom = 0
 801         MatchEnd = 0
 802         PlaceHolderList = []
 803         TemplateSectionList = []
 804         while Template:
 805             MatchObj = gPlaceholderPattern.search(Template, SearchFrom)
 806             if not MatchObj:
 807                 if MatchEnd <= len(Template):
 808                     TemplateSection = TemplateString.Section(Template[SectionStart:], PlaceHolderList)
 809                     TemplateSectionList.append(TemplateSection)
 810                 break
 811
 812             MatchString = MatchObj.group(1)
 813             MatchStart = MatchObj.start()
 814             MatchEnd = MatchObj.end()
 815
 816             if MatchString == self._REPEAT_START_FLAG:
 817                 if MatchStart > SectionStart:
 818                     TemplateSection = TemplateString.Section(Template[SectionStart:MatchStart], PlaceHolderList)
 819                     TemplateSectionList.append(TemplateSection)
 820                 SectionStart = MatchEnd
 821                 PlaceHolderList = []
 822             elif MatchString == self._REPEAT_END_FLAG:
 823                 TemplateSection = TemplateString.Section(Template[SectionStart:MatchStart], PlaceHolderList)
 824                 TemplateSectionList.append(TemplateSection)
 825                 SectionStart = MatchEnd
 826                 PlaceHolderList = []
 827             else:
 828                 PlaceHolderList.append((MatchString, MatchStart - SectionStart, MatchEnd - SectionStart))
 829             SearchFrom = MatchEnd
 830         return TemplateSectionList
 831
 832     ## Replace the string template with dictionary of placeholders and append it to previous one
 833     #
 834     #   @param      AppendString    The string template to append
 835     #   @param      Dictionary      The placeholder dictionaries
 836     #
 837     def Append(self, AppendString, Dictionary=None):
 838         if Dictionary:
 839             SectionList = self._Parse(AppendString)
 840             self.String.append( "".join(S.Instantiate(Dictionary) for S in SectionList))
 841         else:
 842             if isinstance(AppendString,list):
 843                 self.String.extend(AppendString)
 844             else:
 845                 self.String.append(AppendString)
 846
 847     ## Replace the string template with dictionary of placeholders
 848     #
 849     #   @param      Dictionary      The placeholder dictionaries
 850     #
 851     #   @retval     str             The string replaced with placeholder values
 852     #
 853     def Replace(self, Dictionary=None):
 854         return "".join(S.Instantiate(Dictionary) for S in self._TemplateSectionList)
 855
 856 ## Progress indicator class
 857 #
 858 #  This class makes use of thread to print progress on console.
 859 #
 860 class Progressor:
 861     # for avoiding deadloop
 862     _StopFlag = None
 863     _ProgressThread = None
 864     _CheckInterval = 0.25
 865
 866     ## Constructor
 867     #
 868     #   @param      OpenMessage     The string printed before progress charaters
 869     #   @param      CloseMessage    The string printed after progress charaters
 870     #   @param      ProgressChar    The charater used to indicate the progress
 871     #   @param      Interval        The interval in seconds between two progress charaters
 872     #
 873     def __init__(self, OpenMessage="", CloseMessage="", ProgressChar='.', Interval=1.0):
 874         self.PromptMessage = OpenMessage
 875         self.CodaMessage = CloseMessage
 876         self.ProgressChar = ProgressChar
 877         self.Interval = Interval
 878         if Progressor._StopFlag is None:
 879             Progressor._StopFlag = threading.Event()
 880
 881     ## Start to print progress charater
 882     #
 883     #   @param      OpenMessage     The string printed before progress charaters
 884     #
 885     def Start(self, OpenMessage=None):
 886         if OpenMessage is not None:
 887             self.PromptMessage = OpenMessage
 888         Progressor._StopFlag.clear()
 889         if Progressor._ProgressThread is None:
 890             Progressor._ProgressThread = threading.Thread(target=self._ProgressThreadEntry)
 891             Progressor._ProgressThread.setDaemon(False)
 892             Progressor._ProgressThread.start()
 893
 894     ## Stop printing progress charater
 895     #
 896     #   @param      CloseMessage    The string printed after progress charaters
 897     #
 898     def Stop(self, CloseMessage=None):
 899         OriginalCodaMessage = self.CodaMessage
 900         if CloseMessage is not None:
 901             self.CodaMessage = CloseMessage
 902         self.Abort()
 903         self.CodaMessage = OriginalCodaMessage
 904
 905     ## Thread entry method
 906     def _ProgressThreadEntry(self):
 907         sys.stdout.write(self.PromptMessage + " ")
 908         sys.stdout.flush()
 909         TimeUp = 0.0
 910         while not Progressor._StopFlag.isSet():
 911             if TimeUp <= 0.0:
 912                 sys.stdout.write(self.ProgressChar)
 913                 sys.stdout.flush()
 914                 TimeUp = self.Interval
 915             time.sleep(self._CheckInterval)
 916             TimeUp -= self._CheckInterval
 917         sys.stdout.write(" " + self.CodaMessage + "\n")
 918         sys.stdout.flush()
 919
 920     ## Abort the progress display
 921     @staticmethod
 922     def Abort():
 923         if Progressor._StopFlag is not None:
 924             Progressor._StopFlag.set()
 925         if Progressor._ProgressThread is not None:
 926             Progressor._ProgressThread.join()
 927             Progressor._ProgressThread = None
 928
 929 ## A dict which can access its keys and/or values orderly
 930 #
 931 #  The class implements a new kind of dict which its keys or values can be
 932 #  accessed in the order they are added into the dict. It guarantees the order
 933 #  by making use of an internal list to keep a copy of keys.
 934 #
 935 class sdict(IterableUserDict):
 936     ## Constructor
 937     def __init__(self):
 938         IterableUserDict.__init__(self)
 939         self._key_list = []
 940
 941     ## [] operator
 942     def __setitem__(self, key, value):
 943         if key not in self._key_list:
 944             self._key_list.append(key)
 945         IterableUserDict.__setitem__(self, key, value)
 946
 947     ## del operator
 948     def __delitem__(self, key):
 949         self._key_list.remove(key)
 950         IterableUserDict.__delitem__(self, key)
 951
 952     ## used in "for k in dict" loop to ensure the correct order
 953     def __iter__(self):
 954         return self.iterkeys()
 955
 956     ## len() support
 957     def __len__(self):
 958         return len(self._key_list)
 959
 960     ## "in" test support
 961     def __contains__(self, key):
 962         return key in self._key_list
 963
 964     ## indexof support
 965     def index(self, key):
 966         return self._key_list.index(key)
 967
 968     ## insert support
 969     def insert(self, key, newkey, newvalue, order):
 970         index = self._key_list.index(key)
 971         if order == 'BEFORE':
 972             self._key_list.insert(index, newkey)
 973             IterableUserDict.__setitem__(self, newkey, newvalue)
 974         elif order == 'AFTER':
 975             self._key_list.insert(index + 1, newkey)
 976             IterableUserDict.__setitem__(self, newkey, newvalue)
 977
 978     ## append support
 979     def append(self, sdict):
 980         for key in sdict:
 981             if key not in self._key_list:
 982                 self._key_list.append(key)
 983             IterableUserDict.__setitem__(self, key, sdict[key])
 984
 985     def has_key(self, key):
 986         return key in self._key_list
 987
 988     ## Empty the dict
 989     def clear(self):
 990         self._key_list = []
 991         IterableUserDict.clear(self)
 992
 993     ## Return a copy of keys
 994     def keys(self):
 995         keys = []
 996         for key in self._key_list:
 997             keys.append(key)
 998         return keys
 999
1000     ## Return a copy of values
1001     def values(self):
1002         values = []
1003         for key in self._key_list:
1004             values.append(self[key])
1005         return values
1006
1007     ## Return a copy of (key, value) list
1008     def items(self):
1009         items = []
1010         for key in self._key_list:
1011             items.append((key, self[key]))
1012         return items
1013
1014     ## Iteration support
1015     def iteritems(self):
1016         return iter(self.items())
1017
1018     ## Keys interation support
1019     def iterkeys(self):
1020         return iter(self.keys())
1021
1022     ## Values interation support
1023     def itervalues(self):
1024         return iter(self.values())
1025
1026     ## Return value related to a key, and remove the (key, value) from the dict
1027     def pop(self, key, *dv):
1028         value = None
1029         if key in self._key_list:
1030             value = self[key]
1031             self.__delitem__(key)
1032         elif len(dv) != 0 :
1033             value = kv[0]
1034         return value
1035
1036     ## Return (key, value) pair, and remove the (key, value) from the dict
1037     def popitem(self):
1038         key = self._key_list[-1]
1039         value = self[key]
1040         self.__delitem__(key)
1041         return key, value
1042
1043     def update(self, dict=None, **kwargs):
1044         if dict is not None:
1045             for k, v in dict.items():
1046                 self[k] = v
1047         if len(kwargs):
1048             for k, v in kwargs.items():
1049                 self[k] = v
1050
1051 ## Dictionary with restricted keys
1052 #
1053 class rdict(dict):
1054     ## Constructor
1055     def __init__(self, KeyList):
1056         for Key in KeyList:
1057             dict.__setitem__(self, Key, "")
1058
1059     ## []= operator
1060     def __setitem__(self, key, value):
1061         if key not in self:
1062             EdkLogger.error("RestrictedDict", ATTRIBUTE_SET_FAILURE, "Key [%s] is not allowed" % key,
1063                             ExtraData=", ".join(dict.keys(self)))
1064         dict.__setitem__(self, key, value)
1065
1066     ## =[] operator
1067     def __getitem__(self, key):
1068         if key not in self:
1069             return ""
1070         return dict.__getitem__(self, key)
1071
1072     ## del operator
1073     def __delitem__(self, key):
1074         EdkLogger.error("RestrictedDict", ATTRIBUTE_ACCESS_DENIED, ExtraData="del")
1075
1076     ## Empty the dict
1077     def clear(self):
1078         for Key in self:
1079             self.__setitem__(Key, "")
1080
1081     ## Return value related to a key, and remove the (key, value) from the dict
1082     def pop(self, key, *dv):
1083         EdkLogger.error("RestrictedDict", ATTRIBUTE_ACCESS_DENIED, ExtraData="pop")
1084
1085     ## Return (key, value) pair, and remove the (key, value) from the dict
1086     def popitem(self):
1087         EdkLogger.error("RestrictedDict", ATTRIBUTE_ACCESS_DENIED, ExtraData="popitem")
1088
1089 ## Dictionary using prioritized list as key
1090 #
1091 class tdict:
1092     _ListType = type([])
1093     _TupleType = type(())
1094     _Wildcard = 'COMMON'
1095     _ValidWildcardList = ['COMMON', 'DEFAULT', 'ALL', TAB_STAR, 'PLATFORM']
1096
1097     def __init__(self, _Single_=False, _Level_=2):
1098         self._Level_ = _Level_
1099         self.data = {}
1100         self._Single_ = _Single_
1101
1102     # =[] operator
1103     def __getitem__(self, key):
1104         KeyType = type(key)
1105         RestKeys = None
1106         if KeyType == self._ListType or KeyType == self._TupleType:
1107             FirstKey = key[0]
1108             if len(key) > 1:
1109                 RestKeys = key[1:]
1110             elif self._Level_ > 1:
1111                 RestKeys = [self._Wildcard for i in range(0, self._Level_ - 1)]
1112         else:
1113             FirstKey = key
1114             if self._Level_ > 1:
1115                 RestKeys = [self._Wildcard for i in range(0, self._Level_ - 1)]
1116
1117         if FirstKey is None or str(FirstKey).upper() in self._ValidWildcardList:
1118             FirstKey = self._Wildcard
1119
1120         if self._Single_:
1121             return self._GetSingleValue(FirstKey, RestKeys)
1122         else:
1123             return self._GetAllValues(FirstKey, RestKeys)
1124
1125     def _GetSingleValue(self, FirstKey, RestKeys):
1126         Value = None
1127         #print "%s-%s" % (FirstKey, self._Level_) ,
1128         if self._Level_ > 1:
1129             if FirstKey == self._Wildcard:
1130                 if FirstKey in self.data:
1131                     Value = self.data[FirstKey][RestKeys]
1132                 if Value is None:
1133                     for Key in self.data:
1134                         Value = self.data[Key][RestKeys]
1135                         if Value is not None: break
1136             else:
1137                 if FirstKey in self.data:
1138                     Value = self.data[FirstKey][RestKeys]
1139                 if Value is None and self._Wildcard in self.data:
1140                     #print "Value=None"
1141                     Value = self.data[self._Wildcard][RestKeys]
1142         else:
1143             if FirstKey == self._Wildcard:
1144                 if FirstKey in self.data:
1145                     Value = self.data[FirstKey]
1146                 if Value is None:
1147                     for Key in self.data:
1148                         Value = self.data[Key]
1149                         if Value is not None: break
1150             else:
1151                 if FirstKey in self.data:
1152                     Value = self.data[FirstKey]
1153                 elif self._Wildcard in self.data:
1154                     Value = self.data[self._Wildcard]
1155         return Value
1156
1157     def _GetAllValues(self, FirstKey, RestKeys):
1158         Value = []
1159         if self._Level_ > 1:
1160             if FirstKey == self._Wildcard:
1161                 for Key in self.data:
1162                     Value += self.data[Key][RestKeys]
1163             else:
1164                 if FirstKey in self.data:
1165                     Value += self.data[FirstKey][RestKeys]
1166                 if self._Wildcard in self.data:
1167                     Value += self.data[self._Wildcard][RestKeys]
1168         else:
1169             if FirstKey == self._Wildcard:
1170                 for Key in self.data:
1171                     Value.append(self.data[Key])
1172             else:
1173                 if FirstKey in self.data:
1174                     Value.append(self.data[FirstKey])
1175                 if self._Wildcard in self.data:
1176                     Value.append(self.data[self._Wildcard])
1177         return Value
1178
1179     ## []= operator
1180     def __setitem__(self, key, value):
1181         KeyType = type(key)
1182         RestKeys = None
1183         if KeyType == self._ListType or KeyType == self._TupleType:
1184             FirstKey = key[0]
1185             if len(key) > 1:
1186                 RestKeys = key[1:]
1187             else:
1188                 RestKeys = [self._Wildcard for i in range(0, self._Level_ - 1)]
1189         else:
1190             FirstKey = key
1191             if self._Level_ > 1:
1192                 RestKeys = [self._Wildcard for i in range(0, self._Level_ - 1)]
1193
1194         if FirstKey in self._ValidWildcardList:
1195             FirstKey = self._Wildcard
1196
1197         if FirstKey not in self.data and self._Level_ > 0:
1198             self.data[FirstKey] = tdict(self._Single_, self._Level_ - 1)
1199
1200         if self._Level_ > 1:
1201             self.data[FirstKey][RestKeys] = value
1202         else:
1203             self.data[FirstKey] = value
1204
1205     def SetGreedyMode(self):
1206         self._Single_ = False
1207         if self._Level_ > 1:
1208             for Key in self.data:
1209                 self.data[Key].SetGreedyMode()
1210
1211     def SetSingleMode(self):
1212         self._Single_ = True
1213         if self._Level_ > 1:
1214             for Key in self.data:
1215                 self.data[Key].SetSingleMode()
1216
1217     def GetKeys(self, KeyIndex=0):
1218         assert KeyIndex >= 0
1219         if KeyIndex == 0:
1220             return set(self.data.keys())
1221         else:
1222             keys = set()
1223             for Key in self.data:
1224                 keys |= self.data[Key].GetKeys(KeyIndex - 1)
1225             return keys
1226
1227 def IsFieldValueAnArray (Value):
1228     Value = Value.strip()
1229     if Value.startswith(TAB_GUID) and Value.endswith(')'):
1230         return True
1231     if Value.startswith('L"') and Value.endswith('"')  and len(list(Value[2:-1])) > 1:
1232         return True
1233     if Value[0] == '"' and Value[-1] == '"' and len(list(Value[1:-1])) > 1:
1234         return True
1235     if Value[0] == '{' and Value[-1] == '}':
1236         return True
1237     if Value.startswith("L'") and Value.endswith("'") and len(list(Value[2:-1])) > 1:
1238         return True
1239     if Value[0] == "'" and Value[-1] == "'" and len(list(Value[1:-1])) > 1:
1240         return True
1241     return False
1242
1243 def AnalyzePcdExpression(Setting):
1244     RanStr = ''.join(sample(string.ascii_letters + string.digits, 8))
1245     Setting = Setting.replace('\\\\', RanStr).strip()
1246     # There might be escaped quote in a string: \", \\\" , \', \\\'
1247     Data = Setting
1248     # There might be '|' in string and in ( ... | ... ), replace it with '-'
1249     NewStr = ''
1250     InSingleQuoteStr = False
1251     InDoubleQuoteStr = False
1252     Pair = 0
1253     for Index, ch in enumerate(Data):
1254         if ch == '"' and not InSingleQuoteStr:
1255             if Data[Index - 1] != '\\':
1256                 InDoubleQuoteStr = not InDoubleQuoteStr
1257         elif ch == "'" and not InDoubleQuoteStr:
1258             if Data[Index - 1] != '\\':
1259                 InSingleQuoteStr = not InSingleQuoteStr
1260         elif ch == '(' and not (InSingleQuoteStr or InDoubleQuoteStr):
1261             Pair += 1
1262         elif ch == ')' and not (InSingleQuoteStr or InDoubleQuoteStr):
1263             Pair -= 1
1264
1265         if (Pair > 0 or InSingleQuoteStr or InDoubleQuoteStr) and ch == TAB_VALUE_SPLIT:
1266             NewStr += '-'
1267         else:
1268             NewStr += ch
1269     FieldList = []
1270     StartPos = 0
1271     while True:
1272         Pos = NewStr.find(TAB_VALUE_SPLIT, StartPos)
1273         if Pos < 0:
1274             FieldList.append(Setting[StartPos:].strip())
1275             break
1276         FieldList.append(Setting[StartPos:Pos].strip())
1277         StartPos = Pos + 1
1278     for i, ch in enumerate(FieldList):
1279         if RanStr in ch:
1280             FieldList[i] = ch.replace(RanStr,'\\\\')
1281     return FieldList
1282
1283 def ParseDevPathValue (Value):
1284     if '\\' in Value:
1285         Value.replace('\\', '/').replace(' ', '')
1286
1287     Cmd = 'DevicePath ' + '"' + Value + '"'
1288     try:
1289         p = subprocess.Popen(Cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
1290         out, err = p.communicate()
1291     except Exception as X:
1292         raise BadExpression("DevicePath: %s" % (str(X)) )
1293     finally:
1294         subprocess._cleanup()
1295         p.stdout.close()
1296         p.stderr.close()
1297     if err:
1298         raise BadExpression("DevicePath: %s" % str(err))
1299     Size = len(out.split())
1300     out = ','.join(out.split())
1301     return '{' + out + '}', Size
1302
1303 def ParseFieldValue (Value):
1304     if "{CODE(" in Value:
1305         return Value, len(Value.split(","))
1306     if isinstance(Value, type(0)):
1307         return Value, (Value.bit_length() + 7) / 8
1308     if not isinstance(Value, type('')):
1309         raise BadExpression('Type %s is %s' %(Value, type(Value)))
1310     Value = Value.strip()
1311     if Value.startswith(TAB_UINT8) and Value.endswith(')'):
1312         Value, Size = ParseFieldValue(Value.split('(', 1)[1][:-1])
1313         if Size > 1:
1314             raise BadExpression('Value (%s) Size larger than %d' %(Value, Size))
1315         return Value, 1
1316     if Value.startswith(TAB_UINT16) and Value.endswith(')'):
1317         Value, Size = ParseFieldValue(Value.split('(', 1)[1][:-1])
1318         if Size > 2:
1319             raise BadExpression('Value (%s) Size larger than %d' %(Value, Size))
1320         return Value, 2
1321     if Value.startswith(TAB_UINT32) and Value.endswith(')'):
1322         Value, Size = ParseFieldValue(Value.split('(', 1)[1][:-1])
1323         if Size > 4:
1324             raise BadExpression('Value (%s) Size larger than %d' %(Value, Size))
1325         return Value, 4
1326     if Value.startswith(TAB_UINT64) and Value.endswith(')'):
1327         Value, Size = ParseFieldValue(Value.split('(', 1)[1][:-1])
1328         if Size > 8:
1329             raise BadExpression('Value (%s) Size larger than %d' % (Value, Size))
1330         return Value, 8
1331     if Value.startswith(TAB_GUID) and Value.endswith(')'):
1332         Value = Value.split('(', 1)[1][:-1].strip()
1333         if Value[0] == '{' and Value[-1] == '}':
1334             TmpValue = GuidStructureStringToGuidString(Value)
1335             if not TmpValue:
1336                 raise BadExpression("Invalid GUID value string %s" % Value)
1337             Value = TmpValue
1338         if Value[0] == '"' and Value[-1] == '"':
1339             Value = Value[1:-1]
1340         try:
1341             Value = "'" + uuid.UUID(Value).get_bytes_le() + "'"
1342         except ValueError as Message:
1343             raise BadExpression(Message)
1344         Value, Size = ParseFieldValue(Value)
1345         return Value, 16
1346     if Value.startswith('L"') and Value.endswith('"'):
1347         # Unicode String
1348         # translate escape character
1349         Value = Value[1:]
1350         try:
1351             Value = eval(Value)
1352         except:
1353             Value = Value[1:-1]
1354         List = list(Value)
1355         List.reverse()
1356         Value = 0
1357         for Char in List:
1358             Value = (Value << 16) | ord(Char)
1359         return Value, (len(List) + 1) * 2
1360     if Value.startswith('"') and Value.endswith('"'):
1361         # ASCII String
1362         # translate escape character
1363         try:
1364             Value = eval(Value)
1365         except:
1366             Value = Value[1:-1]
1367         List = list(Value)
1368         List.reverse()
1369         Value = 0
1370         for Char in List:
1371             Value = (Value << 8) | ord(Char)
1372         return Value, len(List) + 1
1373     if Value.startswith("L'") and Value.endswith("'"):
1374         # Unicode Character Constant
1375         # translate escape character
1376         Value = Value[1:]
1377         try:
1378             Value = eval(Value)
1379         except:
1380             Value = Value[1:-1]
1381         List = list(Value)
1382         if len(List) == 0:
1383             raise BadExpression('Length %s is %s' % (Value, len(List)))
1384         List.reverse()
1385         Value = 0
1386         for Char in List:
1387             Value = (Value << 16) | ord(Char)
1388         return Value, len(List) * 2
1389     if Value.startswith("'") and Value.endswith("'"):
1390         # Character constant
1391         # translate escape character
1392         try:
1393             Value = eval(Value)
1394         except:
1395             Value = Value[1:-1]
1396         List = list(Value)
1397         if len(List) == 0:
1398             raise BadExpression('Length %s is %s' % (Value, len(List)))
1399         List.reverse()
1400         Value = 0
1401         for Char in List:
1402             Value = (Value << 8) | ord(Char)
1403         return Value, len(List)
1404     if Value.startswith('{') and Value.endswith('}'):
1405         # Byte array
1406         Value = Value[1:-1]
1407         List = [Item.strip() for Item in Value.split(',')]
1408         List.reverse()
1409         Value = 0
1410         RetSize = 0
1411         for Item in List:
1412             ItemValue, Size = ParseFieldValue(Item)
1413             RetSize += Size
1414             for I in range(Size):
1415                 Value = (Value << 8) | ((ItemValue >> 8 * I) & 0xff)
1416         return Value, RetSize
1417     if Value.startswith('DEVICE_PATH(') and Value.endswith(')'):
1418         Value = Value.replace("DEVICE_PATH(", '').rstrip(')')
1419         Value = Value.strip().strip('"')
1420         return ParseDevPathValue(Value)
1421     if Value.lower().startswith('0x'):
1422         try:
1423             Value = int(Value, 16)
1424         except:
1425             raise BadExpression("invalid hex value: %s" % Value)
1426         if Value == 0:
1427             return 0, 1
1428         return Value, (Value.bit_length() + 7) / 8
1429     if Value[0].isdigit():
1430         Value = int(Value, 10)
1431         if Value == 0:
1432             return 0, 1
1433         return Value, (Value.bit_length() + 7) / 8
1434     if Value.lower() == 'true':
1435         return 1, 1
1436     if Value.lower() == 'false':
1437         return 0, 1
1438     return Value, 1
1439
1440 ## AnalyzeDscPcd
1441 #
1442 #  Analyze DSC PCD value, since there is no data type info in DSC
1443 #  This function is used to match functions (AnalyzePcdData) used for retrieving PCD value from database
1444 #  1. Feature flag: TokenSpace.PcdCName|PcdValue
1445 #  2. Fix and Patch:TokenSpace.PcdCName|PcdValue[|VOID*[|MaxSize]]
1446 #  3. Dynamic default:
1447 #     TokenSpace.PcdCName|PcdValue[|VOID*[|MaxSize]]
1448 #     TokenSpace.PcdCName|PcdValue
1449 #  4. Dynamic VPD:
1450 #     TokenSpace.PcdCName|VpdOffset[|VpdValue]
1451 #     TokenSpace.PcdCName|VpdOffset[|MaxSize[|VpdValue]]
1452 #  5. Dynamic HII:
1453 #     TokenSpace.PcdCName|HiiString|VariableGuid|VariableOffset[|HiiValue]
1454 #  PCD value needs to be located in such kind of string, and the PCD value might be an expression in which
1455 #    there might have "|" operator, also in string value.
1456 #
1457 #  @param Setting: String contain information described above with "TokenSpace.PcdCName|" stripped
1458 #  @param PcdType: PCD type: feature, fixed, dynamic default VPD HII
1459 #  @param DataType: The datum type of PCD: VOID*, UNIT, BOOL
1460 #  @retval:
1461 #    ValueList: A List contain fields described above
1462 #    IsValid:   True if conforming EBNF, otherwise False
1463 #    Index:     The index where PcdValue is in ValueList
1464 #
1465 def AnalyzeDscPcd(Setting, PcdType, DataType=''):
1466     FieldList = AnalyzePcdExpression(Setting)
1467
1468     IsValid = True
1469     if PcdType in (MODEL_PCD_FIXED_AT_BUILD, MODEL_PCD_PATCHABLE_IN_MODULE, MODEL_PCD_DYNAMIC_DEFAULT, MODEL_PCD_DYNAMIC_EX_DEFAULT):
1470         Value = FieldList[0]
1471         Size = ''
1472         if len(FieldList) > 1 and FieldList[1]:
1473             DataType = FieldList[1]
1474             if FieldList[1] != TAB_VOID and StructPattern.match(FieldList[1]) is None:
1475                 IsValid = False
1476         if len(FieldList) > 2:
1477             Size = FieldList[2]
1478         if IsValid:
1479             if DataType == "":
1480                 IsValid = (len(FieldList) <= 1)
1481             else:
1482                 IsValid = (len(FieldList) <= 3)
1483
1484         if Size:
1485             try:
1486                 int(Size, 16) if Size.upper().startswith("0X") else int(Size)
1487             except:
1488                 IsValid = False
1489                 Size = -1
1490         return [str(Value), DataType, str(Size)], IsValid, 0
1491     elif PcdType == MODEL_PCD_FEATURE_FLAG:
1492         Value = FieldList[0]
1493         Size = ''
1494         IsValid = (len(FieldList) <= 1)
1495         return [Value, DataType, str(Size)], IsValid, 0
1496     elif PcdType in (MODEL_PCD_DYNAMIC_VPD, MODEL_PCD_DYNAMIC_EX_VPD):
1497         VpdOffset = FieldList[0]
1498         Value = Size = ''
1499         if not DataType == TAB_VOID:
1500             if len(FieldList) > 1:
1501                 Value = FieldList[1]
1502         else:
1503             if len(FieldList) > 1:
1504                 Size = FieldList[1]
1505             if len(FieldList) > 2:
1506                 Value = FieldList[2]
1507         if DataType == "":
1508             IsValid = (len(FieldList) <= 1)
1509         else:
1510             IsValid = (len(FieldList) <= 3)
1511         if Size:
1512             try:
1513                 int(Size, 16) if Size.upper().startswith("0X") else int(Size)
1514             except:
1515                 IsValid = False
1516                 Size = -1
1517         return [VpdOffset, str(Size), Value], IsValid, 2
1518     elif PcdType in (MODEL_PCD_DYNAMIC_HII, MODEL_PCD_DYNAMIC_EX_HII):
1519         IsValid = (3 <= len(FieldList) <= 5)
1520         HiiString = FieldList[0]
1521         Guid = Offset = Value = Attribute = ''
1522         if len(FieldList) > 1:
1523             Guid = FieldList[1]
1524         if len(FieldList) > 2:
1525             Offset = FieldList[2]
1526         if len(FieldList) > 3:
1527             Value = FieldList[3]
1528         if len(FieldList) > 4:
1529             Attribute = FieldList[4]
1530         return [HiiString, Guid, Offset, Value, Attribute], IsValid, 3
1531     return [], False, 0
1532
1533 ## AnalyzePcdData
1534 #
1535 #  Analyze the pcd Value, Datum type and TokenNumber.
1536 #  Used to avoid split issue while the value string contain "|" character
1537 #
1538 #  @param[in] Setting:  A String contain value/datum type/token number information;
1539 #
1540 #  @retval   ValueList: A List contain value, datum type and toke number.
1541 #
1542 def AnalyzePcdData(Setting):
1543     ValueList = ['', '', '']
1544
1545     ValueRe = re.compile(r'^\s*L?\".*\|.*\"')
1546     PtrValue = ValueRe.findall(Setting)
1547
1548     ValueUpdateFlag = False
1549
1550     if len(PtrValue) >= 1:
1551         Setting = re.sub(ValueRe, '', Setting)
1552         ValueUpdateFlag = True
1553
1554     TokenList = Setting.split(TAB_VALUE_SPLIT)
1555     ValueList[0:len(TokenList)] = TokenList
1556
1557     if ValueUpdateFlag:
1558         ValueList[0] = PtrValue[0]
1559
1560     return ValueList
1561
1562 ## check format of PCD value against its the datum type
1563 #
1564 # For PCD value setting
1565 #
1566 def CheckPcdDatum(Type, Value):
1567     if Type == TAB_VOID:
1568         ValueRe = re.compile(r'\s*L?\".*\"\s*$')
1569         if not (((Value.startswith('L"') or Value.startswith('"')) and Value.endswith('"'))
1570                 or (Value.startswith('{') and Value.endswith('}')) or (Value.startswith("L'") or Value.startswith("'") and Value.endswith("'"))
1571                ):
1572             return False, "Invalid value [%s] of type [%s]; must be in the form of {...} for array"\
1573                           ", \"...\" or \'...\' for string, L\"...\" or L\'...\' for unicode string" % (Value, Type)
1574         elif ValueRe.match(Value):
1575             # Check the chars in UnicodeString or CString is printable
1576             if Value.startswith("L"):
1577                 Value = Value[2:-1]
1578             else:
1579                 Value = Value[1:-1]
1580             Printset = set(string.printable)
1581             Printset.remove(TAB_PRINTCHAR_VT)
1582             Printset.add(TAB_PRINTCHAR_BS)
1583             Printset.add(TAB_PRINTCHAR_NUL)
1584             if not set(Value).issubset(Printset):
1585                 PrintList = sorted(Printset)
1586                 return False, "Invalid PCD string value of type [%s]; must be printable chars %s." % (Type, PrintList)
1587     elif Type == 'BOOLEAN':
1588         if Value not in ['TRUE', 'True', 'true', '0x1', '0x01', '1', 'FALSE', 'False', 'false', '0x0', '0x00', '0']:
1589             return False, "Invalid value [%s] of type [%s]; must be one of TRUE, True, true, 0x1, 0x01, 1"\
1590                           ", FALSE, False, false, 0x0, 0x00, 0" % (Value, Type)
1591     elif Type in [TAB_UINT8, TAB_UINT16, TAB_UINT32, TAB_UINT64]:
1592         if Value and int(Value, 0) < 0:
1593             return False, "PCD can't be set to negative value[%s] for datum type [%s]" % (Value, Type)
1594         try:
1595             Value = long(Value, 0)
1596             if Value > MAX_VAL_TYPE[Type]:
1597                 return False, "Too large PCD value[%s] for datum type [%s]" % (Value, Type)
1598         except:
1599             return False, "Invalid value [%s] of type [%s];"\
1600                           " must be a hexadecimal, decimal or octal in C language format." % (Value, Type)
1601     else:
1602         return True, "StructurePcd"
1603
1604     return True, ""
1605
1606 ## Split command line option string to list
1607 #
1608 # subprocess.Popen needs the args to be a sequence. Otherwise there's problem
1609 # in non-windows platform to launch command
1610 #
1611 def SplitOption(OptionString):
1612     OptionList = []
1613     LastChar = " "
1614     OptionStart = 0
1615     QuotationMark = ""
1616     for Index in range(0, len(OptionString)):
1617         CurrentChar = OptionString[Index]
1618         if CurrentChar in ['"', "'"]:
1619             if QuotationMark == CurrentChar:
1620                 QuotationMark = ""
1621             elif QuotationMark == "":
1622                 QuotationMark = CurrentChar
1623             continue
1624         elif QuotationMark:
1625             continue
1626
1627         if CurrentChar in ["/", "-"] and LastChar in [" ", "\t", "\r", "\n"]:
1628             if Index > OptionStart:
1629                 OptionList.append(OptionString[OptionStart:Index - 1])
1630             OptionStart = Index
1631         LastChar = CurrentChar
1632     OptionList.append(OptionString[OptionStart:])
1633     return OptionList
1634
1635 def CommonPath(PathList):
1636     P1 = min(PathList).split(os.path.sep)
1637     P2 = max(PathList).split(os.path.sep)
1638     for Index in xrange(min(len(P1), len(P2))):
1639         if P1[Index] != P2[Index]:
1640             return os.path.sep.join(P1[:Index])
1641     return os.path.sep.join(P1)
1642
1643 #
1644 # Convert string to C format array
1645 #
1646 def ConvertStringToByteArray(Value):
1647     Value = Value.strip()
1648     if not Value:
1649         return None
1650     if Value[0] == '{':
1651         if not Value.endswith('}'):
1652             return None
1653         Value = Value.replace(' ', '').replace('{', '').replace('}', '')
1654         ValFields = Value.split(',')
1655         try:
1656             for Index in range(len(ValFields)):
1657                 ValFields[Index] = str(int(ValFields[Index], 0))
1658         except ValueError:
1659             return None
1660         Value = '{' + ','.join(ValFields) + '}'
1661         return Value
1662
1663     Unicode = False
1664     if Value.startswith('L"'):
1665         if not Value.endswith('"'):
1666             return None
1667         Value = Value[1:]
1668         Unicode = True
1669     elif not Value.startswith('"') or not Value.endswith('"'):
1670         return None
1671
1672     Value = eval(Value)         # translate escape character
1673     NewValue = '{'
1674     for Index in range(0, len(Value)):
1675         if Unicode:
1676             NewValue = NewValue + str(ord(Value[Index]) % 0x10000) + ','
1677         else:
1678             NewValue = NewValue + str(ord(Value[Index]) % 0x100) + ','
1679     Value = NewValue + '0}'
1680     return Value
1681
1682 class PathClass(object):
1683     def __init__(self, File='', Root='', AlterRoot='', Type='', IsBinary=False,
1684                  Arch='COMMON', ToolChainFamily='', Target='', TagName='', ToolCode=''):
1685         self.Arch = Arch
1686         self.File = str(File)
1687         if os.path.isabs(self.File):
1688             self.Root = ''
1689             self.AlterRoot = ''
1690         else:
1691             self.Root = str(Root)
1692             self.AlterRoot = str(AlterRoot)
1693
1694         # Remove any '.' and '..' in path
1695         if self.Root:
1696             self.Root = mws.getWs(self.Root, self.File)
1697             self.Path = os.path.normpath(os.path.join(self.Root, self.File))
1698             self.Root = os.path.normpath(CommonPath([self.Root, self.Path]))
1699             # eliminate the side-effect of 'C:'
1700             if self.Root[-1] == ':':
1701                 self.Root += os.path.sep
1702             # file path should not start with path separator
1703             if self.Root[-1] == os.path.sep:
1704                 self.File = self.Path[len(self.Root):]
1705             else:
1706                 self.File = self.Path[len(self.Root) + 1:]
1707         else:
1708             self.Path = os.path.normpath(self.File)
1709
1710         self.SubDir, self.Name = os.path.split(self.File)
1711         self.BaseName, self.Ext = os.path.splitext(self.Name)
1712
1713         if self.Root:
1714             if self.SubDir:
1715                 self.Dir = os.path.join(self.Root, self.SubDir)
1716             else:
1717                 self.Dir = self.Root
1718         else:
1719             self.Dir = self.SubDir
1720
1721         if IsBinary:
1722             self.Type = Type
1723         else:
1724             self.Type = self.Ext.lower()
1725
1726         self.IsBinary = IsBinary
1727         self.Target = Target
1728         self.TagName = TagName
1729         self.ToolCode = ToolCode
1730         self.ToolChainFamily = ToolChainFamily
1731
1732     ## Convert the object of this class to a string
1733     #
1734     #  Convert member Path of the class to a string
1735     #
1736     #  @retval string Formatted String
1737     #
1738     def __str__(self):
1739         return self.Path
1740
1741     ## Override __eq__ function
1742     #
1743     # Check whether PathClass are the same
1744     #
1745     # @retval False The two PathClass are different
1746     # @retval True  The two PathClass are the same
1747     #
1748     def __eq__(self, Other):
1749         return self.Path == str(Other)
1750
1751     ## Override __cmp__ function
1752     #
1753     # Customize the comparsion operation of two PathClass
1754     #
1755     # @retval 0     The two PathClass are different
1756     # @retval -1    The first PathClass is less than the second PathClass
1757     # @retval 1     The first PathClass is Bigger than the second PathClass
1758     def __cmp__(self, Other):
1759         OtherKey = str(Other)
1760
1761         SelfKey = self.Path
1762         if SelfKey == OtherKey:
1763             return 0
1764         elif SelfKey > OtherKey:
1765             return 1
1766         else:
1767             return -1
1768
1769     ## Override __hash__ function
1770     #
1771     # Use Path as key in hash table
1772     #
1773     # @retval string Key for hash table
1774     #
1775     def __hash__(self):
1776         return hash(self.Path)
1777
1778     @cached_property
1779     def Key(self):
1780         return self.Path.upper()
1781
1782     @property
1783     def TimeStamp(self):
1784         return os.stat(self.Path)[8]
1785
1786     def Validate(self, Type='', CaseSensitive=True):
1787         if GlobalData.gCaseInsensitive:
1788             CaseSensitive = False
1789         if Type and Type.lower() != self.Type:
1790             return FILE_TYPE_MISMATCH, '%s (expect %s but got %s)' % (self.File, Type, self.Type)
1791
1792         RealFile, RealRoot = RealPath2(self.File, self.Root, self.AlterRoot)
1793         if not RealRoot and not RealFile:
1794             RealFile = self.File
1795             if self.AlterRoot:
1796                 RealFile = os.path.join(self.AlterRoot, self.File)
1797             elif self.Root:
1798                 RealFile = os.path.join(self.Root, self.File)
1799             if len (mws.getPkgPath()) == 0:
1800                 return FILE_NOT_FOUND, os.path.join(self.AlterRoot, RealFile)
1801             else:
1802                 return FILE_NOT_FOUND, "%s is not found in packages path:\n\t%s" % (self.File, '\n\t'.join(mws.getPkgPath()))
1803
1804         ErrorCode = 0
1805         ErrorInfo = ''
1806         if RealRoot != self.Root or RealFile != self.File:
1807             if CaseSensitive and (RealFile != self.File or (RealRoot != self.Root and RealRoot != self.AlterRoot)):
1808                 ErrorCode = FILE_CASE_MISMATCH
1809                 ErrorInfo = self.File + '\n\t' + RealFile + " [in file system]"
1810
1811             self.SubDir, self.Name = os.path.split(RealFile)
1812             self.BaseName, self.Ext = os.path.splitext(self.Name)
1813             if self.SubDir:
1814                 self.Dir = os.path.join(RealRoot, self.SubDir)
1815             else:
1816                 self.Dir = RealRoot
1817             self.File = RealFile
1818             self.Root = RealRoot
1819             self.Path = os.path.join(RealRoot, RealFile)
1820         return ErrorCode, ErrorInfo
1821
1822 ## Parse PE image to get the required PE informaion.
1823 #
1824 class PeImageClass():
1825     ## Constructor
1826     #
1827     #   @param  File FilePath of PeImage
1828     #
1829     def __init__(self, PeFile):
1830         self.FileName   = PeFile
1831         self.IsValid    = False
1832         self.Size       = 0
1833         self.EntryPoint = 0
1834         self.SectionAlignment  = 0
1835         self.SectionHeaderList = []
1836         self.ErrorInfo = ''
1837         try:
1838             PeObject = open(PeFile, 'rb')
1839         except:
1840             self.ErrorInfo = self.FileName + ' can not be found\n'
1841             return
1842         # Read DOS header
1843         ByteArray = array.array('B')
1844         ByteArray.fromfile(PeObject, 0x3E)
1845         ByteList = ByteArray.tolist()
1846         # DOS signature should be 'MZ'
1847         if self._ByteListToStr (ByteList[0x0:0x2]) != 'MZ':
1848             self.ErrorInfo = self.FileName + ' has no valid DOS signature MZ'
1849             return
1850
1851         # Read 4 byte PE Signature
1852         PeOffset = self._ByteListToInt(ByteList[0x3C:0x3E])
1853         PeObject.seek(PeOffset)
1854         ByteArray = array.array('B')
1855         ByteArray.fromfile(PeObject, 4)
1856         # PE signature should be 'PE\0\0'
1857         if ByteArray.tostring() != 'PE\0\0':
1858             self.ErrorInfo = self.FileName + ' has no valid PE signature PE00'
1859             return
1860
1861         # Read PE file header
1862         ByteArray = array.array('B')
1863         ByteArray.fromfile(PeObject, 0x14)
1864         ByteList = ByteArray.tolist()
1865         SecNumber = self._ByteListToInt(ByteList[0x2:0x4])
1866         if SecNumber == 0:
1867             self.ErrorInfo = self.FileName + ' has no section header'
1868             return
1869
1870         # Read PE optional header
1871         OptionalHeaderSize = self._ByteListToInt(ByteArray[0x10:0x12])
1872         ByteArray = array.array('B')
1873         ByteArray.fromfile(PeObject, OptionalHeaderSize)
1874         ByteList = ByteArray.tolist()
1875         self.EntryPoint       = self._ByteListToInt(ByteList[0x10:0x14])
1876         self.SectionAlignment = self._ByteListToInt(ByteList[0x20:0x24])
1877         self.Size             = self._ByteListToInt(ByteList[0x38:0x3C])
1878
1879         # Read each Section Header
1880         for Index in range(SecNumber):
1881             ByteArray = array.array('B')
1882             ByteArray.fromfile(PeObject, 0x28)
1883             ByteList = ByteArray.tolist()
1884             SecName  = self._ByteListToStr(ByteList[0:8])
1885             SecVirtualSize = self._ByteListToInt(ByteList[8:12])
1886             SecRawAddress  = self._ByteListToInt(ByteList[20:24])
1887             SecVirtualAddress = self._ByteListToInt(ByteList[12:16])
1888             self.SectionHeaderList.append((SecName, SecVirtualAddress, SecRawAddress, SecVirtualSize))
1889         self.IsValid = True
1890         PeObject.close()
1891
1892     def _ByteListToStr(self, ByteList):
1893         String = ''
1894         for index in range(len(ByteList)):
1895             if ByteList[index] == 0:
1896                 break
1897             String += chr(ByteList[index])
1898         return String
1899
1900     def _ByteListToInt(self, ByteList):
1901         Value = 0
1902         for index in range(len(ByteList) - 1, -1, -1):
1903             Value = (Value << 8) | int(ByteList[index])
1904         return Value
1905
1906 class DefaultStore():
1907     def __init__(self, DefaultStores ):
1908
1909         self.DefaultStores = DefaultStores
1910     def DefaultStoreID(self, DefaultStoreName):
1911         for key, value in self.DefaultStores.items():
1912             if value == DefaultStoreName:
1913                 return key
1914         return None
1915     def GetDefaultDefault(self):
1916         if not self.DefaultStores or "0" in self.DefaultStores:
1917             return "0", TAB_DEFAULT_STORES_DEFAULT
1918         else:
1919             minvalue = min(int(value_str) for value_str in self.DefaultStores)
1920             return (str(minvalue), self.DefaultStores[str(minvalue)])
1921     def GetMin(self, DefaultSIdList):
1922         if not DefaultSIdList:
1923             return TAB_DEFAULT_STORES_DEFAULT
1924         storeidset = {storeid for storeid, storename in self.DefaultStores.values() if storename in DefaultSIdList}
1925         if not storeidset:
1926             return ""
1927         minid = min(storeidset )
1928         for sid, name in self.DefaultStores.values():
1929             if sid == minid:
1930                 return name
1931
1932 class SkuClass():
1933     DEFAULT = 0
1934     SINGLE = 1
1935     MULTIPLE =2
1936
1937     def __init__(self,SkuIdentifier='', SkuIds=None):
1938         if SkuIds is None:
1939             SkuIds = {}
1940
1941         for SkuName in SkuIds:
1942             SkuId = SkuIds[SkuName][0]
1943             skuid_num = int(SkuId, 16) if SkuId.upper().startswith("0X") else int(SkuId)
1944             if skuid_num > 0xFFFFFFFFFFFFFFFF:
1945                 EdkLogger.error("build", PARAMETER_INVALID,
1946                             ExtraData = "SKU-ID [%s] value %s exceeds the max value of UINT64"
1947                                       % (SkuName, SkuId))
1948
1949         self.AvailableSkuIds = sdict()
1950         self.SkuIdSet = []
1951         self.SkuIdNumberSet = []
1952         self.SkuData = SkuIds
1953         self._SkuInherit = {}
1954         self._SkuIdentifier = SkuIdentifier
1955         if SkuIdentifier == '' or SkuIdentifier is None:
1956             self.SkuIdSet = ['DEFAULT']
1957             self.SkuIdNumberSet = ['0U']
1958         elif SkuIdentifier == 'ALL':
1959             self.SkuIdSet = SkuIds.keys()
1960             self.SkuIdNumberSet = [num[0].strip() + 'U' for num in SkuIds.values()]
1961         else:
1962             r = SkuIdentifier.split('|')
1963             self.SkuIdSet=[(r[k].strip()).upper() for k in range(len(r))]
1964             k = None
1965             try:
1966                 self.SkuIdNumberSet = [SkuIds[k][0].strip() + 'U' for k in self.SkuIdSet]
1967             except Exception:
1968                 EdkLogger.error("build", PARAMETER_INVALID,
1969                             ExtraData = "SKU-ID [%s] is not supported by the platform. [Valid SKU-ID: %s]"
1970                                       % (k, " | ".join(SkuIds.keys())))
1971         for each in self.SkuIdSet:
1972             if each in SkuIds:
1973                 self.AvailableSkuIds[each] = SkuIds[each][0]
1974             else:
1975                 EdkLogger.error("build", PARAMETER_INVALID,
1976                             ExtraData="SKU-ID [%s] is not supported by the platform. [Valid SKU-ID: %s]"
1977                                       % (each, " | ".join(SkuIds.keys())))
1978         if self.SkuUsageType != SkuClass.SINGLE:
1979             self.AvailableSkuIds.update({'DEFAULT':0, 'COMMON':0})
1980         if self.SkuIdSet:
1981             GlobalData.gSkuids = (self.SkuIdSet)
1982             if 'COMMON' in GlobalData.gSkuids:
1983                 GlobalData.gSkuids.remove('COMMON')
1984             if self.SkuUsageType == self.SINGLE:
1985                 if len(GlobalData.gSkuids) != 1:
1986                     if 'DEFAULT' in GlobalData.gSkuids:
1987                         GlobalData.gSkuids.remove('DEFAULT')
1988             if GlobalData.gSkuids:
1989                 GlobalData.gSkuids.sort()
1990
1991     def GetNextSkuId(self, skuname):
1992         if not self._SkuInherit:
1993             self._SkuInherit = {}
1994             for item in self.SkuData.values():
1995                 self._SkuInherit[item[1]]=item[2] if item[2] else "DEFAULT"
1996         return self._SkuInherit.get(skuname, "DEFAULT")
1997
1998     def GetSkuChain(self, sku):
1999         if sku == "DEFAULT":
2000             return ["DEFAULT"]
2001         skulist = [sku]
2002         nextsku = sku
2003         while True:
2004             nextsku = self.GetNextSkuId(nextsku)
2005             skulist.append(nextsku)
2006             if nextsku == "DEFAULT":
2007                 break
2008         skulist.reverse()
2009         return skulist
2010     def SkuOverrideOrder(self):
2011         skuorderset = []
2012         for skuname in self.SkuIdSet:
2013             skuorderset.append(self.GetSkuChain(skuname))
2014
2015         skuorder = []
2016         for index in range(max(len(item) for item in skuorderset)):
2017             for subset in skuorderset:
2018                 if index > len(subset)-1:
2019                     continue
2020                 if subset[index] in skuorder:
2021                     continue
2022                 skuorder.append(subset[index])
2023
2024         return skuorder
2025
2026     @property
2027     def SkuUsageType(self):
2028         if self._SkuIdentifier.upper() == "ALL":
2029             return SkuClass.MULTIPLE
2030
2031         if len(self.SkuIdSet) == 1:
2032             if self.SkuIdSet[0] == 'DEFAULT':
2033                 return SkuClass.DEFAULT
2034             return SkuClass.SINGLE
2035         if len(self.SkuIdSet) == 2 and 'DEFAULT' in self.SkuIdSet:
2036             return SkuClass.SINGLE
2037         return SkuClass.MULTIPLE
2038
2039     def DumpSkuIdArrary(self):
2040         if self.SkuUsageType == SkuClass.SINGLE:
2041             return "{0x0}"
2042         ArrayStrList = []
2043         for skuname in self.AvailableSkuIds:
2044             if skuname == "COMMON":
2045                 continue
2046             while skuname != "DEFAULT":
2047                 ArrayStrList.append(hex(int(self.AvailableSkuIds[skuname])))
2048                 skuname = self.GetNextSkuId(skuname)
2049             ArrayStrList.append("0x0")
2050         return "{{{myList}}}".format(myList=",".join(ArrayStrList))
2051
2052     @property
2053     def AvailableSkuIdSet(self):
2054         return self.AvailableSkuIds
2055
2056     @property
2057     def SystemSkuId(self):
2058         if self.SkuUsageType == SkuClass.SINGLE:
2059             if len(self.SkuIdSet) == 1:
2060                 return self.SkuIdSet[0]
2061             else:
2062                 return self.SkuIdSet[0] if self.SkuIdSet[0] != 'DEFAULT' else self.SkuIdSet[1]
2063         else:
2064             return 'DEFAULT'
2065
2066 #
2067 # Pack a registry format GUID
2068 #
2069 def PackRegistryFormatGuid(Guid):
2070     return PackGUID(Guid.split('-'))
2071
2072 ##  Get the integer value from string like "14U" or integer like 2
2073 #
2074 #   @param      Input   The object that may be either a integer value or a string
2075 #
2076 #   @retval     Value    The integer value that the input represents
2077 #
2078 def GetIntegerValue(Input):
2079     if type(Input) in (int, long):
2080         return Input
2081     String = Input
2082     if String.endswith("U"):
2083         String = String[:-1]
2084     if String.endswith("ULL"):
2085         String = String[:-3]
2086     if String.endswith("LL"):
2087         String = String[:-2]
2088
2089     if String.startswith("0x") or String.startswith("0X"):
2090         return int(String, 16)
2091     elif String == '':
2092         return 0
2093     else:
2094         return int(String)
2095
2096 #
2097 # Pack a GUID (registry format) list into a buffer and return it
2098 #
2099 def PackGUID(Guid):
2100     return pack(PACK_PATTERN_GUID,
2101                 int(Guid[0], 16),
2102                 int(Guid[1], 16),
2103                 int(Guid[2], 16),
2104                 int(Guid[3][-4:-2], 16),
2105                 int(Guid[3][-2:], 16),
2106                 int(Guid[4][-12:-10], 16),
2107                 int(Guid[4][-10:-8], 16),
2108                 int(Guid[4][-8:-6], 16),
2109                 int(Guid[4][-6:-4], 16),
2110                 int(Guid[4][-4:-2], 16),
2111                 int(Guid[4][-2:], 16)
2112                 )
2113
2114 #
2115 # Pack a GUID (byte) list into a buffer and return it
2116 #
2117 def PackByteFormatGUID(Guid):
2118     return pack(PACK_PATTERN_GUID,
2119                 Guid[0],
2120                 Guid[1],
2121                 Guid[2],
2122                 Guid[3],
2123                 Guid[4],
2124                 Guid[5],
2125                 Guid[6],
2126                 Guid[7],
2127                 Guid[8],
2128                 Guid[9],
2129                 Guid[10],
2130                 )
2131
2132 ## DeepCopy dict/OrderedDict recusively
2133 #
2134 #   @param      ori_dict    a nested dict or ordereddict
2135 #
2136 #   @retval     new dict or orderdict
2137 #
2138 def CopyDict(ori_dict):
2139     dict_type = ori_dict.__class__
2140     if dict_type not in (dict,OrderedDict):
2141         return ori_dict
2142     new_dict = dict_type()
2143     for key in ori_dict:
2144         if isinstance(ori_dict[key],(dict,OrderedDict)):
2145             new_dict[key] = CopyDict(ori_dict[key])
2146         else:
2147             new_dict[key] = ori_dict[key]
2148     return new_dict
2149
2150 #
2151 # Remove the c/c++ comments: // and /* */
2152 #
2153 def RemoveCComments(ctext):
2154     return re.sub('//.*?\n|/\*.*?\*/', '\n', ctext, flags=re.S)
2155 ##
2156 #
2157 # This acts like the main() function for the script, unless it is 'import'ed into another
2158 # script.
2159 #
2160 if __name__ == '__main__':
2161     pass
2162