BaseTools/Source/Python/Common/Misc.py

   1 ## @file
   2 # Common routines used by all tools
   3 #
   4 # Copyright (c) 2007 - 2018, Intel Corporation. All rights reserved.<BR>
   5 # This program and the accompanying materials
   6 # are licensed and made available under the terms and conditions of the BSD License
   7 # which accompanies this distribution.  The full text of the license may be found at
   8 # http://opensource.org/licenses/bsd-license.php
   9 #
  10 # THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS,
  11 # WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED.
  12 #
  13
  14 ##
  15 # Import Modules
  16 #
  17 from __future__ import absolute_import
  18 import Common.LongFilePathOs as os
  19 import sys
  20 import string
  21 import threading
  22 import time
  23 import re
  24 import pickle
  25 import array
  26 import shutil
  27 from random import sample
  28 from struct import pack
  29 from UserDict import IterableUserDict
  30 from UserList import UserList
  31
  32 from Common import EdkLogger as EdkLogger
  33 from Common import GlobalData as GlobalData
  34 from .DataType import *
  35 from .BuildToolError import *
  36 from CommonDataClass.DataClass import *
  37 from .Parsing import GetSplitValueList
  38 from Common.LongFilePathSupport import OpenLongFilePath as open
  39 from Common.MultipleWorkspace import MultipleWorkspace as mws
  40 import uuid
  41 from CommonDataClass.Exceptions import BadExpression
  42 from Common.caching import cached_property
  43 import subprocess
  44 from collections import OrderedDict
  45 ## Regular expression used to find out place holders in string template
  46 gPlaceholderPattern = re.compile("\$\{([^$()\s]+)\}", re.MULTILINE | re.UNICODE)
  47
  48 ## regular expressions for map file processing
  49 startPatternGeneral = re.compile("^Start[' ']+Length[' ']+Name[' ']+Class")
  50 addressPatternGeneral = re.compile("^Address[' ']+Publics by Value[' ']+Rva\+Base")
  51 valuePatternGcc = re.compile('^([\w_\.]+) +([\da-fA-Fx]+) +([\da-fA-Fx]+)$')
  52 pcdPatternGcc = re.compile('^([\da-fA-Fx]+) +([\da-fA-Fx]+)')
  53 secReGeneral = re.compile('^([\da-fA-F]+):([\da-fA-F]+) +([\da-fA-F]+)[Hh]? +([.\w\$]+) +(\w+)', re.UNICODE)
  54
  55 StructPattern = re.compile(r'[_a-zA-Z][0-9A-Za-z_]*$')
  56
  57 ## Dictionary used to store file time stamp for quick re-access
  58 gFileTimeStampCache = {}    # {file path : file time stamp}
  59
  60 ## Dictionary used to store dependencies of files
  61 gDependencyDatabase = {}    # arch : {file path : [dependent files list]}
  62
  63 #
  64 # If a module is built more than once with different PCDs or library classes
  65 # a temporary INF file with same content is created, the temporary file is removed
  66 # when build exits.
  67 #
  68 _TempInfs = []
  69
  70 def GetVariableOffset(mapfilepath, efifilepath, varnames):
  71     """ Parse map file to get variable offset in current EFI file
  72     @param mapfilepath    Map file absolution path
  73     @param efifilepath:   EFI binary file full path
  74     @param varnames       iteratable container whose elements are variable names to be searched
  75
  76     @return List whos elements are tuple with variable name and raw offset
  77     """
  78     lines = []
  79     try:
  80         f = open(mapfilepath, 'r')
  81         lines = f.readlines()
  82         f.close()
  83     except:
  84         return None
  85
  86     if len(lines) == 0: return None
  87     firstline = lines[0].strip()
  88     if (firstline.startswith("Archive member included ") and
  89         firstline.endswith(" file (symbol)")):
  90         return _parseForGCC(lines, efifilepath, varnames)
  91     if firstline.startswith("# Path:"):
  92         return _parseForXcode(lines, efifilepath, varnames)
  93     return _parseGeneral(lines, efifilepath, varnames)
  94
  95 def _parseForXcode(lines, efifilepath, varnames):
  96     status = 0
  97     ret = []
  98     for line in lines:
  99         line = line.strip()
 100         if status == 0 and line == "# Symbols:":
 101             status = 1
 102             continue
 103         if status == 1 and len(line) != 0:
 104             for varname in varnames:
 105                 if varname in line:
 106                     # cannot pregenerate this RegEx since it uses varname from varnames.
 107                     m = re.match('^([\da-fA-FxX]+)([\s\S]*)([_]*%s)$' % varname, line)
 108                     if m is not None:
 109                         ret.append((varname, m.group(1)))
 110     return ret
 111
 112 def _parseForGCC(lines, efifilepath, varnames):
 113     """ Parse map file generated by GCC linker """
 114     status = 0
 115     sections = []
 116     varoffset = []
 117     for index, line in enumerate(lines):
 118         line = line.strip()
 119         # status machine transection
 120         if status == 0 and line == "Memory Configuration":
 121             status = 1
 122             continue
 123         elif status == 1 and line == 'Linker script and memory map':
 124             status = 2
 125             continue
 126         elif status ==2 and line == 'START GROUP':
 127             status = 3
 128             continue
 129
 130         # status handler
 131         if status == 3:
 132             m = valuePatternGcc.match(line)
 133             if m is not None:
 134                 sections.append(m.groups(0))
 135             for varname in varnames:
 136                 Str = ''
 137                 m = re.match("^.data.(%s)" % varname, line)
 138                 if m is not None:
 139                     m = re.match(".data.(%s)$" % varname, line)
 140                     if m is not None:
 141                         Str = lines[index + 1]
 142                     else:
 143                         Str = line[len(".data.%s" % varname):]
 144                     if Str:
 145                         m = pcdPatternGcc.match(Str.strip())
 146                         if m is not None:
 147                             varoffset.append((varname, int(m.groups(0)[0], 16), int(sections[-1][1], 16), sections[-1][0]))
 148
 149     if not varoffset:
 150         return []
 151     # get section information from efi file
 152     efisecs = PeImageClass(efifilepath).SectionHeaderList
 153     if efisecs is None or len(efisecs) == 0:
 154         return []
 155     #redirection
 156     redirection = 0
 157     for efisec in efisecs:
 158         for section in sections:
 159             if section[0].strip() == efisec[0].strip() and section[0].strip() == '.text':
 160                 redirection = int(section[1], 16) - efisec[1]
 161
 162     ret = []
 163     for var in varoffset:
 164         for efisec in efisecs:
 165             if var[1] >= efisec[1] and var[1] < efisec[1]+efisec[3]:
 166                 ret.append((var[0], hex(efisec[2] + var[1] - efisec[1] - redirection)))
 167     return ret
 168
 169 def _parseGeneral(lines, efifilepath, varnames):
 170     status = 0    #0 - beginning of file; 1 - PE section definition; 2 - symbol table
 171     secs  = []    # key = section name
 172     varoffset = []
 173     symRe = re.compile('^([\da-fA-F]+):([\da-fA-F]+) +([\.:\\\\\w\?@\$]+) +([\da-fA-F]+)', re.UNICODE)
 174
 175     for line in lines:
 176         line = line.strip()
 177         if startPatternGeneral.match(line):
 178             status = 1
 179             continue
 180         if addressPatternGeneral.match(line):
 181             status = 2
 182             continue
 183         if line.startswith("entry point at"):
 184             status = 3
 185             continue
 186         if status == 1 and len(line) != 0:
 187             m =  secReGeneral.match(line)
 188             assert m is not None, "Fail to parse the section in map file , line is %s" % line
 189             sec_no, sec_start, sec_length, sec_name, sec_class = m.groups(0)
 190             secs.append([int(sec_no, 16), int(sec_start, 16), int(sec_length, 16), sec_name, sec_class])
 191         if status == 2 and len(line) != 0:
 192             for varname in varnames:
 193                 m = symRe.match(line)
 194                 assert m is not None, "Fail to parse the symbol in map file, line is %s" % line
 195                 sec_no, sym_offset, sym_name, vir_addr = m.groups(0)
 196                 sec_no     = int(sec_no,     16)
 197                 sym_offset = int(sym_offset, 16)
 198                 vir_addr   = int(vir_addr,   16)
 199                 # cannot pregenerate this RegEx since it uses varname from varnames.
 200                 m2 = re.match('^[_]*(%s)' % varname, sym_name)
 201                 if m2 is not None:
 202                     # fond a binary pcd entry in map file
 203                     for sec in secs:
 204                         if sec[0] == sec_no and (sym_offset >= sec[1] and sym_offset < sec[1] + sec[2]):
 205                             varoffset.append([varname, sec[3], sym_offset, vir_addr, sec_no])
 206
 207     if not varoffset: return []
 208
 209     # get section information from efi file
 210     efisecs = PeImageClass(efifilepath).SectionHeaderList
 211     if efisecs is None or len(efisecs) == 0:
 212         return []
 213
 214     ret = []
 215     for var in varoffset:
 216         index = 0
 217         for efisec in efisecs:
 218             index = index + 1
 219             if var[1].strip() == efisec[0].strip():
 220                 ret.append((var[0], hex(efisec[2] + var[2])))
 221             elif var[4] == index:
 222                 ret.append((var[0], hex(efisec[2] + var[2])))
 223
 224     return ret
 225
 226 ## Routine to process duplicated INF
 227 #
 228 #  This function is called by following two cases:
 229 #  Case 1 in DSC:
 230 #    [components.arch]
 231 #    Pkg/module/module.inf
 232 #    Pkg/module/module.inf {
 233 #      <Defines>
 234 #        FILE_GUID = 0D1B936F-68F3-4589-AFCC-FB8B7AEBC836
 235 #    }
 236 #  Case 2 in FDF:
 237 #    INF Pkg/module/module.inf
 238 #    INF FILE_GUID = 0D1B936F-68F3-4589-AFCC-FB8B7AEBC836 Pkg/module/module.inf
 239 #
 240 #  This function copies Pkg/module/module.inf to
 241 #  Conf/.cache/0D1B936F-68F3-4589-AFCC-FB8B7AEBC836module.inf
 242 #
 243 #  @param Path     Original PathClass object
 244 #  @param BaseName New file base name
 245 #
 246 #  @retval         return the new PathClass object
 247 #
 248 def ProcessDuplicatedInf(Path, BaseName, Workspace):
 249     Filename = os.path.split(Path.File)[1]
 250     if '.' in Filename:
 251         Filename = BaseName + Path.BaseName + Filename[Filename.rfind('.'):]
 252     else:
 253         Filename = BaseName + Path.BaseName
 254
 255     #
 256     # If -N is specified on command line, cache is disabled
 257     # The directory has to be created
 258     #
 259     DbDir = os.path.split(GlobalData.gDatabasePath)[0]
 260     if not os.path.exists(DbDir):
 261         os.makedirs(DbDir)
 262     #
 263     # A temporary INF is copied to database path which must have write permission
 264     # The temporary will be removed at the end of build
 265     # In case of name conflict, the file name is
 266     # FILE_GUIDBaseName (0D1B936F-68F3-4589-AFCC-FB8B7AEBC836module.inf)
 267     #
 268     TempFullPath = os.path.join(DbDir,
 269                                 Filename)
 270     RtPath = PathClass(Path.File, Workspace)
 271     #
 272     # Modify the full path to temporary path, keep other unchanged
 273     #
 274     # To build same module more than once, the module path with FILE_GUID overridden has
 275     # the file name FILE_GUIDmodule.inf, but the relative path (self.MetaFile.File) is the real path
 276     # in DSC which is used as relative path by C files and other files in INF.
 277     # A trick was used: all module paths are PathClass instances, after the initialization
 278     # of PathClass, the PathClass.Path is overridden by the temporary INF path.
 279     #
 280     # The reason for creating a temporary INF is:
 281     # Platform.Modules which is the base to create ModuleAutoGen objects is a dictionary,
 282     # the key is the full path of INF, the value is an object to save overridden library instances, PCDs.
 283     # A different key for the same module is needed to create different output directory,
 284     # retrieve overridden PCDs, library instances.
 285     #
 286     # The BaseName is the FILE_GUID which is also the output directory name.
 287     #
 288     #
 289     RtPath.Path = TempFullPath
 290     RtPath.BaseName = BaseName
 291     #
 292     # If file exists, compare contents
 293     #
 294     if os.path.exists(TempFullPath):
 295         with open(str(Path), 'rb') as f1, open(TempFullPath, 'rb') as f2:
 296             if f1.read() == f2.read():
 297                 return RtPath
 298     _TempInfs.append(TempFullPath)
 299     shutil.copy2(str(Path), TempFullPath)
 300     return RtPath
 301
 302 ## Remove temporary created INFs whose paths were saved in _TempInfs
 303 #
 304 def ClearDuplicatedInf():
 305     while _TempInfs:
 306         File = _TempInfs.pop()
 307         if os.path.exists(File):
 308             os.remove(File)
 309
 310 ## Convert GUID string in xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx style to C structure style
 311 #
 312 #   @param      Guid    The GUID string
 313 #
 314 #   @retval     string  The GUID string in C structure style
 315 #
 316 def GuidStringToGuidStructureString(Guid):
 317     GuidList = Guid.split('-')
 318     Result = '{'
 319     for Index in range(0, 3, 1):
 320         Result = Result + '0x' + GuidList[Index] + ', '
 321     Result = Result + '{0x' + GuidList[3][0:2] + ', 0x' + GuidList[3][2:4]
 322     for Index in range(0, 12, 2):
 323         Result = Result + ', 0x' + GuidList[4][Index:Index + 2]
 324     Result += '}}'
 325     return Result
 326
 327 ## Convert GUID structure in byte array to xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
 328 #
 329 #   @param      GuidValue   The GUID value in byte array
 330 #
 331 #   @retval     string      The GUID value in xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx format
 332 #
 333 def GuidStructureByteArrayToGuidString(GuidValue):
 334     guidValueString = GuidValue.lower().replace("{", "").replace("}", "").replace(" ", "").replace(";", "")
 335     guidValueList = guidValueString.split(",")
 336     if len(guidValueList) != 16:
 337         return ''
 338         #EdkLogger.error(None, None, "Invalid GUID value string %s" % GuidValue)
 339     try:
 340         return "%02x%02x%02x%02x-%02x%02x-%02x%02x-%02x%02x-%02x%02x%02x%02x%02x%02x" % (
 341                 int(guidValueList[3], 16),
 342                 int(guidValueList[2], 16),
 343                 int(guidValueList[1], 16),
 344                 int(guidValueList[0], 16),
 345                 int(guidValueList[5], 16),
 346                 int(guidValueList[4], 16),
 347                 int(guidValueList[7], 16),
 348                 int(guidValueList[6], 16),
 349                 int(guidValueList[8], 16),
 350                 int(guidValueList[9], 16),
 351                 int(guidValueList[10], 16),
 352                 int(guidValueList[11], 16),
 353                 int(guidValueList[12], 16),
 354                 int(guidValueList[13], 16),
 355                 int(guidValueList[14], 16),
 356                 int(guidValueList[15], 16)
 357                 )
 358     except:
 359         return ''
 360
 361 ## Convert GUID string in C structure style to xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
 362 #
 363 #   @param      GuidValue   The GUID value in C structure format
 364 #
 365 #   @retval     string      The GUID value in xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx format
 366 #
 367 def GuidStructureStringToGuidString(GuidValue):
 368     if not GlobalData.gGuidCFormatPattern.match(GuidValue):
 369         return ''
 370     guidValueString = GuidValue.lower().replace("{", "").replace("}", "").replace(" ", "").replace(";", "")
 371     guidValueList = guidValueString.split(",")
 372     if len(guidValueList) != 11:
 373         return ''
 374         #EdkLogger.error(None, None, "Invalid GUID value string %s" % GuidValue)
 375     try:
 376         return "%08x-%04x-%04x-%02x%02x-%02x%02x%02x%02x%02x%02x" % (
 377                 int(guidValueList[0], 16),
 378                 int(guidValueList[1], 16),
 379                 int(guidValueList[2], 16),
 380                 int(guidValueList[3], 16),
 381                 int(guidValueList[4], 16),
 382                 int(guidValueList[5], 16),
 383                 int(guidValueList[6], 16),
 384                 int(guidValueList[7], 16),
 385                 int(guidValueList[8], 16),
 386                 int(guidValueList[9], 16),
 387                 int(guidValueList[10], 16)
 388                 )
 389     except:
 390         return ''
 391
 392 ## Convert GUID string in C structure style to xxxxxxxx_xxxx_xxxx_xxxx_xxxxxxxxxxxx
 393 #
 394 #   @param      GuidValue   The GUID value in C structure format
 395 #
 396 #   @retval     string      The GUID value in xxxxxxxx_xxxx_xxxx_xxxx_xxxxxxxxxxxx format
 397 #
 398 def GuidStructureStringToGuidValueName(GuidValue):
 399     guidValueString = GuidValue.lower().replace("{", "").replace("}", "").replace(" ", "")
 400     guidValueList = guidValueString.split(",")
 401     if len(guidValueList) != 11:
 402         EdkLogger.error(None, FORMAT_INVALID, "Invalid GUID value string [%s]" % GuidValue)
 403     return "%08x_%04x_%04x_%02x%02x_%02x%02x%02x%02x%02x%02x" % (
 404             int(guidValueList[0], 16),
 405             int(guidValueList[1], 16),
 406             int(guidValueList[2], 16),
 407             int(guidValueList[3], 16),
 408             int(guidValueList[4], 16),
 409             int(guidValueList[5], 16),
 410             int(guidValueList[6], 16),
 411             int(guidValueList[7], 16),
 412             int(guidValueList[8], 16),
 413             int(guidValueList[9], 16),
 414             int(guidValueList[10], 16)
 415             )
 416
 417 ## Create directories
 418 #
 419 #   @param      Directory   The directory name
 420 #
 421 def CreateDirectory(Directory):
 422     if Directory is None or Directory.strip() == "":
 423         return True
 424     try:
 425         if not os.access(Directory, os.F_OK):
 426             os.makedirs(Directory)
 427     except:
 428         return False
 429     return True
 430
 431 ## Remove directories, including files and sub-directories in it
 432 #
 433 #   @param      Directory   The directory name
 434 #
 435 def RemoveDirectory(Directory, Recursively=False):
 436     if Directory is None or Directory.strip() == "" or not os.path.exists(Directory):
 437         return
 438     if Recursively:
 439         CurrentDirectory = os.getcwd()
 440         os.chdir(Directory)
 441         for File in os.listdir("."):
 442             if os.path.isdir(File):
 443                 RemoveDirectory(File, Recursively)
 444             else:
 445                 os.remove(File)
 446         os.chdir(CurrentDirectory)
 447     os.rmdir(Directory)
 448
 449 ## Store content in file
 450 #
 451 #  This method is used to save file only when its content is changed. This is
 452 #  quite useful for "make" system to decide what will be re-built and what won't.
 453 #
 454 #   @param      File            The path of file
 455 #   @param      Content         The new content of the file
 456 #   @param      IsBinaryFile    The flag indicating if the file is binary file or not
 457 #
 458 #   @retval     True            If the file content is changed and the file is renewed
 459 #   @retval     False           If the file content is the same
 460 #
 461 def SaveFileOnChange(File, Content, IsBinaryFile=True):
 462     if not IsBinaryFile:
 463         Content = Content.replace("\n", os.linesep)
 464
 465     if os.path.exists(File):
 466         try:
 467             if Content == open(File, "rb").read():
 468                 return False
 469         except:
 470             EdkLogger.error(None, FILE_OPEN_FAILURE, ExtraData=File)
 471
 472     DirName = os.path.dirname(File)
 473     if not CreateDirectory(DirName):
 474         EdkLogger.error(None, FILE_CREATE_FAILURE, "Could not create directory %s" % DirName)
 475     else:
 476         if DirName == '':
 477             DirName = os.getcwd()
 478         if not os.access(DirName, os.W_OK):
 479             EdkLogger.error(None, PERMISSION_FAILURE, "Do not have write permission on directory %s" % DirName)
 480
 481     try:
 482         Fd = open(File, "wb")
 483         Fd.write(Content)
 484         Fd.close()
 485     except IOError as X:
 486         EdkLogger.error(None, FILE_CREATE_FAILURE, ExtraData='IOError %s' % X)
 487
 488     return True
 489
 490 ## Make a Python object persistent on file system
 491 #
 492 #   @param      Data    The object to be stored in file
 493 #   @param      File    The path of file to store the object
 494 #
 495 def DataDump(Data, File):
 496     Fd = None
 497     try:
 498         Fd = open(File, 'wb')
 499         pickle.dump(Data, Fd, pickle.HIGHEST_PROTOCOL)
 500     except:
 501         EdkLogger.error("", FILE_OPEN_FAILURE, ExtraData=File, RaiseError=False)
 502     finally:
 503         if Fd is not None:
 504             Fd.close()
 505
 506 ## Restore a Python object from a file
 507 #
 508 #   @param      File    The path of file stored the object
 509 #
 510 #   @retval     object  A python object
 511 #   @retval     None    If failure in file operation
 512 #
 513 def DataRestore(File):
 514     Data = None
 515     Fd = None
 516     try:
 517         Fd = open(File, 'rb')
 518         Data = pickle.load(Fd)
 519     except Exception as e:
 520         EdkLogger.verbose("Failed to load [%s]\n\t%s" % (File, str(e)))
 521         Data = None
 522     finally:
 523         if Fd is not None:
 524             Fd.close()
 525     return Data
 526
 527 ## Retrieve and cache the real path name in file system
 528 #
 529 #   @param      Root    The root directory of path relative to
 530 #
 531 #   @retval     str     The path string if the path exists
 532 #   @retval     None    If path doesn't exist
 533 #
 534 class DirCache:
 535     _CACHE_ = set()
 536     _UPPER_CACHE_ = {}
 537
 538     def __init__(self, Root):
 539         self._Root = Root
 540         for F in os.listdir(Root):
 541             self._CACHE_.add(F)
 542             self._UPPER_CACHE_[F.upper()] = F
 543
 544     # =[] operator
 545     def __getitem__(self, Path):
 546         Path = Path[len(os.path.commonprefix([Path, self._Root])):]
 547         if not Path:
 548             return self._Root
 549         if Path and Path[0] == os.path.sep:
 550             Path = Path[1:]
 551         if Path in self._CACHE_:
 552             return os.path.join(self._Root, Path)
 553         UpperPath = Path.upper()
 554         if UpperPath in self._UPPER_CACHE_:
 555             return os.path.join(self._Root, self._UPPER_CACHE_[UpperPath])
 556
 557         IndexList = []
 558         LastSepIndex = -1
 559         SepIndex = Path.find(os.path.sep)
 560         while SepIndex > -1:
 561             Parent = UpperPath[:SepIndex]
 562             if Parent not in self._UPPER_CACHE_:
 563                 break
 564             LastSepIndex = SepIndex
 565             SepIndex = Path.find(os.path.sep, LastSepIndex + 1)
 566
 567         if LastSepIndex == -1:
 568             return None
 569
 570         Cwd = os.getcwd()
 571         os.chdir(self._Root)
 572         SepIndex = LastSepIndex
 573         while SepIndex > -1:
 574             Parent = Path[:SepIndex]
 575             ParentKey = UpperPath[:SepIndex]
 576             if ParentKey not in self._UPPER_CACHE_:
 577                 os.chdir(Cwd)
 578                 return None
 579
 580             if Parent in self._CACHE_:
 581                 ParentDir = Parent
 582             else:
 583                 ParentDir = self._UPPER_CACHE_[ParentKey]
 584             for F in os.listdir(ParentDir):
 585                 Dir = os.path.join(ParentDir, F)
 586                 self._CACHE_.add(Dir)
 587                 self._UPPER_CACHE_[Dir.upper()] = Dir
 588
 589             SepIndex = Path.find(os.path.sep, SepIndex + 1)
 590
 591         os.chdir(Cwd)
 592         if Path in self._CACHE_:
 593             return os.path.join(self._Root, Path)
 594         elif UpperPath in self._UPPER_CACHE_:
 595             return os.path.join(self._Root, self._UPPER_CACHE_[UpperPath])
 596         return None
 597
 598 def RealPath(File, Dir='', OverrideDir=''):
 599     NewFile = os.path.normpath(os.path.join(Dir, File))
 600     NewFile = GlobalData.gAllFiles[NewFile]
 601     if not NewFile and OverrideDir:
 602         NewFile = os.path.normpath(os.path.join(OverrideDir, File))
 603         NewFile = GlobalData.gAllFiles[NewFile]
 604     return NewFile
 605
 606 def RealPath2(File, Dir='', OverrideDir=''):
 607     NewFile = None
 608     if OverrideDir:
 609         NewFile = GlobalData.gAllFiles[os.path.normpath(os.path.join(OverrideDir, File))]
 610         if NewFile:
 611             if OverrideDir[-1] == os.path.sep:
 612                 return NewFile[len(OverrideDir):], NewFile[0:len(OverrideDir)]
 613             else:
 614                 return NewFile[len(OverrideDir) + 1:], NewFile[0:len(OverrideDir)]
 615     if GlobalData.gAllFiles:
 616         NewFile = GlobalData.gAllFiles[os.path.normpath(os.path.join(Dir, File))]
 617     if not NewFile:
 618         NewFile = os.path.normpath(os.path.join(Dir, File))
 619         if not os.path.exists(NewFile):
 620             return None, None
 621     if NewFile:
 622         if Dir:
 623             if Dir[-1] == os.path.sep:
 624                 return NewFile[len(Dir):], NewFile[0:len(Dir)]
 625             else:
 626                 return NewFile[len(Dir) + 1:], NewFile[0:len(Dir)]
 627         else:
 628             return NewFile, ''
 629
 630     return None, None
 631
 632 ## Get GUID value from given packages
 633 #
 634 #   @param      CName           The CName of the GUID
 635 #   @param      PackageList     List of packages looking-up in
 636 #   @param      Inffile         The driver file
 637 #
 638 #   @retval     GuidValue   if the CName is found in any given package
 639 #   @retval     None        if the CName is not found in all given packages
 640 #
 641 def GuidValue(CName, PackageList, Inffile = None):
 642     for P in PackageList:
 643         GuidKeys = P.Guids.keys()
 644         if Inffile and P._PrivateGuids:
 645             if not Inffile.startswith(P.MetaFile.Dir):
 646                 GuidKeys = [x for x in P.Guids if x not in P._PrivateGuids]
 647         if CName in GuidKeys:
 648             return P.Guids[CName]
 649     return None
 650
 651 ## Get Protocol value from given packages
 652 #
 653 #   @param      CName           The CName of the GUID
 654 #   @param      PackageList     List of packages looking-up in
 655 #   @param      Inffile         The driver file
 656 #
 657 #   @retval     GuidValue   if the CName is found in any given package
 658 #   @retval     None        if the CName is not found in all given packages
 659 #
 660 def ProtocolValue(CName, PackageList, Inffile = None):
 661     for P in PackageList:
 662         ProtocolKeys = P.Protocols.keys()
 663         if Inffile and P._PrivateProtocols:
 664             if not Inffile.startswith(P.MetaFile.Dir):
 665                 ProtocolKeys = [x for x in P.Protocols if x not in P._PrivateProtocols]
 666         if CName in ProtocolKeys:
 667             return P.Protocols[CName]
 668     return None
 669
 670 ## Get PPI value from given packages
 671 #
 672 #   @param      CName           The CName of the GUID
 673 #   @param      PackageList     List of packages looking-up in
 674 #   @param      Inffile         The driver file
 675 #
 676 #   @retval     GuidValue   if the CName is found in any given package
 677 #   @retval     None        if the CName is not found in all given packages
 678 #
 679 def PpiValue(CName, PackageList, Inffile = None):
 680     for P in PackageList:
 681         PpiKeys = P.Ppis.keys()
 682         if Inffile and P._PrivatePpis:
 683             if not Inffile.startswith(P.MetaFile.Dir):
 684                 PpiKeys = [x for x in P.Ppis if x not in P._PrivatePpis]
 685         if CName in PpiKeys:
 686             return P.Ppis[CName]
 687     return None
 688
 689 ## A string template class
 690 #
 691 #  This class implements a template for string replacement. A string template
 692 #  looks like following
 693 #
 694 #       ${BEGIN} other_string ${placeholder_name} other_string ${END}
 695 #
 696 #  The string between ${BEGIN} and ${END} will be repeated as many times as the
 697 #  length of "placeholder_name", which is a list passed through a dict. The
 698 #  "placeholder_name" is the key name of the dict. The ${BEGIN} and ${END} can
 699 #  be not used and, in this case, the "placeholder_name" must not a list and it
 700 #  will just be replaced once.
 701 #
 702 class TemplateString(object):
 703     _REPEAT_START_FLAG = "BEGIN"
 704     _REPEAT_END_FLAG = "END"
 705
 706     class Section(object):
 707         _LIST_TYPES = [type([]), type(set()), type((0,))]
 708
 709         def __init__(self, TemplateSection, PlaceHolderList):
 710             self._Template = TemplateSection
 711             self._PlaceHolderList = []
 712
 713             # Split the section into sub-sections according to the position of placeholders
 714             if PlaceHolderList:
 715                 self._SubSectionList = []
 716                 SubSectionStart = 0
 717                 #
 718                 # The placeholders passed in must be in the format of
 719                 #
 720                 #   PlaceHolderName, PlaceHolderStartPoint, PlaceHolderEndPoint
 721                 #
 722                 for PlaceHolder, Start, End in PlaceHolderList:
 723                     self._SubSectionList.append(TemplateSection[SubSectionStart:Start])
 724                     self._SubSectionList.append(TemplateSection[Start:End])
 725                     self._PlaceHolderList.append(PlaceHolder)
 726                     SubSectionStart = End
 727                 if SubSectionStart < len(TemplateSection):
 728                     self._SubSectionList.append(TemplateSection[SubSectionStart:])
 729             else:
 730                 self._SubSectionList = [TemplateSection]
 731
 732         def __str__(self):
 733             return self._Template + " : " + str(self._PlaceHolderList)
 734
 735         def Instantiate(self, PlaceHolderValues):
 736             RepeatTime = -1
 737             RepeatPlaceHolders = {}
 738             NonRepeatPlaceHolders = {}
 739
 740             for PlaceHolder in self._PlaceHolderList:
 741                 if PlaceHolder not in PlaceHolderValues:
 742                     continue
 743                 Value = PlaceHolderValues[PlaceHolder]
 744                 if type(Value) in self._LIST_TYPES:
 745                     if RepeatTime < 0:
 746                         RepeatTime = len(Value)
 747                     elif RepeatTime != len(Value):
 748                         EdkLogger.error(
 749                                     "TemplateString",
 750                                     PARAMETER_INVALID,
 751                                     "${%s} has different repeat time from others!" % PlaceHolder,
 752                                     ExtraData=str(self._Template)
 753                                     )
 754                     RepeatPlaceHolders["${%s}" % PlaceHolder] = Value
 755                 else:
 756                     NonRepeatPlaceHolders["${%s}" % PlaceHolder] = Value
 757
 758             if NonRepeatPlaceHolders:
 759                 StringList = []
 760                 for S in self._SubSectionList:
 761                     if S not in NonRepeatPlaceHolders:
 762                         StringList.append(S)
 763                     else:
 764                         StringList.append(str(NonRepeatPlaceHolders[S]))
 765             else:
 766                 StringList = self._SubSectionList
 767
 768             if RepeatPlaceHolders:
 769                 TempStringList = []
 770                 for Index in range(RepeatTime):
 771                     for S in StringList:
 772                         if S not in RepeatPlaceHolders:
 773                             TempStringList.append(S)
 774                         else:
 775                             TempStringList.append(str(RepeatPlaceHolders[S][Index]))
 776                 StringList = TempStringList
 777
 778             return "".join(StringList)
 779
 780     ## Constructor
 781     def __init__(self, Template=None):
 782         self.String = []
 783         self.IsBinary = False
 784         self._Template = Template
 785         self._TemplateSectionList = self._Parse(Template)
 786
 787     ## str() operator
 788     #
 789     #   @retval     string  The string replaced
 790     #
 791     def __str__(self):
 792         return "".join(self.String)
 793
 794     ## Split the template string into fragments per the ${BEGIN} and ${END} flags
 795     #
 796     #   @retval     list    A list of TemplateString.Section objects
 797     #
 798     def _Parse(self, Template):
 799         SectionStart = 0
 800         SearchFrom = 0
 801         MatchEnd = 0
 802         PlaceHolderList = []
 803         TemplateSectionList = []
 804         while Template:
 805             MatchObj = gPlaceholderPattern.search(Template, SearchFrom)
 806             if not MatchObj:
 807                 if MatchEnd <= len(Template):
 808                     TemplateSection = TemplateString.Section(Template[SectionStart:], PlaceHolderList)
 809                     TemplateSectionList.append(TemplateSection)
 810                 break
 811
 812             MatchString = MatchObj.group(1)
 813             MatchStart = MatchObj.start()
 814             MatchEnd = MatchObj.end()
 815
 816             if MatchString == self._REPEAT_START_FLAG:
 817                 if MatchStart > SectionStart:
 818                     TemplateSection = TemplateString.Section(Template[SectionStart:MatchStart], PlaceHolderList)
 819                     TemplateSectionList.append(TemplateSection)
 820                 SectionStart = MatchEnd
 821                 PlaceHolderList = []
 822             elif MatchString == self._REPEAT_END_FLAG:
 823                 TemplateSection = TemplateString.Section(Template[SectionStart:MatchStart], PlaceHolderList)
 824                 TemplateSectionList.append(TemplateSection)
 825                 SectionStart = MatchEnd
 826                 PlaceHolderList = []
 827             else:
 828                 PlaceHolderList.append((MatchString, MatchStart - SectionStart, MatchEnd - SectionStart))
 829             SearchFrom = MatchEnd
 830         return TemplateSectionList
 831
 832     ## Replace the string template with dictionary of placeholders and append it to previous one
 833     #
 834     #   @param      AppendString    The string template to append
 835     #   @param      Dictionary      The placeholder dictionaries
 836     #
 837     def Append(self, AppendString, Dictionary=None):
 838         if Dictionary:
 839             SectionList = self._Parse(AppendString)
 840             self.String.append( "".join(S.Instantiate(Dictionary) for S in SectionList))
 841         else:
 842             if isinstance(AppendString,list):
 843                 self.String.extend(AppendString)
 844             else:
 845                 self.String.append(AppendString)
 846
 847     ## Replace the string template with dictionary of placeholders
 848     #
 849     #   @param      Dictionary      The placeholder dictionaries
 850     #
 851     #   @retval     str             The string replaced with placeholder values
 852     #
 853     def Replace(self, Dictionary=None):
 854         return "".join(S.Instantiate(Dictionary) for S in self._TemplateSectionList)
 855
 856 ## Progress indicator class
 857 #
 858 #  This class makes use of thread to print progress on console.
 859 #
 860 class Progressor:
 861     # for avoiding deadloop
 862     _StopFlag = None
 863     _ProgressThread = None
 864     _CheckInterval = 0.25
 865
 866     ## Constructor
 867     #
 868     #   @param      OpenMessage     The string printed before progress charaters
 869     #   @param      CloseMessage    The string printed after progress charaters
 870     #   @param      ProgressChar    The charater used to indicate the progress
 871     #   @param      Interval        The interval in seconds between two progress charaters
 872     #
 873     def __init__(self, OpenMessage="", CloseMessage="", ProgressChar='.', Interval=1.0):
 874         self.PromptMessage = OpenMessage
 875         self.CodaMessage = CloseMessage
 876         self.ProgressChar = ProgressChar
 877         self.Interval = Interval
 878         if Progressor._StopFlag is None:
 879             Progressor._StopFlag = threading.Event()
 880
 881     ## Start to print progress charater
 882     #
 883     #   @param      OpenMessage     The string printed before progress charaters
 884     #
 885     def Start(self, OpenMessage=None):
 886         if OpenMessage is not None:
 887             self.PromptMessage = OpenMessage
 888         Progressor._StopFlag.clear()
 889         if Progressor._ProgressThread is None:
 890             Progressor._ProgressThread = threading.Thread(target=self._ProgressThreadEntry)
 891             Progressor._ProgressThread.setDaemon(False)
 892             Progressor._ProgressThread.start()
 893
 894     ## Stop printing progress charater
 895     #
 896     #   @param      CloseMessage    The string printed after progress charaters
 897     #
 898     def Stop(self, CloseMessage=None):
 899         OriginalCodaMessage = self.CodaMessage
 900         if CloseMessage is not None:
 901             self.CodaMessage = CloseMessage
 902         self.Abort()
 903         self.CodaMessage = OriginalCodaMessage
 904
 905     ## Thread entry method
 906     def _ProgressThreadEntry(self):
 907         sys.stdout.write(self.PromptMessage + " ")
 908         sys.stdout.flush()
 909         TimeUp = 0.0
 910         while not Progressor._StopFlag.isSet():
 911             if TimeUp <= 0.0:
 912                 sys.stdout.write(self.ProgressChar)
 913                 sys.stdout.flush()
 914                 TimeUp = self.Interval
 915             time.sleep(self._CheckInterval)
 916             TimeUp -= self._CheckInterval
 917         sys.stdout.write(" " + self.CodaMessage + "\n")
 918         sys.stdout.flush()
 919
 920     ## Abort the progress display
 921     @staticmethod
 922     def Abort():
 923         if Progressor._StopFlag is not None:
 924             Progressor._StopFlag.set()
 925         if Progressor._ProgressThread is not None:
 926             Progressor._ProgressThread.join()
 927             Progressor._ProgressThread = None
 928
 929 ## A dict which can access its keys and/or values orderly
 930 #
 931 #  The class implements a new kind of dict which its keys or values can be
 932 #  accessed in the order they are added into the dict. It guarantees the order
 933 #  by making use of an internal list to keep a copy of keys.
 934 #
 935 class sdict(IterableUserDict):
 936     ## Constructor
 937     def __init__(self):
 938         IterableUserDict.__init__(self)
 939         self._key_list = []
 940
 941     ## [] operator
 942     def __setitem__(self, key, value):
 943         if key not in self._key_list:
 944             self._key_list.append(key)
 945         IterableUserDict.__setitem__(self, key, value)
 946
 947     ## del operator
 948     def __delitem__(self, key):
 949         self._key_list.remove(key)
 950         IterableUserDict.__delitem__(self, key)
 951
 952     ## used in "for k in dict" loop to ensure the correct order
 953     def __iter__(self):
 954         return self.iterkeys()
 955
 956     ## len() support
 957     def __len__(self):
 958         return len(self._key_list)
 959
 960     ## "in" test support
 961     def __contains__(self, key):
 962         return key in self._key_list
 963
 964     ## indexof support
 965     def index(self, key):
 966         return self._key_list.index(key)
 967
 968     ## insert support
 969     def insert(self, key, newkey, newvalue, order):
 970         index = self._key_list.index(key)
 971         if order == 'BEFORE':
 972             self._key_list.insert(index, newkey)
 973             IterableUserDict.__setitem__(self, newkey, newvalue)
 974         elif order == 'AFTER':
 975             self._key_list.insert(index + 1, newkey)
 976             IterableUserDict.__setitem__(self, newkey, newvalue)
 977
 978     ## append support
 979     def append(self, sdict):
 980         for key in sdict:
 981             if key not in self._key_list:
 982                 self._key_list.append(key)
 983             IterableUserDict.__setitem__(self, key, sdict[key])
 984
 985     def has_key(self, key):
 986         return key in self._key_list
 987
 988     ## Empty the dict
 989     def clear(self):
 990         self._key_list = []
 991         IterableUserDict.clear(self)
 992
 993     ## Return a copy of keys
 994     def keys(self):
 995         keys = []
 996         for key in self._key_list:
 997             keys.append(key)
 998         return keys
 999
1000     ## Return a copy of values
1001     def values(self):
1002         values = []
1003         for key in self._key_list:
1004             values.append(self[key])
1005         return values
1006
1007     ## Return a copy of (key, value) list
1008     def items(self):
1009         items = []
1010         for key in self._key_list:
1011             items.append((key, self[key]))
1012         return items
1013
1014     ## Iteration support
1015     def iteritems(self):
1016         return iter(self.items())
1017
1018     ## Keys interation support
1019     def iterkeys(self):
1020         return iter(self.keys())
1021
1022     ## Values interation support
1023     def itervalues(self):
1024         return iter(self.values())
1025
1026     ## Return value related to a key, and remove the (key, value) from the dict
1027     def pop(self, key, *dv):
1028         value = None
1029         if key in self._key_list:
1030             value = self[key]
1031             self.__delitem__(key)
1032         elif len(dv) != 0 :
1033             value = kv[0]
1034         return value
1035
1036     ## Return (key, value) pair, and remove the (key, value) from the dict
1037     def popitem(self):
1038         key = self._key_list[-1]
1039         value = self[key]
1040         self.__delitem__(key)
1041         return key, value
1042
1043     def update(self, dict=None, **kwargs):
1044         if dict is not None:
1045             for k, v in dict.items():
1046                 self[k] = v
1047         if len(kwargs):
1048             for k, v in kwargs.items():
1049                 self[k] = v
1050
1051 ## Dictionary with restricted keys
1052 #
1053 class rdict(dict):
1054     ## Constructor
1055     def __init__(self, KeyList):
1056         for Key in KeyList:
1057             dict.__setitem__(self, Key, "")
1058
1059     ## []= operator
1060     def __setitem__(self, key, value):
1061         if key not in self:
1062             EdkLogger.error("RestrictedDict", ATTRIBUTE_SET_FAILURE, "Key [%s] is not allowed" % key,
1063                             ExtraData=", ".join(dict.keys(self)))
1064         dict.__setitem__(self, key, value)
1065
1066     ## =[] operator
1067     def __getitem__(self, key):
1068         if key not in self:
1069             return ""
1070         return dict.__getitem__(self, key)
1071
1072     ## del operator
1073     def __delitem__(self, key):
1074         EdkLogger.error("RestrictedDict", ATTRIBUTE_ACCESS_DENIED, ExtraData="del")
1075
1076     ## Empty the dict
1077     def clear(self):
1078         for Key in self:
1079             self.__setitem__(Key, "")
1080
1081     ## Return value related to a key, and remove the (key, value) from the dict
1082     def pop(self, key, *dv):
1083         EdkLogger.error("RestrictedDict", ATTRIBUTE_ACCESS_DENIED, ExtraData="pop")
1084
1085     ## Return (key, value) pair, and remove the (key, value) from the dict
1086     def popitem(self):
1087         EdkLogger.error("RestrictedDict", ATTRIBUTE_ACCESS_DENIED, ExtraData="popitem")
1088
1089 ## Dictionary using prioritized list as key
1090 #
1091 class tdict:
1092     _ListType = type([])
1093     _TupleType = type(())
1094     _Wildcard = 'COMMON'
1095     _ValidWildcardList = ['COMMON', 'DEFAULT', 'ALL', TAB_STAR, 'PLATFORM']
1096
1097     def __init__(self, _Single_=False, _Level_=2):
1098         self._Level_ = _Level_
1099         self.data = {}
1100         self._Single_ = _Single_
1101
1102     # =[] operator
1103     def __getitem__(self, key):
1104         KeyType = type(key)
1105         RestKeys = None
1106         if KeyType == self._ListType or KeyType == self._TupleType:
1107             FirstKey = key[0]
1108             if len(key) > 1:
1109                 RestKeys = key[1:]
1110             elif self._Level_ > 1:
1111                 RestKeys = [self._Wildcard for i in range(0, self._Level_ - 1)]
1112         else:
1113             FirstKey = key
1114             if self._Level_ > 1:
1115                 RestKeys = [self._Wildcard for i in range(0, self._Level_ - 1)]
1116
1117         if FirstKey is None or str(FirstKey).upper() in self._ValidWildcardList:
1118             FirstKey = self._Wildcard
1119
1120         if self._Single_:
1121             return self._GetSingleValue(FirstKey, RestKeys)
1122         else:
1123             return self._GetAllValues(FirstKey, RestKeys)
1124
1125     def _GetSingleValue(self, FirstKey, RestKeys):
1126         Value = None
1127         #print "%s-%s" % (FirstKey, self._Level_) ,
1128         if self._Level_ > 1:
1129             if FirstKey == self._Wildcard:
1130                 if FirstKey in self.data:
1131                     Value = self.data[FirstKey][RestKeys]
1132                 if Value is None:
1133                     for Key in self.data:
1134                         Value = self.data[Key][RestKeys]
1135                         if Value is not None: break
1136             else:
1137                 if FirstKey in self.data:
1138                     Value = self.data[FirstKey][RestKeys]
1139                 if Value is None and self._Wildcard in self.data:
1140                     #print "Value=None"
1141                     Value = self.data[self._Wildcard][RestKeys]
1142         else:
1143             if FirstKey == self._Wildcard:
1144                 if FirstKey in self.data:
1145                     Value = self.data[FirstKey]
1146                 if Value is None:
1147                     for Key in self.data:
1148                         Value = self.data[Key]
1149                         if Value is not None: break
1150             else:
1151                 if FirstKey in self.data:
1152                     Value = self.data[FirstKey]
1153                 elif self._Wildcard in self.data:
1154                     Value = self.data[self._Wildcard]
1155         return Value
1156
1157     def _GetAllValues(self, FirstKey, RestKeys):
1158         Value = []
1159         if self._Level_ > 1:
1160             if FirstKey == self._Wildcard:
1161                 for Key in self.data:
1162                     Value += self.data[Key][RestKeys]
1163             else:
1164                 if FirstKey in self.data:
1165                     Value += self.data[FirstKey][RestKeys]
1166                 if self._Wildcard in self.data:
1167                     Value += self.data[self._Wildcard][RestKeys]
1168         else:
1169             if FirstKey == self._Wildcard:
1170                 for Key in self.data:
1171                     Value.append(self.data[Key])
1172             else:
1173                 if FirstKey in self.data:
1174                     Value.append(self.data[FirstKey])
1175                 if self._Wildcard in self.data:
1176                     Value.append(self.data[self._Wildcard])
1177         return Value
1178
1179     ## []= operator
1180     def __setitem__(self, key, value):
1181         KeyType = type(key)
1182         RestKeys = None
1183         if KeyType == self._ListType or KeyType == self._TupleType:
1184             FirstKey = key[0]
1185             if len(key) > 1:
1186                 RestKeys = key[1:]
1187             else:
1188                 RestKeys = [self._Wildcard for i in range(0, self._Level_ - 1)]
1189         else:
1190             FirstKey = key
1191             if self._Level_ > 1:
1192                 RestKeys = [self._Wildcard for i in range(0, self._Level_ - 1)]
1193
1194         if FirstKey in self._ValidWildcardList:
1195             FirstKey = self._Wildcard
1196
1197         if FirstKey not in self.data and self._Level_ > 0:
1198             self.data[FirstKey] = tdict(self._Single_, self._Level_ - 1)
1199
1200         if self._Level_ > 1:
1201             self.data[FirstKey][RestKeys] = value
1202         else:
1203             self.data[FirstKey] = value
1204
1205     def SetGreedyMode(self):
1206         self._Single_ = False
1207         if self._Level_ > 1:
1208             for Key in self.data:
1209                 self.data[Key].SetGreedyMode()
1210
1211     def SetSingleMode(self):
1212         self._Single_ = True
1213         if self._Level_ > 1:
1214             for Key in self.data:
1215                 self.data[Key].SetSingleMode()
1216
1217     def GetKeys(self, KeyIndex=0):
1218         assert KeyIndex >= 0
1219         if KeyIndex == 0:
1220             return set(self.data.keys())
1221         else:
1222             keys = set()
1223             for Key in self.data:
1224                 keys |= self.data[Key].GetKeys(KeyIndex - 1)
1225             return keys
1226
1227 def IsFieldValueAnArray (Value):
1228     Value = Value.strip()
1229     if Value.startswith(TAB_GUID) and Value.endswith(')'):
1230         return True
1231     if Value.startswith('L"') and Value.endswith('"')  and len(list(Value[2:-1])) > 1:
1232         return True
1233     if Value[0] == '"' and Value[-1] == '"' and len(list(Value[1:-1])) > 1:
1234         return True
1235     if Value[0] == '{' and Value[-1] == '}':
1236         return True
1237     if Value.startswith("L'") and Value.endswith("'") and len(list(Value[2:-1])) > 1:
1238         return True
1239     if Value[0] == "'" and Value[-1] == "'" and len(list(Value[1:-1])) > 1:
1240         return True
1241     return False
1242
1243 def AnalyzePcdExpression(Setting):
1244     RanStr = ''.join(sample(string.ascii_letters + string.digits, 8))
1245     Setting = Setting.replace('\\\\', RanStr).strip()
1246     # There might be escaped quote in a string: \", \\\" , \', \\\'
1247     Data = Setting
1248     # There might be '|' in string and in ( ... | ... ), replace it with '-'
1249     NewStr = ''
1250     InSingleQuoteStr = False
1251     InDoubleQuoteStr = False
1252     Pair = 0
1253     for Index, ch in enumerate(Data):
1254         if ch == '"' and not InSingleQuoteStr:
1255             if Data[Index - 1] != '\\':
1256                 InDoubleQuoteStr = not InDoubleQuoteStr
1257         elif ch == "'" and not InDoubleQuoteStr:
1258             if Data[Index - 1] != '\\':
1259                 InSingleQuoteStr = not InSingleQuoteStr
1260         elif ch == '(' and not (InSingleQuoteStr or InDoubleQuoteStr):
1261             Pair += 1
1262         elif ch == ')' and not (InSingleQuoteStr or InDoubleQuoteStr):
1263             Pair -= 1
1264
1265         if (Pair > 0 or InSingleQuoteStr or InDoubleQuoteStr) and ch == TAB_VALUE_SPLIT:
1266             NewStr += '-'
1267         else:
1268             NewStr += ch
1269     FieldList = []
1270     StartPos = 0
1271     while True:
1272         Pos = NewStr.find(TAB_VALUE_SPLIT, StartPos)
1273         if Pos < 0:
1274             FieldList.append(Setting[StartPos:].strip())
1275             break
1276         FieldList.append(Setting[StartPos:Pos].strip())
1277         StartPos = Pos + 1
1278     for i, ch in enumerate(FieldList):
1279         if RanStr in ch:
1280             FieldList[i] = ch.replace(RanStr,'\\\\')
1281     return FieldList
1282
1283 def ParseDevPathValue (Value):
1284     if '\\' in Value:
1285         Value.replace('\\', '/').replace(' ', '')
1286
1287     Cmd = 'DevicePath ' + '"' + Value + '"'
1288     try:
1289         p = subprocess.Popen(Cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
1290         out, err = p.communicate()
1291     except Exception as X:
1292         raise BadExpression("DevicePath: %s" % (str(X)) )
1293     finally:
1294         subprocess._cleanup()
1295         p.stdout.close()
1296         p.stderr.close()
1297     if err:
1298         raise BadExpression("DevicePath: %s" % str(err))
1299     Size = len(out.split())
1300     out = ','.join(out.split())
1301     return '{' + out + '}', Size
1302
1303 def ParseFieldValue (Value):
1304     if "{CODE(" in Value:
1305         return Value, len(Value.split(","))
1306     if isinstance(Value, type(0)):
1307         return Value, (Value.bit_length() + 7) / 8
1308     if not isinstance(Value, type('')):
1309         raise BadExpression('Type %s is %s' %(Value, type(Value)))
1310     Value = Value.strip()
1311     if Value.startswith(TAB_UINT8) and Value.endswith(')'):
1312         Value, Size = ParseFieldValue(Value.split('(', 1)[1][:-1])
1313         if Size > 1:
1314             raise BadExpression('Value (%s) Size larger than %d' %(Value, Size))
1315         return Value, 1
1316     if Value.startswith(TAB_UINT16) and Value.endswith(')'):
1317         Value, Size = ParseFieldValue(Value.split('(', 1)[1][:-1])
1318         if Size > 2:
1319             raise BadExpression('Value (%s) Size larger than %d' %(Value, Size))
1320         return Value, 2
1321     if Value.startswith(TAB_UINT32) and Value.endswith(')'):
1322         Value, Size = ParseFieldValue(Value.split('(', 1)[1][:-1])
1323         if Size > 4:
1324             raise BadExpression('Value (%s) Size larger than %d' %(Value, Size))
1325         return Value, 4
1326     if Value.startswith(TAB_UINT64) and Value.endswith(')'):
1327         Value, Size = ParseFieldValue(Value.split('(', 1)[1][:-1])
1328         if Size > 8:
1329             raise BadExpression('Value (%s) Size larger than %d' % (Value, Size))
1330         return Value, 8
1331     if Value.startswith(TAB_GUID) and Value.endswith(')'):
1332         Value = Value.split('(', 1)[1][:-1].strip()
1333         if Value[0] == '{' and Value[-1] == '}':
1334             TmpValue = GuidStructureStringToGuidString(Value)
1335             if not TmpValue:
1336                 raise BadExpression("Invalid GUID value string %s" % Value)
1337             Value = TmpValue
1338         if Value[0] == '"' and Value[-1] == '"':
1339             Value = Value[1:-1]
1340         try:
1341             Value = "'" + uuid.UUID(Value).get_bytes_le() + "'"
1342         except ValueError as Message:
1343             raise BadExpression(Message)
1344         Value, Size = ParseFieldValue(Value)
1345         return Value, 16
1346     if Value.startswith('L"') and Value.endswith('"'):
1347         # Unicode String
1348         # translate escape character
1349         Value = Value[1:]
1350         try:
1351             Value = eval(Value)
1352         except:
1353             Value = Value[1:-1]
1354         List = list(Value)
1355         List.reverse()
1356         Value = 0
1357         for Char in List:
1358             Value = (Value << 16) | ord(Char)
1359         return Value, (len(List) + 1) * 2
1360     if Value.startswith('"') and Value.endswith('"'):
1361         # ASCII String
1362         # translate escape character
1363         try:
1364             Value = eval(Value)
1365         except:
1366             Value = Value[1:-1]
1367         List = list(Value)
1368         List.reverse()
1369         Value = 0
1370         for Char in List:
1371             Value = (Value << 8) | ord(Char)
1372         return Value, len(List) + 1
1373     if Value.startswith("L'") and Value.endswith("'"):
1374         # Unicode Character Constant
1375         # translate escape character
1376         Value = Value[1:]
1377         try:
1378             Value = eval(Value)
1379         except:
1380             Value = Value[1:-1]
1381         List = list(Value)
1382         if len(List) == 0:
1383             raise BadExpression('Length %s is %s' % (Value, len(List)))
1384         List.reverse()
1385         Value = 0
1386         for Char in List:
1387             Value = (Value << 16) | ord(Char)
1388         return Value, len(List) * 2
1389     if Value.startswith("'") and Value.endswith("'"):
1390         # Character constant
1391         # translate escape character
1392         try:
1393             Value = eval(Value)
1394         except:
1395             Value = Value[1:-1]
1396         List = list(Value)
1397         if len(List) == 0:
1398             raise BadExpression('Length %s is %s' % (Value, len(List)))
1399         List.reverse()
1400         Value = 0
1401         for Char in List:
1402             Value = (Value << 8) | ord(Char)
1403         return Value, len(List)
1404     if Value.startswith('{') and Value.endswith('}'):
1405         # Byte array
1406         Value = Value[1:-1]
1407         List = [Item.strip() for Item in Value.split(',')]
1408         List.reverse()
1409         Value = 0
1410         RetSize = 0
1411         for Item in List:
1412             ItemValue, Size = ParseFieldValue(Item)
1413             RetSize += Size
1414             for I in range(Size):
1415                 Value = (Value << 8) | ((ItemValue >> 8 * I) & 0xff)
1416         return Value, RetSize
1417     if Value.startswith('DEVICE_PATH(') and Value.endswith(')'):
1418         Value = Value.replace("DEVICE_PATH(", '').rstrip(')')
1419         Value = Value.strip().strip('"')
1420         return ParseDevPathValue(Value)
1421     if Value.lower().startswith('0x'):
1422         try:
1423             Value = int(Value, 16)
1424         except:
1425             raise BadExpression("invalid hex value: %s" % Value)
1426         if Value == 0:
1427             return 0, 1
1428         return Value, (Value.bit_length() + 7) / 8
1429     if Value[0].isdigit():
1430         Value = int(Value, 10)
1431         if Value == 0:
1432             return 0, 1
1433         return Value, (Value.bit_length() + 7) / 8
1434     if Value.lower() == 'true':
1435         return 1, 1
1436     if Value.lower() == 'false':
1437         return 0, 1
1438     return Value, 1
1439
1440 ## AnalyzeDscPcd
1441 #
1442 #  Analyze DSC PCD value, since there is no data type info in DSC
1443 #  This function is used to match functions (AnalyzePcdData) used for retrieving PCD value from database
1444 #  1. Feature flag: TokenSpace.PcdCName|PcdValue
1445 #  2. Fix and Patch:TokenSpace.PcdCName|PcdValue[|VOID*[|MaxSize]]
1446 #  3. Dynamic default:
1447 #     TokenSpace.PcdCName|PcdValue[|VOID*[|MaxSize]]
1448 #     TokenSpace.PcdCName|PcdValue
1449 #  4. Dynamic VPD:
1450 #     TokenSpace.PcdCName|VpdOffset[|VpdValue]
1451 #     TokenSpace.PcdCName|VpdOffset[|MaxSize[|VpdValue]]
1452 #  5. Dynamic HII:
1453 #     TokenSpace.PcdCName|HiiString|VariableGuid|VariableOffset[|HiiValue]
1454 #  PCD value needs to be located in such kind of string, and the PCD value might be an expression in which
1455 #    there might have "|" operator, also in string value.
1456 #
1457 #  @param Setting: String contain information described above with "TokenSpace.PcdCName|" stripped
1458 #  @param PcdType: PCD type: feature, fixed, dynamic default VPD HII
1459 #  @param DataType: The datum type of PCD: VOID*, UNIT, BOOL
1460 #  @retval:
1461 #    ValueList: A List contain fields described above
1462 #    IsValid:   True if conforming EBNF, otherwise False
1463 #    Index:     The index where PcdValue is in ValueList
1464 #
1465 def AnalyzeDscPcd(Setting, PcdType, DataType=''):
1466     FieldList = AnalyzePcdExpression(Setting)
1467
1468     IsValid = True
1469     if PcdType in (MODEL_PCD_FIXED_AT_BUILD, MODEL_PCD_PATCHABLE_IN_MODULE, MODEL_PCD_DYNAMIC_DEFAULT, MODEL_PCD_DYNAMIC_EX_DEFAULT):
1470         Value = FieldList[0]
1471         Size = ''
1472         if len(FieldList) > 1 and FieldList[1]:
1473             DataType = FieldList[1]
1474             if FieldList[1] != TAB_VOID and StructPattern.match(FieldList[1]) is None:
1475                 IsValid = False
1476         if len(FieldList) > 2:
1477             Size = FieldList[2]
1478         if IsValid:
1479             if DataType == "":
1480                 IsValid = (len(FieldList) <= 1)
1481             else:
1482                 IsValid = (len(FieldList) <= 3)
1483
1484         if Size:
1485             try:
1486                 int(Size, 16) if Size.upper().startswith("0X") else int(Size)
1487             except:
1488                 IsValid = False
1489                 Size = -1
1490         return [str(Value), DataType, str(Size)], IsValid, 0
1491     elif PcdType == MODEL_PCD_FEATURE_FLAG:
1492         Value = FieldList[0]
1493         Size = ''
1494         IsValid = (len(FieldList) <= 1)
1495         return [Value, DataType, str(Size)], IsValid, 0
1496     elif PcdType in (MODEL_PCD_DYNAMIC_VPD, MODEL_PCD_DYNAMIC_EX_VPD):
1497         VpdOffset = FieldList[0]
1498         Value = Size = ''
1499         if not DataType == TAB_VOID:
1500             if len(FieldList) > 1:
1501                 Value = FieldList[1]
1502         else:
1503             if len(FieldList) > 1:
1504                 Size = FieldList[1]
1505             if len(FieldList) > 2:
1506                 Value = FieldList[2]
1507         if DataType == "":
1508             IsValid = (len(FieldList) <= 1)
1509         else:
1510             IsValid = (len(FieldList) <= 3)
1511         if Size:
1512             try:
1513                 int(Size, 16) if Size.upper().startswith("0X") else int(Size)
1514             except:
1515                 IsValid = False
1516                 Size = -1
1517         return [VpdOffset, str(Size), Value], IsValid, 2
1518     elif PcdType in (MODEL_PCD_DYNAMIC_HII, MODEL_PCD_DYNAMIC_EX_HII):
1519         IsValid = (3 <= len(FieldList) <= 5)
1520         HiiString = FieldList[0]
1521         Guid = Offset = Value = Attribute = ''
1522         if len(FieldList) > 1:
1523             Guid = FieldList[1]
1524         if len(FieldList) > 2:
1525             Offset = FieldList[2]
1526         if len(FieldList) > 3:
1527             Value = FieldList[3]
1528             if not Value:
1529                 IsValid = False
1530         if len(FieldList) > 4:
1531             Attribute = FieldList[4]
1532         return [HiiString, Guid, Offset, Value, Attribute], IsValid, 3
1533     return [], False, 0
1534
1535 ## AnalyzePcdData
1536 #
1537 #  Analyze the pcd Value, Datum type and TokenNumber.
1538 #  Used to avoid split issue while the value string contain "|" character
1539 #
1540 #  @param[in] Setting:  A String contain value/datum type/token number information;
1541 #
1542 #  @retval   ValueList: A List contain value, datum type and toke number.
1543 #
1544 def AnalyzePcdData(Setting):
1545     ValueList = ['', '', '']
1546
1547     ValueRe = re.compile(r'^\s*L?\".*\|.*\"')
1548     PtrValue = ValueRe.findall(Setting)
1549
1550     ValueUpdateFlag = False
1551
1552     if len(PtrValue) >= 1:
1553         Setting = re.sub(ValueRe, '', Setting)
1554         ValueUpdateFlag = True
1555
1556     TokenList = Setting.split(TAB_VALUE_SPLIT)
1557     ValueList[0:len(TokenList)] = TokenList
1558
1559     if ValueUpdateFlag:
1560         ValueList[0] = PtrValue[0]
1561
1562     return ValueList
1563
1564 ## check format of PCD value against its the datum type
1565 #
1566 # For PCD value setting
1567 #
1568 def CheckPcdDatum(Type, Value):
1569     if Type == TAB_VOID:
1570         ValueRe = re.compile(r'\s*L?\".*\"\s*$')
1571         if not (((Value.startswith('L"') or Value.startswith('"')) and Value.endswith('"'))
1572                 or (Value.startswith('{') and Value.endswith('}')) or (Value.startswith("L'") or Value.startswith("'") and Value.endswith("'"))
1573                ):
1574             return False, "Invalid value [%s] of type [%s]; must be in the form of {...} for array"\
1575                           ", \"...\" or \'...\' for string, L\"...\" or L\'...\' for unicode string" % (Value, Type)
1576         elif ValueRe.match(Value):
1577             # Check the chars in UnicodeString or CString is printable
1578             if Value.startswith("L"):
1579                 Value = Value[2:-1]
1580             else:
1581                 Value = Value[1:-1]
1582             Printset = set(string.printable)
1583             Printset.remove(TAB_PRINTCHAR_VT)
1584             Printset.add(TAB_PRINTCHAR_BS)
1585             Printset.add(TAB_PRINTCHAR_NUL)
1586             if not set(Value).issubset(Printset):
1587                 PrintList = sorted(Printset)
1588                 return False, "Invalid PCD string value of type [%s]; must be printable chars %s." % (Type, PrintList)
1589     elif Type == 'BOOLEAN':
1590         if Value not in ['TRUE', 'True', 'true', '0x1', '0x01', '1', 'FALSE', 'False', 'false', '0x0', '0x00', '0']:
1591             return False, "Invalid value [%s] of type [%s]; must be one of TRUE, True, true, 0x1, 0x01, 1"\
1592                           ", FALSE, False, false, 0x0, 0x00, 0" % (Value, Type)
1593     elif Type in [TAB_UINT8, TAB_UINT16, TAB_UINT32, TAB_UINT64]:
1594         if Value and int(Value, 0) < 0:
1595             return False, "PCD can't be set to negative value[%s] for datum type [%s]" % (Value, Type)
1596         try:
1597             Value = long(Value, 0)
1598             if Value > MAX_VAL_TYPE[Type]:
1599                 return False, "Too large PCD value[%s] for datum type [%s]" % (Value, Type)
1600         except:
1601             return False, "Invalid value [%s] of type [%s];"\
1602                           " must be a hexadecimal, decimal or octal in C language format." % (Value, Type)
1603     else:
1604         return True, "StructurePcd"
1605
1606     return True, ""
1607
1608 ## Split command line option string to list
1609 #
1610 # subprocess.Popen needs the args to be a sequence. Otherwise there's problem
1611 # in non-windows platform to launch command
1612 #
1613 def SplitOption(OptionString):
1614     OptionList = []
1615     LastChar = " "
1616     OptionStart = 0
1617     QuotationMark = ""
1618     for Index in range(0, len(OptionString)):
1619         CurrentChar = OptionString[Index]
1620         if CurrentChar in ['"', "'"]:
1621             if QuotationMark == CurrentChar:
1622                 QuotationMark = ""
1623             elif QuotationMark == "":
1624                 QuotationMark = CurrentChar
1625             continue
1626         elif QuotationMark:
1627             continue
1628
1629         if CurrentChar in ["/", "-"] and LastChar in [" ", "\t", "\r", "\n"]:
1630             if Index > OptionStart:
1631                 OptionList.append(OptionString[OptionStart:Index - 1])
1632             OptionStart = Index
1633         LastChar = CurrentChar
1634     OptionList.append(OptionString[OptionStart:])
1635     return OptionList
1636
1637 def CommonPath(PathList):
1638     P1 = min(PathList).split(os.path.sep)
1639     P2 = max(PathList).split(os.path.sep)
1640     for Index in xrange(min(len(P1), len(P2))):
1641         if P1[Index] != P2[Index]:
1642             return os.path.sep.join(P1[:Index])
1643     return os.path.sep.join(P1)
1644
1645 #
1646 # Convert string to C format array
1647 #
1648 def ConvertStringToByteArray(Value):
1649     Value = Value.strip()
1650     if not Value:
1651         return None
1652     if Value[0] == '{':
1653         if not Value.endswith('}'):
1654             return None
1655         Value = Value.replace(' ', '').replace('{', '').replace('}', '')
1656         ValFields = Value.split(',')
1657         try:
1658             for Index in range(len(ValFields)):
1659                 ValFields[Index] = str(int(ValFields[Index], 0))
1660         except ValueError:
1661             return None
1662         Value = '{' + ','.join(ValFields) + '}'
1663         return Value
1664
1665     Unicode = False
1666     if Value.startswith('L"'):
1667         if not Value.endswith('"'):
1668             return None
1669         Value = Value[1:]
1670         Unicode = True
1671     elif not Value.startswith('"') or not Value.endswith('"'):
1672         return None
1673
1674     Value = eval(Value)         # translate escape character
1675     NewValue = '{'
1676     for Index in range(0, len(Value)):
1677         if Unicode:
1678             NewValue = NewValue + str(ord(Value[Index]) % 0x10000) + ','
1679         else:
1680             NewValue = NewValue + str(ord(Value[Index]) % 0x100) + ','
1681     Value = NewValue + '0}'
1682     return Value
1683
1684 class PathClass(object):
1685     def __init__(self, File='', Root='', AlterRoot='', Type='', IsBinary=False,
1686                  Arch='COMMON', ToolChainFamily='', Target='', TagName='', ToolCode=''):
1687         self.Arch = Arch
1688         self.File = str(File)
1689         if os.path.isabs(self.File):
1690             self.Root = ''
1691             self.AlterRoot = ''
1692         else:
1693             self.Root = str(Root)
1694             self.AlterRoot = str(AlterRoot)
1695
1696         # Remove any '.' and '..' in path
1697         if self.Root:
1698             self.Root = mws.getWs(self.Root, self.File)
1699             self.Path = os.path.normpath(os.path.join(self.Root, self.File))
1700             self.Root = os.path.normpath(CommonPath([self.Root, self.Path]))
1701             # eliminate the side-effect of 'C:'
1702             if self.Root[-1] == ':':
1703                 self.Root += os.path.sep
1704             # file path should not start with path separator
1705             if self.Root[-1] == os.path.sep:
1706                 self.File = self.Path[len(self.Root):]
1707             else:
1708                 self.File = self.Path[len(self.Root) + 1:]
1709         else:
1710             self.Path = os.path.normpath(self.File)
1711
1712         self.SubDir, self.Name = os.path.split(self.File)
1713         self.BaseName, self.Ext = os.path.splitext(self.Name)
1714
1715         if self.Root:
1716             if self.SubDir:
1717                 self.Dir = os.path.join(self.Root, self.SubDir)
1718             else:
1719                 self.Dir = self.Root
1720         else:
1721             self.Dir = self.SubDir
1722
1723         if IsBinary:
1724             self.Type = Type
1725         else:
1726             self.Type = self.Ext.lower()
1727
1728         self.IsBinary = IsBinary
1729         self.Target = Target
1730         self.TagName = TagName
1731         self.ToolCode = ToolCode
1732         self.ToolChainFamily = ToolChainFamily
1733
1734     ## Convert the object of this class to a string
1735     #
1736     #  Convert member Path of the class to a string
1737     #
1738     #  @retval string Formatted String
1739     #
1740     def __str__(self):
1741         return self.Path
1742
1743     ## Override __eq__ function
1744     #
1745     # Check whether PathClass are the same
1746     #
1747     # @retval False The two PathClass are different
1748     # @retval True  The two PathClass are the same
1749     #
1750     def __eq__(self, Other):
1751         return self.Path == str(Other)
1752
1753     ## Override __cmp__ function
1754     #
1755     # Customize the comparsion operation of two PathClass
1756     #
1757     # @retval 0     The two PathClass are different
1758     # @retval -1    The first PathClass is less than the second PathClass
1759     # @retval 1     The first PathClass is Bigger than the second PathClass
1760     def __cmp__(self, Other):
1761         OtherKey = str(Other)
1762
1763         SelfKey = self.Path
1764         if SelfKey == OtherKey:
1765             return 0
1766         elif SelfKey > OtherKey:
1767             return 1
1768         else:
1769             return -1
1770
1771     ## Override __hash__ function
1772     #
1773     # Use Path as key in hash table
1774     #
1775     # @retval string Key for hash table
1776     #
1777     def __hash__(self):
1778         return hash(self.Path)
1779
1780     @cached_property
1781     def Key(self):
1782         return self.Path.upper()
1783
1784     @property
1785     def TimeStamp(self):
1786         return os.stat(self.Path)[8]
1787
1788     def Validate(self, Type='', CaseSensitive=True):
1789         if GlobalData.gCaseInsensitive:
1790             CaseSensitive = False
1791         if Type and Type.lower() != self.Type:
1792             return FILE_TYPE_MISMATCH, '%s (expect %s but got %s)' % (self.File, Type, self.Type)
1793
1794         RealFile, RealRoot = RealPath2(self.File, self.Root, self.AlterRoot)
1795         if not RealRoot and not RealFile:
1796             RealFile = self.File
1797             if self.AlterRoot:
1798                 RealFile = os.path.join(self.AlterRoot, self.File)
1799             elif self.Root:
1800                 RealFile = os.path.join(self.Root, self.File)
1801             if len (mws.getPkgPath()) == 0:
1802                 return FILE_NOT_FOUND, os.path.join(self.AlterRoot, RealFile)
1803             else:
1804                 return FILE_NOT_FOUND, "%s is not found in packages path:\n\t%s" % (self.File, '\n\t'.join(mws.getPkgPath()))
1805
1806         ErrorCode = 0
1807         ErrorInfo = ''
1808         if RealRoot != self.Root or RealFile != self.File:
1809             if CaseSensitive and (RealFile != self.File or (RealRoot != self.Root and RealRoot != self.AlterRoot)):
1810                 ErrorCode = FILE_CASE_MISMATCH
1811                 ErrorInfo = self.File + '\n\t' + RealFile + " [in file system]"
1812
1813             self.SubDir, self.Name = os.path.split(RealFile)
1814             self.BaseName, self.Ext = os.path.splitext(self.Name)
1815             if self.SubDir:
1816                 self.Dir = os.path.join(RealRoot, self.SubDir)
1817             else:
1818                 self.Dir = RealRoot
1819             self.File = RealFile
1820             self.Root = RealRoot
1821             self.Path = os.path.join(RealRoot, RealFile)
1822         return ErrorCode, ErrorInfo
1823
1824 ## Parse PE image to get the required PE informaion.
1825 #
1826 class PeImageClass():
1827     ## Constructor
1828     #
1829     #   @param  File FilePath of PeImage
1830     #
1831     def __init__(self, PeFile):
1832         self.FileName   = PeFile
1833         self.IsValid    = False
1834         self.Size       = 0
1835         self.EntryPoint = 0
1836         self.SectionAlignment  = 0
1837         self.SectionHeaderList = []
1838         self.ErrorInfo = ''
1839         try:
1840             PeObject = open(PeFile, 'rb')
1841         except:
1842             self.ErrorInfo = self.FileName + ' can not be found\n'
1843             return
1844         # Read DOS header
1845         ByteArray = array.array('B')
1846         ByteArray.fromfile(PeObject, 0x3E)
1847         ByteList = ByteArray.tolist()
1848         # DOS signature should be 'MZ'
1849         if self._ByteListToStr (ByteList[0x0:0x2]) != 'MZ':
1850             self.ErrorInfo = self.FileName + ' has no valid DOS signature MZ'
1851             return
1852
1853         # Read 4 byte PE Signature
1854         PeOffset = self._ByteListToInt(ByteList[0x3C:0x3E])
1855         PeObject.seek(PeOffset)
1856         ByteArray = array.array('B')
1857         ByteArray.fromfile(PeObject, 4)
1858         # PE signature should be 'PE\0\0'
1859         if ByteArray.tostring() != 'PE\0\0':
1860             self.ErrorInfo = self.FileName + ' has no valid PE signature PE00'
1861             return
1862
1863         # Read PE file header
1864         ByteArray = array.array('B')
1865         ByteArray.fromfile(PeObject, 0x14)
1866         ByteList = ByteArray.tolist()
1867         SecNumber = self._ByteListToInt(ByteList[0x2:0x4])
1868         if SecNumber == 0:
1869             self.ErrorInfo = self.FileName + ' has no section header'
1870             return
1871
1872         # Read PE optional header
1873         OptionalHeaderSize = self._ByteListToInt(ByteArray[0x10:0x12])
1874         ByteArray = array.array('B')
1875         ByteArray.fromfile(PeObject, OptionalHeaderSize)
1876         ByteList = ByteArray.tolist()
1877         self.EntryPoint       = self._ByteListToInt(ByteList[0x10:0x14])
1878         self.SectionAlignment = self._ByteListToInt(ByteList[0x20:0x24])
1879         self.Size             = self._ByteListToInt(ByteList[0x38:0x3C])
1880
1881         # Read each Section Header
1882         for Index in range(SecNumber):
1883             ByteArray = array.array('B')
1884             ByteArray.fromfile(PeObject, 0x28)
1885             ByteList = ByteArray.tolist()
1886             SecName  = self._ByteListToStr(ByteList[0:8])
1887             SecVirtualSize = self._ByteListToInt(ByteList[8:12])
1888             SecRawAddress  = self._ByteListToInt(ByteList[20:24])
1889             SecVirtualAddress = self._ByteListToInt(ByteList[12:16])
1890             self.SectionHeaderList.append((SecName, SecVirtualAddress, SecRawAddress, SecVirtualSize))
1891         self.IsValid = True
1892         PeObject.close()
1893
1894     def _ByteListToStr(self, ByteList):
1895         String = ''
1896         for index in range(len(ByteList)):
1897             if ByteList[index] == 0:
1898                 break
1899             String += chr(ByteList[index])
1900         return String
1901
1902     def _ByteListToInt(self, ByteList):
1903         Value = 0
1904         for index in range(len(ByteList) - 1, -1, -1):
1905             Value = (Value << 8) | int(ByteList[index])
1906         return Value
1907
1908 class DefaultStore():
1909     def __init__(self, DefaultStores ):
1910
1911         self.DefaultStores = DefaultStores
1912     def DefaultStoreID(self, DefaultStoreName):
1913         for key, value in self.DefaultStores.items():
1914             if value == DefaultStoreName:
1915                 return key
1916         return None
1917     def GetDefaultDefault(self):
1918         if not self.DefaultStores or "0" in self.DefaultStores:
1919             return "0", TAB_DEFAULT_STORES_DEFAULT
1920         else:
1921             minvalue = min(int(value_str) for value_str in self.DefaultStores)
1922             return (str(minvalue), self.DefaultStores[str(minvalue)])
1923     def GetMin(self, DefaultSIdList):
1924         if not DefaultSIdList:
1925             return TAB_DEFAULT_STORES_DEFAULT
1926         storeidset = {storeid for storeid, storename in self.DefaultStores.values() if storename in DefaultSIdList}
1927         if not storeidset:
1928             return ""
1929         minid = min(storeidset )
1930         for sid, name in self.DefaultStores.values():
1931             if sid == minid:
1932                 return name
1933
1934 class SkuClass():
1935     DEFAULT = 0
1936     SINGLE = 1
1937     MULTIPLE =2
1938
1939     def __init__(self,SkuIdentifier='', SkuIds=None):
1940         if SkuIds is None:
1941             SkuIds = {}
1942
1943         for SkuName in SkuIds:
1944             SkuId = SkuIds[SkuName][0]
1945             skuid_num = int(SkuId, 16) if SkuId.upper().startswith("0X") else int(SkuId)
1946             if skuid_num > 0xFFFFFFFFFFFFFFFF:
1947                 EdkLogger.error("build", PARAMETER_INVALID,
1948                             ExtraData = "SKU-ID [%s] value %s exceeds the max value of UINT64"
1949                                       % (SkuName, SkuId))
1950
1951         self.AvailableSkuIds = sdict()
1952         self.SkuIdSet = []
1953         self.SkuIdNumberSet = []
1954         self.SkuData = SkuIds
1955         self._SkuInherit = {}
1956         self._SkuIdentifier = SkuIdentifier
1957         if SkuIdentifier == '' or SkuIdentifier is None:
1958             self.SkuIdSet = ['DEFAULT']
1959             self.SkuIdNumberSet = ['0U']
1960         elif SkuIdentifier == 'ALL':
1961             self.SkuIdSet = SkuIds.keys()
1962             self.SkuIdNumberSet = [num[0].strip() + 'U' for num in SkuIds.values()]
1963         else:
1964             r = SkuIdentifier.split('|')
1965             self.SkuIdSet=[(r[k].strip()).upper() for k in range(len(r))]
1966             k = None
1967             try:
1968                 self.SkuIdNumberSet = [SkuIds[k][0].strip() + 'U' for k in self.SkuIdSet]
1969             except Exception:
1970                 EdkLogger.error("build", PARAMETER_INVALID,
1971                             ExtraData = "SKU-ID [%s] is not supported by the platform. [Valid SKU-ID: %s]"
1972                                       % (k, " | ".join(SkuIds.keys())))
1973         for each in self.SkuIdSet:
1974             if each in SkuIds:
1975                 self.AvailableSkuIds[each] = SkuIds[each][0]
1976             else:
1977                 EdkLogger.error("build", PARAMETER_INVALID,
1978                             ExtraData="SKU-ID [%s] is not supported by the platform. [Valid SKU-ID: %s]"
1979                                       % (each, " | ".join(SkuIds.keys())))
1980         if self.SkuUsageType != SkuClass.SINGLE:
1981             self.AvailableSkuIds.update({'DEFAULT':0, 'COMMON':0})
1982         if self.SkuIdSet:
1983             GlobalData.gSkuids = (self.SkuIdSet)
1984             if 'COMMON' in GlobalData.gSkuids:
1985                 GlobalData.gSkuids.remove('COMMON')
1986             if self.SkuUsageType == self.SINGLE:
1987                 if len(GlobalData.gSkuids) != 1:
1988                     if 'DEFAULT' in GlobalData.gSkuids:
1989                         GlobalData.gSkuids.remove('DEFAULT')
1990             if GlobalData.gSkuids:
1991                 GlobalData.gSkuids.sort()
1992
1993     def GetNextSkuId(self, skuname):
1994         if not self._SkuInherit:
1995             self._SkuInherit = {}
1996             for item in self.SkuData.values():
1997                 self._SkuInherit[item[1]]=item[2] if item[2] else "DEFAULT"
1998         return self._SkuInherit.get(skuname, "DEFAULT")
1999
2000     def GetSkuChain(self, sku):
2001         if sku == "DEFAULT":
2002             return ["DEFAULT"]
2003         skulist = [sku]
2004         nextsku = sku
2005         while True:
2006             nextsku = self.GetNextSkuId(nextsku)
2007             skulist.append(nextsku)
2008             if nextsku == "DEFAULT":
2009                 break
2010         skulist.reverse()
2011         return skulist
2012     def SkuOverrideOrder(self):
2013         skuorderset = []
2014         for skuname in self.SkuIdSet:
2015             skuorderset.append(self.GetSkuChain(skuname))
2016
2017         skuorder = []
2018         for index in range(max(len(item) for item in skuorderset)):
2019             for subset in skuorderset:
2020                 if index > len(subset)-1:
2021                     continue
2022                 if subset[index] in skuorder:
2023                     continue
2024                 skuorder.append(subset[index])
2025
2026         return skuorder
2027
2028     @property
2029     def SkuUsageType(self):
2030         if self._SkuIdentifier.upper() == "ALL":
2031             return SkuClass.MULTIPLE
2032
2033         if len(self.SkuIdSet) == 1:
2034             if self.SkuIdSet[0] == 'DEFAULT':
2035                 return SkuClass.DEFAULT
2036             return SkuClass.SINGLE
2037         if len(self.SkuIdSet) == 2 and 'DEFAULT' in self.SkuIdSet:
2038             return SkuClass.SINGLE
2039         return SkuClass.MULTIPLE
2040
2041     def DumpSkuIdArrary(self):
2042         if self.SkuUsageType == SkuClass.SINGLE:
2043             return "{0x0}"
2044         ArrayStrList = []
2045         for skuname in self.AvailableSkuIds:
2046             if skuname == "COMMON":
2047                 continue
2048             while skuname != "DEFAULT":
2049                 ArrayStrList.append(hex(int(self.AvailableSkuIds[skuname])))
2050                 skuname = self.GetNextSkuId(skuname)
2051             ArrayStrList.append("0x0")
2052         return "{{{myList}}}".format(myList=",".join(ArrayStrList))
2053
2054     @property
2055     def AvailableSkuIdSet(self):
2056         return self.AvailableSkuIds
2057
2058     @property
2059     def SystemSkuId(self):
2060         if self.SkuUsageType == SkuClass.SINGLE:
2061             if len(self.SkuIdSet) == 1:
2062                 return self.SkuIdSet[0]
2063             else:
2064                 return self.SkuIdSet[0] if self.SkuIdSet[0] != 'DEFAULT' else self.SkuIdSet[1]
2065         else:
2066             return 'DEFAULT'
2067
2068 #
2069 # Pack a registry format GUID
2070 #
2071 def PackRegistryFormatGuid(Guid):
2072     return PackGUID(Guid.split('-'))
2073
2074 ##  Get the integer value from string like "14U" or integer like 2
2075 #
2076 #   @param      Input   The object that may be either a integer value or a string
2077 #
2078 #   @retval     Value    The integer value that the input represents
2079 #
2080 def GetIntegerValue(Input):
2081     if type(Input) in (int, long):
2082         return Input
2083     String = Input
2084     if String.endswith("U"):
2085         String = String[:-1]
2086     if String.endswith("ULL"):
2087         String = String[:-3]
2088     if String.endswith("LL"):
2089         String = String[:-2]
2090
2091     if String.startswith("0x") or String.startswith("0X"):
2092         return int(String, 16)
2093     elif String == '':
2094         return 0
2095     else:
2096         return int(String)
2097
2098 #
2099 # Pack a GUID (registry format) list into a buffer and return it
2100 #
2101 def PackGUID(Guid):
2102     return pack(PACK_PATTERN_GUID,
2103                 int(Guid[0], 16),
2104                 int(Guid[1], 16),
2105                 int(Guid[2], 16),
2106                 int(Guid[3][-4:-2], 16),
2107                 int(Guid[3][-2:], 16),
2108                 int(Guid[4][-12:-10], 16),
2109                 int(Guid[4][-10:-8], 16),
2110                 int(Guid[4][-8:-6], 16),
2111                 int(Guid[4][-6:-4], 16),
2112                 int(Guid[4][-4:-2], 16),
2113                 int(Guid[4][-2:], 16)
2114                 )
2115
2116 #
2117 # Pack a GUID (byte) list into a buffer and return it
2118 #
2119 def PackByteFormatGUID(Guid):
2120     return pack(PACK_PATTERN_GUID,
2121                 Guid[0],
2122                 Guid[1],
2123                 Guid[2],
2124                 Guid[3],
2125                 Guid[4],
2126                 Guid[5],
2127                 Guid[6],
2128                 Guid[7],
2129                 Guid[8],
2130                 Guid[9],
2131                 Guid[10],
2132                 )
2133
2134 ## DeepCopy dict/OrderedDict recusively
2135 #
2136 #   @param      ori_dict    a nested dict or ordereddict
2137 #
2138 #   @retval     new dict or orderdict
2139 #
2140 def CopyDict(ori_dict):
2141     dict_type = ori_dict.__class__
2142     new_dict = dict_type()
2143     for key in ori_dict:
2144         if isinstance(ori_dict[key],(dict,OrderedDict)):
2145             new_dict[key] = CopyDict(ori_dict[key])
2146         else:
2147             new_dict[key] = ori_dict[key]
2148     return new_dict
2149 ##
2150 #
2151 # This acts like the main() function for the script, unless it is 'import'ed into another
2152 # script.
2153 #
2154 if __name__ == '__main__':
2155     pass
2156