BaseTools/Source/Python/Common/Misc.py

   1 ## @file
   2 # Common routines used by all tools
   3 #
   4 # Copyright (c) 2007 - 2018, Intel Corporation. All rights reserved.<BR>
   5 # This program and the accompanying materials
   6 # are licensed and made available under the terms and conditions of the BSD License
   7 # which accompanies this distribution.  The full text of the license may be found at
   8 # http://opensource.org/licenses/bsd-license.php
   9 #
  10 # THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS,
  11 # WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED.
  12 #
  13
  14 ##
  15 # Import Modules
  16 #
  17 from __future__ import absolute_import
  18 import Common.LongFilePathOs as os
  19 import sys
  20 import string
  21 import threading
  22 import time
  23 import re
  24 import pickle
  25 import array
  26 import shutil
  27 from random import sample
  28 from struct import pack
  29 from UserDict import IterableUserDict
  30 from UserList import UserList
  31
  32 from Common import EdkLogger as EdkLogger
  33 from Common import GlobalData as GlobalData
  34 from .DataType import *
  35 from .BuildToolError import *
  36 from CommonDataClass.DataClass import *
  37 from .Parsing import GetSplitValueList
  38 from Common.LongFilePathSupport import OpenLongFilePath as open
  39 from Common.MultipleWorkspace import MultipleWorkspace as mws
  40 import uuid
  41 from CommonDataClass.Exceptions import BadExpression
  42 from Common.caching import cached_property
  43 import subprocess
  44 from collections import OrderedDict
  45 ## Regular expression used to find out place holders in string template
  46 gPlaceholderPattern = re.compile("\$\{([^$()\s]+)\}", re.MULTILINE | re.UNICODE)
  47
  48 ## regular expressions for map file processing
  49 startPatternGeneral = re.compile("^Start[' ']+Length[' ']+Name[' ']+Class")
  50 addressPatternGeneral = re.compile("^Address[' ']+Publics by Value[' ']+Rva\+Base")
  51 valuePatternGcc = re.compile('^([\w_\.]+) +([\da-fA-Fx]+) +([\da-fA-Fx]+)$')
  52 pcdPatternGcc = re.compile('^([\da-fA-Fx]+) +([\da-fA-Fx]+)')
  53 secReGeneral = re.compile('^([\da-fA-F]+):([\da-fA-F]+) +([\da-fA-F]+)[Hh]? +([.\w\$]+) +(\w+)', re.UNICODE)
  54
  55 StructPattern = re.compile(r'[_a-zA-Z][0-9A-Za-z_]*$')
  56
  57 ## Dictionary used to store file time stamp for quick re-access
  58 gFileTimeStampCache = {}    # {file path : file time stamp}
  59
  60 ## Dictionary used to store dependencies of files
  61 gDependencyDatabase = {}    # arch : {file path : [dependent files list]}
  62
  63 #
  64 # If a module is built more than once with different PCDs or library classes
  65 # a temporary INF file with same content is created, the temporary file is removed
  66 # when build exits.
  67 #
  68 _TempInfs = []
  69
  70 def GetVariableOffset(mapfilepath, efifilepath, varnames):
  71     """ Parse map file to get variable offset in current EFI file
  72     @param mapfilepath    Map file absolution path
  73     @param efifilepath:   EFI binary file full path
  74     @param varnames       iteratable container whose elements are variable names to be searched
  75
  76     @return List whos elements are tuple with variable name and raw offset
  77     """
  78     lines = []
  79     try:
  80         f = open(mapfilepath, 'r')
  81         lines = f.readlines()
  82         f.close()
  83     except:
  84         return None
  85
  86     if len(lines) == 0: return None
  87     firstline = lines[0].strip()
  88     if (firstline.startswith("Archive member included ") and
  89         firstline.endswith(" file (symbol)")):
  90         return _parseForGCC(lines, efifilepath, varnames)
  91     if firstline.startswith("# Path:"):
  92         return _parseForXcode(lines, efifilepath, varnames)
  93     return _parseGeneral(lines, efifilepath, varnames)
  94
  95 def _parseForXcode(lines, efifilepath, varnames):
  96     status = 0
  97     ret = []
  98     for line in lines:
  99         line = line.strip()
 100         if status == 0 and line == "# Symbols:":
 101             status = 1
 102             continue
 103         if status == 1 and len(line) != 0:
 104             for varname in varnames:
 105                 if varname in line:
 106                     # cannot pregenerate this RegEx since it uses varname from varnames.
 107                     m = re.match('^([\da-fA-FxX]+)([\s\S]*)([_]*%s)$' % varname, line)
 108                     if m is not None:
 109                         ret.append((varname, m.group(1)))
 110     return ret
 111
 112 def _parseForGCC(lines, efifilepath, varnames):
 113     """ Parse map file generated by GCC linker """
 114     status = 0
 115     sections = []
 116     varoffset = []
 117     for index, line in enumerate(lines):
 118         line = line.strip()
 119         # status machine transection
 120         if status == 0 and line == "Memory Configuration":
 121             status = 1
 122             continue
 123         elif status == 1 and line == 'Linker script and memory map':
 124             status = 2
 125             continue
 126         elif status ==2 and line == 'START GROUP':
 127             status = 3
 128             continue
 129
 130         # status handler
 131         if status == 3:
 132             m = valuePatternGcc.match(line)
 133             if m is not None:
 134                 sections.append(m.groups(0))
 135             for varname in varnames:
 136                 Str = ''
 137                 m = re.match("^.data.(%s)" % varname, line)
 138                 if m is not None:
 139                     m = re.match(".data.(%s)$" % varname, line)
 140                     if m is not None:
 141                         Str = lines[index + 1]
 142                     else:
 143                         Str = line[len(".data.%s" % varname):]
 144                     if Str:
 145                         m = pcdPatternGcc.match(Str.strip())
 146                         if m is not None:
 147                             varoffset.append((varname, int(m.groups(0)[0], 16), int(sections[-1][1], 16), sections[-1][0]))
 148
 149     if not varoffset:
 150         return []
 151     # get section information from efi file
 152     efisecs = PeImageClass(efifilepath).SectionHeaderList
 153     if efisecs is None or len(efisecs) == 0:
 154         return []
 155     #redirection
 156     redirection = 0
 157     for efisec in efisecs:
 158         for section in sections:
 159             if section[0].strip() == efisec[0].strip() and section[0].strip() == '.text':
 160                 redirection = int(section[1], 16) - efisec[1]
 161
 162     ret = []
 163     for var in varoffset:
 164         for efisec in efisecs:
 165             if var[1] >= efisec[1] and var[1] < efisec[1]+efisec[3]:
 166                 ret.append((var[0], hex(efisec[2] + var[1] - efisec[1] - redirection)))
 167     return ret
 168
 169 def _parseGeneral(lines, efifilepath, varnames):
 170     status = 0    #0 - beginning of file; 1 - PE section definition; 2 - symbol table
 171     secs  = []    # key = section name
 172     varoffset = []
 173     symRe = re.compile('^([\da-fA-F]+):([\da-fA-F]+) +([\.:\\\\\w\?@\$]+) +([\da-fA-F]+)', re.UNICODE)
 174
 175     for line in lines:
 176         line = line.strip()
 177         if startPatternGeneral.match(line):
 178             status = 1
 179             continue
 180         if addressPatternGeneral.match(line):
 181             status = 2
 182             continue
 183         if line.startswith("entry point at"):
 184             status = 3
 185             continue
 186         if status == 1 and len(line) != 0:
 187             m =  secReGeneral.match(line)
 188             assert m is not None, "Fail to parse the section in map file , line is %s" % line
 189             sec_no, sec_start, sec_length, sec_name, sec_class = m.groups(0)
 190             secs.append([int(sec_no, 16), int(sec_start, 16), int(sec_length, 16), sec_name, sec_class])
 191         if status == 2 and len(line) != 0:
 192             for varname in varnames:
 193                 m = symRe.match(line)
 194                 assert m is not None, "Fail to parse the symbol in map file, line is %s" % line
 195                 sec_no, sym_offset, sym_name, vir_addr = m.groups(0)
 196                 sec_no     = int(sec_no,     16)
 197                 sym_offset = int(sym_offset, 16)
 198                 vir_addr   = int(vir_addr,   16)
 199                 # cannot pregenerate this RegEx since it uses varname from varnames.
 200                 m2 = re.match('^[_]*(%s)' % varname, sym_name)
 201                 if m2 is not None:
 202                     # fond a binary pcd entry in map file
 203                     for sec in secs:
 204                         if sec[0] == sec_no and (sym_offset >= sec[1] and sym_offset < sec[1] + sec[2]):
 205                             varoffset.append([varname, sec[3], sym_offset, vir_addr, sec_no])
 206
 207     if not varoffset: return []
 208
 209     # get section information from efi file
 210     efisecs = PeImageClass(efifilepath).SectionHeaderList
 211     if efisecs is None or len(efisecs) == 0:
 212         return []
 213
 214     ret = []
 215     for var in varoffset:
 216         index = 0
 217         for efisec in efisecs:
 218             index = index + 1
 219             if var[1].strip() == efisec[0].strip():
 220                 ret.append((var[0], hex(efisec[2] + var[2])))
 221             elif var[4] == index:
 222                 ret.append((var[0], hex(efisec[2] + var[2])))
 223
 224     return ret
 225
 226 ## Routine to process duplicated INF
 227 #
 228 #  This function is called by following two cases:
 229 #  Case 1 in DSC:
 230 #    [components.arch]
 231 #    Pkg/module/module.inf
 232 #    Pkg/module/module.inf {
 233 #      <Defines>
 234 #        FILE_GUID = 0D1B936F-68F3-4589-AFCC-FB8B7AEBC836
 235 #    }
 236 #  Case 2 in FDF:
 237 #    INF Pkg/module/module.inf
 238 #    INF FILE_GUID = 0D1B936F-68F3-4589-AFCC-FB8B7AEBC836 Pkg/module/module.inf
 239 #
 240 #  This function copies Pkg/module/module.inf to
 241 #  Conf/.cache/0D1B936F-68F3-4589-AFCC-FB8B7AEBC836module.inf
 242 #
 243 #  @param Path     Original PathClass object
 244 #  @param BaseName New file base name
 245 #
 246 #  @retval         return the new PathClass object
 247 #
 248 def ProcessDuplicatedInf(Path, BaseName, Workspace):
 249     Filename = os.path.split(Path.File)[1]
 250     if '.' in Filename:
 251         Filename = BaseName + Path.BaseName + Filename[Filename.rfind('.'):]
 252     else:
 253         Filename = BaseName + Path.BaseName
 254
 255     #
 256     # If -N is specified on command line, cache is disabled
 257     # The directory has to be created
 258     #
 259     DbDir = os.path.split(GlobalData.gDatabasePath)[0]
 260     if not os.path.exists(DbDir):
 261         os.makedirs(DbDir)
 262     #
 263     # A temporary INF is copied to database path which must have write permission
 264     # The temporary will be removed at the end of build
 265     # In case of name conflict, the file name is
 266     # FILE_GUIDBaseName (0D1B936F-68F3-4589-AFCC-FB8B7AEBC836module.inf)
 267     #
 268     TempFullPath = os.path.join(DbDir,
 269                                 Filename)
 270     RtPath = PathClass(Path.File, Workspace)
 271     #
 272     # Modify the full path to temporary path, keep other unchanged
 273     #
 274     # To build same module more than once, the module path with FILE_GUID overridden has
 275     # the file name FILE_GUIDmodule.inf, but the relative path (self.MetaFile.File) is the real path
 276     # in DSC which is used as relative path by C files and other files in INF.
 277     # A trick was used: all module paths are PathClass instances, after the initialization
 278     # of PathClass, the PathClass.Path is overridden by the temporary INF path.
 279     #
 280     # The reason for creating a temporary INF is:
 281     # Platform.Modules which is the base to create ModuleAutoGen objects is a dictionary,
 282     # the key is the full path of INF, the value is an object to save overridden library instances, PCDs.
 283     # A different key for the same module is needed to create different output directory,
 284     # retrieve overridden PCDs, library instances.
 285     #
 286     # The BaseName is the FILE_GUID which is also the output directory name.
 287     #
 288     #
 289     RtPath.Path = TempFullPath
 290     RtPath.BaseName = BaseName
 291     #
 292     # If file exists, compare contents
 293     #
 294     if os.path.exists(TempFullPath):
 295         with open(str(Path), 'rb') as f1, open(TempFullPath, 'rb') as f2:
 296             if f1.read() == f2.read():
 297                 return RtPath
 298     _TempInfs.append(TempFullPath)
 299     shutil.copy2(str(Path), TempFullPath)
 300     return RtPath
 301
 302 ## Remove temporary created INFs whose paths were saved in _TempInfs
 303 #
 304 def ClearDuplicatedInf():
 305     while _TempInfs:
 306         File = _TempInfs.pop()
 307         if os.path.exists(File):
 308             os.remove(File)
 309
 310 ## Convert GUID string in xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx style to C structure style
 311 #
 312 #   @param      Guid    The GUID string
 313 #
 314 #   @retval     string  The GUID string in C structure style
 315 #
 316 def GuidStringToGuidStructureString(Guid):
 317     GuidList = Guid.split('-')
 318     Result = '{'
 319     for Index in range(0, 3, 1):
 320         Result = Result + '0x' + GuidList[Index] + ', '
 321     Result = Result + '{0x' + GuidList[3][0:2] + ', 0x' + GuidList[3][2:4]
 322     for Index in range(0, 12, 2):
 323         Result = Result + ', 0x' + GuidList[4][Index:Index + 2]
 324     Result += '}}'
 325     return Result
 326
 327 ## Convert GUID structure in byte array to xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
 328 #
 329 #   @param      GuidValue   The GUID value in byte array
 330 #
 331 #   @retval     string      The GUID value in xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx format
 332 #
 333 def GuidStructureByteArrayToGuidString(GuidValue):
 334     guidValueString = GuidValue.lower().replace("{", "").replace("}", "").replace(" ", "").replace(";", "")
 335     guidValueList = guidValueString.split(",")
 336     if len(guidValueList) != 16:
 337         return ''
 338         #EdkLogger.error(None, None, "Invalid GUID value string %s" % GuidValue)
 339     try:
 340         return "%02x%02x%02x%02x-%02x%02x-%02x%02x-%02x%02x-%02x%02x%02x%02x%02x%02x" % (
 341                 int(guidValueList[3], 16),
 342                 int(guidValueList[2], 16),
 343                 int(guidValueList[1], 16),
 344                 int(guidValueList[0], 16),
 345                 int(guidValueList[5], 16),
 346                 int(guidValueList[4], 16),
 347                 int(guidValueList[7], 16),
 348                 int(guidValueList[6], 16),
 349                 int(guidValueList[8], 16),
 350                 int(guidValueList[9], 16),
 351                 int(guidValueList[10], 16),
 352                 int(guidValueList[11], 16),
 353                 int(guidValueList[12], 16),
 354                 int(guidValueList[13], 16),
 355                 int(guidValueList[14], 16),
 356                 int(guidValueList[15], 16)
 357                 )
 358     except:
 359         return ''
 360
 361 ## Convert GUID string in C structure style to xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
 362 #
 363 #   @param      GuidValue   The GUID value in C structure format
 364 #
 365 #   @retval     string      The GUID value in xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx format
 366 #
 367 def GuidStructureStringToGuidString(GuidValue):
 368     if not GlobalData.gGuidCFormatPattern.match(GuidValue):
 369         return ''
 370     guidValueString = GuidValue.lower().replace("{", "").replace("}", "").replace(" ", "").replace(";", "")
 371     guidValueList = guidValueString.split(",")
 372     if len(guidValueList) != 11:
 373         return ''
 374         #EdkLogger.error(None, None, "Invalid GUID value string %s" % GuidValue)
 375     try:
 376         return "%08x-%04x-%04x-%02x%02x-%02x%02x%02x%02x%02x%02x" % (
 377                 int(guidValueList[0], 16),
 378                 int(guidValueList[1], 16),
 379                 int(guidValueList[2], 16),
 380                 int(guidValueList[3], 16),
 381                 int(guidValueList[4], 16),
 382                 int(guidValueList[5], 16),
 383                 int(guidValueList[6], 16),
 384                 int(guidValueList[7], 16),
 385                 int(guidValueList[8], 16),
 386                 int(guidValueList[9], 16),
 387                 int(guidValueList[10], 16)
 388                 )
 389     except:
 390         return ''
 391
 392 ## Convert GUID string in C structure style to xxxxxxxx_xxxx_xxxx_xxxx_xxxxxxxxxxxx
 393 #
 394 #   @param      GuidValue   The GUID value in C structure format
 395 #
 396 #   @retval     string      The GUID value in xxxxxxxx_xxxx_xxxx_xxxx_xxxxxxxxxxxx format
 397 #
 398 def GuidStructureStringToGuidValueName(GuidValue):
 399     guidValueString = GuidValue.lower().replace("{", "").replace("}", "").replace(" ", "")
 400     guidValueList = guidValueString.split(",")
 401     if len(guidValueList) != 11:
 402         EdkLogger.error(None, FORMAT_INVALID, "Invalid GUID value string [%s]" % GuidValue)
 403     return "%08x_%04x_%04x_%02x%02x_%02x%02x%02x%02x%02x%02x" % (
 404             int(guidValueList[0], 16),
 405             int(guidValueList[1], 16),
 406             int(guidValueList[2], 16),
 407             int(guidValueList[3], 16),
 408             int(guidValueList[4], 16),
 409             int(guidValueList[5], 16),
 410             int(guidValueList[6], 16),
 411             int(guidValueList[7], 16),
 412             int(guidValueList[8], 16),
 413             int(guidValueList[9], 16),
 414             int(guidValueList[10], 16)
 415             )
 416
 417 ## Create directories
 418 #
 419 #   @param      Directory   The directory name
 420 #
 421 def CreateDirectory(Directory):
 422     if Directory is None or Directory.strip() == "":
 423         return True
 424     try:
 425         if not os.access(Directory, os.F_OK):
 426             os.makedirs(Directory)
 427     except:
 428         return False
 429     return True
 430
 431 ## Remove directories, including files and sub-directories in it
 432 #
 433 #   @param      Directory   The directory name
 434 #
 435 def RemoveDirectory(Directory, Recursively=False):
 436     if Directory is None or Directory.strip() == "" or not os.path.exists(Directory):
 437         return
 438     if Recursively:
 439         CurrentDirectory = os.getcwd()
 440         os.chdir(Directory)
 441         for File in os.listdir("."):
 442             if os.path.isdir(File):
 443                 RemoveDirectory(File, Recursively)
 444             else:
 445                 os.remove(File)
 446         os.chdir(CurrentDirectory)
 447     os.rmdir(Directory)
 448
 449 ## Store content in file
 450 #
 451 #  This method is used to save file only when its content is changed. This is
 452 #  quite useful for "make" system to decide what will be re-built and what won't.
 453 #
 454 #   @param      File            The path of file
 455 #   @param      Content         The new content of the file
 456 #   @param      IsBinaryFile    The flag indicating if the file is binary file or not
 457 #
 458 #   @retval     True            If the file content is changed and the file is renewed
 459 #   @retval     False           If the file content is the same
 460 #
 461 def SaveFileOnChange(File, Content, IsBinaryFile=True):
 462     if not IsBinaryFile:
 463         Content = Content.replace("\n", os.linesep)
 464
 465     if os.path.exists(File):
 466         try:
 467             if Content == open(File, "rb").read():
 468                 return False
 469         except:
 470             EdkLogger.error(None, FILE_OPEN_FAILURE, ExtraData=File)
 471
 472     DirName = os.path.dirname(File)
 473     if not CreateDirectory(DirName):
 474         EdkLogger.error(None, FILE_CREATE_FAILURE, "Could not create directory %s" % DirName)
 475     else:
 476         if DirName == '':
 477             DirName = os.getcwd()
 478         if not os.access(DirName, os.W_OK):
 479             EdkLogger.error(None, PERMISSION_FAILURE, "Do not have write permission on directory %s" % DirName)
 480
 481     try:
 482         Fd = open(File, "wb")
 483         Fd.write(Content)
 484         Fd.close()
 485     except IOError as X:
 486         EdkLogger.error(None, FILE_CREATE_FAILURE, ExtraData='IOError %s' % X)
 487
 488     return True
 489
 490 ## Retrieve and cache the real path name in file system
 491 #
 492 #   @param      Root    The root directory of path relative to
 493 #
 494 #   @retval     str     The path string if the path exists
 495 #   @retval     None    If path doesn't exist
 496 #
 497 class DirCache:
 498     _CACHE_ = set()
 499     _UPPER_CACHE_ = {}
 500
 501     def __init__(self, Root):
 502         self._Root = Root
 503         for F in os.listdir(Root):
 504             self._CACHE_.add(F)
 505             self._UPPER_CACHE_[F.upper()] = F
 506
 507     # =[] operator
 508     def __getitem__(self, Path):
 509         Path = Path[len(os.path.commonprefix([Path, self._Root])):]
 510         if not Path:
 511             return self._Root
 512         if Path and Path[0] == os.path.sep:
 513             Path = Path[1:]
 514         if Path in self._CACHE_:
 515             return os.path.join(self._Root, Path)
 516         UpperPath = Path.upper()
 517         if UpperPath in self._UPPER_CACHE_:
 518             return os.path.join(self._Root, self._UPPER_CACHE_[UpperPath])
 519
 520         IndexList = []
 521         LastSepIndex = -1
 522         SepIndex = Path.find(os.path.sep)
 523         while SepIndex > -1:
 524             Parent = UpperPath[:SepIndex]
 525             if Parent not in self._UPPER_CACHE_:
 526                 break
 527             LastSepIndex = SepIndex
 528             SepIndex = Path.find(os.path.sep, LastSepIndex + 1)
 529
 530         if LastSepIndex == -1:
 531             return None
 532
 533         Cwd = os.getcwd()
 534         os.chdir(self._Root)
 535         SepIndex = LastSepIndex
 536         while SepIndex > -1:
 537             Parent = Path[:SepIndex]
 538             ParentKey = UpperPath[:SepIndex]
 539             if ParentKey not in self._UPPER_CACHE_:
 540                 os.chdir(Cwd)
 541                 return None
 542
 543             if Parent in self._CACHE_:
 544                 ParentDir = Parent
 545             else:
 546                 ParentDir = self._UPPER_CACHE_[ParentKey]
 547             for F in os.listdir(ParentDir):
 548                 Dir = os.path.join(ParentDir, F)
 549                 self._CACHE_.add(Dir)
 550                 self._UPPER_CACHE_[Dir.upper()] = Dir
 551
 552             SepIndex = Path.find(os.path.sep, SepIndex + 1)
 553
 554         os.chdir(Cwd)
 555         if Path in self._CACHE_:
 556             return os.path.join(self._Root, Path)
 557         elif UpperPath in self._UPPER_CACHE_:
 558             return os.path.join(self._Root, self._UPPER_CACHE_[UpperPath])
 559         return None
 560
 561 def RealPath(File, Dir='', OverrideDir=''):
 562     NewFile = os.path.normpath(os.path.join(Dir, File))
 563     NewFile = GlobalData.gAllFiles[NewFile]
 564     if not NewFile and OverrideDir:
 565         NewFile = os.path.normpath(os.path.join(OverrideDir, File))
 566         NewFile = GlobalData.gAllFiles[NewFile]
 567     return NewFile
 568
 569 def RealPath2(File, Dir='', OverrideDir=''):
 570     NewFile = None
 571     if OverrideDir:
 572         NewFile = GlobalData.gAllFiles[os.path.normpath(os.path.join(OverrideDir, File))]
 573         if NewFile:
 574             if OverrideDir[-1] == os.path.sep:
 575                 return NewFile[len(OverrideDir):], NewFile[0:len(OverrideDir)]
 576             else:
 577                 return NewFile[len(OverrideDir) + 1:], NewFile[0:len(OverrideDir)]
 578     if GlobalData.gAllFiles:
 579         NewFile = GlobalData.gAllFiles[os.path.normpath(os.path.join(Dir, File))]
 580     if not NewFile:
 581         NewFile = os.path.normpath(os.path.join(Dir, File))
 582         if not os.path.exists(NewFile):
 583             return None, None
 584     if NewFile:
 585         if Dir:
 586             if Dir[-1] == os.path.sep:
 587                 return NewFile[len(Dir):], NewFile[0:len(Dir)]
 588             else:
 589                 return NewFile[len(Dir) + 1:], NewFile[0:len(Dir)]
 590         else:
 591             return NewFile, ''
 592
 593     return None, None
 594
 595 ## Get GUID value from given packages
 596 #
 597 #   @param      CName           The CName of the GUID
 598 #   @param      PackageList     List of packages looking-up in
 599 #   @param      Inffile         The driver file
 600 #
 601 #   @retval     GuidValue   if the CName is found in any given package
 602 #   @retval     None        if the CName is not found in all given packages
 603 #
 604 def GuidValue(CName, PackageList, Inffile = None):
 605     for P in PackageList:
 606         GuidKeys = P.Guids.keys()
 607         if Inffile and P._PrivateGuids:
 608             if not Inffile.startswith(P.MetaFile.Dir):
 609                 GuidKeys = [x for x in P.Guids if x not in P._PrivateGuids]
 610         if CName in GuidKeys:
 611             return P.Guids[CName]
 612     return None
 613
 614 ## Get Protocol value from given packages
 615 #
 616 #   @param      CName           The CName of the GUID
 617 #   @param      PackageList     List of packages looking-up in
 618 #   @param      Inffile         The driver file
 619 #
 620 #   @retval     GuidValue   if the CName is found in any given package
 621 #   @retval     None        if the CName is not found in all given packages
 622 #
 623 def ProtocolValue(CName, PackageList, Inffile = None):
 624     for P in PackageList:
 625         ProtocolKeys = P.Protocols.keys()
 626         if Inffile and P._PrivateProtocols:
 627             if not Inffile.startswith(P.MetaFile.Dir):
 628                 ProtocolKeys = [x for x in P.Protocols if x not in P._PrivateProtocols]
 629         if CName in ProtocolKeys:
 630             return P.Protocols[CName]
 631     return None
 632
 633 ## Get PPI value from given packages
 634 #
 635 #   @param      CName           The CName of the GUID
 636 #   @param      PackageList     List of packages looking-up in
 637 #   @param      Inffile         The driver file
 638 #
 639 #   @retval     GuidValue   if the CName is found in any given package
 640 #   @retval     None        if the CName is not found in all given packages
 641 #
 642 def PpiValue(CName, PackageList, Inffile = None):
 643     for P in PackageList:
 644         PpiKeys = P.Ppis.keys()
 645         if Inffile and P._PrivatePpis:
 646             if not Inffile.startswith(P.MetaFile.Dir):
 647                 PpiKeys = [x for x in P.Ppis if x not in P._PrivatePpis]
 648         if CName in PpiKeys:
 649             return P.Ppis[CName]
 650     return None
 651
 652 ## A string template class
 653 #
 654 #  This class implements a template for string replacement. A string template
 655 #  looks like following
 656 #
 657 #       ${BEGIN} other_string ${placeholder_name} other_string ${END}
 658 #
 659 #  The string between ${BEGIN} and ${END} will be repeated as many times as the
 660 #  length of "placeholder_name", which is a list passed through a dict. The
 661 #  "placeholder_name" is the key name of the dict. The ${BEGIN} and ${END} can
 662 #  be not used and, in this case, the "placeholder_name" must not a list and it
 663 #  will just be replaced once.
 664 #
 665 class TemplateString(object):
 666     _REPEAT_START_FLAG = "BEGIN"
 667     _REPEAT_END_FLAG = "END"
 668
 669     class Section(object):
 670         _LIST_TYPES = [type([]), type(set()), type((0,))]
 671
 672         def __init__(self, TemplateSection, PlaceHolderList):
 673             self._Template = TemplateSection
 674             self._PlaceHolderList = []
 675
 676             # Split the section into sub-sections according to the position of placeholders
 677             if PlaceHolderList:
 678                 self._SubSectionList = []
 679                 SubSectionStart = 0
 680                 #
 681                 # The placeholders passed in must be in the format of
 682                 #
 683                 #   PlaceHolderName, PlaceHolderStartPoint, PlaceHolderEndPoint
 684                 #
 685                 for PlaceHolder, Start, End in PlaceHolderList:
 686                     self._SubSectionList.append(TemplateSection[SubSectionStart:Start])
 687                     self._SubSectionList.append(TemplateSection[Start:End])
 688                     self._PlaceHolderList.append(PlaceHolder)
 689                     SubSectionStart = End
 690                 if SubSectionStart < len(TemplateSection):
 691                     self._SubSectionList.append(TemplateSection[SubSectionStart:])
 692             else:
 693                 self._SubSectionList = [TemplateSection]
 694
 695         def __str__(self):
 696             return self._Template + " : " + str(self._PlaceHolderList)
 697
 698         def Instantiate(self, PlaceHolderValues):
 699             RepeatTime = -1
 700             RepeatPlaceHolders = {}
 701             NonRepeatPlaceHolders = {}
 702
 703             for PlaceHolder in self._PlaceHolderList:
 704                 if PlaceHolder not in PlaceHolderValues:
 705                     continue
 706                 Value = PlaceHolderValues[PlaceHolder]
 707                 if type(Value) in self._LIST_TYPES:
 708                     if RepeatTime < 0:
 709                         RepeatTime = len(Value)
 710                     elif RepeatTime != len(Value):
 711                         EdkLogger.error(
 712                                     "TemplateString",
 713                                     PARAMETER_INVALID,
 714                                     "${%s} has different repeat time from others!" % PlaceHolder,
 715                                     ExtraData=str(self._Template)
 716                                     )
 717                     RepeatPlaceHolders["${%s}" % PlaceHolder] = Value
 718                 else:
 719                     NonRepeatPlaceHolders["${%s}" % PlaceHolder] = Value
 720
 721             if NonRepeatPlaceHolders:
 722                 StringList = []
 723                 for S in self._SubSectionList:
 724                     if S not in NonRepeatPlaceHolders:
 725                         StringList.append(S)
 726                     else:
 727                         StringList.append(str(NonRepeatPlaceHolders[S]))
 728             else:
 729                 StringList = self._SubSectionList
 730
 731             if RepeatPlaceHolders:
 732                 TempStringList = []
 733                 for Index in range(RepeatTime):
 734                     for S in StringList:
 735                         if S not in RepeatPlaceHolders:
 736                             TempStringList.append(S)
 737                         else:
 738                             TempStringList.append(str(RepeatPlaceHolders[S][Index]))
 739                 StringList = TempStringList
 740
 741             return "".join(StringList)
 742
 743     ## Constructor
 744     def __init__(self, Template=None):
 745         self.String = []
 746         self.IsBinary = False
 747         self._Template = Template
 748         self._TemplateSectionList = self._Parse(Template)
 749
 750     ## str() operator
 751     #
 752     #   @retval     string  The string replaced
 753     #
 754     def __str__(self):
 755         return "".join(self.String)
 756
 757     ## Split the template string into fragments per the ${BEGIN} and ${END} flags
 758     #
 759     #   @retval     list    A list of TemplateString.Section objects
 760     #
 761     def _Parse(self, Template):
 762         SectionStart = 0
 763         SearchFrom = 0
 764         MatchEnd = 0
 765         PlaceHolderList = []
 766         TemplateSectionList = []
 767         while Template:
 768             MatchObj = gPlaceholderPattern.search(Template, SearchFrom)
 769             if not MatchObj:
 770                 if MatchEnd <= len(Template):
 771                     TemplateSection = TemplateString.Section(Template[SectionStart:], PlaceHolderList)
 772                     TemplateSectionList.append(TemplateSection)
 773                 break
 774
 775             MatchString = MatchObj.group(1)
 776             MatchStart = MatchObj.start()
 777             MatchEnd = MatchObj.end()
 778
 779             if MatchString == self._REPEAT_START_FLAG:
 780                 if MatchStart > SectionStart:
 781                     TemplateSection = TemplateString.Section(Template[SectionStart:MatchStart], PlaceHolderList)
 782                     TemplateSectionList.append(TemplateSection)
 783                 SectionStart = MatchEnd
 784                 PlaceHolderList = []
 785             elif MatchString == self._REPEAT_END_FLAG:
 786                 TemplateSection = TemplateString.Section(Template[SectionStart:MatchStart], PlaceHolderList)
 787                 TemplateSectionList.append(TemplateSection)
 788                 SectionStart = MatchEnd
 789                 PlaceHolderList = []
 790             else:
 791                 PlaceHolderList.append((MatchString, MatchStart - SectionStart, MatchEnd - SectionStart))
 792             SearchFrom = MatchEnd
 793         return TemplateSectionList
 794
 795     ## Replace the string template with dictionary of placeholders and append it to previous one
 796     #
 797     #   @param      AppendString    The string template to append
 798     #   @param      Dictionary      The placeholder dictionaries
 799     #
 800     def Append(self, AppendString, Dictionary=None):
 801         if Dictionary:
 802             SectionList = self._Parse(AppendString)
 803             self.String.append( "".join(S.Instantiate(Dictionary) for S in SectionList))
 804         else:
 805             if isinstance(AppendString,list):
 806                 self.String.extend(AppendString)
 807             else:
 808                 self.String.append(AppendString)
 809
 810     ## Replace the string template with dictionary of placeholders
 811     #
 812     #   @param      Dictionary      The placeholder dictionaries
 813     #
 814     #   @retval     str             The string replaced with placeholder values
 815     #
 816     def Replace(self, Dictionary=None):
 817         return "".join(S.Instantiate(Dictionary) for S in self._TemplateSectionList)
 818
 819 ## Progress indicator class
 820 #
 821 #  This class makes use of thread to print progress on console.
 822 #
 823 class Progressor:
 824     # for avoiding deadloop
 825     _StopFlag = None
 826     _ProgressThread = None
 827     _CheckInterval = 0.25
 828
 829     ## Constructor
 830     #
 831     #   @param      OpenMessage     The string printed before progress charaters
 832     #   @param      CloseMessage    The string printed after progress charaters
 833     #   @param      ProgressChar    The charater used to indicate the progress
 834     #   @param      Interval        The interval in seconds between two progress charaters
 835     #
 836     def __init__(self, OpenMessage="", CloseMessage="", ProgressChar='.', Interval=1.0):
 837         self.PromptMessage = OpenMessage
 838         self.CodaMessage = CloseMessage
 839         self.ProgressChar = ProgressChar
 840         self.Interval = Interval
 841         if Progressor._StopFlag is None:
 842             Progressor._StopFlag = threading.Event()
 843
 844     ## Start to print progress charater
 845     #
 846     #   @param      OpenMessage     The string printed before progress charaters
 847     #
 848     def Start(self, OpenMessage=None):
 849         if OpenMessage is not None:
 850             self.PromptMessage = OpenMessage
 851         Progressor._StopFlag.clear()
 852         if Progressor._ProgressThread is None:
 853             Progressor._ProgressThread = threading.Thread(target=self._ProgressThreadEntry)
 854             Progressor._ProgressThread.setDaemon(False)
 855             Progressor._ProgressThread.start()
 856
 857     ## Stop printing progress charater
 858     #
 859     #   @param      CloseMessage    The string printed after progress charaters
 860     #
 861     def Stop(self, CloseMessage=None):
 862         OriginalCodaMessage = self.CodaMessage
 863         if CloseMessage is not None:
 864             self.CodaMessage = CloseMessage
 865         self.Abort()
 866         self.CodaMessage = OriginalCodaMessage
 867
 868     ## Thread entry method
 869     def _ProgressThreadEntry(self):
 870         sys.stdout.write(self.PromptMessage + " ")
 871         sys.stdout.flush()
 872         TimeUp = 0.0
 873         while not Progressor._StopFlag.isSet():
 874             if TimeUp <= 0.0:
 875                 sys.stdout.write(self.ProgressChar)
 876                 sys.stdout.flush()
 877                 TimeUp = self.Interval
 878             time.sleep(self._CheckInterval)
 879             TimeUp -= self._CheckInterval
 880         sys.stdout.write(" " + self.CodaMessage + "\n")
 881         sys.stdout.flush()
 882
 883     ## Abort the progress display
 884     @staticmethod
 885     def Abort():
 886         if Progressor._StopFlag is not None:
 887             Progressor._StopFlag.set()
 888         if Progressor._ProgressThread is not None:
 889             Progressor._ProgressThread.join()
 890             Progressor._ProgressThread = None
 891
 892 ## A dict which can access its keys and/or values orderly
 893 #
 894 #  The class implements a new kind of dict which its keys or values can be
 895 #  accessed in the order they are added into the dict. It guarantees the order
 896 #  by making use of an internal list to keep a copy of keys.
 897 #
 898 class sdict(IterableUserDict):
 899     ## Constructor
 900     def __init__(self):
 901         IterableUserDict.__init__(self)
 902         self._key_list = []
 903
 904     ## [] operator
 905     def __setitem__(self, key, value):
 906         if key not in self._key_list:
 907             self._key_list.append(key)
 908         IterableUserDict.__setitem__(self, key, value)
 909
 910     ## del operator
 911     def __delitem__(self, key):
 912         self._key_list.remove(key)
 913         IterableUserDict.__delitem__(self, key)
 914
 915     ## used in "for k in dict" loop to ensure the correct order
 916     def __iter__(self):
 917         return self.iterkeys()
 918
 919     ## len() support
 920     def __len__(self):
 921         return len(self._key_list)
 922
 923     ## "in" test support
 924     def __contains__(self, key):
 925         return key in self._key_list
 926
 927     ## indexof support
 928     def index(self, key):
 929         return self._key_list.index(key)
 930
 931     ## insert support
 932     def insert(self, key, newkey, newvalue, order):
 933         index = self._key_list.index(key)
 934         if order == 'BEFORE':
 935             self._key_list.insert(index, newkey)
 936             IterableUserDict.__setitem__(self, newkey, newvalue)
 937         elif order == 'AFTER':
 938             self._key_list.insert(index + 1, newkey)
 939             IterableUserDict.__setitem__(self, newkey, newvalue)
 940
 941     ## append support
 942     def append(self, sdict):
 943         for key in sdict:
 944             if key not in self._key_list:
 945                 self._key_list.append(key)
 946             IterableUserDict.__setitem__(self, key, sdict[key])
 947
 948     def has_key(self, key):
 949         return key in self._key_list
 950
 951     ## Empty the dict
 952     def clear(self):
 953         self._key_list = []
 954         IterableUserDict.clear(self)
 955
 956     ## Return a copy of keys
 957     def keys(self):
 958         keys = []
 959         for key in self._key_list:
 960             keys.append(key)
 961         return keys
 962
 963     ## Return a copy of values
 964     def values(self):
 965         values = []
 966         for key in self._key_list:
 967             values.append(self[key])
 968         return values
 969
 970     ## Return a copy of (key, value) list
 971     def items(self):
 972         items = []
 973         for key in self._key_list:
 974             items.append((key, self[key]))
 975         return items
 976
 977     ## Iteration support
 978     def iteritems(self):
 979         return iter(self.items())
 980
 981     ## Keys interation support
 982     def iterkeys(self):
 983         return iter(self.keys())
 984
 985     ## Values interation support
 986     def itervalues(self):
 987         return iter(self.values())
 988
 989     ## Return value related to a key, and remove the (key, value) from the dict
 990     def pop(self, key, *dv):
 991         value = None
 992         if key in self._key_list:
 993             value = self[key]
 994             self.__delitem__(key)
 995         elif len(dv) != 0 :
 996             value = kv[0]
 997         return value
 998
 999     ## Return (key, value) pair, and remove the (key, value) from the dict
1000     def popitem(self):
1001         key = self._key_list[-1]
1002         value = self[key]
1003         self.__delitem__(key)
1004         return key, value
1005
1006     def update(self, dict=None, **kwargs):
1007         if dict is not None:
1008             for k, v in dict.items():
1009                 self[k] = v
1010         if len(kwargs):
1011             for k, v in kwargs.items():
1012                 self[k] = v
1013
1014 ## Dictionary with restricted keys
1015 #
1016 class rdict(dict):
1017     ## Constructor
1018     def __init__(self, KeyList):
1019         for Key in KeyList:
1020             dict.__setitem__(self, Key, "")
1021
1022     ## []= operator
1023     def __setitem__(self, key, value):
1024         if key not in self:
1025             EdkLogger.error("RestrictedDict", ATTRIBUTE_SET_FAILURE, "Key [%s] is not allowed" % key,
1026                             ExtraData=", ".join(dict.keys(self)))
1027         dict.__setitem__(self, key, value)
1028
1029     ## =[] operator
1030     def __getitem__(self, key):
1031         if key not in self:
1032             return ""
1033         return dict.__getitem__(self, key)
1034
1035     ## del operator
1036     def __delitem__(self, key):
1037         EdkLogger.error("RestrictedDict", ATTRIBUTE_ACCESS_DENIED, ExtraData="del")
1038
1039     ## Empty the dict
1040     def clear(self):
1041         for Key in self:
1042             self.__setitem__(Key, "")
1043
1044     ## Return value related to a key, and remove the (key, value) from the dict
1045     def pop(self, key, *dv):
1046         EdkLogger.error("RestrictedDict", ATTRIBUTE_ACCESS_DENIED, ExtraData="pop")
1047
1048     ## Return (key, value) pair, and remove the (key, value) from the dict
1049     def popitem(self):
1050         EdkLogger.error("RestrictedDict", ATTRIBUTE_ACCESS_DENIED, ExtraData="popitem")
1051
1052 ## Dictionary using prioritized list as key
1053 #
1054 class tdict:
1055     _ListType = type([])
1056     _TupleType = type(())
1057     _Wildcard = 'COMMON'
1058     _ValidWildcardList = ['COMMON', 'DEFAULT', 'ALL', TAB_STAR, 'PLATFORM']
1059
1060     def __init__(self, _Single_=False, _Level_=2):
1061         self._Level_ = _Level_
1062         self.data = {}
1063         self._Single_ = _Single_
1064
1065     # =[] operator
1066     def __getitem__(self, key):
1067         KeyType = type(key)
1068         RestKeys = None
1069         if KeyType == self._ListType or KeyType == self._TupleType:
1070             FirstKey = key[0]
1071             if len(key) > 1:
1072                 RestKeys = key[1:]
1073             elif self._Level_ > 1:
1074                 RestKeys = [self._Wildcard for i in range(0, self._Level_ - 1)]
1075         else:
1076             FirstKey = key
1077             if self._Level_ > 1:
1078                 RestKeys = [self._Wildcard for i in range(0, self._Level_ - 1)]
1079
1080         if FirstKey is None or str(FirstKey).upper() in self._ValidWildcardList:
1081             FirstKey = self._Wildcard
1082
1083         if self._Single_:
1084             return self._GetSingleValue(FirstKey, RestKeys)
1085         else:
1086             return self._GetAllValues(FirstKey, RestKeys)
1087
1088     def _GetSingleValue(self, FirstKey, RestKeys):
1089         Value = None
1090         #print "%s-%s" % (FirstKey, self._Level_) ,
1091         if self._Level_ > 1:
1092             if FirstKey == self._Wildcard:
1093                 if FirstKey in self.data:
1094                     Value = self.data[FirstKey][RestKeys]
1095                 if Value is None:
1096                     for Key in self.data:
1097                         Value = self.data[Key][RestKeys]
1098                         if Value is not None: break
1099             else:
1100                 if FirstKey in self.data:
1101                     Value = self.data[FirstKey][RestKeys]
1102                 if Value is None and self._Wildcard in self.data:
1103                     #print "Value=None"
1104                     Value = self.data[self._Wildcard][RestKeys]
1105         else:
1106             if FirstKey == self._Wildcard:
1107                 if FirstKey in self.data:
1108                     Value = self.data[FirstKey]
1109                 if Value is None:
1110                     for Key in self.data:
1111                         Value = self.data[Key]
1112                         if Value is not None: break
1113             else:
1114                 if FirstKey in self.data:
1115                     Value = self.data[FirstKey]
1116                 elif self._Wildcard in self.data:
1117                     Value = self.data[self._Wildcard]
1118         return Value
1119
1120     def _GetAllValues(self, FirstKey, RestKeys):
1121         Value = []
1122         if self._Level_ > 1:
1123             if FirstKey == self._Wildcard:
1124                 for Key in self.data:
1125                     Value += self.data[Key][RestKeys]
1126             else:
1127                 if FirstKey in self.data:
1128                     Value += self.data[FirstKey][RestKeys]
1129                 if self._Wildcard in self.data:
1130                     Value += self.data[self._Wildcard][RestKeys]
1131         else:
1132             if FirstKey == self._Wildcard:
1133                 for Key in self.data:
1134                     Value.append(self.data[Key])
1135             else:
1136                 if FirstKey in self.data:
1137                     Value.append(self.data[FirstKey])
1138                 if self._Wildcard in self.data:
1139                     Value.append(self.data[self._Wildcard])
1140         return Value
1141
1142     ## []= operator
1143     def __setitem__(self, key, value):
1144         KeyType = type(key)
1145         RestKeys = None
1146         if KeyType == self._ListType or KeyType == self._TupleType:
1147             FirstKey = key[0]
1148             if len(key) > 1:
1149                 RestKeys = key[1:]
1150             else:
1151                 RestKeys = [self._Wildcard for i in range(0, self._Level_ - 1)]
1152         else:
1153             FirstKey = key
1154             if self._Level_ > 1:
1155                 RestKeys = [self._Wildcard for i in range(0, self._Level_ - 1)]
1156
1157         if FirstKey in self._ValidWildcardList:
1158             FirstKey = self._Wildcard
1159
1160         if FirstKey not in self.data and self._Level_ > 0:
1161             self.data[FirstKey] = tdict(self._Single_, self._Level_ - 1)
1162
1163         if self._Level_ > 1:
1164             self.data[FirstKey][RestKeys] = value
1165         else:
1166             self.data[FirstKey] = value
1167
1168     def SetGreedyMode(self):
1169         self._Single_ = False
1170         if self._Level_ > 1:
1171             for Key in self.data:
1172                 self.data[Key].SetGreedyMode()
1173
1174     def SetSingleMode(self):
1175         self._Single_ = True
1176         if self._Level_ > 1:
1177             for Key in self.data:
1178                 self.data[Key].SetSingleMode()
1179
1180     def GetKeys(self, KeyIndex=0):
1181         assert KeyIndex >= 0
1182         if KeyIndex == 0:
1183             return set(self.data.keys())
1184         else:
1185             keys = set()
1186             for Key in self.data:
1187                 keys |= self.data[Key].GetKeys(KeyIndex - 1)
1188             return keys
1189
1190 def IsFieldValueAnArray (Value):
1191     Value = Value.strip()
1192     if Value.startswith(TAB_GUID) and Value.endswith(')'):
1193         return True
1194     if Value.startswith('L"') and Value.endswith('"')  and len(list(Value[2:-1])) > 1:
1195         return True
1196     if Value[0] == '"' and Value[-1] == '"' and len(list(Value[1:-1])) > 1:
1197         return True
1198     if Value[0] == '{' and Value[-1] == '}':
1199         return True
1200     if Value.startswith("L'") and Value.endswith("'") and len(list(Value[2:-1])) > 1:
1201         return True
1202     if Value[0] == "'" and Value[-1] == "'" and len(list(Value[1:-1])) > 1:
1203         return True
1204     return False
1205
1206 def AnalyzePcdExpression(Setting):
1207     RanStr = ''.join(sample(string.ascii_letters + string.digits, 8))
1208     Setting = Setting.replace('\\\\', RanStr).strip()
1209     # There might be escaped quote in a string: \", \\\" , \', \\\'
1210     Data = Setting
1211     # There might be '|' in string and in ( ... | ... ), replace it with '-'
1212     NewStr = ''
1213     InSingleQuoteStr = False
1214     InDoubleQuoteStr = False
1215     Pair = 0
1216     for Index, ch in enumerate(Data):
1217         if ch == '"' and not InSingleQuoteStr:
1218             if Data[Index - 1] != '\\':
1219                 InDoubleQuoteStr = not InDoubleQuoteStr
1220         elif ch == "'" and not InDoubleQuoteStr:
1221             if Data[Index - 1] != '\\':
1222                 InSingleQuoteStr = not InSingleQuoteStr
1223         elif ch == '(' and not (InSingleQuoteStr or InDoubleQuoteStr):
1224             Pair += 1
1225         elif ch == ')' and not (InSingleQuoteStr or InDoubleQuoteStr):
1226             Pair -= 1
1227
1228         if (Pair > 0 or InSingleQuoteStr or InDoubleQuoteStr) and ch == TAB_VALUE_SPLIT:
1229             NewStr += '-'
1230         else:
1231             NewStr += ch
1232     FieldList = []
1233     StartPos = 0
1234     while True:
1235         Pos = NewStr.find(TAB_VALUE_SPLIT, StartPos)
1236         if Pos < 0:
1237             FieldList.append(Setting[StartPos:].strip())
1238             break
1239         FieldList.append(Setting[StartPos:Pos].strip())
1240         StartPos = Pos + 1
1241     for i, ch in enumerate(FieldList):
1242         if RanStr in ch:
1243             FieldList[i] = ch.replace(RanStr,'\\\\')
1244     return FieldList
1245
1246 def ParseDevPathValue (Value):
1247     if '\\' in Value:
1248         Value.replace('\\', '/').replace(' ', '')
1249
1250     Cmd = 'DevicePath ' + '"' + Value + '"'
1251     try:
1252         p = subprocess.Popen(Cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
1253         out, err = p.communicate()
1254     except Exception as X:
1255         raise BadExpression("DevicePath: %s" % (str(X)) )
1256     finally:
1257         subprocess._cleanup()
1258         p.stdout.close()
1259         p.stderr.close()
1260     if err:
1261         raise BadExpression("DevicePath: %s" % str(err))
1262     Size = len(out.split())
1263     out = ','.join(out.split())
1264     return '{' + out + '}', Size
1265
1266 def ParseFieldValue (Value):
1267     if "{CODE(" in Value:
1268         return Value, len(Value.split(","))
1269     if isinstance(Value, type(0)):
1270         return Value, (Value.bit_length() + 7) / 8
1271     if not isinstance(Value, type('')):
1272         raise BadExpression('Type %s is %s' %(Value, type(Value)))
1273     Value = Value.strip()
1274     if Value.startswith(TAB_UINT8) and Value.endswith(')'):
1275         Value, Size = ParseFieldValue(Value.split('(', 1)[1][:-1])
1276         if Size > 1:
1277             raise BadExpression('Value (%s) Size larger than %d' %(Value, Size))
1278         return Value, 1
1279     if Value.startswith(TAB_UINT16) and Value.endswith(')'):
1280         Value, Size = ParseFieldValue(Value.split('(', 1)[1][:-1])
1281         if Size > 2:
1282             raise BadExpression('Value (%s) Size larger than %d' %(Value, Size))
1283         return Value, 2
1284     if Value.startswith(TAB_UINT32) and Value.endswith(')'):
1285         Value, Size = ParseFieldValue(Value.split('(', 1)[1][:-1])
1286         if Size > 4:
1287             raise BadExpression('Value (%s) Size larger than %d' %(Value, Size))
1288         return Value, 4
1289     if Value.startswith(TAB_UINT64) and Value.endswith(')'):
1290         Value, Size = ParseFieldValue(Value.split('(', 1)[1][:-1])
1291         if Size > 8:
1292             raise BadExpression('Value (%s) Size larger than %d' % (Value, Size))
1293         return Value, 8
1294     if Value.startswith(TAB_GUID) and Value.endswith(')'):
1295         Value = Value.split('(', 1)[1][:-1].strip()
1296         if Value[0] == '{' and Value[-1] == '}':
1297             TmpValue = GuidStructureStringToGuidString(Value)
1298             if not TmpValue:
1299                 raise BadExpression("Invalid GUID value string %s" % Value)
1300             Value = TmpValue
1301         if Value[0] == '"' and Value[-1] == '"':
1302             Value = Value[1:-1]
1303         try:
1304             Value = "'" + uuid.UUID(Value).get_bytes_le() + "'"
1305         except ValueError as Message:
1306             raise BadExpression(Message)
1307         Value, Size = ParseFieldValue(Value)
1308         return Value, 16
1309     if Value.startswith('L"') and Value.endswith('"'):
1310         # Unicode String
1311         # translate escape character
1312         Value = Value[1:]
1313         try:
1314             Value = eval(Value)
1315         except:
1316             Value = Value[1:-1]
1317         List = list(Value)
1318         List.reverse()
1319         Value = 0
1320         for Char in List:
1321             Value = (Value << 16) | ord(Char)
1322         return Value, (len(List) + 1) * 2
1323     if Value.startswith('"') and Value.endswith('"'):
1324         # ASCII String
1325         # translate escape character
1326         try:
1327             Value = eval(Value)
1328         except:
1329             Value = Value[1:-1]
1330         List = list(Value)
1331         List.reverse()
1332         Value = 0
1333         for Char in List:
1334             Value = (Value << 8) | ord(Char)
1335         return Value, len(List) + 1
1336     if Value.startswith("L'") and Value.endswith("'"):
1337         # Unicode Character Constant
1338         # translate escape character
1339         Value = Value[1:]
1340         try:
1341             Value = eval(Value)
1342         except:
1343             Value = Value[1:-1]
1344         List = list(Value)
1345         if len(List) == 0:
1346             raise BadExpression('Length %s is %s' % (Value, len(List)))
1347         List.reverse()
1348         Value = 0
1349         for Char in List:
1350             Value = (Value << 16) | ord(Char)
1351         return Value, len(List) * 2
1352     if Value.startswith("'") and Value.endswith("'"):
1353         # Character constant
1354         # translate escape character
1355         try:
1356             Value = eval(Value)
1357         except:
1358             Value = Value[1:-1]
1359         List = list(Value)
1360         if len(List) == 0:
1361             raise BadExpression('Length %s is %s' % (Value, len(List)))
1362         List.reverse()
1363         Value = 0
1364         for Char in List:
1365             Value = (Value << 8) | ord(Char)
1366         return Value, len(List)
1367     if Value.startswith('{') and Value.endswith('}'):
1368         # Byte array
1369         Value = Value[1:-1]
1370         List = [Item.strip() for Item in Value.split(',')]
1371         List.reverse()
1372         Value = 0
1373         RetSize = 0
1374         for Item in List:
1375             ItemValue, Size = ParseFieldValue(Item)
1376             RetSize += Size
1377             for I in range(Size):
1378                 Value = (Value << 8) | ((ItemValue >> 8 * I) & 0xff)
1379         return Value, RetSize
1380     if Value.startswith('DEVICE_PATH(') and Value.endswith(')'):
1381         Value = Value.replace("DEVICE_PATH(", '').rstrip(')')
1382         Value = Value.strip().strip('"')
1383         return ParseDevPathValue(Value)
1384     if Value.lower().startswith('0x'):
1385         try:
1386             Value = int(Value, 16)
1387         except:
1388             raise BadExpression("invalid hex value: %s" % Value)
1389         if Value == 0:
1390             return 0, 1
1391         return Value, (Value.bit_length() + 7) / 8
1392     if Value[0].isdigit():
1393         Value = int(Value, 10)
1394         if Value == 0:
1395             return 0, 1
1396         return Value, (Value.bit_length() + 7) / 8
1397     if Value.lower() == 'true':
1398         return 1, 1
1399     if Value.lower() == 'false':
1400         return 0, 1
1401     return Value, 1
1402
1403 ## AnalyzeDscPcd
1404 #
1405 #  Analyze DSC PCD value, since there is no data type info in DSC
1406 #  This function is used to match functions (AnalyzePcdData) used for retrieving PCD value from database
1407 #  1. Feature flag: TokenSpace.PcdCName|PcdValue
1408 #  2. Fix and Patch:TokenSpace.PcdCName|PcdValue[|VOID*[|MaxSize]]
1409 #  3. Dynamic default:
1410 #     TokenSpace.PcdCName|PcdValue[|VOID*[|MaxSize]]
1411 #     TokenSpace.PcdCName|PcdValue
1412 #  4. Dynamic VPD:
1413 #     TokenSpace.PcdCName|VpdOffset[|VpdValue]
1414 #     TokenSpace.PcdCName|VpdOffset[|MaxSize[|VpdValue]]
1415 #  5. Dynamic HII:
1416 #     TokenSpace.PcdCName|HiiString|VariableGuid|VariableOffset[|HiiValue]
1417 #  PCD value needs to be located in such kind of string, and the PCD value might be an expression in which
1418 #    there might have "|" operator, also in string value.
1419 #
1420 #  @param Setting: String contain information described above with "TokenSpace.PcdCName|" stripped
1421 #  @param PcdType: PCD type: feature, fixed, dynamic default VPD HII
1422 #  @param DataType: The datum type of PCD: VOID*, UNIT, BOOL
1423 #  @retval:
1424 #    ValueList: A List contain fields described above
1425 #    IsValid:   True if conforming EBNF, otherwise False
1426 #    Index:     The index where PcdValue is in ValueList
1427 #
1428 def AnalyzeDscPcd(Setting, PcdType, DataType=''):
1429     FieldList = AnalyzePcdExpression(Setting)
1430
1431     IsValid = True
1432     if PcdType in (MODEL_PCD_FIXED_AT_BUILD, MODEL_PCD_PATCHABLE_IN_MODULE, MODEL_PCD_DYNAMIC_DEFAULT, MODEL_PCD_DYNAMIC_EX_DEFAULT):
1433         Value = FieldList[0]
1434         Size = ''
1435         if len(FieldList) > 1 and FieldList[1]:
1436             DataType = FieldList[1]
1437             if FieldList[1] != TAB_VOID and StructPattern.match(FieldList[1]) is None:
1438                 IsValid = False
1439         if len(FieldList) > 2:
1440             Size = FieldList[2]
1441         if IsValid:
1442             if DataType == "":
1443                 IsValid = (len(FieldList) <= 1)
1444             else:
1445                 IsValid = (len(FieldList) <= 3)
1446
1447         if Size:
1448             try:
1449                 int(Size, 16) if Size.upper().startswith("0X") else int(Size)
1450             except:
1451                 IsValid = False
1452                 Size = -1
1453         return [str(Value), DataType, str(Size)], IsValid, 0
1454     elif PcdType == MODEL_PCD_FEATURE_FLAG:
1455         Value = FieldList[0]
1456         Size = ''
1457         IsValid = (len(FieldList) <= 1)
1458         return [Value, DataType, str(Size)], IsValid, 0
1459     elif PcdType in (MODEL_PCD_DYNAMIC_VPD, MODEL_PCD_DYNAMIC_EX_VPD):
1460         VpdOffset = FieldList[0]
1461         Value = Size = ''
1462         if not DataType == TAB_VOID:
1463             if len(FieldList) > 1:
1464                 Value = FieldList[1]
1465         else:
1466             if len(FieldList) > 1:
1467                 Size = FieldList[1]
1468             if len(FieldList) > 2:
1469                 Value = FieldList[2]
1470         if DataType == "":
1471             IsValid = (len(FieldList) <= 1)
1472         else:
1473             IsValid = (len(FieldList) <= 3)
1474         if Size:
1475             try:
1476                 int(Size, 16) if Size.upper().startswith("0X") else int(Size)
1477             except:
1478                 IsValid = False
1479                 Size = -1
1480         return [VpdOffset, str(Size), Value], IsValid, 2
1481     elif PcdType in (MODEL_PCD_DYNAMIC_HII, MODEL_PCD_DYNAMIC_EX_HII):
1482         IsValid = (3 <= len(FieldList) <= 5)
1483         HiiString = FieldList[0]
1484         Guid = Offset = Value = Attribute = ''
1485         if len(FieldList) > 1:
1486             Guid = FieldList[1]
1487         if len(FieldList) > 2:
1488             Offset = FieldList[2]
1489         if len(FieldList) > 3:
1490             Value = FieldList[3]
1491         if len(FieldList) > 4:
1492             Attribute = FieldList[4]
1493         return [HiiString, Guid, Offset, Value, Attribute], IsValid, 3
1494     return [], False, 0
1495
1496 ## AnalyzePcdData
1497 #
1498 #  Analyze the pcd Value, Datum type and TokenNumber.
1499 #  Used to avoid split issue while the value string contain "|" character
1500 #
1501 #  @param[in] Setting:  A String contain value/datum type/token number information;
1502 #
1503 #  @retval   ValueList: A List contain value, datum type and toke number.
1504 #
1505 def AnalyzePcdData(Setting):
1506     ValueList = ['', '', '']
1507
1508     ValueRe = re.compile(r'^\s*L?\".*\|.*\"')
1509     PtrValue = ValueRe.findall(Setting)
1510
1511     ValueUpdateFlag = False
1512
1513     if len(PtrValue) >= 1:
1514         Setting = re.sub(ValueRe, '', Setting)
1515         ValueUpdateFlag = True
1516
1517     TokenList = Setting.split(TAB_VALUE_SPLIT)
1518     ValueList[0:len(TokenList)] = TokenList
1519
1520     if ValueUpdateFlag:
1521         ValueList[0] = PtrValue[0]
1522
1523     return ValueList
1524
1525 ## check format of PCD value against its the datum type
1526 #
1527 # For PCD value setting
1528 #
1529 def CheckPcdDatum(Type, Value):
1530     if Type == TAB_VOID:
1531         ValueRe = re.compile(r'\s*L?\".*\"\s*$')
1532         if not (((Value.startswith('L"') or Value.startswith('"')) and Value.endswith('"'))
1533                 or (Value.startswith('{') and Value.endswith('}')) or (Value.startswith("L'") or Value.startswith("'") and Value.endswith("'"))
1534                ):
1535             return False, "Invalid value [%s] of type [%s]; must be in the form of {...} for array"\
1536                           ", \"...\" or \'...\' for string, L\"...\" or L\'...\' for unicode string" % (Value, Type)
1537         elif ValueRe.match(Value):
1538             # Check the chars in UnicodeString or CString is printable
1539             if Value.startswith("L"):
1540                 Value = Value[2:-1]
1541             else:
1542                 Value = Value[1:-1]
1543             Printset = set(string.printable)
1544             Printset.remove(TAB_PRINTCHAR_VT)
1545             Printset.add(TAB_PRINTCHAR_BS)
1546             Printset.add(TAB_PRINTCHAR_NUL)
1547             if not set(Value).issubset(Printset):
1548                 PrintList = sorted(Printset)
1549                 return False, "Invalid PCD string value of type [%s]; must be printable chars %s." % (Type, PrintList)
1550     elif Type == 'BOOLEAN':
1551         if Value not in ['TRUE', 'True', 'true', '0x1', '0x01', '1', 'FALSE', 'False', 'false', '0x0', '0x00', '0']:
1552             return False, "Invalid value [%s] of type [%s]; must be one of TRUE, True, true, 0x1, 0x01, 1"\
1553                           ", FALSE, False, false, 0x0, 0x00, 0" % (Value, Type)
1554     elif Type in [TAB_UINT8, TAB_UINT16, TAB_UINT32, TAB_UINT64]:
1555         if Value and int(Value, 0) < 0:
1556             return False, "PCD can't be set to negative value[%s] for datum type [%s]" % (Value, Type)
1557         try:
1558             Value = long(Value, 0)
1559             if Value > MAX_VAL_TYPE[Type]:
1560                 return False, "Too large PCD value[%s] for datum type [%s]" % (Value, Type)
1561         except:
1562             return False, "Invalid value [%s] of type [%s];"\
1563                           " must be a hexadecimal, decimal or octal in C language format." % (Value, Type)
1564     else:
1565         return True, "StructurePcd"
1566
1567     return True, ""
1568
1569 ## Split command line option string to list
1570 #
1571 # subprocess.Popen needs the args to be a sequence. Otherwise there's problem
1572 # in non-windows platform to launch command
1573 #
1574 def SplitOption(OptionString):
1575     OptionList = []
1576     LastChar = " "
1577     OptionStart = 0
1578     QuotationMark = ""
1579     for Index in range(0, len(OptionString)):
1580         CurrentChar = OptionString[Index]
1581         if CurrentChar in ['"', "'"]:
1582             if QuotationMark == CurrentChar:
1583                 QuotationMark = ""
1584             elif QuotationMark == "":
1585                 QuotationMark = CurrentChar
1586             continue
1587         elif QuotationMark:
1588             continue
1589
1590         if CurrentChar in ["/", "-"] and LastChar in [" ", "\t", "\r", "\n"]:
1591             if Index > OptionStart:
1592                 OptionList.append(OptionString[OptionStart:Index - 1])
1593             OptionStart = Index
1594         LastChar = CurrentChar
1595     OptionList.append(OptionString[OptionStart:])
1596     return OptionList
1597
1598 def CommonPath(PathList):
1599     P1 = min(PathList).split(os.path.sep)
1600     P2 = max(PathList).split(os.path.sep)
1601     for Index in xrange(min(len(P1), len(P2))):
1602         if P1[Index] != P2[Index]:
1603             return os.path.sep.join(P1[:Index])
1604     return os.path.sep.join(P1)
1605
1606 #
1607 # Convert string to C format array
1608 #
1609 def ConvertStringToByteArray(Value):
1610     Value = Value.strip()
1611     if not Value:
1612         return None
1613     if Value[0] == '{':
1614         if not Value.endswith('}'):
1615             return None
1616         Value = Value.replace(' ', '').replace('{', '').replace('}', '')
1617         ValFields = Value.split(',')
1618         try:
1619             for Index in range(len(ValFields)):
1620                 ValFields[Index] = str(int(ValFields[Index], 0))
1621         except ValueError:
1622             return None
1623         Value = '{' + ','.join(ValFields) + '}'
1624         return Value
1625
1626     Unicode = False
1627     if Value.startswith('L"'):
1628         if not Value.endswith('"'):
1629             return None
1630         Value = Value[1:]
1631         Unicode = True
1632     elif not Value.startswith('"') or not Value.endswith('"'):
1633         return None
1634
1635     Value = eval(Value)         # translate escape character
1636     NewValue = '{'
1637     for Index in range(0, len(Value)):
1638         if Unicode:
1639             NewValue = NewValue + str(ord(Value[Index]) % 0x10000) + ','
1640         else:
1641             NewValue = NewValue + str(ord(Value[Index]) % 0x100) + ','
1642     Value = NewValue + '0}'
1643     return Value
1644
1645 class PathClass(object):
1646     def __init__(self, File='', Root='', AlterRoot='', Type='', IsBinary=False,
1647                  Arch='COMMON', ToolChainFamily='', Target='', TagName='', ToolCode=''):
1648         self.Arch = Arch
1649         self.File = str(File)
1650         if os.path.isabs(self.File):
1651             self.Root = ''
1652             self.AlterRoot = ''
1653         else:
1654             self.Root = str(Root)
1655             self.AlterRoot = str(AlterRoot)
1656
1657         # Remove any '.' and '..' in path
1658         if self.Root:
1659             self.Root = mws.getWs(self.Root, self.File)
1660             self.Path = os.path.normpath(os.path.join(self.Root, self.File))
1661             self.Root = os.path.normpath(CommonPath([self.Root, self.Path]))
1662             # eliminate the side-effect of 'C:'
1663             if self.Root[-1] == ':':
1664                 self.Root += os.path.sep
1665             # file path should not start with path separator
1666             if self.Root[-1] == os.path.sep:
1667                 self.File = self.Path[len(self.Root):]
1668             else:
1669                 self.File = self.Path[len(self.Root) + 1:]
1670         else:
1671             self.Path = os.path.normpath(self.File)
1672
1673         self.SubDir, self.Name = os.path.split(self.File)
1674         self.BaseName, self.Ext = os.path.splitext(self.Name)
1675
1676         if self.Root:
1677             if self.SubDir:
1678                 self.Dir = os.path.join(self.Root, self.SubDir)
1679             else:
1680                 self.Dir = self.Root
1681         else:
1682             self.Dir = self.SubDir
1683
1684         if IsBinary:
1685             self.Type = Type
1686         else:
1687             self.Type = self.Ext.lower()
1688
1689         self.IsBinary = IsBinary
1690         self.Target = Target
1691         self.TagName = TagName
1692         self.ToolCode = ToolCode
1693         self.ToolChainFamily = ToolChainFamily
1694
1695     ## Convert the object of this class to a string
1696     #
1697     #  Convert member Path of the class to a string
1698     #
1699     #  @retval string Formatted String
1700     #
1701     def __str__(self):
1702         return self.Path
1703
1704     ## Override __eq__ function
1705     #
1706     # Check whether PathClass are the same
1707     #
1708     # @retval False The two PathClass are different
1709     # @retval True  The two PathClass are the same
1710     #
1711     def __eq__(self, Other):
1712         return self.Path == str(Other)
1713
1714     ## Override __cmp__ function
1715     #
1716     # Customize the comparsion operation of two PathClass
1717     #
1718     # @retval 0     The two PathClass are different
1719     # @retval -1    The first PathClass is less than the second PathClass
1720     # @retval 1     The first PathClass is Bigger than the second PathClass
1721     def __cmp__(self, Other):
1722         OtherKey = str(Other)
1723
1724         SelfKey = self.Path
1725         if SelfKey == OtherKey:
1726             return 0
1727         elif SelfKey > OtherKey:
1728             return 1
1729         else:
1730             return -1
1731
1732     ## Override __hash__ function
1733     #
1734     # Use Path as key in hash table
1735     #
1736     # @retval string Key for hash table
1737     #
1738     def __hash__(self):
1739         return hash(self.Path)
1740
1741     @cached_property
1742     def Key(self):
1743         return self.Path.upper()
1744
1745     @property
1746     def TimeStamp(self):
1747         return os.stat(self.Path)[8]
1748
1749     def Validate(self, Type='', CaseSensitive=True):
1750         if GlobalData.gCaseInsensitive:
1751             CaseSensitive = False
1752         if Type and Type.lower() != self.Type:
1753             return FILE_TYPE_MISMATCH, '%s (expect %s but got %s)' % (self.File, Type, self.Type)
1754
1755         RealFile, RealRoot = RealPath2(self.File, self.Root, self.AlterRoot)
1756         if not RealRoot and not RealFile:
1757             RealFile = self.File
1758             if self.AlterRoot:
1759                 RealFile = os.path.join(self.AlterRoot, self.File)
1760             elif self.Root:
1761                 RealFile = os.path.join(self.Root, self.File)
1762             if len (mws.getPkgPath()) == 0:
1763                 return FILE_NOT_FOUND, os.path.join(self.AlterRoot, RealFile)
1764             else:
1765                 return FILE_NOT_FOUND, "%s is not found in packages path:\n\t%s" % (self.File, '\n\t'.join(mws.getPkgPath()))
1766
1767         ErrorCode = 0
1768         ErrorInfo = ''
1769         if RealRoot != self.Root or RealFile != self.File:
1770             if CaseSensitive and (RealFile != self.File or (RealRoot != self.Root and RealRoot != self.AlterRoot)):
1771                 ErrorCode = FILE_CASE_MISMATCH
1772                 ErrorInfo = self.File + '\n\t' + RealFile + " [in file system]"
1773
1774             self.SubDir, self.Name = os.path.split(RealFile)
1775             self.BaseName, self.Ext = os.path.splitext(self.Name)
1776             if self.SubDir:
1777                 self.Dir = os.path.join(RealRoot, self.SubDir)
1778             else:
1779                 self.Dir = RealRoot
1780             self.File = RealFile
1781             self.Root = RealRoot
1782             self.Path = os.path.join(RealRoot, RealFile)
1783         return ErrorCode, ErrorInfo
1784
1785 ## Parse PE image to get the required PE informaion.
1786 #
1787 class PeImageClass():
1788     ## Constructor
1789     #
1790     #   @param  File FilePath of PeImage
1791     #
1792     def __init__(self, PeFile):
1793         self.FileName   = PeFile
1794         self.IsValid    = False
1795         self.Size       = 0
1796         self.EntryPoint = 0
1797         self.SectionAlignment  = 0
1798         self.SectionHeaderList = []
1799         self.ErrorInfo = ''
1800         try:
1801             PeObject = open(PeFile, 'rb')
1802         except:
1803             self.ErrorInfo = self.FileName + ' can not be found\n'
1804             return
1805         # Read DOS header
1806         ByteArray = array.array('B')
1807         ByteArray.fromfile(PeObject, 0x3E)
1808         ByteList = ByteArray.tolist()
1809         # DOS signature should be 'MZ'
1810         if self._ByteListToStr (ByteList[0x0:0x2]) != 'MZ':
1811             self.ErrorInfo = self.FileName + ' has no valid DOS signature MZ'
1812             return
1813
1814         # Read 4 byte PE Signature
1815         PeOffset = self._ByteListToInt(ByteList[0x3C:0x3E])
1816         PeObject.seek(PeOffset)
1817         ByteArray = array.array('B')
1818         ByteArray.fromfile(PeObject, 4)
1819         # PE signature should be 'PE\0\0'
1820         if ByteArray.tostring() != 'PE\0\0':
1821             self.ErrorInfo = self.FileName + ' has no valid PE signature PE00'
1822             return
1823
1824         # Read PE file header
1825         ByteArray = array.array('B')
1826         ByteArray.fromfile(PeObject, 0x14)
1827         ByteList = ByteArray.tolist()
1828         SecNumber = self._ByteListToInt(ByteList[0x2:0x4])
1829         if SecNumber == 0:
1830             self.ErrorInfo = self.FileName + ' has no section header'
1831             return
1832
1833         # Read PE optional header
1834         OptionalHeaderSize = self._ByteListToInt(ByteArray[0x10:0x12])
1835         ByteArray = array.array('B')
1836         ByteArray.fromfile(PeObject, OptionalHeaderSize)
1837         ByteList = ByteArray.tolist()
1838         self.EntryPoint       = self._ByteListToInt(ByteList[0x10:0x14])
1839         self.SectionAlignment = self._ByteListToInt(ByteList[0x20:0x24])
1840         self.Size             = self._ByteListToInt(ByteList[0x38:0x3C])
1841
1842         # Read each Section Header
1843         for Index in range(SecNumber):
1844             ByteArray = array.array('B')
1845             ByteArray.fromfile(PeObject, 0x28)
1846             ByteList = ByteArray.tolist()
1847             SecName  = self._ByteListToStr(ByteList[0:8])
1848             SecVirtualSize = self._ByteListToInt(ByteList[8:12])
1849             SecRawAddress  = self._ByteListToInt(ByteList[20:24])
1850             SecVirtualAddress = self._ByteListToInt(ByteList[12:16])
1851             self.SectionHeaderList.append((SecName, SecVirtualAddress, SecRawAddress, SecVirtualSize))
1852         self.IsValid = True
1853         PeObject.close()
1854
1855     def _ByteListToStr(self, ByteList):
1856         String = ''
1857         for index in range(len(ByteList)):
1858             if ByteList[index] == 0:
1859                 break
1860             String += chr(ByteList[index])
1861         return String
1862
1863     def _ByteListToInt(self, ByteList):
1864         Value = 0
1865         for index in range(len(ByteList) - 1, -1, -1):
1866             Value = (Value << 8) | int(ByteList[index])
1867         return Value
1868
1869 class DefaultStore():
1870     def __init__(self, DefaultStores ):
1871
1872         self.DefaultStores = DefaultStores
1873     def DefaultStoreID(self, DefaultStoreName):
1874         for key, value in self.DefaultStores.items():
1875             if value == DefaultStoreName:
1876                 return key
1877         return None
1878     def GetDefaultDefault(self):
1879         if not self.DefaultStores or "0" in self.DefaultStores:
1880             return "0", TAB_DEFAULT_STORES_DEFAULT
1881         else:
1882             minvalue = min(int(value_str) for value_str in self.DefaultStores)
1883             return (str(minvalue), self.DefaultStores[str(minvalue)])
1884     def GetMin(self, DefaultSIdList):
1885         if not DefaultSIdList:
1886             return TAB_DEFAULT_STORES_DEFAULT
1887         storeidset = {storeid for storeid, storename in self.DefaultStores.values() if storename in DefaultSIdList}
1888         if not storeidset:
1889             return ""
1890         minid = min(storeidset )
1891         for sid, name in self.DefaultStores.values():
1892             if sid == minid:
1893                 return name
1894
1895 class SkuClass():
1896     DEFAULT = 0
1897     SINGLE = 1
1898     MULTIPLE =2
1899
1900     def __init__(self,SkuIdentifier='', SkuIds=None):
1901         if SkuIds is None:
1902             SkuIds = {}
1903
1904         for SkuName in SkuIds:
1905             SkuId = SkuIds[SkuName][0]
1906             skuid_num = int(SkuId, 16) if SkuId.upper().startswith("0X") else int(SkuId)
1907             if skuid_num > 0xFFFFFFFFFFFFFFFF:
1908                 EdkLogger.error("build", PARAMETER_INVALID,
1909                             ExtraData = "SKU-ID [%s] value %s exceeds the max value of UINT64"
1910                                       % (SkuName, SkuId))
1911
1912         self.AvailableSkuIds = sdict()
1913         self.SkuIdSet = []
1914         self.SkuIdNumberSet = []
1915         self.SkuData = SkuIds
1916         self._SkuInherit = {}
1917         self._SkuIdentifier = SkuIdentifier
1918         if SkuIdentifier == '' or SkuIdentifier is None:
1919             self.SkuIdSet = ['DEFAULT']
1920             self.SkuIdNumberSet = ['0U']
1921         elif SkuIdentifier == 'ALL':
1922             self.SkuIdSet = SkuIds.keys()
1923             self.SkuIdNumberSet = [num[0].strip() + 'U' for num in SkuIds.values()]
1924         else:
1925             r = SkuIdentifier.split('|')
1926             self.SkuIdSet=[(r[k].strip()).upper() for k in range(len(r))]
1927             k = None
1928             try:
1929                 self.SkuIdNumberSet = [SkuIds[k][0].strip() + 'U' for k in self.SkuIdSet]
1930             except Exception:
1931                 EdkLogger.error("build", PARAMETER_INVALID,
1932                             ExtraData = "SKU-ID [%s] is not supported by the platform. [Valid SKU-ID: %s]"
1933                                       % (k, " | ".join(SkuIds.keys())))
1934         for each in self.SkuIdSet:
1935             if each in SkuIds:
1936                 self.AvailableSkuIds[each] = SkuIds[each][0]
1937             else:
1938                 EdkLogger.error("build", PARAMETER_INVALID,
1939                             ExtraData="SKU-ID [%s] is not supported by the platform. [Valid SKU-ID: %s]"
1940                                       % (each, " | ".join(SkuIds.keys())))
1941         if self.SkuUsageType != SkuClass.SINGLE:
1942             self.AvailableSkuIds.update({'DEFAULT':0, 'COMMON':0})
1943         if self.SkuIdSet:
1944             GlobalData.gSkuids = (self.SkuIdSet)
1945             if 'COMMON' in GlobalData.gSkuids:
1946                 GlobalData.gSkuids.remove('COMMON')
1947             if self.SkuUsageType == self.SINGLE:
1948                 if len(GlobalData.gSkuids) != 1:
1949                     if 'DEFAULT' in GlobalData.gSkuids:
1950                         GlobalData.gSkuids.remove('DEFAULT')
1951             if GlobalData.gSkuids:
1952                 GlobalData.gSkuids.sort()
1953
1954     def GetNextSkuId(self, skuname):
1955         if not self._SkuInherit:
1956             self._SkuInherit = {}
1957             for item in self.SkuData.values():
1958                 self._SkuInherit[item[1]]=item[2] if item[2] else "DEFAULT"
1959         return self._SkuInherit.get(skuname, "DEFAULT")
1960
1961     def GetSkuChain(self, sku):
1962         if sku == "DEFAULT":
1963             return ["DEFAULT"]
1964         skulist = [sku]
1965         nextsku = sku
1966         while True:
1967             nextsku = self.GetNextSkuId(nextsku)
1968             skulist.append(nextsku)
1969             if nextsku == "DEFAULT":
1970                 break
1971         skulist.reverse()
1972         return skulist
1973     def SkuOverrideOrder(self):
1974         skuorderset = []
1975         for skuname in self.SkuIdSet:
1976             skuorderset.append(self.GetSkuChain(skuname))
1977
1978         skuorder = []
1979         for index in range(max(len(item) for item in skuorderset)):
1980             for subset in skuorderset:
1981                 if index > len(subset)-1:
1982                     continue
1983                 if subset[index] in skuorder:
1984                     continue
1985                 skuorder.append(subset[index])
1986
1987         return skuorder
1988
1989     @property
1990     def SkuUsageType(self):
1991         if self._SkuIdentifier.upper() == "ALL":
1992             return SkuClass.MULTIPLE
1993
1994         if len(self.SkuIdSet) == 1:
1995             if self.SkuIdSet[0] == 'DEFAULT':
1996                 return SkuClass.DEFAULT
1997             return SkuClass.SINGLE
1998         if len(self.SkuIdSet) == 2 and 'DEFAULT' in self.SkuIdSet:
1999             return SkuClass.SINGLE
2000         return SkuClass.MULTIPLE
2001
2002     def DumpSkuIdArrary(self):
2003         if self.SkuUsageType == SkuClass.SINGLE:
2004             return "{0x0}"
2005         ArrayStrList = []
2006         for skuname in self.AvailableSkuIds:
2007             if skuname == "COMMON":
2008                 continue
2009             while skuname != "DEFAULT":
2010                 ArrayStrList.append(hex(int(self.AvailableSkuIds[skuname])))
2011                 skuname = self.GetNextSkuId(skuname)
2012             ArrayStrList.append("0x0")
2013         return "{{{myList}}}".format(myList=",".join(ArrayStrList))
2014
2015     @property
2016     def AvailableSkuIdSet(self):
2017         return self.AvailableSkuIds
2018
2019     @property
2020     def SystemSkuId(self):
2021         if self.SkuUsageType == SkuClass.SINGLE:
2022             if len(self.SkuIdSet) == 1:
2023                 return self.SkuIdSet[0]
2024             else:
2025                 return self.SkuIdSet[0] if self.SkuIdSet[0] != 'DEFAULT' else self.SkuIdSet[1]
2026         else:
2027             return 'DEFAULT'
2028
2029 #
2030 # Pack a registry format GUID
2031 #
2032 def PackRegistryFormatGuid(Guid):
2033     return PackGUID(Guid.split('-'))
2034
2035 ##  Get the integer value from string like "14U" or integer like 2
2036 #
2037 #   @param      Input   The object that may be either a integer value or a string
2038 #
2039 #   @retval     Value    The integer value that the input represents
2040 #
2041 def GetIntegerValue(Input):
2042     if type(Input) in (int, long):
2043         return Input
2044     String = Input
2045     if String.endswith("U"):
2046         String = String[:-1]
2047     if String.endswith("ULL"):
2048         String = String[:-3]
2049     if String.endswith("LL"):
2050         String = String[:-2]
2051
2052     if String.startswith("0x") or String.startswith("0X"):
2053         return int(String, 16)
2054     elif String == '':
2055         return 0
2056     else:
2057         return int(String)
2058
2059 #
2060 # Pack a GUID (registry format) list into a buffer and return it
2061 #
2062 def PackGUID(Guid):
2063     return pack(PACK_PATTERN_GUID,
2064                 int(Guid[0], 16),
2065                 int(Guid[1], 16),
2066                 int(Guid[2], 16),
2067                 int(Guid[3][-4:-2], 16),
2068                 int(Guid[3][-2:], 16),
2069                 int(Guid[4][-12:-10], 16),
2070                 int(Guid[4][-10:-8], 16),
2071                 int(Guid[4][-8:-6], 16),
2072                 int(Guid[4][-6:-4], 16),
2073                 int(Guid[4][-4:-2], 16),
2074                 int(Guid[4][-2:], 16)
2075                 )
2076
2077 #
2078 # Pack a GUID (byte) list into a buffer and return it
2079 #
2080 def PackByteFormatGUID(Guid):
2081     return pack(PACK_PATTERN_GUID,
2082                 Guid[0],
2083                 Guid[1],
2084                 Guid[2],
2085                 Guid[3],
2086                 Guid[4],
2087                 Guid[5],
2088                 Guid[6],
2089                 Guid[7],
2090                 Guid[8],
2091                 Guid[9],
2092                 Guid[10],
2093                 )
2094
2095 ## DeepCopy dict/OrderedDict recusively
2096 #
2097 #   @param      ori_dict    a nested dict or ordereddict
2098 #
2099 #   @retval     new dict or orderdict
2100 #
2101 def CopyDict(ori_dict):
2102     dict_type = ori_dict.__class__
2103     if dict_type not in (dict,OrderedDict):
2104         return ori_dict
2105     new_dict = dict_type()
2106     for key in ori_dict:
2107         if isinstance(ori_dict[key],(dict,OrderedDict)):
2108             new_dict[key] = CopyDict(ori_dict[key])
2109         else:
2110             new_dict[key] = ori_dict[key]
2111     return new_dict
2112
2113 #
2114 # Remove the c/c++ comments: // and /* */
2115 #
2116 def RemoveCComments(ctext):
2117     return re.sub('//.*?\n|/\*.*?\*/', '\n', ctext, flags=re.S)
2118 ##
2119 #
2120 # This acts like the main() function for the script, unless it is 'import'ed into another
2121 # script.
2122 #
2123 if __name__ == '__main__':
2124     pass
2125