BaseTools/Source/Python/Common/Misc.py

   1 ## @file
   2 # Common routines used by all tools
   3 #
   4 # Copyright (c) 2007 - 2019, Intel Corporation. All rights reserved.<BR>
   5 # SPDX-License-Identifier: BSD-2-Clause-Patent
   6 #
   7
   8 ##
   9 # Import Modules
  10 #
  11 from __future__ import absolute_import
  12
  13 import sys
  14 import string
  15 import threading
  16 import time
  17 import re
  18 import pickle
  19 import array
  20 import shutil
  21 from random import sample
  22 from struct import pack
  23 import uuid
  24 import subprocess
  25 import tempfile
  26 from collections import OrderedDict
  27
  28 import Common.LongFilePathOs as os
  29 from Common import EdkLogger as EdkLogger
  30 from Common import GlobalData as GlobalData
  31 from Common.DataType import *
  32 from Common.BuildToolError import *
  33 from CommonDataClass.DataClass import *
  34 from Common.Parsing import GetSplitValueList
  35 from Common.LongFilePathSupport import OpenLongFilePath as open
  36 from Common.MultipleWorkspace import MultipleWorkspace as mws
  37 from CommonDataClass.Exceptions import BadExpression
  38 from Common.caching import cached_property
  39
  40 ArrayIndex = re.compile("\[\s*[0-9a-fA-FxX]*\s*\]")
  41 ## Regular expression used to find out place holders in string template
  42 gPlaceholderPattern = re.compile("\$\{([^$()\s]+)\}", re.MULTILINE | re.UNICODE)
  43
  44 ## regular expressions for map file processing
  45 startPatternGeneral = re.compile("^Start[' ']+Length[' ']+Name[' ']+Class")
  46 addressPatternGeneral = re.compile("^Address[' ']+Publics by Value[' ']+Rva\+Base")
  47 valuePatternGcc = re.compile('^([\w_\.]+) +([\da-fA-Fx]+) +([\da-fA-Fx]+)$')
  48 pcdPatternGcc = re.compile('^([\da-fA-Fx]+) +([\da-fA-Fx]+)')
  49 secReGeneral = re.compile('^([\da-fA-F]+):([\da-fA-F]+) +([\da-fA-F]+)[Hh]? +([.\w\$]+) +(\w+)', re.UNICODE)
  50
  51 StructPattern = re.compile(r'[_a-zA-Z][0-9A-Za-z_]*$')
  52
  53 ## Dictionary used to store dependencies of files
  54 gDependencyDatabase = {}    # arch : {file path : [dependent files list]}
  55
  56 #
  57 # If a module is built more than once with different PCDs or library classes
  58 # a temporary INF file with same content is created, the temporary file is removed
  59 # when build exits.
  60 #
  61 _TempInfs = []
  62
  63 def GetVariableOffset(mapfilepath, efifilepath, varnames):
  64     """ Parse map file to get variable offset in current EFI file
  65     @param mapfilepath    Map file absolution path
  66     @param efifilepath:   EFI binary file full path
  67     @param varnames       iteratable container whose elements are variable names to be searched
  68
  69     @return List whos elements are tuple with variable name and raw offset
  70     """
  71     lines = []
  72     try:
  73         f = open(mapfilepath, 'r')
  74         lines = f.readlines()
  75         f.close()
  76     except:
  77         return None
  78
  79     if len(lines) == 0: return None
  80     firstline = lines[0].strip()
  81     if (firstline.startswith("Archive member included ") and
  82         firstline.endswith(" file (symbol)")):
  83         return _parseForGCC(lines, efifilepath, varnames)
  84     if firstline.startswith("# Path:"):
  85         return _parseForXcode(lines, efifilepath, varnames)
  86     return _parseGeneral(lines, efifilepath, varnames)
  87
  88 def _parseForXcode(lines, efifilepath, varnames):
  89     status = 0
  90     ret = []
  91     for line in lines:
  92         line = line.strip()
  93         if status == 0 and line == "# Symbols:":
  94             status = 1
  95             continue
  96         if status == 1 and len(line) != 0:
  97             for varname in varnames:
  98                 if varname in line:
  99                     # cannot pregenerate this RegEx since it uses varname from varnames.
 100                     m = re.match('^([\da-fA-FxX]+)([\s\S]*)([_]*%s)$' % varname, line)
 101                     if m is not None:
 102                         ret.append((varname, m.group(1)))
 103     return ret
 104
 105 def _parseForGCC(lines, efifilepath, varnames):
 106     """ Parse map file generated by GCC linker """
 107     status = 0
 108     sections = []
 109     varoffset = []
 110     for index, line in enumerate(lines):
 111         line = line.strip()
 112         # status machine transection
 113         if status == 0 and line == "Memory Configuration":
 114             status = 1
 115             continue
 116         elif status == 1 and line == 'Linker script and memory map':
 117             status = 2
 118             continue
 119         elif status ==2 and line == 'START GROUP':
 120             status = 3
 121             continue
 122
 123         # status handler
 124         if status == 3:
 125             m = valuePatternGcc.match(line)
 126             if m is not None:
 127                 sections.append(m.groups(0))
 128             for varname in varnames:
 129                 Str = ''
 130                 m = re.match("^.data.(%s)" % varname, line)
 131                 if m is not None:
 132                     m = re.match(".data.(%s)$" % varname, line)
 133                     if m is not None:
 134                         Str = lines[index + 1]
 135                     else:
 136                         Str = line[len(".data.%s" % varname):]
 137                     if Str:
 138                         m = pcdPatternGcc.match(Str.strip())
 139                         if m is not None:
 140                             varoffset.append((varname, int(m.groups(0)[0], 16), int(sections[-1][1], 16), sections[-1][0]))
 141
 142     if not varoffset:
 143         return []
 144     # get section information from efi file
 145     efisecs = PeImageClass(efifilepath).SectionHeaderList
 146     if efisecs is None or len(efisecs) == 0:
 147         return []
 148     #redirection
 149     redirection = 0
 150     for efisec in efisecs:
 151         for section in sections:
 152             if section[0].strip() == efisec[0].strip() and section[0].strip() == '.text':
 153                 redirection = int(section[1], 16) - efisec[1]
 154
 155     ret = []
 156     for var in varoffset:
 157         for efisec in efisecs:
 158             if var[1] >= efisec[1] and var[1] < efisec[1]+efisec[3]:
 159                 ret.append((var[0], hex(efisec[2] + var[1] - efisec[1] - redirection)))
 160     return ret
 161
 162 def _parseGeneral(lines, efifilepath, varnames):
 163     status = 0    #0 - beginning of file; 1 - PE section definition; 2 - symbol table
 164     secs  = []    # key = section name
 165     varoffset = []
 166     symRe = re.compile('^([\da-fA-F]+):([\da-fA-F]+) +([\.:\\\\\w\?@\$]+) +([\da-fA-F]+)', re.UNICODE)
 167
 168     for line in lines:
 169         line = line.strip()
 170         if startPatternGeneral.match(line):
 171             status = 1
 172             continue
 173         if addressPatternGeneral.match(line):
 174             status = 2
 175             continue
 176         if line.startswith("entry point at"):
 177             status = 3
 178             continue
 179         if status == 1 and len(line) != 0:
 180             m =  secReGeneral.match(line)
 181             assert m is not None, "Fail to parse the section in map file , line is %s" % line
 182             sec_no, sec_start, sec_length, sec_name, sec_class = m.groups(0)
 183             secs.append([int(sec_no, 16), int(sec_start, 16), int(sec_length, 16), sec_name, sec_class])
 184         if status == 2 and len(line) != 0:
 185             for varname in varnames:
 186                 m = symRe.match(line)
 187                 assert m is not None, "Fail to parse the symbol in map file, line is %s" % line
 188                 sec_no, sym_offset, sym_name, vir_addr = m.groups(0)
 189                 sec_no     = int(sec_no,     16)
 190                 sym_offset = int(sym_offset, 16)
 191                 vir_addr   = int(vir_addr,   16)
 192                 # cannot pregenerate this RegEx since it uses varname from varnames.
 193                 m2 = re.match('^[_]*(%s)' % varname, sym_name)
 194                 if m2 is not None:
 195                     # fond a binary pcd entry in map file
 196                     for sec in secs:
 197                         if sec[0] == sec_no and (sym_offset >= sec[1] and sym_offset < sec[1] + sec[2]):
 198                             varoffset.append([varname, sec[3], sym_offset, vir_addr, sec_no])
 199
 200     if not varoffset: return []
 201
 202     # get section information from efi file
 203     efisecs = PeImageClass(efifilepath).SectionHeaderList
 204     if efisecs is None or len(efisecs) == 0:
 205         return []
 206
 207     ret = []
 208     for var in varoffset:
 209         index = 0
 210         for efisec in efisecs:
 211             index = index + 1
 212             if var[1].strip() == efisec[0].strip():
 213                 ret.append((var[0], hex(efisec[2] + var[2])))
 214             elif var[4] == index:
 215                 ret.append((var[0], hex(efisec[2] + var[2])))
 216
 217     return ret
 218
 219 ## Routine to process duplicated INF
 220 #
 221 #  This function is called by following two cases:
 222 #  Case 1 in DSC:
 223 #    [components.arch]
 224 #    Pkg/module/module.inf
 225 #    Pkg/module/module.inf {
 226 #      <Defines>
 227 #        FILE_GUID = 0D1B936F-68F3-4589-AFCC-FB8B7AEBC836
 228 #    }
 229 #  Case 2 in FDF:
 230 #    INF Pkg/module/module.inf
 231 #    INF FILE_GUID = 0D1B936F-68F3-4589-AFCC-FB8B7AEBC836 Pkg/module/module.inf
 232 #
 233 #  This function copies Pkg/module/module.inf to
 234 #  Conf/.cache/0D1B936F-68F3-4589-AFCC-FB8B7AEBC836module.inf
 235 #
 236 #  @param Path     Original PathClass object
 237 #  @param BaseName New file base name
 238 #
 239 #  @retval         return the new PathClass object
 240 #
 241 def ProcessDuplicatedInf(Path, BaseName, Workspace):
 242     Filename = os.path.split(Path.File)[1]
 243     if '.' in Filename:
 244         Filename = BaseName + Path.BaseName + Filename[Filename.rfind('.'):]
 245     else:
 246         Filename = BaseName + Path.BaseName
 247
 248     #
 249     # If -N is specified on command line, cache is disabled
 250     # The directory has to be created
 251     #
 252     DbDir = os.path.split(GlobalData.gDatabasePath)[0]
 253     if not os.path.exists(DbDir):
 254         os.makedirs(DbDir)
 255     #
 256     # A temporary INF is copied to database path which must have write permission
 257     # The temporary will be removed at the end of build
 258     # In case of name conflict, the file name is
 259     # FILE_GUIDBaseName (0D1B936F-68F3-4589-AFCC-FB8B7AEBC836module.inf)
 260     #
 261     TempFullPath = os.path.join(DbDir,
 262                                 Filename)
 263     RtPath = PathClass(Path.File, Workspace)
 264     #
 265     # Modify the full path to temporary path, keep other unchanged
 266     #
 267     # To build same module more than once, the module path with FILE_GUID overridden has
 268     # the file name FILE_GUIDmodule.inf, but the relative path (self.MetaFile.File) is the real path
 269     # in DSC which is used as relative path by C files and other files in INF.
 270     # A trick was used: all module paths are PathClass instances, after the initialization
 271     # of PathClass, the PathClass.Path is overridden by the temporary INF path.
 272     #
 273     # The reason for creating a temporary INF is:
 274     # Platform.Modules which is the base to create ModuleAutoGen objects is a dictionary,
 275     # the key is the full path of INF, the value is an object to save overridden library instances, PCDs.
 276     # A different key for the same module is needed to create different output directory,
 277     # retrieve overridden PCDs, library instances.
 278     #
 279     # The BaseName is the FILE_GUID which is also the output directory name.
 280     #
 281     #
 282     RtPath.Path = TempFullPath
 283     RtPath.BaseName = BaseName
 284     RtPath.OriginalPath = Path
 285     #
 286     # If file exists, compare contents
 287     #
 288     if os.path.exists(TempFullPath):
 289         with open(str(Path), 'rb') as f1, open(TempFullPath, 'rb') as f2:
 290             if f1.read() == f2.read():
 291                 return RtPath
 292     _TempInfs.append(TempFullPath)
 293     shutil.copy2(str(Path), TempFullPath)
 294     return RtPath
 295
 296 ## Remove temporary created INFs whose paths were saved in _TempInfs
 297 #
 298 def ClearDuplicatedInf():
 299     while _TempInfs:
 300         File = _TempInfs.pop()
 301         if os.path.exists(File):
 302             os.remove(File)
 303
 304 ## Convert GUID string in xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx style to C structure style
 305 #
 306 #   @param      Guid    The GUID string
 307 #
 308 #   @retval     string  The GUID string in C structure style
 309 #
 310 def GuidStringToGuidStructureString(Guid):
 311     GuidList = Guid.split('-')
 312     Result = '{'
 313     for Index in range(0, 3, 1):
 314         Result = Result + '0x' + GuidList[Index] + ', '
 315     Result = Result + '{0x' + GuidList[3][0:2] + ', 0x' + GuidList[3][2:4]
 316     for Index in range(0, 12, 2):
 317         Result = Result + ', 0x' + GuidList[4][Index:Index + 2]
 318     Result += '}}'
 319     return Result
 320
 321 ## Convert GUID structure in byte array to xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
 322 #
 323 #   @param      GuidValue   The GUID value in byte array
 324 #
 325 #   @retval     string      The GUID value in xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx format
 326 #
 327 def GuidStructureByteArrayToGuidString(GuidValue):
 328     guidValueString = GuidValue.lower().replace("{", "").replace("}", "").replace(" ", "").replace(";", "")
 329     guidValueList = guidValueString.split(",")
 330     if len(guidValueList) != 16:
 331         return ''
 332         #EdkLogger.error(None, None, "Invalid GUID value string %s" % GuidValue)
 333     try:
 334         return "%02x%02x%02x%02x-%02x%02x-%02x%02x-%02x%02x-%02x%02x%02x%02x%02x%02x" % (
 335                 int(guidValueList[3], 16),
 336                 int(guidValueList[2], 16),
 337                 int(guidValueList[1], 16),
 338                 int(guidValueList[0], 16),
 339                 int(guidValueList[5], 16),
 340                 int(guidValueList[4], 16),
 341                 int(guidValueList[7], 16),
 342                 int(guidValueList[6], 16),
 343                 int(guidValueList[8], 16),
 344                 int(guidValueList[9], 16),
 345                 int(guidValueList[10], 16),
 346                 int(guidValueList[11], 16),
 347                 int(guidValueList[12], 16),
 348                 int(guidValueList[13], 16),
 349                 int(guidValueList[14], 16),
 350                 int(guidValueList[15], 16)
 351                 )
 352     except:
 353         return ''
 354
 355 ## Convert GUID string in C structure style to xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
 356 #
 357 #   @param      GuidValue   The GUID value in C structure format
 358 #
 359 #   @retval     string      The GUID value in xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx format
 360 #
 361 def GuidStructureStringToGuidString(GuidValue):
 362     if not GlobalData.gGuidCFormatPattern.match(GuidValue):
 363         return ''
 364     guidValueString = GuidValue.lower().replace("{", "").replace("}", "").replace(" ", "").replace(";", "")
 365     guidValueList = guidValueString.split(",")
 366     if len(guidValueList) != 11:
 367         return ''
 368         #EdkLogger.error(None, None, "Invalid GUID value string %s" % GuidValue)
 369     try:
 370         return "%08x-%04x-%04x-%02x%02x-%02x%02x%02x%02x%02x%02x" % (
 371                 int(guidValueList[0], 16),
 372                 int(guidValueList[1], 16),
 373                 int(guidValueList[2], 16),
 374                 int(guidValueList[3], 16),
 375                 int(guidValueList[4], 16),
 376                 int(guidValueList[5], 16),
 377                 int(guidValueList[6], 16),
 378                 int(guidValueList[7], 16),
 379                 int(guidValueList[8], 16),
 380                 int(guidValueList[9], 16),
 381                 int(guidValueList[10], 16)
 382                 )
 383     except:
 384         return ''
 385
 386 ## Convert GUID string in C structure style to xxxxxxxx_xxxx_xxxx_xxxx_xxxxxxxxxxxx
 387 #
 388 #   @param      GuidValue   The GUID value in C structure format
 389 #
 390 #   @retval     string      The GUID value in xxxxxxxx_xxxx_xxxx_xxxx_xxxxxxxxxxxx format
 391 #
 392 def GuidStructureStringToGuidValueName(GuidValue):
 393     guidValueString = GuidValue.lower().replace("{", "").replace("}", "").replace(" ", "")
 394     guidValueList = guidValueString.split(",")
 395     if len(guidValueList) != 11:
 396         EdkLogger.error(None, FORMAT_INVALID, "Invalid GUID value string [%s]" % GuidValue)
 397     return "%08x_%04x_%04x_%02x%02x_%02x%02x%02x%02x%02x%02x" % (
 398             int(guidValueList[0], 16),
 399             int(guidValueList[1], 16),
 400             int(guidValueList[2], 16),
 401             int(guidValueList[3], 16),
 402             int(guidValueList[4], 16),
 403             int(guidValueList[5], 16),
 404             int(guidValueList[6], 16),
 405             int(guidValueList[7], 16),
 406             int(guidValueList[8], 16),
 407             int(guidValueList[9], 16),
 408             int(guidValueList[10], 16)
 409             )
 410
 411 ## Create directories
 412 #
 413 #   @param      Directory   The directory name
 414 #
 415 def CreateDirectory(Directory):
 416     if Directory is None or Directory.strip() == "":
 417         return True
 418     try:
 419         if not os.access(Directory, os.F_OK):
 420             os.makedirs(Directory)
 421     except:
 422         return False
 423     return True
 424
 425 ## Remove directories, including files and sub-directories in it
 426 #
 427 #   @param      Directory   The directory name
 428 #
 429 def RemoveDirectory(Directory, Recursively=False):
 430     if Directory is None or Directory.strip() == "" or not os.path.exists(Directory):
 431         return
 432     if Recursively:
 433         CurrentDirectory = os.getcwd()
 434         os.chdir(Directory)
 435         for File in os.listdir("."):
 436             if os.path.isdir(File):
 437                 RemoveDirectory(File, Recursively)
 438             else:
 439                 os.remove(File)
 440         os.chdir(CurrentDirectory)
 441     os.rmdir(Directory)
 442
 443 ## Store content in file
 444 #
 445 #  This method is used to save file only when its content is changed. This is
 446 #  quite useful for "make" system to decide what will be re-built and what won't.
 447 #
 448 #   @param      File            The path of file
 449 #   @param      Content         The new content of the file
 450 #   @param      IsBinaryFile    The flag indicating if the file is binary file or not
 451 #
 452 #   @retval     True            If the file content is changed and the file is renewed
 453 #   @retval     False           If the file content is the same
 454 #
 455 def SaveFileOnChange(File, Content, IsBinaryFile=True):
 456
 457     if os.path.exists(File):
 458         if IsBinaryFile:
 459             try:
 460                 with open(File, "rb") as f:
 461                     if Content == f.read():
 462                         return False
 463             except:
 464                 EdkLogger.error(None, FILE_OPEN_FAILURE, ExtraData=File)
 465         else:
 466             try:
 467                 with open(File, "r") as f:
 468                     if Content == f.read():
 469                         return False
 470             except:
 471                 EdkLogger.error(None, FILE_OPEN_FAILURE, ExtraData=File)
 472
 473     DirName = os.path.dirname(File)
 474     if not CreateDirectory(DirName):
 475         EdkLogger.error(None, FILE_CREATE_FAILURE, "Could not create directory %s" % DirName)
 476     else:
 477         if DirName == '':
 478             DirName = os.getcwd()
 479         if not os.access(DirName, os.W_OK):
 480             EdkLogger.error(None, PERMISSION_FAILURE, "Do not have write permission on directory %s" % DirName)
 481
 482     OpenMode = "w"
 483     if IsBinaryFile:
 484         OpenMode = "wb"
 485
 486     if GlobalData.gIsWindows and not os.path.exists(File):
 487         # write temp file, then rename the temp file to the real file
 488         # to make sure the file be immediate saved to disk
 489         with tempfile.NamedTemporaryFile(OpenMode, dir=os.path.dirname(File), delete=False) as tf:
 490             tf.write(Content)
 491             tempname = tf.name
 492         try:
 493             os.rename(tempname, File)
 494         except:
 495             EdkLogger.error(None, FILE_CREATE_FAILURE, ExtraData='IOError %s' % X)
 496     else:
 497         try:
 498             with open(File, OpenMode) as Fd:
 499                 Fd.write(Content)
 500         except IOError as X:
 501             EdkLogger.error(None, FILE_CREATE_FAILURE, ExtraData='IOError %s' % X)
 502
 503     return True
 504
 505 ## Retrieve and cache the real path name in file system
 506 #
 507 #   @param      Root    The root directory of path relative to
 508 #
 509 #   @retval     str     The path string if the path exists
 510 #   @retval     None    If path doesn't exist
 511 #
 512 class DirCache:
 513     _CACHE_ = set()
 514     _UPPER_CACHE_ = {}
 515
 516     def __init__(self, Root):
 517         self._Root = Root
 518         for F in os.listdir(Root):
 519             self._CACHE_.add(F)
 520             self._UPPER_CACHE_[F.upper()] = F
 521
 522     # =[] operator
 523     def __getitem__(self, Path):
 524         Path = Path[len(os.path.commonprefix([Path, self._Root])):]
 525         if not Path:
 526             return self._Root
 527         if Path and Path[0] == os.path.sep:
 528             Path = Path[1:]
 529         if Path in self._CACHE_:
 530             return os.path.join(self._Root, Path)
 531         UpperPath = Path.upper()
 532         if UpperPath in self._UPPER_CACHE_:
 533             return os.path.join(self._Root, self._UPPER_CACHE_[UpperPath])
 534
 535         IndexList = []
 536         LastSepIndex = -1
 537         SepIndex = Path.find(os.path.sep)
 538         while SepIndex > -1:
 539             Parent = UpperPath[:SepIndex]
 540             if Parent not in self._UPPER_CACHE_:
 541                 break
 542             LastSepIndex = SepIndex
 543             SepIndex = Path.find(os.path.sep, LastSepIndex + 1)
 544
 545         if LastSepIndex == -1:
 546             return None
 547
 548         Cwd = os.getcwd()
 549         os.chdir(self._Root)
 550         SepIndex = LastSepIndex
 551         while SepIndex > -1:
 552             Parent = Path[:SepIndex]
 553             ParentKey = UpperPath[:SepIndex]
 554             if ParentKey not in self._UPPER_CACHE_:
 555                 os.chdir(Cwd)
 556                 return None
 557
 558             if Parent in self._CACHE_:
 559                 ParentDir = Parent
 560             else:
 561                 ParentDir = self._UPPER_CACHE_[ParentKey]
 562             for F in os.listdir(ParentDir):
 563                 Dir = os.path.join(ParentDir, F)
 564                 self._CACHE_.add(Dir)
 565                 self._UPPER_CACHE_[Dir.upper()] = Dir
 566
 567             SepIndex = Path.find(os.path.sep, SepIndex + 1)
 568
 569         os.chdir(Cwd)
 570         if Path in self._CACHE_:
 571             return os.path.join(self._Root, Path)
 572         elif UpperPath in self._UPPER_CACHE_:
 573             return os.path.join(self._Root, self._UPPER_CACHE_[UpperPath])
 574         return None
 575
 576 def RealPath(File, Dir='', OverrideDir=''):
 577     NewFile = os.path.normpath(os.path.join(Dir, File))
 578     NewFile = GlobalData.gAllFiles[NewFile]
 579     if not NewFile and OverrideDir:
 580         NewFile = os.path.normpath(os.path.join(OverrideDir, File))
 581         NewFile = GlobalData.gAllFiles[NewFile]
 582     return NewFile
 583
 584 ## Get GUID value from given packages
 585 #
 586 #   @param      CName           The CName of the GUID
 587 #   @param      PackageList     List of packages looking-up in
 588 #   @param      Inffile         The driver file
 589 #
 590 #   @retval     GuidValue   if the CName is found in any given package
 591 #   @retval     None        if the CName is not found in all given packages
 592 #
 593 def GuidValue(CName, PackageList, Inffile = None):
 594     for P in PackageList:
 595         GuidKeys = list(P.Guids.keys())
 596         if Inffile and P._PrivateGuids:
 597             if not Inffile.startswith(P.MetaFile.Dir):
 598                 GuidKeys = [x for x in P.Guids if x not in P._PrivateGuids]
 599         if CName in GuidKeys:
 600             return P.Guids[CName]
 601     return None
 602     return None
 603
 604 ## A string template class
 605 #
 606 #  This class implements a template for string replacement. A string template
 607 #  looks like following
 608 #
 609 #       ${BEGIN} other_string ${placeholder_name} other_string ${END}
 610 #
 611 #  The string between ${BEGIN} and ${END} will be repeated as many times as the
 612 #  length of "placeholder_name", which is a list passed through a dict. The
 613 #  "placeholder_name" is the key name of the dict. The ${BEGIN} and ${END} can
 614 #  be not used and, in this case, the "placeholder_name" must not a list and it
 615 #  will just be replaced once.
 616 #
 617 class TemplateString(object):
 618     _REPEAT_START_FLAG = "BEGIN"
 619     _REPEAT_END_FLAG = "END"
 620
 621     class Section(object):
 622         _LIST_TYPES = [type([]), type(set()), type((0,))]
 623
 624         def __init__(self, TemplateSection, PlaceHolderList):
 625             self._Template = TemplateSection
 626             self._PlaceHolderList = []
 627
 628             # Split the section into sub-sections according to the position of placeholders
 629             if PlaceHolderList:
 630                 self._SubSectionList = []
 631                 SubSectionStart = 0
 632                 #
 633                 # The placeholders passed in must be in the format of
 634                 #
 635                 #   PlaceHolderName, PlaceHolderStartPoint, PlaceHolderEndPoint
 636                 #
 637                 for PlaceHolder, Start, End in PlaceHolderList:
 638                     self._SubSectionList.append(TemplateSection[SubSectionStart:Start])
 639                     self._SubSectionList.append(TemplateSection[Start:End])
 640                     self._PlaceHolderList.append(PlaceHolder)
 641                     SubSectionStart = End
 642                 if SubSectionStart < len(TemplateSection):
 643                     self._SubSectionList.append(TemplateSection[SubSectionStart:])
 644             else:
 645                 self._SubSectionList = [TemplateSection]
 646
 647         def __str__(self):
 648             return self._Template + " : " + str(self._PlaceHolderList)
 649
 650         def Instantiate(self, PlaceHolderValues):
 651             RepeatTime = -1
 652             RepeatPlaceHolders = {}
 653             NonRepeatPlaceHolders = {}
 654
 655             for PlaceHolder in self._PlaceHolderList:
 656                 if PlaceHolder not in PlaceHolderValues:
 657                     continue
 658                 Value = PlaceHolderValues[PlaceHolder]
 659                 if type(Value) in self._LIST_TYPES:
 660                     if RepeatTime < 0:
 661                         RepeatTime = len(Value)
 662                     elif RepeatTime != len(Value):
 663                         EdkLogger.error(
 664                                     "TemplateString",
 665                                     PARAMETER_INVALID,
 666                                     "${%s} has different repeat time from others!" % PlaceHolder,
 667                                     ExtraData=str(self._Template)
 668                                     )
 669                     RepeatPlaceHolders["${%s}" % PlaceHolder] = Value
 670                 else:
 671                     NonRepeatPlaceHolders["${%s}" % PlaceHolder] = Value
 672
 673             if NonRepeatPlaceHolders:
 674                 StringList = []
 675                 for S in self._SubSectionList:
 676                     if S not in NonRepeatPlaceHolders:
 677                         StringList.append(S)
 678                     else:
 679                         StringList.append(str(NonRepeatPlaceHolders[S]))
 680             else:
 681                 StringList = self._SubSectionList
 682
 683             if RepeatPlaceHolders:
 684                 TempStringList = []
 685                 for Index in range(RepeatTime):
 686                     for S in StringList:
 687                         if S not in RepeatPlaceHolders:
 688                             TempStringList.append(S)
 689                         else:
 690                             TempStringList.append(str(RepeatPlaceHolders[S][Index]))
 691                 StringList = TempStringList
 692
 693             return "".join(StringList)
 694
 695     ## Constructor
 696     def __init__(self, Template=None):
 697         self.String = []
 698         self.IsBinary = False
 699         self._Template = Template
 700         self._TemplateSectionList = self._Parse(Template)
 701
 702     ## str() operator
 703     #
 704     #   @retval     string  The string replaced
 705     #
 706     def __str__(self):
 707         return "".join(self.String)
 708
 709     ## Split the template string into fragments per the ${BEGIN} and ${END} flags
 710     #
 711     #   @retval     list    A list of TemplateString.Section objects
 712     #
 713     def _Parse(self, Template):
 714         SectionStart = 0
 715         SearchFrom = 0
 716         MatchEnd = 0
 717         PlaceHolderList = []
 718         TemplateSectionList = []
 719         while Template:
 720             MatchObj = gPlaceholderPattern.search(Template, SearchFrom)
 721             if not MatchObj:
 722                 if MatchEnd <= len(Template):
 723                     TemplateSection = TemplateString.Section(Template[SectionStart:], PlaceHolderList)
 724                     TemplateSectionList.append(TemplateSection)
 725                 break
 726
 727             MatchString = MatchObj.group(1)
 728             MatchStart = MatchObj.start()
 729             MatchEnd = MatchObj.end()
 730
 731             if MatchString == self._REPEAT_START_FLAG:
 732                 if MatchStart > SectionStart:
 733                     TemplateSection = TemplateString.Section(Template[SectionStart:MatchStart], PlaceHolderList)
 734                     TemplateSectionList.append(TemplateSection)
 735                 SectionStart = MatchEnd
 736                 PlaceHolderList = []
 737             elif MatchString == self._REPEAT_END_FLAG:
 738                 TemplateSection = TemplateString.Section(Template[SectionStart:MatchStart], PlaceHolderList)
 739                 TemplateSectionList.append(TemplateSection)
 740                 SectionStart = MatchEnd
 741                 PlaceHolderList = []
 742             else:
 743                 PlaceHolderList.append((MatchString, MatchStart - SectionStart, MatchEnd - SectionStart))
 744             SearchFrom = MatchEnd
 745         return TemplateSectionList
 746
 747     ## Replace the string template with dictionary of placeholders and append it to previous one
 748     #
 749     #   @param      AppendString    The string template to append
 750     #   @param      Dictionary      The placeholder dictionaries
 751     #
 752     def Append(self, AppendString, Dictionary=None):
 753         if Dictionary:
 754             SectionList = self._Parse(AppendString)
 755             self.String.append( "".join(S.Instantiate(Dictionary) for S in SectionList))
 756         else:
 757             if isinstance(AppendString,list):
 758                 self.String.extend(AppendString)
 759             else:
 760                 self.String.append(AppendString)
 761
 762     ## Replace the string template with dictionary of placeholders
 763     #
 764     #   @param      Dictionary      The placeholder dictionaries
 765     #
 766     #   @retval     str             The string replaced with placeholder values
 767     #
 768     def Replace(self, Dictionary=None):
 769         return "".join(S.Instantiate(Dictionary) for S in self._TemplateSectionList)
 770
 771 ## Progress indicator class
 772 #
 773 #  This class makes use of thread to print progress on console.
 774 #
 775 class Progressor:
 776     # for avoiding deadloop
 777     _StopFlag = None
 778     _ProgressThread = None
 779     _CheckInterval = 0.25
 780
 781     ## Constructor
 782     #
 783     #   @param      OpenMessage     The string printed before progress characters
 784     #   @param      CloseMessage    The string printed after progress characters
 785     #   @param      ProgressChar    The character used to indicate the progress
 786     #   @param      Interval        The interval in seconds between two progress characters
 787     #
 788     def __init__(self, OpenMessage="", CloseMessage="", ProgressChar='.', Interval=1.0):
 789         self.PromptMessage = OpenMessage
 790         self.CodaMessage = CloseMessage
 791         self.ProgressChar = ProgressChar
 792         self.Interval = Interval
 793         if Progressor._StopFlag is None:
 794             Progressor._StopFlag = threading.Event()
 795
 796     ## Start to print progress character
 797     #
 798     #   @param      OpenMessage     The string printed before progress characters
 799     #
 800     def Start(self, OpenMessage=None):
 801         if OpenMessage is not None:
 802             self.PromptMessage = OpenMessage
 803         Progressor._StopFlag.clear()
 804         if Progressor._ProgressThread is None:
 805             Progressor._ProgressThread = threading.Thread(target=self._ProgressThreadEntry)
 806             Progressor._ProgressThread.setDaemon(False)
 807             Progressor._ProgressThread.start()
 808
 809     ## Stop printing progress character
 810     #
 811     #   @param      CloseMessage    The string printed after progress characters
 812     #
 813     def Stop(self, CloseMessage=None):
 814         OriginalCodaMessage = self.CodaMessage
 815         if CloseMessage is not None:
 816             self.CodaMessage = CloseMessage
 817         self.Abort()
 818         self.CodaMessage = OriginalCodaMessage
 819
 820     ## Thread entry method
 821     def _ProgressThreadEntry(self):
 822         sys.stdout.write(self.PromptMessage + " ")
 823         sys.stdout.flush()
 824         TimeUp = 0.0
 825         while not Progressor._StopFlag.isSet():
 826             if TimeUp <= 0.0:
 827                 sys.stdout.write(self.ProgressChar)
 828                 sys.stdout.flush()
 829                 TimeUp = self.Interval
 830             time.sleep(self._CheckInterval)
 831             TimeUp -= self._CheckInterval
 832         sys.stdout.write(" " + self.CodaMessage + "\n")
 833         sys.stdout.flush()
 834
 835     ## Abort the progress display
 836     @staticmethod
 837     def Abort():
 838         if Progressor._StopFlag is not None:
 839             Progressor._StopFlag.set()
 840         if Progressor._ProgressThread is not None:
 841             Progressor._ProgressThread.join()
 842             Progressor._ProgressThread = None
 843
 844
 845 ## Dictionary using prioritized list as key
 846 #
 847 class tdict:
 848     _ListType = type([])
 849     _TupleType = type(())
 850     _Wildcard = 'COMMON'
 851     _ValidWildcardList = ['COMMON', 'DEFAULT', 'ALL', TAB_STAR, 'PLATFORM']
 852
 853     def __init__(self, _Single_=False, _Level_=2):
 854         self._Level_ = _Level_
 855         self.data = {}
 856         self._Single_ = _Single_
 857
 858     # =[] operator
 859     def __getitem__(self, key):
 860         KeyType = type(key)
 861         RestKeys = None
 862         if KeyType == self._ListType or KeyType == self._TupleType:
 863             FirstKey = key[0]
 864             if len(key) > 1:
 865                 RestKeys = key[1:]
 866             elif self._Level_ > 1:
 867                 RestKeys = [self._Wildcard for i in range(0, self._Level_ - 1)]
 868         else:
 869             FirstKey = key
 870             if self._Level_ > 1:
 871                 RestKeys = [self._Wildcard for i in range(0, self._Level_ - 1)]
 872
 873         if FirstKey is None or str(FirstKey).upper() in self._ValidWildcardList:
 874             FirstKey = self._Wildcard
 875
 876         if self._Single_:
 877             return self._GetSingleValue(FirstKey, RestKeys)
 878         else:
 879             return self._GetAllValues(FirstKey, RestKeys)
 880
 881     def _GetSingleValue(self, FirstKey, RestKeys):
 882         Value = None
 883         #print "%s-%s" % (FirstKey, self._Level_) ,
 884         if self._Level_ > 1:
 885             if FirstKey == self._Wildcard:
 886                 if FirstKey in self.data:
 887                     Value = self.data[FirstKey][RestKeys]
 888                 if Value is None:
 889                     for Key in self.data:
 890                         Value = self.data[Key][RestKeys]
 891                         if Value is not None: break
 892             else:
 893                 if FirstKey in self.data:
 894                     Value = self.data[FirstKey][RestKeys]
 895                 if Value is None and self._Wildcard in self.data:
 896                     #print "Value=None"
 897                     Value = self.data[self._Wildcard][RestKeys]
 898         else:
 899             if FirstKey == self._Wildcard:
 900                 if FirstKey in self.data:
 901                     Value = self.data[FirstKey]
 902                 if Value is None:
 903                     for Key in self.data:
 904                         Value = self.data[Key]
 905                         if Value is not None: break
 906             else:
 907                 if FirstKey in self.data:
 908                     Value = self.data[FirstKey]
 909                 elif self._Wildcard in self.data:
 910                     Value = self.data[self._Wildcard]
 911         return Value
 912
 913     def _GetAllValues(self, FirstKey, RestKeys):
 914         Value = []
 915         if self._Level_ > 1:
 916             if FirstKey == self._Wildcard:
 917                 for Key in self.data:
 918                     Value += self.data[Key][RestKeys]
 919             else:
 920                 if FirstKey in self.data:
 921                     Value += self.data[FirstKey][RestKeys]
 922                 if self._Wildcard in self.data:
 923                     Value += self.data[self._Wildcard][RestKeys]
 924         else:
 925             if FirstKey == self._Wildcard:
 926                 for Key in self.data:
 927                     Value.append(self.data[Key])
 928             else:
 929                 if FirstKey in self.data:
 930                     Value.append(self.data[FirstKey])
 931                 if self._Wildcard in self.data:
 932                     Value.append(self.data[self._Wildcard])
 933         return Value
 934
 935     ## []= operator
 936     def __setitem__(self, key, value):
 937         KeyType = type(key)
 938         RestKeys = None
 939         if KeyType == self._ListType or KeyType == self._TupleType:
 940             FirstKey = key[0]
 941             if len(key) > 1:
 942                 RestKeys = key[1:]
 943             else:
 944                 RestKeys = [self._Wildcard for i in range(0, self._Level_ - 1)]
 945         else:
 946             FirstKey = key
 947             if self._Level_ > 1:
 948                 RestKeys = [self._Wildcard for i in range(0, self._Level_ - 1)]
 949
 950         if FirstKey in self._ValidWildcardList:
 951             FirstKey = self._Wildcard
 952
 953         if FirstKey not in self.data and self._Level_ > 0:
 954             self.data[FirstKey] = tdict(self._Single_, self._Level_ - 1)
 955
 956         if self._Level_ > 1:
 957             self.data[FirstKey][RestKeys] = value
 958         else:
 959             self.data[FirstKey] = value
 960
 961     def SetGreedyMode(self):
 962         self._Single_ = False
 963         if self._Level_ > 1:
 964             for Key in self.data:
 965                 self.data[Key].SetGreedyMode()
 966
 967     def SetSingleMode(self):
 968         self._Single_ = True
 969         if self._Level_ > 1:
 970             for Key in self.data:
 971                 self.data[Key].SetSingleMode()
 972
 973     def GetKeys(self, KeyIndex=0):
 974         assert KeyIndex >= 0
 975         if KeyIndex == 0:
 976             return set(self.data.keys())
 977         else:
 978             keys = set()
 979             for Key in self.data:
 980                 keys |= self.data[Key].GetKeys(KeyIndex - 1)
 981             return keys
 982
 983 def AnalyzePcdExpression(Setting):
 984     RanStr = ''.join(sample(string.ascii_letters + string.digits, 8))
 985     Setting = Setting.replace('\\\\', RanStr).strip()
 986     # There might be escaped quote in a string: \", \\\" , \', \\\'
 987     Data = Setting
 988     # There might be '|' in string and in ( ... | ... ), replace it with '-'
 989     NewStr = ''
 990     InSingleQuoteStr = False
 991     InDoubleQuoteStr = False
 992     Pair = 0
 993     for Index, ch in enumerate(Data):
 994         if ch == '"' and not InSingleQuoteStr:
 995             if Data[Index - 1] != '\\':
 996                 InDoubleQuoteStr = not InDoubleQuoteStr
 997         elif ch == "'" and not InDoubleQuoteStr:
 998             if Data[Index - 1] != '\\':
 999                 InSingleQuoteStr = not InSingleQuoteStr
1000         elif ch == '(' and not (InSingleQuoteStr or InDoubleQuoteStr):
1001             Pair += 1
1002         elif ch == ')' and not (InSingleQuoteStr or InDoubleQuoteStr):
1003             Pair -= 1
1004
1005         if (Pair > 0 or InSingleQuoteStr or InDoubleQuoteStr) and ch == TAB_VALUE_SPLIT:
1006             NewStr += '-'
1007         else:
1008             NewStr += ch
1009     FieldList = []
1010     StartPos = 0
1011     while True:
1012         Pos = NewStr.find(TAB_VALUE_SPLIT, StartPos)
1013         if Pos < 0:
1014             FieldList.append(Setting[StartPos:].strip())
1015             break
1016         FieldList.append(Setting[StartPos:Pos].strip())
1017         StartPos = Pos + 1
1018     for i, ch in enumerate(FieldList):
1019         if RanStr in ch:
1020             FieldList[i] = ch.replace(RanStr,'\\\\')
1021     return FieldList
1022
1023 def ParseFieldValue (Value):
1024     def ParseDevPathValue (Value):
1025         if '\\' in Value:
1026             Value.replace('\\', '/').replace(' ', '')
1027
1028         Cmd = 'DevicePath ' + '"' + Value + '"'
1029         try:
1030             p = subprocess.Popen(Cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
1031             out, err = p.communicate()
1032         except Exception as X:
1033             raise BadExpression("DevicePath: %s" % (str(X)) )
1034         finally:
1035             subprocess._cleanup()
1036             p.stdout.close()
1037             p.stderr.close()
1038         if err:
1039             raise BadExpression("DevicePath: %s" % str(err))
1040         out = out.decode(encoding='utf-8', errors='ignore')
1041         Size = len(out.split())
1042         out = ','.join(out.split())
1043         return '{' + out + '}', Size
1044
1045     if "{CODE(" in Value:
1046         return Value, len(Value.split(","))
1047     if isinstance(Value, type(0)):
1048         return Value, (Value.bit_length() + 7) // 8
1049     if not isinstance(Value, type('')):
1050         raise BadExpression('Type %s is %s' %(Value, type(Value)))
1051     Value = Value.strip()
1052     if Value.startswith(TAB_UINT8) and Value.endswith(')'):
1053         Value, Size = ParseFieldValue(Value.split('(', 1)[1][:-1])
1054         if Size > 1:
1055             raise BadExpression('Value (%s) Size larger than %d' %(Value, Size))
1056         return Value, 1
1057     if Value.startswith(TAB_UINT16) and Value.endswith(')'):
1058         Value, Size = ParseFieldValue(Value.split('(', 1)[1][:-1])
1059         if Size > 2:
1060             raise BadExpression('Value (%s) Size larger than %d' %(Value, Size))
1061         return Value, 2
1062     if Value.startswith(TAB_UINT32) and Value.endswith(')'):
1063         Value, Size = ParseFieldValue(Value.split('(', 1)[1][:-1])
1064         if Size > 4:
1065             raise BadExpression('Value (%s) Size larger than %d' %(Value, Size))
1066         return Value, 4
1067     if Value.startswith(TAB_UINT64) and Value.endswith(')'):
1068         Value, Size = ParseFieldValue(Value.split('(', 1)[1][:-1])
1069         if Size > 8:
1070             raise BadExpression('Value (%s) Size larger than %d' % (Value, Size))
1071         return Value, 8
1072     if Value.startswith(TAB_GUID) and Value.endswith(')'):
1073         Value = Value.split('(', 1)[1][:-1].strip()
1074         if Value[0] == '{' and Value[-1] == '}':
1075             TmpValue = GuidStructureStringToGuidString(Value)
1076             if not TmpValue:
1077                 raise BadExpression("Invalid GUID value string %s" % Value)
1078             Value = TmpValue
1079         if Value[0] == '"' and Value[-1] == '"':
1080             Value = Value[1:-1]
1081         try:
1082             Value = str(uuid.UUID(Value).bytes_le)
1083             if Value.startswith("b'"):
1084                 Value = Value[2:-1]
1085             Value = "'" + Value + "'"
1086         except ValueError as Message:
1087             raise BadExpression(Message)
1088         Value, Size = ParseFieldValue(Value)
1089         return Value, 16
1090     if Value.startswith('L"') and Value.endswith('"'):
1091         # Unicode String
1092         # translate escape character
1093         Value = Value[1:]
1094         try:
1095             Value = eval(Value)
1096         except:
1097             Value = Value[1:-1]
1098         List = list(Value)
1099         List.reverse()
1100         Value = 0
1101         for Char in List:
1102             Value = (Value << 16) | ord(Char)
1103         return Value, (len(List) + 1) * 2
1104     if Value.startswith('"') and Value.endswith('"'):
1105         # ASCII String
1106         # translate escape character
1107         try:
1108             Value = eval(Value)
1109         except:
1110             Value = Value[1:-1]
1111         List = list(Value)
1112         List.reverse()
1113         Value = 0
1114         for Char in List:
1115             Value = (Value << 8) | ord(Char)
1116         return Value, len(List) + 1
1117     if Value.startswith("L'") and Value.endswith("'"):
1118         # Unicode Character Constant
1119         # translate escape character
1120         Value = Value[1:]
1121         try:
1122             Value = eval(Value)
1123         except:
1124             Value = Value[1:-1]
1125         List = list(Value)
1126         if len(List) == 0:
1127             raise BadExpression('Length %s is %s' % (Value, len(List)))
1128         List.reverse()
1129         Value = 0
1130         for Char in List:
1131             Value = (Value << 16) | ord(Char)
1132         return Value, len(List) * 2
1133     if Value.startswith("'") and Value.endswith("'"):
1134         # Character constant
1135         # translate escape character
1136         try:
1137             Value = eval(Value)
1138         except:
1139             Value = Value[1:-1]
1140         List = list(Value)
1141         if len(List) == 0:
1142             raise BadExpression('Length %s is %s' % (Value, len(List)))
1143         List.reverse()
1144         Value = 0
1145         for Char in List:
1146             Value = (Value << 8) | ord(Char)
1147         return Value, len(List)
1148     if Value.startswith('{') and Value.endswith('}'):
1149         # Byte array
1150         Value = Value[1:-1]
1151         List = [Item.strip() for Item in Value.split(',')]
1152         List.reverse()
1153         Value = 0
1154         RetSize = 0
1155         for Item in List:
1156             ItemValue, Size = ParseFieldValue(Item)
1157             RetSize += Size
1158             for I in range(Size):
1159                 Value = (Value << 8) | ((ItemValue >> 8 * I) & 0xff)
1160         return Value, RetSize
1161     if Value.startswith('DEVICE_PATH(') and Value.endswith(')'):
1162         Value = Value.replace("DEVICE_PATH(", '').rstrip(')')
1163         Value = Value.strip().strip('"')
1164         return ParseDevPathValue(Value)
1165     if Value.lower().startswith('0x'):
1166         try:
1167             Value = int(Value, 16)
1168         except:
1169             raise BadExpression("invalid hex value: %s" % Value)
1170         if Value == 0:
1171             return 0, 1
1172         return Value, (Value.bit_length() + 7) // 8
1173     if Value[0].isdigit():
1174         Value = int(Value, 10)
1175         if Value == 0:
1176             return 0, 1
1177         return Value, (Value.bit_length() + 7) // 8
1178     if Value.lower() == 'true':
1179         return 1, 1
1180     if Value.lower() == 'false':
1181         return 0, 1
1182     return Value, 1
1183
1184 ## AnalyzeDscPcd
1185 #
1186 #  Analyze DSC PCD value, since there is no data type info in DSC
1187 #  This function is used to match functions (AnalyzePcdData) used for retrieving PCD value from database
1188 #  1. Feature flag: TokenSpace.PcdCName|PcdValue
1189 #  2. Fix and Patch:TokenSpace.PcdCName|PcdValue[|VOID*[|MaxSize]]
1190 #  3. Dynamic default:
1191 #     TokenSpace.PcdCName|PcdValue[|VOID*[|MaxSize]]
1192 #     TokenSpace.PcdCName|PcdValue
1193 #  4. Dynamic VPD:
1194 #     TokenSpace.PcdCName|VpdOffset[|VpdValue]
1195 #     TokenSpace.PcdCName|VpdOffset[|MaxSize[|VpdValue]]
1196 #  5. Dynamic HII:
1197 #     TokenSpace.PcdCName|HiiString|VariableGuid|VariableOffset[|HiiValue]
1198 #  PCD value needs to be located in such kind of string, and the PCD value might be an expression in which
1199 #    there might have "|" operator, also in string value.
1200 #
1201 #  @param Setting: String contain information described above with "TokenSpace.PcdCName|" stripped
1202 #  @param PcdType: PCD type: feature, fixed, dynamic default VPD HII
1203 #  @param DataType: The datum type of PCD: VOID*, UNIT, BOOL
1204 #  @retval:
1205 #    ValueList: A List contain fields described above
1206 #    IsValid:   True if conforming EBNF, otherwise False
1207 #    Index:     The index where PcdValue is in ValueList
1208 #
1209 def AnalyzeDscPcd(Setting, PcdType, DataType=''):
1210     FieldList = AnalyzePcdExpression(Setting)
1211
1212     IsValid = True
1213     if PcdType in (MODEL_PCD_FIXED_AT_BUILD, MODEL_PCD_PATCHABLE_IN_MODULE, MODEL_PCD_DYNAMIC_DEFAULT, MODEL_PCD_DYNAMIC_EX_DEFAULT):
1214         Value = FieldList[0]
1215         Size = ''
1216         if len(FieldList) > 1 and FieldList[1]:
1217             DataType = FieldList[1]
1218             if FieldList[1] != TAB_VOID and StructPattern.match(FieldList[1]) is None:
1219                 IsValid = False
1220         if len(FieldList) > 2:
1221             Size = FieldList[2]
1222         if IsValid:
1223             if DataType == "":
1224                 IsValid = (len(FieldList) <= 1)
1225             else:
1226                 IsValid = (len(FieldList) <= 3)
1227
1228         if Size:
1229             try:
1230                 int(Size, 16) if Size.upper().startswith("0X") else int(Size)
1231             except:
1232                 IsValid = False
1233                 Size = -1
1234         return [str(Value), DataType, str(Size)], IsValid, 0
1235     elif PcdType == MODEL_PCD_FEATURE_FLAG:
1236         Value = FieldList[0]
1237         Size = ''
1238         IsValid = (len(FieldList) <= 1)
1239         return [Value, DataType, str(Size)], IsValid, 0
1240     elif PcdType in (MODEL_PCD_DYNAMIC_VPD, MODEL_PCD_DYNAMIC_EX_VPD):
1241         VpdOffset = FieldList[0]
1242         Value = Size = ''
1243         if not DataType == TAB_VOID:
1244             if len(FieldList) > 1:
1245                 Value = FieldList[1]
1246         else:
1247             if len(FieldList) > 1:
1248                 Size = FieldList[1]
1249             if len(FieldList) > 2:
1250                 Value = FieldList[2]
1251         if DataType == "":
1252             IsValid = (len(FieldList) <= 1)
1253         else:
1254             IsValid = (len(FieldList) <= 3)
1255         if Size:
1256             try:
1257                 int(Size, 16) if Size.upper().startswith("0X") else int(Size)
1258             except:
1259                 IsValid = False
1260                 Size = -1
1261         return [VpdOffset, str(Size), Value], IsValid, 2
1262     elif PcdType in (MODEL_PCD_DYNAMIC_HII, MODEL_PCD_DYNAMIC_EX_HII):
1263         IsValid = (3 <= len(FieldList) <= 5)
1264         HiiString = FieldList[0]
1265         Guid = Offset = Value = Attribute = ''
1266         if len(FieldList) > 1:
1267             Guid = FieldList[1]
1268         if len(FieldList) > 2:
1269             Offset = FieldList[2]
1270         if len(FieldList) > 3:
1271             Value = FieldList[3]
1272         if len(FieldList) > 4:
1273             Attribute = FieldList[4]
1274         return [HiiString, Guid, Offset, Value, Attribute], IsValid, 3
1275     return [], False, 0
1276
1277 ## AnalyzePcdData
1278 #
1279 #  Analyze the pcd Value, Datum type and TokenNumber.
1280 #  Used to avoid split issue while the value string contain "|" character
1281 #
1282 #  @param[in] Setting:  A String contain value/datum type/token number information;
1283 #
1284 #  @retval   ValueList: A List contain value, datum type and toke number.
1285 #
1286 def AnalyzePcdData(Setting):
1287     ValueList = ['', '', '']
1288
1289     ValueRe = re.compile(r'^\s*L?\".*\|.*\"')
1290     PtrValue = ValueRe.findall(Setting)
1291
1292     ValueUpdateFlag = False
1293
1294     if len(PtrValue) >= 1:
1295         Setting = re.sub(ValueRe, '', Setting)
1296         ValueUpdateFlag = True
1297
1298     TokenList = Setting.split(TAB_VALUE_SPLIT)
1299     ValueList[0:len(TokenList)] = TokenList
1300
1301     if ValueUpdateFlag:
1302         ValueList[0] = PtrValue[0]
1303
1304     return ValueList
1305
1306 ## check format of PCD value against its the datum type
1307 #
1308 # For PCD value setting
1309 #
1310 def CheckPcdDatum(Type, Value):
1311     if Type == TAB_VOID:
1312         ValueRe = re.compile(r'\s*L?\".*\"\s*$')
1313         if not (((Value.startswith('L"') or Value.startswith('"')) and Value.endswith('"'))
1314                 or (Value.startswith('{') and Value.endswith('}')) or (Value.startswith("L'") or Value.startswith("'") and Value.endswith("'"))
1315                ):
1316             return False, "Invalid value [%s] of type [%s]; must be in the form of {...} for array"\
1317                           ", \"...\" or \'...\' for string, L\"...\" or L\'...\' for unicode string" % (Value, Type)
1318         elif ValueRe.match(Value):
1319             # Check the chars in UnicodeString or CString is printable
1320             if Value.startswith("L"):
1321                 Value = Value[2:-1]
1322             else:
1323                 Value = Value[1:-1]
1324             Printset = set(string.printable)
1325             Printset.remove(TAB_PRINTCHAR_VT)
1326             Printset.add(TAB_PRINTCHAR_BS)
1327             Printset.add(TAB_PRINTCHAR_NUL)
1328             if not set(Value).issubset(Printset):
1329                 PrintList = sorted(Printset)
1330                 return False, "Invalid PCD string value of type [%s]; must be printable chars %s." % (Type, PrintList)
1331     elif Type == 'BOOLEAN':
1332         if Value not in ['TRUE', 'True', 'true', '0x1', '0x01', '1', 'FALSE', 'False', 'false', '0x0', '0x00', '0']:
1333             return False, "Invalid value [%s] of type [%s]; must be one of TRUE, True, true, 0x1, 0x01, 1"\
1334                           ", FALSE, False, false, 0x0, 0x00, 0" % (Value, Type)
1335     elif Type in [TAB_UINT8, TAB_UINT16, TAB_UINT32, TAB_UINT64]:
1336         if Value.startswith('0') and not Value.lower().startswith('0x') and len(Value) > 1 and Value.lstrip('0'):
1337             Value = Value.lstrip('0')
1338         try:
1339             if Value and int(Value, 0) < 0:
1340                 return False, "PCD can't be set to negative value[%s] for datum type [%s]" % (Value, Type)
1341             Value = int(Value, 0)
1342             if Value > MAX_VAL_TYPE[Type]:
1343                 return False, "Too large PCD value[%s] for datum type [%s]" % (Value, Type)
1344         except:
1345             return False, "Invalid value [%s] of type [%s];"\
1346                           " must be a hexadecimal, decimal or octal in C language format." % (Value, Type)
1347     else:
1348         return True, "StructurePcd"
1349
1350     return True, ""
1351
1352 def CommonPath(PathList):
1353     P1 = min(PathList).split(os.path.sep)
1354     P2 = max(PathList).split(os.path.sep)
1355     for Index in range(min(len(P1), len(P2))):
1356         if P1[Index] != P2[Index]:
1357             return os.path.sep.join(P1[:Index])
1358     return os.path.sep.join(P1)
1359
1360 class PathClass(object):
1361     def __init__(self, File='', Root='', AlterRoot='', Type='', IsBinary=False,
1362                  Arch='COMMON', ToolChainFamily='', Target='', TagName='', ToolCode=''):
1363         self.Arch = Arch
1364         self.File = str(File)
1365         if os.path.isabs(self.File):
1366             self.Root = ''
1367             self.AlterRoot = ''
1368         else:
1369             self.Root = str(Root)
1370             self.AlterRoot = str(AlterRoot)
1371
1372         # Remove any '.' and '..' in path
1373         if self.Root:
1374             self.Root = mws.getWs(self.Root, self.File)
1375             self.Path = os.path.normpath(os.path.join(self.Root, self.File))
1376             self.Root = os.path.normpath(CommonPath([self.Root, self.Path]))
1377             # eliminate the side-effect of 'C:'
1378             if self.Root[-1] == ':':
1379                 self.Root += os.path.sep
1380             # file path should not start with path separator
1381             if self.Root[-1] == os.path.sep:
1382                 self.File = self.Path[len(self.Root):]
1383             else:
1384                 self.File = self.Path[len(self.Root) + 1:]
1385         else:
1386             self.Path = os.path.normpath(self.File)
1387
1388         self.SubDir, self.Name = os.path.split(self.File)
1389         self.BaseName, self.Ext = os.path.splitext(self.Name)
1390
1391         if self.Root:
1392             if self.SubDir:
1393                 self.Dir = os.path.join(self.Root, self.SubDir)
1394             else:
1395                 self.Dir = self.Root
1396         else:
1397             self.Dir = self.SubDir
1398
1399         if IsBinary:
1400             self.Type = Type
1401         else:
1402             self.Type = self.Ext.lower()
1403
1404         self.IsBinary = IsBinary
1405         self.Target = Target
1406         self.TagName = TagName
1407         self.ToolCode = ToolCode
1408         self.ToolChainFamily = ToolChainFamily
1409         self.OriginalPath = self
1410
1411     ## Convert the object of this class to a string
1412     #
1413     #  Convert member Path of the class to a string
1414     #
1415     #  @retval string Formatted String
1416     #
1417     def __str__(self):
1418         return self.Path
1419
1420     ## Override __eq__ function
1421     #
1422     # Check whether PathClass are the same
1423     #
1424     # @retval False The two PathClass are different
1425     # @retval True  The two PathClass are the same
1426     #
1427     def __eq__(self, Other):
1428         return self.Path == str(Other)
1429
1430     ## Override __cmp__ function
1431     #
1432     # Customize the comparison operation of two PathClass
1433     #
1434     # @retval 0     The two PathClass are different
1435     # @retval -1    The first PathClass is less than the second PathClass
1436     # @retval 1     The first PathClass is Bigger than the second PathClass
1437     def __cmp__(self, Other):
1438         OtherKey = str(Other)
1439
1440         SelfKey = self.Path
1441         if SelfKey == OtherKey:
1442             return 0
1443         elif SelfKey > OtherKey:
1444             return 1
1445         else:
1446             return -1
1447
1448     ## Override __hash__ function
1449     #
1450     # Use Path as key in hash table
1451     #
1452     # @retval string Key for hash table
1453     #
1454     def __hash__(self):
1455         return hash(self.Path)
1456
1457     @cached_property
1458     def Key(self):
1459         return self.Path.upper()
1460
1461     @property
1462     def TimeStamp(self):
1463         return os.stat(self.Path)[8]
1464
1465     def Validate(self, Type='', CaseSensitive=True):
1466         def RealPath2(File, Dir='', OverrideDir=''):
1467             NewFile = None
1468             if OverrideDir:
1469                 NewFile = GlobalData.gAllFiles[os.path.normpath(os.path.join(OverrideDir, File))]
1470                 if NewFile:
1471                     if OverrideDir[-1] == os.path.sep:
1472                         return NewFile[len(OverrideDir):], NewFile[0:len(OverrideDir)]
1473                     else:
1474                         return NewFile[len(OverrideDir) + 1:], NewFile[0:len(OverrideDir)]
1475             if GlobalData.gAllFiles:
1476                 NewFile = GlobalData.gAllFiles[os.path.normpath(os.path.join(Dir, File))]
1477             if not NewFile:
1478                 NewFile = os.path.normpath(os.path.join(Dir, File))
1479                 if not os.path.exists(NewFile):
1480                     return None, None
1481             if NewFile:
1482                 if Dir:
1483                     if Dir[-1] == os.path.sep:
1484                         return NewFile[len(Dir):], NewFile[0:len(Dir)]
1485                     else:
1486                         return NewFile[len(Dir) + 1:], NewFile[0:len(Dir)]
1487                 else:
1488                     return NewFile, ''
1489
1490             return None, None
1491
1492         if GlobalData.gCaseInsensitive:
1493             CaseSensitive = False
1494         if Type and Type.lower() != self.Type:
1495             return FILE_TYPE_MISMATCH, '%s (expect %s but got %s)' % (self.File, Type, self.Type)
1496
1497         RealFile, RealRoot = RealPath2(self.File, self.Root, self.AlterRoot)
1498         if not RealRoot and not RealFile:
1499             RealFile = self.File
1500             if self.AlterRoot:
1501                 RealFile = os.path.join(self.AlterRoot, self.File)
1502             elif self.Root:
1503                 RealFile = os.path.join(self.Root, self.File)
1504             if len (mws.getPkgPath()) == 0:
1505                 return FILE_NOT_FOUND, os.path.join(self.AlterRoot, RealFile)
1506             else:
1507                 return FILE_NOT_FOUND, "%s is not found in packages path:\n\t%s" % (self.File, '\n\t'.join(mws.getPkgPath()))
1508
1509         ErrorCode = 0
1510         ErrorInfo = ''
1511         if RealRoot != self.Root or RealFile != self.File:
1512             if CaseSensitive and (RealFile != self.File or (RealRoot != self.Root and RealRoot != self.AlterRoot)):
1513                 ErrorCode = FILE_CASE_MISMATCH
1514                 ErrorInfo = self.File + '\n\t' + RealFile + " [in file system]"
1515
1516             self.SubDir, self.Name = os.path.split(RealFile)
1517             self.BaseName, self.Ext = os.path.splitext(self.Name)
1518             if self.SubDir:
1519                 self.Dir = os.path.join(RealRoot, self.SubDir)
1520             else:
1521                 self.Dir = RealRoot
1522             self.File = RealFile
1523             self.Root = RealRoot
1524             self.Path = os.path.join(RealRoot, RealFile)
1525         return ErrorCode, ErrorInfo
1526
1527 ## Parse PE image to get the required PE information.
1528 #
1529 class PeImageClass():
1530     ## Constructor
1531     #
1532     #   @param  File FilePath of PeImage
1533     #
1534     def __init__(self, PeFile):
1535         self.FileName   = PeFile
1536         self.IsValid    = False
1537         self.Size       = 0
1538         self.EntryPoint = 0
1539         self.SectionAlignment  = 0
1540         self.SectionHeaderList = []
1541         self.ErrorInfo = ''
1542         try:
1543             PeObject = open(PeFile, 'rb')
1544         except:
1545             self.ErrorInfo = self.FileName + ' can not be found\n'
1546             return
1547         # Read DOS header
1548         ByteArray = array.array('B')
1549         ByteArray.fromfile(PeObject, 0x3E)
1550         ByteList = ByteArray.tolist()
1551         # DOS signature should be 'MZ'
1552         if self._ByteListToStr (ByteList[0x0:0x2]) != 'MZ':
1553             self.ErrorInfo = self.FileName + ' has no valid DOS signature MZ'
1554             return
1555
1556         # Read 4 byte PE Signature
1557         PeOffset = self._ByteListToInt(ByteList[0x3C:0x3E])
1558         PeObject.seek(PeOffset)
1559         ByteArray = array.array('B')
1560         ByteArray.fromfile(PeObject, 4)
1561         # PE signature should be 'PE\0\0'
1562         if ByteArray.tostring() != b'PE\0\0':
1563             self.ErrorInfo = self.FileName + ' has no valid PE signature PE00'
1564             return
1565
1566         # Read PE file header
1567         ByteArray = array.array('B')
1568         ByteArray.fromfile(PeObject, 0x14)
1569         ByteList = ByteArray.tolist()
1570         SecNumber = self._ByteListToInt(ByteList[0x2:0x4])
1571         if SecNumber == 0:
1572             self.ErrorInfo = self.FileName + ' has no section header'
1573             return
1574
1575         # Read PE optional header
1576         OptionalHeaderSize = self._ByteListToInt(ByteArray[0x10:0x12])
1577         ByteArray = array.array('B')
1578         ByteArray.fromfile(PeObject, OptionalHeaderSize)
1579         ByteList = ByteArray.tolist()
1580         self.EntryPoint       = self._ByteListToInt(ByteList[0x10:0x14])
1581         self.SectionAlignment = self._ByteListToInt(ByteList[0x20:0x24])
1582         self.Size             = self._ByteListToInt(ByteList[0x38:0x3C])
1583
1584         # Read each Section Header
1585         for Index in range(SecNumber):
1586             ByteArray = array.array('B')
1587             ByteArray.fromfile(PeObject, 0x28)
1588             ByteList = ByteArray.tolist()
1589             SecName  = self._ByteListToStr(ByteList[0:8])
1590             SecVirtualSize = self._ByteListToInt(ByteList[8:12])
1591             SecRawAddress  = self._ByteListToInt(ByteList[20:24])
1592             SecVirtualAddress = self._ByteListToInt(ByteList[12:16])
1593             self.SectionHeaderList.append((SecName, SecVirtualAddress, SecRawAddress, SecVirtualSize))
1594         self.IsValid = True
1595         PeObject.close()
1596
1597     def _ByteListToStr(self, ByteList):
1598         String = ''
1599         for index in range(len(ByteList)):
1600             if ByteList[index] == 0:
1601                 break
1602             String += chr(ByteList[index])
1603         return String
1604
1605     def _ByteListToInt(self, ByteList):
1606         Value = 0
1607         for index in range(len(ByteList) - 1, -1, -1):
1608             Value = (Value << 8) | int(ByteList[index])
1609         return Value
1610
1611 class DefaultStore():
1612     def __init__(self, DefaultStores ):
1613
1614         self.DefaultStores = DefaultStores
1615     def DefaultStoreID(self, DefaultStoreName):
1616         for key, value in self.DefaultStores.items():
1617             if value == DefaultStoreName:
1618                 return key
1619         return None
1620     def GetDefaultDefault(self):
1621         if not self.DefaultStores or "0" in self.DefaultStores:
1622             return "0", TAB_DEFAULT_STORES_DEFAULT
1623         else:
1624             minvalue = min(int(value_str) for value_str in self.DefaultStores)
1625             return (str(minvalue), self.DefaultStores[str(minvalue)])
1626     def GetMin(self, DefaultSIdList):
1627         if not DefaultSIdList:
1628             return TAB_DEFAULT_STORES_DEFAULT
1629         storeidset = {storeid for storeid, storename in self.DefaultStores.values() if storename in DefaultSIdList}
1630         if not storeidset:
1631             return ""
1632         minid = min(storeidset )
1633         for sid, name in self.DefaultStores.values():
1634             if sid == minid:
1635                 return name
1636
1637 class SkuClass():
1638     DEFAULT = 0
1639     SINGLE = 1
1640     MULTIPLE =2
1641
1642     def __init__(self,SkuIdentifier='', SkuIds=None):
1643         if SkuIds is None:
1644             SkuIds = {}
1645
1646         for SkuName in SkuIds:
1647             SkuId = SkuIds[SkuName][0]
1648             skuid_num = int(SkuId, 16) if SkuId.upper().startswith("0X") else int(SkuId)
1649             if skuid_num > 0xFFFFFFFFFFFFFFFF:
1650                 EdkLogger.error("build", PARAMETER_INVALID,
1651                             ExtraData = "SKU-ID [%s] value %s exceeds the max value of UINT64"
1652                                       % (SkuName, SkuId))
1653
1654         self.AvailableSkuIds = OrderedDict()
1655         self.SkuIdSet = []
1656         self.SkuIdNumberSet = []
1657         self.SkuData = SkuIds
1658         self._SkuInherit = {}
1659         self._SkuIdentifier = SkuIdentifier
1660         if SkuIdentifier == '' or SkuIdentifier is None:
1661             self.SkuIdSet = ['DEFAULT']
1662             self.SkuIdNumberSet = ['0U']
1663         elif SkuIdentifier == 'ALL':
1664             self.SkuIdSet = list(SkuIds.keys())
1665             self.SkuIdNumberSet = [num[0].strip() + 'U' for num in SkuIds.values()]
1666         else:
1667             r = SkuIdentifier.split('|')
1668             self.SkuIdSet=[(r[k].strip()).upper() for k in range(len(r))]
1669             k = None
1670             try:
1671                 self.SkuIdNumberSet = [SkuIds[k][0].strip() + 'U' for k in self.SkuIdSet]
1672             except Exception:
1673                 EdkLogger.error("build", PARAMETER_INVALID,
1674                             ExtraData = "SKU-ID [%s] is not supported by the platform. [Valid SKU-ID: %s]"
1675                                       % (k, " | ".join(SkuIds.keys())))
1676         for each in self.SkuIdSet:
1677             if each in SkuIds:
1678                 self.AvailableSkuIds[each] = SkuIds[each][0]
1679             else:
1680                 EdkLogger.error("build", PARAMETER_INVALID,
1681                             ExtraData="SKU-ID [%s] is not supported by the platform. [Valid SKU-ID: %s]"
1682                                       % (each, " | ".join(SkuIds.keys())))
1683         if self.SkuUsageType != SkuClass.SINGLE:
1684             self.AvailableSkuIds.update({'DEFAULT':0, 'COMMON':0})
1685         if self.SkuIdSet:
1686             GlobalData.gSkuids = (self.SkuIdSet)
1687             if 'COMMON' in GlobalData.gSkuids:
1688                 GlobalData.gSkuids.remove('COMMON')
1689             if self.SkuUsageType == self.SINGLE:
1690                 if len(GlobalData.gSkuids) != 1:
1691                     if 'DEFAULT' in GlobalData.gSkuids:
1692                         GlobalData.gSkuids.remove('DEFAULT')
1693             if GlobalData.gSkuids:
1694                 GlobalData.gSkuids.sort()
1695
1696     def GetNextSkuId(self, skuname):
1697         if not self._SkuInherit:
1698             self._SkuInherit = {}
1699             for item in self.SkuData.values():
1700                 self._SkuInherit[item[1]]=item[2] if item[2] else "DEFAULT"
1701         return self._SkuInherit.get(skuname, "DEFAULT")
1702
1703     def GetSkuChain(self, sku):
1704         if sku == "DEFAULT":
1705             return ["DEFAULT"]
1706         skulist = [sku]
1707         nextsku = sku
1708         while True:
1709             nextsku = self.GetNextSkuId(nextsku)
1710             skulist.append(nextsku)
1711             if nextsku == "DEFAULT":
1712                 break
1713         skulist.reverse()
1714         return skulist
1715     def SkuOverrideOrder(self):
1716         skuorderset = []
1717         for skuname in self.SkuIdSet:
1718             skuorderset.append(self.GetSkuChain(skuname))
1719
1720         skuorder = []
1721         for index in range(max(len(item) for item in skuorderset)):
1722             for subset in skuorderset:
1723                 if index > len(subset)-1:
1724                     continue
1725                 if subset[index] in skuorder:
1726                     continue
1727                 skuorder.append(subset[index])
1728
1729         return skuorder
1730
1731     @property
1732     def SkuUsageType(self):
1733         if self._SkuIdentifier.upper() == "ALL":
1734             return SkuClass.MULTIPLE
1735
1736         if len(self.SkuIdSet) == 1:
1737             if self.SkuIdSet[0] == 'DEFAULT':
1738                 return SkuClass.DEFAULT
1739             return SkuClass.SINGLE
1740         if len(self.SkuIdSet) == 2 and 'DEFAULT' in self.SkuIdSet:
1741             return SkuClass.SINGLE
1742         return SkuClass.MULTIPLE
1743
1744     def DumpSkuIdArrary(self):
1745         if self.SkuUsageType == SkuClass.SINGLE:
1746             return "{0x0}"
1747         ArrayStrList = []
1748         for skuname in self.AvailableSkuIds:
1749             if skuname == "COMMON":
1750                 continue
1751             while skuname != "DEFAULT":
1752                 ArrayStrList.append(hex(int(self.AvailableSkuIds[skuname])))
1753                 skuname = self.GetNextSkuId(skuname)
1754             ArrayStrList.append("0x0")
1755         return "{{{myList}}}".format(myList=",".join(ArrayStrList))
1756
1757     @property
1758     def AvailableSkuIdSet(self):
1759         return self.AvailableSkuIds
1760
1761     @property
1762     def SystemSkuId(self):
1763         if self.SkuUsageType == SkuClass.SINGLE:
1764             if len(self.SkuIdSet) == 1:
1765                 return self.SkuIdSet[0]
1766             else:
1767                 return self.SkuIdSet[0] if self.SkuIdSet[0] != 'DEFAULT' else self.SkuIdSet[1]
1768         else:
1769             return 'DEFAULT'
1770
1771 ##  Get the integer value from string like "14U" or integer like 2
1772 #
1773 #   @param      Input   The object that may be either a integer value or a string
1774 #
1775 #   @retval     Value    The integer value that the input represents
1776 #
1777 def GetIntegerValue(Input):
1778     if not isinstance(Input, str):
1779         return Input
1780     String = Input
1781     if String.endswith("U"):
1782         String = String[:-1]
1783     if String.endswith("ULL"):
1784         String = String[:-3]
1785     if String.endswith("LL"):
1786         String = String[:-2]
1787
1788     if String.startswith("0x") or String.startswith("0X"):
1789         return int(String, 16)
1790     elif String == '':
1791         return 0
1792     else:
1793         return int(String)
1794
1795 #
1796 # Pack a GUID (registry format) list into a buffer and return it
1797 #
1798 def PackGUID(Guid):
1799     return pack(PACK_PATTERN_GUID,
1800                 int(Guid[0], 16),
1801                 int(Guid[1], 16),
1802                 int(Guid[2], 16),
1803                 int(Guid[3][-4:-2], 16),
1804                 int(Guid[3][-2:], 16),
1805                 int(Guid[4][-12:-10], 16),
1806                 int(Guid[4][-10:-8], 16),
1807                 int(Guid[4][-8:-6], 16),
1808                 int(Guid[4][-6:-4], 16),
1809                 int(Guid[4][-4:-2], 16),
1810                 int(Guid[4][-2:], 16)
1811                 )
1812
1813 #
1814 # Pack a GUID (byte) list into a buffer and return it
1815 #
1816 def PackByteFormatGUID(Guid):
1817     return pack(PACK_PATTERN_GUID,
1818                 Guid[0],
1819                 Guid[1],
1820                 Guid[2],
1821                 Guid[3],
1822                 Guid[4],
1823                 Guid[5],
1824                 Guid[6],
1825                 Guid[7],
1826                 Guid[8],
1827                 Guid[9],
1828                 Guid[10],
1829                 )
1830
1831 ## DeepCopy dict/OrderedDict recusively
1832 #
1833 #   @param      ori_dict    a nested dict or ordereddict
1834 #
1835 #   @retval     new dict or orderdict
1836 #
1837 def CopyDict(ori_dict):
1838     dict_type = ori_dict.__class__
1839     if dict_type not in (dict,OrderedDict):
1840         return ori_dict
1841     new_dict = dict_type()
1842     for key in ori_dict:
1843         if isinstance(ori_dict[key],(dict,OrderedDict)):
1844             new_dict[key] = CopyDict(ori_dict[key])
1845         else:
1846             new_dict[key] = ori_dict[key]
1847     return new_dict
1848
1849 #
1850 # Remove the c/c++ comments: // and /* */
1851 #
1852 def RemoveCComments(ctext):
1853     return re.sub('//.*?\n|/\*.*?\*/', '\n', ctext, flags=re.S)