BaseTools/Source/Python/Common/Misc.py

   1 ## @file
   2 # Common routines used by all tools
   3 #
   4 # Copyright (c) 2007 - 2019, Intel Corporation. All rights reserved.<BR>
   5 # SPDX-License-Identifier: BSD-2-Clause-Patent
   6 #
   7
   8 ##
   9 # Import Modules
  10 #
  11 from __future__ import absolute_import
  12
  13 import sys
  14 import string
  15 import threading
  16 import time
  17 import re
  18 import pickle
  19 import array
  20 import shutil
  21 from random import sample
  22 from struct import pack
  23 import uuid
  24 import subprocess
  25 from collections import OrderedDict
  26
  27 import Common.LongFilePathOs as os
  28 from Common import EdkLogger as EdkLogger
  29 from Common import GlobalData as GlobalData
  30 from Common.DataType import *
  31 from Common.BuildToolError import *
  32 from CommonDataClass.DataClass import *
  33 from Common.Parsing import GetSplitValueList
  34 from Common.LongFilePathSupport import OpenLongFilePath as open
  35 from Common.MultipleWorkspace import MultipleWorkspace as mws
  36 from CommonDataClass.Exceptions import BadExpression
  37 from Common.caching import cached_property
  38
  39 ## Regular expression used to find out place holders in string template
  40 gPlaceholderPattern = re.compile("\$\{([^$()\s]+)\}", re.MULTILINE | re.UNICODE)
  41
  42 ## regular expressions for map file processing
  43 startPatternGeneral = re.compile("^Start[' ']+Length[' ']+Name[' ']+Class")
  44 addressPatternGeneral = re.compile("^Address[' ']+Publics by Value[' ']+Rva\+Base")
  45 valuePatternGcc = re.compile('^([\w_\.]+) +([\da-fA-Fx]+) +([\da-fA-Fx]+)$')
  46 pcdPatternGcc = re.compile('^([\da-fA-Fx]+) +([\da-fA-Fx]+)')
  47 secReGeneral = re.compile('^([\da-fA-F]+):([\da-fA-F]+) +([\da-fA-F]+)[Hh]? +([.\w\$]+) +(\w+)', re.UNICODE)
  48
  49 StructPattern = re.compile(r'[_a-zA-Z][0-9A-Za-z_]*$')
  50
  51 ## Dictionary used to store dependencies of files
  52 gDependencyDatabase = {}    # arch : {file path : [dependent files list]}
  53
  54 #
  55 # If a module is built more than once with different PCDs or library classes
  56 # a temporary INF file with same content is created, the temporary file is removed
  57 # when build exits.
  58 #
  59 _TempInfs = []
  60
  61 def GetVariableOffset(mapfilepath, efifilepath, varnames):
  62     """ Parse map file to get variable offset in current EFI file
  63     @param mapfilepath    Map file absolution path
  64     @param efifilepath:   EFI binary file full path
  65     @param varnames       iteratable container whose elements are variable names to be searched
  66
  67     @return List whos elements are tuple with variable name and raw offset
  68     """
  69     lines = []
  70     try:
  71         f = open(mapfilepath, 'r')
  72         lines = f.readlines()
  73         f.close()
  74     except:
  75         return None
  76
  77     if len(lines) == 0: return None
  78     firstline = lines[0].strip()
  79     if (firstline.startswith("Archive member included ") and
  80         firstline.endswith(" file (symbol)")):
  81         return _parseForGCC(lines, efifilepath, varnames)
  82     if firstline.startswith("# Path:"):
  83         return _parseForXcode(lines, efifilepath, varnames)
  84     return _parseGeneral(lines, efifilepath, varnames)
  85
  86 def _parseForXcode(lines, efifilepath, varnames):
  87     status = 0
  88     ret = []
  89     for line in lines:
  90         line = line.strip()
  91         if status == 0 and line == "# Symbols:":
  92             status = 1
  93             continue
  94         if status == 1 and len(line) != 0:
  95             for varname in varnames:
  96                 if varname in line:
  97                     # cannot pregenerate this RegEx since it uses varname from varnames.
  98                     m = re.match('^([\da-fA-FxX]+)([\s\S]*)([_]*%s)$' % varname, line)
  99                     if m is not None:
 100                         ret.append((varname, m.group(1)))
 101     return ret
 102
 103 def _parseForGCC(lines, efifilepath, varnames):
 104     """ Parse map file generated by GCC linker """
 105     status = 0
 106     sections = []
 107     varoffset = []
 108     for index, line in enumerate(lines):
 109         line = line.strip()
 110         # status machine transection
 111         if status == 0 and line == "Memory Configuration":
 112             status = 1
 113             continue
 114         elif status == 1 and line == 'Linker script and memory map':
 115             status = 2
 116             continue
 117         elif status ==2 and line == 'START GROUP':
 118             status = 3
 119             continue
 120
 121         # status handler
 122         if status == 3:
 123             m = valuePatternGcc.match(line)
 124             if m is not None:
 125                 sections.append(m.groups(0))
 126             for varname in varnames:
 127                 Str = ''
 128                 m = re.match("^.data.(%s)" % varname, line)
 129                 if m is not None:
 130                     m = re.match(".data.(%s)$" % varname, line)
 131                     if m is not None:
 132                         Str = lines[index + 1]
 133                     else:
 134                         Str = line[len(".data.%s" % varname):]
 135                     if Str:
 136                         m = pcdPatternGcc.match(Str.strip())
 137                         if m is not None:
 138                             varoffset.append((varname, int(m.groups(0)[0], 16), int(sections[-1][1], 16), sections[-1][0]))
 139
 140     if not varoffset:
 141         return []
 142     # get section information from efi file
 143     efisecs = PeImageClass(efifilepath).SectionHeaderList
 144     if efisecs is None or len(efisecs) == 0:
 145         return []
 146     #redirection
 147     redirection = 0
 148     for efisec in efisecs:
 149         for section in sections:
 150             if section[0].strip() == efisec[0].strip() and section[0].strip() == '.text':
 151                 redirection = int(section[1], 16) - efisec[1]
 152
 153     ret = []
 154     for var in varoffset:
 155         for efisec in efisecs:
 156             if var[1] >= efisec[1] and var[1] < efisec[1]+efisec[3]:
 157                 ret.append((var[0], hex(efisec[2] + var[1] - efisec[1] - redirection)))
 158     return ret
 159
 160 def _parseGeneral(lines, efifilepath, varnames):
 161     status = 0    #0 - beginning of file; 1 - PE section definition; 2 - symbol table
 162     secs  = []    # key = section name
 163     varoffset = []
 164     symRe = re.compile('^([\da-fA-F]+):([\da-fA-F]+) +([\.:\\\\\w\?@\$]+) +([\da-fA-F]+)', re.UNICODE)
 165
 166     for line in lines:
 167         line = line.strip()
 168         if startPatternGeneral.match(line):
 169             status = 1
 170             continue
 171         if addressPatternGeneral.match(line):
 172             status = 2
 173             continue
 174         if line.startswith("entry point at"):
 175             status = 3
 176             continue
 177         if status == 1 and len(line) != 0:
 178             m =  secReGeneral.match(line)
 179             assert m is not None, "Fail to parse the section in map file , line is %s" % line
 180             sec_no, sec_start, sec_length, sec_name, sec_class = m.groups(0)
 181             secs.append([int(sec_no, 16), int(sec_start, 16), int(sec_length, 16), sec_name, sec_class])
 182         if status == 2 and len(line) != 0:
 183             for varname in varnames:
 184                 m = symRe.match(line)
 185                 assert m is not None, "Fail to parse the symbol in map file, line is %s" % line
 186                 sec_no, sym_offset, sym_name, vir_addr = m.groups(0)
 187                 sec_no     = int(sec_no,     16)
 188                 sym_offset = int(sym_offset, 16)
 189                 vir_addr   = int(vir_addr,   16)
 190                 # cannot pregenerate this RegEx since it uses varname from varnames.
 191                 m2 = re.match('^[_]*(%s)' % varname, sym_name)
 192                 if m2 is not None:
 193                     # fond a binary pcd entry in map file
 194                     for sec in secs:
 195                         if sec[0] == sec_no and (sym_offset >= sec[1] and sym_offset < sec[1] + sec[2]):
 196                             varoffset.append([varname, sec[3], sym_offset, vir_addr, sec_no])
 197
 198     if not varoffset: return []
 199
 200     # get section information from efi file
 201     efisecs = PeImageClass(efifilepath).SectionHeaderList
 202     if efisecs is None or len(efisecs) == 0:
 203         return []
 204
 205     ret = []
 206     for var in varoffset:
 207         index = 0
 208         for efisec in efisecs:
 209             index = index + 1
 210             if var[1].strip() == efisec[0].strip():
 211                 ret.append((var[0], hex(efisec[2] + var[2])))
 212             elif var[4] == index:
 213                 ret.append((var[0], hex(efisec[2] + var[2])))
 214
 215     return ret
 216
 217 ## Routine to process duplicated INF
 218 #
 219 #  This function is called by following two cases:
 220 #  Case 1 in DSC:
 221 #    [components.arch]
 222 #    Pkg/module/module.inf
 223 #    Pkg/module/module.inf {
 224 #      <Defines>
 225 #        FILE_GUID = 0D1B936F-68F3-4589-AFCC-FB8B7AEBC836
 226 #    }
 227 #  Case 2 in FDF:
 228 #    INF Pkg/module/module.inf
 229 #    INF FILE_GUID = 0D1B936F-68F3-4589-AFCC-FB8B7AEBC836 Pkg/module/module.inf
 230 #
 231 #  This function copies Pkg/module/module.inf to
 232 #  Conf/.cache/0D1B936F-68F3-4589-AFCC-FB8B7AEBC836module.inf
 233 #
 234 #  @param Path     Original PathClass object
 235 #  @param BaseName New file base name
 236 #
 237 #  @retval         return the new PathClass object
 238 #
 239 def ProcessDuplicatedInf(Path, BaseName, Workspace):
 240     Filename = os.path.split(Path.File)[1]
 241     if '.' in Filename:
 242         Filename = BaseName + Path.BaseName + Filename[Filename.rfind('.'):]
 243     else:
 244         Filename = BaseName + Path.BaseName
 245
 246     #
 247     # If -N is specified on command line, cache is disabled
 248     # The directory has to be created
 249     #
 250     DbDir = os.path.split(GlobalData.gDatabasePath)[0]
 251     if not os.path.exists(DbDir):
 252         os.makedirs(DbDir)
 253     #
 254     # A temporary INF is copied to database path which must have write permission
 255     # The temporary will be removed at the end of build
 256     # In case of name conflict, the file name is
 257     # FILE_GUIDBaseName (0D1B936F-68F3-4589-AFCC-FB8B7AEBC836module.inf)
 258     #
 259     TempFullPath = os.path.join(DbDir,
 260                                 Filename)
 261     RtPath = PathClass(Path.File, Workspace)
 262     #
 263     # Modify the full path to temporary path, keep other unchanged
 264     #
 265     # To build same module more than once, the module path with FILE_GUID overridden has
 266     # the file name FILE_GUIDmodule.inf, but the relative path (self.MetaFile.File) is the real path
 267     # in DSC which is used as relative path by C files and other files in INF.
 268     # A trick was used: all module paths are PathClass instances, after the initialization
 269     # of PathClass, the PathClass.Path is overridden by the temporary INF path.
 270     #
 271     # The reason for creating a temporary INF is:
 272     # Platform.Modules which is the base to create ModuleAutoGen objects is a dictionary,
 273     # the key is the full path of INF, the value is an object to save overridden library instances, PCDs.
 274     # A different key for the same module is needed to create different output directory,
 275     # retrieve overridden PCDs, library instances.
 276     #
 277     # The BaseName is the FILE_GUID which is also the output directory name.
 278     #
 279     #
 280     RtPath.Path = TempFullPath
 281     RtPath.BaseName = BaseName
 282     #
 283     # If file exists, compare contents
 284     #
 285     if os.path.exists(TempFullPath):
 286         with open(str(Path), 'rb') as f1, open(TempFullPath, 'rb') as f2:
 287             if f1.read() == f2.read():
 288                 return RtPath
 289     _TempInfs.append(TempFullPath)
 290     shutil.copy2(str(Path), TempFullPath)
 291     return RtPath
 292
 293 ## Remove temporary created INFs whose paths were saved in _TempInfs
 294 #
 295 def ClearDuplicatedInf():
 296     while _TempInfs:
 297         File = _TempInfs.pop()
 298         if os.path.exists(File):
 299             os.remove(File)
 300
 301 ## Convert GUID string in xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx style to C structure style
 302 #
 303 #   @param      Guid    The GUID string
 304 #
 305 #   @retval     string  The GUID string in C structure style
 306 #
 307 def GuidStringToGuidStructureString(Guid):
 308     GuidList = Guid.split('-')
 309     Result = '{'
 310     for Index in range(0, 3, 1):
 311         Result = Result + '0x' + GuidList[Index] + ', '
 312     Result = Result + '{0x' + GuidList[3][0:2] + ', 0x' + GuidList[3][2:4]
 313     for Index in range(0, 12, 2):
 314         Result = Result + ', 0x' + GuidList[4][Index:Index + 2]
 315     Result += '}}'
 316     return Result
 317
 318 ## Convert GUID structure in byte array to xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
 319 #
 320 #   @param      GuidValue   The GUID value in byte array
 321 #
 322 #   @retval     string      The GUID value in xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx format
 323 #
 324 def GuidStructureByteArrayToGuidString(GuidValue):
 325     guidValueString = GuidValue.lower().replace("{", "").replace("}", "").replace(" ", "").replace(";", "")
 326     guidValueList = guidValueString.split(",")
 327     if len(guidValueList) != 16:
 328         return ''
 329         #EdkLogger.error(None, None, "Invalid GUID value string %s" % GuidValue)
 330     try:
 331         return "%02x%02x%02x%02x-%02x%02x-%02x%02x-%02x%02x-%02x%02x%02x%02x%02x%02x" % (
 332                 int(guidValueList[3], 16),
 333                 int(guidValueList[2], 16),
 334                 int(guidValueList[1], 16),
 335                 int(guidValueList[0], 16),
 336                 int(guidValueList[5], 16),
 337                 int(guidValueList[4], 16),
 338                 int(guidValueList[7], 16),
 339                 int(guidValueList[6], 16),
 340                 int(guidValueList[8], 16),
 341                 int(guidValueList[9], 16),
 342                 int(guidValueList[10], 16),
 343                 int(guidValueList[11], 16),
 344                 int(guidValueList[12], 16),
 345                 int(guidValueList[13], 16),
 346                 int(guidValueList[14], 16),
 347                 int(guidValueList[15], 16)
 348                 )
 349     except:
 350         return ''
 351
 352 ## Convert GUID string in C structure style to xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
 353 #
 354 #   @param      GuidValue   The GUID value in C structure format
 355 #
 356 #   @retval     string      The GUID value in xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx format
 357 #
 358 def GuidStructureStringToGuidString(GuidValue):
 359     if not GlobalData.gGuidCFormatPattern.match(GuidValue):
 360         return ''
 361     guidValueString = GuidValue.lower().replace("{", "").replace("}", "").replace(" ", "").replace(";", "")
 362     guidValueList = guidValueString.split(",")
 363     if len(guidValueList) != 11:
 364         return ''
 365         #EdkLogger.error(None, None, "Invalid GUID value string %s" % GuidValue)
 366     try:
 367         return "%08x-%04x-%04x-%02x%02x-%02x%02x%02x%02x%02x%02x" % (
 368                 int(guidValueList[0], 16),
 369                 int(guidValueList[1], 16),
 370                 int(guidValueList[2], 16),
 371                 int(guidValueList[3], 16),
 372                 int(guidValueList[4], 16),
 373                 int(guidValueList[5], 16),
 374                 int(guidValueList[6], 16),
 375                 int(guidValueList[7], 16),
 376                 int(guidValueList[8], 16),
 377                 int(guidValueList[9], 16),
 378                 int(guidValueList[10], 16)
 379                 )
 380     except:
 381         return ''
 382
 383 ## Convert GUID string in C structure style to xxxxxxxx_xxxx_xxxx_xxxx_xxxxxxxxxxxx
 384 #
 385 #   @param      GuidValue   The GUID value in C structure format
 386 #
 387 #   @retval     string      The GUID value in xxxxxxxx_xxxx_xxxx_xxxx_xxxxxxxxxxxx format
 388 #
 389 def GuidStructureStringToGuidValueName(GuidValue):
 390     guidValueString = GuidValue.lower().replace("{", "").replace("}", "").replace(" ", "")
 391     guidValueList = guidValueString.split(",")
 392     if len(guidValueList) != 11:
 393         EdkLogger.error(None, FORMAT_INVALID, "Invalid GUID value string [%s]" % GuidValue)
 394     return "%08x_%04x_%04x_%02x%02x_%02x%02x%02x%02x%02x%02x" % (
 395             int(guidValueList[0], 16),
 396             int(guidValueList[1], 16),
 397             int(guidValueList[2], 16),
 398             int(guidValueList[3], 16),
 399             int(guidValueList[4], 16),
 400             int(guidValueList[5], 16),
 401             int(guidValueList[6], 16),
 402             int(guidValueList[7], 16),
 403             int(guidValueList[8], 16),
 404             int(guidValueList[9], 16),
 405             int(guidValueList[10], 16)
 406             )
 407
 408 ## Create directories
 409 #
 410 #   @param      Directory   The directory name
 411 #
 412 def CreateDirectory(Directory):
 413     if Directory is None or Directory.strip() == "":
 414         return True
 415     try:
 416         if not os.access(Directory, os.F_OK):
 417             os.makedirs(Directory)
 418     except:
 419         return False
 420     return True
 421
 422 ## Remove directories, including files and sub-directories in it
 423 #
 424 #   @param      Directory   The directory name
 425 #
 426 def RemoveDirectory(Directory, Recursively=False):
 427     if Directory is None or Directory.strip() == "" or not os.path.exists(Directory):
 428         return
 429     if Recursively:
 430         CurrentDirectory = os.getcwd()
 431         os.chdir(Directory)
 432         for File in os.listdir("."):
 433             if os.path.isdir(File):
 434                 RemoveDirectory(File, Recursively)
 435             else:
 436                 os.remove(File)
 437         os.chdir(CurrentDirectory)
 438     os.rmdir(Directory)
 439
 440 ## Store content in file
 441 #
 442 #  This method is used to save file only when its content is changed. This is
 443 #  quite useful for "make" system to decide what will be re-built and what won't.
 444 #
 445 #   @param      File            The path of file
 446 #   @param      Content         The new content of the file
 447 #   @param      IsBinaryFile    The flag indicating if the file is binary file or not
 448 #
 449 #   @retval     True            If the file content is changed and the file is renewed
 450 #   @retval     False           If the file content is the same
 451 #
 452 def SaveFileOnChange(File, Content, IsBinaryFile=True):
 453
 454     if os.path.exists(File):
 455         if IsBinaryFile:
 456             try:
 457                 with open(File, "rb") as f:
 458                     if Content == f.read():
 459                         return False
 460             except:
 461                 EdkLogger.error(None, FILE_OPEN_FAILURE, ExtraData=File)
 462         else:
 463             try:
 464                 with open(File, "r") as f:
 465                     if Content == f.read():
 466                         return False
 467             except:
 468                 EdkLogger.error(None, FILE_OPEN_FAILURE, ExtraData=File)
 469
 470     DirName = os.path.dirname(File)
 471     if not CreateDirectory(DirName):
 472         EdkLogger.error(None, FILE_CREATE_FAILURE, "Could not create directory %s" % DirName)
 473     else:
 474         if DirName == '':
 475             DirName = os.getcwd()
 476         if not os.access(DirName, os.W_OK):
 477             EdkLogger.error(None, PERMISSION_FAILURE, "Do not have write permission on directory %s" % DirName)
 478
 479     if IsBinaryFile:
 480         try:
 481             with open(File, "wb") as Fd:
 482                 Fd.write(Content)
 483         except IOError as X:
 484             EdkLogger.error(None, FILE_CREATE_FAILURE, ExtraData='IOError %s' % X)
 485     else:
 486         try:
 487             with open(File, 'w') as Fd:
 488                 Fd.write(Content)
 489         except IOError as X:
 490             EdkLogger.error(None, FILE_CREATE_FAILURE, ExtraData='IOError %s' % X)
 491
 492     return True
 493
 494 ## Retrieve and cache the real path name in file system
 495 #
 496 #   @param      Root    The root directory of path relative to
 497 #
 498 #   @retval     str     The path string if the path exists
 499 #   @retval     None    If path doesn't exist
 500 #
 501 class DirCache:
 502     _CACHE_ = set()
 503     _UPPER_CACHE_ = {}
 504
 505     def __init__(self, Root):
 506         self._Root = Root
 507         for F in os.listdir(Root):
 508             self._CACHE_.add(F)
 509             self._UPPER_CACHE_[F.upper()] = F
 510
 511     # =[] operator
 512     def __getitem__(self, Path):
 513         Path = Path[len(os.path.commonprefix([Path, self._Root])):]
 514         if not Path:
 515             return self._Root
 516         if Path and Path[0] == os.path.sep:
 517             Path = Path[1:]
 518         if Path in self._CACHE_:
 519             return os.path.join(self._Root, Path)
 520         UpperPath = Path.upper()
 521         if UpperPath in self._UPPER_CACHE_:
 522             return os.path.join(self._Root, self._UPPER_CACHE_[UpperPath])
 523
 524         IndexList = []
 525         LastSepIndex = -1
 526         SepIndex = Path.find(os.path.sep)
 527         while SepIndex > -1:
 528             Parent = UpperPath[:SepIndex]
 529             if Parent not in self._UPPER_CACHE_:
 530                 break
 531             LastSepIndex = SepIndex
 532             SepIndex = Path.find(os.path.sep, LastSepIndex + 1)
 533
 534         if LastSepIndex == -1:
 535             return None
 536
 537         Cwd = os.getcwd()
 538         os.chdir(self._Root)
 539         SepIndex = LastSepIndex
 540         while SepIndex > -1:
 541             Parent = Path[:SepIndex]
 542             ParentKey = UpperPath[:SepIndex]
 543             if ParentKey not in self._UPPER_CACHE_:
 544                 os.chdir(Cwd)
 545                 return None
 546
 547             if Parent in self._CACHE_:
 548                 ParentDir = Parent
 549             else:
 550                 ParentDir = self._UPPER_CACHE_[ParentKey]
 551             for F in os.listdir(ParentDir):
 552                 Dir = os.path.join(ParentDir, F)
 553                 self._CACHE_.add(Dir)
 554                 self._UPPER_CACHE_[Dir.upper()] = Dir
 555
 556             SepIndex = Path.find(os.path.sep, SepIndex + 1)
 557
 558         os.chdir(Cwd)
 559         if Path in self._CACHE_:
 560             return os.path.join(self._Root, Path)
 561         elif UpperPath in self._UPPER_CACHE_:
 562             return os.path.join(self._Root, self._UPPER_CACHE_[UpperPath])
 563         return None
 564
 565 def RealPath(File, Dir='', OverrideDir=''):
 566     NewFile = os.path.normpath(os.path.join(Dir, File))
 567     NewFile = GlobalData.gAllFiles[NewFile]
 568     if not NewFile and OverrideDir:
 569         NewFile = os.path.normpath(os.path.join(OverrideDir, File))
 570         NewFile = GlobalData.gAllFiles[NewFile]
 571     return NewFile
 572
 573 ## Get GUID value from given packages
 574 #
 575 #   @param      CName           The CName of the GUID
 576 #   @param      PackageList     List of packages looking-up in
 577 #   @param      Inffile         The driver file
 578 #
 579 #   @retval     GuidValue   if the CName is found in any given package
 580 #   @retval     None        if the CName is not found in all given packages
 581 #
 582 def GuidValue(CName, PackageList, Inffile = None):
 583     for P in PackageList:
 584         GuidKeys = list(P.Guids.keys())
 585         if Inffile and P._PrivateGuids:
 586             if not Inffile.startswith(P.MetaFile.Dir):
 587                 GuidKeys = [x for x in P.Guids if x not in P._PrivateGuids]
 588         if CName in GuidKeys:
 589             return P.Guids[CName]
 590     return None
 591     return None
 592
 593 ## A string template class
 594 #
 595 #  This class implements a template for string replacement. A string template
 596 #  looks like following
 597 #
 598 #       ${BEGIN} other_string ${placeholder_name} other_string ${END}
 599 #
 600 #  The string between ${BEGIN} and ${END} will be repeated as many times as the
 601 #  length of "placeholder_name", which is a list passed through a dict. The
 602 #  "placeholder_name" is the key name of the dict. The ${BEGIN} and ${END} can
 603 #  be not used and, in this case, the "placeholder_name" must not a list and it
 604 #  will just be replaced once.
 605 #
 606 class TemplateString(object):
 607     _REPEAT_START_FLAG = "BEGIN"
 608     _REPEAT_END_FLAG = "END"
 609
 610     class Section(object):
 611         _LIST_TYPES = [type([]), type(set()), type((0,))]
 612
 613         def __init__(self, TemplateSection, PlaceHolderList):
 614             self._Template = TemplateSection
 615             self._PlaceHolderList = []
 616
 617             # Split the section into sub-sections according to the position of placeholders
 618             if PlaceHolderList:
 619                 self._SubSectionList = []
 620                 SubSectionStart = 0
 621                 #
 622                 # The placeholders passed in must be in the format of
 623                 #
 624                 #   PlaceHolderName, PlaceHolderStartPoint, PlaceHolderEndPoint
 625                 #
 626                 for PlaceHolder, Start, End in PlaceHolderList:
 627                     self._SubSectionList.append(TemplateSection[SubSectionStart:Start])
 628                     self._SubSectionList.append(TemplateSection[Start:End])
 629                     self._PlaceHolderList.append(PlaceHolder)
 630                     SubSectionStart = End
 631                 if SubSectionStart < len(TemplateSection):
 632                     self._SubSectionList.append(TemplateSection[SubSectionStart:])
 633             else:
 634                 self._SubSectionList = [TemplateSection]
 635
 636         def __str__(self):
 637             return self._Template + " : " + str(self._PlaceHolderList)
 638
 639         def Instantiate(self, PlaceHolderValues):
 640             RepeatTime = -1
 641             RepeatPlaceHolders = {}
 642             NonRepeatPlaceHolders = {}
 643
 644             for PlaceHolder in self._PlaceHolderList:
 645                 if PlaceHolder not in PlaceHolderValues:
 646                     continue
 647                 Value = PlaceHolderValues[PlaceHolder]
 648                 if type(Value) in self._LIST_TYPES:
 649                     if RepeatTime < 0:
 650                         RepeatTime = len(Value)
 651                     elif RepeatTime != len(Value):
 652                         EdkLogger.error(
 653                                     "TemplateString",
 654                                     PARAMETER_INVALID,
 655                                     "${%s} has different repeat time from others!" % PlaceHolder,
 656                                     ExtraData=str(self._Template)
 657                                     )
 658                     RepeatPlaceHolders["${%s}" % PlaceHolder] = Value
 659                 else:
 660                     NonRepeatPlaceHolders["${%s}" % PlaceHolder] = Value
 661
 662             if NonRepeatPlaceHolders:
 663                 StringList = []
 664                 for S in self._SubSectionList:
 665                     if S not in NonRepeatPlaceHolders:
 666                         StringList.append(S)
 667                     else:
 668                         StringList.append(str(NonRepeatPlaceHolders[S]))
 669             else:
 670                 StringList = self._SubSectionList
 671
 672             if RepeatPlaceHolders:
 673                 TempStringList = []
 674                 for Index in range(RepeatTime):
 675                     for S in StringList:
 676                         if S not in RepeatPlaceHolders:
 677                             TempStringList.append(S)
 678                         else:
 679                             TempStringList.append(str(RepeatPlaceHolders[S][Index]))
 680                 StringList = TempStringList
 681
 682             return "".join(StringList)
 683
 684     ## Constructor
 685     def __init__(self, Template=None):
 686         self.String = []
 687         self.IsBinary = False
 688         self._Template = Template
 689         self._TemplateSectionList = self._Parse(Template)
 690
 691     ## str() operator
 692     #
 693     #   @retval     string  The string replaced
 694     #
 695     def __str__(self):
 696         return "".join(self.String)
 697
 698     ## Split the template string into fragments per the ${BEGIN} and ${END} flags
 699     #
 700     #   @retval     list    A list of TemplateString.Section objects
 701     #
 702     def _Parse(self, Template):
 703         SectionStart = 0
 704         SearchFrom = 0
 705         MatchEnd = 0
 706         PlaceHolderList = []
 707         TemplateSectionList = []
 708         while Template:
 709             MatchObj = gPlaceholderPattern.search(Template, SearchFrom)
 710             if not MatchObj:
 711                 if MatchEnd <= len(Template):
 712                     TemplateSection = TemplateString.Section(Template[SectionStart:], PlaceHolderList)
 713                     TemplateSectionList.append(TemplateSection)
 714                 break
 715
 716             MatchString = MatchObj.group(1)
 717             MatchStart = MatchObj.start()
 718             MatchEnd = MatchObj.end()
 719
 720             if MatchString == self._REPEAT_START_FLAG:
 721                 if MatchStart > SectionStart:
 722                     TemplateSection = TemplateString.Section(Template[SectionStart:MatchStart], PlaceHolderList)
 723                     TemplateSectionList.append(TemplateSection)
 724                 SectionStart = MatchEnd
 725                 PlaceHolderList = []
 726             elif MatchString == self._REPEAT_END_FLAG:
 727                 TemplateSection = TemplateString.Section(Template[SectionStart:MatchStart], PlaceHolderList)
 728                 TemplateSectionList.append(TemplateSection)
 729                 SectionStart = MatchEnd
 730                 PlaceHolderList = []
 731             else:
 732                 PlaceHolderList.append((MatchString, MatchStart - SectionStart, MatchEnd - SectionStart))
 733             SearchFrom = MatchEnd
 734         return TemplateSectionList
 735
 736     ## Replace the string template with dictionary of placeholders and append it to previous one
 737     #
 738     #   @param      AppendString    The string template to append
 739     #   @param      Dictionary      The placeholder dictionaries
 740     #
 741     def Append(self, AppendString, Dictionary=None):
 742         if Dictionary:
 743             SectionList = self._Parse(AppendString)
 744             self.String.append( "".join(S.Instantiate(Dictionary) for S in SectionList))
 745         else:
 746             if isinstance(AppendString,list):
 747                 self.String.extend(AppendString)
 748             else:
 749                 self.String.append(AppendString)
 750
 751     ## Replace the string template with dictionary of placeholders
 752     #
 753     #   @param      Dictionary      The placeholder dictionaries
 754     #
 755     #   @retval     str             The string replaced with placeholder values
 756     #
 757     def Replace(self, Dictionary=None):
 758         return "".join(S.Instantiate(Dictionary) for S in self._TemplateSectionList)
 759
 760 ## Progress indicator class
 761 #
 762 #  This class makes use of thread to print progress on console.
 763 #
 764 class Progressor:
 765     # for avoiding deadloop
 766     _StopFlag = None
 767     _ProgressThread = None
 768     _CheckInterval = 0.25
 769
 770     ## Constructor
 771     #
 772     #   @param      OpenMessage     The string printed before progress characters
 773     #   @param      CloseMessage    The string printed after progress characters
 774     #   @param      ProgressChar    The character used to indicate the progress
 775     #   @param      Interval        The interval in seconds between two progress characters
 776     #
 777     def __init__(self, OpenMessage="", CloseMessage="", ProgressChar='.', Interval=1.0):
 778         self.PromptMessage = OpenMessage
 779         self.CodaMessage = CloseMessage
 780         self.ProgressChar = ProgressChar
 781         self.Interval = Interval
 782         if Progressor._StopFlag is None:
 783             Progressor._StopFlag = threading.Event()
 784
 785     ## Start to print progress character
 786     #
 787     #   @param      OpenMessage     The string printed before progress characters
 788     #
 789     def Start(self, OpenMessage=None):
 790         if OpenMessage is not None:
 791             self.PromptMessage = OpenMessage
 792         Progressor._StopFlag.clear()
 793         if Progressor._ProgressThread is None:
 794             Progressor._ProgressThread = threading.Thread(target=self._ProgressThreadEntry)
 795             Progressor._ProgressThread.setDaemon(False)
 796             Progressor._ProgressThread.start()
 797
 798     ## Stop printing progress character
 799     #
 800     #   @param      CloseMessage    The string printed after progress characters
 801     #
 802     def Stop(self, CloseMessage=None):
 803         OriginalCodaMessage = self.CodaMessage
 804         if CloseMessage is not None:
 805             self.CodaMessage = CloseMessage
 806         self.Abort()
 807         self.CodaMessage = OriginalCodaMessage
 808
 809     ## Thread entry method
 810     def _ProgressThreadEntry(self):
 811         sys.stdout.write(self.PromptMessage + " ")
 812         sys.stdout.flush()
 813         TimeUp = 0.0
 814         while not Progressor._StopFlag.isSet():
 815             if TimeUp <= 0.0:
 816                 sys.stdout.write(self.ProgressChar)
 817                 sys.stdout.flush()
 818                 TimeUp = self.Interval
 819             time.sleep(self._CheckInterval)
 820             TimeUp -= self._CheckInterval
 821         sys.stdout.write(" " + self.CodaMessage + "\n")
 822         sys.stdout.flush()
 823
 824     ## Abort the progress display
 825     @staticmethod
 826     def Abort():
 827         if Progressor._StopFlag is not None:
 828             Progressor._StopFlag.set()
 829         if Progressor._ProgressThread is not None:
 830             Progressor._ProgressThread.join()
 831             Progressor._ProgressThread = None
 832
 833
 834 ## Dictionary using prioritized list as key
 835 #
 836 class tdict:
 837     _ListType = type([])
 838     _TupleType = type(())
 839     _Wildcard = 'COMMON'
 840     _ValidWildcardList = ['COMMON', 'DEFAULT', 'ALL', TAB_STAR, 'PLATFORM']
 841
 842     def __init__(self, _Single_=False, _Level_=2):
 843         self._Level_ = _Level_
 844         self.data = {}
 845         self._Single_ = _Single_
 846
 847     # =[] operator
 848     def __getitem__(self, key):
 849         KeyType = type(key)
 850         RestKeys = None
 851         if KeyType == self._ListType or KeyType == self._TupleType:
 852             FirstKey = key[0]
 853             if len(key) > 1:
 854                 RestKeys = key[1:]
 855             elif self._Level_ > 1:
 856                 RestKeys = [self._Wildcard for i in range(0, self._Level_ - 1)]
 857         else:
 858             FirstKey = key
 859             if self._Level_ > 1:
 860                 RestKeys = [self._Wildcard for i in range(0, self._Level_ - 1)]
 861
 862         if FirstKey is None or str(FirstKey).upper() in self._ValidWildcardList:
 863             FirstKey = self._Wildcard
 864
 865         if self._Single_:
 866             return self._GetSingleValue(FirstKey, RestKeys)
 867         else:
 868             return self._GetAllValues(FirstKey, RestKeys)
 869
 870     def _GetSingleValue(self, FirstKey, RestKeys):
 871         Value = None
 872         #print "%s-%s" % (FirstKey, self._Level_) ,
 873         if self._Level_ > 1:
 874             if FirstKey == self._Wildcard:
 875                 if FirstKey in self.data:
 876                     Value = self.data[FirstKey][RestKeys]
 877                 if Value is None:
 878                     for Key in self.data:
 879                         Value = self.data[Key][RestKeys]
 880                         if Value is not None: break
 881             else:
 882                 if FirstKey in self.data:
 883                     Value = self.data[FirstKey][RestKeys]
 884                 if Value is None and self._Wildcard in self.data:
 885                     #print "Value=None"
 886                     Value = self.data[self._Wildcard][RestKeys]
 887         else:
 888             if FirstKey == self._Wildcard:
 889                 if FirstKey in self.data:
 890                     Value = self.data[FirstKey]
 891                 if Value is None:
 892                     for Key in self.data:
 893                         Value = self.data[Key]
 894                         if Value is not None: break
 895             else:
 896                 if FirstKey in self.data:
 897                     Value = self.data[FirstKey]
 898                 elif self._Wildcard in self.data:
 899                     Value = self.data[self._Wildcard]
 900         return Value
 901
 902     def _GetAllValues(self, FirstKey, RestKeys):
 903         Value = []
 904         if self._Level_ > 1:
 905             if FirstKey == self._Wildcard:
 906                 for Key in self.data:
 907                     Value += self.data[Key][RestKeys]
 908             else:
 909                 if FirstKey in self.data:
 910                     Value += self.data[FirstKey][RestKeys]
 911                 if self._Wildcard in self.data:
 912                     Value += self.data[self._Wildcard][RestKeys]
 913         else:
 914             if FirstKey == self._Wildcard:
 915                 for Key in self.data:
 916                     Value.append(self.data[Key])
 917             else:
 918                 if FirstKey in self.data:
 919                     Value.append(self.data[FirstKey])
 920                 if self._Wildcard in self.data:
 921                     Value.append(self.data[self._Wildcard])
 922         return Value
 923
 924     ## []= operator
 925     def __setitem__(self, key, value):
 926         KeyType = type(key)
 927         RestKeys = None
 928         if KeyType == self._ListType or KeyType == self._TupleType:
 929             FirstKey = key[0]
 930             if len(key) > 1:
 931                 RestKeys = key[1:]
 932             else:
 933                 RestKeys = [self._Wildcard for i in range(0, self._Level_ - 1)]
 934         else:
 935             FirstKey = key
 936             if self._Level_ > 1:
 937                 RestKeys = [self._Wildcard for i in range(0, self._Level_ - 1)]
 938
 939         if FirstKey in self._ValidWildcardList:
 940             FirstKey = self._Wildcard
 941
 942         if FirstKey not in self.data and self._Level_ > 0:
 943             self.data[FirstKey] = tdict(self._Single_, self._Level_ - 1)
 944
 945         if self._Level_ > 1:
 946             self.data[FirstKey][RestKeys] = value
 947         else:
 948             self.data[FirstKey] = value
 949
 950     def SetGreedyMode(self):
 951         self._Single_ = False
 952         if self._Level_ > 1:
 953             for Key in self.data:
 954                 self.data[Key].SetGreedyMode()
 955
 956     def SetSingleMode(self):
 957         self._Single_ = True
 958         if self._Level_ > 1:
 959             for Key in self.data:
 960                 self.data[Key].SetSingleMode()
 961
 962     def GetKeys(self, KeyIndex=0):
 963         assert KeyIndex >= 0
 964         if KeyIndex == 0:
 965             return set(self.data.keys())
 966         else:
 967             keys = set()
 968             for Key in self.data:
 969                 keys |= self.data[Key].GetKeys(KeyIndex - 1)
 970             return keys
 971
 972 def AnalyzePcdExpression(Setting):
 973     RanStr = ''.join(sample(string.ascii_letters + string.digits, 8))
 974     Setting = Setting.replace('\\\\', RanStr).strip()
 975     # There might be escaped quote in a string: \", \\\" , \', \\\'
 976     Data = Setting
 977     # There might be '|' in string and in ( ... | ... ), replace it with '-'
 978     NewStr = ''
 979     InSingleQuoteStr = False
 980     InDoubleQuoteStr = False
 981     Pair = 0
 982     for Index, ch in enumerate(Data):
 983         if ch == '"' and not InSingleQuoteStr:
 984             if Data[Index - 1] != '\\':
 985                 InDoubleQuoteStr = not InDoubleQuoteStr
 986         elif ch == "'" and not InDoubleQuoteStr:
 987             if Data[Index - 1] != '\\':
 988                 InSingleQuoteStr = not InSingleQuoteStr
 989         elif ch == '(' and not (InSingleQuoteStr or InDoubleQuoteStr):
 990             Pair += 1
 991         elif ch == ')' and not (InSingleQuoteStr or InDoubleQuoteStr):
 992             Pair -= 1
 993
 994         if (Pair > 0 or InSingleQuoteStr or InDoubleQuoteStr) and ch == TAB_VALUE_SPLIT:
 995             NewStr += '-'
 996         else:
 997             NewStr += ch
 998     FieldList = []
 999     StartPos = 0
1000     while True:
1001         Pos = NewStr.find(TAB_VALUE_SPLIT, StartPos)
1002         if Pos < 0:
1003             FieldList.append(Setting[StartPos:].strip())
1004             break
1005         FieldList.append(Setting[StartPos:Pos].strip())
1006         StartPos = Pos + 1
1007     for i, ch in enumerate(FieldList):
1008         if RanStr in ch:
1009             FieldList[i] = ch.replace(RanStr,'\\\\')
1010     return FieldList
1011
1012 def ParseFieldValue (Value):
1013     def ParseDevPathValue (Value):
1014         if '\\' in Value:
1015             Value.replace('\\', '/').replace(' ', '')
1016
1017         Cmd = 'DevicePath ' + '"' + Value + '"'
1018         try:
1019             p = subprocess.Popen(Cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
1020             out, err = p.communicate()
1021         except Exception as X:
1022             raise BadExpression("DevicePath: %s" % (str(X)) )
1023         finally:
1024             subprocess._cleanup()
1025             p.stdout.close()
1026             p.stderr.close()
1027         if err:
1028             raise BadExpression("DevicePath: %s" % str(err))
1029         out = out.decode(encoding='utf-8', errors='ignore')
1030         Size = len(out.split())
1031         out = ','.join(out.split())
1032         return '{' + out + '}', Size
1033
1034     if "{CODE(" in Value:
1035         return Value, len(Value.split(","))
1036     if isinstance(Value, type(0)):
1037         return Value, (Value.bit_length() + 7) // 8
1038     if not isinstance(Value, type('')):
1039         raise BadExpression('Type %s is %s' %(Value, type(Value)))
1040     Value = Value.strip()
1041     if Value.startswith(TAB_UINT8) and Value.endswith(')'):
1042         Value, Size = ParseFieldValue(Value.split('(', 1)[1][:-1])
1043         if Size > 1:
1044             raise BadExpression('Value (%s) Size larger than %d' %(Value, Size))
1045         return Value, 1
1046     if Value.startswith(TAB_UINT16) and Value.endswith(')'):
1047         Value, Size = ParseFieldValue(Value.split('(', 1)[1][:-1])
1048         if Size > 2:
1049             raise BadExpression('Value (%s) Size larger than %d' %(Value, Size))
1050         return Value, 2
1051     if Value.startswith(TAB_UINT32) and Value.endswith(')'):
1052         Value, Size = ParseFieldValue(Value.split('(', 1)[1][:-1])
1053         if Size > 4:
1054             raise BadExpression('Value (%s) Size larger than %d' %(Value, Size))
1055         return Value, 4
1056     if Value.startswith(TAB_UINT64) and Value.endswith(')'):
1057         Value, Size = ParseFieldValue(Value.split('(', 1)[1][:-1])
1058         if Size > 8:
1059             raise BadExpression('Value (%s) Size larger than %d' % (Value, Size))
1060         return Value, 8
1061     if Value.startswith(TAB_GUID) and Value.endswith(')'):
1062         Value = Value.split('(', 1)[1][:-1].strip()
1063         if Value[0] == '{' and Value[-1] == '}':
1064             TmpValue = GuidStructureStringToGuidString(Value)
1065             if not TmpValue:
1066                 raise BadExpression("Invalid GUID value string %s" % Value)
1067             Value = TmpValue
1068         if Value[0] == '"' and Value[-1] == '"':
1069             Value = Value[1:-1]
1070         try:
1071             Value = str(uuid.UUID(Value).bytes_le)
1072             if Value.startswith("b'"):
1073                 Value = Value[2:-1]
1074             Value = "'" + Value + "'"
1075         except ValueError as Message:
1076             raise BadExpression(Message)
1077         Value, Size = ParseFieldValue(Value)
1078         return Value, 16
1079     if Value.startswith('L"') and Value.endswith('"'):
1080         # Unicode String
1081         # translate escape character
1082         Value = Value[1:]
1083         try:
1084             Value = eval(Value)
1085         except:
1086             Value = Value[1:-1]
1087         List = list(Value)
1088         List.reverse()
1089         Value = 0
1090         for Char in List:
1091             Value = (Value << 16) | ord(Char)
1092         return Value, (len(List) + 1) * 2
1093     if Value.startswith('"') and Value.endswith('"'):
1094         # ASCII String
1095         # translate escape character
1096         try:
1097             Value = eval(Value)
1098         except:
1099             Value = Value[1:-1]
1100         List = list(Value)
1101         List.reverse()
1102         Value = 0
1103         for Char in List:
1104             Value = (Value << 8) | ord(Char)
1105         return Value, len(List) + 1
1106     if Value.startswith("L'") and Value.endswith("'"):
1107         # Unicode Character Constant
1108         # translate escape character
1109         Value = Value[1:]
1110         try:
1111             Value = eval(Value)
1112         except:
1113             Value = Value[1:-1]
1114         List = list(Value)
1115         if len(List) == 0:
1116             raise BadExpression('Length %s is %s' % (Value, len(List)))
1117         List.reverse()
1118         Value = 0
1119         for Char in List:
1120             Value = (Value << 16) | ord(Char)
1121         return Value, len(List) * 2
1122     if Value.startswith("'") and Value.endswith("'"):
1123         # Character constant
1124         # translate escape character
1125         try:
1126             Value = eval(Value)
1127         except:
1128             Value = Value[1:-1]
1129         List = list(Value)
1130         if len(List) == 0:
1131             raise BadExpression('Length %s is %s' % (Value, len(List)))
1132         List.reverse()
1133         Value = 0
1134         for Char in List:
1135             Value = (Value << 8) | ord(Char)
1136         return Value, len(List)
1137     if Value.startswith('{') and Value.endswith('}'):
1138         # Byte array
1139         Value = Value[1:-1]
1140         List = [Item.strip() for Item in Value.split(',')]
1141         List.reverse()
1142         Value = 0
1143         RetSize = 0
1144         for Item in List:
1145             ItemValue, Size = ParseFieldValue(Item)
1146             RetSize += Size
1147             for I in range(Size):
1148                 Value = (Value << 8) | ((ItemValue >> 8 * I) & 0xff)
1149         return Value, RetSize
1150     if Value.startswith('DEVICE_PATH(') and Value.endswith(')'):
1151         Value = Value.replace("DEVICE_PATH(", '').rstrip(')')
1152         Value = Value.strip().strip('"')
1153         return ParseDevPathValue(Value)
1154     if Value.lower().startswith('0x'):
1155         try:
1156             Value = int(Value, 16)
1157         except:
1158             raise BadExpression("invalid hex value: %s" % Value)
1159         if Value == 0:
1160             return 0, 1
1161         return Value, (Value.bit_length() + 7) // 8
1162     if Value[0].isdigit():
1163         Value = int(Value, 10)
1164         if Value == 0:
1165             return 0, 1
1166         return Value, (Value.bit_length() + 7) // 8
1167     if Value.lower() == 'true':
1168         return 1, 1
1169     if Value.lower() == 'false':
1170         return 0, 1
1171     return Value, 1
1172
1173 ## AnalyzeDscPcd
1174 #
1175 #  Analyze DSC PCD value, since there is no data type info in DSC
1176 #  This function is used to match functions (AnalyzePcdData) used for retrieving PCD value from database
1177 #  1. Feature flag: TokenSpace.PcdCName|PcdValue
1178 #  2. Fix and Patch:TokenSpace.PcdCName|PcdValue[|VOID*[|MaxSize]]
1179 #  3. Dynamic default:
1180 #     TokenSpace.PcdCName|PcdValue[|VOID*[|MaxSize]]
1181 #     TokenSpace.PcdCName|PcdValue
1182 #  4. Dynamic VPD:
1183 #     TokenSpace.PcdCName|VpdOffset[|VpdValue]
1184 #     TokenSpace.PcdCName|VpdOffset[|MaxSize[|VpdValue]]
1185 #  5. Dynamic HII:
1186 #     TokenSpace.PcdCName|HiiString|VariableGuid|VariableOffset[|HiiValue]
1187 #  PCD value needs to be located in such kind of string, and the PCD value might be an expression in which
1188 #    there might have "|" operator, also in string value.
1189 #
1190 #  @param Setting: String contain information described above with "TokenSpace.PcdCName|" stripped
1191 #  @param PcdType: PCD type: feature, fixed, dynamic default VPD HII
1192 #  @param DataType: The datum type of PCD: VOID*, UNIT, BOOL
1193 #  @retval:
1194 #    ValueList: A List contain fields described above
1195 #    IsValid:   True if conforming EBNF, otherwise False
1196 #    Index:     The index where PcdValue is in ValueList
1197 #
1198 def AnalyzeDscPcd(Setting, PcdType, DataType=''):
1199     FieldList = AnalyzePcdExpression(Setting)
1200
1201     IsValid = True
1202     if PcdType in (MODEL_PCD_FIXED_AT_BUILD, MODEL_PCD_PATCHABLE_IN_MODULE, MODEL_PCD_DYNAMIC_DEFAULT, MODEL_PCD_DYNAMIC_EX_DEFAULT):
1203         Value = FieldList[0]
1204         Size = ''
1205         if len(FieldList) > 1 and FieldList[1]:
1206             DataType = FieldList[1]
1207             if FieldList[1] != TAB_VOID and StructPattern.match(FieldList[1]) is None:
1208                 IsValid = False
1209         if len(FieldList) > 2:
1210             Size = FieldList[2]
1211         if IsValid:
1212             if DataType == "":
1213                 IsValid = (len(FieldList) <= 1)
1214             else:
1215                 IsValid = (len(FieldList) <= 3)
1216
1217         if Size:
1218             try:
1219                 int(Size, 16) if Size.upper().startswith("0X") else int(Size)
1220             except:
1221                 IsValid = False
1222                 Size = -1
1223         return [str(Value), DataType, str(Size)], IsValid, 0
1224     elif PcdType == MODEL_PCD_FEATURE_FLAG:
1225         Value = FieldList[0]
1226         Size = ''
1227         IsValid = (len(FieldList) <= 1)
1228         return [Value, DataType, str(Size)], IsValid, 0
1229     elif PcdType in (MODEL_PCD_DYNAMIC_VPD, MODEL_PCD_DYNAMIC_EX_VPD):
1230         VpdOffset = FieldList[0]
1231         Value = Size = ''
1232         if not DataType == TAB_VOID:
1233             if len(FieldList) > 1:
1234                 Value = FieldList[1]
1235         else:
1236             if len(FieldList) > 1:
1237                 Size = FieldList[1]
1238             if len(FieldList) > 2:
1239                 Value = FieldList[2]
1240         if DataType == "":
1241             IsValid = (len(FieldList) <= 1)
1242         else:
1243             IsValid = (len(FieldList) <= 3)
1244         if Size:
1245             try:
1246                 int(Size, 16) if Size.upper().startswith("0X") else int(Size)
1247             except:
1248                 IsValid = False
1249                 Size = -1
1250         return [VpdOffset, str(Size), Value], IsValid, 2
1251     elif PcdType in (MODEL_PCD_DYNAMIC_HII, MODEL_PCD_DYNAMIC_EX_HII):
1252         IsValid = (3 <= len(FieldList) <= 5)
1253         HiiString = FieldList[0]
1254         Guid = Offset = Value = Attribute = ''
1255         if len(FieldList) > 1:
1256             Guid = FieldList[1]
1257         if len(FieldList) > 2:
1258             Offset = FieldList[2]
1259         if len(FieldList) > 3:
1260             Value = FieldList[3]
1261         if len(FieldList) > 4:
1262             Attribute = FieldList[4]
1263         return [HiiString, Guid, Offset, Value, Attribute], IsValid, 3
1264     return [], False, 0
1265
1266 ## AnalyzePcdData
1267 #
1268 #  Analyze the pcd Value, Datum type and TokenNumber.
1269 #  Used to avoid split issue while the value string contain "|" character
1270 #
1271 #  @param[in] Setting:  A String contain value/datum type/token number information;
1272 #
1273 #  @retval   ValueList: A List contain value, datum type and toke number.
1274 #
1275 def AnalyzePcdData(Setting):
1276     ValueList = ['', '', '']
1277
1278     ValueRe = re.compile(r'^\s*L?\".*\|.*\"')
1279     PtrValue = ValueRe.findall(Setting)
1280
1281     ValueUpdateFlag = False
1282
1283     if len(PtrValue) >= 1:
1284         Setting = re.sub(ValueRe, '', Setting)
1285         ValueUpdateFlag = True
1286
1287     TokenList = Setting.split(TAB_VALUE_SPLIT)
1288     ValueList[0:len(TokenList)] = TokenList
1289
1290     if ValueUpdateFlag:
1291         ValueList[0] = PtrValue[0]
1292
1293     return ValueList
1294
1295 ## check format of PCD value against its the datum type
1296 #
1297 # For PCD value setting
1298 #
1299 def CheckPcdDatum(Type, Value):
1300     if Type == TAB_VOID:
1301         ValueRe = re.compile(r'\s*L?\".*\"\s*$')
1302         if not (((Value.startswith('L"') or Value.startswith('"')) and Value.endswith('"'))
1303                 or (Value.startswith('{') and Value.endswith('}')) or (Value.startswith("L'") or Value.startswith("'") and Value.endswith("'"))
1304                ):
1305             return False, "Invalid value [%s] of type [%s]; must be in the form of {...} for array"\
1306                           ", \"...\" or \'...\' for string, L\"...\" or L\'...\' for unicode string" % (Value, Type)
1307         elif ValueRe.match(Value):
1308             # Check the chars in UnicodeString or CString is printable
1309             if Value.startswith("L"):
1310                 Value = Value[2:-1]
1311             else:
1312                 Value = Value[1:-1]
1313             Printset = set(string.printable)
1314             Printset.remove(TAB_PRINTCHAR_VT)
1315             Printset.add(TAB_PRINTCHAR_BS)
1316             Printset.add(TAB_PRINTCHAR_NUL)
1317             if not set(Value).issubset(Printset):
1318                 PrintList = sorted(Printset)
1319                 return False, "Invalid PCD string value of type [%s]; must be printable chars %s." % (Type, PrintList)
1320     elif Type == 'BOOLEAN':
1321         if Value not in ['TRUE', 'True', 'true', '0x1', '0x01', '1', 'FALSE', 'False', 'false', '0x0', '0x00', '0']:
1322             return False, "Invalid value [%s] of type [%s]; must be one of TRUE, True, true, 0x1, 0x01, 1"\
1323                           ", FALSE, False, false, 0x0, 0x00, 0" % (Value, Type)
1324     elif Type in [TAB_UINT8, TAB_UINT16, TAB_UINT32, TAB_UINT64]:
1325         if Value.startswith('0') and not Value.lower().startswith('0x') and len(Value) > 1 and Value.lstrip('0'):
1326             Value = Value.lstrip('0')
1327         try:
1328             if Value and int(Value, 0) < 0:
1329                 return False, "PCD can't be set to negative value[%s] for datum type [%s]" % (Value, Type)
1330             Value = int(Value, 0)
1331             if Value > MAX_VAL_TYPE[Type]:
1332                 return False, "Too large PCD value[%s] for datum type [%s]" % (Value, Type)
1333         except:
1334             return False, "Invalid value [%s] of type [%s];"\
1335                           " must be a hexadecimal, decimal or octal in C language format." % (Value, Type)
1336     else:
1337         return True, "StructurePcd"
1338
1339     return True, ""
1340
1341 def CommonPath(PathList):
1342     P1 = min(PathList).split(os.path.sep)
1343     P2 = max(PathList).split(os.path.sep)
1344     for Index in range(min(len(P1), len(P2))):
1345         if P1[Index] != P2[Index]:
1346             return os.path.sep.join(P1[:Index])
1347     return os.path.sep.join(P1)
1348
1349 class PathClass(object):
1350     def __init__(self, File='', Root='', AlterRoot='', Type='', IsBinary=False,
1351                  Arch='COMMON', ToolChainFamily='', Target='', TagName='', ToolCode=''):
1352         self.Arch = Arch
1353         self.File = str(File)
1354         if os.path.isabs(self.File):
1355             self.Root = ''
1356             self.AlterRoot = ''
1357         else:
1358             self.Root = str(Root)
1359             self.AlterRoot = str(AlterRoot)
1360
1361         # Remove any '.' and '..' in path
1362         if self.Root:
1363             self.Root = mws.getWs(self.Root, self.File)
1364             self.Path = os.path.normpath(os.path.join(self.Root, self.File))
1365             self.Root = os.path.normpath(CommonPath([self.Root, self.Path]))
1366             # eliminate the side-effect of 'C:'
1367             if self.Root[-1] == ':':
1368                 self.Root += os.path.sep
1369             # file path should not start with path separator
1370             if self.Root[-1] == os.path.sep:
1371                 self.File = self.Path[len(self.Root):]
1372             else:
1373                 self.File = self.Path[len(self.Root) + 1:]
1374         else:
1375             self.Path = os.path.normpath(self.File)
1376
1377         self.SubDir, self.Name = os.path.split(self.File)
1378         self.BaseName, self.Ext = os.path.splitext(self.Name)
1379
1380         if self.Root:
1381             if self.SubDir:
1382                 self.Dir = os.path.join(self.Root, self.SubDir)
1383             else:
1384                 self.Dir = self.Root
1385         else:
1386             self.Dir = self.SubDir
1387
1388         if IsBinary:
1389             self.Type = Type
1390         else:
1391             self.Type = self.Ext.lower()
1392
1393         self.IsBinary = IsBinary
1394         self.Target = Target
1395         self.TagName = TagName
1396         self.ToolCode = ToolCode
1397         self.ToolChainFamily = ToolChainFamily
1398
1399     ## Convert the object of this class to a string
1400     #
1401     #  Convert member Path of the class to a string
1402     #
1403     #  @retval string Formatted String
1404     #
1405     def __str__(self):
1406         return self.Path
1407
1408     ## Override __eq__ function
1409     #
1410     # Check whether PathClass are the same
1411     #
1412     # @retval False The two PathClass are different
1413     # @retval True  The two PathClass are the same
1414     #
1415     def __eq__(self, Other):
1416         return self.Path == str(Other)
1417
1418     ## Override __cmp__ function
1419     #
1420     # Customize the comparison operation of two PathClass
1421     #
1422     # @retval 0     The two PathClass are different
1423     # @retval -1    The first PathClass is less than the second PathClass
1424     # @retval 1     The first PathClass is Bigger than the second PathClass
1425     def __cmp__(self, Other):
1426         OtherKey = str(Other)
1427
1428         SelfKey = self.Path
1429         if SelfKey == OtherKey:
1430             return 0
1431         elif SelfKey > OtherKey:
1432             return 1
1433         else:
1434             return -1
1435
1436     ## Override __hash__ function
1437     #
1438     # Use Path as key in hash table
1439     #
1440     # @retval string Key for hash table
1441     #
1442     def __hash__(self):
1443         return hash(self.Path)
1444
1445     @cached_property
1446     def Key(self):
1447         return self.Path.upper()
1448
1449     @property
1450     def TimeStamp(self):
1451         return os.stat(self.Path)[8]
1452
1453     def Validate(self, Type='', CaseSensitive=True):
1454         def RealPath2(File, Dir='', OverrideDir=''):
1455             NewFile = None
1456             if OverrideDir:
1457                 NewFile = GlobalData.gAllFiles[os.path.normpath(os.path.join(OverrideDir, File))]
1458                 if NewFile:
1459                     if OverrideDir[-1] == os.path.sep:
1460                         return NewFile[len(OverrideDir):], NewFile[0:len(OverrideDir)]
1461                     else:
1462                         return NewFile[len(OverrideDir) + 1:], NewFile[0:len(OverrideDir)]
1463             if GlobalData.gAllFiles:
1464                 NewFile = GlobalData.gAllFiles[os.path.normpath(os.path.join(Dir, File))]
1465             if not NewFile:
1466                 NewFile = os.path.normpath(os.path.join(Dir, File))
1467                 if not os.path.exists(NewFile):
1468                     return None, None
1469             if NewFile:
1470                 if Dir:
1471                     if Dir[-1] == os.path.sep:
1472                         return NewFile[len(Dir):], NewFile[0:len(Dir)]
1473                     else:
1474                         return NewFile[len(Dir) + 1:], NewFile[0:len(Dir)]
1475                 else:
1476                     return NewFile, ''
1477
1478             return None, None
1479
1480         if GlobalData.gCaseInsensitive:
1481             CaseSensitive = False
1482         if Type and Type.lower() != self.Type:
1483             return FILE_TYPE_MISMATCH, '%s (expect %s but got %s)' % (self.File, Type, self.Type)
1484
1485         RealFile, RealRoot = RealPath2(self.File, self.Root, self.AlterRoot)
1486         if not RealRoot and not RealFile:
1487             RealFile = self.File
1488             if self.AlterRoot:
1489                 RealFile = os.path.join(self.AlterRoot, self.File)
1490             elif self.Root:
1491                 RealFile = os.path.join(self.Root, self.File)
1492             if len (mws.getPkgPath()) == 0:
1493                 return FILE_NOT_FOUND, os.path.join(self.AlterRoot, RealFile)
1494             else:
1495                 return FILE_NOT_FOUND, "%s is not found in packages path:\n\t%s" % (self.File, '\n\t'.join(mws.getPkgPath()))
1496
1497         ErrorCode = 0
1498         ErrorInfo = ''
1499         if RealRoot != self.Root or RealFile != self.File:
1500             if CaseSensitive and (RealFile != self.File or (RealRoot != self.Root and RealRoot != self.AlterRoot)):
1501                 ErrorCode = FILE_CASE_MISMATCH
1502                 ErrorInfo = self.File + '\n\t' + RealFile + " [in file system]"
1503
1504             self.SubDir, self.Name = os.path.split(RealFile)
1505             self.BaseName, self.Ext = os.path.splitext(self.Name)
1506             if self.SubDir:
1507                 self.Dir = os.path.join(RealRoot, self.SubDir)
1508             else:
1509                 self.Dir = RealRoot
1510             self.File = RealFile
1511             self.Root = RealRoot
1512             self.Path = os.path.join(RealRoot, RealFile)
1513         return ErrorCode, ErrorInfo
1514
1515 ## Parse PE image to get the required PE information.
1516 #
1517 class PeImageClass():
1518     ## Constructor
1519     #
1520     #   @param  File FilePath of PeImage
1521     #
1522     def __init__(self, PeFile):
1523         self.FileName   = PeFile
1524         self.IsValid    = False
1525         self.Size       = 0
1526         self.EntryPoint = 0
1527         self.SectionAlignment  = 0
1528         self.SectionHeaderList = []
1529         self.ErrorInfo = ''
1530         try:
1531             PeObject = open(PeFile, 'rb')
1532         except:
1533             self.ErrorInfo = self.FileName + ' can not be found\n'
1534             return
1535         # Read DOS header
1536         ByteArray = array.array('B')
1537         ByteArray.fromfile(PeObject, 0x3E)
1538         ByteList = ByteArray.tolist()
1539         # DOS signature should be 'MZ'
1540         if self._ByteListToStr (ByteList[0x0:0x2]) != 'MZ':
1541             self.ErrorInfo = self.FileName + ' has no valid DOS signature MZ'
1542             return
1543
1544         # Read 4 byte PE Signature
1545         PeOffset = self._ByteListToInt(ByteList[0x3C:0x3E])
1546         PeObject.seek(PeOffset)
1547         ByteArray = array.array('B')
1548         ByteArray.fromfile(PeObject, 4)
1549         # PE signature should be 'PE\0\0'
1550         if ByteArray.tostring() != b'PE\0\0':
1551             self.ErrorInfo = self.FileName + ' has no valid PE signature PE00'
1552             return
1553
1554         # Read PE file header
1555         ByteArray = array.array('B')
1556         ByteArray.fromfile(PeObject, 0x14)
1557         ByteList = ByteArray.tolist()
1558         SecNumber = self._ByteListToInt(ByteList[0x2:0x4])
1559         if SecNumber == 0:
1560             self.ErrorInfo = self.FileName + ' has no section header'
1561             return
1562
1563         # Read PE optional header
1564         OptionalHeaderSize = self._ByteListToInt(ByteArray[0x10:0x12])
1565         ByteArray = array.array('B')
1566         ByteArray.fromfile(PeObject, OptionalHeaderSize)
1567         ByteList = ByteArray.tolist()
1568         self.EntryPoint       = self._ByteListToInt(ByteList[0x10:0x14])
1569         self.SectionAlignment = self._ByteListToInt(ByteList[0x20:0x24])
1570         self.Size             = self._ByteListToInt(ByteList[0x38:0x3C])
1571
1572         # Read each Section Header
1573         for Index in range(SecNumber):
1574             ByteArray = array.array('B')
1575             ByteArray.fromfile(PeObject, 0x28)
1576             ByteList = ByteArray.tolist()
1577             SecName  = self._ByteListToStr(ByteList[0:8])
1578             SecVirtualSize = self._ByteListToInt(ByteList[8:12])
1579             SecRawAddress  = self._ByteListToInt(ByteList[20:24])
1580             SecVirtualAddress = self._ByteListToInt(ByteList[12:16])
1581             self.SectionHeaderList.append((SecName, SecVirtualAddress, SecRawAddress, SecVirtualSize))
1582         self.IsValid = True
1583         PeObject.close()
1584
1585     def _ByteListToStr(self, ByteList):
1586         String = ''
1587         for index in range(len(ByteList)):
1588             if ByteList[index] == 0:
1589                 break
1590             String += chr(ByteList[index])
1591         return String
1592
1593     def _ByteListToInt(self, ByteList):
1594         Value = 0
1595         for index in range(len(ByteList) - 1, -1, -1):
1596             Value = (Value << 8) | int(ByteList[index])
1597         return Value
1598
1599 class DefaultStore():
1600     def __init__(self, DefaultStores ):
1601
1602         self.DefaultStores = DefaultStores
1603     def DefaultStoreID(self, DefaultStoreName):
1604         for key, value in self.DefaultStores.items():
1605             if value == DefaultStoreName:
1606                 return key
1607         return None
1608     def GetDefaultDefault(self):
1609         if not self.DefaultStores or "0" in self.DefaultStores:
1610             return "0", TAB_DEFAULT_STORES_DEFAULT
1611         else:
1612             minvalue = min(int(value_str) for value_str in self.DefaultStores)
1613             return (str(minvalue), self.DefaultStores[str(minvalue)])
1614     def GetMin(self, DefaultSIdList):
1615         if not DefaultSIdList:
1616             return TAB_DEFAULT_STORES_DEFAULT
1617         storeidset = {storeid for storeid, storename in self.DefaultStores.values() if storename in DefaultSIdList}
1618         if not storeidset:
1619             return ""
1620         minid = min(storeidset )
1621         for sid, name in self.DefaultStores.values():
1622             if sid == minid:
1623                 return name
1624
1625 class SkuClass():
1626     DEFAULT = 0
1627     SINGLE = 1
1628     MULTIPLE =2
1629
1630     def __init__(self,SkuIdentifier='', SkuIds=None):
1631         if SkuIds is None:
1632             SkuIds = {}
1633
1634         for SkuName in SkuIds:
1635             SkuId = SkuIds[SkuName][0]
1636             skuid_num = int(SkuId, 16) if SkuId.upper().startswith("0X") else int(SkuId)
1637             if skuid_num > 0xFFFFFFFFFFFFFFFF:
1638                 EdkLogger.error("build", PARAMETER_INVALID,
1639                             ExtraData = "SKU-ID [%s] value %s exceeds the max value of UINT64"
1640                                       % (SkuName, SkuId))
1641
1642         self.AvailableSkuIds = OrderedDict()
1643         self.SkuIdSet = []
1644         self.SkuIdNumberSet = []
1645         self.SkuData = SkuIds
1646         self._SkuInherit = {}
1647         self._SkuIdentifier = SkuIdentifier
1648         if SkuIdentifier == '' or SkuIdentifier is None:
1649             self.SkuIdSet = ['DEFAULT']
1650             self.SkuIdNumberSet = ['0U']
1651         elif SkuIdentifier == 'ALL':
1652             self.SkuIdSet = list(SkuIds.keys())
1653             self.SkuIdNumberSet = [num[0].strip() + 'U' for num in SkuIds.values()]
1654         else:
1655             r = SkuIdentifier.split('|')
1656             self.SkuIdSet=[(r[k].strip()).upper() for k in range(len(r))]
1657             k = None
1658             try:
1659                 self.SkuIdNumberSet = [SkuIds[k][0].strip() + 'U' for k in self.SkuIdSet]
1660             except Exception:
1661                 EdkLogger.error("build", PARAMETER_INVALID,
1662                             ExtraData = "SKU-ID [%s] is not supported by the platform. [Valid SKU-ID: %s]"
1663                                       % (k, " | ".join(SkuIds.keys())))
1664         for each in self.SkuIdSet:
1665             if each in SkuIds:
1666                 self.AvailableSkuIds[each] = SkuIds[each][0]
1667             else:
1668                 EdkLogger.error("build", PARAMETER_INVALID,
1669                             ExtraData="SKU-ID [%s] is not supported by the platform. [Valid SKU-ID: %s]"
1670                                       % (each, " | ".join(SkuIds.keys())))
1671         if self.SkuUsageType != SkuClass.SINGLE:
1672             self.AvailableSkuIds.update({'DEFAULT':0, 'COMMON':0})
1673         if self.SkuIdSet:
1674             GlobalData.gSkuids = (self.SkuIdSet)
1675             if 'COMMON' in GlobalData.gSkuids:
1676                 GlobalData.gSkuids.remove('COMMON')
1677             if self.SkuUsageType == self.SINGLE:
1678                 if len(GlobalData.gSkuids) != 1:
1679                     if 'DEFAULT' in GlobalData.gSkuids:
1680                         GlobalData.gSkuids.remove('DEFAULT')
1681             if GlobalData.gSkuids:
1682                 GlobalData.gSkuids.sort()
1683
1684     def GetNextSkuId(self, skuname):
1685         if not self._SkuInherit:
1686             self._SkuInherit = {}
1687             for item in self.SkuData.values():
1688                 self._SkuInherit[item[1]]=item[2] if item[2] else "DEFAULT"
1689         return self._SkuInherit.get(skuname, "DEFAULT")
1690
1691     def GetSkuChain(self, sku):
1692         if sku == "DEFAULT":
1693             return ["DEFAULT"]
1694         skulist = [sku]
1695         nextsku = sku
1696         while True:
1697             nextsku = self.GetNextSkuId(nextsku)
1698             skulist.append(nextsku)
1699             if nextsku == "DEFAULT":
1700                 break
1701         skulist.reverse()
1702         return skulist
1703     def SkuOverrideOrder(self):
1704         skuorderset = []
1705         for skuname in self.SkuIdSet:
1706             skuorderset.append(self.GetSkuChain(skuname))
1707
1708         skuorder = []
1709         for index in range(max(len(item) for item in skuorderset)):
1710             for subset in skuorderset:
1711                 if index > len(subset)-1:
1712                     continue
1713                 if subset[index] in skuorder:
1714                     continue
1715                 skuorder.append(subset[index])
1716
1717         return skuorder
1718
1719     @property
1720     def SkuUsageType(self):
1721         if self._SkuIdentifier.upper() == "ALL":
1722             return SkuClass.MULTIPLE
1723
1724         if len(self.SkuIdSet) == 1:
1725             if self.SkuIdSet[0] == 'DEFAULT':
1726                 return SkuClass.DEFAULT
1727             return SkuClass.SINGLE
1728         if len(self.SkuIdSet) == 2 and 'DEFAULT' in self.SkuIdSet:
1729             return SkuClass.SINGLE
1730         return SkuClass.MULTIPLE
1731
1732     def DumpSkuIdArrary(self):
1733         if self.SkuUsageType == SkuClass.SINGLE:
1734             return "{0x0}"
1735         ArrayStrList = []
1736         for skuname in self.AvailableSkuIds:
1737             if skuname == "COMMON":
1738                 continue
1739             while skuname != "DEFAULT":
1740                 ArrayStrList.append(hex(int(self.AvailableSkuIds[skuname])))
1741                 skuname = self.GetNextSkuId(skuname)
1742             ArrayStrList.append("0x0")
1743         return "{{{myList}}}".format(myList=",".join(ArrayStrList))
1744
1745     @property
1746     def AvailableSkuIdSet(self):
1747         return self.AvailableSkuIds
1748
1749     @property
1750     def SystemSkuId(self):
1751         if self.SkuUsageType == SkuClass.SINGLE:
1752             if len(self.SkuIdSet) == 1:
1753                 return self.SkuIdSet[0]
1754             else:
1755                 return self.SkuIdSet[0] if self.SkuIdSet[0] != 'DEFAULT' else self.SkuIdSet[1]
1756         else:
1757             return 'DEFAULT'
1758
1759 ##  Get the integer value from string like "14U" or integer like 2
1760 #
1761 #   @param      Input   The object that may be either a integer value or a string
1762 #
1763 #   @retval     Value    The integer value that the input represents
1764 #
1765 def GetIntegerValue(Input):
1766     if not isinstance(Input, str):
1767         return Input
1768     String = Input
1769     if String.endswith("U"):
1770         String = String[:-1]
1771     if String.endswith("ULL"):
1772         String = String[:-3]
1773     if String.endswith("LL"):
1774         String = String[:-2]
1775
1776     if String.startswith("0x") or String.startswith("0X"):
1777         return int(String, 16)
1778     elif String == '':
1779         return 0
1780     else:
1781         return int(String)
1782
1783 #
1784 # Pack a GUID (registry format) list into a buffer and return it
1785 #
1786 def PackGUID(Guid):
1787     return pack(PACK_PATTERN_GUID,
1788                 int(Guid[0], 16),
1789                 int(Guid[1], 16),
1790                 int(Guid[2], 16),
1791                 int(Guid[3][-4:-2], 16),
1792                 int(Guid[3][-2:], 16),
1793                 int(Guid[4][-12:-10], 16),
1794                 int(Guid[4][-10:-8], 16),
1795                 int(Guid[4][-8:-6], 16),
1796                 int(Guid[4][-6:-4], 16),
1797                 int(Guid[4][-4:-2], 16),
1798                 int(Guid[4][-2:], 16)
1799                 )
1800
1801 #
1802 # Pack a GUID (byte) list into a buffer and return it
1803 #
1804 def PackByteFormatGUID(Guid):
1805     return pack(PACK_PATTERN_GUID,
1806                 Guid[0],
1807                 Guid[1],
1808                 Guid[2],
1809                 Guid[3],
1810                 Guid[4],
1811                 Guid[5],
1812                 Guid[6],
1813                 Guid[7],
1814                 Guid[8],
1815                 Guid[9],
1816                 Guid[10],
1817                 )
1818
1819 ## DeepCopy dict/OrderedDict recusively
1820 #
1821 #   @param      ori_dict    a nested dict or ordereddict
1822 #
1823 #   @retval     new dict or orderdict
1824 #
1825 def CopyDict(ori_dict):
1826     dict_type = ori_dict.__class__
1827     if dict_type not in (dict,OrderedDict):
1828         return ori_dict
1829     new_dict = dict_type()
1830     for key in ori_dict:
1831         if isinstance(ori_dict[key],(dict,OrderedDict)):
1832             new_dict[key] = CopyDict(ori_dict[key])
1833         else:
1834             new_dict[key] = ori_dict[key]
1835     return new_dict
1836
1837 #
1838 # Remove the c/c++ comments: // and /* */
1839 #
1840 def RemoveCComments(ctext):
1841     return re.sub('//.*?\n|/\*.*?\*/', '\n', ctext, flags=re.S)