BaseTools/Source/Python/Common/Misc.py

   1 ## @file
   2 # Common routines used by all tools
   3 #
   4 # Copyright (c) 2007 - 2019, Intel Corporation. All rights reserved.<BR>
   5 # This program and the accompanying materials
   6 # are licensed and made available under the terms and conditions of the BSD License
   7 # which accompanies this distribution.  The full text of the license may be found at
   8 # http://opensource.org/licenses/bsd-license.php
   9 #
  10 # THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS,
  11 # WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED.
  12 #
  13
  14 ##
  15 # Import Modules
  16 #
  17 from __future__ import absolute_import
  18
  19 import sys
  20 import string
  21 import threading
  22 import time
  23 import re
  24 import pickle
  25 import array
  26 import shutil
  27 from random import sample
  28 from struct import pack
  29 import uuid
  30 import subprocess
  31 from collections import OrderedDict
  32
  33 import Common.LongFilePathOs as os
  34 from Common import EdkLogger as EdkLogger
  35 from Common import GlobalData as GlobalData
  36 from Common.DataType import *
  37 from Common.BuildToolError import *
  38 from CommonDataClass.DataClass import *
  39 from Common.Parsing import GetSplitValueList
  40 from Common.LongFilePathSupport import OpenLongFilePath as open
  41 from Common.MultipleWorkspace import MultipleWorkspace as mws
  42 from CommonDataClass.Exceptions import BadExpression
  43 from Common.caching import cached_property
  44
  45 ## Regular expression used to find out place holders in string template
  46 gPlaceholderPattern = re.compile("\$\{([^$()\s]+)\}", re.MULTILINE | re.UNICODE)
  47
  48 ## regular expressions for map file processing
  49 startPatternGeneral = re.compile("^Start[' ']+Length[' ']+Name[' ']+Class")
  50 addressPatternGeneral = re.compile("^Address[' ']+Publics by Value[' ']+Rva\+Base")
  51 valuePatternGcc = re.compile('^([\w_\.]+) +([\da-fA-Fx]+) +([\da-fA-Fx]+)$')
  52 pcdPatternGcc = re.compile('^([\da-fA-Fx]+) +([\da-fA-Fx]+)')
  53 secReGeneral = re.compile('^([\da-fA-F]+):([\da-fA-F]+) +([\da-fA-F]+)[Hh]? +([.\w\$]+) +(\w+)', re.UNICODE)
  54
  55 StructPattern = re.compile(r'[_a-zA-Z][0-9A-Za-z_]*$')
  56
  57 ## Dictionary used to store dependencies of files
  58 gDependencyDatabase = {}    # arch : {file path : [dependent files list]}
  59
  60 #
  61 # If a module is built more than once with different PCDs or library classes
  62 # a temporary INF file with same content is created, the temporary file is removed
  63 # when build exits.
  64 #
  65 _TempInfs = []
  66
  67 def GetVariableOffset(mapfilepath, efifilepath, varnames):
  68     """ Parse map file to get variable offset in current EFI file
  69     @param mapfilepath    Map file absolution path
  70     @param efifilepath:   EFI binary file full path
  71     @param varnames       iteratable container whose elements are variable names to be searched
  72
  73     @return List whos elements are tuple with variable name and raw offset
  74     """
  75     lines = []
  76     try:
  77         f = open(mapfilepath, 'r')
  78         lines = f.readlines()
  79         f.close()
  80     except:
  81         return None
  82
  83     if len(lines) == 0: return None
  84     firstline = lines[0].strip()
  85     if (firstline.startswith("Archive member included ") and
  86         firstline.endswith(" file (symbol)")):
  87         return _parseForGCC(lines, efifilepath, varnames)
  88     if firstline.startswith("# Path:"):
  89         return _parseForXcode(lines, efifilepath, varnames)
  90     return _parseGeneral(lines, efifilepath, varnames)
  91
  92 def _parseForXcode(lines, efifilepath, varnames):
  93     status = 0
  94     ret = []
  95     for line in lines:
  96         line = line.strip()
  97         if status == 0 and line == "# Symbols:":
  98             status = 1
  99             continue
 100         if status == 1 and len(line) != 0:
 101             for varname in varnames:
 102                 if varname in line:
 103                     # cannot pregenerate this RegEx since it uses varname from varnames.
 104                     m = re.match('^([\da-fA-FxX]+)([\s\S]*)([_]*%s)$' % varname, line)
 105                     if m is not None:
 106                         ret.append((varname, m.group(1)))
 107     return ret
 108
 109 def _parseForGCC(lines, efifilepath, varnames):
 110     """ Parse map file generated by GCC linker """
 111     status = 0
 112     sections = []
 113     varoffset = []
 114     for index, line in enumerate(lines):
 115         line = line.strip()
 116         # status machine transection
 117         if status == 0 and line == "Memory Configuration":
 118             status = 1
 119             continue
 120         elif status == 1 and line == 'Linker script and memory map':
 121             status = 2
 122             continue
 123         elif status ==2 and line == 'START GROUP':
 124             status = 3
 125             continue
 126
 127         # status handler
 128         if status == 3:
 129             m = valuePatternGcc.match(line)
 130             if m is not None:
 131                 sections.append(m.groups(0))
 132             for varname in varnames:
 133                 Str = ''
 134                 m = re.match("^.data.(%s)" % varname, line)
 135                 if m is not None:
 136                     m = re.match(".data.(%s)$" % varname, line)
 137                     if m is not None:
 138                         Str = lines[index + 1]
 139                     else:
 140                         Str = line[len(".data.%s" % varname):]
 141                     if Str:
 142                         m = pcdPatternGcc.match(Str.strip())
 143                         if m is not None:
 144                             varoffset.append((varname, int(m.groups(0)[0], 16), int(sections[-1][1], 16), sections[-1][0]))
 145
 146     if not varoffset:
 147         return []
 148     # get section information from efi file
 149     efisecs = PeImageClass(efifilepath).SectionHeaderList
 150     if efisecs is None or len(efisecs) == 0:
 151         return []
 152     #redirection
 153     redirection = 0
 154     for efisec in efisecs:
 155         for section in sections:
 156             if section[0].strip() == efisec[0].strip() and section[0].strip() == '.text':
 157                 redirection = int(section[1], 16) - efisec[1]
 158
 159     ret = []
 160     for var in varoffset:
 161         for efisec in efisecs:
 162             if var[1] >= efisec[1] and var[1] < efisec[1]+efisec[3]:
 163                 ret.append((var[0], hex(efisec[2] + var[1] - efisec[1] - redirection)))
 164     return ret
 165
 166 def _parseGeneral(lines, efifilepath, varnames):
 167     status = 0    #0 - beginning of file; 1 - PE section definition; 2 - symbol table
 168     secs  = []    # key = section name
 169     varoffset = []
 170     symRe = re.compile('^([\da-fA-F]+):([\da-fA-F]+) +([\.:\\\\\w\?@\$]+) +([\da-fA-F]+)', re.UNICODE)
 171
 172     for line in lines:
 173         line = line.strip()
 174         if startPatternGeneral.match(line):
 175             status = 1
 176             continue
 177         if addressPatternGeneral.match(line):
 178             status = 2
 179             continue
 180         if line.startswith("entry point at"):
 181             status = 3
 182             continue
 183         if status == 1 and len(line) != 0:
 184             m =  secReGeneral.match(line)
 185             assert m is not None, "Fail to parse the section in map file , line is %s" % line
 186             sec_no, sec_start, sec_length, sec_name, sec_class = m.groups(0)
 187             secs.append([int(sec_no, 16), int(sec_start, 16), int(sec_length, 16), sec_name, sec_class])
 188         if status == 2 and len(line) != 0:
 189             for varname in varnames:
 190                 m = symRe.match(line)
 191                 assert m is not None, "Fail to parse the symbol in map file, line is %s" % line
 192                 sec_no, sym_offset, sym_name, vir_addr = m.groups(0)
 193                 sec_no     = int(sec_no,     16)
 194                 sym_offset = int(sym_offset, 16)
 195                 vir_addr   = int(vir_addr,   16)
 196                 # cannot pregenerate this RegEx since it uses varname from varnames.
 197                 m2 = re.match('^[_]*(%s)' % varname, sym_name)
 198                 if m2 is not None:
 199                     # fond a binary pcd entry in map file
 200                     for sec in secs:
 201                         if sec[0] == sec_no and (sym_offset >= sec[1] and sym_offset < sec[1] + sec[2]):
 202                             varoffset.append([varname, sec[3], sym_offset, vir_addr, sec_no])
 203
 204     if not varoffset: return []
 205
 206     # get section information from efi file
 207     efisecs = PeImageClass(efifilepath).SectionHeaderList
 208     if efisecs is None or len(efisecs) == 0:
 209         return []
 210
 211     ret = []
 212     for var in varoffset:
 213         index = 0
 214         for efisec in efisecs:
 215             index = index + 1
 216             if var[1].strip() == efisec[0].strip():
 217                 ret.append((var[0], hex(efisec[2] + var[2])))
 218             elif var[4] == index:
 219                 ret.append((var[0], hex(efisec[2] + var[2])))
 220
 221     return ret
 222
 223 ## Routine to process duplicated INF
 224 #
 225 #  This function is called by following two cases:
 226 #  Case 1 in DSC:
 227 #    [components.arch]
 228 #    Pkg/module/module.inf
 229 #    Pkg/module/module.inf {
 230 #      <Defines>
 231 #        FILE_GUID = 0D1B936F-68F3-4589-AFCC-FB8B7AEBC836
 232 #    }
 233 #  Case 2 in FDF:
 234 #    INF Pkg/module/module.inf
 235 #    INF FILE_GUID = 0D1B936F-68F3-4589-AFCC-FB8B7AEBC836 Pkg/module/module.inf
 236 #
 237 #  This function copies Pkg/module/module.inf to
 238 #  Conf/.cache/0D1B936F-68F3-4589-AFCC-FB8B7AEBC836module.inf
 239 #
 240 #  @param Path     Original PathClass object
 241 #  @param BaseName New file base name
 242 #
 243 #  @retval         return the new PathClass object
 244 #
 245 def ProcessDuplicatedInf(Path, BaseName, Workspace):
 246     Filename = os.path.split(Path.File)[1]
 247     if '.' in Filename:
 248         Filename = BaseName + Path.BaseName + Filename[Filename.rfind('.'):]
 249     else:
 250         Filename = BaseName + Path.BaseName
 251
 252     #
 253     # If -N is specified on command line, cache is disabled
 254     # The directory has to be created
 255     #
 256     DbDir = os.path.split(GlobalData.gDatabasePath)[0]
 257     if not os.path.exists(DbDir):
 258         os.makedirs(DbDir)
 259     #
 260     # A temporary INF is copied to database path which must have write permission
 261     # The temporary will be removed at the end of build
 262     # In case of name conflict, the file name is
 263     # FILE_GUIDBaseName (0D1B936F-68F3-4589-AFCC-FB8B7AEBC836module.inf)
 264     #
 265     TempFullPath = os.path.join(DbDir,
 266                                 Filename)
 267     RtPath = PathClass(Path.File, Workspace)
 268     #
 269     # Modify the full path to temporary path, keep other unchanged
 270     #
 271     # To build same module more than once, the module path with FILE_GUID overridden has
 272     # the file name FILE_GUIDmodule.inf, but the relative path (self.MetaFile.File) is the real path
 273     # in DSC which is used as relative path by C files and other files in INF.
 274     # A trick was used: all module paths are PathClass instances, after the initialization
 275     # of PathClass, the PathClass.Path is overridden by the temporary INF path.
 276     #
 277     # The reason for creating a temporary INF is:
 278     # Platform.Modules which is the base to create ModuleAutoGen objects is a dictionary,
 279     # the key is the full path of INF, the value is an object to save overridden library instances, PCDs.
 280     # A different key for the same module is needed to create different output directory,
 281     # retrieve overridden PCDs, library instances.
 282     #
 283     # The BaseName is the FILE_GUID which is also the output directory name.
 284     #
 285     #
 286     RtPath.Path = TempFullPath
 287     RtPath.BaseName = BaseName
 288     #
 289     # If file exists, compare contents
 290     #
 291     if os.path.exists(TempFullPath):
 292         with open(str(Path), 'rb') as f1, open(TempFullPath, 'rb') as f2:
 293             if f1.read() == f2.read():
 294                 return RtPath
 295     _TempInfs.append(TempFullPath)
 296     shutil.copy2(str(Path), TempFullPath)
 297     return RtPath
 298
 299 ## Remove temporary created INFs whose paths were saved in _TempInfs
 300 #
 301 def ClearDuplicatedInf():
 302     while _TempInfs:
 303         File = _TempInfs.pop()
 304         if os.path.exists(File):
 305             os.remove(File)
 306
 307 ## Convert GUID string in xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx style to C structure style
 308 #
 309 #   @param      Guid    The GUID string
 310 #
 311 #   @retval     string  The GUID string in C structure style
 312 #
 313 def GuidStringToGuidStructureString(Guid):
 314     GuidList = Guid.split('-')
 315     Result = '{'
 316     for Index in range(0, 3, 1):
 317         Result = Result + '0x' + GuidList[Index] + ', '
 318     Result = Result + '{0x' + GuidList[3][0:2] + ', 0x' + GuidList[3][2:4]
 319     for Index in range(0, 12, 2):
 320         Result = Result + ', 0x' + GuidList[4][Index:Index + 2]
 321     Result += '}}'
 322     return Result
 323
 324 ## Convert GUID structure in byte array to xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
 325 #
 326 #   @param      GuidValue   The GUID value in byte array
 327 #
 328 #   @retval     string      The GUID value in xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx format
 329 #
 330 def GuidStructureByteArrayToGuidString(GuidValue):
 331     guidValueString = GuidValue.lower().replace("{", "").replace("}", "").replace(" ", "").replace(";", "")
 332     guidValueList = guidValueString.split(",")
 333     if len(guidValueList) != 16:
 334         return ''
 335         #EdkLogger.error(None, None, "Invalid GUID value string %s" % GuidValue)
 336     try:
 337         return "%02x%02x%02x%02x-%02x%02x-%02x%02x-%02x%02x-%02x%02x%02x%02x%02x%02x" % (
 338                 int(guidValueList[3], 16),
 339                 int(guidValueList[2], 16),
 340                 int(guidValueList[1], 16),
 341                 int(guidValueList[0], 16),
 342                 int(guidValueList[5], 16),
 343                 int(guidValueList[4], 16),
 344                 int(guidValueList[7], 16),
 345                 int(guidValueList[6], 16),
 346                 int(guidValueList[8], 16),
 347                 int(guidValueList[9], 16),
 348                 int(guidValueList[10], 16),
 349                 int(guidValueList[11], 16),
 350                 int(guidValueList[12], 16),
 351                 int(guidValueList[13], 16),
 352                 int(guidValueList[14], 16),
 353                 int(guidValueList[15], 16)
 354                 )
 355     except:
 356         return ''
 357
 358 ## Convert GUID string in C structure style to xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
 359 #
 360 #   @param      GuidValue   The GUID value in C structure format
 361 #
 362 #   @retval     string      The GUID value in xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx format
 363 #
 364 def GuidStructureStringToGuidString(GuidValue):
 365     if not GlobalData.gGuidCFormatPattern.match(GuidValue):
 366         return ''
 367     guidValueString = GuidValue.lower().replace("{", "").replace("}", "").replace(" ", "").replace(";", "")
 368     guidValueList = guidValueString.split(",")
 369     if len(guidValueList) != 11:
 370         return ''
 371         #EdkLogger.error(None, None, "Invalid GUID value string %s" % GuidValue)
 372     try:
 373         return "%08x-%04x-%04x-%02x%02x-%02x%02x%02x%02x%02x%02x" % (
 374                 int(guidValueList[0], 16),
 375                 int(guidValueList[1], 16),
 376                 int(guidValueList[2], 16),
 377                 int(guidValueList[3], 16),
 378                 int(guidValueList[4], 16),
 379                 int(guidValueList[5], 16),
 380                 int(guidValueList[6], 16),
 381                 int(guidValueList[7], 16),
 382                 int(guidValueList[8], 16),
 383                 int(guidValueList[9], 16),
 384                 int(guidValueList[10], 16)
 385                 )
 386     except:
 387         return ''
 388
 389 ## Convert GUID string in C structure style to xxxxxxxx_xxxx_xxxx_xxxx_xxxxxxxxxxxx
 390 #
 391 #   @param      GuidValue   The GUID value in C structure format
 392 #
 393 #   @retval     string      The GUID value in xxxxxxxx_xxxx_xxxx_xxxx_xxxxxxxxxxxx format
 394 #
 395 def GuidStructureStringToGuidValueName(GuidValue):
 396     guidValueString = GuidValue.lower().replace("{", "").replace("}", "").replace(" ", "")
 397     guidValueList = guidValueString.split(",")
 398     if len(guidValueList) != 11:
 399         EdkLogger.error(None, FORMAT_INVALID, "Invalid GUID value string [%s]" % GuidValue)
 400     return "%08x_%04x_%04x_%02x%02x_%02x%02x%02x%02x%02x%02x" % (
 401             int(guidValueList[0], 16),
 402             int(guidValueList[1], 16),
 403             int(guidValueList[2], 16),
 404             int(guidValueList[3], 16),
 405             int(guidValueList[4], 16),
 406             int(guidValueList[5], 16),
 407             int(guidValueList[6], 16),
 408             int(guidValueList[7], 16),
 409             int(guidValueList[8], 16),
 410             int(guidValueList[9], 16),
 411             int(guidValueList[10], 16)
 412             )
 413
 414 ## Create directories
 415 #
 416 #   @param      Directory   The directory name
 417 #
 418 def CreateDirectory(Directory):
 419     if Directory is None or Directory.strip() == "":
 420         return True
 421     try:
 422         if not os.access(Directory, os.F_OK):
 423             os.makedirs(Directory)
 424     except:
 425         return False
 426     return True
 427
 428 ## Remove directories, including files and sub-directories in it
 429 #
 430 #   @param      Directory   The directory name
 431 #
 432 def RemoveDirectory(Directory, Recursively=False):
 433     if Directory is None or Directory.strip() == "" or not os.path.exists(Directory):
 434         return
 435     if Recursively:
 436         CurrentDirectory = os.getcwd()
 437         os.chdir(Directory)
 438         for File in os.listdir("."):
 439             if os.path.isdir(File):
 440                 RemoveDirectory(File, Recursively)
 441             else:
 442                 os.remove(File)
 443         os.chdir(CurrentDirectory)
 444     os.rmdir(Directory)
 445
 446 ## Store content in file
 447 #
 448 #  This method is used to save file only when its content is changed. This is
 449 #  quite useful for "make" system to decide what will be re-built and what won't.
 450 #
 451 #   @param      File            The path of file
 452 #   @param      Content         The new content of the file
 453 #   @param      IsBinaryFile    The flag indicating if the file is binary file or not
 454 #
 455 #   @retval     True            If the file content is changed and the file is renewed
 456 #   @retval     False           If the file content is the same
 457 #
 458 def SaveFileOnChange(File, Content, IsBinaryFile=True):
 459     if not IsBinaryFile:
 460         Content = Content.replace("\n", os.linesep)
 461
 462     if os.path.exists(File):
 463         try:
 464             if Content == open(File, "rb").read():
 465                 return False
 466         except:
 467             EdkLogger.error(None, FILE_OPEN_FAILURE, ExtraData=File)
 468
 469     DirName = os.path.dirname(File)
 470     if not CreateDirectory(DirName):
 471         EdkLogger.error(None, FILE_CREATE_FAILURE, "Could not create directory %s" % DirName)
 472     else:
 473         if DirName == '':
 474             DirName = os.getcwd()
 475         if not os.access(DirName, os.W_OK):
 476             EdkLogger.error(None, PERMISSION_FAILURE, "Do not have write permission on directory %s" % DirName)
 477
 478     try:
 479         Fd = open(File, "wb")
 480         Fd.write(Content)
 481         Fd.close()
 482     except IOError as X:
 483         EdkLogger.error(None, FILE_CREATE_FAILURE, ExtraData='IOError %s' % X)
 484
 485     return True
 486
 487 ## Retrieve and cache the real path name in file system
 488 #
 489 #   @param      Root    The root directory of path relative to
 490 #
 491 #   @retval     str     The path string if the path exists
 492 #   @retval     None    If path doesn't exist
 493 #
 494 class DirCache:
 495     _CACHE_ = set()
 496     _UPPER_CACHE_ = {}
 497
 498     def __init__(self, Root):
 499         self._Root = Root
 500         for F in os.listdir(Root):
 501             self._CACHE_.add(F)
 502             self._UPPER_CACHE_[F.upper()] = F
 503
 504     # =[] operator
 505     def __getitem__(self, Path):
 506         Path = Path[len(os.path.commonprefix([Path, self._Root])):]
 507         if not Path:
 508             return self._Root
 509         if Path and Path[0] == os.path.sep:
 510             Path = Path[1:]
 511         if Path in self._CACHE_:
 512             return os.path.join(self._Root, Path)
 513         UpperPath = Path.upper()
 514         if UpperPath in self._UPPER_CACHE_:
 515             return os.path.join(self._Root, self._UPPER_CACHE_[UpperPath])
 516
 517         IndexList = []
 518         LastSepIndex = -1
 519         SepIndex = Path.find(os.path.sep)
 520         while SepIndex > -1:
 521             Parent = UpperPath[:SepIndex]
 522             if Parent not in self._UPPER_CACHE_:
 523                 break
 524             LastSepIndex = SepIndex
 525             SepIndex = Path.find(os.path.sep, LastSepIndex + 1)
 526
 527         if LastSepIndex == -1:
 528             return None
 529
 530         Cwd = os.getcwd()
 531         os.chdir(self._Root)
 532         SepIndex = LastSepIndex
 533         while SepIndex > -1:
 534             Parent = Path[:SepIndex]
 535             ParentKey = UpperPath[:SepIndex]
 536             if ParentKey not in self._UPPER_CACHE_:
 537                 os.chdir(Cwd)
 538                 return None
 539
 540             if Parent in self._CACHE_:
 541                 ParentDir = Parent
 542             else:
 543                 ParentDir = self._UPPER_CACHE_[ParentKey]
 544             for F in os.listdir(ParentDir):
 545                 Dir = os.path.join(ParentDir, F)
 546                 self._CACHE_.add(Dir)
 547                 self._UPPER_CACHE_[Dir.upper()] = Dir
 548
 549             SepIndex = Path.find(os.path.sep, SepIndex + 1)
 550
 551         os.chdir(Cwd)
 552         if Path in self._CACHE_:
 553             return os.path.join(self._Root, Path)
 554         elif UpperPath in self._UPPER_CACHE_:
 555             return os.path.join(self._Root, self._UPPER_CACHE_[UpperPath])
 556         return None
 557
 558 def RealPath(File, Dir='', OverrideDir=''):
 559     NewFile = os.path.normpath(os.path.join(Dir, File))
 560     NewFile = GlobalData.gAllFiles[NewFile]
 561     if not NewFile and OverrideDir:
 562         NewFile = os.path.normpath(os.path.join(OverrideDir, File))
 563         NewFile = GlobalData.gAllFiles[NewFile]
 564     return NewFile
 565
 566 ## Get GUID value from given packages
 567 #
 568 #   @param      CName           The CName of the GUID
 569 #   @param      PackageList     List of packages looking-up in
 570 #   @param      Inffile         The driver file
 571 #
 572 #   @retval     GuidValue   if the CName is found in any given package
 573 #   @retval     None        if the CName is not found in all given packages
 574 #
 575 def GuidValue(CName, PackageList, Inffile = None):
 576     for P in PackageList:
 577         GuidKeys = list(P.Guids.keys())
 578         if Inffile and P._PrivateGuids:
 579             if not Inffile.startswith(P.MetaFile.Dir):
 580                 GuidKeys = [x for x in P.Guids if x not in P._PrivateGuids]
 581         if CName in GuidKeys:
 582             return P.Guids[CName]
 583     return None
 584     return None
 585
 586 ## A string template class
 587 #
 588 #  This class implements a template for string replacement. A string template
 589 #  looks like following
 590 #
 591 #       ${BEGIN} other_string ${placeholder_name} other_string ${END}
 592 #
 593 #  The string between ${BEGIN} and ${END} will be repeated as many times as the
 594 #  length of "placeholder_name", which is a list passed through a dict. The
 595 #  "placeholder_name" is the key name of the dict. The ${BEGIN} and ${END} can
 596 #  be not used and, in this case, the "placeholder_name" must not a list and it
 597 #  will just be replaced once.
 598 #
 599 class TemplateString(object):
 600     _REPEAT_START_FLAG = "BEGIN"
 601     _REPEAT_END_FLAG = "END"
 602
 603     class Section(object):
 604         _LIST_TYPES = [type([]), type(set()), type((0,))]
 605
 606         def __init__(self, TemplateSection, PlaceHolderList):
 607             self._Template = TemplateSection
 608             self._PlaceHolderList = []
 609
 610             # Split the section into sub-sections according to the position of placeholders
 611             if PlaceHolderList:
 612                 self._SubSectionList = []
 613                 SubSectionStart = 0
 614                 #
 615                 # The placeholders passed in must be in the format of
 616                 #
 617                 #   PlaceHolderName, PlaceHolderStartPoint, PlaceHolderEndPoint
 618                 #
 619                 for PlaceHolder, Start, End in PlaceHolderList:
 620                     self._SubSectionList.append(TemplateSection[SubSectionStart:Start])
 621                     self._SubSectionList.append(TemplateSection[Start:End])
 622                     self._PlaceHolderList.append(PlaceHolder)
 623                     SubSectionStart = End
 624                 if SubSectionStart < len(TemplateSection):
 625                     self._SubSectionList.append(TemplateSection[SubSectionStart:])
 626             else:
 627                 self._SubSectionList = [TemplateSection]
 628
 629         def __str__(self):
 630             return self._Template + " : " + str(self._PlaceHolderList)
 631
 632         def Instantiate(self, PlaceHolderValues):
 633             RepeatTime = -1
 634             RepeatPlaceHolders = {}
 635             NonRepeatPlaceHolders = {}
 636
 637             for PlaceHolder in self._PlaceHolderList:
 638                 if PlaceHolder not in PlaceHolderValues:
 639                     continue
 640                 Value = PlaceHolderValues[PlaceHolder]
 641                 if type(Value) in self._LIST_TYPES:
 642                     if RepeatTime < 0:
 643                         RepeatTime = len(Value)
 644                     elif RepeatTime != len(Value):
 645                         EdkLogger.error(
 646                                     "TemplateString",
 647                                     PARAMETER_INVALID,
 648                                     "${%s} has different repeat time from others!" % PlaceHolder,
 649                                     ExtraData=str(self._Template)
 650                                     )
 651                     RepeatPlaceHolders["${%s}" % PlaceHolder] = Value
 652                 else:
 653                     NonRepeatPlaceHolders["${%s}" % PlaceHolder] = Value
 654
 655             if NonRepeatPlaceHolders:
 656                 StringList = []
 657                 for S in self._SubSectionList:
 658                     if S not in NonRepeatPlaceHolders:
 659                         StringList.append(S)
 660                     else:
 661                         StringList.append(str(NonRepeatPlaceHolders[S]))
 662             else:
 663                 StringList = self._SubSectionList
 664
 665             if RepeatPlaceHolders:
 666                 TempStringList = []
 667                 for Index in range(RepeatTime):
 668                     for S in StringList:
 669                         if S not in RepeatPlaceHolders:
 670                             TempStringList.append(S)
 671                         else:
 672                             TempStringList.append(str(RepeatPlaceHolders[S][Index]))
 673                 StringList = TempStringList
 674
 675             return "".join(StringList)
 676
 677     ## Constructor
 678     def __init__(self, Template=None):
 679         self.String = []
 680         self.IsBinary = False
 681         self._Template = Template
 682         self._TemplateSectionList = self._Parse(Template)
 683
 684     ## str() operator
 685     #
 686     #   @retval     string  The string replaced
 687     #
 688     def __str__(self):
 689         return "".join(self.String)
 690
 691     ## Split the template string into fragments per the ${BEGIN} and ${END} flags
 692     #
 693     #   @retval     list    A list of TemplateString.Section objects
 694     #
 695     def _Parse(self, Template):
 696         SectionStart = 0
 697         SearchFrom = 0
 698         MatchEnd = 0
 699         PlaceHolderList = []
 700         TemplateSectionList = []
 701         while Template:
 702             MatchObj = gPlaceholderPattern.search(Template, SearchFrom)
 703             if not MatchObj:
 704                 if MatchEnd <= len(Template):
 705                     TemplateSection = TemplateString.Section(Template[SectionStart:], PlaceHolderList)
 706                     TemplateSectionList.append(TemplateSection)
 707                 break
 708
 709             MatchString = MatchObj.group(1)
 710             MatchStart = MatchObj.start()
 711             MatchEnd = MatchObj.end()
 712
 713             if MatchString == self._REPEAT_START_FLAG:
 714                 if MatchStart > SectionStart:
 715                     TemplateSection = TemplateString.Section(Template[SectionStart:MatchStart], PlaceHolderList)
 716                     TemplateSectionList.append(TemplateSection)
 717                 SectionStart = MatchEnd
 718                 PlaceHolderList = []
 719             elif MatchString == self._REPEAT_END_FLAG:
 720                 TemplateSection = TemplateString.Section(Template[SectionStart:MatchStart], PlaceHolderList)
 721                 TemplateSectionList.append(TemplateSection)
 722                 SectionStart = MatchEnd
 723                 PlaceHolderList = []
 724             else:
 725                 PlaceHolderList.append((MatchString, MatchStart - SectionStart, MatchEnd - SectionStart))
 726             SearchFrom = MatchEnd
 727         return TemplateSectionList
 728
 729     ## Replace the string template with dictionary of placeholders and append it to previous one
 730     #
 731     #   @param      AppendString    The string template to append
 732     #   @param      Dictionary      The placeholder dictionaries
 733     #
 734     def Append(self, AppendString, Dictionary=None):
 735         if Dictionary:
 736             SectionList = self._Parse(AppendString)
 737             self.String.append( "".join(S.Instantiate(Dictionary) for S in SectionList))
 738         else:
 739             if isinstance(AppendString,list):
 740                 self.String.extend(AppendString)
 741             else:
 742                 self.String.append(AppendString)
 743
 744     ## Replace the string template with dictionary of placeholders
 745     #
 746     #   @param      Dictionary      The placeholder dictionaries
 747     #
 748     #   @retval     str             The string replaced with placeholder values
 749     #
 750     def Replace(self, Dictionary=None):
 751         return "".join(S.Instantiate(Dictionary) for S in self._TemplateSectionList)
 752
 753 ## Progress indicator class
 754 #
 755 #  This class makes use of thread to print progress on console.
 756 #
 757 class Progressor:
 758     # for avoiding deadloop
 759     _StopFlag = None
 760     _ProgressThread = None
 761     _CheckInterval = 0.25
 762
 763     ## Constructor
 764     #
 765     #   @param      OpenMessage     The string printed before progress charaters
 766     #   @param      CloseMessage    The string printed after progress charaters
 767     #   @param      ProgressChar    The charater used to indicate the progress
 768     #   @param      Interval        The interval in seconds between two progress charaters
 769     #
 770     def __init__(self, OpenMessage="", CloseMessage="", ProgressChar='.', Interval=1.0):
 771         self.PromptMessage = OpenMessage
 772         self.CodaMessage = CloseMessage
 773         self.ProgressChar = ProgressChar
 774         self.Interval = Interval
 775         if Progressor._StopFlag is None:
 776             Progressor._StopFlag = threading.Event()
 777
 778     ## Start to print progress charater
 779     #
 780     #   @param      OpenMessage     The string printed before progress charaters
 781     #
 782     def Start(self, OpenMessage=None):
 783         if OpenMessage is not None:
 784             self.PromptMessage = OpenMessage
 785         Progressor._StopFlag.clear()
 786         if Progressor._ProgressThread is None:
 787             Progressor._ProgressThread = threading.Thread(target=self._ProgressThreadEntry)
 788             Progressor._ProgressThread.setDaemon(False)
 789             Progressor._ProgressThread.start()
 790
 791     ## Stop printing progress charater
 792     #
 793     #   @param      CloseMessage    The string printed after progress charaters
 794     #
 795     def Stop(self, CloseMessage=None):
 796         OriginalCodaMessage = self.CodaMessage
 797         if CloseMessage is not None:
 798             self.CodaMessage = CloseMessage
 799         self.Abort()
 800         self.CodaMessage = OriginalCodaMessage
 801
 802     ## Thread entry method
 803     def _ProgressThreadEntry(self):
 804         sys.stdout.write(self.PromptMessage + " ")
 805         sys.stdout.flush()
 806         TimeUp = 0.0
 807         while not Progressor._StopFlag.isSet():
 808             if TimeUp <= 0.0:
 809                 sys.stdout.write(self.ProgressChar)
 810                 sys.stdout.flush()
 811                 TimeUp = self.Interval
 812             time.sleep(self._CheckInterval)
 813             TimeUp -= self._CheckInterval
 814         sys.stdout.write(" " + self.CodaMessage + "\n")
 815         sys.stdout.flush()
 816
 817     ## Abort the progress display
 818     @staticmethod
 819     def Abort():
 820         if Progressor._StopFlag is not None:
 821             Progressor._StopFlag.set()
 822         if Progressor._ProgressThread is not None:
 823             Progressor._ProgressThread.join()
 824             Progressor._ProgressThread = None
 825
 826
 827 ## Dictionary using prioritized list as key
 828 #
 829 class tdict:
 830     _ListType = type([])
 831     _TupleType = type(())
 832     _Wildcard = 'COMMON'
 833     _ValidWildcardList = ['COMMON', 'DEFAULT', 'ALL', TAB_STAR, 'PLATFORM']
 834
 835     def __init__(self, _Single_=False, _Level_=2):
 836         self._Level_ = _Level_
 837         self.data = {}
 838         self._Single_ = _Single_
 839
 840     # =[] operator
 841     def __getitem__(self, key):
 842         KeyType = type(key)
 843         RestKeys = None
 844         if KeyType == self._ListType or KeyType == self._TupleType:
 845             FirstKey = key[0]
 846             if len(key) > 1:
 847                 RestKeys = key[1:]
 848             elif self._Level_ > 1:
 849                 RestKeys = [self._Wildcard for i in range(0, self._Level_ - 1)]
 850         else:
 851             FirstKey = key
 852             if self._Level_ > 1:
 853                 RestKeys = [self._Wildcard for i in range(0, self._Level_ - 1)]
 854
 855         if FirstKey is None or str(FirstKey).upper() in self._ValidWildcardList:
 856             FirstKey = self._Wildcard
 857
 858         if self._Single_:
 859             return self._GetSingleValue(FirstKey, RestKeys)
 860         else:
 861             return self._GetAllValues(FirstKey, RestKeys)
 862
 863     def _GetSingleValue(self, FirstKey, RestKeys):
 864         Value = None
 865         #print "%s-%s" % (FirstKey, self._Level_) ,
 866         if self._Level_ > 1:
 867             if FirstKey == self._Wildcard:
 868                 if FirstKey in self.data:
 869                     Value = self.data[FirstKey][RestKeys]
 870                 if Value is None:
 871                     for Key in self.data:
 872                         Value = self.data[Key][RestKeys]
 873                         if Value is not None: break
 874             else:
 875                 if FirstKey in self.data:
 876                     Value = self.data[FirstKey][RestKeys]
 877                 if Value is None and self._Wildcard in self.data:
 878                     #print "Value=None"
 879                     Value = self.data[self._Wildcard][RestKeys]
 880         else:
 881             if FirstKey == self._Wildcard:
 882                 if FirstKey in self.data:
 883                     Value = self.data[FirstKey]
 884                 if Value is None:
 885                     for Key in self.data:
 886                         Value = self.data[Key]
 887                         if Value is not None: break
 888             else:
 889                 if FirstKey in self.data:
 890                     Value = self.data[FirstKey]
 891                 elif self._Wildcard in self.data:
 892                     Value = self.data[self._Wildcard]
 893         return Value
 894
 895     def _GetAllValues(self, FirstKey, RestKeys):
 896         Value = []
 897         if self._Level_ > 1:
 898             if FirstKey == self._Wildcard:
 899                 for Key in self.data:
 900                     Value += self.data[Key][RestKeys]
 901             else:
 902                 if FirstKey in self.data:
 903                     Value += self.data[FirstKey][RestKeys]
 904                 if self._Wildcard in self.data:
 905                     Value += self.data[self._Wildcard][RestKeys]
 906         else:
 907             if FirstKey == self._Wildcard:
 908                 for Key in self.data:
 909                     Value.append(self.data[Key])
 910             else:
 911                 if FirstKey in self.data:
 912                     Value.append(self.data[FirstKey])
 913                 if self._Wildcard in self.data:
 914                     Value.append(self.data[self._Wildcard])
 915         return Value
 916
 917     ## []= operator
 918     def __setitem__(self, key, value):
 919         KeyType = type(key)
 920         RestKeys = None
 921         if KeyType == self._ListType or KeyType == self._TupleType:
 922             FirstKey = key[0]
 923             if len(key) > 1:
 924                 RestKeys = key[1:]
 925             else:
 926                 RestKeys = [self._Wildcard for i in range(0, self._Level_ - 1)]
 927         else:
 928             FirstKey = key
 929             if self._Level_ > 1:
 930                 RestKeys = [self._Wildcard for i in range(0, self._Level_ - 1)]
 931
 932         if FirstKey in self._ValidWildcardList:
 933             FirstKey = self._Wildcard
 934
 935         if FirstKey not in self.data and self._Level_ > 0:
 936             self.data[FirstKey] = tdict(self._Single_, self._Level_ - 1)
 937
 938         if self._Level_ > 1:
 939             self.data[FirstKey][RestKeys] = value
 940         else:
 941             self.data[FirstKey] = value
 942
 943     def SetGreedyMode(self):
 944         self._Single_ = False
 945         if self._Level_ > 1:
 946             for Key in self.data:
 947                 self.data[Key].SetGreedyMode()
 948
 949     def SetSingleMode(self):
 950         self._Single_ = True
 951         if self._Level_ > 1:
 952             for Key in self.data:
 953                 self.data[Key].SetSingleMode()
 954
 955     def GetKeys(self, KeyIndex=0):
 956         assert KeyIndex >= 0
 957         if KeyIndex == 0:
 958             return set(self.data.keys())
 959         else:
 960             keys = set()
 961             for Key in self.data:
 962                 keys |= self.data[Key].GetKeys(KeyIndex - 1)
 963             return keys
 964
 965 def AnalyzePcdExpression(Setting):
 966     RanStr = ''.join(sample(string.ascii_letters + string.digits, 8))
 967     Setting = Setting.replace('\\\\', RanStr).strip()
 968     # There might be escaped quote in a string: \", \\\" , \', \\\'
 969     Data = Setting
 970     # There might be '|' in string and in ( ... | ... ), replace it with '-'
 971     NewStr = ''
 972     InSingleQuoteStr = False
 973     InDoubleQuoteStr = False
 974     Pair = 0
 975     for Index, ch in enumerate(Data):
 976         if ch == '"' and not InSingleQuoteStr:
 977             if Data[Index - 1] != '\\':
 978                 InDoubleQuoteStr = not InDoubleQuoteStr
 979         elif ch == "'" and not InDoubleQuoteStr:
 980             if Data[Index - 1] != '\\':
 981                 InSingleQuoteStr = not InSingleQuoteStr
 982         elif ch == '(' and not (InSingleQuoteStr or InDoubleQuoteStr):
 983             Pair += 1
 984         elif ch == ')' and not (InSingleQuoteStr or InDoubleQuoteStr):
 985             Pair -= 1
 986
 987         if (Pair > 0 or InSingleQuoteStr or InDoubleQuoteStr) and ch == TAB_VALUE_SPLIT:
 988             NewStr += '-'
 989         else:
 990             NewStr += ch
 991     FieldList = []
 992     StartPos = 0
 993     while True:
 994         Pos = NewStr.find(TAB_VALUE_SPLIT, StartPos)
 995         if Pos < 0:
 996             FieldList.append(Setting[StartPos:].strip())
 997             break
 998         FieldList.append(Setting[StartPos:Pos].strip())
 999         StartPos = Pos + 1
1000     for i, ch in enumerate(FieldList):
1001         if RanStr in ch:
1002             FieldList[i] = ch.replace(RanStr,'\\\\')
1003     return FieldList
1004
1005 def ParseFieldValue (Value):
1006     def ParseDevPathValue (Value):
1007         if '\\' in Value:
1008             Value.replace('\\', '/').replace(' ', '')
1009
1010         Cmd = 'DevicePath ' + '"' + Value + '"'
1011         try:
1012             p = subprocess.Popen(Cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
1013             out, err = p.communicate()
1014         except Exception as X:
1015             raise BadExpression("DevicePath: %s" % (str(X)) )
1016         finally:
1017             subprocess._cleanup()
1018             p.stdout.close()
1019             p.stderr.close()
1020         if err:
1021             raise BadExpression("DevicePath: %s" % str(err))
1022         Size = len(out.split())
1023         out = ','.join(out.split())
1024         return '{' + out + '}', Size
1025
1026     if "{CODE(" in Value:
1027         return Value, len(Value.split(","))
1028     if isinstance(Value, type(0)):
1029         return Value, (Value.bit_length() + 7) // 8
1030     if not isinstance(Value, type('')):
1031         raise BadExpression('Type %s is %s' %(Value, type(Value)))
1032     Value = Value.strip()
1033     if Value.startswith(TAB_UINT8) and Value.endswith(')'):
1034         Value, Size = ParseFieldValue(Value.split('(', 1)[1][:-1])
1035         if Size > 1:
1036             raise BadExpression('Value (%s) Size larger than %d' %(Value, Size))
1037         return Value, 1
1038     if Value.startswith(TAB_UINT16) and Value.endswith(')'):
1039         Value, Size = ParseFieldValue(Value.split('(', 1)[1][:-1])
1040         if Size > 2:
1041             raise BadExpression('Value (%s) Size larger than %d' %(Value, Size))
1042         return Value, 2
1043     if Value.startswith(TAB_UINT32) and Value.endswith(')'):
1044         Value, Size = ParseFieldValue(Value.split('(', 1)[1][:-1])
1045         if Size > 4:
1046             raise BadExpression('Value (%s) Size larger than %d' %(Value, Size))
1047         return Value, 4
1048     if Value.startswith(TAB_UINT64) and Value.endswith(')'):
1049         Value, Size = ParseFieldValue(Value.split('(', 1)[1][:-1])
1050         if Size > 8:
1051             raise BadExpression('Value (%s) Size larger than %d' % (Value, Size))
1052         return Value, 8
1053     if Value.startswith(TAB_GUID) and Value.endswith(')'):
1054         Value = Value.split('(', 1)[1][:-1].strip()
1055         if Value[0] == '{' and Value[-1] == '}':
1056             TmpValue = GuidStructureStringToGuidString(Value)
1057             if not TmpValue:
1058                 raise BadExpression("Invalid GUID value string %s" % Value)
1059             Value = TmpValue
1060         if Value[0] == '"' and Value[-1] == '"':
1061             Value = Value[1:-1]
1062         try:
1063             Value = "'" + uuid.UUID(Value).bytes_le + "'"
1064         except ValueError as Message:
1065             raise BadExpression(Message)
1066         Value, Size = ParseFieldValue(Value)
1067         return Value, 16
1068     if Value.startswith('L"') and Value.endswith('"'):
1069         # Unicode String
1070         # translate escape character
1071         Value = Value[1:]
1072         try:
1073             Value = eval(Value)
1074         except:
1075             Value = Value[1:-1]
1076         List = list(Value)
1077         List.reverse()
1078         Value = 0
1079         for Char in List:
1080             Value = (Value << 16) | ord(Char)
1081         return Value, (len(List) + 1) * 2
1082     if Value.startswith('"') and Value.endswith('"'):
1083         # ASCII String
1084         # translate escape character
1085         try:
1086             Value = eval(Value)
1087         except:
1088             Value = Value[1:-1]
1089         List = list(Value)
1090         List.reverse()
1091         Value = 0
1092         for Char in List:
1093             Value = (Value << 8) | ord(Char)
1094         return Value, len(List) + 1
1095     if Value.startswith("L'") and Value.endswith("'"):
1096         # Unicode Character Constant
1097         # translate escape character
1098         Value = Value[1:]
1099         try:
1100             Value = eval(Value)
1101         except:
1102             Value = Value[1:-1]
1103         List = list(Value)
1104         if len(List) == 0:
1105             raise BadExpression('Length %s is %s' % (Value, len(List)))
1106         List.reverse()
1107         Value = 0
1108         for Char in List:
1109             Value = (Value << 16) | ord(Char)
1110         return Value, len(List) * 2
1111     if Value.startswith("'") and Value.endswith("'"):
1112         # Character constant
1113         # translate escape character
1114         try:
1115             Value = eval(Value)
1116         except:
1117             Value = Value[1:-1]
1118         List = list(Value)
1119         if len(List) == 0:
1120             raise BadExpression('Length %s is %s' % (Value, len(List)))
1121         List.reverse()
1122         Value = 0
1123         for Char in List:
1124             Value = (Value << 8) | ord(Char)
1125         return Value, len(List)
1126     if Value.startswith('{') and Value.endswith('}'):
1127         # Byte array
1128         Value = Value[1:-1]
1129         List = [Item.strip() for Item in Value.split(',')]
1130         List.reverse()
1131         Value = 0
1132         RetSize = 0
1133         for Item in List:
1134             ItemValue, Size = ParseFieldValue(Item)
1135             RetSize += Size
1136             for I in range(Size):
1137                 Value = (Value << 8) | ((ItemValue >> 8 * I) & 0xff)
1138         return Value, RetSize
1139     if Value.startswith('DEVICE_PATH(') and Value.endswith(')'):
1140         Value = Value.replace("DEVICE_PATH(", '').rstrip(')')
1141         Value = Value.strip().strip('"')
1142         return ParseDevPathValue(Value)
1143     if Value.lower().startswith('0x'):
1144         try:
1145             Value = int(Value, 16)
1146         except:
1147             raise BadExpression("invalid hex value: %s" % Value)
1148         if Value == 0:
1149             return 0, 1
1150         return Value, (Value.bit_length() + 7) // 8
1151     if Value[0].isdigit():
1152         Value = int(Value, 10)
1153         if Value == 0:
1154             return 0, 1
1155         return Value, (Value.bit_length() + 7) // 8
1156     if Value.lower() == 'true':
1157         return 1, 1
1158     if Value.lower() == 'false':
1159         return 0, 1
1160     return Value, 1
1161
1162 ## AnalyzeDscPcd
1163 #
1164 #  Analyze DSC PCD value, since there is no data type info in DSC
1165 #  This function is used to match functions (AnalyzePcdData) used for retrieving PCD value from database
1166 #  1. Feature flag: TokenSpace.PcdCName|PcdValue
1167 #  2. Fix and Patch:TokenSpace.PcdCName|PcdValue[|VOID*[|MaxSize]]
1168 #  3. Dynamic default:
1169 #     TokenSpace.PcdCName|PcdValue[|VOID*[|MaxSize]]
1170 #     TokenSpace.PcdCName|PcdValue
1171 #  4. Dynamic VPD:
1172 #     TokenSpace.PcdCName|VpdOffset[|VpdValue]
1173 #     TokenSpace.PcdCName|VpdOffset[|MaxSize[|VpdValue]]
1174 #  5. Dynamic HII:
1175 #     TokenSpace.PcdCName|HiiString|VariableGuid|VariableOffset[|HiiValue]
1176 #  PCD value needs to be located in such kind of string, and the PCD value might be an expression in which
1177 #    there might have "|" operator, also in string value.
1178 #
1179 #  @param Setting: String contain information described above with "TokenSpace.PcdCName|" stripped
1180 #  @param PcdType: PCD type: feature, fixed, dynamic default VPD HII
1181 #  @param DataType: The datum type of PCD: VOID*, UNIT, BOOL
1182 #  @retval:
1183 #    ValueList: A List contain fields described above
1184 #    IsValid:   True if conforming EBNF, otherwise False
1185 #    Index:     The index where PcdValue is in ValueList
1186 #
1187 def AnalyzeDscPcd(Setting, PcdType, DataType=''):
1188     FieldList = AnalyzePcdExpression(Setting)
1189
1190     IsValid = True
1191     if PcdType in (MODEL_PCD_FIXED_AT_BUILD, MODEL_PCD_PATCHABLE_IN_MODULE, MODEL_PCD_DYNAMIC_DEFAULT, MODEL_PCD_DYNAMIC_EX_DEFAULT):
1192         Value = FieldList[0]
1193         Size = ''
1194         if len(FieldList) > 1 and FieldList[1]:
1195             DataType = FieldList[1]
1196             if FieldList[1] != TAB_VOID and StructPattern.match(FieldList[1]) is None:
1197                 IsValid = False
1198         if len(FieldList) > 2:
1199             Size = FieldList[2]
1200         if IsValid:
1201             if DataType == "":
1202                 IsValid = (len(FieldList) <= 1)
1203             else:
1204                 IsValid = (len(FieldList) <= 3)
1205
1206         if Size:
1207             try:
1208                 int(Size, 16) if Size.upper().startswith("0X") else int(Size)
1209             except:
1210                 IsValid = False
1211                 Size = -1
1212         return [str(Value), DataType, str(Size)], IsValid, 0
1213     elif PcdType == MODEL_PCD_FEATURE_FLAG:
1214         Value = FieldList[0]
1215         Size = ''
1216         IsValid = (len(FieldList) <= 1)
1217         return [Value, DataType, str(Size)], IsValid, 0
1218     elif PcdType in (MODEL_PCD_DYNAMIC_VPD, MODEL_PCD_DYNAMIC_EX_VPD):
1219         VpdOffset = FieldList[0]
1220         Value = Size = ''
1221         if not DataType == TAB_VOID:
1222             if len(FieldList) > 1:
1223                 Value = FieldList[1]
1224         else:
1225             if len(FieldList) > 1:
1226                 Size = FieldList[1]
1227             if len(FieldList) > 2:
1228                 Value = FieldList[2]
1229         if DataType == "":
1230             IsValid = (len(FieldList) <= 1)
1231         else:
1232             IsValid = (len(FieldList) <= 3)
1233         if Size:
1234             try:
1235                 int(Size, 16) if Size.upper().startswith("0X") else int(Size)
1236             except:
1237                 IsValid = False
1238                 Size = -1
1239         return [VpdOffset, str(Size), Value], IsValid, 2
1240     elif PcdType in (MODEL_PCD_DYNAMIC_HII, MODEL_PCD_DYNAMIC_EX_HII):
1241         IsValid = (3 <= len(FieldList) <= 5)
1242         HiiString = FieldList[0]
1243         Guid = Offset = Value = Attribute = ''
1244         if len(FieldList) > 1:
1245             Guid = FieldList[1]
1246         if len(FieldList) > 2:
1247             Offset = FieldList[2]
1248         if len(FieldList) > 3:
1249             Value = FieldList[3]
1250         if len(FieldList) > 4:
1251             Attribute = FieldList[4]
1252         return [HiiString, Guid, Offset, Value, Attribute], IsValid, 3
1253     return [], False, 0
1254
1255 ## AnalyzePcdData
1256 #
1257 #  Analyze the pcd Value, Datum type and TokenNumber.
1258 #  Used to avoid split issue while the value string contain "|" character
1259 #
1260 #  @param[in] Setting:  A String contain value/datum type/token number information;
1261 #
1262 #  @retval   ValueList: A List contain value, datum type and toke number.
1263 #
1264 def AnalyzePcdData(Setting):
1265     ValueList = ['', '', '']
1266
1267     ValueRe = re.compile(r'^\s*L?\".*\|.*\"')
1268     PtrValue = ValueRe.findall(Setting)
1269
1270     ValueUpdateFlag = False
1271
1272     if len(PtrValue) >= 1:
1273         Setting = re.sub(ValueRe, '', Setting)
1274         ValueUpdateFlag = True
1275
1276     TokenList = Setting.split(TAB_VALUE_SPLIT)
1277     ValueList[0:len(TokenList)] = TokenList
1278
1279     if ValueUpdateFlag:
1280         ValueList[0] = PtrValue[0]
1281
1282     return ValueList
1283
1284 ## check format of PCD value against its the datum type
1285 #
1286 # For PCD value setting
1287 #
1288 def CheckPcdDatum(Type, Value):
1289     if Type == TAB_VOID:
1290         ValueRe = re.compile(r'\s*L?\".*\"\s*$')
1291         if not (((Value.startswith('L"') or Value.startswith('"')) and Value.endswith('"'))
1292                 or (Value.startswith('{') and Value.endswith('}')) or (Value.startswith("L'") or Value.startswith("'") and Value.endswith("'"))
1293                ):
1294             return False, "Invalid value [%s] of type [%s]; must be in the form of {...} for array"\
1295                           ", \"...\" or \'...\' for string, L\"...\" or L\'...\' for unicode string" % (Value, Type)
1296         elif ValueRe.match(Value):
1297             # Check the chars in UnicodeString or CString is printable
1298             if Value.startswith("L"):
1299                 Value = Value[2:-1]
1300             else:
1301                 Value = Value[1:-1]
1302             Printset = set(string.printable)
1303             Printset.remove(TAB_PRINTCHAR_VT)
1304             Printset.add(TAB_PRINTCHAR_BS)
1305             Printset.add(TAB_PRINTCHAR_NUL)
1306             if not set(Value).issubset(Printset):
1307                 PrintList = sorted(Printset)
1308                 return False, "Invalid PCD string value of type [%s]; must be printable chars %s." % (Type, PrintList)
1309     elif Type == 'BOOLEAN':
1310         if Value not in ['TRUE', 'True', 'true', '0x1', '0x01', '1', 'FALSE', 'False', 'false', '0x0', '0x00', '0']:
1311             return False, "Invalid value [%s] of type [%s]; must be one of TRUE, True, true, 0x1, 0x01, 1"\
1312                           ", FALSE, False, false, 0x0, 0x00, 0" % (Value, Type)
1313     elif Type in [TAB_UINT8, TAB_UINT16, TAB_UINT32, TAB_UINT64]:
1314         if Value.startswith('0') and not Value.lower().startswith('0x') and len(Value) > 1 and Value.lstrip('0'):
1315             Value = Value.lstrip('0')
1316         try:
1317             if Value and int(Value, 0) < 0:
1318                 return False, "PCD can't be set to negative value[%s] for datum type [%s]" % (Value, Type)
1319             Value = int(Value, 0)
1320             if Value > MAX_VAL_TYPE[Type]:
1321                 return False, "Too large PCD value[%s] for datum type [%s]" % (Value, Type)
1322         except:
1323             return False, "Invalid value [%s] of type [%s];"\
1324                           " must be a hexadecimal, decimal or octal in C language format." % (Value, Type)
1325     else:
1326         return True, "StructurePcd"
1327
1328     return True, ""
1329
1330 def CommonPath(PathList):
1331     P1 = min(PathList).split(os.path.sep)
1332     P2 = max(PathList).split(os.path.sep)
1333     for Index in range(min(len(P1), len(P2))):
1334         if P1[Index] != P2[Index]:
1335             return os.path.sep.join(P1[:Index])
1336     return os.path.sep.join(P1)
1337
1338 class PathClass(object):
1339     def __init__(self, File='', Root='', AlterRoot='', Type='', IsBinary=False,
1340                  Arch='COMMON', ToolChainFamily='', Target='', TagName='', ToolCode=''):
1341         self.Arch = Arch
1342         self.File = str(File)
1343         if os.path.isabs(self.File):
1344             self.Root = ''
1345             self.AlterRoot = ''
1346         else:
1347             self.Root = str(Root)
1348             self.AlterRoot = str(AlterRoot)
1349
1350         # Remove any '.' and '..' in path
1351         if self.Root:
1352             self.Root = mws.getWs(self.Root, self.File)
1353             self.Path = os.path.normpath(os.path.join(self.Root, self.File))
1354             self.Root = os.path.normpath(CommonPath([self.Root, self.Path]))
1355             # eliminate the side-effect of 'C:'
1356             if self.Root[-1] == ':':
1357                 self.Root += os.path.sep
1358             # file path should not start with path separator
1359             if self.Root[-1] == os.path.sep:
1360                 self.File = self.Path[len(self.Root):]
1361             else:
1362                 self.File = self.Path[len(self.Root) + 1:]
1363         else:
1364             self.Path = os.path.normpath(self.File)
1365
1366         self.SubDir, self.Name = os.path.split(self.File)
1367         self.BaseName, self.Ext = os.path.splitext(self.Name)
1368
1369         if self.Root:
1370             if self.SubDir:
1371                 self.Dir = os.path.join(self.Root, self.SubDir)
1372             else:
1373                 self.Dir = self.Root
1374         else:
1375             self.Dir = self.SubDir
1376
1377         if IsBinary:
1378             self.Type = Type
1379         else:
1380             self.Type = self.Ext.lower()
1381
1382         self.IsBinary = IsBinary
1383         self.Target = Target
1384         self.TagName = TagName
1385         self.ToolCode = ToolCode
1386         self.ToolChainFamily = ToolChainFamily
1387
1388     ## Convert the object of this class to a string
1389     #
1390     #  Convert member Path of the class to a string
1391     #
1392     #  @retval string Formatted String
1393     #
1394     def __str__(self):
1395         return self.Path
1396
1397     ## Override __eq__ function
1398     #
1399     # Check whether PathClass are the same
1400     #
1401     # @retval False The two PathClass are different
1402     # @retval True  The two PathClass are the same
1403     #
1404     def __eq__(self, Other):
1405         return self.Path == str(Other)
1406
1407     ## Override __cmp__ function
1408     #
1409     # Customize the comparsion operation of two PathClass
1410     #
1411     # @retval 0     The two PathClass are different
1412     # @retval -1    The first PathClass is less than the second PathClass
1413     # @retval 1     The first PathClass is Bigger than the second PathClass
1414     def __cmp__(self, Other):
1415         OtherKey = str(Other)
1416
1417         SelfKey = self.Path
1418         if SelfKey == OtherKey:
1419             return 0
1420         elif SelfKey > OtherKey:
1421             return 1
1422         else:
1423             return -1
1424
1425     ## Override __hash__ function
1426     #
1427     # Use Path as key in hash table
1428     #
1429     # @retval string Key for hash table
1430     #
1431     def __hash__(self):
1432         return hash(self.Path)
1433
1434     @cached_property
1435     def Key(self):
1436         return self.Path.upper()
1437
1438     @property
1439     def TimeStamp(self):
1440         return os.stat(self.Path)[8]
1441
1442     def Validate(self, Type='', CaseSensitive=True):
1443         def RealPath2(File, Dir='', OverrideDir=''):
1444             NewFile = None
1445             if OverrideDir:
1446                 NewFile = GlobalData.gAllFiles[os.path.normpath(os.path.join(OverrideDir, File))]
1447                 if NewFile:
1448                     if OverrideDir[-1] == os.path.sep:
1449                         return NewFile[len(OverrideDir):], NewFile[0:len(OverrideDir)]
1450                     else:
1451                         return NewFile[len(OverrideDir) + 1:], NewFile[0:len(OverrideDir)]
1452             if GlobalData.gAllFiles:
1453                 NewFile = GlobalData.gAllFiles[os.path.normpath(os.path.join(Dir, File))]
1454             if not NewFile:
1455                 NewFile = os.path.normpath(os.path.join(Dir, File))
1456                 if not os.path.exists(NewFile):
1457                     return None, None
1458             if NewFile:
1459                 if Dir:
1460                     if Dir[-1] == os.path.sep:
1461                         return NewFile[len(Dir):], NewFile[0:len(Dir)]
1462                     else:
1463                         return NewFile[len(Dir) + 1:], NewFile[0:len(Dir)]
1464                 else:
1465                     return NewFile, ''
1466
1467             return None, None
1468
1469         if GlobalData.gCaseInsensitive:
1470             CaseSensitive = False
1471         if Type and Type.lower() != self.Type:
1472             return FILE_TYPE_MISMATCH, '%s (expect %s but got %s)' % (self.File, Type, self.Type)
1473
1474         RealFile, RealRoot = RealPath2(self.File, self.Root, self.AlterRoot)
1475         if not RealRoot and not RealFile:
1476             RealFile = self.File
1477             if self.AlterRoot:
1478                 RealFile = os.path.join(self.AlterRoot, self.File)
1479             elif self.Root:
1480                 RealFile = os.path.join(self.Root, self.File)
1481             if len (mws.getPkgPath()) == 0:
1482                 return FILE_NOT_FOUND, os.path.join(self.AlterRoot, RealFile)
1483             else:
1484                 return FILE_NOT_FOUND, "%s is not found in packages path:\n\t%s" % (self.File, '\n\t'.join(mws.getPkgPath()))
1485
1486         ErrorCode = 0
1487         ErrorInfo = ''
1488         if RealRoot != self.Root or RealFile != self.File:
1489             if CaseSensitive and (RealFile != self.File or (RealRoot != self.Root and RealRoot != self.AlterRoot)):
1490                 ErrorCode = FILE_CASE_MISMATCH
1491                 ErrorInfo = self.File + '\n\t' + RealFile + " [in file system]"
1492
1493             self.SubDir, self.Name = os.path.split(RealFile)
1494             self.BaseName, self.Ext = os.path.splitext(self.Name)
1495             if self.SubDir:
1496                 self.Dir = os.path.join(RealRoot, self.SubDir)
1497             else:
1498                 self.Dir = RealRoot
1499             self.File = RealFile
1500             self.Root = RealRoot
1501             self.Path = os.path.join(RealRoot, RealFile)
1502         return ErrorCode, ErrorInfo
1503
1504 ## Parse PE image to get the required PE informaion.
1505 #
1506 class PeImageClass():
1507     ## Constructor
1508     #
1509     #   @param  File FilePath of PeImage
1510     #
1511     def __init__(self, PeFile):
1512         self.FileName   = PeFile
1513         self.IsValid    = False
1514         self.Size       = 0
1515         self.EntryPoint = 0
1516         self.SectionAlignment  = 0
1517         self.SectionHeaderList = []
1518         self.ErrorInfo = ''
1519         try:
1520             PeObject = open(PeFile, 'rb')
1521         except:
1522             self.ErrorInfo = self.FileName + ' can not be found\n'
1523             return
1524         # Read DOS header
1525         ByteArray = array.array('B')
1526         ByteArray.fromfile(PeObject, 0x3E)
1527         ByteList = ByteArray.tolist()
1528         # DOS signature should be 'MZ'
1529         if self._ByteListToStr (ByteList[0x0:0x2]) != 'MZ':
1530             self.ErrorInfo = self.FileName + ' has no valid DOS signature MZ'
1531             return
1532
1533         # Read 4 byte PE Signature
1534         PeOffset = self._ByteListToInt(ByteList[0x3C:0x3E])
1535         PeObject.seek(PeOffset)
1536         ByteArray = array.array('B')
1537         ByteArray.fromfile(PeObject, 4)
1538         # PE signature should be 'PE\0\0'
1539         if ByteArray.tostring() != 'PE\0\0':
1540             self.ErrorInfo = self.FileName + ' has no valid PE signature PE00'
1541             return
1542
1543         # Read PE file header
1544         ByteArray = array.array('B')
1545         ByteArray.fromfile(PeObject, 0x14)
1546         ByteList = ByteArray.tolist()
1547         SecNumber = self._ByteListToInt(ByteList[0x2:0x4])
1548         if SecNumber == 0:
1549             self.ErrorInfo = self.FileName + ' has no section header'
1550             return
1551
1552         # Read PE optional header
1553         OptionalHeaderSize = self._ByteListToInt(ByteArray[0x10:0x12])
1554         ByteArray = array.array('B')
1555         ByteArray.fromfile(PeObject, OptionalHeaderSize)
1556         ByteList = ByteArray.tolist()
1557         self.EntryPoint       = self._ByteListToInt(ByteList[0x10:0x14])
1558         self.SectionAlignment = self._ByteListToInt(ByteList[0x20:0x24])
1559         self.Size             = self._ByteListToInt(ByteList[0x38:0x3C])
1560
1561         # Read each Section Header
1562         for Index in range(SecNumber):
1563             ByteArray = array.array('B')
1564             ByteArray.fromfile(PeObject, 0x28)
1565             ByteList = ByteArray.tolist()
1566             SecName  = self._ByteListToStr(ByteList[0:8])
1567             SecVirtualSize = self._ByteListToInt(ByteList[8:12])
1568             SecRawAddress  = self._ByteListToInt(ByteList[20:24])
1569             SecVirtualAddress = self._ByteListToInt(ByteList[12:16])
1570             self.SectionHeaderList.append((SecName, SecVirtualAddress, SecRawAddress, SecVirtualSize))
1571         self.IsValid = True
1572         PeObject.close()
1573
1574     def _ByteListToStr(self, ByteList):
1575         String = ''
1576         for index in range(len(ByteList)):
1577             if ByteList[index] == 0:
1578                 break
1579             String += chr(ByteList[index])
1580         return String
1581
1582     def _ByteListToInt(self, ByteList):
1583         Value = 0
1584         for index in range(len(ByteList) - 1, -1, -1):
1585             Value = (Value << 8) | int(ByteList[index])
1586         return Value
1587
1588 class DefaultStore():
1589     def __init__(self, DefaultStores ):
1590
1591         self.DefaultStores = DefaultStores
1592     def DefaultStoreID(self, DefaultStoreName):
1593         for key, value in self.DefaultStores.items():
1594             if value == DefaultStoreName:
1595                 return key
1596         return None
1597     def GetDefaultDefault(self):
1598         if not self.DefaultStores or "0" in self.DefaultStores:
1599             return "0", TAB_DEFAULT_STORES_DEFAULT
1600         else:
1601             minvalue = min(int(value_str) for value_str in self.DefaultStores)
1602             return (str(minvalue), self.DefaultStores[str(minvalue)])
1603     def GetMin(self, DefaultSIdList):
1604         if not DefaultSIdList:
1605             return TAB_DEFAULT_STORES_DEFAULT
1606         storeidset = {storeid for storeid, storename in self.DefaultStores.values() if storename in DefaultSIdList}
1607         if not storeidset:
1608             return ""
1609         minid = min(storeidset )
1610         for sid, name in self.DefaultStores.values():
1611             if sid == minid:
1612                 return name
1613
1614 class SkuClass():
1615     DEFAULT = 0
1616     SINGLE = 1
1617     MULTIPLE =2
1618
1619     def __init__(self,SkuIdentifier='', SkuIds=None):
1620         if SkuIds is None:
1621             SkuIds = {}
1622
1623         for SkuName in SkuIds:
1624             SkuId = SkuIds[SkuName][0]
1625             skuid_num = int(SkuId, 16) if SkuId.upper().startswith("0X") else int(SkuId)
1626             if skuid_num > 0xFFFFFFFFFFFFFFFF:
1627                 EdkLogger.error("build", PARAMETER_INVALID,
1628                             ExtraData = "SKU-ID [%s] value %s exceeds the max value of UINT64"
1629                                       % (SkuName, SkuId))
1630
1631         self.AvailableSkuIds = OrderedDict()
1632         self.SkuIdSet = []
1633         self.SkuIdNumberSet = []
1634         self.SkuData = SkuIds
1635         self._SkuInherit = {}
1636         self._SkuIdentifier = SkuIdentifier
1637         if SkuIdentifier == '' or SkuIdentifier is None:
1638             self.SkuIdSet = ['DEFAULT']
1639             self.SkuIdNumberSet = ['0U']
1640         elif SkuIdentifier == 'ALL':
1641             self.SkuIdSet = list(SkuIds.keys())
1642             self.SkuIdNumberSet = [num[0].strip() + 'U' for num in SkuIds.values()]
1643         else:
1644             r = SkuIdentifier.split('|')
1645             self.SkuIdSet=[(r[k].strip()).upper() for k in range(len(r))]
1646             k = None
1647             try:
1648                 self.SkuIdNumberSet = [SkuIds[k][0].strip() + 'U' for k in self.SkuIdSet]
1649             except Exception:
1650                 EdkLogger.error("build", PARAMETER_INVALID,
1651                             ExtraData = "SKU-ID [%s] is not supported by the platform. [Valid SKU-ID: %s]"
1652                                       % (k, " | ".join(SkuIds.keys())))
1653         for each in self.SkuIdSet:
1654             if each in SkuIds:
1655                 self.AvailableSkuIds[each] = SkuIds[each][0]
1656             else:
1657                 EdkLogger.error("build", PARAMETER_INVALID,
1658                             ExtraData="SKU-ID [%s] is not supported by the platform. [Valid SKU-ID: %s]"
1659                                       % (each, " | ".join(SkuIds.keys())))
1660         if self.SkuUsageType != SkuClass.SINGLE:
1661             self.AvailableSkuIds.update({'DEFAULT':0, 'COMMON':0})
1662         if self.SkuIdSet:
1663             GlobalData.gSkuids = (self.SkuIdSet)
1664             if 'COMMON' in GlobalData.gSkuids:
1665                 GlobalData.gSkuids.remove('COMMON')
1666             if self.SkuUsageType == self.SINGLE:
1667                 if len(GlobalData.gSkuids) != 1:
1668                     if 'DEFAULT' in GlobalData.gSkuids:
1669                         GlobalData.gSkuids.remove('DEFAULT')
1670             if GlobalData.gSkuids:
1671                 GlobalData.gSkuids.sort()
1672
1673     def GetNextSkuId(self, skuname):
1674         if not self._SkuInherit:
1675             self._SkuInherit = {}
1676             for item in self.SkuData.values():
1677                 self._SkuInherit[item[1]]=item[2] if item[2] else "DEFAULT"
1678         return self._SkuInherit.get(skuname, "DEFAULT")
1679
1680     def GetSkuChain(self, sku):
1681         if sku == "DEFAULT":
1682             return ["DEFAULT"]
1683         skulist = [sku]
1684         nextsku = sku
1685         while True:
1686             nextsku = self.GetNextSkuId(nextsku)
1687             skulist.append(nextsku)
1688             if nextsku == "DEFAULT":
1689                 break
1690         skulist.reverse()
1691         return skulist
1692     def SkuOverrideOrder(self):
1693         skuorderset = []
1694         for skuname in self.SkuIdSet:
1695             skuorderset.append(self.GetSkuChain(skuname))
1696
1697         skuorder = []
1698         for index in range(max(len(item) for item in skuorderset)):
1699             for subset in skuorderset:
1700                 if index > len(subset)-1:
1701                     continue
1702                 if subset[index] in skuorder:
1703                     continue
1704                 skuorder.append(subset[index])
1705
1706         return skuorder
1707
1708     @property
1709     def SkuUsageType(self):
1710         if self._SkuIdentifier.upper() == "ALL":
1711             return SkuClass.MULTIPLE
1712
1713         if len(self.SkuIdSet) == 1:
1714             if self.SkuIdSet[0] == 'DEFAULT':
1715                 return SkuClass.DEFAULT
1716             return SkuClass.SINGLE
1717         if len(self.SkuIdSet) == 2 and 'DEFAULT' in self.SkuIdSet:
1718             return SkuClass.SINGLE
1719         return SkuClass.MULTIPLE
1720
1721     def DumpSkuIdArrary(self):
1722         if self.SkuUsageType == SkuClass.SINGLE:
1723             return "{0x0}"
1724         ArrayStrList = []
1725         for skuname in self.AvailableSkuIds:
1726             if skuname == "COMMON":
1727                 continue
1728             while skuname != "DEFAULT":
1729                 ArrayStrList.append(hex(int(self.AvailableSkuIds[skuname])))
1730                 skuname = self.GetNextSkuId(skuname)
1731             ArrayStrList.append("0x0")
1732         return "{{{myList}}}".format(myList=",".join(ArrayStrList))
1733
1734     @property
1735     def AvailableSkuIdSet(self):
1736         return self.AvailableSkuIds
1737
1738     @property
1739     def SystemSkuId(self):
1740         if self.SkuUsageType == SkuClass.SINGLE:
1741             if len(self.SkuIdSet) == 1:
1742                 return self.SkuIdSet[0]
1743             else:
1744                 return self.SkuIdSet[0] if self.SkuIdSet[0] != 'DEFAULT' else self.SkuIdSet[1]
1745         else:
1746             return 'DEFAULT'
1747
1748 ##  Get the integer value from string like "14U" or integer like 2
1749 #
1750 #   @param      Input   The object that may be either a integer value or a string
1751 #
1752 #   @retval     Value    The integer value that the input represents
1753 #
1754 def GetIntegerValue(Input):
1755     if type(Input) in (int, long):
1756         return Input
1757     String = Input
1758     if String.endswith("U"):
1759         String = String[:-1]
1760     if String.endswith("ULL"):
1761         String = String[:-3]
1762     if String.endswith("LL"):
1763         String = String[:-2]
1764
1765     if String.startswith("0x") or String.startswith("0X"):
1766         return int(String, 16)
1767     elif String == '':
1768         return 0
1769     else:
1770         return int(String)
1771
1772 #
1773 # Pack a GUID (registry format) list into a buffer and return it
1774 #
1775 def PackGUID(Guid):
1776     return pack(PACK_PATTERN_GUID,
1777                 int(Guid[0], 16),
1778                 int(Guid[1], 16),
1779                 int(Guid[2], 16),
1780                 int(Guid[3][-4:-2], 16),
1781                 int(Guid[3][-2:], 16),
1782                 int(Guid[4][-12:-10], 16),
1783                 int(Guid[4][-10:-8], 16),
1784                 int(Guid[4][-8:-6], 16),
1785                 int(Guid[4][-6:-4], 16),
1786                 int(Guid[4][-4:-2], 16),
1787                 int(Guid[4][-2:], 16)
1788                 )
1789
1790 #
1791 # Pack a GUID (byte) list into a buffer and return it
1792 #
1793 def PackByteFormatGUID(Guid):
1794     return pack(PACK_PATTERN_GUID,
1795                 Guid[0],
1796                 Guid[1],
1797                 Guid[2],
1798                 Guid[3],
1799                 Guid[4],
1800                 Guid[5],
1801                 Guid[6],
1802                 Guid[7],
1803                 Guid[8],
1804                 Guid[9],
1805                 Guid[10],
1806                 )
1807
1808 ## DeepCopy dict/OrderedDict recusively
1809 #
1810 #   @param      ori_dict    a nested dict or ordereddict
1811 #
1812 #   @retval     new dict or orderdict
1813 #
1814 def CopyDict(ori_dict):
1815     dict_type = ori_dict.__class__
1816     if dict_type not in (dict,OrderedDict):
1817         return ori_dict
1818     new_dict = dict_type()
1819     for key in ori_dict:
1820         if isinstance(ori_dict[key],(dict,OrderedDict)):
1821             new_dict[key] = CopyDict(ori_dict[key])
1822         else:
1823             new_dict[key] = ori_dict[key]
1824     return new_dict
1825
1826 #
1827 # Remove the c/c++ comments: // and /* */
1828 #
1829 def RemoveCComments(ctext):
1830     return re.sub('//.*?\n|/\*.*?\*/', '\n', ctext, flags=re.S)