BaseTools/Source/Python/Common/Misc.py

   1 ## @file
   2 # Common routines used by all tools
   3 #
   4 # Copyright (c) 2007 - 2019, Intel Corporation. All rights reserved.<BR>
   5 # SPDX-License-Identifier: BSD-2-Clause-Patent
   6 #
   7
   8 ##
   9 # Import Modules
  10 #
  11 from __future__ import absolute_import
  12
  13 import sys
  14 import string
  15 import threading
  16 import time
  17 import re
  18 import pickle
  19 import array
  20 import shutil
  21 import filecmp
  22 from random import sample
  23 from struct import pack
  24 import uuid
  25 import subprocess
  26 import tempfile
  27 from collections import OrderedDict
  28
  29 import Common.LongFilePathOs as os
  30 from Common import EdkLogger as EdkLogger
  31 from Common import GlobalData as GlobalData
  32 from Common.DataType import *
  33 from Common.BuildToolError import *
  34 from CommonDataClass.DataClass import *
  35 from Common.Parsing import GetSplitValueList
  36 from Common.LongFilePathSupport import OpenLongFilePath as open
  37 from Common.LongFilePathSupport import CopyLongFilePath as CopyLong
  38 from Common.LongFilePathSupport import LongFilePath as LongFilePath
  39 from Common.MultipleWorkspace import MultipleWorkspace as mws
  40 from CommonDataClass.Exceptions import BadExpression
  41 from Common.caching import cached_property
  42
  43 ArrayIndex = re.compile("\[\s*[0-9a-fA-FxX]*\s*\]")
  44 ## Regular expression used to find out place holders in string template
  45 gPlaceholderPattern = re.compile("\$\{([^$()\s]+)\}", re.MULTILINE | re.UNICODE)
  46
  47 ## regular expressions for map file processing
  48 startPatternGeneral = re.compile("^Start[' ']+Length[' ']+Name[' ']+Class")
  49 addressPatternGeneral = re.compile("^Address[' ']+Publics by Value[' ']+Rva\+Base")
  50 valuePatternGcc = re.compile('^([\w_\.]+) +([\da-fA-Fx]+) +([\da-fA-Fx]+)$')
  51 pcdPatternGcc = re.compile('^([\da-fA-Fx]+) +([\da-fA-Fx]+)')
  52 secReGeneral = re.compile('^([\da-fA-F]+):([\da-fA-F]+) +([\da-fA-F]+)[Hh]? +([.\w\$]+) +(\w+)', re.UNICODE)
  53
  54 StructPattern = re.compile(r'[_a-zA-Z][0-9A-Za-z_]*$')
  55
  56 ## Dictionary used to store dependencies of files
  57 gDependencyDatabase = {}    # arch : {file path : [dependent files list]}
  58
  59 #
  60 # If a module is built more than once with different PCDs or library classes
  61 # a temporary INF file with same content is created, the temporary file is removed
  62 # when build exits.
  63 #
  64 _TempInfs = []
  65
  66 def GetVariableOffset(mapfilepath, efifilepath, varnames):
  67     """ Parse map file to get variable offset in current EFI file
  68     @param mapfilepath    Map file absolution path
  69     @param efifilepath:   EFI binary file full path
  70     @param varnames       iteratable container whose elements are variable names to be searched
  71
  72     @return List whos elements are tuple with variable name and raw offset
  73     """
  74     lines = []
  75     try:
  76         f = open(mapfilepath, 'r')
  77         lines = f.readlines()
  78         f.close()
  79     except:
  80         return None
  81
  82     if len(lines) == 0: return None
  83     firstline = lines[0].strip()
  84     if (firstline.startswith("Archive member included ") and
  85         firstline.endswith(" file (symbol)")):
  86         return _parseForGCC(lines, efifilepath, varnames)
  87     if firstline.startswith("# Path:"):
  88         return _parseForXcode(lines, efifilepath, varnames)
  89     return _parseGeneral(lines, efifilepath, varnames)
  90
  91 def _parseForXcode(lines, efifilepath, varnames):
  92     status = 0
  93     ret = []
  94     for line in lines:
  95         line = line.strip()
  96         if status == 0 and line == "# Symbols:":
  97             status = 1
  98             continue
  99         if status == 1 and len(line) != 0:
 100             for varname in varnames:
 101                 if varname in line:
 102                     # cannot pregenerate this RegEx since it uses varname from varnames.
 103                     m = re.match('^([\da-fA-FxX]+)([\s\S]*)([_]*%s)$' % varname, line)
 104                     if m is not None:
 105                         ret.append((varname, m.group(1)))
 106     return ret
 107
 108 def _parseForGCC(lines, efifilepath, varnames):
 109     """ Parse map file generated by GCC linker """
 110     status = 0
 111     sections = []
 112     varoffset = []
 113     for index, line in enumerate(lines):
 114         line = line.strip()
 115         # status machine transection
 116         if status == 0 and line == "Memory Configuration":
 117             status = 1
 118             continue
 119         elif status == 1 and line == 'Linker script and memory map':
 120             status = 2
 121             continue
 122         elif status ==2 and line == 'START GROUP':
 123             status = 3
 124             continue
 125
 126         # status handler
 127         if status == 3:
 128             m = valuePatternGcc.match(line)
 129             if m is not None:
 130                 sections.append(m.groups(0))
 131             for varname in varnames:
 132                 Str = ''
 133                 m = re.match("^.data.(%s)" % varname, line)
 134                 if m is not None:
 135                     m = re.match(".data.(%s)$" % varname, line)
 136                     if m is not None:
 137                         Str = lines[index + 1]
 138                     else:
 139                         Str = line[len(".data.%s" % varname):]
 140                     if Str:
 141                         m = pcdPatternGcc.match(Str.strip())
 142                         if m is not None:
 143                             varoffset.append((varname, int(m.groups(0)[0], 16), int(sections[-1][1], 16), sections[-1][0]))
 144
 145     if not varoffset:
 146         return []
 147     # get section information from efi file
 148     efisecs = PeImageClass(efifilepath).SectionHeaderList
 149     if efisecs is None or len(efisecs) == 0:
 150         return []
 151     #redirection
 152     redirection = 0
 153     for efisec in efisecs:
 154         for section in sections:
 155             if section[0].strip() == efisec[0].strip() and section[0].strip() == '.text':
 156                 redirection = int(section[1], 16) - efisec[1]
 157
 158     ret = []
 159     for var in varoffset:
 160         for efisec in efisecs:
 161             if var[1] >= efisec[1] and var[1] < efisec[1]+efisec[3]:
 162                 ret.append((var[0], hex(efisec[2] + var[1] - efisec[1] - redirection)))
 163     return ret
 164
 165 def _parseGeneral(lines, efifilepath, varnames):
 166     status = 0    #0 - beginning of file; 1 - PE section definition; 2 - symbol table
 167     secs  = []    # key = section name
 168     varoffset = []
 169     symRe = re.compile('^([\da-fA-F]+):([\da-fA-F]+) +([\.:\\\\\w\?@\$-]+) +([\da-fA-F]+)', re.UNICODE)
 170
 171     for line in lines:
 172         line = line.strip()
 173         if startPatternGeneral.match(line):
 174             status = 1
 175             continue
 176         if addressPatternGeneral.match(line):
 177             status = 2
 178             continue
 179         if line.startswith("entry point at"):
 180             status = 3
 181             continue
 182         if status == 1 and len(line) != 0:
 183             m =  secReGeneral.match(line)
 184             assert m is not None, "Fail to parse the section in map file , line is %s" % line
 185             sec_no, sec_start, sec_length, sec_name, sec_class = m.groups(0)
 186             secs.append([int(sec_no, 16), int(sec_start, 16), int(sec_length, 16), sec_name, sec_class])
 187         if status == 2 and len(line) != 0:
 188             for varname in varnames:
 189                 m = symRe.match(line)
 190                 assert m is not None, "Fail to parse the symbol in map file, line is %s" % line
 191                 sec_no, sym_offset, sym_name, vir_addr = m.groups(0)
 192                 sec_no     = int(sec_no,     16)
 193                 sym_offset = int(sym_offset, 16)
 194                 vir_addr   = int(vir_addr,   16)
 195                 # cannot pregenerate this RegEx since it uses varname from varnames.
 196                 m2 = re.match('^[_]*(%s)' % varname, sym_name)
 197                 if m2 is not None:
 198                     # fond a binary pcd entry in map file
 199                     for sec in secs:
 200                         if sec[0] == sec_no and (sym_offset >= sec[1] and sym_offset < sec[1] + sec[2]):
 201                             varoffset.append([varname, sec[3], sym_offset, vir_addr, sec_no])
 202
 203     if not varoffset: return []
 204
 205     # get section information from efi file
 206     efisecs = PeImageClass(efifilepath).SectionHeaderList
 207     if efisecs is None or len(efisecs) == 0:
 208         return []
 209
 210     ret = []
 211     for var in varoffset:
 212         index = 0
 213         for efisec in efisecs:
 214             index = index + 1
 215             if var[1].strip() == efisec[0].strip():
 216                 ret.append((var[0], hex(efisec[2] + var[2])))
 217             elif var[4] == index:
 218                 ret.append((var[0], hex(efisec[2] + var[2])))
 219
 220     return ret
 221
 222 ## Routine to process duplicated INF
 223 #
 224 #  This function is called by following two cases:
 225 #  Case 1 in DSC:
 226 #    [components.arch]
 227 #    Pkg/module/module.inf
 228 #    Pkg/module/module.inf {
 229 #      <Defines>
 230 #        FILE_GUID = 0D1B936F-68F3-4589-AFCC-FB8B7AEBC836
 231 #    }
 232 #  Case 2 in FDF:
 233 #    INF Pkg/module/module.inf
 234 #    INF FILE_GUID = 0D1B936F-68F3-4589-AFCC-FB8B7AEBC836 Pkg/module/module.inf
 235 #
 236 #  This function copies Pkg/module/module.inf to
 237 #  Conf/.cache/0D1B936F-68F3-4589-AFCC-FB8B7AEBC836module.inf
 238 #
 239 #  @param Path     Original PathClass object
 240 #  @param BaseName New file base name
 241 #
 242 #  @retval         return the new PathClass object
 243 #
 244 def ProcessDuplicatedInf(Path, BaseName, Workspace):
 245     Filename = os.path.split(Path.File)[1]
 246     if '.' in Filename:
 247         Filename = BaseName + Path.BaseName + Filename[Filename.rfind('.'):]
 248     else:
 249         Filename = BaseName + Path.BaseName
 250
 251     DbDir = os.path.split(GlobalData.gDatabasePath)[0]
 252
 253     #
 254     # A temporary INF is copied to database path which must have write permission
 255     # The temporary will be removed at the end of build
 256     # In case of name conflict, the file name is
 257     # FILE_GUIDBaseName (0D1B936F-68F3-4589-AFCC-FB8B7AEBC836module.inf)
 258     #
 259     TempFullPath = os.path.join(DbDir,
 260                                 Filename)
 261     RtPath = PathClass(Path.File, Workspace)
 262     #
 263     # Modify the full path to temporary path, keep other unchanged
 264     #
 265     # To build same module more than once, the module path with FILE_GUID overridden has
 266     # the file name FILE_GUIDmodule.inf, but the relative path (self.MetaFile.File) is the real path
 267     # in DSC which is used as relative path by C files and other files in INF.
 268     # A trick was used: all module paths are PathClass instances, after the initialization
 269     # of PathClass, the PathClass.Path is overridden by the temporary INF path.
 270     #
 271     # The reason for creating a temporary INF is:
 272     # Platform.Modules which is the base to create ModuleAutoGen objects is a dictionary,
 273     # the key is the full path of INF, the value is an object to save overridden library instances, PCDs.
 274     # A different key for the same module is needed to create different output directory,
 275     # retrieve overridden PCDs, library instances.
 276     #
 277     # The BaseName is the FILE_GUID which is also the output directory name.
 278     #
 279     #
 280     RtPath.Path = TempFullPath
 281     RtPath.BaseName = BaseName
 282     RtPath.OriginalPath = Path
 283     #
 284     # If file exists, compare contents
 285     #
 286     if os.path.exists(TempFullPath):
 287         with open(str(Path), 'rb') as f1, open(TempFullPath, 'rb') as f2:
 288             if f1.read() == f2.read():
 289                 return RtPath
 290     _TempInfs.append(TempFullPath)
 291     shutil.copy2(str(Path), TempFullPath)
 292     return RtPath
 293
 294 ## Remove temporary created INFs whose paths were saved in _TempInfs
 295 #
 296 def ClearDuplicatedInf():
 297     while _TempInfs:
 298         File = _TempInfs.pop()
 299         if os.path.exists(File):
 300             os.remove(File)
 301
 302 ## Convert GUID string in xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx style to C structure style
 303 #
 304 #   @param      Guid    The GUID string
 305 #
 306 #   @retval     string  The GUID string in C structure style
 307 #
 308 def GuidStringToGuidStructureString(Guid):
 309     GuidList = Guid.split('-')
 310     Result = '{'
 311     for Index in range(0, 3, 1):
 312         Result = Result + '0x' + GuidList[Index] + ', '
 313     Result = Result + '{0x' + GuidList[3][0:2] + ', 0x' + GuidList[3][2:4]
 314     for Index in range(0, 12, 2):
 315         Result = Result + ', 0x' + GuidList[4][Index:Index + 2]
 316     Result += '}}'
 317     return Result
 318
 319 ## Convert GUID structure in byte array to xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
 320 #
 321 #   @param      GuidValue   The GUID value in byte array
 322 #
 323 #   @retval     string      The GUID value in xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx format
 324 #
 325 def GuidStructureByteArrayToGuidString(GuidValue):
 326     guidValueString = GuidValue.lower().replace("{", "").replace("}", "").replace(" ", "").replace(";", "")
 327     guidValueList = guidValueString.split(",")
 328     if len(guidValueList) != 16:
 329         return ''
 330         #EdkLogger.error(None, None, "Invalid GUID value string %s" % GuidValue)
 331     try:
 332         return "%02x%02x%02x%02x-%02x%02x-%02x%02x-%02x%02x-%02x%02x%02x%02x%02x%02x" % (
 333                 int(guidValueList[3], 16),
 334                 int(guidValueList[2], 16),
 335                 int(guidValueList[1], 16),
 336                 int(guidValueList[0], 16),
 337                 int(guidValueList[5], 16),
 338                 int(guidValueList[4], 16),
 339                 int(guidValueList[7], 16),
 340                 int(guidValueList[6], 16),
 341                 int(guidValueList[8], 16),
 342                 int(guidValueList[9], 16),
 343                 int(guidValueList[10], 16),
 344                 int(guidValueList[11], 16),
 345                 int(guidValueList[12], 16),
 346                 int(guidValueList[13], 16),
 347                 int(guidValueList[14], 16),
 348                 int(guidValueList[15], 16)
 349                 )
 350     except:
 351         return ''
 352
 353 ## Convert GUID string in C structure style to xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
 354 #
 355 #   @param      GuidValue   The GUID value in C structure format
 356 #
 357 #   @retval     string      The GUID value in xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx format
 358 #
 359 def GuidStructureStringToGuidString(GuidValue):
 360     if not GlobalData.gGuidCFormatPattern.match(GuidValue):
 361         return ''
 362     guidValueString = GuidValue.lower().replace("{", "").replace("}", "").replace(" ", "").replace(";", "")
 363     guidValueList = guidValueString.split(",")
 364     if len(guidValueList) != 11:
 365         return ''
 366         #EdkLogger.error(None, None, "Invalid GUID value string %s" % GuidValue)
 367     try:
 368         return "%08x-%04x-%04x-%02x%02x-%02x%02x%02x%02x%02x%02x" % (
 369                 int(guidValueList[0], 16),
 370                 int(guidValueList[1], 16),
 371                 int(guidValueList[2], 16),
 372                 int(guidValueList[3], 16),
 373                 int(guidValueList[4], 16),
 374                 int(guidValueList[5], 16),
 375                 int(guidValueList[6], 16),
 376                 int(guidValueList[7], 16),
 377                 int(guidValueList[8], 16),
 378                 int(guidValueList[9], 16),
 379                 int(guidValueList[10], 16)
 380                 )
 381     except:
 382         return ''
 383
 384 ## Convert GUID string in C structure style to xxxxxxxx_xxxx_xxxx_xxxx_xxxxxxxxxxxx
 385 #
 386 #   @param      GuidValue   The GUID value in C structure format
 387 #
 388 #   @retval     string      The GUID value in xxxxxxxx_xxxx_xxxx_xxxx_xxxxxxxxxxxx format
 389 #
 390 def GuidStructureStringToGuidValueName(GuidValue):
 391     guidValueString = GuidValue.lower().replace("{", "").replace("}", "").replace(" ", "")
 392     guidValueList = guidValueString.split(",")
 393     if len(guidValueList) != 11:
 394         EdkLogger.error(None, FORMAT_INVALID, "Invalid GUID value string [%s]" % GuidValue)
 395     return "%08x_%04x_%04x_%02x%02x_%02x%02x%02x%02x%02x%02x" % (
 396             int(guidValueList[0], 16),
 397             int(guidValueList[1], 16),
 398             int(guidValueList[2], 16),
 399             int(guidValueList[3], 16),
 400             int(guidValueList[4], 16),
 401             int(guidValueList[5], 16),
 402             int(guidValueList[6], 16),
 403             int(guidValueList[7], 16),
 404             int(guidValueList[8], 16),
 405             int(guidValueList[9], 16),
 406             int(guidValueList[10], 16)
 407             )
 408
 409 ## Create directories
 410 #
 411 #   @param      Directory   The directory name
 412 #
 413 def CreateDirectory(Directory):
 414     if Directory is None or Directory.strip() == "":
 415         return True
 416     try:
 417         if not os.access(Directory, os.F_OK):
 418             os.makedirs(Directory)
 419     except:
 420         return False
 421     return True
 422
 423 ## Remove directories, including files and sub-directories in it
 424 #
 425 #   @param      Directory   The directory name
 426 #
 427 def RemoveDirectory(Directory, Recursively=False):
 428     if Directory is None or Directory.strip() == "" or not os.path.exists(Directory):
 429         return
 430     if Recursively:
 431         CurrentDirectory = os.getcwd()
 432         os.chdir(Directory)
 433         for File in os.listdir("."):
 434             if os.path.isdir(File):
 435                 RemoveDirectory(File, Recursively)
 436             else:
 437                 os.remove(File)
 438         os.chdir(CurrentDirectory)
 439     os.rmdir(Directory)
 440
 441 ## Store content in file
 442 #
 443 #  This method is used to save file only when its content is changed. This is
 444 #  quite useful for "make" system to decide what will be re-built and what won't.
 445 #
 446 #   @param      File            The path of file
 447 #   @param      Content         The new content of the file
 448 #   @param      IsBinaryFile    The flag indicating if the file is binary file or not
 449 #
 450 #   @retval     True            If the file content is changed and the file is renewed
 451 #   @retval     False           If the file content is the same
 452 #
 453 def SaveFileOnChange(File, Content, IsBinaryFile=True, FileLock=None):
 454
 455     # Convert to long file path format
 456     File = LongFilePath(File)
 457
 458     if os.path.exists(File):
 459         if IsBinaryFile:
 460             try:
 461                 with open(File, "rb") as f:
 462                     if Content == f.read():
 463                         return False
 464             except:
 465                 EdkLogger.error(None, FILE_OPEN_FAILURE, ExtraData=File)
 466         else:
 467             try:
 468                 with open(File, "r") as f:
 469                     if Content == f.read():
 470                         return False
 471             except:
 472                 EdkLogger.error(None, FILE_OPEN_FAILURE, ExtraData=File)
 473
 474     DirName = os.path.dirname(File)
 475     if not CreateDirectory(DirName):
 476         EdkLogger.error(None, FILE_CREATE_FAILURE, "Could not create directory %s" % DirName)
 477     else:
 478         if DirName == '':
 479             DirName = os.getcwd()
 480         if not os.access(DirName, os.W_OK):
 481             EdkLogger.error(None, PERMISSION_FAILURE, "Do not have write permission on directory %s" % DirName)
 482
 483     OpenMode = "w"
 484     if IsBinaryFile:
 485         OpenMode = "wb"
 486
 487     # use default file_lock if no input new lock
 488     if not FileLock:
 489         FileLock = GlobalData.file_lock
 490     if FileLock:
 491         FileLock.acquire()
 492
 493
 494     if GlobalData.gIsWindows and not os.path.exists(File):
 495         try:
 496             with open(File, OpenMode) as tf:
 497                 tf.write(Content)
 498         except IOError as X:
 499             if GlobalData.gBinCacheSource:
 500                 EdkLogger.quiet("[cache error]:fails to save file with error: %s" % (X))
 501             else:
 502                 EdkLogger.error(None, FILE_CREATE_FAILURE, ExtraData='IOError %s' % X)
 503         finally:
 504             if FileLock:
 505                 FileLock.release()
 506     else:
 507         try:
 508             with open(File, OpenMode) as Fd:
 509                 Fd.write(Content)
 510         except IOError as X:
 511             if GlobalData.gBinCacheSource:
 512                 EdkLogger.quiet("[cache error]:fails to save file with error: %s" % (X))
 513             else:
 514                 EdkLogger.error(None, FILE_CREATE_FAILURE, ExtraData='IOError %s' % X)
 515         finally:
 516             if FileLock:
 517                 FileLock.release()
 518
 519     return True
 520
 521 ## Copy source file only if it is different from the destination file
 522 #
 523 #  This method is used to copy file only if the source file and destination
 524 #  file content are different. This is quite useful to avoid duplicated
 525 #  file writing.
 526 #
 527 #   @param      SrcFile   The path of source file
 528 #   @param      Dst       The path of destination file or folder
 529 #
 530 #   @retval     True      The two files content are different and the file is copied
 531 #   @retval     False     No copy really happen
 532 #
 533 def CopyFileOnChange(SrcFile, Dst, FileLock=None):
 534
 535     # Convert to long file path format
 536     SrcFile = LongFilePath(SrcFile)
 537     Dst = LongFilePath(Dst)
 538
 539     if os.path.isdir(SrcFile):
 540         EdkLogger.error(None, FILE_COPY_FAILURE, ExtraData='CopyFileOnChange SrcFile is a dir, not a file: %s' % SrcFile)
 541         return False
 542
 543     if os.path.isdir(Dst):
 544         DstFile = os.path.join(Dst, os.path.basename(SrcFile))
 545     else:
 546         DstFile = Dst
 547
 548     if os.path.exists(DstFile) and filecmp.cmp(SrcFile, DstFile, shallow=False):
 549         return False
 550
 551     DirName = os.path.dirname(DstFile)
 552     if not CreateDirectory(DirName):
 553         EdkLogger.error(None, FILE_CREATE_FAILURE, "Could not create directory %s" % DirName)
 554     else:
 555         if DirName == '':
 556             DirName = os.getcwd()
 557         if not os.access(DirName, os.W_OK):
 558             EdkLogger.error(None, PERMISSION_FAILURE, "Do not have write permission on directory %s" % DirName)
 559
 560     # use default file_lock if no input new lock
 561     if not FileLock:
 562         FileLock = GlobalData.file_lock
 563     if FileLock:
 564         FileLock.acquire()
 565
 566     try:
 567         CopyLong(SrcFile, DstFile)
 568     except IOError as X:
 569         if GlobalData.gBinCacheSource:
 570             EdkLogger.quiet("[cache error]:fails to copy file with error: %s" % (X))
 571         else:
 572             EdkLogger.error(None, FILE_COPY_FAILURE, ExtraData='IOError %s' % X)
 573     finally:
 574         if FileLock:
 575             FileLock.release()
 576
 577     return True
 578
 579 ## Retrieve and cache the real path name in file system
 580 #
 581 #   @param      Root    The root directory of path relative to
 582 #
 583 #   @retval     str     The path string if the path exists
 584 #   @retval     None    If path doesn't exist
 585 #
 586 class DirCache:
 587     _CACHE_ = set()
 588     _UPPER_CACHE_ = {}
 589
 590     def __init__(self, Root):
 591         self._Root = Root
 592         for F in os.listdir(Root):
 593             self._CACHE_.add(F)
 594             self._UPPER_CACHE_[F.upper()] = F
 595
 596     # =[] operator
 597     def __getitem__(self, Path):
 598         Path = Path[len(os.path.commonprefix([Path, self._Root])):]
 599         if not Path:
 600             return self._Root
 601         if Path and Path[0] == os.path.sep:
 602             Path = Path[1:]
 603         if Path in self._CACHE_:
 604             return os.path.join(self._Root, Path)
 605         UpperPath = Path.upper()
 606         if UpperPath in self._UPPER_CACHE_:
 607             return os.path.join(self._Root, self._UPPER_CACHE_[UpperPath])
 608
 609         IndexList = []
 610         LastSepIndex = -1
 611         SepIndex = Path.find(os.path.sep)
 612         while SepIndex > -1:
 613             Parent = UpperPath[:SepIndex]
 614             if Parent not in self._UPPER_CACHE_:
 615                 break
 616             LastSepIndex = SepIndex
 617             SepIndex = Path.find(os.path.sep, LastSepIndex + 1)
 618
 619         if LastSepIndex == -1:
 620             return None
 621
 622         Cwd = os.getcwd()
 623         os.chdir(self._Root)
 624         SepIndex = LastSepIndex
 625         while SepIndex > -1:
 626             Parent = Path[:SepIndex]
 627             ParentKey = UpperPath[:SepIndex]
 628             if ParentKey not in self._UPPER_CACHE_:
 629                 os.chdir(Cwd)
 630                 return None
 631
 632             if Parent in self._CACHE_:
 633                 ParentDir = Parent
 634             else:
 635                 ParentDir = self._UPPER_CACHE_[ParentKey]
 636             for F in os.listdir(ParentDir):
 637                 Dir = os.path.join(ParentDir, F)
 638                 self._CACHE_.add(Dir)
 639                 self._UPPER_CACHE_[Dir.upper()] = Dir
 640
 641             SepIndex = Path.find(os.path.sep, SepIndex + 1)
 642
 643         os.chdir(Cwd)
 644         if Path in self._CACHE_:
 645             return os.path.join(self._Root, Path)
 646         elif UpperPath in self._UPPER_CACHE_:
 647             return os.path.join(self._Root, self._UPPER_CACHE_[UpperPath])
 648         return None
 649
 650 def RealPath(File, Dir='', OverrideDir=''):
 651     NewFile = os.path.normpath(os.path.join(Dir, File))
 652     NewFile = GlobalData.gAllFiles[NewFile]
 653     if not NewFile and OverrideDir:
 654         NewFile = os.path.normpath(os.path.join(OverrideDir, File))
 655         NewFile = GlobalData.gAllFiles[NewFile]
 656     return NewFile
 657
 658 ## Get GUID value from given packages
 659 #
 660 #   @param      CName           The CName of the GUID
 661 #   @param      PackageList     List of packages looking-up in
 662 #   @param      Inffile         The driver file
 663 #
 664 #   @retval     GuidValue   if the CName is found in any given package
 665 #   @retval     None        if the CName is not found in all given packages
 666 #
 667 def GuidValue(CName, PackageList, Inffile = None):
 668     for P in PackageList:
 669         GuidKeys = list(P.Guids.keys())
 670         if Inffile and P._PrivateGuids:
 671             if not Inffile.startswith(P.MetaFile.Dir):
 672                 GuidKeys = [x for x in P.Guids if x not in P._PrivateGuids]
 673         if CName in GuidKeys:
 674             return P.Guids[CName]
 675     return None
 676
 677 ## A string template class
 678 #
 679 #  This class implements a template for string replacement. A string template
 680 #  looks like following
 681 #
 682 #       ${BEGIN} other_string ${placeholder_name} other_string ${END}
 683 #
 684 #  The string between ${BEGIN} and ${END} will be repeated as many times as the
 685 #  length of "placeholder_name", which is a list passed through a dict. The
 686 #  "placeholder_name" is the key name of the dict. The ${BEGIN} and ${END} can
 687 #  be not used and, in this case, the "placeholder_name" must not a list and it
 688 #  will just be replaced once.
 689 #
 690 class TemplateString(object):
 691     _REPEAT_START_FLAG = "BEGIN"
 692     _REPEAT_END_FLAG = "END"
 693
 694     class Section(object):
 695         _LIST_TYPES = [type([]), type(set()), type((0,))]
 696
 697         def __init__(self, TemplateSection, PlaceHolderList):
 698             self._Template = TemplateSection
 699             self._PlaceHolderList = []
 700
 701             # Split the section into sub-sections according to the position of placeholders
 702             if PlaceHolderList:
 703                 self._SubSectionList = []
 704                 SubSectionStart = 0
 705                 #
 706                 # The placeholders passed in must be in the format of
 707                 #
 708                 #   PlaceHolderName, PlaceHolderStartPoint, PlaceHolderEndPoint
 709                 #
 710                 for PlaceHolder, Start, End in PlaceHolderList:
 711                     self._SubSectionList.append(TemplateSection[SubSectionStart:Start])
 712                     self._SubSectionList.append(TemplateSection[Start:End])
 713                     self._PlaceHolderList.append(PlaceHolder)
 714                     SubSectionStart = End
 715                 if SubSectionStart < len(TemplateSection):
 716                     self._SubSectionList.append(TemplateSection[SubSectionStart:])
 717             else:
 718                 self._SubSectionList = [TemplateSection]
 719
 720         def __str__(self):
 721             return self._Template + " : " + str(self._PlaceHolderList)
 722
 723         def Instantiate(self, PlaceHolderValues):
 724             RepeatTime = -1
 725             RepeatPlaceHolders = {}
 726             NonRepeatPlaceHolders = {}
 727
 728             for PlaceHolder in self._PlaceHolderList:
 729                 if PlaceHolder not in PlaceHolderValues:
 730                     continue
 731                 Value = PlaceHolderValues[PlaceHolder]
 732                 if type(Value) in self._LIST_TYPES:
 733                     if RepeatTime < 0:
 734                         RepeatTime = len(Value)
 735                     elif RepeatTime != len(Value):
 736                         EdkLogger.error(
 737                                     "TemplateString",
 738                                     PARAMETER_INVALID,
 739                                     "${%s} has different repeat time from others!" % PlaceHolder,
 740                                     ExtraData=str(self._Template)
 741                                     )
 742                     RepeatPlaceHolders["${%s}" % PlaceHolder] = Value
 743                 else:
 744                     NonRepeatPlaceHolders["${%s}" % PlaceHolder] = Value
 745
 746             if NonRepeatPlaceHolders:
 747                 StringList = []
 748                 for S in self._SubSectionList:
 749                     if S not in NonRepeatPlaceHolders:
 750                         StringList.append(S)
 751                     else:
 752                         StringList.append(str(NonRepeatPlaceHolders[S]))
 753             else:
 754                 StringList = self._SubSectionList
 755
 756             if RepeatPlaceHolders:
 757                 TempStringList = []
 758                 for Index in range(RepeatTime):
 759                     for S in StringList:
 760                         if S not in RepeatPlaceHolders:
 761                             TempStringList.append(S)
 762                         else:
 763                             TempStringList.append(str(RepeatPlaceHolders[S][Index]))
 764                 StringList = TempStringList
 765
 766             return "".join(StringList)
 767
 768     ## Constructor
 769     def __init__(self, Template=None):
 770         self.String = []
 771         self.IsBinary = False
 772         self._Template = Template
 773         self._TemplateSectionList = self._Parse(Template)
 774
 775     ## str() operator
 776     #
 777     #   @retval     string  The string replaced
 778     #
 779     def __str__(self):
 780         return "".join(self.String)
 781
 782     ## Split the template string into fragments per the ${BEGIN} and ${END} flags
 783     #
 784     #   @retval     list    A list of TemplateString.Section objects
 785     #
 786     def _Parse(self, Template):
 787         SectionStart = 0
 788         SearchFrom = 0
 789         MatchEnd = 0
 790         PlaceHolderList = []
 791         TemplateSectionList = []
 792         while Template:
 793             MatchObj = gPlaceholderPattern.search(Template, SearchFrom)
 794             if not MatchObj:
 795                 if MatchEnd <= len(Template):
 796                     TemplateSection = TemplateString.Section(Template[SectionStart:], PlaceHolderList)
 797                     TemplateSectionList.append(TemplateSection)
 798                 break
 799
 800             MatchString = MatchObj.group(1)
 801             MatchStart = MatchObj.start()
 802             MatchEnd = MatchObj.end()
 803
 804             if MatchString == self._REPEAT_START_FLAG:
 805                 if MatchStart > SectionStart:
 806                     TemplateSection = TemplateString.Section(Template[SectionStart:MatchStart], PlaceHolderList)
 807                     TemplateSectionList.append(TemplateSection)
 808                 SectionStart = MatchEnd
 809                 PlaceHolderList = []
 810             elif MatchString == self._REPEAT_END_FLAG:
 811                 TemplateSection = TemplateString.Section(Template[SectionStart:MatchStart], PlaceHolderList)
 812                 TemplateSectionList.append(TemplateSection)
 813                 SectionStart = MatchEnd
 814                 PlaceHolderList = []
 815             else:
 816                 PlaceHolderList.append((MatchString, MatchStart - SectionStart, MatchEnd - SectionStart))
 817             SearchFrom = MatchEnd
 818         return TemplateSectionList
 819
 820     ## Replace the string template with dictionary of placeholders and append it to previous one
 821     #
 822     #   @param      AppendString    The string template to append
 823     #   @param      Dictionary      The placeholder dictionaries
 824     #
 825     def Append(self, AppendString, Dictionary=None):
 826         if Dictionary:
 827             SectionList = self._Parse(AppendString)
 828             self.String.append( "".join(S.Instantiate(Dictionary) for S in SectionList))
 829         else:
 830             if isinstance(AppendString,list):
 831                 self.String.extend(AppendString)
 832             else:
 833                 self.String.append(AppendString)
 834
 835     ## Replace the string template with dictionary of placeholders
 836     #
 837     #   @param      Dictionary      The placeholder dictionaries
 838     #
 839     #   @retval     str             The string replaced with placeholder values
 840     #
 841     def Replace(self, Dictionary=None):
 842         return "".join(S.Instantiate(Dictionary) for S in self._TemplateSectionList)
 843
 844 ## Progress indicator class
 845 #
 846 #  This class makes use of thread to print progress on console.
 847 #
 848 class Progressor:
 849     # for avoiding deadloop
 850     _StopFlag = None
 851     _ProgressThread = None
 852     _CheckInterval = 0.25
 853
 854     ## Constructor
 855     #
 856     #   @param      OpenMessage     The string printed before progress characters
 857     #   @param      CloseMessage    The string printed after progress characters
 858     #   @param      ProgressChar    The character used to indicate the progress
 859     #   @param      Interval        The interval in seconds between two progress characters
 860     #
 861     def __init__(self, OpenMessage="", CloseMessage="", ProgressChar='.', Interval=1.0):
 862         self.PromptMessage = OpenMessage
 863         self.CodaMessage = CloseMessage
 864         self.ProgressChar = ProgressChar
 865         self.Interval = Interval
 866         if Progressor._StopFlag is None:
 867             Progressor._StopFlag = threading.Event()
 868
 869     ## Start to print progress character
 870     #
 871     #   @param      OpenMessage     The string printed before progress characters
 872     #
 873     def Start(self, OpenMessage=None):
 874         if OpenMessage is not None:
 875             self.PromptMessage = OpenMessage
 876         Progressor._StopFlag.clear()
 877         if Progressor._ProgressThread is None:
 878             Progressor._ProgressThread = threading.Thread(target=self._ProgressThreadEntry)
 879             Progressor._ProgressThread.setDaemon(False)
 880             Progressor._ProgressThread.start()
 881
 882     ## Stop printing progress character
 883     #
 884     #   @param      CloseMessage    The string printed after progress characters
 885     #
 886     def Stop(self, CloseMessage=None):
 887         OriginalCodaMessage = self.CodaMessage
 888         if CloseMessage is not None:
 889             self.CodaMessage = CloseMessage
 890         self.Abort()
 891         self.CodaMessage = OriginalCodaMessage
 892
 893     ## Thread entry method
 894     def _ProgressThreadEntry(self):
 895         sys.stdout.write(self.PromptMessage + " ")
 896         sys.stdout.flush()
 897         TimeUp = 0.0
 898         while not Progressor._StopFlag.isSet():
 899             if TimeUp <= 0.0:
 900                 sys.stdout.write(self.ProgressChar)
 901                 sys.stdout.flush()
 902                 TimeUp = self.Interval
 903             time.sleep(self._CheckInterval)
 904             TimeUp -= self._CheckInterval
 905         sys.stdout.write(" " + self.CodaMessage + "\n")
 906         sys.stdout.flush()
 907
 908     ## Abort the progress display
 909     @staticmethod
 910     def Abort():
 911         if Progressor._StopFlag is not None:
 912             Progressor._StopFlag.set()
 913         if Progressor._ProgressThread is not None:
 914             Progressor._ProgressThread.join()
 915             Progressor._ProgressThread = None
 916
 917
 918 ## Dictionary using prioritized list as key
 919 #
 920 class tdict:
 921     _ListType = type([])
 922     _TupleType = type(())
 923     _Wildcard = 'COMMON'
 924     _ValidWildcardList = ['COMMON', 'DEFAULT', 'ALL', TAB_STAR, 'PLATFORM']
 925
 926     def __init__(self, _Single_=False, _Level_=2):
 927         self._Level_ = _Level_
 928         self.data = {}
 929         self._Single_ = _Single_
 930
 931     # =[] operator
 932     def __getitem__(self, key):
 933         KeyType = type(key)
 934         RestKeys = None
 935         if KeyType == self._ListType or KeyType == self._TupleType:
 936             FirstKey = key[0]
 937             if len(key) > 1:
 938                 RestKeys = key[1:]
 939             elif self._Level_ > 1:
 940                 RestKeys = [self._Wildcard for i in range(0, self._Level_ - 1)]
 941         else:
 942             FirstKey = key
 943             if self._Level_ > 1:
 944                 RestKeys = [self._Wildcard for i in range(0, self._Level_ - 1)]
 945
 946         if FirstKey is None or str(FirstKey).upper() in self._ValidWildcardList:
 947             FirstKey = self._Wildcard
 948
 949         if self._Single_:
 950             return self._GetSingleValue(FirstKey, RestKeys)
 951         else:
 952             return self._GetAllValues(FirstKey, RestKeys)
 953
 954     def _GetSingleValue(self, FirstKey, RestKeys):
 955         Value = None
 956         #print "%s-%s" % (FirstKey, self._Level_) ,
 957         if self._Level_ > 1:
 958             if FirstKey == self._Wildcard:
 959                 if FirstKey in self.data:
 960                     Value = self.data[FirstKey][RestKeys]
 961                 if Value is None:
 962                     for Key in self.data:
 963                         Value = self.data[Key][RestKeys]
 964                         if Value is not None: break
 965             else:
 966                 if FirstKey in self.data:
 967                     Value = self.data[FirstKey][RestKeys]
 968                 if Value is None and self._Wildcard in self.data:
 969                     #print "Value=None"
 970                     Value = self.data[self._Wildcard][RestKeys]
 971         else:
 972             if FirstKey == self._Wildcard:
 973                 if FirstKey in self.data:
 974                     Value = self.data[FirstKey]
 975                 if Value is None:
 976                     for Key in self.data:
 977                         Value = self.data[Key]
 978                         if Value is not None: break
 979             else:
 980                 if FirstKey in self.data:
 981                     Value = self.data[FirstKey]
 982                 elif self._Wildcard in self.data:
 983                     Value = self.data[self._Wildcard]
 984         return Value
 985
 986     def _GetAllValues(self, FirstKey, RestKeys):
 987         Value = []
 988         if self._Level_ > 1:
 989             if FirstKey == self._Wildcard:
 990                 for Key in self.data:
 991                     Value += self.data[Key][RestKeys]
 992             else:
 993                 if FirstKey in self.data:
 994                     Value += self.data[FirstKey][RestKeys]
 995                 if self._Wildcard in self.data:
 996                     Value += self.data[self._Wildcard][RestKeys]
 997         else:
 998             if FirstKey == self._Wildcard:
 999                 for Key in self.data:
1000                     Value.append(self.data[Key])
1001             else:
1002                 if FirstKey in self.data:
1003                     Value.append(self.data[FirstKey])
1004                 if self._Wildcard in self.data:
1005                     Value.append(self.data[self._Wildcard])
1006         return Value
1007
1008     ## []= operator
1009     def __setitem__(self, key, value):
1010         KeyType = type(key)
1011         RestKeys = None
1012         if KeyType == self._ListType or KeyType == self._TupleType:
1013             FirstKey = key[0]
1014             if len(key) > 1:
1015                 RestKeys = key[1:]
1016             else:
1017                 RestKeys = [self._Wildcard for i in range(0, self._Level_ - 1)]
1018         else:
1019             FirstKey = key
1020             if self._Level_ > 1:
1021                 RestKeys = [self._Wildcard for i in range(0, self._Level_ - 1)]
1022
1023         if FirstKey in self._ValidWildcardList:
1024             FirstKey = self._Wildcard
1025
1026         if FirstKey not in self.data and self._Level_ > 0:
1027             self.data[FirstKey] = tdict(self._Single_, self._Level_ - 1)
1028
1029         if self._Level_ > 1:
1030             self.data[FirstKey][RestKeys] = value
1031         else:
1032             self.data[FirstKey] = value
1033
1034     def SetGreedyMode(self):
1035         self._Single_ = False
1036         if self._Level_ > 1:
1037             for Key in self.data:
1038                 self.data[Key].SetGreedyMode()
1039
1040     def SetSingleMode(self):
1041         self._Single_ = True
1042         if self._Level_ > 1:
1043             for Key in self.data:
1044                 self.data[Key].SetSingleMode()
1045
1046     def GetKeys(self, KeyIndex=0):
1047         assert KeyIndex >= 0
1048         if KeyIndex == 0:
1049             return set(self.data.keys())
1050         else:
1051             keys = set()
1052             for Key in self.data:
1053                 keys |= self.data[Key].GetKeys(KeyIndex - 1)
1054             return keys
1055
1056 def AnalyzePcdExpression(Setting):
1057     RanStr = ''.join(sample(string.ascii_letters + string.digits, 8))
1058     Setting = Setting.replace('\\\\', RanStr).strip()
1059     # There might be escaped quote in a string: \", \\\" , \', \\\'
1060     Data = Setting
1061     # There might be '|' in string and in ( ... | ... ), replace it with '-'
1062     NewStr = ''
1063     InSingleQuoteStr = False
1064     InDoubleQuoteStr = False
1065     Pair = 0
1066     for Index, ch in enumerate(Data):
1067         if ch == '"' and not InSingleQuoteStr:
1068             if Data[Index - 1] != '\\':
1069                 InDoubleQuoteStr = not InDoubleQuoteStr
1070         elif ch == "'" and not InDoubleQuoteStr:
1071             if Data[Index - 1] != '\\':
1072                 InSingleQuoteStr = not InSingleQuoteStr
1073         elif ch == '(' and not (InSingleQuoteStr or InDoubleQuoteStr):
1074             Pair += 1
1075         elif ch == ')' and not (InSingleQuoteStr or InDoubleQuoteStr):
1076             Pair -= 1
1077
1078         if (Pair > 0 or InSingleQuoteStr or InDoubleQuoteStr) and ch == TAB_VALUE_SPLIT:
1079             NewStr += '-'
1080         else:
1081             NewStr += ch
1082     FieldList = []
1083     StartPos = 0
1084     while True:
1085         Pos = NewStr.find(TAB_VALUE_SPLIT, StartPos)
1086         if Pos < 0:
1087             FieldList.append(Setting[StartPos:].strip())
1088             break
1089         FieldList.append(Setting[StartPos:Pos].strip())
1090         StartPos = Pos + 1
1091     for i, ch in enumerate(FieldList):
1092         if RanStr in ch:
1093             FieldList[i] = ch.replace(RanStr,'\\\\')
1094     return FieldList
1095
1096 def ParseFieldValue (Value):
1097     def ParseDevPathValue (Value):
1098         if '\\' in Value:
1099             Value.replace('\\', '/').replace(' ', '')
1100
1101         Cmd = 'DevicePath ' + '"' + Value + '"'
1102         try:
1103             p = subprocess.Popen(Cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
1104             out, err = p.communicate()
1105         except Exception as X:
1106             raise BadExpression("DevicePath: %s" % (str(X)) )
1107         finally:
1108             subprocess._cleanup()
1109             p.stdout.close()
1110             p.stderr.close()
1111         if err:
1112             raise BadExpression("DevicePath: %s" % str(err))
1113         out = out.decode()
1114         Size = len(out.split())
1115         out = ','.join(out.split())
1116         return '{' + out + '}', Size
1117
1118     if "{CODE(" in Value:
1119         return Value, len(Value.split(","))
1120     if isinstance(Value, type(0)):
1121         return Value, (Value.bit_length() + 7) // 8
1122     if not isinstance(Value, type('')):
1123         raise BadExpression('Type %s is %s' %(Value, type(Value)))
1124     Value = Value.strip()
1125     if Value.startswith(TAB_UINT8) and Value.endswith(')'):
1126         Value, Size = ParseFieldValue(Value.split('(', 1)[1][:-1])
1127         if Size > 1:
1128             raise BadExpression('Value (%s) Size larger than %d' %(Value, Size))
1129         return Value, 1
1130     if Value.startswith(TAB_UINT16) and Value.endswith(')'):
1131         Value, Size = ParseFieldValue(Value.split('(', 1)[1][:-1])
1132         if Size > 2:
1133             raise BadExpression('Value (%s) Size larger than %d' %(Value, Size))
1134         return Value, 2
1135     if Value.startswith(TAB_UINT32) and Value.endswith(')'):
1136         Value, Size = ParseFieldValue(Value.split('(', 1)[1][:-1])
1137         if Size > 4:
1138             raise BadExpression('Value (%s) Size larger than %d' %(Value, Size))
1139         return Value, 4
1140     if Value.startswith(TAB_UINT64) and Value.endswith(')'):
1141         Value, Size = ParseFieldValue(Value.split('(', 1)[1][:-1])
1142         if Size > 8:
1143             raise BadExpression('Value (%s) Size larger than %d' % (Value, Size))
1144         return Value, 8
1145     if Value.startswith(TAB_GUID) and Value.endswith(')'):
1146         Value = Value.split('(', 1)[1][:-1].strip()
1147         if Value[0] == '{' and Value[-1] == '}':
1148             TmpValue = GuidStructureStringToGuidString(Value)
1149             if not TmpValue:
1150                 raise BadExpression("Invalid GUID value string %s" % Value)
1151             Value = TmpValue
1152         if Value[0] == '"' and Value[-1] == '"':
1153             Value = Value[1:-1]
1154         try:
1155             Value = str(uuid.UUID(Value).bytes_le)
1156             if Value.startswith("b'"):
1157                 Value = Value[2:-1]
1158             Value = "'" + Value + "'"
1159         except ValueError as Message:
1160             raise BadExpression(Message)
1161         Value, Size = ParseFieldValue(Value)
1162         return Value, 16
1163     if Value.startswith('L"') and Value.endswith('"'):
1164         # Unicode String
1165         # translate escape character
1166         Value = Value[1:]
1167         try:
1168             Value = eval(Value)
1169         except:
1170             Value = Value[1:-1]
1171         List = list(Value)
1172         List.reverse()
1173         Value = 0
1174         for Char in List:
1175             Value = (Value << 16) | ord(Char)
1176         return Value, (len(List) + 1) * 2
1177     if Value.startswith('"') and Value.endswith('"'):
1178         # ASCII String
1179         # translate escape character
1180         try:
1181             Value = eval(Value)
1182         except:
1183             Value = Value[1:-1]
1184         List = list(Value)
1185         List.reverse()
1186         Value = 0
1187         for Char in List:
1188             Value = (Value << 8) | ord(Char)
1189         return Value, len(List) + 1
1190     if Value.startswith("L'") and Value.endswith("'"):
1191         # Unicode Character Constant
1192         # translate escape character
1193         Value = Value[1:]
1194         try:
1195             Value = eval(Value)
1196         except:
1197             Value = Value[1:-1]
1198         List = list(Value)
1199         if len(List) == 0:
1200             raise BadExpression('Length %s is %s' % (Value, len(List)))
1201         List.reverse()
1202         Value = 0
1203         for Char in List:
1204             Value = (Value << 16) | ord(Char)
1205         return Value, len(List) * 2
1206     if Value.startswith("'") and Value.endswith("'"):
1207         # Character constant
1208         # translate escape character
1209         try:
1210             Value = eval(Value)
1211         except:
1212             Value = Value[1:-1]
1213         List = list(Value)
1214         if len(List) == 0:
1215             raise BadExpression('Length %s is %s' % (Value, len(List)))
1216         List.reverse()
1217         Value = 0
1218         for Char in List:
1219             Value = (Value << 8) | ord(Char)
1220         return Value, len(List)
1221     if Value.startswith('{') and Value.endswith('}'):
1222         # Byte array
1223         Value = Value[1:-1]
1224         List = [Item.strip() for Item in Value.split(',')]
1225         List.reverse()
1226         Value = 0
1227         RetSize = 0
1228         for Item in List:
1229             ItemValue, Size = ParseFieldValue(Item)
1230             RetSize += Size
1231             for I in range(Size):
1232                 Value = (Value << 8) | ((ItemValue >> 8 * I) & 0xff)
1233         return Value, RetSize
1234     if Value.startswith('DEVICE_PATH(') and Value.endswith(')'):
1235         Value = Value.replace("DEVICE_PATH(", '').rstrip(')')
1236         Value = Value.strip().strip('"')
1237         return ParseDevPathValue(Value)
1238     if Value.lower().startswith('0x'):
1239         try:
1240             Value = int(Value, 16)
1241         except:
1242             raise BadExpression("invalid hex value: %s" % Value)
1243         if Value == 0:
1244             return 0, 1
1245         return Value, (Value.bit_length() + 7) // 8
1246     if Value[0].isdigit():
1247         Value = int(Value, 10)
1248         if Value == 0:
1249             return 0, 1
1250         return Value, (Value.bit_length() + 7) // 8
1251     if Value.lower() == 'true':
1252         return 1, 1
1253     if Value.lower() == 'false':
1254         return 0, 1
1255     return Value, 1
1256
1257 ## AnalyzeDscPcd
1258 #
1259 #  Analyze DSC PCD value, since there is no data type info in DSC
1260 #  This function is used to match functions (AnalyzePcdData) used for retrieving PCD value from database
1261 #  1. Feature flag: TokenSpace.PcdCName|PcdValue
1262 #  2. Fix and Patch:TokenSpace.PcdCName|PcdValue[|VOID*[|MaxSize]]
1263 #  3. Dynamic default:
1264 #     TokenSpace.PcdCName|PcdValue[|VOID*[|MaxSize]]
1265 #     TokenSpace.PcdCName|PcdValue
1266 #  4. Dynamic VPD:
1267 #     TokenSpace.PcdCName|VpdOffset[|VpdValue]
1268 #     TokenSpace.PcdCName|VpdOffset[|MaxSize[|VpdValue]]
1269 #  5. Dynamic HII:
1270 #     TokenSpace.PcdCName|HiiString|VariableGuid|VariableOffset[|HiiValue]
1271 #  PCD value needs to be located in such kind of string, and the PCD value might be an expression in which
1272 #    there might have "|" operator, also in string value.
1273 #
1274 #  @param Setting: String contain information described above with "TokenSpace.PcdCName|" stripped
1275 #  @param PcdType: PCD type: feature, fixed, dynamic default VPD HII
1276 #  @param DataType: The datum type of PCD: VOID*, UNIT, BOOL
1277 #  @retval:
1278 #    ValueList: A List contain fields described above
1279 #    IsValid:   True if conforming EBNF, otherwise False
1280 #    Index:     The index where PcdValue is in ValueList
1281 #
1282 def AnalyzeDscPcd(Setting, PcdType, DataType=''):
1283     FieldList = AnalyzePcdExpression(Setting)
1284
1285     IsValid = True
1286     if PcdType in (MODEL_PCD_FIXED_AT_BUILD, MODEL_PCD_PATCHABLE_IN_MODULE, MODEL_PCD_DYNAMIC_DEFAULT, MODEL_PCD_DYNAMIC_EX_DEFAULT):
1287         Value = FieldList[0]
1288         Size = ''
1289         if len(FieldList) > 1 and FieldList[1]:
1290             DataType = FieldList[1]
1291             if FieldList[1] != TAB_VOID and StructPattern.match(FieldList[1]) is None:
1292                 IsValid = False
1293         if len(FieldList) > 2:
1294             Size = FieldList[2]
1295         if IsValid:
1296             if DataType == "":
1297                 IsValid = (len(FieldList) <= 1)
1298             else:
1299                 IsValid = (len(FieldList) <= 3)
1300
1301         if Size:
1302             try:
1303                 int(Size, 16) if Size.upper().startswith("0X") else int(Size)
1304             except:
1305                 IsValid = False
1306                 Size = -1
1307         return [str(Value), DataType, str(Size)], IsValid, 0
1308     elif PcdType == MODEL_PCD_FEATURE_FLAG:
1309         Value = FieldList[0]
1310         Size = ''
1311         IsValid = (len(FieldList) <= 1)
1312         return [Value, DataType, str(Size)], IsValid, 0
1313     elif PcdType in (MODEL_PCD_DYNAMIC_VPD, MODEL_PCD_DYNAMIC_EX_VPD):
1314         VpdOffset = FieldList[0]
1315         Value = Size = ''
1316         if not DataType == TAB_VOID:
1317             if len(FieldList) > 1:
1318                 Value = FieldList[1]
1319         else:
1320             if len(FieldList) > 1:
1321                 Size = FieldList[1]
1322             if len(FieldList) > 2:
1323                 Value = FieldList[2]
1324         if DataType == "":
1325             IsValid = (len(FieldList) <= 1)
1326         else:
1327             IsValid = (len(FieldList) <= 3)
1328         if Size:
1329             try:
1330                 int(Size, 16) if Size.upper().startswith("0X") else int(Size)
1331             except:
1332                 IsValid = False
1333                 Size = -1
1334         return [VpdOffset, str(Size), Value], IsValid, 2
1335     elif PcdType in (MODEL_PCD_DYNAMIC_HII, MODEL_PCD_DYNAMIC_EX_HII):
1336         IsValid = (3 <= len(FieldList) <= 5)
1337         HiiString = FieldList[0]
1338         Guid = Offset = Value = Attribute = ''
1339         if len(FieldList) > 1:
1340             Guid = FieldList[1]
1341         if len(FieldList) > 2:
1342             Offset = FieldList[2]
1343         if len(FieldList) > 3:
1344             Value = FieldList[3]
1345         if len(FieldList) > 4:
1346             Attribute = FieldList[4]
1347         return [HiiString, Guid, Offset, Value, Attribute], IsValid, 3
1348     return [], False, 0
1349
1350 ## AnalyzePcdData
1351 #
1352 #  Analyze the pcd Value, Datum type and TokenNumber.
1353 #  Used to avoid split issue while the value string contain "|" character
1354 #
1355 #  @param[in] Setting:  A String contain value/datum type/token number information;
1356 #
1357 #  @retval   ValueList: A List contain value, datum type and toke number.
1358 #
1359 def AnalyzePcdData(Setting):
1360     ValueList = ['', '', '']
1361
1362     ValueRe = re.compile(r'^\s*L?\".*\|.*\"')
1363     PtrValue = ValueRe.findall(Setting)
1364
1365     ValueUpdateFlag = False
1366
1367     if len(PtrValue) >= 1:
1368         Setting = re.sub(ValueRe, '', Setting)
1369         ValueUpdateFlag = True
1370
1371     TokenList = Setting.split(TAB_VALUE_SPLIT)
1372     ValueList[0:len(TokenList)] = TokenList
1373
1374     if ValueUpdateFlag:
1375         ValueList[0] = PtrValue[0]
1376
1377     return ValueList
1378
1379 ## check format of PCD value against its the datum type
1380 #
1381 # For PCD value setting
1382 #
1383 def CheckPcdDatum(Type, Value):
1384     if Type == TAB_VOID:
1385         ValueRe = re.compile(r'\s*L?\".*\"\s*$')
1386         if not (((Value.startswith('L"') or Value.startswith('"')) and Value.endswith('"'))
1387                 or (Value.startswith('{') and Value.endswith('}')) or (Value.startswith("L'") or Value.startswith("'") and Value.endswith("'"))
1388                ):
1389             return False, "Invalid value [%s] of type [%s]; must be in the form of {...} for array"\
1390                           ", \"...\" or \'...\' for string, L\"...\" or L\'...\' for unicode string" % (Value, Type)
1391         elif ValueRe.match(Value):
1392             # Check the chars in UnicodeString or CString is printable
1393             if Value.startswith("L"):
1394                 Value = Value[2:-1]
1395             else:
1396                 Value = Value[1:-1]
1397             Printset = set(string.printable)
1398             Printset.remove(TAB_PRINTCHAR_VT)
1399             Printset.add(TAB_PRINTCHAR_BS)
1400             Printset.add(TAB_PRINTCHAR_NUL)
1401             if not set(Value).issubset(Printset):
1402                 PrintList = sorted(Printset)
1403                 return False, "Invalid PCD string value of type [%s]; must be printable chars %s." % (Type, PrintList)
1404     elif Type == 'BOOLEAN':
1405         if Value not in ['TRUE', 'True', 'true', '0x1', '0x01', '1', 'FALSE', 'False', 'false', '0x0', '0x00', '0']:
1406             return False, "Invalid value [%s] of type [%s]; must be one of TRUE, True, true, 0x1, 0x01, 1"\
1407                           ", FALSE, False, false, 0x0, 0x00, 0" % (Value, Type)
1408     elif Type in [TAB_UINT8, TAB_UINT16, TAB_UINT32, TAB_UINT64]:
1409         if Value.startswith('0') and not Value.lower().startswith('0x') and len(Value) > 1 and Value.lstrip('0'):
1410             Value = Value.lstrip('0')
1411         try:
1412             if Value and int(Value, 0) < 0:
1413                 return False, "PCD can't be set to negative value[%s] for datum type [%s]" % (Value, Type)
1414             Value = int(Value, 0)
1415             if Value > MAX_VAL_TYPE[Type]:
1416                 return False, "Too large PCD value[%s] for datum type [%s]" % (Value, Type)
1417         except:
1418             return False, "Invalid value [%s] of type [%s];"\
1419                           " must be a hexadecimal, decimal or octal in C language format." % (Value, Type)
1420     else:
1421         return True, "StructurePcd"
1422
1423     return True, ""
1424
1425 def CommonPath(PathList):
1426     P1 = min(PathList).split(os.path.sep)
1427     P2 = max(PathList).split(os.path.sep)
1428     for Index in range(min(len(P1), len(P2))):
1429         if P1[Index] != P2[Index]:
1430             return os.path.sep.join(P1[:Index])
1431     return os.path.sep.join(P1)
1432
1433 class PathClass(object):
1434     def __init__(self, File='', Root='', AlterRoot='', Type='', IsBinary=False,
1435                  Arch='COMMON', ToolChainFamily='', Target='', TagName='', ToolCode=''):
1436         self.Arch = Arch
1437         self.File = str(File)
1438         if os.path.isabs(self.File):
1439             self.Root = ''
1440             self.AlterRoot = ''
1441         else:
1442             self.Root = str(Root)
1443             self.AlterRoot = str(AlterRoot)
1444
1445         # Remove any '.' and '..' in path
1446         if self.Root:
1447             self.Root = mws.getWs(self.Root, self.File)
1448             self.Path = os.path.normpath(os.path.join(self.Root, self.File))
1449             self.Root = os.path.normpath(CommonPath([self.Root, self.Path]))
1450             # eliminate the side-effect of 'C:'
1451             if self.Root[-1] == ':':
1452                 self.Root += os.path.sep
1453             # file path should not start with path separator
1454             if self.Root[-1] == os.path.sep:
1455                 self.File = self.Path[len(self.Root):]
1456             else:
1457                 self.File = self.Path[len(self.Root) + 1:]
1458         else:
1459             self.Path = os.path.normpath(self.File)
1460
1461         self.SubDir, self.Name = os.path.split(self.File)
1462         self.BaseName, self.Ext = os.path.splitext(self.Name)
1463
1464         if self.Root:
1465             if self.SubDir:
1466                 self.Dir = os.path.join(self.Root, self.SubDir)
1467             else:
1468                 self.Dir = self.Root
1469         else:
1470             self.Dir = self.SubDir
1471
1472         if IsBinary:
1473             self.Type = Type
1474         else:
1475             self.Type = self.Ext.lower()
1476
1477         self.IsBinary = IsBinary
1478         self.Target = Target
1479         self.TagName = TagName
1480         self.ToolCode = ToolCode
1481         self.ToolChainFamily = ToolChainFamily
1482         self.OriginalPath = self
1483
1484     ## Convert the object of this class to a string
1485     #
1486     #  Convert member Path of the class to a string
1487     #
1488     #  @retval string Formatted String
1489     #
1490     def __str__(self):
1491         return self.Path
1492
1493     ## Override __eq__ function
1494     #
1495     # Check whether PathClass are the same
1496     #
1497     # @retval False The two PathClass are different
1498     # @retval True  The two PathClass are the same
1499     #
1500     def __eq__(self, Other):
1501         return self.Path == str(Other)
1502
1503     ## Override __cmp__ function
1504     #
1505     # Customize the comparison operation of two PathClass
1506     #
1507     # @retval 0     The two PathClass are different
1508     # @retval -1    The first PathClass is less than the second PathClass
1509     # @retval 1     The first PathClass is Bigger than the second PathClass
1510     def __cmp__(self, Other):
1511         OtherKey = str(Other)
1512
1513         SelfKey = self.Path
1514         if SelfKey == OtherKey:
1515             return 0
1516         elif SelfKey > OtherKey:
1517             return 1
1518         else:
1519             return -1
1520
1521     ## Override __hash__ function
1522     #
1523     # Use Path as key in hash table
1524     #
1525     # @retval string Key for hash table
1526     #
1527     def __hash__(self):
1528         return hash(self.Path)
1529
1530     @cached_property
1531     def Key(self):
1532         return self.Path.upper()
1533
1534     @property
1535     def TimeStamp(self):
1536         return os.stat(self.Path)[8]
1537
1538     def Validate(self, Type='', CaseSensitive=True):
1539         def RealPath2(File, Dir='', OverrideDir=''):
1540             NewFile = None
1541             if OverrideDir:
1542                 NewFile = GlobalData.gAllFiles[os.path.normpath(os.path.join(OverrideDir, File))]
1543                 if NewFile:
1544                     if OverrideDir[-1] == os.path.sep:
1545                         return NewFile[len(OverrideDir):], NewFile[0:len(OverrideDir)]
1546                     else:
1547                         return NewFile[len(OverrideDir) + 1:], NewFile[0:len(OverrideDir)]
1548             if GlobalData.gAllFiles:
1549                 NewFile = GlobalData.gAllFiles[os.path.normpath(os.path.join(Dir, File))]
1550             if not NewFile:
1551                 NewFile = os.path.normpath(os.path.join(Dir, File))
1552                 if not os.path.exists(NewFile):
1553                     return None, None
1554             if NewFile:
1555                 if Dir:
1556                     if Dir[-1] == os.path.sep:
1557                         return NewFile[len(Dir):], NewFile[0:len(Dir)]
1558                     else:
1559                         return NewFile[len(Dir) + 1:], NewFile[0:len(Dir)]
1560                 else:
1561                     return NewFile, ''
1562
1563             return None, None
1564
1565         if GlobalData.gCaseInsensitive:
1566             CaseSensitive = False
1567         if Type and Type.lower() != self.Type:
1568             return FILE_TYPE_MISMATCH, '%s (expect %s but got %s)' % (self.File, Type, self.Type)
1569
1570         RealFile, RealRoot = RealPath2(self.File, self.Root, self.AlterRoot)
1571         if not RealRoot and not RealFile:
1572             RealFile = self.File
1573             if self.AlterRoot:
1574                 RealFile = os.path.join(self.AlterRoot, self.File)
1575             elif self.Root:
1576                 RealFile = os.path.join(self.Root, self.File)
1577             if len (mws.getPkgPath()) == 0:
1578                 return FILE_NOT_FOUND, os.path.join(self.AlterRoot, RealFile)
1579             else:
1580                 return FILE_NOT_FOUND, "%s is not found in packages path:\n\t%s" % (self.File, '\n\t'.join(mws.getPkgPath()))
1581
1582         ErrorCode = 0
1583         ErrorInfo = ''
1584         if RealRoot != self.Root or RealFile != self.File:
1585             if CaseSensitive and (RealFile != self.File or (RealRoot != self.Root and RealRoot != self.AlterRoot)):
1586                 ErrorCode = FILE_CASE_MISMATCH
1587                 ErrorInfo = self.File + '\n\t' + RealFile + " [in file system]"
1588
1589             self.SubDir, self.Name = os.path.split(RealFile)
1590             self.BaseName, self.Ext = os.path.splitext(self.Name)
1591             if self.SubDir:
1592                 self.Dir = os.path.join(RealRoot, self.SubDir)
1593             else:
1594                 self.Dir = RealRoot
1595             self.File = RealFile
1596             self.Root = RealRoot
1597             self.Path = os.path.join(RealRoot, RealFile)
1598         return ErrorCode, ErrorInfo
1599
1600 ## Parse PE image to get the required PE information.
1601 #
1602 class PeImageClass():
1603     ## Constructor
1604     #
1605     #   @param  File FilePath of PeImage
1606     #
1607     def __init__(self, PeFile):
1608         self.FileName   = PeFile
1609         self.IsValid    = False
1610         self.Size       = 0
1611         self.EntryPoint = 0
1612         self.SectionAlignment  = 0
1613         self.SectionHeaderList = []
1614         self.ErrorInfo = ''
1615         try:
1616             PeObject = open(PeFile, 'rb')
1617         except:
1618             self.ErrorInfo = self.FileName + ' can not be found\n'
1619             return
1620         # Read DOS header
1621         ByteArray = array.array('B')
1622         ByteArray.fromfile(PeObject, 0x3E)
1623         ByteList = ByteArray.tolist()
1624         # DOS signature should be 'MZ'
1625         if self._ByteListToStr (ByteList[0x0:0x2]) != 'MZ':
1626             self.ErrorInfo = self.FileName + ' has no valid DOS signature MZ'
1627             return
1628
1629         # Read 4 byte PE Signature
1630         PeOffset = self._ByteListToInt(ByteList[0x3C:0x3E])
1631         PeObject.seek(PeOffset)
1632         ByteArray = array.array('B')
1633         ByteArray.fromfile(PeObject, 4)
1634         # PE signature should be 'PE\0\0'
1635         if ByteArray.tostring() != b'PE\0\0':
1636             self.ErrorInfo = self.FileName + ' has no valid PE signature PE00'
1637             return
1638
1639         # Read PE file header
1640         ByteArray = array.array('B')
1641         ByteArray.fromfile(PeObject, 0x14)
1642         ByteList = ByteArray.tolist()
1643         SecNumber = self._ByteListToInt(ByteList[0x2:0x4])
1644         if SecNumber == 0:
1645             self.ErrorInfo = self.FileName + ' has no section header'
1646             return
1647
1648         # Read PE optional header
1649         OptionalHeaderSize = self._ByteListToInt(ByteArray[0x10:0x12])
1650         ByteArray = array.array('B')
1651         ByteArray.fromfile(PeObject, OptionalHeaderSize)
1652         ByteList = ByteArray.tolist()
1653         self.EntryPoint       = self._ByteListToInt(ByteList[0x10:0x14])
1654         self.SectionAlignment = self._ByteListToInt(ByteList[0x20:0x24])
1655         self.Size             = self._ByteListToInt(ByteList[0x38:0x3C])
1656
1657         # Read each Section Header
1658         for Index in range(SecNumber):
1659             ByteArray = array.array('B')
1660             ByteArray.fromfile(PeObject, 0x28)
1661             ByteList = ByteArray.tolist()
1662             SecName  = self._ByteListToStr(ByteList[0:8])
1663             SecVirtualSize = self._ByteListToInt(ByteList[8:12])
1664             SecRawAddress  = self._ByteListToInt(ByteList[20:24])
1665             SecVirtualAddress = self._ByteListToInt(ByteList[12:16])
1666             self.SectionHeaderList.append((SecName, SecVirtualAddress, SecRawAddress, SecVirtualSize))
1667         self.IsValid = True
1668         PeObject.close()
1669
1670     def _ByteListToStr(self, ByteList):
1671         String = ''
1672         for index in range(len(ByteList)):
1673             if ByteList[index] == 0:
1674                 break
1675             String += chr(ByteList[index])
1676         return String
1677
1678     def _ByteListToInt(self, ByteList):
1679         Value = 0
1680         for index in range(len(ByteList) - 1, -1, -1):
1681             Value = (Value << 8) | int(ByteList[index])
1682         return Value
1683
1684 class DefaultStore():
1685     def __init__(self, DefaultStores ):
1686
1687         self.DefaultStores = DefaultStores
1688     def DefaultStoreID(self, DefaultStoreName):
1689         for key, value in self.DefaultStores.items():
1690             if value == DefaultStoreName:
1691                 return key
1692         return None
1693     def GetDefaultDefault(self):
1694         if not self.DefaultStores or "0" in self.DefaultStores:
1695             return "0", TAB_DEFAULT_STORES_DEFAULT
1696         else:
1697             minvalue = min(int(value_str) for value_str in self.DefaultStores)
1698             return (str(minvalue), self.DefaultStores[str(minvalue)])
1699     def GetMin(self, DefaultSIdList):
1700         if not DefaultSIdList:
1701             return TAB_DEFAULT_STORES_DEFAULT
1702         storeidset = {storeid for storeid, storename in self.DefaultStores.values() if storename in DefaultSIdList}
1703         if not storeidset:
1704             return ""
1705         minid = min(storeidset )
1706         for sid, name in self.DefaultStores.values():
1707             if sid == minid:
1708                 return name
1709
1710 class SkuClass():
1711     DEFAULT = 0
1712     SINGLE = 1
1713     MULTIPLE =2
1714
1715     def __init__(self,SkuIdentifier='', SkuIds=None):
1716         if SkuIds is None:
1717             SkuIds = {}
1718
1719         for SkuName in SkuIds:
1720             SkuId = SkuIds[SkuName][0]
1721             skuid_num = int(SkuId, 16) if SkuId.upper().startswith("0X") else int(SkuId)
1722             if skuid_num > 0xFFFFFFFFFFFFFFFF:
1723                 EdkLogger.error("build", PARAMETER_INVALID,
1724                             ExtraData = "SKU-ID [%s] value %s exceeds the max value of UINT64"
1725                                       % (SkuName, SkuId))
1726
1727         self.AvailableSkuIds = OrderedDict()
1728         self.SkuIdSet = []
1729         self.SkuIdNumberSet = []
1730         self.SkuData = SkuIds
1731         self._SkuInherit = {}
1732         self._SkuIdentifier = SkuIdentifier
1733         if SkuIdentifier == '' or SkuIdentifier is None:
1734             self.SkuIdSet = ['DEFAULT']
1735             self.SkuIdNumberSet = ['0U']
1736         elif SkuIdentifier == 'ALL':
1737             self.SkuIdSet = list(SkuIds.keys())
1738             self.SkuIdNumberSet = [num[0].strip() + 'U' for num in SkuIds.values()]
1739         else:
1740             r = SkuIdentifier.split('|')
1741             self.SkuIdSet=[(r[k].strip()).upper() for k in range(len(r))]
1742             k = None
1743             try:
1744                 self.SkuIdNumberSet = [SkuIds[k][0].strip() + 'U' for k in self.SkuIdSet]
1745             except Exception:
1746                 EdkLogger.error("build", PARAMETER_INVALID,
1747                             ExtraData = "SKU-ID [%s] is not supported by the platform. [Valid SKU-ID: %s]"
1748                                       % (k, " | ".join(SkuIds.keys())))
1749         for each in self.SkuIdSet:
1750             if each in SkuIds:
1751                 self.AvailableSkuIds[each] = SkuIds[each][0]
1752             else:
1753                 EdkLogger.error("build", PARAMETER_INVALID,
1754                             ExtraData="SKU-ID [%s] is not supported by the platform. [Valid SKU-ID: %s]"
1755                                       % (each, " | ".join(SkuIds.keys())))
1756         if self.SkuUsageType != SkuClass.SINGLE:
1757             self.AvailableSkuIds.update({'DEFAULT':0, 'COMMON':0})
1758         if self.SkuIdSet:
1759             GlobalData.gSkuids = (self.SkuIdSet)
1760             if 'COMMON' in GlobalData.gSkuids:
1761                 GlobalData.gSkuids.remove('COMMON')
1762             if self.SkuUsageType == self.SINGLE:
1763                 if len(GlobalData.gSkuids) != 1:
1764                     if 'DEFAULT' in GlobalData.gSkuids:
1765                         GlobalData.gSkuids.remove('DEFAULT')
1766             if GlobalData.gSkuids:
1767                 GlobalData.gSkuids.sort()
1768
1769     def GetNextSkuId(self, skuname):
1770         if not self._SkuInherit:
1771             self._SkuInherit = {}
1772             for item in self.SkuData.values():
1773                 self._SkuInherit[item[1]]=item[2] if item[2] else "DEFAULT"
1774         return self._SkuInherit.get(skuname, "DEFAULT")
1775
1776     def GetSkuChain(self, sku):
1777         if sku == "DEFAULT":
1778             return ["DEFAULT"]
1779         skulist = [sku]
1780         nextsku = sku
1781         while True:
1782             nextsku = self.GetNextSkuId(nextsku)
1783             skulist.append(nextsku)
1784             if nextsku == "DEFAULT":
1785                 break
1786         skulist.reverse()
1787         return skulist
1788     def SkuOverrideOrder(self):
1789         skuorderset = []
1790         for skuname in self.SkuIdSet:
1791             skuorderset.append(self.GetSkuChain(skuname))
1792
1793         skuorder = []
1794         for index in range(max(len(item) for item in skuorderset)):
1795             for subset in skuorderset:
1796                 if index > len(subset)-1:
1797                     continue
1798                 if subset[index] in skuorder:
1799                     continue
1800                 skuorder.append(subset[index])
1801
1802         return skuorder
1803
1804     @property
1805     def SkuUsageType(self):
1806         if self._SkuIdentifier.upper() == "ALL":
1807             return SkuClass.MULTIPLE
1808
1809         if len(self.SkuIdSet) == 1:
1810             if self.SkuIdSet[0] == 'DEFAULT':
1811                 return SkuClass.DEFAULT
1812             return SkuClass.SINGLE
1813         if len(self.SkuIdSet) == 2 and 'DEFAULT' in self.SkuIdSet:
1814             return SkuClass.SINGLE
1815         return SkuClass.MULTIPLE
1816
1817     def DumpSkuIdArrary(self):
1818         if self.SkuUsageType == SkuClass.SINGLE:
1819             return "{0x0}"
1820         ArrayStrList = []
1821         for skuname in self.AvailableSkuIds:
1822             if skuname == "COMMON":
1823                 continue
1824             while skuname != "DEFAULT":
1825                 ArrayStrList.append(hex(int(self.AvailableSkuIds[skuname])))
1826                 skuname = self.GetNextSkuId(skuname)
1827             ArrayStrList.append("0x0")
1828         return "{{{myList}}}".format(myList=",".join(ArrayStrList))
1829
1830     @property
1831     def AvailableSkuIdSet(self):
1832         return self.AvailableSkuIds
1833
1834     @property
1835     def SystemSkuId(self):
1836         if self.SkuUsageType == SkuClass.SINGLE:
1837             if len(self.SkuIdSet) == 1:
1838                 return self.SkuIdSet[0]
1839             else:
1840                 return self.SkuIdSet[0] if self.SkuIdSet[0] != 'DEFAULT' else self.SkuIdSet[1]
1841         else:
1842             return 'DEFAULT'
1843
1844 ##  Get the integer value from string like "14U" or integer like 2
1845 #
1846 #   @param      Input   The object that may be either a integer value or a string
1847 #
1848 #   @retval     Value    The integer value that the input represents
1849 #
1850 def GetIntegerValue(Input):
1851     if not isinstance(Input, str):
1852         return Input
1853     String = Input
1854     if String.endswith("U"):
1855         String = String[:-1]
1856     if String.endswith("ULL"):
1857         String = String[:-3]
1858     if String.endswith("LL"):
1859         String = String[:-2]
1860
1861     if String.startswith("0x") or String.startswith("0X"):
1862         return int(String, 16)
1863     elif String == '':
1864         return 0
1865     else:
1866         return int(String)
1867
1868 #
1869 # Pack a GUID (registry format) list into a buffer and return it
1870 #
1871 def PackGUID(Guid):
1872     return pack(PACK_PATTERN_GUID,
1873                 int(Guid[0], 16),
1874                 int(Guid[1], 16),
1875                 int(Guid[2], 16),
1876                 int(Guid[3][-4:-2], 16),
1877                 int(Guid[3][-2:], 16),
1878                 int(Guid[4][-12:-10], 16),
1879                 int(Guid[4][-10:-8], 16),
1880                 int(Guid[4][-8:-6], 16),
1881                 int(Guid[4][-6:-4], 16),
1882                 int(Guid[4][-4:-2], 16),
1883                 int(Guid[4][-2:], 16)
1884                 )
1885
1886 #
1887 # Pack a GUID (byte) list into a buffer and return it
1888 #
1889 def PackByteFormatGUID(Guid):
1890     return pack(PACK_PATTERN_GUID,
1891                 Guid[0],
1892                 Guid[1],
1893                 Guid[2],
1894                 Guid[3],
1895                 Guid[4],
1896                 Guid[5],
1897                 Guid[6],
1898                 Guid[7],
1899                 Guid[8],
1900                 Guid[9],
1901                 Guid[10],
1902                 )
1903
1904 ## DeepCopy dict/OrderedDict recusively
1905 #
1906 #   @param      ori_dict    a nested dict or ordereddict
1907 #
1908 #   @retval     new dict or orderdict
1909 #
1910 def CopyDict(ori_dict):
1911     dict_type = ori_dict.__class__
1912     if dict_type not in (dict,OrderedDict):
1913         return ori_dict
1914     new_dict = dict_type()
1915     for key in ori_dict:
1916         if isinstance(ori_dict[key],(dict,OrderedDict)):
1917             new_dict[key] = CopyDict(ori_dict[key])
1918         else:
1919             new_dict[key] = ori_dict[key]
1920     return new_dict
1921
1922 #
1923 # Remove the c/c++ comments: // and /* */
1924 #
1925 def RemoveCComments(ctext):
1926     return re.sub('//.*?\n|/\*.*?\*/', '\n', ctext, flags=re.S)