BaseTools/Source/Python/Common/Misc.py

   1 ## @file
   2 # Common routines used by all tools
   3 #
   4 # Copyright (c) 2007 - 2019, Intel Corporation. All rights reserved.<BR>
   5 # This program and the accompanying materials
   6 # are licensed and made available under the terms and conditions of the BSD License
   7 # which accompanies this distribution.  The full text of the license may be found at
   8 # http://opensource.org/licenses/bsd-license.php
   9 #
  10 # THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS,
  11 # WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED.
  12 #
  13
  14 ##
  15 # Import Modules
  16 #
  17 from __future__ import absolute_import
  18
  19 import sys
  20 import string
  21 import threading
  22 import time
  23 import re
  24 import pickle
  25 import array
  26 import shutil
  27 from random import sample
  28 from struct import pack
  29 import uuid
  30 import subprocess
  31 from collections import OrderedDict
  32
  33 import Common.LongFilePathOs as os
  34 from Common import EdkLogger as EdkLogger
  35 from Common import GlobalData as GlobalData
  36 from Common.DataType import *
  37 from Common.BuildToolError import *
  38 from CommonDataClass.DataClass import *
  39 from Common.Parsing import GetSplitValueList
  40 from Common.LongFilePathSupport import OpenLongFilePath as open
  41 from Common.MultipleWorkspace import MultipleWorkspace as mws
  42 from CommonDataClass.Exceptions import BadExpression
  43 from Common.caching import cached_property
  44
  45 ## Regular expression used to find out place holders in string template
  46 gPlaceholderPattern = re.compile("\$\{([^$()\s]+)\}", re.MULTILINE | re.UNICODE)
  47
  48 ## regular expressions for map file processing
  49 startPatternGeneral = re.compile("^Start[' ']+Length[' ']+Name[' ']+Class")
  50 addressPatternGeneral = re.compile("^Address[' ']+Publics by Value[' ']+Rva\+Base")
  51 valuePatternGcc = re.compile('^([\w_\.]+) +([\da-fA-Fx]+) +([\da-fA-Fx]+)$')
  52 pcdPatternGcc = re.compile('^([\da-fA-Fx]+) +([\da-fA-Fx]+)')
  53 secReGeneral = re.compile('^([\da-fA-F]+):([\da-fA-F]+) +([\da-fA-F]+)[Hh]? +([.\w\$]+) +(\w+)', re.UNICODE)
  54
  55 StructPattern = re.compile(r'[_a-zA-Z][0-9A-Za-z_]*$')
  56
  57 ## Dictionary used to store dependencies of files
  58 gDependencyDatabase = {}    # arch : {file path : [dependent files list]}
  59
  60 #
  61 # If a module is built more than once with different PCDs or library classes
  62 # a temporary INF file with same content is created, the temporary file is removed
  63 # when build exits.
  64 #
  65 _TempInfs = []
  66
  67 def GetVariableOffset(mapfilepath, efifilepath, varnames):
  68     """ Parse map file to get variable offset in current EFI file
  69     @param mapfilepath    Map file absolution path
  70     @param efifilepath:   EFI binary file full path
  71     @param varnames       iteratable container whose elements are variable names to be searched
  72
  73     @return List whos elements are tuple with variable name and raw offset
  74     """
  75     lines = []
  76     try:
  77         f = open(mapfilepath, 'r')
  78         lines = f.readlines()
  79         f.close()
  80     except:
  81         return None
  82
  83     if len(lines) == 0: return None
  84     firstline = lines[0].strip()
  85     if (firstline.startswith("Archive member included ") and
  86         firstline.endswith(" file (symbol)")):
  87         return _parseForGCC(lines, efifilepath, varnames)
  88     if firstline.startswith("# Path:"):
  89         return _parseForXcode(lines, efifilepath, varnames)
  90     return _parseGeneral(lines, efifilepath, varnames)
  91
  92 def _parseForXcode(lines, efifilepath, varnames):
  93     status = 0
  94     ret = []
  95     for line in lines:
  96         line = line.strip()
  97         if status == 0 and line == "# Symbols:":
  98             status = 1
  99             continue
 100         if status == 1 and len(line) != 0:
 101             for varname in varnames:
 102                 if varname in line:
 103                     # cannot pregenerate this RegEx since it uses varname from varnames.
 104                     m = re.match('^([\da-fA-FxX]+)([\s\S]*)([_]*%s)$' % varname, line)
 105                     if m is not None:
 106                         ret.append((varname, m.group(1)))
 107     return ret
 108
 109 def _parseForGCC(lines, efifilepath, varnames):
 110     """ Parse map file generated by GCC linker """
 111     status = 0
 112     sections = []
 113     varoffset = []
 114     for index, line in enumerate(lines):
 115         line = line.strip()
 116         # status machine transection
 117         if status == 0 and line == "Memory Configuration":
 118             status = 1
 119             continue
 120         elif status == 1 and line == 'Linker script and memory map':
 121             status = 2
 122             continue
 123         elif status ==2 and line == 'START GROUP':
 124             status = 3
 125             continue
 126
 127         # status handler
 128         if status == 3:
 129             m = valuePatternGcc.match(line)
 130             if m is not None:
 131                 sections.append(m.groups(0))
 132             for varname in varnames:
 133                 Str = ''
 134                 m = re.match("^.data.(%s)" % varname, line)
 135                 if m is not None:
 136                     m = re.match(".data.(%s)$" % varname, line)
 137                     if m is not None:
 138                         Str = lines[index + 1]
 139                     else:
 140                         Str = line[len(".data.%s" % varname):]
 141                     if Str:
 142                         m = pcdPatternGcc.match(Str.strip())
 143                         if m is not None:
 144                             varoffset.append((varname, int(m.groups(0)[0], 16), int(sections[-1][1], 16), sections[-1][0]))
 145
 146     if not varoffset:
 147         return []
 148     # get section information from efi file
 149     efisecs = PeImageClass(efifilepath).SectionHeaderList
 150     if efisecs is None or len(efisecs) == 0:
 151         return []
 152     #redirection
 153     redirection = 0
 154     for efisec in efisecs:
 155         for section in sections:
 156             if section[0].strip() == efisec[0].strip() and section[0].strip() == '.text':
 157                 redirection = int(section[1], 16) - efisec[1]
 158
 159     ret = []
 160     for var in varoffset:
 161         for efisec in efisecs:
 162             if var[1] >= efisec[1] and var[1] < efisec[1]+efisec[3]:
 163                 ret.append((var[0], hex(efisec[2] + var[1] - efisec[1] - redirection)))
 164     return ret
 165
 166 def _parseGeneral(lines, efifilepath, varnames):
 167     status = 0    #0 - beginning of file; 1 - PE section definition; 2 - symbol table
 168     secs  = []    # key = section name
 169     varoffset = []
 170     symRe = re.compile('^([\da-fA-F]+):([\da-fA-F]+) +([\.:\\\\\w\?@\$]+) +([\da-fA-F]+)', re.UNICODE)
 171
 172     for line in lines:
 173         line = line.strip()
 174         if startPatternGeneral.match(line):
 175             status = 1
 176             continue
 177         if addressPatternGeneral.match(line):
 178             status = 2
 179             continue
 180         if line.startswith("entry point at"):
 181             status = 3
 182             continue
 183         if status == 1 and len(line) != 0:
 184             m =  secReGeneral.match(line)
 185             assert m is not None, "Fail to parse the section in map file , line is %s" % line
 186             sec_no, sec_start, sec_length, sec_name, sec_class = m.groups(0)
 187             secs.append([int(sec_no, 16), int(sec_start, 16), int(sec_length, 16), sec_name, sec_class])
 188         if status == 2 and len(line) != 0:
 189             for varname in varnames:
 190                 m = symRe.match(line)
 191                 assert m is not None, "Fail to parse the symbol in map file, line is %s" % line
 192                 sec_no, sym_offset, sym_name, vir_addr = m.groups(0)
 193                 sec_no     = int(sec_no,     16)
 194                 sym_offset = int(sym_offset, 16)
 195                 vir_addr   = int(vir_addr,   16)
 196                 # cannot pregenerate this RegEx since it uses varname from varnames.
 197                 m2 = re.match('^[_]*(%s)' % varname, sym_name)
 198                 if m2 is not None:
 199                     # fond a binary pcd entry in map file
 200                     for sec in secs:
 201                         if sec[0] == sec_no and (sym_offset >= sec[1] and sym_offset < sec[1] + sec[2]):
 202                             varoffset.append([varname, sec[3], sym_offset, vir_addr, sec_no])
 203
 204     if not varoffset: return []
 205
 206     # get section information from efi file
 207     efisecs = PeImageClass(efifilepath).SectionHeaderList
 208     if efisecs is None or len(efisecs) == 0:
 209         return []
 210
 211     ret = []
 212     for var in varoffset:
 213         index = 0
 214         for efisec in efisecs:
 215             index = index + 1
 216             if var[1].strip() == efisec[0].strip():
 217                 ret.append((var[0], hex(efisec[2] + var[2])))
 218             elif var[4] == index:
 219                 ret.append((var[0], hex(efisec[2] + var[2])))
 220
 221     return ret
 222
 223 ## Routine to process duplicated INF
 224 #
 225 #  This function is called by following two cases:
 226 #  Case 1 in DSC:
 227 #    [components.arch]
 228 #    Pkg/module/module.inf
 229 #    Pkg/module/module.inf {
 230 #      <Defines>
 231 #        FILE_GUID = 0D1B936F-68F3-4589-AFCC-FB8B7AEBC836
 232 #    }
 233 #  Case 2 in FDF:
 234 #    INF Pkg/module/module.inf
 235 #    INF FILE_GUID = 0D1B936F-68F3-4589-AFCC-FB8B7AEBC836 Pkg/module/module.inf
 236 #
 237 #  This function copies Pkg/module/module.inf to
 238 #  Conf/.cache/0D1B936F-68F3-4589-AFCC-FB8B7AEBC836module.inf
 239 #
 240 #  @param Path     Original PathClass object
 241 #  @param BaseName New file base name
 242 #
 243 #  @retval         return the new PathClass object
 244 #
 245 def ProcessDuplicatedInf(Path, BaseName, Workspace):
 246     Filename = os.path.split(Path.File)[1]
 247     if '.' in Filename:
 248         Filename = BaseName + Path.BaseName + Filename[Filename.rfind('.'):]
 249     else:
 250         Filename = BaseName + Path.BaseName
 251
 252     #
 253     # If -N is specified on command line, cache is disabled
 254     # The directory has to be created
 255     #
 256     DbDir = os.path.split(GlobalData.gDatabasePath)[0]
 257     if not os.path.exists(DbDir):
 258         os.makedirs(DbDir)
 259     #
 260     # A temporary INF is copied to database path which must have write permission
 261     # The temporary will be removed at the end of build
 262     # In case of name conflict, the file name is
 263     # FILE_GUIDBaseName (0D1B936F-68F3-4589-AFCC-FB8B7AEBC836module.inf)
 264     #
 265     TempFullPath = os.path.join(DbDir,
 266                                 Filename)
 267     RtPath = PathClass(Path.File, Workspace)
 268     #
 269     # Modify the full path to temporary path, keep other unchanged
 270     #
 271     # To build same module more than once, the module path with FILE_GUID overridden has
 272     # the file name FILE_GUIDmodule.inf, but the relative path (self.MetaFile.File) is the real path
 273     # in DSC which is used as relative path by C files and other files in INF.
 274     # A trick was used: all module paths are PathClass instances, after the initialization
 275     # of PathClass, the PathClass.Path is overridden by the temporary INF path.
 276     #
 277     # The reason for creating a temporary INF is:
 278     # Platform.Modules which is the base to create ModuleAutoGen objects is a dictionary,
 279     # the key is the full path of INF, the value is an object to save overridden library instances, PCDs.
 280     # A different key for the same module is needed to create different output directory,
 281     # retrieve overridden PCDs, library instances.
 282     #
 283     # The BaseName is the FILE_GUID which is also the output directory name.
 284     #
 285     #
 286     RtPath.Path = TempFullPath
 287     RtPath.BaseName = BaseName
 288     #
 289     # If file exists, compare contents
 290     #
 291     if os.path.exists(TempFullPath):
 292         with open(str(Path), 'rb') as f1, open(TempFullPath, 'rb') as f2:
 293             if f1.read() == f2.read():
 294                 return RtPath
 295     _TempInfs.append(TempFullPath)
 296     shutil.copy2(str(Path), TempFullPath)
 297     return RtPath
 298
 299 ## Remove temporary created INFs whose paths were saved in _TempInfs
 300 #
 301 def ClearDuplicatedInf():
 302     while _TempInfs:
 303         File = _TempInfs.pop()
 304         if os.path.exists(File):
 305             os.remove(File)
 306
 307 ## Convert GUID string in xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx style to C structure style
 308 #
 309 #   @param      Guid    The GUID string
 310 #
 311 #   @retval     string  The GUID string in C structure style
 312 #
 313 def GuidStringToGuidStructureString(Guid):
 314     GuidList = Guid.split('-')
 315     Result = '{'
 316     for Index in range(0, 3, 1):
 317         Result = Result + '0x' + GuidList[Index] + ', '
 318     Result = Result + '{0x' + GuidList[3][0:2] + ', 0x' + GuidList[3][2:4]
 319     for Index in range(0, 12, 2):
 320         Result = Result + ', 0x' + GuidList[4][Index:Index + 2]
 321     Result += '}}'
 322     return Result
 323
 324 ## Convert GUID structure in byte array to xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
 325 #
 326 #   @param      GuidValue   The GUID value in byte array
 327 #
 328 #   @retval     string      The GUID value in xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx format
 329 #
 330 def GuidStructureByteArrayToGuidString(GuidValue):
 331     guidValueString = GuidValue.lower().replace("{", "").replace("}", "").replace(" ", "").replace(";", "")
 332     guidValueList = guidValueString.split(",")
 333     if len(guidValueList) != 16:
 334         return ''
 335         #EdkLogger.error(None, None, "Invalid GUID value string %s" % GuidValue)
 336     try:
 337         return "%02x%02x%02x%02x-%02x%02x-%02x%02x-%02x%02x-%02x%02x%02x%02x%02x%02x" % (
 338                 int(guidValueList[3], 16),
 339                 int(guidValueList[2], 16),
 340                 int(guidValueList[1], 16),
 341                 int(guidValueList[0], 16),
 342                 int(guidValueList[5], 16),
 343                 int(guidValueList[4], 16),
 344                 int(guidValueList[7], 16),
 345                 int(guidValueList[6], 16),
 346                 int(guidValueList[8], 16),
 347                 int(guidValueList[9], 16),
 348                 int(guidValueList[10], 16),
 349                 int(guidValueList[11], 16),
 350                 int(guidValueList[12], 16),
 351                 int(guidValueList[13], 16),
 352                 int(guidValueList[14], 16),
 353                 int(guidValueList[15], 16)
 354                 )
 355     except:
 356         return ''
 357
 358 ## Convert GUID string in C structure style to xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
 359 #
 360 #   @param      GuidValue   The GUID value in C structure format
 361 #
 362 #   @retval     string      The GUID value in xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx format
 363 #
 364 def GuidStructureStringToGuidString(GuidValue):
 365     if not GlobalData.gGuidCFormatPattern.match(GuidValue):
 366         return ''
 367     guidValueString = GuidValue.lower().replace("{", "").replace("}", "").replace(" ", "").replace(";", "")
 368     guidValueList = guidValueString.split(",")
 369     if len(guidValueList) != 11:
 370         return ''
 371         #EdkLogger.error(None, None, "Invalid GUID value string %s" % GuidValue)
 372     try:
 373         return "%08x-%04x-%04x-%02x%02x-%02x%02x%02x%02x%02x%02x" % (
 374                 int(guidValueList[0], 16),
 375                 int(guidValueList[1], 16),
 376                 int(guidValueList[2], 16),
 377                 int(guidValueList[3], 16),
 378                 int(guidValueList[4], 16),
 379                 int(guidValueList[5], 16),
 380                 int(guidValueList[6], 16),
 381                 int(guidValueList[7], 16),
 382                 int(guidValueList[8], 16),
 383                 int(guidValueList[9], 16),
 384                 int(guidValueList[10], 16)
 385                 )
 386     except:
 387         return ''
 388
 389 ## Convert GUID string in C structure style to xxxxxxxx_xxxx_xxxx_xxxx_xxxxxxxxxxxx
 390 #
 391 #   @param      GuidValue   The GUID value in C structure format
 392 #
 393 #   @retval     string      The GUID value in xxxxxxxx_xxxx_xxxx_xxxx_xxxxxxxxxxxx format
 394 #
 395 def GuidStructureStringToGuidValueName(GuidValue):
 396     guidValueString = GuidValue.lower().replace("{", "").replace("}", "").replace(" ", "")
 397     guidValueList = guidValueString.split(",")
 398     if len(guidValueList) != 11:
 399         EdkLogger.error(None, FORMAT_INVALID, "Invalid GUID value string [%s]" % GuidValue)
 400     return "%08x_%04x_%04x_%02x%02x_%02x%02x%02x%02x%02x%02x" % (
 401             int(guidValueList[0], 16),
 402             int(guidValueList[1], 16),
 403             int(guidValueList[2], 16),
 404             int(guidValueList[3], 16),
 405             int(guidValueList[4], 16),
 406             int(guidValueList[5], 16),
 407             int(guidValueList[6], 16),
 408             int(guidValueList[7], 16),
 409             int(guidValueList[8], 16),
 410             int(guidValueList[9], 16),
 411             int(guidValueList[10], 16)
 412             )
 413
 414 ## Create directories
 415 #
 416 #   @param      Directory   The directory name
 417 #
 418 def CreateDirectory(Directory):
 419     if Directory is None or Directory.strip() == "":
 420         return True
 421     try:
 422         if not os.access(Directory, os.F_OK):
 423             os.makedirs(Directory)
 424     except:
 425         return False
 426     return True
 427
 428 ## Remove directories, including files and sub-directories in it
 429 #
 430 #   @param      Directory   The directory name
 431 #
 432 def RemoveDirectory(Directory, Recursively=False):
 433     if Directory is None or Directory.strip() == "" or not os.path.exists(Directory):
 434         return
 435     if Recursively:
 436         CurrentDirectory = os.getcwd()
 437         os.chdir(Directory)
 438         for File in os.listdir("."):
 439             if os.path.isdir(File):
 440                 RemoveDirectory(File, Recursively)
 441             else:
 442                 os.remove(File)
 443         os.chdir(CurrentDirectory)
 444     os.rmdir(Directory)
 445
 446 ## Store content in file
 447 #
 448 #  This method is used to save file only when its content is changed. This is
 449 #  quite useful for "make" system to decide what will be re-built and what won't.
 450 #
 451 #   @param      File            The path of file
 452 #   @param      Content         The new content of the file
 453 #   @param      IsBinaryFile    The flag indicating if the file is binary file or not
 454 #
 455 #   @retval     True            If the file content is changed and the file is renewed
 456 #   @retval     False           If the file content is the same
 457 #
 458 def SaveFileOnChange(File, Content, IsBinaryFile=True):
 459     if not IsBinaryFile:
 460         Content = Content.replace("\n", os.linesep)
 461
 462     if os.path.exists(File):
 463         try:
 464             if Content == open(File, "rb").read():
 465                 return False
 466         except:
 467             EdkLogger.error(None, FILE_OPEN_FAILURE, ExtraData=File)
 468
 469     DirName = os.path.dirname(File)
 470     if not CreateDirectory(DirName):
 471         EdkLogger.error(None, FILE_CREATE_FAILURE, "Could not create directory %s" % DirName)
 472     else:
 473         if DirName == '':
 474             DirName = os.getcwd()
 475         if not os.access(DirName, os.W_OK):
 476             EdkLogger.error(None, PERMISSION_FAILURE, "Do not have write permission on directory %s" % DirName)
 477
 478     try:
 479         Fd = open(File, "wb")
 480         Fd.write(Content)
 481         Fd.close()
 482     except IOError as X:
 483         EdkLogger.error(None, FILE_CREATE_FAILURE, ExtraData='IOError %s' % X)
 484
 485     return True
 486
 487 ## Retrieve and cache the real path name in file system
 488 #
 489 #   @param      Root    The root directory of path relative to
 490 #
 491 #   @retval     str     The path string if the path exists
 492 #   @retval     None    If path doesn't exist
 493 #
 494 class DirCache:
 495     _CACHE_ = set()
 496     _UPPER_CACHE_ = {}
 497
 498     def __init__(self, Root):
 499         self._Root = Root
 500         for F in os.listdir(Root):
 501             self._CACHE_.add(F)
 502             self._UPPER_CACHE_[F.upper()] = F
 503
 504     # =[] operator
 505     def __getitem__(self, Path):
 506         Path = Path[len(os.path.commonprefix([Path, self._Root])):]
 507         if not Path:
 508             return self._Root
 509         if Path and Path[0] == os.path.sep:
 510             Path = Path[1:]
 511         if Path in self._CACHE_:
 512             return os.path.join(self._Root, Path)
 513         UpperPath = Path.upper()
 514         if UpperPath in self._UPPER_CACHE_:
 515             return os.path.join(self._Root, self._UPPER_CACHE_[UpperPath])
 516
 517         IndexList = []
 518         LastSepIndex = -1
 519         SepIndex = Path.find(os.path.sep)
 520         while SepIndex > -1:
 521             Parent = UpperPath[:SepIndex]
 522             if Parent not in self._UPPER_CACHE_:
 523                 break
 524             LastSepIndex = SepIndex
 525             SepIndex = Path.find(os.path.sep, LastSepIndex + 1)
 526
 527         if LastSepIndex == -1:
 528             return None
 529
 530         Cwd = os.getcwd()
 531         os.chdir(self._Root)
 532         SepIndex = LastSepIndex
 533         while SepIndex > -1:
 534             Parent = Path[:SepIndex]
 535             ParentKey = UpperPath[:SepIndex]
 536             if ParentKey not in self._UPPER_CACHE_:
 537                 os.chdir(Cwd)
 538                 return None
 539
 540             if Parent in self._CACHE_:
 541                 ParentDir = Parent
 542             else:
 543                 ParentDir = self._UPPER_CACHE_[ParentKey]
 544             for F in os.listdir(ParentDir):
 545                 Dir = os.path.join(ParentDir, F)
 546                 self._CACHE_.add(Dir)
 547                 self._UPPER_CACHE_[Dir.upper()] = Dir
 548
 549             SepIndex = Path.find(os.path.sep, SepIndex + 1)
 550
 551         os.chdir(Cwd)
 552         if Path in self._CACHE_:
 553             return os.path.join(self._Root, Path)
 554         elif UpperPath in self._UPPER_CACHE_:
 555             return os.path.join(self._Root, self._UPPER_CACHE_[UpperPath])
 556         return None
 557
 558 def RealPath(File, Dir='', OverrideDir=''):
 559     NewFile = os.path.normpath(os.path.join(Dir, File))
 560     NewFile = GlobalData.gAllFiles[NewFile]
 561     if not NewFile and OverrideDir:
 562         NewFile = os.path.normpath(os.path.join(OverrideDir, File))
 563         NewFile = GlobalData.gAllFiles[NewFile]
 564     return NewFile
 565
 566 ## Get GUID value from given packages
 567 #
 568 #   @param      CName           The CName of the GUID
 569 #   @param      PackageList     List of packages looking-up in
 570 #   @param      Inffile         The driver file
 571 #
 572 #   @retval     GuidValue   if the CName is found in any given package
 573 #   @retval     None        if the CName is not found in all given packages
 574 #
 575 def GuidValue(CName, PackageList, Inffile = None):
 576     for P in PackageList:
 577         GuidKeys = P.Guids.keys()
 578         if Inffile and P._PrivateGuids:
 579             if not Inffile.startswith(P.MetaFile.Dir):
 580                 GuidKeys = [x for x in P.Guids if x not in P._PrivateGuids]
 581         if CName in GuidKeys:
 582             return P.Guids[CName]
 583     return None
 584
 585 ## A string template class
 586 #
 587 #  This class implements a template for string replacement. A string template
 588 #  looks like following
 589 #
 590 #       ${BEGIN} other_string ${placeholder_name} other_string ${END}
 591 #
 592 #  The string between ${BEGIN} and ${END} will be repeated as many times as the
 593 #  length of "placeholder_name", which is a list passed through a dict. The
 594 #  "placeholder_name" is the key name of the dict. The ${BEGIN} and ${END} can
 595 #  be not used and, in this case, the "placeholder_name" must not a list and it
 596 #  will just be replaced once.
 597 #
 598 class TemplateString(object):
 599     _REPEAT_START_FLAG = "BEGIN"
 600     _REPEAT_END_FLAG = "END"
 601
 602     class Section(object):
 603         _LIST_TYPES = [type([]), type(set()), type((0,))]
 604
 605         def __init__(self, TemplateSection, PlaceHolderList):
 606             self._Template = TemplateSection
 607             self._PlaceHolderList = []
 608
 609             # Split the section into sub-sections according to the position of placeholders
 610             if PlaceHolderList:
 611                 self._SubSectionList = []
 612                 SubSectionStart = 0
 613                 #
 614                 # The placeholders passed in must be in the format of
 615                 #
 616                 #   PlaceHolderName, PlaceHolderStartPoint, PlaceHolderEndPoint
 617                 #
 618                 for PlaceHolder, Start, End in PlaceHolderList:
 619                     self._SubSectionList.append(TemplateSection[SubSectionStart:Start])
 620                     self._SubSectionList.append(TemplateSection[Start:End])
 621                     self._PlaceHolderList.append(PlaceHolder)
 622                     SubSectionStart = End
 623                 if SubSectionStart < len(TemplateSection):
 624                     self._SubSectionList.append(TemplateSection[SubSectionStart:])
 625             else:
 626                 self._SubSectionList = [TemplateSection]
 627
 628         def __str__(self):
 629             return self._Template + " : " + str(self._PlaceHolderList)
 630
 631         def Instantiate(self, PlaceHolderValues):
 632             RepeatTime = -1
 633             RepeatPlaceHolders = {}
 634             NonRepeatPlaceHolders = {}
 635
 636             for PlaceHolder in self._PlaceHolderList:
 637                 if PlaceHolder not in PlaceHolderValues:
 638                     continue
 639                 Value = PlaceHolderValues[PlaceHolder]
 640                 if type(Value) in self._LIST_TYPES:
 641                     if RepeatTime < 0:
 642                         RepeatTime = len(Value)
 643                     elif RepeatTime != len(Value):
 644                         EdkLogger.error(
 645                                     "TemplateString",
 646                                     PARAMETER_INVALID,
 647                                     "${%s} has different repeat time from others!" % PlaceHolder,
 648                                     ExtraData=str(self._Template)
 649                                     )
 650                     RepeatPlaceHolders["${%s}" % PlaceHolder] = Value
 651                 else:
 652                     NonRepeatPlaceHolders["${%s}" % PlaceHolder] = Value
 653
 654             if NonRepeatPlaceHolders:
 655                 StringList = []
 656                 for S in self._SubSectionList:
 657                     if S not in NonRepeatPlaceHolders:
 658                         StringList.append(S)
 659                     else:
 660                         StringList.append(str(NonRepeatPlaceHolders[S]))
 661             else:
 662                 StringList = self._SubSectionList
 663
 664             if RepeatPlaceHolders:
 665                 TempStringList = []
 666                 for Index in range(RepeatTime):
 667                     for S in StringList:
 668                         if S not in RepeatPlaceHolders:
 669                             TempStringList.append(S)
 670                         else:
 671                             TempStringList.append(str(RepeatPlaceHolders[S][Index]))
 672                 StringList = TempStringList
 673
 674             return "".join(StringList)
 675
 676     ## Constructor
 677     def __init__(self, Template=None):
 678         self.String = []
 679         self.IsBinary = False
 680         self._Template = Template
 681         self._TemplateSectionList = self._Parse(Template)
 682
 683     ## str() operator
 684     #
 685     #   @retval     string  The string replaced
 686     #
 687     def __str__(self):
 688         return "".join(self.String)
 689
 690     ## Split the template string into fragments per the ${BEGIN} and ${END} flags
 691     #
 692     #   @retval     list    A list of TemplateString.Section objects
 693     #
 694     def _Parse(self, Template):
 695         SectionStart = 0
 696         SearchFrom = 0
 697         MatchEnd = 0
 698         PlaceHolderList = []
 699         TemplateSectionList = []
 700         while Template:
 701             MatchObj = gPlaceholderPattern.search(Template, SearchFrom)
 702             if not MatchObj:
 703                 if MatchEnd <= len(Template):
 704                     TemplateSection = TemplateString.Section(Template[SectionStart:], PlaceHolderList)
 705                     TemplateSectionList.append(TemplateSection)
 706                 break
 707
 708             MatchString = MatchObj.group(1)
 709             MatchStart = MatchObj.start()
 710             MatchEnd = MatchObj.end()
 711
 712             if MatchString == self._REPEAT_START_FLAG:
 713                 if MatchStart > SectionStart:
 714                     TemplateSection = TemplateString.Section(Template[SectionStart:MatchStart], PlaceHolderList)
 715                     TemplateSectionList.append(TemplateSection)
 716                 SectionStart = MatchEnd
 717                 PlaceHolderList = []
 718             elif MatchString == self._REPEAT_END_FLAG:
 719                 TemplateSection = TemplateString.Section(Template[SectionStart:MatchStart], PlaceHolderList)
 720                 TemplateSectionList.append(TemplateSection)
 721                 SectionStart = MatchEnd
 722                 PlaceHolderList = []
 723             else:
 724                 PlaceHolderList.append((MatchString, MatchStart - SectionStart, MatchEnd - SectionStart))
 725             SearchFrom = MatchEnd
 726         return TemplateSectionList
 727
 728     ## Replace the string template with dictionary of placeholders and append it to previous one
 729     #
 730     #   @param      AppendString    The string template to append
 731     #   @param      Dictionary      The placeholder dictionaries
 732     #
 733     def Append(self, AppendString, Dictionary=None):
 734         if Dictionary:
 735             SectionList = self._Parse(AppendString)
 736             self.String.append( "".join(S.Instantiate(Dictionary) for S in SectionList))
 737         else:
 738             if isinstance(AppendString,list):
 739                 self.String.extend(AppendString)
 740             else:
 741                 self.String.append(AppendString)
 742
 743     ## Replace the string template with dictionary of placeholders
 744     #
 745     #   @param      Dictionary      The placeholder dictionaries
 746     #
 747     #   @retval     str             The string replaced with placeholder values
 748     #
 749     def Replace(self, Dictionary=None):
 750         return "".join(S.Instantiate(Dictionary) for S in self._TemplateSectionList)
 751
 752 ## Progress indicator class
 753 #
 754 #  This class makes use of thread to print progress on console.
 755 #
 756 class Progressor:
 757     # for avoiding deadloop
 758     _StopFlag = None
 759     _ProgressThread = None
 760     _CheckInterval = 0.25
 761
 762     ## Constructor
 763     #
 764     #   @param      OpenMessage     The string printed before progress charaters
 765     #   @param      CloseMessage    The string printed after progress charaters
 766     #   @param      ProgressChar    The charater used to indicate the progress
 767     #   @param      Interval        The interval in seconds between two progress charaters
 768     #
 769     def __init__(self, OpenMessage="", CloseMessage="", ProgressChar='.', Interval=1.0):
 770         self.PromptMessage = OpenMessage
 771         self.CodaMessage = CloseMessage
 772         self.ProgressChar = ProgressChar
 773         self.Interval = Interval
 774         if Progressor._StopFlag is None:
 775             Progressor._StopFlag = threading.Event()
 776
 777     ## Start to print progress charater
 778     #
 779     #   @param      OpenMessage     The string printed before progress charaters
 780     #
 781     def Start(self, OpenMessage=None):
 782         if OpenMessage is not None:
 783             self.PromptMessage = OpenMessage
 784         Progressor._StopFlag.clear()
 785         if Progressor._ProgressThread is None:
 786             Progressor._ProgressThread = threading.Thread(target=self._ProgressThreadEntry)
 787             Progressor._ProgressThread.setDaemon(False)
 788             Progressor._ProgressThread.start()
 789
 790     ## Stop printing progress charater
 791     #
 792     #   @param      CloseMessage    The string printed after progress charaters
 793     #
 794     def Stop(self, CloseMessage=None):
 795         OriginalCodaMessage = self.CodaMessage
 796         if CloseMessage is not None:
 797             self.CodaMessage = CloseMessage
 798         self.Abort()
 799         self.CodaMessage = OriginalCodaMessage
 800
 801     ## Thread entry method
 802     def _ProgressThreadEntry(self):
 803         sys.stdout.write(self.PromptMessage + " ")
 804         sys.stdout.flush()
 805         TimeUp = 0.0
 806         while not Progressor._StopFlag.isSet():
 807             if TimeUp <= 0.0:
 808                 sys.stdout.write(self.ProgressChar)
 809                 sys.stdout.flush()
 810                 TimeUp = self.Interval
 811             time.sleep(self._CheckInterval)
 812             TimeUp -= self._CheckInterval
 813         sys.stdout.write(" " + self.CodaMessage + "\n")
 814         sys.stdout.flush()
 815
 816     ## Abort the progress display
 817     @staticmethod
 818     def Abort():
 819         if Progressor._StopFlag is not None:
 820             Progressor._StopFlag.set()
 821         if Progressor._ProgressThread is not None:
 822             Progressor._ProgressThread.join()
 823             Progressor._ProgressThread = None
 824
 825 ## A dict which can access its keys and/or values orderly
 826 #
 827 #  The class implements a new kind of dict which its keys or values can be
 828 #  accessed in the order they are added into the dict. It guarantees the order
 829 #  by making use of an internal list to keep a copy of keys.
 830 #
 831 class sdict(dict):
 832     ## Constructor
 833     def __init__(self):
 834         dict.__init__(self)
 835         self._key_list = []
 836
 837     ## [] operator
 838     def __setitem__(self, key, value):
 839         if key not in self._key_list:
 840             self._key_list.append(key)
 841         dict.__setitem__(self, key, value)
 842
 843     ## del operator
 844     def __delitem__(self, key):
 845         self._key_list.remove(key)
 846         dict.__delitem__(self, key)
 847
 848     ## used in "for k in dict" loop to ensure the correct order
 849     def __iter__(self):
 850         return self.iterkeys()
 851
 852     ## len() support
 853     def __len__(self):
 854         return len(self._key_list)
 855
 856     ## "in" test support
 857     def __contains__(self, key):
 858         return key in self._key_list
 859
 860     ## indexof support
 861     def index(self, key):
 862         return self._key_list.index(key)
 863
 864     ## insert support
 865     def insert(self, key, newkey, newvalue, order):
 866         index = self._key_list.index(key)
 867         if order == 'BEFORE':
 868             self._key_list.insert(index, newkey)
 869             dict.__setitem__(self, newkey, newvalue)
 870         elif order == 'AFTER':
 871             self._key_list.insert(index + 1, newkey)
 872             dict.__setitem__(self, newkey, newvalue)
 873
 874     ## append support
 875     def append(self, sdict):
 876         for key in sdict:
 877             if key not in self._key_list:
 878                 self._key_list.append(key)
 879             dict.__setitem__(self, key, sdict[key])
 880
 881     def has_key(self, key):
 882         return key in self._key_list
 883
 884     ## Empty the dict
 885     def clear(self):
 886         self._key_list = []
 887         dict.clear(self)
 888
 889     ## Return a copy of keys
 890     def keys(self):
 891         keys = []
 892         for key in self._key_list:
 893             keys.append(key)
 894         return keys
 895
 896     ## Return a copy of values
 897     def values(self):
 898         values = []
 899         for key in self._key_list:
 900             values.append(self[key])
 901         return values
 902
 903     ## Return a copy of (key, value) list
 904     def items(self):
 905         items = []
 906         for key in self._key_list:
 907             items.append((key, self[key]))
 908         return items
 909
 910     ## Iteration support
 911     def iteritems(self):
 912         return iter(self.items())
 913
 914     ## Keys interation support
 915     def iterkeys(self):
 916         return iter(self.keys())
 917
 918     ## Values interation support
 919     def itervalues(self):
 920         return iter(self.values())
 921
 922     ## Return value related to a key, and remove the (key, value) from the dict
 923     def pop(self, key, *dv):
 924         value = None
 925         if key in self._key_list:
 926             value = self[key]
 927             self.__delitem__(key)
 928         elif len(dv) != 0 :
 929             value = kv[0]
 930         return value
 931
 932     ## Return (key, value) pair, and remove the (key, value) from the dict
 933     def popitem(self):
 934         key = self._key_list[-1]
 935         value = self[key]
 936         self.__delitem__(key)
 937         return key, value
 938
 939     def update(self, dict=None, **kwargs):
 940         if dict is not None:
 941             for k, v in dict.items():
 942                 self[k] = v
 943         if len(kwargs):
 944             for k, v in kwargs.items():
 945                 self[k] = v
 946
 947 ## Dictionary using prioritized list as key
 948 #
 949 class tdict:
 950     _ListType = type([])
 951     _TupleType = type(())
 952     _Wildcard = 'COMMON'
 953     _ValidWildcardList = ['COMMON', 'DEFAULT', 'ALL', TAB_STAR, 'PLATFORM']
 954
 955     def __init__(self, _Single_=False, _Level_=2):
 956         self._Level_ = _Level_
 957         self.data = {}
 958         self._Single_ = _Single_
 959
 960     # =[] operator
 961     def __getitem__(self, key):
 962         KeyType = type(key)
 963         RestKeys = None
 964         if KeyType == self._ListType or KeyType == self._TupleType:
 965             FirstKey = key[0]
 966             if len(key) > 1:
 967                 RestKeys = key[1:]
 968             elif self._Level_ > 1:
 969                 RestKeys = [self._Wildcard for i in range(0, self._Level_ - 1)]
 970         else:
 971             FirstKey = key
 972             if self._Level_ > 1:
 973                 RestKeys = [self._Wildcard for i in range(0, self._Level_ - 1)]
 974
 975         if FirstKey is None or str(FirstKey).upper() in self._ValidWildcardList:
 976             FirstKey = self._Wildcard
 977
 978         if self._Single_:
 979             return self._GetSingleValue(FirstKey, RestKeys)
 980         else:
 981             return self._GetAllValues(FirstKey, RestKeys)
 982
 983     def _GetSingleValue(self, FirstKey, RestKeys):
 984         Value = None
 985         #print "%s-%s" % (FirstKey, self._Level_) ,
 986         if self._Level_ > 1:
 987             if FirstKey == self._Wildcard:
 988                 if FirstKey in self.data:
 989                     Value = self.data[FirstKey][RestKeys]
 990                 if Value is None:
 991                     for Key in self.data:
 992                         Value = self.data[Key][RestKeys]
 993                         if Value is not None: break
 994             else:
 995                 if FirstKey in self.data:
 996                     Value = self.data[FirstKey][RestKeys]
 997                 if Value is None and self._Wildcard in self.data:
 998                     #print "Value=None"
 999                     Value = self.data[self._Wildcard][RestKeys]
1000         else:
1001             if FirstKey == self._Wildcard:
1002                 if FirstKey in self.data:
1003                     Value = self.data[FirstKey]
1004                 if Value is None:
1005                     for Key in self.data:
1006                         Value = self.data[Key]
1007                         if Value is not None: break
1008             else:
1009                 if FirstKey in self.data:
1010                     Value = self.data[FirstKey]
1011                 elif self._Wildcard in self.data:
1012                     Value = self.data[self._Wildcard]
1013         return Value
1014
1015     def _GetAllValues(self, FirstKey, RestKeys):
1016         Value = []
1017         if self._Level_ > 1:
1018             if FirstKey == self._Wildcard:
1019                 for Key in self.data:
1020                     Value += self.data[Key][RestKeys]
1021             else:
1022                 if FirstKey in self.data:
1023                     Value += self.data[FirstKey][RestKeys]
1024                 if self._Wildcard in self.data:
1025                     Value += self.data[self._Wildcard][RestKeys]
1026         else:
1027             if FirstKey == self._Wildcard:
1028                 for Key in self.data:
1029                     Value.append(self.data[Key])
1030             else:
1031                 if FirstKey in self.data:
1032                     Value.append(self.data[FirstKey])
1033                 if self._Wildcard in self.data:
1034                     Value.append(self.data[self._Wildcard])
1035         return Value
1036
1037     ## []= operator
1038     def __setitem__(self, key, value):
1039         KeyType = type(key)
1040         RestKeys = None
1041         if KeyType == self._ListType or KeyType == self._TupleType:
1042             FirstKey = key[0]
1043             if len(key) > 1:
1044                 RestKeys = key[1:]
1045             else:
1046                 RestKeys = [self._Wildcard for i in range(0, self._Level_ - 1)]
1047         else:
1048             FirstKey = key
1049             if self._Level_ > 1:
1050                 RestKeys = [self._Wildcard for i in range(0, self._Level_ - 1)]
1051
1052         if FirstKey in self._ValidWildcardList:
1053             FirstKey = self._Wildcard
1054
1055         if FirstKey not in self.data and self._Level_ > 0:
1056             self.data[FirstKey] = tdict(self._Single_, self._Level_ - 1)
1057
1058         if self._Level_ > 1:
1059             self.data[FirstKey][RestKeys] = value
1060         else:
1061             self.data[FirstKey] = value
1062
1063     def SetGreedyMode(self):
1064         self._Single_ = False
1065         if self._Level_ > 1:
1066             for Key in self.data:
1067                 self.data[Key].SetGreedyMode()
1068
1069     def SetSingleMode(self):
1070         self._Single_ = True
1071         if self._Level_ > 1:
1072             for Key in self.data:
1073                 self.data[Key].SetSingleMode()
1074
1075     def GetKeys(self, KeyIndex=0):
1076         assert KeyIndex >= 0
1077         if KeyIndex == 0:
1078             return set(self.data.keys())
1079         else:
1080             keys = set()
1081             for Key in self.data:
1082                 keys |= self.data[Key].GetKeys(KeyIndex - 1)
1083             return keys
1084
1085 def AnalyzePcdExpression(Setting):
1086     RanStr = ''.join(sample(string.ascii_letters + string.digits, 8))
1087     Setting = Setting.replace('\\\\', RanStr).strip()
1088     # There might be escaped quote in a string: \", \\\" , \', \\\'
1089     Data = Setting
1090     # There might be '|' in string and in ( ... | ... ), replace it with '-'
1091     NewStr = ''
1092     InSingleQuoteStr = False
1093     InDoubleQuoteStr = False
1094     Pair = 0
1095     for Index, ch in enumerate(Data):
1096         if ch == '"' and not InSingleQuoteStr:
1097             if Data[Index - 1] != '\\':
1098                 InDoubleQuoteStr = not InDoubleQuoteStr
1099         elif ch == "'" and not InDoubleQuoteStr:
1100             if Data[Index - 1] != '\\':
1101                 InSingleQuoteStr = not InSingleQuoteStr
1102         elif ch == '(' and not (InSingleQuoteStr or InDoubleQuoteStr):
1103             Pair += 1
1104         elif ch == ')' and not (InSingleQuoteStr or InDoubleQuoteStr):
1105             Pair -= 1
1106
1107         if (Pair > 0 or InSingleQuoteStr or InDoubleQuoteStr) and ch == TAB_VALUE_SPLIT:
1108             NewStr += '-'
1109         else:
1110             NewStr += ch
1111     FieldList = []
1112     StartPos = 0
1113     while True:
1114         Pos = NewStr.find(TAB_VALUE_SPLIT, StartPos)
1115         if Pos < 0:
1116             FieldList.append(Setting[StartPos:].strip())
1117             break
1118         FieldList.append(Setting[StartPos:Pos].strip())
1119         StartPos = Pos + 1
1120     for i, ch in enumerate(FieldList):
1121         if RanStr in ch:
1122             FieldList[i] = ch.replace(RanStr,'\\\\')
1123     return FieldList
1124
1125 def ParseFieldValue (Value):
1126     def ParseDevPathValue (Value):
1127         if '\\' in Value:
1128             Value.replace('\\', '/').replace(' ', '')
1129
1130         Cmd = 'DevicePath ' + '"' + Value + '"'
1131         try:
1132             p = subprocess.Popen(Cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
1133             out, err = p.communicate()
1134         except Exception as X:
1135             raise BadExpression("DevicePath: %s" % (str(X)) )
1136         finally:
1137             subprocess._cleanup()
1138             p.stdout.close()
1139             p.stderr.close()
1140         if err:
1141             raise BadExpression("DevicePath: %s" % str(err))
1142         Size = len(out.split())
1143         out = ','.join(out.split())
1144         return '{' + out + '}', Size
1145
1146     if "{CODE(" in Value:
1147         return Value, len(Value.split(","))
1148     if isinstance(Value, type(0)):
1149         return Value, (Value.bit_length() + 7) / 8
1150     if not isinstance(Value, type('')):
1151         raise BadExpression('Type %s is %s' %(Value, type(Value)))
1152     Value = Value.strip()
1153     if Value.startswith(TAB_UINT8) and Value.endswith(')'):
1154         Value, Size = ParseFieldValue(Value.split('(', 1)[1][:-1])
1155         if Size > 1:
1156             raise BadExpression('Value (%s) Size larger than %d' %(Value, Size))
1157         return Value, 1
1158     if Value.startswith(TAB_UINT16) and Value.endswith(')'):
1159         Value, Size = ParseFieldValue(Value.split('(', 1)[1][:-1])
1160         if Size > 2:
1161             raise BadExpression('Value (%s) Size larger than %d' %(Value, Size))
1162         return Value, 2
1163     if Value.startswith(TAB_UINT32) and Value.endswith(')'):
1164         Value, Size = ParseFieldValue(Value.split('(', 1)[1][:-1])
1165         if Size > 4:
1166             raise BadExpression('Value (%s) Size larger than %d' %(Value, Size))
1167         return Value, 4
1168     if Value.startswith(TAB_UINT64) and Value.endswith(')'):
1169         Value, Size = ParseFieldValue(Value.split('(', 1)[1][:-1])
1170         if Size > 8:
1171             raise BadExpression('Value (%s) Size larger than %d' % (Value, Size))
1172         return Value, 8
1173     if Value.startswith(TAB_GUID) and Value.endswith(')'):
1174         Value = Value.split('(', 1)[1][:-1].strip()
1175         if Value[0] == '{' and Value[-1] == '}':
1176             TmpValue = GuidStructureStringToGuidString(Value)
1177             if not TmpValue:
1178                 raise BadExpression("Invalid GUID value string %s" % Value)
1179             Value = TmpValue
1180         if Value[0] == '"' and Value[-1] == '"':
1181             Value = Value[1:-1]
1182         try:
1183             Value = "'" + uuid.UUID(Value).get_bytes_le() + "'"
1184         except ValueError as Message:
1185             raise BadExpression(Message)
1186         Value, Size = ParseFieldValue(Value)
1187         return Value, 16
1188     if Value.startswith('L"') and Value.endswith('"'):
1189         # Unicode String
1190         # translate escape character
1191         Value = Value[1:]
1192         try:
1193             Value = eval(Value)
1194         except:
1195             Value = Value[1:-1]
1196         List = list(Value)
1197         List.reverse()
1198         Value = 0
1199         for Char in List:
1200             Value = (Value << 16) | ord(Char)
1201         return Value, (len(List) + 1) * 2
1202     if Value.startswith('"') and Value.endswith('"'):
1203         # ASCII String
1204         # translate escape character
1205         try:
1206             Value = eval(Value)
1207         except:
1208             Value = Value[1:-1]
1209         List = list(Value)
1210         List.reverse()
1211         Value = 0
1212         for Char in List:
1213             Value = (Value << 8) | ord(Char)
1214         return Value, len(List) + 1
1215     if Value.startswith("L'") and Value.endswith("'"):
1216         # Unicode Character Constant
1217         # translate escape character
1218         Value = Value[1:]
1219         try:
1220             Value = eval(Value)
1221         except:
1222             Value = Value[1:-1]
1223         List = list(Value)
1224         if len(List) == 0:
1225             raise BadExpression('Length %s is %s' % (Value, len(List)))
1226         List.reverse()
1227         Value = 0
1228         for Char in List:
1229             Value = (Value << 16) | ord(Char)
1230         return Value, len(List) * 2
1231     if Value.startswith("'") and Value.endswith("'"):
1232         # Character constant
1233         # translate escape character
1234         try:
1235             Value = eval(Value)
1236         except:
1237             Value = Value[1:-1]
1238         List = list(Value)
1239         if len(List) == 0:
1240             raise BadExpression('Length %s is %s' % (Value, len(List)))
1241         List.reverse()
1242         Value = 0
1243         for Char in List:
1244             Value = (Value << 8) | ord(Char)
1245         return Value, len(List)
1246     if Value.startswith('{') and Value.endswith('}'):
1247         # Byte array
1248         Value = Value[1:-1]
1249         List = [Item.strip() for Item in Value.split(',')]
1250         List.reverse()
1251         Value = 0
1252         RetSize = 0
1253         for Item in List:
1254             ItemValue, Size = ParseFieldValue(Item)
1255             RetSize += Size
1256             for I in range(Size):
1257                 Value = (Value << 8) | ((ItemValue >> 8 * I) & 0xff)
1258         return Value, RetSize
1259     if Value.startswith('DEVICE_PATH(') and Value.endswith(')'):
1260         Value = Value.replace("DEVICE_PATH(", '').rstrip(')')
1261         Value = Value.strip().strip('"')
1262         return ParseDevPathValue(Value)
1263     if Value.lower().startswith('0x'):
1264         try:
1265             Value = int(Value, 16)
1266         except:
1267             raise BadExpression("invalid hex value: %s" % Value)
1268         if Value == 0:
1269             return 0, 1
1270         return Value, (Value.bit_length() + 7) / 8
1271     if Value[0].isdigit():
1272         Value = int(Value, 10)
1273         if Value == 0:
1274             return 0, 1
1275         return Value, (Value.bit_length() + 7) / 8
1276     if Value.lower() == 'true':
1277         return 1, 1
1278     if Value.lower() == 'false':
1279         return 0, 1
1280     return Value, 1
1281
1282 ## AnalyzeDscPcd
1283 #
1284 #  Analyze DSC PCD value, since there is no data type info in DSC
1285 #  This function is used to match functions (AnalyzePcdData) used for retrieving PCD value from database
1286 #  1. Feature flag: TokenSpace.PcdCName|PcdValue
1287 #  2. Fix and Patch:TokenSpace.PcdCName|PcdValue[|VOID*[|MaxSize]]
1288 #  3. Dynamic default:
1289 #     TokenSpace.PcdCName|PcdValue[|VOID*[|MaxSize]]
1290 #     TokenSpace.PcdCName|PcdValue
1291 #  4. Dynamic VPD:
1292 #     TokenSpace.PcdCName|VpdOffset[|VpdValue]
1293 #     TokenSpace.PcdCName|VpdOffset[|MaxSize[|VpdValue]]
1294 #  5. Dynamic HII:
1295 #     TokenSpace.PcdCName|HiiString|VariableGuid|VariableOffset[|HiiValue]
1296 #  PCD value needs to be located in such kind of string, and the PCD value might be an expression in which
1297 #    there might have "|" operator, also in string value.
1298 #
1299 #  @param Setting: String contain information described above with "TokenSpace.PcdCName|" stripped
1300 #  @param PcdType: PCD type: feature, fixed, dynamic default VPD HII
1301 #  @param DataType: The datum type of PCD: VOID*, UNIT, BOOL
1302 #  @retval:
1303 #    ValueList: A List contain fields described above
1304 #    IsValid:   True if conforming EBNF, otherwise False
1305 #    Index:     The index where PcdValue is in ValueList
1306 #
1307 def AnalyzeDscPcd(Setting, PcdType, DataType=''):
1308     FieldList = AnalyzePcdExpression(Setting)
1309
1310     IsValid = True
1311     if PcdType in (MODEL_PCD_FIXED_AT_BUILD, MODEL_PCD_PATCHABLE_IN_MODULE, MODEL_PCD_DYNAMIC_DEFAULT, MODEL_PCD_DYNAMIC_EX_DEFAULT):
1312         Value = FieldList[0]
1313         Size = ''
1314         if len(FieldList) > 1 and FieldList[1]:
1315             DataType = FieldList[1]
1316             if FieldList[1] != TAB_VOID and StructPattern.match(FieldList[1]) is None:
1317                 IsValid = False
1318         if len(FieldList) > 2:
1319             Size = FieldList[2]
1320         if IsValid:
1321             if DataType == "":
1322                 IsValid = (len(FieldList) <= 1)
1323             else:
1324                 IsValid = (len(FieldList) <= 3)
1325
1326         if Size:
1327             try:
1328                 int(Size, 16) if Size.upper().startswith("0X") else int(Size)
1329             except:
1330                 IsValid = False
1331                 Size = -1
1332         return [str(Value), DataType, str(Size)], IsValid, 0
1333     elif PcdType == MODEL_PCD_FEATURE_FLAG:
1334         Value = FieldList[0]
1335         Size = ''
1336         IsValid = (len(FieldList) <= 1)
1337         return [Value, DataType, str(Size)], IsValid, 0
1338     elif PcdType in (MODEL_PCD_DYNAMIC_VPD, MODEL_PCD_DYNAMIC_EX_VPD):
1339         VpdOffset = FieldList[0]
1340         Value = Size = ''
1341         if not DataType == TAB_VOID:
1342             if len(FieldList) > 1:
1343                 Value = FieldList[1]
1344         else:
1345             if len(FieldList) > 1:
1346                 Size = FieldList[1]
1347             if len(FieldList) > 2:
1348                 Value = FieldList[2]
1349         if DataType == "":
1350             IsValid = (len(FieldList) <= 1)
1351         else:
1352             IsValid = (len(FieldList) <= 3)
1353         if Size:
1354             try:
1355                 int(Size, 16) if Size.upper().startswith("0X") else int(Size)
1356             except:
1357                 IsValid = False
1358                 Size = -1
1359         return [VpdOffset, str(Size), Value], IsValid, 2
1360     elif PcdType in (MODEL_PCD_DYNAMIC_HII, MODEL_PCD_DYNAMIC_EX_HII):
1361         IsValid = (3 <= len(FieldList) <= 5)
1362         HiiString = FieldList[0]
1363         Guid = Offset = Value = Attribute = ''
1364         if len(FieldList) > 1:
1365             Guid = FieldList[1]
1366         if len(FieldList) > 2:
1367             Offset = FieldList[2]
1368         if len(FieldList) > 3:
1369             Value = FieldList[3]
1370         if len(FieldList) > 4:
1371             Attribute = FieldList[4]
1372         return [HiiString, Guid, Offset, Value, Attribute], IsValid, 3
1373     return [], False, 0
1374
1375 ## AnalyzePcdData
1376 #
1377 #  Analyze the pcd Value, Datum type and TokenNumber.
1378 #  Used to avoid split issue while the value string contain "|" character
1379 #
1380 #  @param[in] Setting:  A String contain value/datum type/token number information;
1381 #
1382 #  @retval   ValueList: A List contain value, datum type and toke number.
1383 #
1384 def AnalyzePcdData(Setting):
1385     ValueList = ['', '', '']
1386
1387     ValueRe = re.compile(r'^\s*L?\".*\|.*\"')
1388     PtrValue = ValueRe.findall(Setting)
1389
1390     ValueUpdateFlag = False
1391
1392     if len(PtrValue) >= 1:
1393         Setting = re.sub(ValueRe, '', Setting)
1394         ValueUpdateFlag = True
1395
1396     TokenList = Setting.split(TAB_VALUE_SPLIT)
1397     ValueList[0:len(TokenList)] = TokenList
1398
1399     if ValueUpdateFlag:
1400         ValueList[0] = PtrValue[0]
1401
1402     return ValueList
1403
1404 ## check format of PCD value against its the datum type
1405 #
1406 # For PCD value setting
1407 #
1408 def CheckPcdDatum(Type, Value):
1409     if Type == TAB_VOID:
1410         ValueRe = re.compile(r'\s*L?\".*\"\s*$')
1411         if not (((Value.startswith('L"') or Value.startswith('"')) and Value.endswith('"'))
1412                 or (Value.startswith('{') and Value.endswith('}')) or (Value.startswith("L'") or Value.startswith("'") and Value.endswith("'"))
1413                ):
1414             return False, "Invalid value [%s] of type [%s]; must be in the form of {...} for array"\
1415                           ", \"...\" or \'...\' for string, L\"...\" or L\'...\' for unicode string" % (Value, Type)
1416         elif ValueRe.match(Value):
1417             # Check the chars in UnicodeString or CString is printable
1418             if Value.startswith("L"):
1419                 Value = Value[2:-1]
1420             else:
1421                 Value = Value[1:-1]
1422             Printset = set(string.printable)
1423             Printset.remove(TAB_PRINTCHAR_VT)
1424             Printset.add(TAB_PRINTCHAR_BS)
1425             Printset.add(TAB_PRINTCHAR_NUL)
1426             if not set(Value).issubset(Printset):
1427                 PrintList = sorted(Printset)
1428                 return False, "Invalid PCD string value of type [%s]; must be printable chars %s." % (Type, PrintList)
1429     elif Type == 'BOOLEAN':
1430         if Value not in ['TRUE', 'True', 'true', '0x1', '0x01', '1', 'FALSE', 'False', 'false', '0x0', '0x00', '0']:
1431             return False, "Invalid value [%s] of type [%s]; must be one of TRUE, True, true, 0x1, 0x01, 1"\
1432                           ", FALSE, False, false, 0x0, 0x00, 0" % (Value, Type)
1433     elif Type in [TAB_UINT8, TAB_UINT16, TAB_UINT32, TAB_UINT64]:
1434         if Value and int(Value, 0) < 0:
1435             return False, "PCD can't be set to negative value[%s] for datum type [%s]" % (Value, Type)
1436         try:
1437             Value = long(Value, 0)
1438             if Value > MAX_VAL_TYPE[Type]:
1439                 return False, "Too large PCD value[%s] for datum type [%s]" % (Value, Type)
1440         except:
1441             return False, "Invalid value [%s] of type [%s];"\
1442                           " must be a hexadecimal, decimal or octal in C language format." % (Value, Type)
1443     else:
1444         return True, "StructurePcd"
1445
1446     return True, ""
1447
1448 def CommonPath(PathList):
1449     P1 = min(PathList).split(os.path.sep)
1450     P2 = max(PathList).split(os.path.sep)
1451     for Index in xrange(min(len(P1), len(P2))):
1452         if P1[Index] != P2[Index]:
1453             return os.path.sep.join(P1[:Index])
1454     return os.path.sep.join(P1)
1455
1456 class PathClass(object):
1457     def __init__(self, File='', Root='', AlterRoot='', Type='', IsBinary=False,
1458                  Arch='COMMON', ToolChainFamily='', Target='', TagName='', ToolCode=''):
1459         self.Arch = Arch
1460         self.File = str(File)
1461         if os.path.isabs(self.File):
1462             self.Root = ''
1463             self.AlterRoot = ''
1464         else:
1465             self.Root = str(Root)
1466             self.AlterRoot = str(AlterRoot)
1467
1468         # Remove any '.' and '..' in path
1469         if self.Root:
1470             self.Root = mws.getWs(self.Root, self.File)
1471             self.Path = os.path.normpath(os.path.join(self.Root, self.File))
1472             self.Root = os.path.normpath(CommonPath([self.Root, self.Path]))
1473             # eliminate the side-effect of 'C:'
1474             if self.Root[-1] == ':':
1475                 self.Root += os.path.sep
1476             # file path should not start with path separator
1477             if self.Root[-1] == os.path.sep:
1478                 self.File = self.Path[len(self.Root):]
1479             else:
1480                 self.File = self.Path[len(self.Root) + 1:]
1481         else:
1482             self.Path = os.path.normpath(self.File)
1483
1484         self.SubDir, self.Name = os.path.split(self.File)
1485         self.BaseName, self.Ext = os.path.splitext(self.Name)
1486
1487         if self.Root:
1488             if self.SubDir:
1489                 self.Dir = os.path.join(self.Root, self.SubDir)
1490             else:
1491                 self.Dir = self.Root
1492         else:
1493             self.Dir = self.SubDir
1494
1495         if IsBinary:
1496             self.Type = Type
1497         else:
1498             self.Type = self.Ext.lower()
1499
1500         self.IsBinary = IsBinary
1501         self.Target = Target
1502         self.TagName = TagName
1503         self.ToolCode = ToolCode
1504         self.ToolChainFamily = ToolChainFamily
1505
1506     ## Convert the object of this class to a string
1507     #
1508     #  Convert member Path of the class to a string
1509     #
1510     #  @retval string Formatted String
1511     #
1512     def __str__(self):
1513         return self.Path
1514
1515     ## Override __eq__ function
1516     #
1517     # Check whether PathClass are the same
1518     #
1519     # @retval False The two PathClass are different
1520     # @retval True  The two PathClass are the same
1521     #
1522     def __eq__(self, Other):
1523         return self.Path == str(Other)
1524
1525     ## Override __cmp__ function
1526     #
1527     # Customize the comparsion operation of two PathClass
1528     #
1529     # @retval 0     The two PathClass are different
1530     # @retval -1    The first PathClass is less than the second PathClass
1531     # @retval 1     The first PathClass is Bigger than the second PathClass
1532     def __cmp__(self, Other):
1533         OtherKey = str(Other)
1534
1535         SelfKey = self.Path
1536         if SelfKey == OtherKey:
1537             return 0
1538         elif SelfKey > OtherKey:
1539             return 1
1540         else:
1541             return -1
1542
1543     ## Override __hash__ function
1544     #
1545     # Use Path as key in hash table
1546     #
1547     # @retval string Key for hash table
1548     #
1549     def __hash__(self):
1550         return hash(self.Path)
1551
1552     @cached_property
1553     def Key(self):
1554         return self.Path.upper()
1555
1556     @property
1557     def TimeStamp(self):
1558         return os.stat(self.Path)[8]
1559
1560     def Validate(self, Type='', CaseSensitive=True):
1561         def RealPath2(File, Dir='', OverrideDir=''):
1562             NewFile = None
1563             if OverrideDir:
1564                 NewFile = GlobalData.gAllFiles[os.path.normpath(os.path.join(OverrideDir, File))]
1565                 if NewFile:
1566                     if OverrideDir[-1] == os.path.sep:
1567                         return NewFile[len(OverrideDir):], NewFile[0:len(OverrideDir)]
1568                     else:
1569                         return NewFile[len(OverrideDir) + 1:], NewFile[0:len(OverrideDir)]
1570             if GlobalData.gAllFiles:
1571                 NewFile = GlobalData.gAllFiles[os.path.normpath(os.path.join(Dir, File))]
1572             if not NewFile:
1573                 NewFile = os.path.normpath(os.path.join(Dir, File))
1574                 if not os.path.exists(NewFile):
1575                     return None, None
1576             if NewFile:
1577                 if Dir:
1578                     if Dir[-1] == os.path.sep:
1579                         return NewFile[len(Dir):], NewFile[0:len(Dir)]
1580                     else:
1581                         return NewFile[len(Dir) + 1:], NewFile[0:len(Dir)]
1582                 else:
1583                     return NewFile, ''
1584
1585             return None, None
1586
1587         if GlobalData.gCaseInsensitive:
1588             CaseSensitive = False
1589         if Type and Type.lower() != self.Type:
1590             return FILE_TYPE_MISMATCH, '%s (expect %s but got %s)' % (self.File, Type, self.Type)
1591
1592         RealFile, RealRoot = RealPath2(self.File, self.Root, self.AlterRoot)
1593         if not RealRoot and not RealFile:
1594             RealFile = self.File
1595             if self.AlterRoot:
1596                 RealFile = os.path.join(self.AlterRoot, self.File)
1597             elif self.Root:
1598                 RealFile = os.path.join(self.Root, self.File)
1599             if len (mws.getPkgPath()) == 0:
1600                 return FILE_NOT_FOUND, os.path.join(self.AlterRoot, RealFile)
1601             else:
1602                 return FILE_NOT_FOUND, "%s is not found in packages path:\n\t%s" % (self.File, '\n\t'.join(mws.getPkgPath()))
1603
1604         ErrorCode = 0
1605         ErrorInfo = ''
1606         if RealRoot != self.Root or RealFile != self.File:
1607             if CaseSensitive and (RealFile != self.File or (RealRoot != self.Root and RealRoot != self.AlterRoot)):
1608                 ErrorCode = FILE_CASE_MISMATCH
1609                 ErrorInfo = self.File + '\n\t' + RealFile + " [in file system]"
1610
1611             self.SubDir, self.Name = os.path.split(RealFile)
1612             self.BaseName, self.Ext = os.path.splitext(self.Name)
1613             if self.SubDir:
1614                 self.Dir = os.path.join(RealRoot, self.SubDir)
1615             else:
1616                 self.Dir = RealRoot
1617             self.File = RealFile
1618             self.Root = RealRoot
1619             self.Path = os.path.join(RealRoot, RealFile)
1620         return ErrorCode, ErrorInfo
1621
1622 ## Parse PE image to get the required PE informaion.
1623 #
1624 class PeImageClass():
1625     ## Constructor
1626     #
1627     #   @param  File FilePath of PeImage
1628     #
1629     def __init__(self, PeFile):
1630         self.FileName   = PeFile
1631         self.IsValid    = False
1632         self.Size       = 0
1633         self.EntryPoint = 0
1634         self.SectionAlignment  = 0
1635         self.SectionHeaderList = []
1636         self.ErrorInfo = ''
1637         try:
1638             PeObject = open(PeFile, 'rb')
1639         except:
1640             self.ErrorInfo = self.FileName + ' can not be found\n'
1641             return
1642         # Read DOS header
1643         ByteArray = array.array('B')
1644         ByteArray.fromfile(PeObject, 0x3E)
1645         ByteList = ByteArray.tolist()
1646         # DOS signature should be 'MZ'
1647         if self._ByteListToStr (ByteList[0x0:0x2]) != 'MZ':
1648             self.ErrorInfo = self.FileName + ' has no valid DOS signature MZ'
1649             return
1650
1651         # Read 4 byte PE Signature
1652         PeOffset = self._ByteListToInt(ByteList[0x3C:0x3E])
1653         PeObject.seek(PeOffset)
1654         ByteArray = array.array('B')
1655         ByteArray.fromfile(PeObject, 4)
1656         # PE signature should be 'PE\0\0'
1657         if ByteArray.tostring() != 'PE\0\0':
1658             self.ErrorInfo = self.FileName + ' has no valid PE signature PE00'
1659             return
1660
1661         # Read PE file header
1662         ByteArray = array.array('B')
1663         ByteArray.fromfile(PeObject, 0x14)
1664         ByteList = ByteArray.tolist()
1665         SecNumber = self._ByteListToInt(ByteList[0x2:0x4])
1666         if SecNumber == 0:
1667             self.ErrorInfo = self.FileName + ' has no section header'
1668             return
1669
1670         # Read PE optional header
1671         OptionalHeaderSize = self._ByteListToInt(ByteArray[0x10:0x12])
1672         ByteArray = array.array('B')
1673         ByteArray.fromfile(PeObject, OptionalHeaderSize)
1674         ByteList = ByteArray.tolist()
1675         self.EntryPoint       = self._ByteListToInt(ByteList[0x10:0x14])
1676         self.SectionAlignment = self._ByteListToInt(ByteList[0x20:0x24])
1677         self.Size             = self._ByteListToInt(ByteList[0x38:0x3C])
1678
1679         # Read each Section Header
1680         for Index in range(SecNumber):
1681             ByteArray = array.array('B')
1682             ByteArray.fromfile(PeObject, 0x28)
1683             ByteList = ByteArray.tolist()
1684             SecName  = self._ByteListToStr(ByteList[0:8])
1685             SecVirtualSize = self._ByteListToInt(ByteList[8:12])
1686             SecRawAddress  = self._ByteListToInt(ByteList[20:24])
1687             SecVirtualAddress = self._ByteListToInt(ByteList[12:16])
1688             self.SectionHeaderList.append((SecName, SecVirtualAddress, SecRawAddress, SecVirtualSize))
1689         self.IsValid = True
1690         PeObject.close()
1691
1692     def _ByteListToStr(self, ByteList):
1693         String = ''
1694         for index in range(len(ByteList)):
1695             if ByteList[index] == 0:
1696                 break
1697             String += chr(ByteList[index])
1698         return String
1699
1700     def _ByteListToInt(self, ByteList):
1701         Value = 0
1702         for index in range(len(ByteList) - 1, -1, -1):
1703             Value = (Value << 8) | int(ByteList[index])
1704         return Value
1705
1706 class DefaultStore():
1707     def __init__(self, DefaultStores ):
1708
1709         self.DefaultStores = DefaultStores
1710     def DefaultStoreID(self, DefaultStoreName):
1711         for key, value in self.DefaultStores.items():
1712             if value == DefaultStoreName:
1713                 return key
1714         return None
1715     def GetDefaultDefault(self):
1716         if not self.DefaultStores or "0" in self.DefaultStores:
1717             return "0", TAB_DEFAULT_STORES_DEFAULT
1718         else:
1719             minvalue = min(int(value_str) for value_str in self.DefaultStores)
1720             return (str(minvalue), self.DefaultStores[str(minvalue)])
1721     def GetMin(self, DefaultSIdList):
1722         if not DefaultSIdList:
1723             return TAB_DEFAULT_STORES_DEFAULT
1724         storeidset = {storeid for storeid, storename in self.DefaultStores.values() if storename in DefaultSIdList}
1725         if not storeidset:
1726             return ""
1727         minid = min(storeidset )
1728         for sid, name in self.DefaultStores.values():
1729             if sid == minid:
1730                 return name
1731
1732 class SkuClass():
1733     DEFAULT = 0
1734     SINGLE = 1
1735     MULTIPLE =2
1736
1737     def __init__(self,SkuIdentifier='', SkuIds=None):
1738         if SkuIds is None:
1739             SkuIds = {}
1740
1741         for SkuName in SkuIds:
1742             SkuId = SkuIds[SkuName][0]
1743             skuid_num = int(SkuId, 16) if SkuId.upper().startswith("0X") else int(SkuId)
1744             if skuid_num > 0xFFFFFFFFFFFFFFFF:
1745                 EdkLogger.error("build", PARAMETER_INVALID,
1746                             ExtraData = "SKU-ID [%s] value %s exceeds the max value of UINT64"
1747                                       % (SkuName, SkuId))
1748
1749         self.AvailableSkuIds = sdict()
1750         self.SkuIdSet = []
1751         self.SkuIdNumberSet = []
1752         self.SkuData = SkuIds
1753         self._SkuInherit = {}
1754         self._SkuIdentifier = SkuIdentifier
1755         if SkuIdentifier == '' or SkuIdentifier is None:
1756             self.SkuIdSet = ['DEFAULT']
1757             self.SkuIdNumberSet = ['0U']
1758         elif SkuIdentifier == 'ALL':
1759             self.SkuIdSet = SkuIds.keys()
1760             self.SkuIdNumberSet = [num[0].strip() + 'U' for num in SkuIds.values()]
1761         else:
1762             r = SkuIdentifier.split('|')
1763             self.SkuIdSet=[(r[k].strip()).upper() for k in range(len(r))]
1764             k = None
1765             try:
1766                 self.SkuIdNumberSet = [SkuIds[k][0].strip() + 'U' for k in self.SkuIdSet]
1767             except Exception:
1768                 EdkLogger.error("build", PARAMETER_INVALID,
1769                             ExtraData = "SKU-ID [%s] is not supported by the platform. [Valid SKU-ID: %s]"
1770                                       % (k, " | ".join(SkuIds.keys())))
1771         for each in self.SkuIdSet:
1772             if each in SkuIds:
1773                 self.AvailableSkuIds[each] = SkuIds[each][0]
1774             else:
1775                 EdkLogger.error("build", PARAMETER_INVALID,
1776                             ExtraData="SKU-ID [%s] is not supported by the platform. [Valid SKU-ID: %s]"
1777                                       % (each, " | ".join(SkuIds.keys())))
1778         if self.SkuUsageType != SkuClass.SINGLE:
1779             self.AvailableSkuIds.update({'DEFAULT':0, 'COMMON':0})
1780         if self.SkuIdSet:
1781             GlobalData.gSkuids = (self.SkuIdSet)
1782             if 'COMMON' in GlobalData.gSkuids:
1783                 GlobalData.gSkuids.remove('COMMON')
1784             if self.SkuUsageType == self.SINGLE:
1785                 if len(GlobalData.gSkuids) != 1:
1786                     if 'DEFAULT' in GlobalData.gSkuids:
1787                         GlobalData.gSkuids.remove('DEFAULT')
1788             if GlobalData.gSkuids:
1789                 GlobalData.gSkuids.sort()
1790
1791     def GetNextSkuId(self, skuname):
1792         if not self._SkuInherit:
1793             self._SkuInherit = {}
1794             for item in self.SkuData.values():
1795                 self._SkuInherit[item[1]]=item[2] if item[2] else "DEFAULT"
1796         return self._SkuInherit.get(skuname, "DEFAULT")
1797
1798     def GetSkuChain(self, sku):
1799         if sku == "DEFAULT":
1800             return ["DEFAULT"]
1801         skulist = [sku]
1802         nextsku = sku
1803         while True:
1804             nextsku = self.GetNextSkuId(nextsku)
1805             skulist.append(nextsku)
1806             if nextsku == "DEFAULT":
1807                 break
1808         skulist.reverse()
1809         return skulist
1810     def SkuOverrideOrder(self):
1811         skuorderset = []
1812         for skuname in self.SkuIdSet:
1813             skuorderset.append(self.GetSkuChain(skuname))
1814
1815         skuorder = []
1816         for index in range(max(len(item) for item in skuorderset)):
1817             for subset in skuorderset:
1818                 if index > len(subset)-1:
1819                     continue
1820                 if subset[index] in skuorder:
1821                     continue
1822                 skuorder.append(subset[index])
1823
1824         return skuorder
1825
1826     @property
1827     def SkuUsageType(self):
1828         if self._SkuIdentifier.upper() == "ALL":
1829             return SkuClass.MULTIPLE
1830
1831         if len(self.SkuIdSet) == 1:
1832             if self.SkuIdSet[0] == 'DEFAULT':
1833                 return SkuClass.DEFAULT
1834             return SkuClass.SINGLE
1835         if len(self.SkuIdSet) == 2 and 'DEFAULT' in self.SkuIdSet:
1836             return SkuClass.SINGLE
1837         return SkuClass.MULTIPLE
1838
1839     def DumpSkuIdArrary(self):
1840         if self.SkuUsageType == SkuClass.SINGLE:
1841             return "{0x0}"
1842         ArrayStrList = []
1843         for skuname in self.AvailableSkuIds:
1844             if skuname == "COMMON":
1845                 continue
1846             while skuname != "DEFAULT":
1847                 ArrayStrList.append(hex(int(self.AvailableSkuIds[skuname])))
1848                 skuname = self.GetNextSkuId(skuname)
1849             ArrayStrList.append("0x0")
1850         return "{{{myList}}}".format(myList=",".join(ArrayStrList))
1851
1852     @property
1853     def AvailableSkuIdSet(self):
1854         return self.AvailableSkuIds
1855
1856     @property
1857     def SystemSkuId(self):
1858         if self.SkuUsageType == SkuClass.SINGLE:
1859             if len(self.SkuIdSet) == 1:
1860                 return self.SkuIdSet[0]
1861             else:
1862                 return self.SkuIdSet[0] if self.SkuIdSet[0] != 'DEFAULT' else self.SkuIdSet[1]
1863         else:
1864             return 'DEFAULT'
1865
1866 ##  Get the integer value from string like "14U" or integer like 2
1867 #
1868 #   @param      Input   The object that may be either a integer value or a string
1869 #
1870 #   @retval     Value    The integer value that the input represents
1871 #
1872 def GetIntegerValue(Input):
1873     if type(Input) in (int, long):
1874         return Input
1875     String = Input
1876     if String.endswith("U"):
1877         String = String[:-1]
1878     if String.endswith("ULL"):
1879         String = String[:-3]
1880     if String.endswith("LL"):
1881         String = String[:-2]
1882
1883     if String.startswith("0x") or String.startswith("0X"):
1884         return int(String, 16)
1885     elif String == '':
1886         return 0
1887     else:
1888         return int(String)
1889
1890 #
1891 # Pack a GUID (registry format) list into a buffer and return it
1892 #
1893 def PackGUID(Guid):
1894     return pack(PACK_PATTERN_GUID,
1895                 int(Guid[0], 16),
1896                 int(Guid[1], 16),
1897                 int(Guid[2], 16),
1898                 int(Guid[3][-4:-2], 16),
1899                 int(Guid[3][-2:], 16),
1900                 int(Guid[4][-12:-10], 16),
1901                 int(Guid[4][-10:-8], 16),
1902                 int(Guid[4][-8:-6], 16),
1903                 int(Guid[4][-6:-4], 16),
1904                 int(Guid[4][-4:-2], 16),
1905                 int(Guid[4][-2:], 16)
1906                 )
1907
1908 #
1909 # Pack a GUID (byte) list into a buffer and return it
1910 #
1911 def PackByteFormatGUID(Guid):
1912     return pack(PACK_PATTERN_GUID,
1913                 Guid[0],
1914                 Guid[1],
1915                 Guid[2],
1916                 Guid[3],
1917                 Guid[4],
1918                 Guid[5],
1919                 Guid[6],
1920                 Guid[7],
1921                 Guid[8],
1922                 Guid[9],
1923                 Guid[10],
1924                 )
1925
1926 ## DeepCopy dict/OrderedDict recusively
1927 #
1928 #   @param      ori_dict    a nested dict or ordereddict
1929 #
1930 #   @retval     new dict or orderdict
1931 #
1932 def CopyDict(ori_dict):
1933     dict_type = ori_dict.__class__
1934     if dict_type not in (dict,OrderedDict):
1935         return ori_dict
1936     new_dict = dict_type()
1937     for key in ori_dict:
1938         if isinstance(ori_dict[key],(dict,OrderedDict)):
1939             new_dict[key] = CopyDict(ori_dict[key])
1940         else:
1941             new_dict[key] = ori_dict[key]
1942     return new_dict
1943
1944 #
1945 # Remove the c/c++ comments: // and /* */
1946 #
1947 def RemoveCComments(ctext):
1948     return re.sub('//.*?\n|/\*.*?\*/', '\n', ctext, flags=re.S)