BaseTools/Source/Python/Common/Misc.py

   1 ## @file
   2 # Common routines used by all tools
   3 #
   4 # Copyright (c) 2007 - 2019, Intel Corporation. All rights reserved.<BR>
   5 # This program and the accompanying materials
   6 # are licensed and made available under the terms and conditions of the BSD License
   7 # which accompanies this distribution.  The full text of the license may be found at
   8 # http://opensource.org/licenses/bsd-license.php
   9 #
  10 # THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS,
  11 # WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED.
  12 #
  13
  14 ##
  15 # Import Modules
  16 #
  17 from __future__ import absolute_import
  18 import Common.LongFilePathOs as os
  19 import sys
  20 import string
  21 import threading
  22 import time
  23 import re
  24 import pickle
  25 import array
  26 import shutil
  27 from random import sample
  28 from struct import pack
  29 from UserDict import IterableUserDict
  30 from UserList import UserList
  31
  32 from Common import EdkLogger as EdkLogger
  33 from Common import GlobalData as GlobalData
  34 from .DataType import *
  35 from .BuildToolError import *
  36 from CommonDataClass.DataClass import *
  37 from .Parsing import GetSplitValueList
  38 from Common.LongFilePathSupport import OpenLongFilePath as open
  39 from Common.MultipleWorkspace import MultipleWorkspace as mws
  40 import uuid
  41 from CommonDataClass.Exceptions import BadExpression
  42 from Common.caching import cached_property
  43 import subprocess
  44 from collections import OrderedDict
  45 ## Regular expression used to find out place holders in string template
  46 gPlaceholderPattern = re.compile("\$\{([^$()\s]+)\}", re.MULTILINE | re.UNICODE)
  47
  48 ## regular expressions for map file processing
  49 startPatternGeneral = re.compile("^Start[' ']+Length[' ']+Name[' ']+Class")
  50 addressPatternGeneral = re.compile("^Address[' ']+Publics by Value[' ']+Rva\+Base")
  51 valuePatternGcc = re.compile('^([\w_\.]+) +([\da-fA-Fx]+) +([\da-fA-Fx]+)$')
  52 pcdPatternGcc = re.compile('^([\da-fA-Fx]+) +([\da-fA-Fx]+)')
  53 secReGeneral = re.compile('^([\da-fA-F]+):([\da-fA-F]+) +([\da-fA-F]+)[Hh]? +([.\w\$]+) +(\w+)', re.UNICODE)
  54
  55 StructPattern = re.compile(r'[_a-zA-Z][0-9A-Za-z_]*$')
  56
  57 ## Dictionary used to store file time stamp for quick re-access
  58 gFileTimeStampCache = {}    # {file path : file time stamp}
  59
  60 ## Dictionary used to store dependencies of files
  61 gDependencyDatabase = {}    # arch : {file path : [dependent files list]}
  62
  63 #
  64 # If a module is built more than once with different PCDs or library classes
  65 # a temporary INF file with same content is created, the temporary file is removed
  66 # when build exits.
  67 #
  68 _TempInfs = []
  69
  70 def GetVariableOffset(mapfilepath, efifilepath, varnames):
  71     """ Parse map file to get variable offset in current EFI file
  72     @param mapfilepath    Map file absolution path
  73     @param efifilepath:   EFI binary file full path
  74     @param varnames       iteratable container whose elements are variable names to be searched
  75
  76     @return List whos elements are tuple with variable name and raw offset
  77     """
  78     lines = []
  79     try:
  80         f = open(mapfilepath, 'r')
  81         lines = f.readlines()
  82         f.close()
  83     except:
  84         return None
  85
  86     if len(lines) == 0: return None
  87     firstline = lines[0].strip()
  88     if (firstline.startswith("Archive member included ") and
  89         firstline.endswith(" file (symbol)")):
  90         return _parseForGCC(lines, efifilepath, varnames)
  91     if firstline.startswith("# Path:"):
  92         return _parseForXcode(lines, efifilepath, varnames)
  93     return _parseGeneral(lines, efifilepath, varnames)
  94
  95 def _parseForXcode(lines, efifilepath, varnames):
  96     status = 0
  97     ret = []
  98     for line in lines:
  99         line = line.strip()
 100         if status == 0 and line == "# Symbols:":
 101             status = 1
 102             continue
 103         if status == 1 and len(line) != 0:
 104             for varname in varnames:
 105                 if varname in line:
 106                     # cannot pregenerate this RegEx since it uses varname from varnames.
 107                     m = re.match('^([\da-fA-FxX]+)([\s\S]*)([_]*%s)$' % varname, line)
 108                     if m is not None:
 109                         ret.append((varname, m.group(1)))
 110     return ret
 111
 112 def _parseForGCC(lines, efifilepath, varnames):
 113     """ Parse map file generated by GCC linker """
 114     status = 0
 115     sections = []
 116     varoffset = []
 117     for index, line in enumerate(lines):
 118         line = line.strip()
 119         # status machine transection
 120         if status == 0 and line == "Memory Configuration":
 121             status = 1
 122             continue
 123         elif status == 1 and line == 'Linker script and memory map':
 124             status = 2
 125             continue
 126         elif status ==2 and line == 'START GROUP':
 127             status = 3
 128             continue
 129
 130         # status handler
 131         if status == 3:
 132             m = valuePatternGcc.match(line)
 133             if m is not None:
 134                 sections.append(m.groups(0))
 135             for varname in varnames:
 136                 Str = ''
 137                 m = re.match("^.data.(%s)" % varname, line)
 138                 if m is not None:
 139                     m = re.match(".data.(%s)$" % varname, line)
 140                     if m is not None:
 141                         Str = lines[index + 1]
 142                     else:
 143                         Str = line[len(".data.%s" % varname):]
 144                     if Str:
 145                         m = pcdPatternGcc.match(Str.strip())
 146                         if m is not None:
 147                             varoffset.append((varname, int(m.groups(0)[0], 16), int(sections[-1][1], 16), sections[-1][0]))
 148
 149     if not varoffset:
 150         return []
 151     # get section information from efi file
 152     efisecs = PeImageClass(efifilepath).SectionHeaderList
 153     if efisecs is None or len(efisecs) == 0:
 154         return []
 155     #redirection
 156     redirection = 0
 157     for efisec in efisecs:
 158         for section in sections:
 159             if section[0].strip() == efisec[0].strip() and section[0].strip() == '.text':
 160                 redirection = int(section[1], 16) - efisec[1]
 161
 162     ret = []
 163     for var in varoffset:
 164         for efisec in efisecs:
 165             if var[1] >= efisec[1] and var[1] < efisec[1]+efisec[3]:
 166                 ret.append((var[0], hex(efisec[2] + var[1] - efisec[1] - redirection)))
 167     return ret
 168
 169 def _parseGeneral(lines, efifilepath, varnames):
 170     status = 0    #0 - beginning of file; 1 - PE section definition; 2 - symbol table
 171     secs  = []    # key = section name
 172     varoffset = []
 173     symRe = re.compile('^([\da-fA-F]+):([\da-fA-F]+) +([\.:\\\\\w\?@\$]+) +([\da-fA-F]+)', re.UNICODE)
 174
 175     for line in lines:
 176         line = line.strip()
 177         if startPatternGeneral.match(line):
 178             status = 1
 179             continue
 180         if addressPatternGeneral.match(line):
 181             status = 2
 182             continue
 183         if line.startswith("entry point at"):
 184             status = 3
 185             continue
 186         if status == 1 and len(line) != 0:
 187             m =  secReGeneral.match(line)
 188             assert m is not None, "Fail to parse the section in map file , line is %s" % line
 189             sec_no, sec_start, sec_length, sec_name, sec_class = m.groups(0)
 190             secs.append([int(sec_no, 16), int(sec_start, 16), int(sec_length, 16), sec_name, sec_class])
 191         if status == 2 and len(line) != 0:
 192             for varname in varnames:
 193                 m = symRe.match(line)
 194                 assert m is not None, "Fail to parse the symbol in map file, line is %s" % line
 195                 sec_no, sym_offset, sym_name, vir_addr = m.groups(0)
 196                 sec_no     = int(sec_no,     16)
 197                 sym_offset = int(sym_offset, 16)
 198                 vir_addr   = int(vir_addr,   16)
 199                 # cannot pregenerate this RegEx since it uses varname from varnames.
 200                 m2 = re.match('^[_]*(%s)' % varname, sym_name)
 201                 if m2 is not None:
 202                     # fond a binary pcd entry in map file
 203                     for sec in secs:
 204                         if sec[0] == sec_no and (sym_offset >= sec[1] and sym_offset < sec[1] + sec[2]):
 205                             varoffset.append([varname, sec[3], sym_offset, vir_addr, sec_no])
 206
 207     if not varoffset: return []
 208
 209     # get section information from efi file
 210     efisecs = PeImageClass(efifilepath).SectionHeaderList
 211     if efisecs is None or len(efisecs) == 0:
 212         return []
 213
 214     ret = []
 215     for var in varoffset:
 216         index = 0
 217         for efisec in efisecs:
 218             index = index + 1
 219             if var[1].strip() == efisec[0].strip():
 220                 ret.append((var[0], hex(efisec[2] + var[2])))
 221             elif var[4] == index:
 222                 ret.append((var[0], hex(efisec[2] + var[2])))
 223
 224     return ret
 225
 226 ## Routine to process duplicated INF
 227 #
 228 #  This function is called by following two cases:
 229 #  Case 1 in DSC:
 230 #    [components.arch]
 231 #    Pkg/module/module.inf
 232 #    Pkg/module/module.inf {
 233 #      <Defines>
 234 #        FILE_GUID = 0D1B936F-68F3-4589-AFCC-FB8B7AEBC836
 235 #    }
 236 #  Case 2 in FDF:
 237 #    INF Pkg/module/module.inf
 238 #    INF FILE_GUID = 0D1B936F-68F3-4589-AFCC-FB8B7AEBC836 Pkg/module/module.inf
 239 #
 240 #  This function copies Pkg/module/module.inf to
 241 #  Conf/.cache/0D1B936F-68F3-4589-AFCC-FB8B7AEBC836module.inf
 242 #
 243 #  @param Path     Original PathClass object
 244 #  @param BaseName New file base name
 245 #
 246 #  @retval         return the new PathClass object
 247 #
 248 def ProcessDuplicatedInf(Path, BaseName, Workspace):
 249     Filename = os.path.split(Path.File)[1]
 250     if '.' in Filename:
 251         Filename = BaseName + Path.BaseName + Filename[Filename.rfind('.'):]
 252     else:
 253         Filename = BaseName + Path.BaseName
 254
 255     #
 256     # If -N is specified on command line, cache is disabled
 257     # The directory has to be created
 258     #
 259     DbDir = os.path.split(GlobalData.gDatabasePath)[0]
 260     if not os.path.exists(DbDir):
 261         os.makedirs(DbDir)
 262     #
 263     # A temporary INF is copied to database path which must have write permission
 264     # The temporary will be removed at the end of build
 265     # In case of name conflict, the file name is
 266     # FILE_GUIDBaseName (0D1B936F-68F3-4589-AFCC-FB8B7AEBC836module.inf)
 267     #
 268     TempFullPath = os.path.join(DbDir,
 269                                 Filename)
 270     RtPath = PathClass(Path.File, Workspace)
 271     #
 272     # Modify the full path to temporary path, keep other unchanged
 273     #
 274     # To build same module more than once, the module path with FILE_GUID overridden has
 275     # the file name FILE_GUIDmodule.inf, but the relative path (self.MetaFile.File) is the real path
 276     # in DSC which is used as relative path by C files and other files in INF.
 277     # A trick was used: all module paths are PathClass instances, after the initialization
 278     # of PathClass, the PathClass.Path is overridden by the temporary INF path.
 279     #
 280     # The reason for creating a temporary INF is:
 281     # Platform.Modules which is the base to create ModuleAutoGen objects is a dictionary,
 282     # the key is the full path of INF, the value is an object to save overridden library instances, PCDs.
 283     # A different key for the same module is needed to create different output directory,
 284     # retrieve overridden PCDs, library instances.
 285     #
 286     # The BaseName is the FILE_GUID which is also the output directory name.
 287     #
 288     #
 289     RtPath.Path = TempFullPath
 290     RtPath.BaseName = BaseName
 291     #
 292     # If file exists, compare contents
 293     #
 294     if os.path.exists(TempFullPath):
 295         with open(str(Path), 'rb') as f1, open(TempFullPath, 'rb') as f2:
 296             if f1.read() == f2.read():
 297                 return RtPath
 298     _TempInfs.append(TempFullPath)
 299     shutil.copy2(str(Path), TempFullPath)
 300     return RtPath
 301
 302 ## Remove temporary created INFs whose paths were saved in _TempInfs
 303 #
 304 def ClearDuplicatedInf():
 305     while _TempInfs:
 306         File = _TempInfs.pop()
 307         if os.path.exists(File):
 308             os.remove(File)
 309
 310 ## Convert GUID string in xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx style to C structure style
 311 #
 312 #   @param      Guid    The GUID string
 313 #
 314 #   @retval     string  The GUID string in C structure style
 315 #
 316 def GuidStringToGuidStructureString(Guid):
 317     GuidList = Guid.split('-')
 318     Result = '{'
 319     for Index in range(0, 3, 1):
 320         Result = Result + '0x' + GuidList[Index] + ', '
 321     Result = Result + '{0x' + GuidList[3][0:2] + ', 0x' + GuidList[3][2:4]
 322     for Index in range(0, 12, 2):
 323         Result = Result + ', 0x' + GuidList[4][Index:Index + 2]
 324     Result += '}}'
 325     return Result
 326
 327 ## Convert GUID structure in byte array to xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
 328 #
 329 #   @param      GuidValue   The GUID value in byte array
 330 #
 331 #   @retval     string      The GUID value in xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx format
 332 #
 333 def GuidStructureByteArrayToGuidString(GuidValue):
 334     guidValueString = GuidValue.lower().replace("{", "").replace("}", "").replace(" ", "").replace(";", "")
 335     guidValueList = guidValueString.split(",")
 336     if len(guidValueList) != 16:
 337         return ''
 338         #EdkLogger.error(None, None, "Invalid GUID value string %s" % GuidValue)
 339     try:
 340         return "%02x%02x%02x%02x-%02x%02x-%02x%02x-%02x%02x-%02x%02x%02x%02x%02x%02x" % (
 341                 int(guidValueList[3], 16),
 342                 int(guidValueList[2], 16),
 343                 int(guidValueList[1], 16),
 344                 int(guidValueList[0], 16),
 345                 int(guidValueList[5], 16),
 346                 int(guidValueList[4], 16),
 347                 int(guidValueList[7], 16),
 348                 int(guidValueList[6], 16),
 349                 int(guidValueList[8], 16),
 350                 int(guidValueList[9], 16),
 351                 int(guidValueList[10], 16),
 352                 int(guidValueList[11], 16),
 353                 int(guidValueList[12], 16),
 354                 int(guidValueList[13], 16),
 355                 int(guidValueList[14], 16),
 356                 int(guidValueList[15], 16)
 357                 )
 358     except:
 359         return ''
 360
 361 ## Convert GUID string in C structure style to xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
 362 #
 363 #   @param      GuidValue   The GUID value in C structure format
 364 #
 365 #   @retval     string      The GUID value in xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx format
 366 #
 367 def GuidStructureStringToGuidString(GuidValue):
 368     if not GlobalData.gGuidCFormatPattern.match(GuidValue):
 369         return ''
 370     guidValueString = GuidValue.lower().replace("{", "").replace("}", "").replace(" ", "").replace(";", "")
 371     guidValueList = guidValueString.split(",")
 372     if len(guidValueList) != 11:
 373         return ''
 374         #EdkLogger.error(None, None, "Invalid GUID value string %s" % GuidValue)
 375     try:
 376         return "%08x-%04x-%04x-%02x%02x-%02x%02x%02x%02x%02x%02x" % (
 377                 int(guidValueList[0], 16),
 378                 int(guidValueList[1], 16),
 379                 int(guidValueList[2], 16),
 380                 int(guidValueList[3], 16),
 381                 int(guidValueList[4], 16),
 382                 int(guidValueList[5], 16),
 383                 int(guidValueList[6], 16),
 384                 int(guidValueList[7], 16),
 385                 int(guidValueList[8], 16),
 386                 int(guidValueList[9], 16),
 387                 int(guidValueList[10], 16)
 388                 )
 389     except:
 390         return ''
 391
 392 ## Convert GUID string in C structure style to xxxxxxxx_xxxx_xxxx_xxxx_xxxxxxxxxxxx
 393 #
 394 #   @param      GuidValue   The GUID value in C structure format
 395 #
 396 #   @retval     string      The GUID value in xxxxxxxx_xxxx_xxxx_xxxx_xxxxxxxxxxxx format
 397 #
 398 def GuidStructureStringToGuidValueName(GuidValue):
 399     guidValueString = GuidValue.lower().replace("{", "").replace("}", "").replace(" ", "")
 400     guidValueList = guidValueString.split(",")
 401     if len(guidValueList) != 11:
 402         EdkLogger.error(None, FORMAT_INVALID, "Invalid GUID value string [%s]" % GuidValue)
 403     return "%08x_%04x_%04x_%02x%02x_%02x%02x%02x%02x%02x%02x" % (
 404             int(guidValueList[0], 16),
 405             int(guidValueList[1], 16),
 406             int(guidValueList[2], 16),
 407             int(guidValueList[3], 16),
 408             int(guidValueList[4], 16),
 409             int(guidValueList[5], 16),
 410             int(guidValueList[6], 16),
 411             int(guidValueList[7], 16),
 412             int(guidValueList[8], 16),
 413             int(guidValueList[9], 16),
 414             int(guidValueList[10], 16)
 415             )
 416
 417 ## Create directories
 418 #
 419 #   @param      Directory   The directory name
 420 #
 421 def CreateDirectory(Directory):
 422     if Directory is None or Directory.strip() == "":
 423         return True
 424     try:
 425         if not os.access(Directory, os.F_OK):
 426             os.makedirs(Directory)
 427     except:
 428         return False
 429     return True
 430
 431 ## Remove directories, including files and sub-directories in it
 432 #
 433 #   @param      Directory   The directory name
 434 #
 435 def RemoveDirectory(Directory, Recursively=False):
 436     if Directory is None or Directory.strip() == "" or not os.path.exists(Directory):
 437         return
 438     if Recursively:
 439         CurrentDirectory = os.getcwd()
 440         os.chdir(Directory)
 441         for File in os.listdir("."):
 442             if os.path.isdir(File):
 443                 RemoveDirectory(File, Recursively)
 444             else:
 445                 os.remove(File)
 446         os.chdir(CurrentDirectory)
 447     os.rmdir(Directory)
 448
 449 ## Store content in file
 450 #
 451 #  This method is used to save file only when its content is changed. This is
 452 #  quite useful for "make" system to decide what will be re-built and what won't.
 453 #
 454 #   @param      File            The path of file
 455 #   @param      Content         The new content of the file
 456 #   @param      IsBinaryFile    The flag indicating if the file is binary file or not
 457 #
 458 #   @retval     True            If the file content is changed and the file is renewed
 459 #   @retval     False           If the file content is the same
 460 #
 461 def SaveFileOnChange(File, Content, IsBinaryFile=True):
 462     if not IsBinaryFile:
 463         Content = Content.replace("\n", os.linesep)
 464
 465     if os.path.exists(File):
 466         try:
 467             if Content == open(File, "rb").read():
 468                 return False
 469         except:
 470             EdkLogger.error(None, FILE_OPEN_FAILURE, ExtraData=File)
 471
 472     DirName = os.path.dirname(File)
 473     if not CreateDirectory(DirName):
 474         EdkLogger.error(None, FILE_CREATE_FAILURE, "Could not create directory %s" % DirName)
 475     else:
 476         if DirName == '':
 477             DirName = os.getcwd()
 478         if not os.access(DirName, os.W_OK):
 479             EdkLogger.error(None, PERMISSION_FAILURE, "Do not have write permission on directory %s" % DirName)
 480
 481     try:
 482         Fd = open(File, "wb")
 483         Fd.write(Content)
 484         Fd.close()
 485     except IOError as X:
 486         EdkLogger.error(None, FILE_CREATE_FAILURE, ExtraData='IOError %s' % X)
 487
 488     return True
 489
 490 ## Retrieve and cache the real path name in file system
 491 #
 492 #   @param      Root    The root directory of path relative to
 493 #
 494 #   @retval     str     The path string if the path exists
 495 #   @retval     None    If path doesn't exist
 496 #
 497 class DirCache:
 498     _CACHE_ = set()
 499     _UPPER_CACHE_ = {}
 500
 501     def __init__(self, Root):
 502         self._Root = Root
 503         for F in os.listdir(Root):
 504             self._CACHE_.add(F)
 505             self._UPPER_CACHE_[F.upper()] = F
 506
 507     # =[] operator
 508     def __getitem__(self, Path):
 509         Path = Path[len(os.path.commonprefix([Path, self._Root])):]
 510         if not Path:
 511             return self._Root
 512         if Path and Path[0] == os.path.sep:
 513             Path = Path[1:]
 514         if Path in self._CACHE_:
 515             return os.path.join(self._Root, Path)
 516         UpperPath = Path.upper()
 517         if UpperPath in self._UPPER_CACHE_:
 518             return os.path.join(self._Root, self._UPPER_CACHE_[UpperPath])
 519
 520         IndexList = []
 521         LastSepIndex = -1
 522         SepIndex = Path.find(os.path.sep)
 523         while SepIndex > -1:
 524             Parent = UpperPath[:SepIndex]
 525             if Parent not in self._UPPER_CACHE_:
 526                 break
 527             LastSepIndex = SepIndex
 528             SepIndex = Path.find(os.path.sep, LastSepIndex + 1)
 529
 530         if LastSepIndex == -1:
 531             return None
 532
 533         Cwd = os.getcwd()
 534         os.chdir(self._Root)
 535         SepIndex = LastSepIndex
 536         while SepIndex > -1:
 537             Parent = Path[:SepIndex]
 538             ParentKey = UpperPath[:SepIndex]
 539             if ParentKey not in self._UPPER_CACHE_:
 540                 os.chdir(Cwd)
 541                 return None
 542
 543             if Parent in self._CACHE_:
 544                 ParentDir = Parent
 545             else:
 546                 ParentDir = self._UPPER_CACHE_[ParentKey]
 547             for F in os.listdir(ParentDir):
 548                 Dir = os.path.join(ParentDir, F)
 549                 self._CACHE_.add(Dir)
 550                 self._UPPER_CACHE_[Dir.upper()] = Dir
 551
 552             SepIndex = Path.find(os.path.sep, SepIndex + 1)
 553
 554         os.chdir(Cwd)
 555         if Path in self._CACHE_:
 556             return os.path.join(self._Root, Path)
 557         elif UpperPath in self._UPPER_CACHE_:
 558             return os.path.join(self._Root, self._UPPER_CACHE_[UpperPath])
 559         return None
 560
 561 def RealPath(File, Dir='', OverrideDir=''):
 562     NewFile = os.path.normpath(os.path.join(Dir, File))
 563     NewFile = GlobalData.gAllFiles[NewFile]
 564     if not NewFile and OverrideDir:
 565         NewFile = os.path.normpath(os.path.join(OverrideDir, File))
 566         NewFile = GlobalData.gAllFiles[NewFile]
 567     return NewFile
 568
 569 def RealPath2(File, Dir='', OverrideDir=''):
 570     NewFile = None
 571     if OverrideDir:
 572         NewFile = GlobalData.gAllFiles[os.path.normpath(os.path.join(OverrideDir, File))]
 573         if NewFile:
 574             if OverrideDir[-1] == os.path.sep:
 575                 return NewFile[len(OverrideDir):], NewFile[0:len(OverrideDir)]
 576             else:
 577                 return NewFile[len(OverrideDir) + 1:], NewFile[0:len(OverrideDir)]
 578     if GlobalData.gAllFiles:
 579         NewFile = GlobalData.gAllFiles[os.path.normpath(os.path.join(Dir, File))]
 580     if not NewFile:
 581         NewFile = os.path.normpath(os.path.join(Dir, File))
 582         if not os.path.exists(NewFile):
 583             return None, None
 584     if NewFile:
 585         if Dir:
 586             if Dir[-1] == os.path.sep:
 587                 return NewFile[len(Dir):], NewFile[0:len(Dir)]
 588             else:
 589                 return NewFile[len(Dir) + 1:], NewFile[0:len(Dir)]
 590         else:
 591             return NewFile, ''
 592
 593     return None, None
 594
 595 ## Get GUID value from given packages
 596 #
 597 #   @param      CName           The CName of the GUID
 598 #   @param      PackageList     List of packages looking-up in
 599 #   @param      Inffile         The driver file
 600 #
 601 #   @retval     GuidValue   if the CName is found in any given package
 602 #   @retval     None        if the CName is not found in all given packages
 603 #
 604 def GuidValue(CName, PackageList, Inffile = None):
 605     for P in PackageList:
 606         GuidKeys = P.Guids.keys()
 607         if Inffile and P._PrivateGuids:
 608             if not Inffile.startswith(P.MetaFile.Dir):
 609                 GuidKeys = [x for x in P.Guids if x not in P._PrivateGuids]
 610         if CName in GuidKeys:
 611             return P.Guids[CName]
 612     return None
 613
 614 ## A string template class
 615 #
 616 #  This class implements a template for string replacement. A string template
 617 #  looks like following
 618 #
 619 #       ${BEGIN} other_string ${placeholder_name} other_string ${END}
 620 #
 621 #  The string between ${BEGIN} and ${END} will be repeated as many times as the
 622 #  length of "placeholder_name", which is a list passed through a dict. The
 623 #  "placeholder_name" is the key name of the dict. The ${BEGIN} and ${END} can
 624 #  be not used and, in this case, the "placeholder_name" must not a list and it
 625 #  will just be replaced once.
 626 #
 627 class TemplateString(object):
 628     _REPEAT_START_FLAG = "BEGIN"
 629     _REPEAT_END_FLAG = "END"
 630
 631     class Section(object):
 632         _LIST_TYPES = [type([]), type(set()), type((0,))]
 633
 634         def __init__(self, TemplateSection, PlaceHolderList):
 635             self._Template = TemplateSection
 636             self._PlaceHolderList = []
 637
 638             # Split the section into sub-sections according to the position of placeholders
 639             if PlaceHolderList:
 640                 self._SubSectionList = []
 641                 SubSectionStart = 0
 642                 #
 643                 # The placeholders passed in must be in the format of
 644                 #
 645                 #   PlaceHolderName, PlaceHolderStartPoint, PlaceHolderEndPoint
 646                 #
 647                 for PlaceHolder, Start, End in PlaceHolderList:
 648                     self._SubSectionList.append(TemplateSection[SubSectionStart:Start])
 649                     self._SubSectionList.append(TemplateSection[Start:End])
 650                     self._PlaceHolderList.append(PlaceHolder)
 651                     SubSectionStart = End
 652                 if SubSectionStart < len(TemplateSection):
 653                     self._SubSectionList.append(TemplateSection[SubSectionStart:])
 654             else:
 655                 self._SubSectionList = [TemplateSection]
 656
 657         def __str__(self):
 658             return self._Template + " : " + str(self._PlaceHolderList)
 659
 660         def Instantiate(self, PlaceHolderValues):
 661             RepeatTime = -1
 662             RepeatPlaceHolders = {}
 663             NonRepeatPlaceHolders = {}
 664
 665             for PlaceHolder in self._PlaceHolderList:
 666                 if PlaceHolder not in PlaceHolderValues:
 667                     continue
 668                 Value = PlaceHolderValues[PlaceHolder]
 669                 if type(Value) in self._LIST_TYPES:
 670                     if RepeatTime < 0:
 671                         RepeatTime = len(Value)
 672                     elif RepeatTime != len(Value):
 673                         EdkLogger.error(
 674                                     "TemplateString",
 675                                     PARAMETER_INVALID,
 676                                     "${%s} has different repeat time from others!" % PlaceHolder,
 677                                     ExtraData=str(self._Template)
 678                                     )
 679                     RepeatPlaceHolders["${%s}" % PlaceHolder] = Value
 680                 else:
 681                     NonRepeatPlaceHolders["${%s}" % PlaceHolder] = Value
 682
 683             if NonRepeatPlaceHolders:
 684                 StringList = []
 685                 for S in self._SubSectionList:
 686                     if S not in NonRepeatPlaceHolders:
 687                         StringList.append(S)
 688                     else:
 689                         StringList.append(str(NonRepeatPlaceHolders[S]))
 690             else:
 691                 StringList = self._SubSectionList
 692
 693             if RepeatPlaceHolders:
 694                 TempStringList = []
 695                 for Index in range(RepeatTime):
 696                     for S in StringList:
 697                         if S not in RepeatPlaceHolders:
 698                             TempStringList.append(S)
 699                         else:
 700                             TempStringList.append(str(RepeatPlaceHolders[S][Index]))
 701                 StringList = TempStringList
 702
 703             return "".join(StringList)
 704
 705     ## Constructor
 706     def __init__(self, Template=None):
 707         self.String = []
 708         self.IsBinary = False
 709         self._Template = Template
 710         self._TemplateSectionList = self._Parse(Template)
 711
 712     ## str() operator
 713     #
 714     #   @retval     string  The string replaced
 715     #
 716     def __str__(self):
 717         return "".join(self.String)
 718
 719     ## Split the template string into fragments per the ${BEGIN} and ${END} flags
 720     #
 721     #   @retval     list    A list of TemplateString.Section objects
 722     #
 723     def _Parse(self, Template):
 724         SectionStart = 0
 725         SearchFrom = 0
 726         MatchEnd = 0
 727         PlaceHolderList = []
 728         TemplateSectionList = []
 729         while Template:
 730             MatchObj = gPlaceholderPattern.search(Template, SearchFrom)
 731             if not MatchObj:
 732                 if MatchEnd <= len(Template):
 733                     TemplateSection = TemplateString.Section(Template[SectionStart:], PlaceHolderList)
 734                     TemplateSectionList.append(TemplateSection)
 735                 break
 736
 737             MatchString = MatchObj.group(1)
 738             MatchStart = MatchObj.start()
 739             MatchEnd = MatchObj.end()
 740
 741             if MatchString == self._REPEAT_START_FLAG:
 742                 if MatchStart > SectionStart:
 743                     TemplateSection = TemplateString.Section(Template[SectionStart:MatchStart], PlaceHolderList)
 744                     TemplateSectionList.append(TemplateSection)
 745                 SectionStart = MatchEnd
 746                 PlaceHolderList = []
 747             elif MatchString == self._REPEAT_END_FLAG:
 748                 TemplateSection = TemplateString.Section(Template[SectionStart:MatchStart], PlaceHolderList)
 749                 TemplateSectionList.append(TemplateSection)
 750                 SectionStart = MatchEnd
 751                 PlaceHolderList = []
 752             else:
 753                 PlaceHolderList.append((MatchString, MatchStart - SectionStart, MatchEnd - SectionStart))
 754             SearchFrom = MatchEnd
 755         return TemplateSectionList
 756
 757     ## Replace the string template with dictionary of placeholders and append it to previous one
 758     #
 759     #   @param      AppendString    The string template to append
 760     #   @param      Dictionary      The placeholder dictionaries
 761     #
 762     def Append(self, AppendString, Dictionary=None):
 763         if Dictionary:
 764             SectionList = self._Parse(AppendString)
 765             self.String.append( "".join(S.Instantiate(Dictionary) for S in SectionList))
 766         else:
 767             if isinstance(AppendString,list):
 768                 self.String.extend(AppendString)
 769             else:
 770                 self.String.append(AppendString)
 771
 772     ## Replace the string template with dictionary of placeholders
 773     #
 774     #   @param      Dictionary      The placeholder dictionaries
 775     #
 776     #   @retval     str             The string replaced with placeholder values
 777     #
 778     def Replace(self, Dictionary=None):
 779         return "".join(S.Instantiate(Dictionary) for S in self._TemplateSectionList)
 780
 781 ## Progress indicator class
 782 #
 783 #  This class makes use of thread to print progress on console.
 784 #
 785 class Progressor:
 786     # for avoiding deadloop
 787     _StopFlag = None
 788     _ProgressThread = None
 789     _CheckInterval = 0.25
 790
 791     ## Constructor
 792     #
 793     #   @param      OpenMessage     The string printed before progress charaters
 794     #   @param      CloseMessage    The string printed after progress charaters
 795     #   @param      ProgressChar    The charater used to indicate the progress
 796     #   @param      Interval        The interval in seconds between two progress charaters
 797     #
 798     def __init__(self, OpenMessage="", CloseMessage="", ProgressChar='.', Interval=1.0):
 799         self.PromptMessage = OpenMessage
 800         self.CodaMessage = CloseMessage
 801         self.ProgressChar = ProgressChar
 802         self.Interval = Interval
 803         if Progressor._StopFlag is None:
 804             Progressor._StopFlag = threading.Event()
 805
 806     ## Start to print progress charater
 807     #
 808     #   @param      OpenMessage     The string printed before progress charaters
 809     #
 810     def Start(self, OpenMessage=None):
 811         if OpenMessage is not None:
 812             self.PromptMessage = OpenMessage
 813         Progressor._StopFlag.clear()
 814         if Progressor._ProgressThread is None:
 815             Progressor._ProgressThread = threading.Thread(target=self._ProgressThreadEntry)
 816             Progressor._ProgressThread.setDaemon(False)
 817             Progressor._ProgressThread.start()
 818
 819     ## Stop printing progress charater
 820     #
 821     #   @param      CloseMessage    The string printed after progress charaters
 822     #
 823     def Stop(self, CloseMessage=None):
 824         OriginalCodaMessage = self.CodaMessage
 825         if CloseMessage is not None:
 826             self.CodaMessage = CloseMessage
 827         self.Abort()
 828         self.CodaMessage = OriginalCodaMessage
 829
 830     ## Thread entry method
 831     def _ProgressThreadEntry(self):
 832         sys.stdout.write(self.PromptMessage + " ")
 833         sys.stdout.flush()
 834         TimeUp = 0.0
 835         while not Progressor._StopFlag.isSet():
 836             if TimeUp <= 0.0:
 837                 sys.stdout.write(self.ProgressChar)
 838                 sys.stdout.flush()
 839                 TimeUp = self.Interval
 840             time.sleep(self._CheckInterval)
 841             TimeUp -= self._CheckInterval
 842         sys.stdout.write(" " + self.CodaMessage + "\n")
 843         sys.stdout.flush()
 844
 845     ## Abort the progress display
 846     @staticmethod
 847     def Abort():
 848         if Progressor._StopFlag is not None:
 849             Progressor._StopFlag.set()
 850         if Progressor._ProgressThread is not None:
 851             Progressor._ProgressThread.join()
 852             Progressor._ProgressThread = None
 853
 854 ## A dict which can access its keys and/or values orderly
 855 #
 856 #  The class implements a new kind of dict which its keys or values can be
 857 #  accessed in the order they are added into the dict. It guarantees the order
 858 #  by making use of an internal list to keep a copy of keys.
 859 #
 860 class sdict(IterableUserDict):
 861     ## Constructor
 862     def __init__(self):
 863         IterableUserDict.__init__(self)
 864         self._key_list = []
 865
 866     ## [] operator
 867     def __setitem__(self, key, value):
 868         if key not in self._key_list:
 869             self._key_list.append(key)
 870         IterableUserDict.__setitem__(self, key, value)
 871
 872     ## del operator
 873     def __delitem__(self, key):
 874         self._key_list.remove(key)
 875         IterableUserDict.__delitem__(self, key)
 876
 877     ## used in "for k in dict" loop to ensure the correct order
 878     def __iter__(self):
 879         return self.iterkeys()
 880
 881     ## len() support
 882     def __len__(self):
 883         return len(self._key_list)
 884
 885     ## "in" test support
 886     def __contains__(self, key):
 887         return key in self._key_list
 888
 889     ## indexof support
 890     def index(self, key):
 891         return self._key_list.index(key)
 892
 893     ## insert support
 894     def insert(self, key, newkey, newvalue, order):
 895         index = self._key_list.index(key)
 896         if order == 'BEFORE':
 897             self._key_list.insert(index, newkey)
 898             IterableUserDict.__setitem__(self, newkey, newvalue)
 899         elif order == 'AFTER':
 900             self._key_list.insert(index + 1, newkey)
 901             IterableUserDict.__setitem__(self, newkey, newvalue)
 902
 903     ## append support
 904     def append(self, sdict):
 905         for key in sdict:
 906             if key not in self._key_list:
 907                 self._key_list.append(key)
 908             IterableUserDict.__setitem__(self, key, sdict[key])
 909
 910     def has_key(self, key):
 911         return key in self._key_list
 912
 913     ## Empty the dict
 914     def clear(self):
 915         self._key_list = []
 916         IterableUserDict.clear(self)
 917
 918     ## Return a copy of keys
 919     def keys(self):
 920         keys = []
 921         for key in self._key_list:
 922             keys.append(key)
 923         return keys
 924
 925     ## Return a copy of values
 926     def values(self):
 927         values = []
 928         for key in self._key_list:
 929             values.append(self[key])
 930         return values
 931
 932     ## Return a copy of (key, value) list
 933     def items(self):
 934         items = []
 935         for key in self._key_list:
 936             items.append((key, self[key]))
 937         return items
 938
 939     ## Iteration support
 940     def iteritems(self):
 941         return iter(self.items())
 942
 943     ## Keys interation support
 944     def iterkeys(self):
 945         return iter(self.keys())
 946
 947     ## Values interation support
 948     def itervalues(self):
 949         return iter(self.values())
 950
 951     ## Return value related to a key, and remove the (key, value) from the dict
 952     def pop(self, key, *dv):
 953         value = None
 954         if key in self._key_list:
 955             value = self[key]
 956             self.__delitem__(key)
 957         elif len(dv) != 0 :
 958             value = kv[0]
 959         return value
 960
 961     ## Return (key, value) pair, and remove the (key, value) from the dict
 962     def popitem(self):
 963         key = self._key_list[-1]
 964         value = self[key]
 965         self.__delitem__(key)
 966         return key, value
 967
 968     def update(self, dict=None, **kwargs):
 969         if dict is not None:
 970             for k, v in dict.items():
 971                 self[k] = v
 972         if len(kwargs):
 973             for k, v in kwargs.items():
 974                 self[k] = v
 975
 976 ## Dictionary with restricted keys
 977 #
 978 class rdict(dict):
 979     ## Constructor
 980     def __init__(self, KeyList):
 981         for Key in KeyList:
 982             dict.__setitem__(self, Key, "")
 983
 984     ## []= operator
 985     def __setitem__(self, key, value):
 986         if key not in self:
 987             EdkLogger.error("RestrictedDict", ATTRIBUTE_SET_FAILURE, "Key [%s] is not allowed" % key,
 988                             ExtraData=", ".join(dict.keys(self)))
 989         dict.__setitem__(self, key, value)
 990
 991     ## =[] operator
 992     def __getitem__(self, key):
 993         if key not in self:
 994             return ""
 995         return dict.__getitem__(self, key)
 996
 997     ## del operator
 998     def __delitem__(self, key):
 999         EdkLogger.error("RestrictedDict", ATTRIBUTE_ACCESS_DENIED, ExtraData="del")
1000
1001     ## Empty the dict
1002     def clear(self):
1003         for Key in self:
1004             self.__setitem__(Key, "")
1005
1006     ## Return value related to a key, and remove the (key, value) from the dict
1007     def pop(self, key, *dv):
1008         EdkLogger.error("RestrictedDict", ATTRIBUTE_ACCESS_DENIED, ExtraData="pop")
1009
1010     ## Return (key, value) pair, and remove the (key, value) from the dict
1011     def popitem(self):
1012         EdkLogger.error("RestrictedDict", ATTRIBUTE_ACCESS_DENIED, ExtraData="popitem")
1013
1014 ## Dictionary using prioritized list as key
1015 #
1016 class tdict:
1017     _ListType = type([])
1018     _TupleType = type(())
1019     _Wildcard = 'COMMON'
1020     _ValidWildcardList = ['COMMON', 'DEFAULT', 'ALL', TAB_STAR, 'PLATFORM']
1021
1022     def __init__(self, _Single_=False, _Level_=2):
1023         self._Level_ = _Level_
1024         self.data = {}
1025         self._Single_ = _Single_
1026
1027     # =[] operator
1028     def __getitem__(self, key):
1029         KeyType = type(key)
1030         RestKeys = None
1031         if KeyType == self._ListType or KeyType == self._TupleType:
1032             FirstKey = key[0]
1033             if len(key) > 1:
1034                 RestKeys = key[1:]
1035             elif self._Level_ > 1:
1036                 RestKeys = [self._Wildcard for i in range(0, self._Level_ - 1)]
1037         else:
1038             FirstKey = key
1039             if self._Level_ > 1:
1040                 RestKeys = [self._Wildcard for i in range(0, self._Level_ - 1)]
1041
1042         if FirstKey is None or str(FirstKey).upper() in self._ValidWildcardList:
1043             FirstKey = self._Wildcard
1044
1045         if self._Single_:
1046             return self._GetSingleValue(FirstKey, RestKeys)
1047         else:
1048             return self._GetAllValues(FirstKey, RestKeys)
1049
1050     def _GetSingleValue(self, FirstKey, RestKeys):
1051         Value = None
1052         #print "%s-%s" % (FirstKey, self._Level_) ,
1053         if self._Level_ > 1:
1054             if FirstKey == self._Wildcard:
1055                 if FirstKey in self.data:
1056                     Value = self.data[FirstKey][RestKeys]
1057                 if Value is None:
1058                     for Key in self.data:
1059                         Value = self.data[Key][RestKeys]
1060                         if Value is not None: break
1061             else:
1062                 if FirstKey in self.data:
1063                     Value = self.data[FirstKey][RestKeys]
1064                 if Value is None and self._Wildcard in self.data:
1065                     #print "Value=None"
1066                     Value = self.data[self._Wildcard][RestKeys]
1067         else:
1068             if FirstKey == self._Wildcard:
1069                 if FirstKey in self.data:
1070                     Value = self.data[FirstKey]
1071                 if Value is None:
1072                     for Key in self.data:
1073                         Value = self.data[Key]
1074                         if Value is not None: break
1075             else:
1076                 if FirstKey in self.data:
1077                     Value = self.data[FirstKey]
1078                 elif self._Wildcard in self.data:
1079                     Value = self.data[self._Wildcard]
1080         return Value
1081
1082     def _GetAllValues(self, FirstKey, RestKeys):
1083         Value = []
1084         if self._Level_ > 1:
1085             if FirstKey == self._Wildcard:
1086                 for Key in self.data:
1087                     Value += self.data[Key][RestKeys]
1088             else:
1089                 if FirstKey in self.data:
1090                     Value += self.data[FirstKey][RestKeys]
1091                 if self._Wildcard in self.data:
1092                     Value += self.data[self._Wildcard][RestKeys]
1093         else:
1094             if FirstKey == self._Wildcard:
1095                 for Key in self.data:
1096                     Value.append(self.data[Key])
1097             else:
1098                 if FirstKey in self.data:
1099                     Value.append(self.data[FirstKey])
1100                 if self._Wildcard in self.data:
1101                     Value.append(self.data[self._Wildcard])
1102         return Value
1103
1104     ## []= operator
1105     def __setitem__(self, key, value):
1106         KeyType = type(key)
1107         RestKeys = None
1108         if KeyType == self._ListType or KeyType == self._TupleType:
1109             FirstKey = key[0]
1110             if len(key) > 1:
1111                 RestKeys = key[1:]
1112             else:
1113                 RestKeys = [self._Wildcard for i in range(0, self._Level_ - 1)]
1114         else:
1115             FirstKey = key
1116             if self._Level_ > 1:
1117                 RestKeys = [self._Wildcard for i in range(0, self._Level_ - 1)]
1118
1119         if FirstKey in self._ValidWildcardList:
1120             FirstKey = self._Wildcard
1121
1122         if FirstKey not in self.data and self._Level_ > 0:
1123             self.data[FirstKey] = tdict(self._Single_, self._Level_ - 1)
1124
1125         if self._Level_ > 1:
1126             self.data[FirstKey][RestKeys] = value
1127         else:
1128             self.data[FirstKey] = value
1129
1130     def SetGreedyMode(self):
1131         self._Single_ = False
1132         if self._Level_ > 1:
1133             for Key in self.data:
1134                 self.data[Key].SetGreedyMode()
1135
1136     def SetSingleMode(self):
1137         self._Single_ = True
1138         if self._Level_ > 1:
1139             for Key in self.data:
1140                 self.data[Key].SetSingleMode()
1141
1142     def GetKeys(self, KeyIndex=0):
1143         assert KeyIndex >= 0
1144         if KeyIndex == 0:
1145             return set(self.data.keys())
1146         else:
1147             keys = set()
1148             for Key in self.data:
1149                 keys |= self.data[Key].GetKeys(KeyIndex - 1)
1150             return keys
1151
1152 def AnalyzePcdExpression(Setting):
1153     RanStr = ''.join(sample(string.ascii_letters + string.digits, 8))
1154     Setting = Setting.replace('\\\\', RanStr).strip()
1155     # There might be escaped quote in a string: \", \\\" , \', \\\'
1156     Data = Setting
1157     # There might be '|' in string and in ( ... | ... ), replace it with '-'
1158     NewStr = ''
1159     InSingleQuoteStr = False
1160     InDoubleQuoteStr = False
1161     Pair = 0
1162     for Index, ch in enumerate(Data):
1163         if ch == '"' and not InSingleQuoteStr:
1164             if Data[Index - 1] != '\\':
1165                 InDoubleQuoteStr = not InDoubleQuoteStr
1166         elif ch == "'" and not InDoubleQuoteStr:
1167             if Data[Index - 1] != '\\':
1168                 InSingleQuoteStr = not InSingleQuoteStr
1169         elif ch == '(' and not (InSingleQuoteStr or InDoubleQuoteStr):
1170             Pair += 1
1171         elif ch == ')' and not (InSingleQuoteStr or InDoubleQuoteStr):
1172             Pair -= 1
1173
1174         if (Pair > 0 or InSingleQuoteStr or InDoubleQuoteStr) and ch == TAB_VALUE_SPLIT:
1175             NewStr += '-'
1176         else:
1177             NewStr += ch
1178     FieldList = []
1179     StartPos = 0
1180     while True:
1181         Pos = NewStr.find(TAB_VALUE_SPLIT, StartPos)
1182         if Pos < 0:
1183             FieldList.append(Setting[StartPos:].strip())
1184             break
1185         FieldList.append(Setting[StartPos:Pos].strip())
1186         StartPos = Pos + 1
1187     for i, ch in enumerate(FieldList):
1188         if RanStr in ch:
1189             FieldList[i] = ch.replace(RanStr,'\\\\')
1190     return FieldList
1191
1192 def ParseDevPathValue (Value):
1193     if '\\' in Value:
1194         Value.replace('\\', '/').replace(' ', '')
1195
1196     Cmd = 'DevicePath ' + '"' + Value + '"'
1197     try:
1198         p = subprocess.Popen(Cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
1199         out, err = p.communicate()
1200     except Exception as X:
1201         raise BadExpression("DevicePath: %s" % (str(X)) )
1202     finally:
1203         subprocess._cleanup()
1204         p.stdout.close()
1205         p.stderr.close()
1206     if err:
1207         raise BadExpression("DevicePath: %s" % str(err))
1208     Size = len(out.split())
1209     out = ','.join(out.split())
1210     return '{' + out + '}', Size
1211
1212 def ParseFieldValue (Value):
1213     if "{CODE(" in Value:
1214         return Value, len(Value.split(","))
1215     if isinstance(Value, type(0)):
1216         return Value, (Value.bit_length() + 7) / 8
1217     if not isinstance(Value, type('')):
1218         raise BadExpression('Type %s is %s' %(Value, type(Value)))
1219     Value = Value.strip()
1220     if Value.startswith(TAB_UINT8) and Value.endswith(')'):
1221         Value, Size = ParseFieldValue(Value.split('(', 1)[1][:-1])
1222         if Size > 1:
1223             raise BadExpression('Value (%s) Size larger than %d' %(Value, Size))
1224         return Value, 1
1225     if Value.startswith(TAB_UINT16) and Value.endswith(')'):
1226         Value, Size = ParseFieldValue(Value.split('(', 1)[1][:-1])
1227         if Size > 2:
1228             raise BadExpression('Value (%s) Size larger than %d' %(Value, Size))
1229         return Value, 2
1230     if Value.startswith(TAB_UINT32) and Value.endswith(')'):
1231         Value, Size = ParseFieldValue(Value.split('(', 1)[1][:-1])
1232         if Size > 4:
1233             raise BadExpression('Value (%s) Size larger than %d' %(Value, Size))
1234         return Value, 4
1235     if Value.startswith(TAB_UINT64) and Value.endswith(')'):
1236         Value, Size = ParseFieldValue(Value.split('(', 1)[1][:-1])
1237         if Size > 8:
1238             raise BadExpression('Value (%s) Size larger than %d' % (Value, Size))
1239         return Value, 8
1240     if Value.startswith(TAB_GUID) and Value.endswith(')'):
1241         Value = Value.split('(', 1)[1][:-1].strip()
1242         if Value[0] == '{' and Value[-1] == '}':
1243             TmpValue = GuidStructureStringToGuidString(Value)
1244             if not TmpValue:
1245                 raise BadExpression("Invalid GUID value string %s" % Value)
1246             Value = TmpValue
1247         if Value[0] == '"' and Value[-1] == '"':
1248             Value = Value[1:-1]
1249         try:
1250             Value = "'" + uuid.UUID(Value).get_bytes_le() + "'"
1251         except ValueError as Message:
1252             raise BadExpression(Message)
1253         Value, Size = ParseFieldValue(Value)
1254         return Value, 16
1255     if Value.startswith('L"') and Value.endswith('"'):
1256         # Unicode String
1257         # translate escape character
1258         Value = Value[1:]
1259         try:
1260             Value = eval(Value)
1261         except:
1262             Value = Value[1:-1]
1263         List = list(Value)
1264         List.reverse()
1265         Value = 0
1266         for Char in List:
1267             Value = (Value << 16) | ord(Char)
1268         return Value, (len(List) + 1) * 2
1269     if Value.startswith('"') and Value.endswith('"'):
1270         # ASCII String
1271         # translate escape character
1272         try:
1273             Value = eval(Value)
1274         except:
1275             Value = Value[1:-1]
1276         List = list(Value)
1277         List.reverse()
1278         Value = 0
1279         for Char in List:
1280             Value = (Value << 8) | ord(Char)
1281         return Value, len(List) + 1
1282     if Value.startswith("L'") and Value.endswith("'"):
1283         # Unicode Character Constant
1284         # translate escape character
1285         Value = Value[1:]
1286         try:
1287             Value = eval(Value)
1288         except:
1289             Value = Value[1:-1]
1290         List = list(Value)
1291         if len(List) == 0:
1292             raise BadExpression('Length %s is %s' % (Value, len(List)))
1293         List.reverse()
1294         Value = 0
1295         for Char in List:
1296             Value = (Value << 16) | ord(Char)
1297         return Value, len(List) * 2
1298     if Value.startswith("'") and Value.endswith("'"):
1299         # Character constant
1300         # translate escape character
1301         try:
1302             Value = eval(Value)
1303         except:
1304             Value = Value[1:-1]
1305         List = list(Value)
1306         if len(List) == 0:
1307             raise BadExpression('Length %s is %s' % (Value, len(List)))
1308         List.reverse()
1309         Value = 0
1310         for Char in List:
1311             Value = (Value << 8) | ord(Char)
1312         return Value, len(List)
1313     if Value.startswith('{') and Value.endswith('}'):
1314         # Byte array
1315         Value = Value[1:-1]
1316         List = [Item.strip() for Item in Value.split(',')]
1317         List.reverse()
1318         Value = 0
1319         RetSize = 0
1320         for Item in List:
1321             ItemValue, Size = ParseFieldValue(Item)
1322             RetSize += Size
1323             for I in range(Size):
1324                 Value = (Value << 8) | ((ItemValue >> 8 * I) & 0xff)
1325         return Value, RetSize
1326     if Value.startswith('DEVICE_PATH(') and Value.endswith(')'):
1327         Value = Value.replace("DEVICE_PATH(", '').rstrip(')')
1328         Value = Value.strip().strip('"')
1329         return ParseDevPathValue(Value)
1330     if Value.lower().startswith('0x'):
1331         try:
1332             Value = int(Value, 16)
1333         except:
1334             raise BadExpression("invalid hex value: %s" % Value)
1335         if Value == 0:
1336             return 0, 1
1337         return Value, (Value.bit_length() + 7) / 8
1338     if Value[0].isdigit():
1339         Value = int(Value, 10)
1340         if Value == 0:
1341             return 0, 1
1342         return Value, (Value.bit_length() + 7) / 8
1343     if Value.lower() == 'true':
1344         return 1, 1
1345     if Value.lower() == 'false':
1346         return 0, 1
1347     return Value, 1
1348
1349 ## AnalyzeDscPcd
1350 #
1351 #  Analyze DSC PCD value, since there is no data type info in DSC
1352 #  This function is used to match functions (AnalyzePcdData) used for retrieving PCD value from database
1353 #  1. Feature flag: TokenSpace.PcdCName|PcdValue
1354 #  2. Fix and Patch:TokenSpace.PcdCName|PcdValue[|VOID*[|MaxSize]]
1355 #  3. Dynamic default:
1356 #     TokenSpace.PcdCName|PcdValue[|VOID*[|MaxSize]]
1357 #     TokenSpace.PcdCName|PcdValue
1358 #  4. Dynamic VPD:
1359 #     TokenSpace.PcdCName|VpdOffset[|VpdValue]
1360 #     TokenSpace.PcdCName|VpdOffset[|MaxSize[|VpdValue]]
1361 #  5. Dynamic HII:
1362 #     TokenSpace.PcdCName|HiiString|VariableGuid|VariableOffset[|HiiValue]
1363 #  PCD value needs to be located in such kind of string, and the PCD value might be an expression in which
1364 #    there might have "|" operator, also in string value.
1365 #
1366 #  @param Setting: String contain information described above with "TokenSpace.PcdCName|" stripped
1367 #  @param PcdType: PCD type: feature, fixed, dynamic default VPD HII
1368 #  @param DataType: The datum type of PCD: VOID*, UNIT, BOOL
1369 #  @retval:
1370 #    ValueList: A List contain fields described above
1371 #    IsValid:   True if conforming EBNF, otherwise False
1372 #    Index:     The index where PcdValue is in ValueList
1373 #
1374 def AnalyzeDscPcd(Setting, PcdType, DataType=''):
1375     FieldList = AnalyzePcdExpression(Setting)
1376
1377     IsValid = True
1378     if PcdType in (MODEL_PCD_FIXED_AT_BUILD, MODEL_PCD_PATCHABLE_IN_MODULE, MODEL_PCD_DYNAMIC_DEFAULT, MODEL_PCD_DYNAMIC_EX_DEFAULT):
1379         Value = FieldList[0]
1380         Size = ''
1381         if len(FieldList) > 1 and FieldList[1]:
1382             DataType = FieldList[1]
1383             if FieldList[1] != TAB_VOID and StructPattern.match(FieldList[1]) is None:
1384                 IsValid = False
1385         if len(FieldList) > 2:
1386             Size = FieldList[2]
1387         if IsValid:
1388             if DataType == "":
1389                 IsValid = (len(FieldList) <= 1)
1390             else:
1391                 IsValid = (len(FieldList) <= 3)
1392
1393         if Size:
1394             try:
1395                 int(Size, 16) if Size.upper().startswith("0X") else int(Size)
1396             except:
1397                 IsValid = False
1398                 Size = -1
1399         return [str(Value), DataType, str(Size)], IsValid, 0
1400     elif PcdType == MODEL_PCD_FEATURE_FLAG:
1401         Value = FieldList[0]
1402         Size = ''
1403         IsValid = (len(FieldList) <= 1)
1404         return [Value, DataType, str(Size)], IsValid, 0
1405     elif PcdType in (MODEL_PCD_DYNAMIC_VPD, MODEL_PCD_DYNAMIC_EX_VPD):
1406         VpdOffset = FieldList[0]
1407         Value = Size = ''
1408         if not DataType == TAB_VOID:
1409             if len(FieldList) > 1:
1410                 Value = FieldList[1]
1411         else:
1412             if len(FieldList) > 1:
1413                 Size = FieldList[1]
1414             if len(FieldList) > 2:
1415                 Value = FieldList[2]
1416         if DataType == "":
1417             IsValid = (len(FieldList) <= 1)
1418         else:
1419             IsValid = (len(FieldList) <= 3)
1420         if Size:
1421             try:
1422                 int(Size, 16) if Size.upper().startswith("0X") else int(Size)
1423             except:
1424                 IsValid = False
1425                 Size = -1
1426         return [VpdOffset, str(Size), Value], IsValid, 2
1427     elif PcdType in (MODEL_PCD_DYNAMIC_HII, MODEL_PCD_DYNAMIC_EX_HII):
1428         IsValid = (3 <= len(FieldList) <= 5)
1429         HiiString = FieldList[0]
1430         Guid = Offset = Value = Attribute = ''
1431         if len(FieldList) > 1:
1432             Guid = FieldList[1]
1433         if len(FieldList) > 2:
1434             Offset = FieldList[2]
1435         if len(FieldList) > 3:
1436             Value = FieldList[3]
1437         if len(FieldList) > 4:
1438             Attribute = FieldList[4]
1439         return [HiiString, Guid, Offset, Value, Attribute], IsValid, 3
1440     return [], False, 0
1441
1442 ## AnalyzePcdData
1443 #
1444 #  Analyze the pcd Value, Datum type and TokenNumber.
1445 #  Used to avoid split issue while the value string contain "|" character
1446 #
1447 #  @param[in] Setting:  A String contain value/datum type/token number information;
1448 #
1449 #  @retval   ValueList: A List contain value, datum type and toke number.
1450 #
1451 def AnalyzePcdData(Setting):
1452     ValueList = ['', '', '']
1453
1454     ValueRe = re.compile(r'^\s*L?\".*\|.*\"')
1455     PtrValue = ValueRe.findall(Setting)
1456
1457     ValueUpdateFlag = False
1458
1459     if len(PtrValue) >= 1:
1460         Setting = re.sub(ValueRe, '', Setting)
1461         ValueUpdateFlag = True
1462
1463     TokenList = Setting.split(TAB_VALUE_SPLIT)
1464     ValueList[0:len(TokenList)] = TokenList
1465
1466     if ValueUpdateFlag:
1467         ValueList[0] = PtrValue[0]
1468
1469     return ValueList
1470
1471 ## check format of PCD value against its the datum type
1472 #
1473 # For PCD value setting
1474 #
1475 def CheckPcdDatum(Type, Value):
1476     if Type == TAB_VOID:
1477         ValueRe = re.compile(r'\s*L?\".*\"\s*$')
1478         if not (((Value.startswith('L"') or Value.startswith('"')) and Value.endswith('"'))
1479                 or (Value.startswith('{') and Value.endswith('}')) or (Value.startswith("L'") or Value.startswith("'") and Value.endswith("'"))
1480                ):
1481             return False, "Invalid value [%s] of type [%s]; must be in the form of {...} for array"\
1482                           ", \"...\" or \'...\' for string, L\"...\" or L\'...\' for unicode string" % (Value, Type)
1483         elif ValueRe.match(Value):
1484             # Check the chars in UnicodeString or CString is printable
1485             if Value.startswith("L"):
1486                 Value = Value[2:-1]
1487             else:
1488                 Value = Value[1:-1]
1489             Printset = set(string.printable)
1490             Printset.remove(TAB_PRINTCHAR_VT)
1491             Printset.add(TAB_PRINTCHAR_BS)
1492             Printset.add(TAB_PRINTCHAR_NUL)
1493             if not set(Value).issubset(Printset):
1494                 PrintList = sorted(Printset)
1495                 return False, "Invalid PCD string value of type [%s]; must be printable chars %s." % (Type, PrintList)
1496     elif Type == 'BOOLEAN':
1497         if Value not in ['TRUE', 'True', 'true', '0x1', '0x01', '1', 'FALSE', 'False', 'false', '0x0', '0x00', '0']:
1498             return False, "Invalid value [%s] of type [%s]; must be one of TRUE, True, true, 0x1, 0x01, 1"\
1499                           ", FALSE, False, false, 0x0, 0x00, 0" % (Value, Type)
1500     elif Type in [TAB_UINT8, TAB_UINT16, TAB_UINT32, TAB_UINT64]:
1501         if Value and int(Value, 0) < 0:
1502             return False, "PCD can't be set to negative value[%s] for datum type [%s]" % (Value, Type)
1503         try:
1504             Value = long(Value, 0)
1505             if Value > MAX_VAL_TYPE[Type]:
1506                 return False, "Too large PCD value[%s] for datum type [%s]" % (Value, Type)
1507         except:
1508             return False, "Invalid value [%s] of type [%s];"\
1509                           " must be a hexadecimal, decimal or octal in C language format." % (Value, Type)
1510     else:
1511         return True, "StructurePcd"
1512
1513     return True, ""
1514
1515 def CommonPath(PathList):
1516     P1 = min(PathList).split(os.path.sep)
1517     P2 = max(PathList).split(os.path.sep)
1518     for Index in xrange(min(len(P1), len(P2))):
1519         if P1[Index] != P2[Index]:
1520             return os.path.sep.join(P1[:Index])
1521     return os.path.sep.join(P1)
1522
1523 class PathClass(object):
1524     def __init__(self, File='', Root='', AlterRoot='', Type='', IsBinary=False,
1525                  Arch='COMMON', ToolChainFamily='', Target='', TagName='', ToolCode=''):
1526         self.Arch = Arch
1527         self.File = str(File)
1528         if os.path.isabs(self.File):
1529             self.Root = ''
1530             self.AlterRoot = ''
1531         else:
1532             self.Root = str(Root)
1533             self.AlterRoot = str(AlterRoot)
1534
1535         # Remove any '.' and '..' in path
1536         if self.Root:
1537             self.Root = mws.getWs(self.Root, self.File)
1538             self.Path = os.path.normpath(os.path.join(self.Root, self.File))
1539             self.Root = os.path.normpath(CommonPath([self.Root, self.Path]))
1540             # eliminate the side-effect of 'C:'
1541             if self.Root[-1] == ':':
1542                 self.Root += os.path.sep
1543             # file path should not start with path separator
1544             if self.Root[-1] == os.path.sep:
1545                 self.File = self.Path[len(self.Root):]
1546             else:
1547                 self.File = self.Path[len(self.Root) + 1:]
1548         else:
1549             self.Path = os.path.normpath(self.File)
1550
1551         self.SubDir, self.Name = os.path.split(self.File)
1552         self.BaseName, self.Ext = os.path.splitext(self.Name)
1553
1554         if self.Root:
1555             if self.SubDir:
1556                 self.Dir = os.path.join(self.Root, self.SubDir)
1557             else:
1558                 self.Dir = self.Root
1559         else:
1560             self.Dir = self.SubDir
1561
1562         if IsBinary:
1563             self.Type = Type
1564         else:
1565             self.Type = self.Ext.lower()
1566
1567         self.IsBinary = IsBinary
1568         self.Target = Target
1569         self.TagName = TagName
1570         self.ToolCode = ToolCode
1571         self.ToolChainFamily = ToolChainFamily
1572
1573     ## Convert the object of this class to a string
1574     #
1575     #  Convert member Path of the class to a string
1576     #
1577     #  @retval string Formatted String
1578     #
1579     def __str__(self):
1580         return self.Path
1581
1582     ## Override __eq__ function
1583     #
1584     # Check whether PathClass are the same
1585     #
1586     # @retval False The two PathClass are different
1587     # @retval True  The two PathClass are the same
1588     #
1589     def __eq__(self, Other):
1590         return self.Path == str(Other)
1591
1592     ## Override __cmp__ function
1593     #
1594     # Customize the comparsion operation of two PathClass
1595     #
1596     # @retval 0     The two PathClass are different
1597     # @retval -1    The first PathClass is less than the second PathClass
1598     # @retval 1     The first PathClass is Bigger than the second PathClass
1599     def __cmp__(self, Other):
1600         OtherKey = str(Other)
1601
1602         SelfKey = self.Path
1603         if SelfKey == OtherKey:
1604             return 0
1605         elif SelfKey > OtherKey:
1606             return 1
1607         else:
1608             return -1
1609
1610     ## Override __hash__ function
1611     #
1612     # Use Path as key in hash table
1613     #
1614     # @retval string Key for hash table
1615     #
1616     def __hash__(self):
1617         return hash(self.Path)
1618
1619     @cached_property
1620     def Key(self):
1621         return self.Path.upper()
1622
1623     @property
1624     def TimeStamp(self):
1625         return os.stat(self.Path)[8]
1626
1627     def Validate(self, Type='', CaseSensitive=True):
1628         if GlobalData.gCaseInsensitive:
1629             CaseSensitive = False
1630         if Type and Type.lower() != self.Type:
1631             return FILE_TYPE_MISMATCH, '%s (expect %s but got %s)' % (self.File, Type, self.Type)
1632
1633         RealFile, RealRoot = RealPath2(self.File, self.Root, self.AlterRoot)
1634         if not RealRoot and not RealFile:
1635             RealFile = self.File
1636             if self.AlterRoot:
1637                 RealFile = os.path.join(self.AlterRoot, self.File)
1638             elif self.Root:
1639                 RealFile = os.path.join(self.Root, self.File)
1640             if len (mws.getPkgPath()) == 0:
1641                 return FILE_NOT_FOUND, os.path.join(self.AlterRoot, RealFile)
1642             else:
1643                 return FILE_NOT_FOUND, "%s is not found in packages path:\n\t%s" % (self.File, '\n\t'.join(mws.getPkgPath()))
1644
1645         ErrorCode = 0
1646         ErrorInfo = ''
1647         if RealRoot != self.Root or RealFile != self.File:
1648             if CaseSensitive and (RealFile != self.File or (RealRoot != self.Root and RealRoot != self.AlterRoot)):
1649                 ErrorCode = FILE_CASE_MISMATCH
1650                 ErrorInfo = self.File + '\n\t' + RealFile + " [in file system]"
1651
1652             self.SubDir, self.Name = os.path.split(RealFile)
1653             self.BaseName, self.Ext = os.path.splitext(self.Name)
1654             if self.SubDir:
1655                 self.Dir = os.path.join(RealRoot, self.SubDir)
1656             else:
1657                 self.Dir = RealRoot
1658             self.File = RealFile
1659             self.Root = RealRoot
1660             self.Path = os.path.join(RealRoot, RealFile)
1661         return ErrorCode, ErrorInfo
1662
1663 ## Parse PE image to get the required PE informaion.
1664 #
1665 class PeImageClass():
1666     ## Constructor
1667     #
1668     #   @param  File FilePath of PeImage
1669     #
1670     def __init__(self, PeFile):
1671         self.FileName   = PeFile
1672         self.IsValid    = False
1673         self.Size       = 0
1674         self.EntryPoint = 0
1675         self.SectionAlignment  = 0
1676         self.SectionHeaderList = []
1677         self.ErrorInfo = ''
1678         try:
1679             PeObject = open(PeFile, 'rb')
1680         except:
1681             self.ErrorInfo = self.FileName + ' can not be found\n'
1682             return
1683         # Read DOS header
1684         ByteArray = array.array('B')
1685         ByteArray.fromfile(PeObject, 0x3E)
1686         ByteList = ByteArray.tolist()
1687         # DOS signature should be 'MZ'
1688         if self._ByteListToStr (ByteList[0x0:0x2]) != 'MZ':
1689             self.ErrorInfo = self.FileName + ' has no valid DOS signature MZ'
1690             return
1691
1692         # Read 4 byte PE Signature
1693         PeOffset = self._ByteListToInt(ByteList[0x3C:0x3E])
1694         PeObject.seek(PeOffset)
1695         ByteArray = array.array('B')
1696         ByteArray.fromfile(PeObject, 4)
1697         # PE signature should be 'PE\0\0'
1698         if ByteArray.tostring() != 'PE\0\0':
1699             self.ErrorInfo = self.FileName + ' has no valid PE signature PE00'
1700             return
1701
1702         # Read PE file header
1703         ByteArray = array.array('B')
1704         ByteArray.fromfile(PeObject, 0x14)
1705         ByteList = ByteArray.tolist()
1706         SecNumber = self._ByteListToInt(ByteList[0x2:0x4])
1707         if SecNumber == 0:
1708             self.ErrorInfo = self.FileName + ' has no section header'
1709             return
1710
1711         # Read PE optional header
1712         OptionalHeaderSize = self._ByteListToInt(ByteArray[0x10:0x12])
1713         ByteArray = array.array('B')
1714         ByteArray.fromfile(PeObject, OptionalHeaderSize)
1715         ByteList = ByteArray.tolist()
1716         self.EntryPoint       = self._ByteListToInt(ByteList[0x10:0x14])
1717         self.SectionAlignment = self._ByteListToInt(ByteList[0x20:0x24])
1718         self.Size             = self._ByteListToInt(ByteList[0x38:0x3C])
1719
1720         # Read each Section Header
1721         for Index in range(SecNumber):
1722             ByteArray = array.array('B')
1723             ByteArray.fromfile(PeObject, 0x28)
1724             ByteList = ByteArray.tolist()
1725             SecName  = self._ByteListToStr(ByteList[0:8])
1726             SecVirtualSize = self._ByteListToInt(ByteList[8:12])
1727             SecRawAddress  = self._ByteListToInt(ByteList[20:24])
1728             SecVirtualAddress = self._ByteListToInt(ByteList[12:16])
1729             self.SectionHeaderList.append((SecName, SecVirtualAddress, SecRawAddress, SecVirtualSize))
1730         self.IsValid = True
1731         PeObject.close()
1732
1733     def _ByteListToStr(self, ByteList):
1734         String = ''
1735         for index in range(len(ByteList)):
1736             if ByteList[index] == 0:
1737                 break
1738             String += chr(ByteList[index])
1739         return String
1740
1741     def _ByteListToInt(self, ByteList):
1742         Value = 0
1743         for index in range(len(ByteList) - 1, -1, -1):
1744             Value = (Value << 8) | int(ByteList[index])
1745         return Value
1746
1747 class DefaultStore():
1748     def __init__(self, DefaultStores ):
1749
1750         self.DefaultStores = DefaultStores
1751     def DefaultStoreID(self, DefaultStoreName):
1752         for key, value in self.DefaultStores.items():
1753             if value == DefaultStoreName:
1754                 return key
1755         return None
1756     def GetDefaultDefault(self):
1757         if not self.DefaultStores or "0" in self.DefaultStores:
1758             return "0", TAB_DEFAULT_STORES_DEFAULT
1759         else:
1760             minvalue = min(int(value_str) for value_str in self.DefaultStores)
1761             return (str(minvalue), self.DefaultStores[str(minvalue)])
1762     def GetMin(self, DefaultSIdList):
1763         if not DefaultSIdList:
1764             return TAB_DEFAULT_STORES_DEFAULT
1765         storeidset = {storeid for storeid, storename in self.DefaultStores.values() if storename in DefaultSIdList}
1766         if not storeidset:
1767             return ""
1768         minid = min(storeidset )
1769         for sid, name in self.DefaultStores.values():
1770             if sid == minid:
1771                 return name
1772
1773 class SkuClass():
1774     DEFAULT = 0
1775     SINGLE = 1
1776     MULTIPLE =2
1777
1778     def __init__(self,SkuIdentifier='', SkuIds=None):
1779         if SkuIds is None:
1780             SkuIds = {}
1781
1782         for SkuName in SkuIds:
1783             SkuId = SkuIds[SkuName][0]
1784             skuid_num = int(SkuId, 16) if SkuId.upper().startswith("0X") else int(SkuId)
1785             if skuid_num > 0xFFFFFFFFFFFFFFFF:
1786                 EdkLogger.error("build", PARAMETER_INVALID,
1787                             ExtraData = "SKU-ID [%s] value %s exceeds the max value of UINT64"
1788                                       % (SkuName, SkuId))
1789
1790         self.AvailableSkuIds = sdict()
1791         self.SkuIdSet = []
1792         self.SkuIdNumberSet = []
1793         self.SkuData = SkuIds
1794         self._SkuInherit = {}
1795         self._SkuIdentifier = SkuIdentifier
1796         if SkuIdentifier == '' or SkuIdentifier is None:
1797             self.SkuIdSet = ['DEFAULT']
1798             self.SkuIdNumberSet = ['0U']
1799         elif SkuIdentifier == 'ALL':
1800             self.SkuIdSet = SkuIds.keys()
1801             self.SkuIdNumberSet = [num[0].strip() + 'U' for num in SkuIds.values()]
1802         else:
1803             r = SkuIdentifier.split('|')
1804             self.SkuIdSet=[(r[k].strip()).upper() for k in range(len(r))]
1805             k = None
1806             try:
1807                 self.SkuIdNumberSet = [SkuIds[k][0].strip() + 'U' for k in self.SkuIdSet]
1808             except Exception:
1809                 EdkLogger.error("build", PARAMETER_INVALID,
1810                             ExtraData = "SKU-ID [%s] is not supported by the platform. [Valid SKU-ID: %s]"
1811                                       % (k, " | ".join(SkuIds.keys())))
1812         for each in self.SkuIdSet:
1813             if each in SkuIds:
1814                 self.AvailableSkuIds[each] = SkuIds[each][0]
1815             else:
1816                 EdkLogger.error("build", PARAMETER_INVALID,
1817                             ExtraData="SKU-ID [%s] is not supported by the platform. [Valid SKU-ID: %s]"
1818                                       % (each, " | ".join(SkuIds.keys())))
1819         if self.SkuUsageType != SkuClass.SINGLE:
1820             self.AvailableSkuIds.update({'DEFAULT':0, 'COMMON':0})
1821         if self.SkuIdSet:
1822             GlobalData.gSkuids = (self.SkuIdSet)
1823             if 'COMMON' in GlobalData.gSkuids:
1824                 GlobalData.gSkuids.remove('COMMON')
1825             if self.SkuUsageType == self.SINGLE:
1826                 if len(GlobalData.gSkuids) != 1:
1827                     if 'DEFAULT' in GlobalData.gSkuids:
1828                         GlobalData.gSkuids.remove('DEFAULT')
1829             if GlobalData.gSkuids:
1830                 GlobalData.gSkuids.sort()
1831
1832     def GetNextSkuId(self, skuname):
1833         if not self._SkuInherit:
1834             self._SkuInherit = {}
1835             for item in self.SkuData.values():
1836                 self._SkuInherit[item[1]]=item[2] if item[2] else "DEFAULT"
1837         return self._SkuInherit.get(skuname, "DEFAULT")
1838
1839     def GetSkuChain(self, sku):
1840         if sku == "DEFAULT":
1841             return ["DEFAULT"]
1842         skulist = [sku]
1843         nextsku = sku
1844         while True:
1845             nextsku = self.GetNextSkuId(nextsku)
1846             skulist.append(nextsku)
1847             if nextsku == "DEFAULT":
1848                 break
1849         skulist.reverse()
1850         return skulist
1851     def SkuOverrideOrder(self):
1852         skuorderset = []
1853         for skuname in self.SkuIdSet:
1854             skuorderset.append(self.GetSkuChain(skuname))
1855
1856         skuorder = []
1857         for index in range(max(len(item) for item in skuorderset)):
1858             for subset in skuorderset:
1859                 if index > len(subset)-1:
1860                     continue
1861                 if subset[index] in skuorder:
1862                     continue
1863                 skuorder.append(subset[index])
1864
1865         return skuorder
1866
1867     @property
1868     def SkuUsageType(self):
1869         if self._SkuIdentifier.upper() == "ALL":
1870             return SkuClass.MULTIPLE
1871
1872         if len(self.SkuIdSet) == 1:
1873             if self.SkuIdSet[0] == 'DEFAULT':
1874                 return SkuClass.DEFAULT
1875             return SkuClass.SINGLE
1876         if len(self.SkuIdSet) == 2 and 'DEFAULT' in self.SkuIdSet:
1877             return SkuClass.SINGLE
1878         return SkuClass.MULTIPLE
1879
1880     def DumpSkuIdArrary(self):
1881         if self.SkuUsageType == SkuClass.SINGLE:
1882             return "{0x0}"
1883         ArrayStrList = []
1884         for skuname in self.AvailableSkuIds:
1885             if skuname == "COMMON":
1886                 continue
1887             while skuname != "DEFAULT":
1888                 ArrayStrList.append(hex(int(self.AvailableSkuIds[skuname])))
1889                 skuname = self.GetNextSkuId(skuname)
1890             ArrayStrList.append("0x0")
1891         return "{{{myList}}}".format(myList=",".join(ArrayStrList))
1892
1893     @property
1894     def AvailableSkuIdSet(self):
1895         return self.AvailableSkuIds
1896
1897     @property
1898     def SystemSkuId(self):
1899         if self.SkuUsageType == SkuClass.SINGLE:
1900             if len(self.SkuIdSet) == 1:
1901                 return self.SkuIdSet[0]
1902             else:
1903                 return self.SkuIdSet[0] if self.SkuIdSet[0] != 'DEFAULT' else self.SkuIdSet[1]
1904         else:
1905             return 'DEFAULT'
1906
1907 #
1908 # Pack a registry format GUID
1909 #
1910 def PackRegistryFormatGuid(Guid):
1911     return PackGUID(Guid.split('-'))
1912
1913 ##  Get the integer value from string like "14U" or integer like 2
1914 #
1915 #   @param      Input   The object that may be either a integer value or a string
1916 #
1917 #   @retval     Value    The integer value that the input represents
1918 #
1919 def GetIntegerValue(Input):
1920     if type(Input) in (int, long):
1921         return Input
1922     String = Input
1923     if String.endswith("U"):
1924         String = String[:-1]
1925     if String.endswith("ULL"):
1926         String = String[:-3]
1927     if String.endswith("LL"):
1928         String = String[:-2]
1929
1930     if String.startswith("0x") or String.startswith("0X"):
1931         return int(String, 16)
1932     elif String == '':
1933         return 0
1934     else:
1935         return int(String)
1936
1937 #
1938 # Pack a GUID (registry format) list into a buffer and return it
1939 #
1940 def PackGUID(Guid):
1941     return pack(PACK_PATTERN_GUID,
1942                 int(Guid[0], 16),
1943                 int(Guid[1], 16),
1944                 int(Guid[2], 16),
1945                 int(Guid[3][-4:-2], 16),
1946                 int(Guid[3][-2:], 16),
1947                 int(Guid[4][-12:-10], 16),
1948                 int(Guid[4][-10:-8], 16),
1949                 int(Guid[4][-8:-6], 16),
1950                 int(Guid[4][-6:-4], 16),
1951                 int(Guid[4][-4:-2], 16),
1952                 int(Guid[4][-2:], 16)
1953                 )
1954
1955 #
1956 # Pack a GUID (byte) list into a buffer and return it
1957 #
1958 def PackByteFormatGUID(Guid):
1959     return pack(PACK_PATTERN_GUID,
1960                 Guid[0],
1961                 Guid[1],
1962                 Guid[2],
1963                 Guid[3],
1964                 Guid[4],
1965                 Guid[5],
1966                 Guid[6],
1967                 Guid[7],
1968                 Guid[8],
1969                 Guid[9],
1970                 Guid[10],
1971                 )
1972
1973 ## DeepCopy dict/OrderedDict recusively
1974 #
1975 #   @param      ori_dict    a nested dict or ordereddict
1976 #
1977 #   @retval     new dict or orderdict
1978 #
1979 def CopyDict(ori_dict):
1980     dict_type = ori_dict.__class__
1981     if dict_type not in (dict,OrderedDict):
1982         return ori_dict
1983     new_dict = dict_type()
1984     for key in ori_dict:
1985         if isinstance(ori_dict[key],(dict,OrderedDict)):
1986             new_dict[key] = CopyDict(ori_dict[key])
1987         else:
1988             new_dict[key] = ori_dict[key]
1989     return new_dict
1990
1991 #
1992 # Remove the c/c++ comments: // and /* */
1993 #
1994 def RemoveCComments(ctext):
1995     return re.sub('//.*?\n|/\*.*?\*/', '\n', ctext, flags=re.S)
1996 ##
1997 #
1998 # This acts like the main() function for the script, unless it is 'import'ed into another
1999 # script.
2000 #
2001 if __name__ == '__main__':
2002     pass
2003