BaseTools/Source/Python/Common/Misc.py

   1 ## @file
   2 # Common routines used by all tools
   3 #
   4 # Copyright (c) 2007 - 2018, Intel Corporation. All rights reserved.<BR>
   5 # This program and the accompanying materials
   6 # are licensed and made available under the terms and conditions of the BSD License
   7 # which accompanies this distribution.  The full text of the license may be found at
   8 # http://opensource.org/licenses/bsd-license.php
   9 #
  10 # THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS,
  11 # WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED.
  12 #
  13
  14 ##
  15 # Import Modules
  16 #
  17 from __future__ import absolute_import
  18 import Common.LongFilePathOs as os
  19 import sys
  20 import string
  21 import threading
  22 import time
  23 import re
  24 import pickle
  25 import array
  26 import shutil
  27 from random import sample
  28 from struct import pack
  29 from UserDict import IterableUserDict
  30 from UserList import UserList
  31
  32 from Common import EdkLogger as EdkLogger
  33 from Common import GlobalData as GlobalData
  34 from .DataType import *
  35 from .BuildToolError import *
  36 from CommonDataClass.DataClass import *
  37 from .Parsing import GetSplitValueList
  38 from Common.LongFilePathSupport import OpenLongFilePath as open
  39 from Common.MultipleWorkspace import MultipleWorkspace as mws
  40 import uuid
  41 from CommonDataClass.Exceptions import BadExpression
  42 from Common.caching import cached_property
  43 import subprocess
  44 from collections import OrderedDict
  45 ## Regular expression used to find out place holders in string template
  46 gPlaceholderPattern = re.compile("\$\{([^$()\s]+)\}", re.MULTILINE | re.UNICODE)
  47
  48 ## regular expressions for map file processing
  49 startPatternGeneral = re.compile("^Start[' ']+Length[' ']+Name[' ']+Class")
  50 addressPatternGeneral = re.compile("^Address[' ']+Publics by Value[' ']+Rva\+Base")
  51 valuePatternGcc = re.compile('^([\w_\.]+) +([\da-fA-Fx]+) +([\da-fA-Fx]+)$')
  52 pcdPatternGcc = re.compile('^([\da-fA-Fx]+) +([\da-fA-Fx]+)')
  53 secReGeneral = re.compile('^([\da-fA-F]+):([\da-fA-F]+) +([\da-fA-F]+)[Hh]? +([.\w\$]+) +(\w+)', re.UNICODE)
  54
  55 StructPattern = re.compile(r'[_a-zA-Z][0-9A-Za-z_]*$')
  56
  57 ## Dictionary used to store file time stamp for quick re-access
  58 gFileTimeStampCache = {}    # {file path : file time stamp}
  59
  60 ## Dictionary used to store dependencies of files
  61 gDependencyDatabase = {}    # arch : {file path : [dependent files list]}
  62
  63 #
  64 # If a module is built more than once with different PCDs or library classes
  65 # a temporary INF file with same content is created, the temporary file is removed
  66 # when build exits.
  67 #
  68 _TempInfs = []
  69
  70 def GetVariableOffset(mapfilepath, efifilepath, varnames):
  71     """ Parse map file to get variable offset in current EFI file
  72     @param mapfilepath    Map file absolution path
  73     @param efifilepath:   EFI binary file full path
  74     @param varnames       iteratable container whose elements are variable names to be searched
  75
  76     @return List whos elements are tuple with variable name and raw offset
  77     """
  78     lines = []
  79     try:
  80         f = open(mapfilepath, 'r')
  81         lines = f.readlines()
  82         f.close()
  83     except:
  84         return None
  85
  86     if len(lines) == 0: return None
  87     firstline = lines[0].strip()
  88     if (firstline.startswith("Archive member included ") and
  89         firstline.endswith(" file (symbol)")):
  90         return _parseForGCC(lines, efifilepath, varnames)
  91     if firstline.startswith("# Path:"):
  92         return _parseForXcode(lines, efifilepath, varnames)
  93     return _parseGeneral(lines, efifilepath, varnames)
  94
  95 def _parseForXcode(lines, efifilepath, varnames):
  96     status = 0
  97     ret = []
  98     for line in lines:
  99         line = line.strip()
 100         if status == 0 and line == "# Symbols:":
 101             status = 1
 102             continue
 103         if status == 1 and len(line) != 0:
 104             for varname in varnames:
 105                 if varname in line:
 106                     # cannot pregenerate this RegEx since it uses varname from varnames.
 107                     m = re.match('^([\da-fA-FxX]+)([\s\S]*)([_]*%s)$' % varname, line)
 108                     if m is not None:
 109                         ret.append((varname, m.group(1)))
 110     return ret
 111
 112 def _parseForGCC(lines, efifilepath, varnames):
 113     """ Parse map file generated by GCC linker """
 114     status = 0
 115     sections = []
 116     varoffset = []
 117     for index, line in enumerate(lines):
 118         line = line.strip()
 119         # status machine transection
 120         if status == 0 and line == "Memory Configuration":
 121             status = 1
 122             continue
 123         elif status == 1 and line == 'Linker script and memory map':
 124             status = 2
 125             continue
 126         elif status ==2 and line == 'START GROUP':
 127             status = 3
 128             continue
 129
 130         # status handler
 131         if status == 3:
 132             m = valuePatternGcc.match(line)
 133             if m is not None:
 134                 sections.append(m.groups(0))
 135             for varname in varnames:
 136                 Str = ''
 137                 m = re.match("^.data.(%s)" % varname, line)
 138                 if m is not None:
 139                     m = re.match(".data.(%s)$" % varname, line)
 140                     if m is not None:
 141                         Str = lines[index + 1]
 142                     else:
 143                         Str = line[len(".data.%s" % varname):]
 144                     if Str:
 145                         m = pcdPatternGcc.match(Str.strip())
 146                         if m is not None:
 147                             varoffset.append((varname, int(m.groups(0)[0], 16), int(sections[-1][1], 16), sections[-1][0]))
 148
 149     if not varoffset:
 150         return []
 151     # get section information from efi file
 152     efisecs = PeImageClass(efifilepath).SectionHeaderList
 153     if efisecs is None or len(efisecs) == 0:
 154         return []
 155     #redirection
 156     redirection = 0
 157     for efisec in efisecs:
 158         for section in sections:
 159             if section[0].strip() == efisec[0].strip() and section[0].strip() == '.text':
 160                 redirection = int(section[1], 16) - efisec[1]
 161
 162     ret = []
 163     for var in varoffset:
 164         for efisec in efisecs:
 165             if var[1] >= efisec[1] and var[1] < efisec[1]+efisec[3]:
 166                 ret.append((var[0], hex(efisec[2] + var[1] - efisec[1] - redirection)))
 167     return ret
 168
 169 def _parseGeneral(lines, efifilepath, varnames):
 170     status = 0    #0 - beginning of file; 1 - PE section definition; 2 - symbol table
 171     secs  = []    # key = section name
 172     varoffset = []
 173     symRe = re.compile('^([\da-fA-F]+):([\da-fA-F]+) +([\.:\\\\\w\?@\$]+) +([\da-fA-F]+)', re.UNICODE)
 174
 175     for line in lines:
 176         line = line.strip()
 177         if startPatternGeneral.match(line):
 178             status = 1
 179             continue
 180         if addressPatternGeneral.match(line):
 181             status = 2
 182             continue
 183         if line.startswith("entry point at"):
 184             status = 3
 185             continue
 186         if status == 1 and len(line) != 0:
 187             m =  secReGeneral.match(line)
 188             assert m is not None, "Fail to parse the section in map file , line is %s" % line
 189             sec_no, sec_start, sec_length, sec_name, sec_class = m.groups(0)
 190             secs.append([int(sec_no, 16), int(sec_start, 16), int(sec_length, 16), sec_name, sec_class])
 191         if status == 2 and len(line) != 0:
 192             for varname in varnames:
 193                 m = symRe.match(line)
 194                 assert m is not None, "Fail to parse the symbol in map file, line is %s" % line
 195                 sec_no, sym_offset, sym_name, vir_addr = m.groups(0)
 196                 sec_no     = int(sec_no,     16)
 197                 sym_offset = int(sym_offset, 16)
 198                 vir_addr   = int(vir_addr,   16)
 199                 # cannot pregenerate this RegEx since it uses varname from varnames.
 200                 m2 = re.match('^[_]*(%s)' % varname, sym_name)
 201                 if m2 is not None:
 202                     # fond a binary pcd entry in map file
 203                     for sec in secs:
 204                         if sec[0] == sec_no and (sym_offset >= sec[1] and sym_offset < sec[1] + sec[2]):
 205                             varoffset.append([varname, sec[3], sym_offset, vir_addr, sec_no])
 206
 207     if not varoffset: return []
 208
 209     # get section information from efi file
 210     efisecs = PeImageClass(efifilepath).SectionHeaderList
 211     if efisecs is None or len(efisecs) == 0:
 212         return []
 213
 214     ret = []
 215     for var in varoffset:
 216         index = 0
 217         for efisec in efisecs:
 218             index = index + 1
 219             if var[1].strip() == efisec[0].strip():
 220                 ret.append((var[0], hex(efisec[2] + var[2])))
 221             elif var[4] == index:
 222                 ret.append((var[0], hex(efisec[2] + var[2])))
 223
 224     return ret
 225
 226 ## Routine to process duplicated INF
 227 #
 228 #  This function is called by following two cases:
 229 #  Case 1 in DSC:
 230 #    [components.arch]
 231 #    Pkg/module/module.inf
 232 #    Pkg/module/module.inf {
 233 #      <Defines>
 234 #        FILE_GUID = 0D1B936F-68F3-4589-AFCC-FB8B7AEBC836
 235 #    }
 236 #  Case 2 in FDF:
 237 #    INF Pkg/module/module.inf
 238 #    INF FILE_GUID = 0D1B936F-68F3-4589-AFCC-FB8B7AEBC836 Pkg/module/module.inf
 239 #
 240 #  This function copies Pkg/module/module.inf to
 241 #  Conf/.cache/0D1B936F-68F3-4589-AFCC-FB8B7AEBC836module.inf
 242 #
 243 #  @param Path     Original PathClass object
 244 #  @param BaseName New file base name
 245 #
 246 #  @retval         return the new PathClass object
 247 #
 248 def ProcessDuplicatedInf(Path, BaseName, Workspace):
 249     Filename = os.path.split(Path.File)[1]
 250     if '.' in Filename:
 251         Filename = BaseName + Path.BaseName + Filename[Filename.rfind('.'):]
 252     else:
 253         Filename = BaseName + Path.BaseName
 254
 255     #
 256     # If -N is specified on command line, cache is disabled
 257     # The directory has to be created
 258     #
 259     DbDir = os.path.split(GlobalData.gDatabasePath)[0]
 260     if not os.path.exists(DbDir):
 261         os.makedirs(DbDir)
 262     #
 263     # A temporary INF is copied to database path which must have write permission
 264     # The temporary will be removed at the end of build
 265     # In case of name conflict, the file name is
 266     # FILE_GUIDBaseName (0D1B936F-68F3-4589-AFCC-FB8B7AEBC836module.inf)
 267     #
 268     TempFullPath = os.path.join(DbDir,
 269                                 Filename)
 270     RtPath = PathClass(Path.File, Workspace)
 271     #
 272     # Modify the full path to temporary path, keep other unchanged
 273     #
 274     # To build same module more than once, the module path with FILE_GUID overridden has
 275     # the file name FILE_GUIDmodule.inf, but the relative path (self.MetaFile.File) is the real path
 276     # in DSC which is used as relative path by C files and other files in INF.
 277     # A trick was used: all module paths are PathClass instances, after the initialization
 278     # of PathClass, the PathClass.Path is overridden by the temporary INF path.
 279     #
 280     # The reason for creating a temporary INF is:
 281     # Platform.Modules which is the base to create ModuleAutoGen objects is a dictionary,
 282     # the key is the full path of INF, the value is an object to save overridden library instances, PCDs.
 283     # A different key for the same module is needed to create different output directory,
 284     # retrieve overridden PCDs, library instances.
 285     #
 286     # The BaseName is the FILE_GUID which is also the output directory name.
 287     #
 288     #
 289     RtPath.Path = TempFullPath
 290     RtPath.BaseName = BaseName
 291     #
 292     # If file exists, compare contents
 293     #
 294     if os.path.exists(TempFullPath):
 295         with open(str(Path), 'rb') as f1, open(TempFullPath, 'rb') as f2:
 296             if f1.read() == f2.read():
 297                 return RtPath
 298     _TempInfs.append(TempFullPath)
 299     shutil.copy2(str(Path), TempFullPath)
 300     return RtPath
 301
 302 ## Remove temporary created INFs whose paths were saved in _TempInfs
 303 #
 304 def ClearDuplicatedInf():
 305     while _TempInfs:
 306         File = _TempInfs.pop()
 307         if os.path.exists(File):
 308             os.remove(File)
 309
 310 ## Convert GUID string in xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx style to C structure style
 311 #
 312 #   @param      Guid    The GUID string
 313 #
 314 #   @retval     string  The GUID string in C structure style
 315 #
 316 def GuidStringToGuidStructureString(Guid):
 317     GuidList = Guid.split('-')
 318     Result = '{'
 319     for Index in range(0, 3, 1):
 320         Result = Result + '0x' + GuidList[Index] + ', '
 321     Result = Result + '{0x' + GuidList[3][0:2] + ', 0x' + GuidList[3][2:4]
 322     for Index in range(0, 12, 2):
 323         Result = Result + ', 0x' + GuidList[4][Index:Index + 2]
 324     Result += '}}'
 325     return Result
 326
 327 ## Convert GUID structure in byte array to xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
 328 #
 329 #   @param      GuidValue   The GUID value in byte array
 330 #
 331 #   @retval     string      The GUID value in xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx format
 332 #
 333 def GuidStructureByteArrayToGuidString(GuidValue):
 334     guidValueString = GuidValue.lower().replace("{", "").replace("}", "").replace(" ", "").replace(";", "")
 335     guidValueList = guidValueString.split(",")
 336     if len(guidValueList) != 16:
 337         return ''
 338         #EdkLogger.error(None, None, "Invalid GUID value string %s" % GuidValue)
 339     try:
 340         return "%02x%02x%02x%02x-%02x%02x-%02x%02x-%02x%02x-%02x%02x%02x%02x%02x%02x" % (
 341                 int(guidValueList[3], 16),
 342                 int(guidValueList[2], 16),
 343                 int(guidValueList[1], 16),
 344                 int(guidValueList[0], 16),
 345                 int(guidValueList[5], 16),
 346                 int(guidValueList[4], 16),
 347                 int(guidValueList[7], 16),
 348                 int(guidValueList[6], 16),
 349                 int(guidValueList[8], 16),
 350                 int(guidValueList[9], 16),
 351                 int(guidValueList[10], 16),
 352                 int(guidValueList[11], 16),
 353                 int(guidValueList[12], 16),
 354                 int(guidValueList[13], 16),
 355                 int(guidValueList[14], 16),
 356                 int(guidValueList[15], 16)
 357                 )
 358     except:
 359         return ''
 360
 361 ## Convert GUID string in C structure style to xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
 362 #
 363 #   @param      GuidValue   The GUID value in C structure format
 364 #
 365 #   @retval     string      The GUID value in xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx format
 366 #
 367 def GuidStructureStringToGuidString(GuidValue):
 368     if not GlobalData.gGuidCFormatPattern.match(GuidValue):
 369         return ''
 370     guidValueString = GuidValue.lower().replace("{", "").replace("}", "").replace(" ", "").replace(";", "")
 371     guidValueList = guidValueString.split(",")
 372     if len(guidValueList) != 11:
 373         return ''
 374         #EdkLogger.error(None, None, "Invalid GUID value string %s" % GuidValue)
 375     try:
 376         return "%08x-%04x-%04x-%02x%02x-%02x%02x%02x%02x%02x%02x" % (
 377                 int(guidValueList[0], 16),
 378                 int(guidValueList[1], 16),
 379                 int(guidValueList[2], 16),
 380                 int(guidValueList[3], 16),
 381                 int(guidValueList[4], 16),
 382                 int(guidValueList[5], 16),
 383                 int(guidValueList[6], 16),
 384                 int(guidValueList[7], 16),
 385                 int(guidValueList[8], 16),
 386                 int(guidValueList[9], 16),
 387                 int(guidValueList[10], 16)
 388                 )
 389     except:
 390         return ''
 391
 392 ## Convert GUID string in C structure style to xxxxxxxx_xxxx_xxxx_xxxx_xxxxxxxxxxxx
 393 #
 394 #   @param      GuidValue   The GUID value in C structure format
 395 #
 396 #   @retval     string      The GUID value in xxxxxxxx_xxxx_xxxx_xxxx_xxxxxxxxxxxx format
 397 #
 398 def GuidStructureStringToGuidValueName(GuidValue):
 399     guidValueString = GuidValue.lower().replace("{", "").replace("}", "").replace(" ", "")
 400     guidValueList = guidValueString.split(",")
 401     if len(guidValueList) != 11:
 402         EdkLogger.error(None, FORMAT_INVALID, "Invalid GUID value string [%s]" % GuidValue)
 403     return "%08x_%04x_%04x_%02x%02x_%02x%02x%02x%02x%02x%02x" % (
 404             int(guidValueList[0], 16),
 405             int(guidValueList[1], 16),
 406             int(guidValueList[2], 16),
 407             int(guidValueList[3], 16),
 408             int(guidValueList[4], 16),
 409             int(guidValueList[5], 16),
 410             int(guidValueList[6], 16),
 411             int(guidValueList[7], 16),
 412             int(guidValueList[8], 16),
 413             int(guidValueList[9], 16),
 414             int(guidValueList[10], 16)
 415             )
 416
 417 ## Create directories
 418 #
 419 #   @param      Directory   The directory name
 420 #
 421 def CreateDirectory(Directory):
 422     if Directory is None or Directory.strip() == "":
 423         return True
 424     try:
 425         if not os.access(Directory, os.F_OK):
 426             os.makedirs(Directory)
 427     except:
 428         return False
 429     return True
 430
 431 ## Remove directories, including files and sub-directories in it
 432 #
 433 #   @param      Directory   The directory name
 434 #
 435 def RemoveDirectory(Directory, Recursively=False):
 436     if Directory is None or Directory.strip() == "" or not os.path.exists(Directory):
 437         return
 438     if Recursively:
 439         CurrentDirectory = os.getcwd()
 440         os.chdir(Directory)
 441         for File in os.listdir("."):
 442             if os.path.isdir(File):
 443                 RemoveDirectory(File, Recursively)
 444             else:
 445                 os.remove(File)
 446         os.chdir(CurrentDirectory)
 447     os.rmdir(Directory)
 448
 449 ## Store content in file
 450 #
 451 #  This method is used to save file only when its content is changed. This is
 452 #  quite useful for "make" system to decide what will be re-built and what won't.
 453 #
 454 #   @param      File            The path of file
 455 #   @param      Content         The new content of the file
 456 #   @param      IsBinaryFile    The flag indicating if the file is binary file or not
 457 #
 458 #   @retval     True            If the file content is changed and the file is renewed
 459 #   @retval     False           If the file content is the same
 460 #
 461 def SaveFileOnChange(File, Content, IsBinaryFile=True):
 462     if not IsBinaryFile:
 463         Content = Content.replace("\n", os.linesep)
 464
 465     if os.path.exists(File):
 466         try:
 467             if Content == open(File, "rb").read():
 468                 return False
 469         except:
 470             EdkLogger.error(None, FILE_OPEN_FAILURE, ExtraData=File)
 471
 472     DirName = os.path.dirname(File)
 473     if not CreateDirectory(DirName):
 474         EdkLogger.error(None, FILE_CREATE_FAILURE, "Could not create directory %s" % DirName)
 475     else:
 476         if DirName == '':
 477             DirName = os.getcwd()
 478         if not os.access(DirName, os.W_OK):
 479             EdkLogger.error(None, PERMISSION_FAILURE, "Do not have write permission on directory %s" % DirName)
 480
 481     try:
 482         Fd = open(File, "wb")
 483         Fd.write(Content)
 484         Fd.close()
 485     except IOError as X:
 486         EdkLogger.error(None, FILE_CREATE_FAILURE, ExtraData='IOError %s' % X)
 487
 488     return True
 489
 490 ## Retrieve and cache the real path name in file system
 491 #
 492 #   @param      Root    The root directory of path relative to
 493 #
 494 #   @retval     str     The path string if the path exists
 495 #   @retval     None    If path doesn't exist
 496 #
 497 class DirCache:
 498     _CACHE_ = set()
 499     _UPPER_CACHE_ = {}
 500
 501     def __init__(self, Root):
 502         self._Root = Root
 503         for F in os.listdir(Root):
 504             self._CACHE_.add(F)
 505             self._UPPER_CACHE_[F.upper()] = F
 506
 507     # =[] operator
 508     def __getitem__(self, Path):
 509         Path = Path[len(os.path.commonprefix([Path, self._Root])):]
 510         if not Path:
 511             return self._Root
 512         if Path and Path[0] == os.path.sep:
 513             Path = Path[1:]
 514         if Path in self._CACHE_:
 515             return os.path.join(self._Root, Path)
 516         UpperPath = Path.upper()
 517         if UpperPath in self._UPPER_CACHE_:
 518             return os.path.join(self._Root, self._UPPER_CACHE_[UpperPath])
 519
 520         IndexList = []
 521         LastSepIndex = -1
 522         SepIndex = Path.find(os.path.sep)
 523         while SepIndex > -1:
 524             Parent = UpperPath[:SepIndex]
 525             if Parent not in self._UPPER_CACHE_:
 526                 break
 527             LastSepIndex = SepIndex
 528             SepIndex = Path.find(os.path.sep, LastSepIndex + 1)
 529
 530         if LastSepIndex == -1:
 531             return None
 532
 533         Cwd = os.getcwd()
 534         os.chdir(self._Root)
 535         SepIndex = LastSepIndex
 536         while SepIndex > -1:
 537             Parent = Path[:SepIndex]
 538             ParentKey = UpperPath[:SepIndex]
 539             if ParentKey not in self._UPPER_CACHE_:
 540                 os.chdir(Cwd)
 541                 return None
 542
 543             if Parent in self._CACHE_:
 544                 ParentDir = Parent
 545             else:
 546                 ParentDir = self._UPPER_CACHE_[ParentKey]
 547             for F in os.listdir(ParentDir):
 548                 Dir = os.path.join(ParentDir, F)
 549                 self._CACHE_.add(Dir)
 550                 self._UPPER_CACHE_[Dir.upper()] = Dir
 551
 552             SepIndex = Path.find(os.path.sep, SepIndex + 1)
 553
 554         os.chdir(Cwd)
 555         if Path in self._CACHE_:
 556             return os.path.join(self._Root, Path)
 557         elif UpperPath in self._UPPER_CACHE_:
 558             return os.path.join(self._Root, self._UPPER_CACHE_[UpperPath])
 559         return None
 560
 561 def RealPath(File, Dir='', OverrideDir=''):
 562     NewFile = os.path.normpath(os.path.join(Dir, File))
 563     NewFile = GlobalData.gAllFiles[NewFile]
 564     if not NewFile and OverrideDir:
 565         NewFile = os.path.normpath(os.path.join(OverrideDir, File))
 566         NewFile = GlobalData.gAllFiles[NewFile]
 567     return NewFile
 568
 569 def RealPath2(File, Dir='', OverrideDir=''):
 570     NewFile = None
 571     if OverrideDir:
 572         NewFile = GlobalData.gAllFiles[os.path.normpath(os.path.join(OverrideDir, File))]
 573         if NewFile:
 574             if OverrideDir[-1] == os.path.sep:
 575                 return NewFile[len(OverrideDir):], NewFile[0:len(OverrideDir)]
 576             else:
 577                 return NewFile[len(OverrideDir) + 1:], NewFile[0:len(OverrideDir)]
 578     if GlobalData.gAllFiles:
 579         NewFile = GlobalData.gAllFiles[os.path.normpath(os.path.join(Dir, File))]
 580     if not NewFile:
 581         NewFile = os.path.normpath(os.path.join(Dir, File))
 582         if not os.path.exists(NewFile):
 583             return None, None
 584     if NewFile:
 585         if Dir:
 586             if Dir[-1] == os.path.sep:
 587                 return NewFile[len(Dir):], NewFile[0:len(Dir)]
 588             else:
 589                 return NewFile[len(Dir) + 1:], NewFile[0:len(Dir)]
 590         else:
 591             return NewFile, ''
 592
 593     return None, None
 594
 595 ## Get GUID value from given packages
 596 #
 597 #   @param      CName           The CName of the GUID
 598 #   @param      PackageList     List of packages looking-up in
 599 #   @param      Inffile         The driver file
 600 #
 601 #   @retval     GuidValue   if the CName is found in any given package
 602 #   @retval     None        if the CName is not found in all given packages
 603 #
 604 def GuidValue(CName, PackageList, Inffile = None):
 605     for P in PackageList:
 606         GuidKeys = P.Guids.keys()
 607         if Inffile and P._PrivateGuids:
 608             if not Inffile.startswith(P.MetaFile.Dir):
 609                 GuidKeys = [x for x in P.Guids if x not in P._PrivateGuids]
 610         if CName in GuidKeys:
 611             return P.Guids[CName]
 612     return None
 613
 614 ## A string template class
 615 #
 616 #  This class implements a template for string replacement. A string template
 617 #  looks like following
 618 #
 619 #       ${BEGIN} other_string ${placeholder_name} other_string ${END}
 620 #
 621 #  The string between ${BEGIN} and ${END} will be repeated as many times as the
 622 #  length of "placeholder_name", which is a list passed through a dict. The
 623 #  "placeholder_name" is the key name of the dict. The ${BEGIN} and ${END} can
 624 #  be not used and, in this case, the "placeholder_name" must not a list and it
 625 #  will just be replaced once.
 626 #
 627 class TemplateString(object):
 628     _REPEAT_START_FLAG = "BEGIN"
 629     _REPEAT_END_FLAG = "END"
 630
 631     class Section(object):
 632         _LIST_TYPES = [type([]), type(set()), type((0,))]
 633
 634         def __init__(self, TemplateSection, PlaceHolderList):
 635             self._Template = TemplateSection
 636             self._PlaceHolderList = []
 637
 638             # Split the section into sub-sections according to the position of placeholders
 639             if PlaceHolderList:
 640                 self._SubSectionList = []
 641                 SubSectionStart = 0
 642                 #
 643                 # The placeholders passed in must be in the format of
 644                 #
 645                 #   PlaceHolderName, PlaceHolderStartPoint, PlaceHolderEndPoint
 646                 #
 647                 for PlaceHolder, Start, End in PlaceHolderList:
 648                     self._SubSectionList.append(TemplateSection[SubSectionStart:Start])
 649                     self._SubSectionList.append(TemplateSection[Start:End])
 650                     self._PlaceHolderList.append(PlaceHolder)
 651                     SubSectionStart = End
 652                 if SubSectionStart < len(TemplateSection):
 653                     self._SubSectionList.append(TemplateSection[SubSectionStart:])
 654             else:
 655                 self._SubSectionList = [TemplateSection]
 656
 657         def __str__(self):
 658             return self._Template + " : " + str(self._PlaceHolderList)
 659
 660         def Instantiate(self, PlaceHolderValues):
 661             RepeatTime = -1
 662             RepeatPlaceHolders = {}
 663             NonRepeatPlaceHolders = {}
 664
 665             for PlaceHolder in self._PlaceHolderList:
 666                 if PlaceHolder not in PlaceHolderValues:
 667                     continue
 668                 Value = PlaceHolderValues[PlaceHolder]
 669                 if type(Value) in self._LIST_TYPES:
 670                     if RepeatTime < 0:
 671                         RepeatTime = len(Value)
 672                     elif RepeatTime != len(Value):
 673                         EdkLogger.error(
 674                                     "TemplateString",
 675                                     PARAMETER_INVALID,
 676                                     "${%s} has different repeat time from others!" % PlaceHolder,
 677                                     ExtraData=str(self._Template)
 678                                     )
 679                     RepeatPlaceHolders["${%s}" % PlaceHolder] = Value
 680                 else:
 681                     NonRepeatPlaceHolders["${%s}" % PlaceHolder] = Value
 682
 683             if NonRepeatPlaceHolders:
 684                 StringList = []
 685                 for S in self._SubSectionList:
 686                     if S not in NonRepeatPlaceHolders:
 687                         StringList.append(S)
 688                     else:
 689                         StringList.append(str(NonRepeatPlaceHolders[S]))
 690             else:
 691                 StringList = self._SubSectionList
 692
 693             if RepeatPlaceHolders:
 694                 TempStringList = []
 695                 for Index in range(RepeatTime):
 696                     for S in StringList:
 697                         if S not in RepeatPlaceHolders:
 698                             TempStringList.append(S)
 699                         else:
 700                             TempStringList.append(str(RepeatPlaceHolders[S][Index]))
 701                 StringList = TempStringList
 702
 703             return "".join(StringList)
 704
 705     ## Constructor
 706     def __init__(self, Template=None):
 707         self.String = []
 708         self.IsBinary = False
 709         self._Template = Template
 710         self._TemplateSectionList = self._Parse(Template)
 711
 712     ## str() operator
 713     #
 714     #   @retval     string  The string replaced
 715     #
 716     def __str__(self):
 717         return "".join(self.String)
 718
 719     ## Split the template string into fragments per the ${BEGIN} and ${END} flags
 720     #
 721     #   @retval     list    A list of TemplateString.Section objects
 722     #
 723     def _Parse(self, Template):
 724         SectionStart = 0
 725         SearchFrom = 0
 726         MatchEnd = 0
 727         PlaceHolderList = []
 728         TemplateSectionList = []
 729         while Template:
 730             MatchObj = gPlaceholderPattern.search(Template, SearchFrom)
 731             if not MatchObj:
 732                 if MatchEnd <= len(Template):
 733                     TemplateSection = TemplateString.Section(Template[SectionStart:], PlaceHolderList)
 734                     TemplateSectionList.append(TemplateSection)
 735                 break
 736
 737             MatchString = MatchObj.group(1)
 738             MatchStart = MatchObj.start()
 739             MatchEnd = MatchObj.end()
 740
 741             if MatchString == self._REPEAT_START_FLAG:
 742                 if MatchStart > SectionStart:
 743                     TemplateSection = TemplateString.Section(Template[SectionStart:MatchStart], PlaceHolderList)
 744                     TemplateSectionList.append(TemplateSection)
 745                 SectionStart = MatchEnd
 746                 PlaceHolderList = []
 747             elif MatchString == self._REPEAT_END_FLAG:
 748                 TemplateSection = TemplateString.Section(Template[SectionStart:MatchStart], PlaceHolderList)
 749                 TemplateSectionList.append(TemplateSection)
 750                 SectionStart = MatchEnd
 751                 PlaceHolderList = []
 752             else:
 753                 PlaceHolderList.append((MatchString, MatchStart - SectionStart, MatchEnd - SectionStart))
 754             SearchFrom = MatchEnd
 755         return TemplateSectionList
 756
 757     ## Replace the string template with dictionary of placeholders and append it to previous one
 758     #
 759     #   @param      AppendString    The string template to append
 760     #   @param      Dictionary      The placeholder dictionaries
 761     #
 762     def Append(self, AppendString, Dictionary=None):
 763         if Dictionary:
 764             SectionList = self._Parse(AppendString)
 765             self.String.append( "".join(S.Instantiate(Dictionary) for S in SectionList))
 766         else:
 767             if isinstance(AppendString,list):
 768                 self.String.extend(AppendString)
 769             else:
 770                 self.String.append(AppendString)
 771
 772     ## Replace the string template with dictionary of placeholders
 773     #
 774     #   @param      Dictionary      The placeholder dictionaries
 775     #
 776     #   @retval     str             The string replaced with placeholder values
 777     #
 778     def Replace(self, Dictionary=None):
 779         return "".join(S.Instantiate(Dictionary) for S in self._TemplateSectionList)
 780
 781 ## Progress indicator class
 782 #
 783 #  This class makes use of thread to print progress on console.
 784 #
 785 class Progressor:
 786     # for avoiding deadloop
 787     _StopFlag = None
 788     _ProgressThread = None
 789     _CheckInterval = 0.25
 790
 791     ## Constructor
 792     #
 793     #   @param      OpenMessage     The string printed before progress charaters
 794     #   @param      CloseMessage    The string printed after progress charaters
 795     #   @param      ProgressChar    The charater used to indicate the progress
 796     #   @param      Interval        The interval in seconds between two progress charaters
 797     #
 798     def __init__(self, OpenMessage="", CloseMessage="", ProgressChar='.', Interval=1.0):
 799         self.PromptMessage = OpenMessage
 800         self.CodaMessage = CloseMessage
 801         self.ProgressChar = ProgressChar
 802         self.Interval = Interval
 803         if Progressor._StopFlag is None:
 804             Progressor._StopFlag = threading.Event()
 805
 806     ## Start to print progress charater
 807     #
 808     #   @param      OpenMessage     The string printed before progress charaters
 809     #
 810     def Start(self, OpenMessage=None):
 811         if OpenMessage is not None:
 812             self.PromptMessage = OpenMessage
 813         Progressor._StopFlag.clear()
 814         if Progressor._ProgressThread is None:
 815             Progressor._ProgressThread = threading.Thread(target=self._ProgressThreadEntry)
 816             Progressor._ProgressThread.setDaemon(False)
 817             Progressor._ProgressThread.start()
 818
 819     ## Stop printing progress charater
 820     #
 821     #   @param      CloseMessage    The string printed after progress charaters
 822     #
 823     def Stop(self, CloseMessage=None):
 824         OriginalCodaMessage = self.CodaMessage
 825         if CloseMessage is not None:
 826             self.CodaMessage = CloseMessage
 827         self.Abort()
 828         self.CodaMessage = OriginalCodaMessage
 829
 830     ## Thread entry method
 831     def _ProgressThreadEntry(self):
 832         sys.stdout.write(self.PromptMessage + " ")
 833         sys.stdout.flush()
 834         TimeUp = 0.0
 835         while not Progressor._StopFlag.isSet():
 836             if TimeUp <= 0.0:
 837                 sys.stdout.write(self.ProgressChar)
 838                 sys.stdout.flush()
 839                 TimeUp = self.Interval
 840             time.sleep(self._CheckInterval)
 841             TimeUp -= self._CheckInterval
 842         sys.stdout.write(" " + self.CodaMessage + "\n")
 843         sys.stdout.flush()
 844
 845     ## Abort the progress display
 846     @staticmethod
 847     def Abort():
 848         if Progressor._StopFlag is not None:
 849             Progressor._StopFlag.set()
 850         if Progressor._ProgressThread is not None:
 851             Progressor._ProgressThread.join()
 852             Progressor._ProgressThread = None
 853
 854 ## A dict which can access its keys and/or values orderly
 855 #
 856 #  The class implements a new kind of dict which its keys or values can be
 857 #  accessed in the order they are added into the dict. It guarantees the order
 858 #  by making use of an internal list to keep a copy of keys.
 859 #
 860 class sdict(IterableUserDict):
 861     ## Constructor
 862     def __init__(self):
 863         IterableUserDict.__init__(self)
 864         self._key_list = []
 865
 866     ## [] operator
 867     def __setitem__(self, key, value):
 868         if key not in self._key_list:
 869             self._key_list.append(key)
 870         IterableUserDict.__setitem__(self, key, value)
 871
 872     ## del operator
 873     def __delitem__(self, key):
 874         self._key_list.remove(key)
 875         IterableUserDict.__delitem__(self, key)
 876
 877     ## used in "for k in dict" loop to ensure the correct order
 878     def __iter__(self):
 879         return self.iterkeys()
 880
 881     ## len() support
 882     def __len__(self):
 883         return len(self._key_list)
 884
 885     ## "in" test support
 886     def __contains__(self, key):
 887         return key in self._key_list
 888
 889     ## indexof support
 890     def index(self, key):
 891         return self._key_list.index(key)
 892
 893     ## insert support
 894     def insert(self, key, newkey, newvalue, order):
 895         index = self._key_list.index(key)
 896         if order == 'BEFORE':
 897             self._key_list.insert(index, newkey)
 898             IterableUserDict.__setitem__(self, newkey, newvalue)
 899         elif order == 'AFTER':
 900             self._key_list.insert(index + 1, newkey)
 901             IterableUserDict.__setitem__(self, newkey, newvalue)
 902
 903     ## append support
 904     def append(self, sdict):
 905         for key in sdict:
 906             if key not in self._key_list:
 907                 self._key_list.append(key)
 908             IterableUserDict.__setitem__(self, key, sdict[key])
 909
 910     def has_key(self, key):
 911         return key in self._key_list
 912
 913     ## Empty the dict
 914     def clear(self):
 915         self._key_list = []
 916         IterableUserDict.clear(self)
 917
 918     ## Return a copy of keys
 919     def keys(self):
 920         keys = []
 921         for key in self._key_list:
 922             keys.append(key)
 923         return keys
 924
 925     ## Return a copy of values
 926     def values(self):
 927         values = []
 928         for key in self._key_list:
 929             values.append(self[key])
 930         return values
 931
 932     ## Return a copy of (key, value) list
 933     def items(self):
 934         items = []
 935         for key in self._key_list:
 936             items.append((key, self[key]))
 937         return items
 938
 939     ## Iteration support
 940     def iteritems(self):
 941         return iter(self.items())
 942
 943     ## Keys interation support
 944     def iterkeys(self):
 945         return iter(self.keys())
 946
 947     ## Values interation support
 948     def itervalues(self):
 949         return iter(self.values())
 950
 951     ## Return value related to a key, and remove the (key, value) from the dict
 952     def pop(self, key, *dv):
 953         value = None
 954         if key in self._key_list:
 955             value = self[key]
 956             self.__delitem__(key)
 957         elif len(dv) != 0 :
 958             value = kv[0]
 959         return value
 960
 961     ## Return (key, value) pair, and remove the (key, value) from the dict
 962     def popitem(self):
 963         key = self._key_list[-1]
 964         value = self[key]
 965         self.__delitem__(key)
 966         return key, value
 967
 968     def update(self, dict=None, **kwargs):
 969         if dict is not None:
 970             for k, v in dict.items():
 971                 self[k] = v
 972         if len(kwargs):
 973             for k, v in kwargs.items():
 974                 self[k] = v
 975
 976 ## Dictionary with restricted keys
 977 #
 978 class rdict(dict):
 979     ## Constructor
 980     def __init__(self, KeyList):
 981         for Key in KeyList:
 982             dict.__setitem__(self, Key, "")
 983
 984     ## []= operator
 985     def __setitem__(self, key, value):
 986         if key not in self:
 987             EdkLogger.error("RestrictedDict", ATTRIBUTE_SET_FAILURE, "Key [%s] is not allowed" % key,
 988                             ExtraData=", ".join(dict.keys(self)))
 989         dict.__setitem__(self, key, value)
 990
 991     ## =[] operator
 992     def __getitem__(self, key):
 993         if key not in self:
 994             return ""
 995         return dict.__getitem__(self, key)
 996
 997     ## del operator
 998     def __delitem__(self, key):
 999         EdkLogger.error("RestrictedDict", ATTRIBUTE_ACCESS_DENIED, ExtraData="del")
1000
1001     ## Empty the dict
1002     def clear(self):
1003         for Key in self:
1004             self.__setitem__(Key, "")
1005
1006     ## Return value related to a key, and remove the (key, value) from the dict
1007     def pop(self, key, *dv):
1008         EdkLogger.error("RestrictedDict", ATTRIBUTE_ACCESS_DENIED, ExtraData="pop")
1009
1010     ## Return (key, value) pair, and remove the (key, value) from the dict
1011     def popitem(self):
1012         EdkLogger.error("RestrictedDict", ATTRIBUTE_ACCESS_DENIED, ExtraData="popitem")
1013
1014 ## Dictionary using prioritized list as key
1015 #
1016 class tdict:
1017     _ListType = type([])
1018     _TupleType = type(())
1019     _Wildcard = 'COMMON'
1020     _ValidWildcardList = ['COMMON', 'DEFAULT', 'ALL', TAB_STAR, 'PLATFORM']
1021
1022     def __init__(self, _Single_=False, _Level_=2):
1023         self._Level_ = _Level_
1024         self.data = {}
1025         self._Single_ = _Single_
1026
1027     # =[] operator
1028     def __getitem__(self, key):
1029         KeyType = type(key)
1030         RestKeys = None
1031         if KeyType == self._ListType or KeyType == self._TupleType:
1032             FirstKey = key[0]
1033             if len(key) > 1:
1034                 RestKeys = key[1:]
1035             elif self._Level_ > 1:
1036                 RestKeys = [self._Wildcard for i in range(0, self._Level_ - 1)]
1037         else:
1038             FirstKey = key
1039             if self._Level_ > 1:
1040                 RestKeys = [self._Wildcard for i in range(0, self._Level_ - 1)]
1041
1042         if FirstKey is None or str(FirstKey).upper() in self._ValidWildcardList:
1043             FirstKey = self._Wildcard
1044
1045         if self._Single_:
1046             return self._GetSingleValue(FirstKey, RestKeys)
1047         else:
1048             return self._GetAllValues(FirstKey, RestKeys)
1049
1050     def _GetSingleValue(self, FirstKey, RestKeys):
1051         Value = None
1052         #print "%s-%s" % (FirstKey, self._Level_) ,
1053         if self._Level_ > 1:
1054             if FirstKey == self._Wildcard:
1055                 if FirstKey in self.data:
1056                     Value = self.data[FirstKey][RestKeys]
1057                 if Value is None:
1058                     for Key in self.data:
1059                         Value = self.data[Key][RestKeys]
1060                         if Value is not None: break
1061             else:
1062                 if FirstKey in self.data:
1063                     Value = self.data[FirstKey][RestKeys]
1064                 if Value is None and self._Wildcard in self.data:
1065                     #print "Value=None"
1066                     Value = self.data[self._Wildcard][RestKeys]
1067         else:
1068             if FirstKey == self._Wildcard:
1069                 if FirstKey in self.data:
1070                     Value = self.data[FirstKey]
1071                 if Value is None:
1072                     for Key in self.data:
1073                         Value = self.data[Key]
1074                         if Value is not None: break
1075             else:
1076                 if FirstKey in self.data:
1077                     Value = self.data[FirstKey]
1078                 elif self._Wildcard in self.data:
1079                     Value = self.data[self._Wildcard]
1080         return Value
1081
1082     def _GetAllValues(self, FirstKey, RestKeys):
1083         Value = []
1084         if self._Level_ > 1:
1085             if FirstKey == self._Wildcard:
1086                 for Key in self.data:
1087                     Value += self.data[Key][RestKeys]
1088             else:
1089                 if FirstKey in self.data:
1090                     Value += self.data[FirstKey][RestKeys]
1091                 if self._Wildcard in self.data:
1092                     Value += self.data[self._Wildcard][RestKeys]
1093         else:
1094             if FirstKey == self._Wildcard:
1095                 for Key in self.data:
1096                     Value.append(self.data[Key])
1097             else:
1098                 if FirstKey in self.data:
1099                     Value.append(self.data[FirstKey])
1100                 if self._Wildcard in self.data:
1101                     Value.append(self.data[self._Wildcard])
1102         return Value
1103
1104     ## []= operator
1105     def __setitem__(self, key, value):
1106         KeyType = type(key)
1107         RestKeys = None
1108         if KeyType == self._ListType or KeyType == self._TupleType:
1109             FirstKey = key[0]
1110             if len(key) > 1:
1111                 RestKeys = key[1:]
1112             else:
1113                 RestKeys = [self._Wildcard for i in range(0, self._Level_ - 1)]
1114         else:
1115             FirstKey = key
1116             if self._Level_ > 1:
1117                 RestKeys = [self._Wildcard for i in range(0, self._Level_ - 1)]
1118
1119         if FirstKey in self._ValidWildcardList:
1120             FirstKey = self._Wildcard
1121
1122         if FirstKey not in self.data and self._Level_ > 0:
1123             self.data[FirstKey] = tdict(self._Single_, self._Level_ - 1)
1124
1125         if self._Level_ > 1:
1126             self.data[FirstKey][RestKeys] = value
1127         else:
1128             self.data[FirstKey] = value
1129
1130     def SetGreedyMode(self):
1131         self._Single_ = False
1132         if self._Level_ > 1:
1133             for Key in self.data:
1134                 self.data[Key].SetGreedyMode()
1135
1136     def SetSingleMode(self):
1137         self._Single_ = True
1138         if self._Level_ > 1:
1139             for Key in self.data:
1140                 self.data[Key].SetSingleMode()
1141
1142     def GetKeys(self, KeyIndex=0):
1143         assert KeyIndex >= 0
1144         if KeyIndex == 0:
1145             return set(self.data.keys())
1146         else:
1147             keys = set()
1148             for Key in self.data:
1149                 keys |= self.data[Key].GetKeys(KeyIndex - 1)
1150             return keys
1151
1152 def IsFieldValueAnArray (Value):
1153     Value = Value.strip()
1154     if Value.startswith(TAB_GUID) and Value.endswith(')'):
1155         return True
1156     if Value.startswith('L"') and Value.endswith('"')  and len(list(Value[2:-1])) > 1:
1157         return True
1158     if Value[0] == '"' and Value[-1] == '"' and len(list(Value[1:-1])) > 1:
1159         return True
1160     if Value[0] == '{' and Value[-1] == '}':
1161         return True
1162     if Value.startswith("L'") and Value.endswith("'") and len(list(Value[2:-1])) > 1:
1163         return True
1164     if Value[0] == "'" and Value[-1] == "'" and len(list(Value[1:-1])) > 1:
1165         return True
1166     return False
1167
1168 def AnalyzePcdExpression(Setting):
1169     RanStr = ''.join(sample(string.ascii_letters + string.digits, 8))
1170     Setting = Setting.replace('\\\\', RanStr).strip()
1171     # There might be escaped quote in a string: \", \\\" , \', \\\'
1172     Data = Setting
1173     # There might be '|' in string and in ( ... | ... ), replace it with '-'
1174     NewStr = ''
1175     InSingleQuoteStr = False
1176     InDoubleQuoteStr = False
1177     Pair = 0
1178     for Index, ch in enumerate(Data):
1179         if ch == '"' and not InSingleQuoteStr:
1180             if Data[Index - 1] != '\\':
1181                 InDoubleQuoteStr = not InDoubleQuoteStr
1182         elif ch == "'" and not InDoubleQuoteStr:
1183             if Data[Index - 1] != '\\':
1184                 InSingleQuoteStr = not InSingleQuoteStr
1185         elif ch == '(' and not (InSingleQuoteStr or InDoubleQuoteStr):
1186             Pair += 1
1187         elif ch == ')' and not (InSingleQuoteStr or InDoubleQuoteStr):
1188             Pair -= 1
1189
1190         if (Pair > 0 or InSingleQuoteStr or InDoubleQuoteStr) and ch == TAB_VALUE_SPLIT:
1191             NewStr += '-'
1192         else:
1193             NewStr += ch
1194     FieldList = []
1195     StartPos = 0
1196     while True:
1197         Pos = NewStr.find(TAB_VALUE_SPLIT, StartPos)
1198         if Pos < 0:
1199             FieldList.append(Setting[StartPos:].strip())
1200             break
1201         FieldList.append(Setting[StartPos:Pos].strip())
1202         StartPos = Pos + 1
1203     for i, ch in enumerate(FieldList):
1204         if RanStr in ch:
1205             FieldList[i] = ch.replace(RanStr,'\\\\')
1206     return FieldList
1207
1208 def ParseDevPathValue (Value):
1209     if '\\' in Value:
1210         Value.replace('\\', '/').replace(' ', '')
1211
1212     Cmd = 'DevicePath ' + '"' + Value + '"'
1213     try:
1214         p = subprocess.Popen(Cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
1215         out, err = p.communicate()
1216     except Exception as X:
1217         raise BadExpression("DevicePath: %s" % (str(X)) )
1218     finally:
1219         subprocess._cleanup()
1220         p.stdout.close()
1221         p.stderr.close()
1222     if err:
1223         raise BadExpression("DevicePath: %s" % str(err))
1224     Size = len(out.split())
1225     out = ','.join(out.split())
1226     return '{' + out + '}', Size
1227
1228 def ParseFieldValue (Value):
1229     if "{CODE(" in Value:
1230         return Value, len(Value.split(","))
1231     if isinstance(Value, type(0)):
1232         return Value, (Value.bit_length() + 7) / 8
1233     if not isinstance(Value, type('')):
1234         raise BadExpression('Type %s is %s' %(Value, type(Value)))
1235     Value = Value.strip()
1236     if Value.startswith(TAB_UINT8) and Value.endswith(')'):
1237         Value, Size = ParseFieldValue(Value.split('(', 1)[1][:-1])
1238         if Size > 1:
1239             raise BadExpression('Value (%s) Size larger than %d' %(Value, Size))
1240         return Value, 1
1241     if Value.startswith(TAB_UINT16) and Value.endswith(')'):
1242         Value, Size = ParseFieldValue(Value.split('(', 1)[1][:-1])
1243         if Size > 2:
1244             raise BadExpression('Value (%s) Size larger than %d' %(Value, Size))
1245         return Value, 2
1246     if Value.startswith(TAB_UINT32) and Value.endswith(')'):
1247         Value, Size = ParseFieldValue(Value.split('(', 1)[1][:-1])
1248         if Size > 4:
1249             raise BadExpression('Value (%s) Size larger than %d' %(Value, Size))
1250         return Value, 4
1251     if Value.startswith(TAB_UINT64) and Value.endswith(')'):
1252         Value, Size = ParseFieldValue(Value.split('(', 1)[1][:-1])
1253         if Size > 8:
1254             raise BadExpression('Value (%s) Size larger than %d' % (Value, Size))
1255         return Value, 8
1256     if Value.startswith(TAB_GUID) and Value.endswith(')'):
1257         Value = Value.split('(', 1)[1][:-1].strip()
1258         if Value[0] == '{' and Value[-1] == '}':
1259             TmpValue = GuidStructureStringToGuidString(Value)
1260             if not TmpValue:
1261                 raise BadExpression("Invalid GUID value string %s" % Value)
1262             Value = TmpValue
1263         if Value[0] == '"' and Value[-1] == '"':
1264             Value = Value[1:-1]
1265         try:
1266             Value = "'" + uuid.UUID(Value).get_bytes_le() + "'"
1267         except ValueError as Message:
1268             raise BadExpression(Message)
1269         Value, Size = ParseFieldValue(Value)
1270         return Value, 16
1271     if Value.startswith('L"') and Value.endswith('"'):
1272         # Unicode String
1273         # translate escape character
1274         Value = Value[1:]
1275         try:
1276             Value = eval(Value)
1277         except:
1278             Value = Value[1:-1]
1279         List = list(Value)
1280         List.reverse()
1281         Value = 0
1282         for Char in List:
1283             Value = (Value << 16) | ord(Char)
1284         return Value, (len(List) + 1) * 2
1285     if Value.startswith('"') and Value.endswith('"'):
1286         # ASCII String
1287         # translate escape character
1288         try:
1289             Value = eval(Value)
1290         except:
1291             Value = Value[1:-1]
1292         List = list(Value)
1293         List.reverse()
1294         Value = 0
1295         for Char in List:
1296             Value = (Value << 8) | ord(Char)
1297         return Value, len(List) + 1
1298     if Value.startswith("L'") and Value.endswith("'"):
1299         # Unicode Character Constant
1300         # translate escape character
1301         Value = Value[1:]
1302         try:
1303             Value = eval(Value)
1304         except:
1305             Value = Value[1:-1]
1306         List = list(Value)
1307         if len(List) == 0:
1308             raise BadExpression('Length %s is %s' % (Value, len(List)))
1309         List.reverse()
1310         Value = 0
1311         for Char in List:
1312             Value = (Value << 16) | ord(Char)
1313         return Value, len(List) * 2
1314     if Value.startswith("'") and Value.endswith("'"):
1315         # Character constant
1316         # translate escape character
1317         try:
1318             Value = eval(Value)
1319         except:
1320             Value = Value[1:-1]
1321         List = list(Value)
1322         if len(List) == 0:
1323             raise BadExpression('Length %s is %s' % (Value, len(List)))
1324         List.reverse()
1325         Value = 0
1326         for Char in List:
1327             Value = (Value << 8) | ord(Char)
1328         return Value, len(List)
1329     if Value.startswith('{') and Value.endswith('}'):
1330         # Byte array
1331         Value = Value[1:-1]
1332         List = [Item.strip() for Item in Value.split(',')]
1333         List.reverse()
1334         Value = 0
1335         RetSize = 0
1336         for Item in List:
1337             ItemValue, Size = ParseFieldValue(Item)
1338             RetSize += Size
1339             for I in range(Size):
1340                 Value = (Value << 8) | ((ItemValue >> 8 * I) & 0xff)
1341         return Value, RetSize
1342     if Value.startswith('DEVICE_PATH(') and Value.endswith(')'):
1343         Value = Value.replace("DEVICE_PATH(", '').rstrip(')')
1344         Value = Value.strip().strip('"')
1345         return ParseDevPathValue(Value)
1346     if Value.lower().startswith('0x'):
1347         try:
1348             Value = int(Value, 16)
1349         except:
1350             raise BadExpression("invalid hex value: %s" % Value)
1351         if Value == 0:
1352             return 0, 1
1353         return Value, (Value.bit_length() + 7) / 8
1354     if Value[0].isdigit():
1355         Value = int(Value, 10)
1356         if Value == 0:
1357             return 0, 1
1358         return Value, (Value.bit_length() + 7) / 8
1359     if Value.lower() == 'true':
1360         return 1, 1
1361     if Value.lower() == 'false':
1362         return 0, 1
1363     return Value, 1
1364
1365 ## AnalyzeDscPcd
1366 #
1367 #  Analyze DSC PCD value, since there is no data type info in DSC
1368 #  This function is used to match functions (AnalyzePcdData) used for retrieving PCD value from database
1369 #  1. Feature flag: TokenSpace.PcdCName|PcdValue
1370 #  2. Fix and Patch:TokenSpace.PcdCName|PcdValue[|VOID*[|MaxSize]]
1371 #  3. Dynamic default:
1372 #     TokenSpace.PcdCName|PcdValue[|VOID*[|MaxSize]]
1373 #     TokenSpace.PcdCName|PcdValue
1374 #  4. Dynamic VPD:
1375 #     TokenSpace.PcdCName|VpdOffset[|VpdValue]
1376 #     TokenSpace.PcdCName|VpdOffset[|MaxSize[|VpdValue]]
1377 #  5. Dynamic HII:
1378 #     TokenSpace.PcdCName|HiiString|VariableGuid|VariableOffset[|HiiValue]
1379 #  PCD value needs to be located in such kind of string, and the PCD value might be an expression in which
1380 #    there might have "|" operator, also in string value.
1381 #
1382 #  @param Setting: String contain information described above with "TokenSpace.PcdCName|" stripped
1383 #  @param PcdType: PCD type: feature, fixed, dynamic default VPD HII
1384 #  @param DataType: The datum type of PCD: VOID*, UNIT, BOOL
1385 #  @retval:
1386 #    ValueList: A List contain fields described above
1387 #    IsValid:   True if conforming EBNF, otherwise False
1388 #    Index:     The index where PcdValue is in ValueList
1389 #
1390 def AnalyzeDscPcd(Setting, PcdType, DataType=''):
1391     FieldList = AnalyzePcdExpression(Setting)
1392
1393     IsValid = True
1394     if PcdType in (MODEL_PCD_FIXED_AT_BUILD, MODEL_PCD_PATCHABLE_IN_MODULE, MODEL_PCD_DYNAMIC_DEFAULT, MODEL_PCD_DYNAMIC_EX_DEFAULT):
1395         Value = FieldList[0]
1396         Size = ''
1397         if len(FieldList) > 1 and FieldList[1]:
1398             DataType = FieldList[1]
1399             if FieldList[1] != TAB_VOID and StructPattern.match(FieldList[1]) is None:
1400                 IsValid = False
1401         if len(FieldList) > 2:
1402             Size = FieldList[2]
1403         if IsValid:
1404             if DataType == "":
1405                 IsValid = (len(FieldList) <= 1)
1406             else:
1407                 IsValid = (len(FieldList) <= 3)
1408
1409         if Size:
1410             try:
1411                 int(Size, 16) if Size.upper().startswith("0X") else int(Size)
1412             except:
1413                 IsValid = False
1414                 Size = -1
1415         return [str(Value), DataType, str(Size)], IsValid, 0
1416     elif PcdType == MODEL_PCD_FEATURE_FLAG:
1417         Value = FieldList[0]
1418         Size = ''
1419         IsValid = (len(FieldList) <= 1)
1420         return [Value, DataType, str(Size)], IsValid, 0
1421     elif PcdType in (MODEL_PCD_DYNAMIC_VPD, MODEL_PCD_DYNAMIC_EX_VPD):
1422         VpdOffset = FieldList[0]
1423         Value = Size = ''
1424         if not DataType == TAB_VOID:
1425             if len(FieldList) > 1:
1426                 Value = FieldList[1]
1427         else:
1428             if len(FieldList) > 1:
1429                 Size = FieldList[1]
1430             if len(FieldList) > 2:
1431                 Value = FieldList[2]
1432         if DataType == "":
1433             IsValid = (len(FieldList) <= 1)
1434         else:
1435             IsValid = (len(FieldList) <= 3)
1436         if Size:
1437             try:
1438                 int(Size, 16) if Size.upper().startswith("0X") else int(Size)
1439             except:
1440                 IsValid = False
1441                 Size = -1
1442         return [VpdOffset, str(Size), Value], IsValid, 2
1443     elif PcdType in (MODEL_PCD_DYNAMIC_HII, MODEL_PCD_DYNAMIC_EX_HII):
1444         IsValid = (3 <= len(FieldList) <= 5)
1445         HiiString = FieldList[0]
1446         Guid = Offset = Value = Attribute = ''
1447         if len(FieldList) > 1:
1448             Guid = FieldList[1]
1449         if len(FieldList) > 2:
1450             Offset = FieldList[2]
1451         if len(FieldList) > 3:
1452             Value = FieldList[3]
1453         if len(FieldList) > 4:
1454             Attribute = FieldList[4]
1455         return [HiiString, Guid, Offset, Value, Attribute], IsValid, 3
1456     return [], False, 0
1457
1458 ## AnalyzePcdData
1459 #
1460 #  Analyze the pcd Value, Datum type and TokenNumber.
1461 #  Used to avoid split issue while the value string contain "|" character
1462 #
1463 #  @param[in] Setting:  A String contain value/datum type/token number information;
1464 #
1465 #  @retval   ValueList: A List contain value, datum type and toke number.
1466 #
1467 def AnalyzePcdData(Setting):
1468     ValueList = ['', '', '']
1469
1470     ValueRe = re.compile(r'^\s*L?\".*\|.*\"')
1471     PtrValue = ValueRe.findall(Setting)
1472
1473     ValueUpdateFlag = False
1474
1475     if len(PtrValue) >= 1:
1476         Setting = re.sub(ValueRe, '', Setting)
1477         ValueUpdateFlag = True
1478
1479     TokenList = Setting.split(TAB_VALUE_SPLIT)
1480     ValueList[0:len(TokenList)] = TokenList
1481
1482     if ValueUpdateFlag:
1483         ValueList[0] = PtrValue[0]
1484
1485     return ValueList
1486
1487 ## check format of PCD value against its the datum type
1488 #
1489 # For PCD value setting
1490 #
1491 def CheckPcdDatum(Type, Value):
1492     if Type == TAB_VOID:
1493         ValueRe = re.compile(r'\s*L?\".*\"\s*$')
1494         if not (((Value.startswith('L"') or Value.startswith('"')) and Value.endswith('"'))
1495                 or (Value.startswith('{') and Value.endswith('}')) or (Value.startswith("L'") or Value.startswith("'") and Value.endswith("'"))
1496                ):
1497             return False, "Invalid value [%s] of type [%s]; must be in the form of {...} for array"\
1498                           ", \"...\" or \'...\' for string, L\"...\" or L\'...\' for unicode string" % (Value, Type)
1499         elif ValueRe.match(Value):
1500             # Check the chars in UnicodeString or CString is printable
1501             if Value.startswith("L"):
1502                 Value = Value[2:-1]
1503             else:
1504                 Value = Value[1:-1]
1505             Printset = set(string.printable)
1506             Printset.remove(TAB_PRINTCHAR_VT)
1507             Printset.add(TAB_PRINTCHAR_BS)
1508             Printset.add(TAB_PRINTCHAR_NUL)
1509             if not set(Value).issubset(Printset):
1510                 PrintList = sorted(Printset)
1511                 return False, "Invalid PCD string value of type [%s]; must be printable chars %s." % (Type, PrintList)
1512     elif Type == 'BOOLEAN':
1513         if Value not in ['TRUE', 'True', 'true', '0x1', '0x01', '1', 'FALSE', 'False', 'false', '0x0', '0x00', '0']:
1514             return False, "Invalid value [%s] of type [%s]; must be one of TRUE, True, true, 0x1, 0x01, 1"\
1515                           ", FALSE, False, false, 0x0, 0x00, 0" % (Value, Type)
1516     elif Type in [TAB_UINT8, TAB_UINT16, TAB_UINT32, TAB_UINT64]:
1517         if Value and int(Value, 0) < 0:
1518             return False, "PCD can't be set to negative value[%s] for datum type [%s]" % (Value, Type)
1519         try:
1520             Value = long(Value, 0)
1521             if Value > MAX_VAL_TYPE[Type]:
1522                 return False, "Too large PCD value[%s] for datum type [%s]" % (Value, Type)
1523         except:
1524             return False, "Invalid value [%s] of type [%s];"\
1525                           " must be a hexadecimal, decimal or octal in C language format." % (Value, Type)
1526     else:
1527         return True, "StructurePcd"
1528
1529     return True, ""
1530
1531 ## Split command line option string to list
1532 #
1533 # subprocess.Popen needs the args to be a sequence. Otherwise there's problem
1534 # in non-windows platform to launch command
1535 #
1536 def SplitOption(OptionString):
1537     OptionList = []
1538     LastChar = " "
1539     OptionStart = 0
1540     QuotationMark = ""
1541     for Index in range(0, len(OptionString)):
1542         CurrentChar = OptionString[Index]
1543         if CurrentChar in ['"', "'"]:
1544             if QuotationMark == CurrentChar:
1545                 QuotationMark = ""
1546             elif QuotationMark == "":
1547                 QuotationMark = CurrentChar
1548             continue
1549         elif QuotationMark:
1550             continue
1551
1552         if CurrentChar in ["/", "-"] and LastChar in [" ", "\t", "\r", "\n"]:
1553             if Index > OptionStart:
1554                 OptionList.append(OptionString[OptionStart:Index - 1])
1555             OptionStart = Index
1556         LastChar = CurrentChar
1557     OptionList.append(OptionString[OptionStart:])
1558     return OptionList
1559
1560 def CommonPath(PathList):
1561     P1 = min(PathList).split(os.path.sep)
1562     P2 = max(PathList).split(os.path.sep)
1563     for Index in xrange(min(len(P1), len(P2))):
1564         if P1[Index] != P2[Index]:
1565             return os.path.sep.join(P1[:Index])
1566     return os.path.sep.join(P1)
1567
1568 #
1569 # Convert string to C format array
1570 #
1571 def ConvertStringToByteArray(Value):
1572     Value = Value.strip()
1573     if not Value:
1574         return None
1575     if Value[0] == '{':
1576         if not Value.endswith('}'):
1577             return None
1578         Value = Value.replace(' ', '').replace('{', '').replace('}', '')
1579         ValFields = Value.split(',')
1580         try:
1581             for Index in range(len(ValFields)):
1582                 ValFields[Index] = str(int(ValFields[Index], 0))
1583         except ValueError:
1584             return None
1585         Value = '{' + ','.join(ValFields) + '}'
1586         return Value
1587
1588     Unicode = False
1589     if Value.startswith('L"'):
1590         if not Value.endswith('"'):
1591             return None
1592         Value = Value[1:]
1593         Unicode = True
1594     elif not Value.startswith('"') or not Value.endswith('"'):
1595         return None
1596
1597     Value = eval(Value)         # translate escape character
1598     NewValue = '{'
1599     for Index in range(0, len(Value)):
1600         if Unicode:
1601             NewValue = NewValue + str(ord(Value[Index]) % 0x10000) + ','
1602         else:
1603             NewValue = NewValue + str(ord(Value[Index]) % 0x100) + ','
1604     Value = NewValue + '0}'
1605     return Value
1606
1607 class PathClass(object):
1608     def __init__(self, File='', Root='', AlterRoot='', Type='', IsBinary=False,
1609                  Arch='COMMON', ToolChainFamily='', Target='', TagName='', ToolCode=''):
1610         self.Arch = Arch
1611         self.File = str(File)
1612         if os.path.isabs(self.File):
1613             self.Root = ''
1614             self.AlterRoot = ''
1615         else:
1616             self.Root = str(Root)
1617             self.AlterRoot = str(AlterRoot)
1618
1619         # Remove any '.' and '..' in path
1620         if self.Root:
1621             self.Root = mws.getWs(self.Root, self.File)
1622             self.Path = os.path.normpath(os.path.join(self.Root, self.File))
1623             self.Root = os.path.normpath(CommonPath([self.Root, self.Path]))
1624             # eliminate the side-effect of 'C:'
1625             if self.Root[-1] == ':':
1626                 self.Root += os.path.sep
1627             # file path should not start with path separator
1628             if self.Root[-1] == os.path.sep:
1629                 self.File = self.Path[len(self.Root):]
1630             else:
1631                 self.File = self.Path[len(self.Root) + 1:]
1632         else:
1633             self.Path = os.path.normpath(self.File)
1634
1635         self.SubDir, self.Name = os.path.split(self.File)
1636         self.BaseName, self.Ext = os.path.splitext(self.Name)
1637
1638         if self.Root:
1639             if self.SubDir:
1640                 self.Dir = os.path.join(self.Root, self.SubDir)
1641             else:
1642                 self.Dir = self.Root
1643         else:
1644             self.Dir = self.SubDir
1645
1646         if IsBinary:
1647             self.Type = Type
1648         else:
1649             self.Type = self.Ext.lower()
1650
1651         self.IsBinary = IsBinary
1652         self.Target = Target
1653         self.TagName = TagName
1654         self.ToolCode = ToolCode
1655         self.ToolChainFamily = ToolChainFamily
1656
1657     ## Convert the object of this class to a string
1658     #
1659     #  Convert member Path of the class to a string
1660     #
1661     #  @retval string Formatted String
1662     #
1663     def __str__(self):
1664         return self.Path
1665
1666     ## Override __eq__ function
1667     #
1668     # Check whether PathClass are the same
1669     #
1670     # @retval False The two PathClass are different
1671     # @retval True  The two PathClass are the same
1672     #
1673     def __eq__(self, Other):
1674         return self.Path == str(Other)
1675
1676     ## Override __cmp__ function
1677     #
1678     # Customize the comparsion operation of two PathClass
1679     #
1680     # @retval 0     The two PathClass are different
1681     # @retval -1    The first PathClass is less than the second PathClass
1682     # @retval 1     The first PathClass is Bigger than the second PathClass
1683     def __cmp__(self, Other):
1684         OtherKey = str(Other)
1685
1686         SelfKey = self.Path
1687         if SelfKey == OtherKey:
1688             return 0
1689         elif SelfKey > OtherKey:
1690             return 1
1691         else:
1692             return -1
1693
1694     ## Override __hash__ function
1695     #
1696     # Use Path as key in hash table
1697     #
1698     # @retval string Key for hash table
1699     #
1700     def __hash__(self):
1701         return hash(self.Path)
1702
1703     @cached_property
1704     def Key(self):
1705         return self.Path.upper()
1706
1707     @property
1708     def TimeStamp(self):
1709         return os.stat(self.Path)[8]
1710
1711     def Validate(self, Type='', CaseSensitive=True):
1712         if GlobalData.gCaseInsensitive:
1713             CaseSensitive = False
1714         if Type and Type.lower() != self.Type:
1715             return FILE_TYPE_MISMATCH, '%s (expect %s but got %s)' % (self.File, Type, self.Type)
1716
1717         RealFile, RealRoot = RealPath2(self.File, self.Root, self.AlterRoot)
1718         if not RealRoot and not RealFile:
1719             RealFile = self.File
1720             if self.AlterRoot:
1721                 RealFile = os.path.join(self.AlterRoot, self.File)
1722             elif self.Root:
1723                 RealFile = os.path.join(self.Root, self.File)
1724             if len (mws.getPkgPath()) == 0:
1725                 return FILE_NOT_FOUND, os.path.join(self.AlterRoot, RealFile)
1726             else:
1727                 return FILE_NOT_FOUND, "%s is not found in packages path:\n\t%s" % (self.File, '\n\t'.join(mws.getPkgPath()))
1728
1729         ErrorCode = 0
1730         ErrorInfo = ''
1731         if RealRoot != self.Root or RealFile != self.File:
1732             if CaseSensitive and (RealFile != self.File or (RealRoot != self.Root and RealRoot != self.AlterRoot)):
1733                 ErrorCode = FILE_CASE_MISMATCH
1734                 ErrorInfo = self.File + '\n\t' + RealFile + " [in file system]"
1735
1736             self.SubDir, self.Name = os.path.split(RealFile)
1737             self.BaseName, self.Ext = os.path.splitext(self.Name)
1738             if self.SubDir:
1739                 self.Dir = os.path.join(RealRoot, self.SubDir)
1740             else:
1741                 self.Dir = RealRoot
1742             self.File = RealFile
1743             self.Root = RealRoot
1744             self.Path = os.path.join(RealRoot, RealFile)
1745         return ErrorCode, ErrorInfo
1746
1747 ## Parse PE image to get the required PE informaion.
1748 #
1749 class PeImageClass():
1750     ## Constructor
1751     #
1752     #   @param  File FilePath of PeImage
1753     #
1754     def __init__(self, PeFile):
1755         self.FileName   = PeFile
1756         self.IsValid    = False
1757         self.Size       = 0
1758         self.EntryPoint = 0
1759         self.SectionAlignment  = 0
1760         self.SectionHeaderList = []
1761         self.ErrorInfo = ''
1762         try:
1763             PeObject = open(PeFile, 'rb')
1764         except:
1765             self.ErrorInfo = self.FileName + ' can not be found\n'
1766             return
1767         # Read DOS header
1768         ByteArray = array.array('B')
1769         ByteArray.fromfile(PeObject, 0x3E)
1770         ByteList = ByteArray.tolist()
1771         # DOS signature should be 'MZ'
1772         if self._ByteListToStr (ByteList[0x0:0x2]) != 'MZ':
1773             self.ErrorInfo = self.FileName + ' has no valid DOS signature MZ'
1774             return
1775
1776         # Read 4 byte PE Signature
1777         PeOffset = self._ByteListToInt(ByteList[0x3C:0x3E])
1778         PeObject.seek(PeOffset)
1779         ByteArray = array.array('B')
1780         ByteArray.fromfile(PeObject, 4)
1781         # PE signature should be 'PE\0\0'
1782         if ByteArray.tostring() != 'PE\0\0':
1783             self.ErrorInfo = self.FileName + ' has no valid PE signature PE00'
1784             return
1785
1786         # Read PE file header
1787         ByteArray = array.array('B')
1788         ByteArray.fromfile(PeObject, 0x14)
1789         ByteList = ByteArray.tolist()
1790         SecNumber = self._ByteListToInt(ByteList[0x2:0x4])
1791         if SecNumber == 0:
1792             self.ErrorInfo = self.FileName + ' has no section header'
1793             return
1794
1795         # Read PE optional header
1796         OptionalHeaderSize = self._ByteListToInt(ByteArray[0x10:0x12])
1797         ByteArray = array.array('B')
1798         ByteArray.fromfile(PeObject, OptionalHeaderSize)
1799         ByteList = ByteArray.tolist()
1800         self.EntryPoint       = self._ByteListToInt(ByteList[0x10:0x14])
1801         self.SectionAlignment = self._ByteListToInt(ByteList[0x20:0x24])
1802         self.Size             = self._ByteListToInt(ByteList[0x38:0x3C])
1803
1804         # Read each Section Header
1805         for Index in range(SecNumber):
1806             ByteArray = array.array('B')
1807             ByteArray.fromfile(PeObject, 0x28)
1808             ByteList = ByteArray.tolist()
1809             SecName  = self._ByteListToStr(ByteList[0:8])
1810             SecVirtualSize = self._ByteListToInt(ByteList[8:12])
1811             SecRawAddress  = self._ByteListToInt(ByteList[20:24])
1812             SecVirtualAddress = self._ByteListToInt(ByteList[12:16])
1813             self.SectionHeaderList.append((SecName, SecVirtualAddress, SecRawAddress, SecVirtualSize))
1814         self.IsValid = True
1815         PeObject.close()
1816
1817     def _ByteListToStr(self, ByteList):
1818         String = ''
1819         for index in range(len(ByteList)):
1820             if ByteList[index] == 0:
1821                 break
1822             String += chr(ByteList[index])
1823         return String
1824
1825     def _ByteListToInt(self, ByteList):
1826         Value = 0
1827         for index in range(len(ByteList) - 1, -1, -1):
1828             Value = (Value << 8) | int(ByteList[index])
1829         return Value
1830
1831 class DefaultStore():
1832     def __init__(self, DefaultStores ):
1833
1834         self.DefaultStores = DefaultStores
1835     def DefaultStoreID(self, DefaultStoreName):
1836         for key, value in self.DefaultStores.items():
1837             if value == DefaultStoreName:
1838                 return key
1839         return None
1840     def GetDefaultDefault(self):
1841         if not self.DefaultStores or "0" in self.DefaultStores:
1842             return "0", TAB_DEFAULT_STORES_DEFAULT
1843         else:
1844             minvalue = min(int(value_str) for value_str in self.DefaultStores)
1845             return (str(minvalue), self.DefaultStores[str(minvalue)])
1846     def GetMin(self, DefaultSIdList):
1847         if not DefaultSIdList:
1848             return TAB_DEFAULT_STORES_DEFAULT
1849         storeidset = {storeid for storeid, storename in self.DefaultStores.values() if storename in DefaultSIdList}
1850         if not storeidset:
1851             return ""
1852         minid = min(storeidset )
1853         for sid, name in self.DefaultStores.values():
1854             if sid == minid:
1855                 return name
1856
1857 class SkuClass():
1858     DEFAULT = 0
1859     SINGLE = 1
1860     MULTIPLE =2
1861
1862     def __init__(self,SkuIdentifier='', SkuIds=None):
1863         if SkuIds is None:
1864             SkuIds = {}
1865
1866         for SkuName in SkuIds:
1867             SkuId = SkuIds[SkuName][0]
1868             skuid_num = int(SkuId, 16) if SkuId.upper().startswith("0X") else int(SkuId)
1869             if skuid_num > 0xFFFFFFFFFFFFFFFF:
1870                 EdkLogger.error("build", PARAMETER_INVALID,
1871                             ExtraData = "SKU-ID [%s] value %s exceeds the max value of UINT64"
1872                                       % (SkuName, SkuId))
1873
1874         self.AvailableSkuIds = sdict()
1875         self.SkuIdSet = []
1876         self.SkuIdNumberSet = []
1877         self.SkuData = SkuIds
1878         self._SkuInherit = {}
1879         self._SkuIdentifier = SkuIdentifier
1880         if SkuIdentifier == '' or SkuIdentifier is None:
1881             self.SkuIdSet = ['DEFAULT']
1882             self.SkuIdNumberSet = ['0U']
1883         elif SkuIdentifier == 'ALL':
1884             self.SkuIdSet = SkuIds.keys()
1885             self.SkuIdNumberSet = [num[0].strip() + 'U' for num in SkuIds.values()]
1886         else:
1887             r = SkuIdentifier.split('|')
1888             self.SkuIdSet=[(r[k].strip()).upper() for k in range(len(r))]
1889             k = None
1890             try:
1891                 self.SkuIdNumberSet = [SkuIds[k][0].strip() + 'U' for k in self.SkuIdSet]
1892             except Exception:
1893                 EdkLogger.error("build", PARAMETER_INVALID,
1894                             ExtraData = "SKU-ID [%s] is not supported by the platform. [Valid SKU-ID: %s]"
1895                                       % (k, " | ".join(SkuIds.keys())))
1896         for each in self.SkuIdSet:
1897             if each in SkuIds:
1898                 self.AvailableSkuIds[each] = SkuIds[each][0]
1899             else:
1900                 EdkLogger.error("build", PARAMETER_INVALID,
1901                             ExtraData="SKU-ID [%s] is not supported by the platform. [Valid SKU-ID: %s]"
1902                                       % (each, " | ".join(SkuIds.keys())))
1903         if self.SkuUsageType != SkuClass.SINGLE:
1904             self.AvailableSkuIds.update({'DEFAULT':0, 'COMMON':0})
1905         if self.SkuIdSet:
1906             GlobalData.gSkuids = (self.SkuIdSet)
1907             if 'COMMON' in GlobalData.gSkuids:
1908                 GlobalData.gSkuids.remove('COMMON')
1909             if self.SkuUsageType == self.SINGLE:
1910                 if len(GlobalData.gSkuids) != 1:
1911                     if 'DEFAULT' in GlobalData.gSkuids:
1912                         GlobalData.gSkuids.remove('DEFAULT')
1913             if GlobalData.gSkuids:
1914                 GlobalData.gSkuids.sort()
1915
1916     def GetNextSkuId(self, skuname):
1917         if not self._SkuInherit:
1918             self._SkuInherit = {}
1919             for item in self.SkuData.values():
1920                 self._SkuInherit[item[1]]=item[2] if item[2] else "DEFAULT"
1921         return self._SkuInherit.get(skuname, "DEFAULT")
1922
1923     def GetSkuChain(self, sku):
1924         if sku == "DEFAULT":
1925             return ["DEFAULT"]
1926         skulist = [sku]
1927         nextsku = sku
1928         while True:
1929             nextsku = self.GetNextSkuId(nextsku)
1930             skulist.append(nextsku)
1931             if nextsku == "DEFAULT":
1932                 break
1933         skulist.reverse()
1934         return skulist
1935     def SkuOverrideOrder(self):
1936         skuorderset = []
1937         for skuname in self.SkuIdSet:
1938             skuorderset.append(self.GetSkuChain(skuname))
1939
1940         skuorder = []
1941         for index in range(max(len(item) for item in skuorderset)):
1942             for subset in skuorderset:
1943                 if index > len(subset)-1:
1944                     continue
1945                 if subset[index] in skuorder:
1946                     continue
1947                 skuorder.append(subset[index])
1948
1949         return skuorder
1950
1951     @property
1952     def SkuUsageType(self):
1953         if self._SkuIdentifier.upper() == "ALL":
1954             return SkuClass.MULTIPLE
1955
1956         if len(self.SkuIdSet) == 1:
1957             if self.SkuIdSet[0] == 'DEFAULT':
1958                 return SkuClass.DEFAULT
1959             return SkuClass.SINGLE
1960         if len(self.SkuIdSet) == 2 and 'DEFAULT' in self.SkuIdSet:
1961             return SkuClass.SINGLE
1962         return SkuClass.MULTIPLE
1963
1964     def DumpSkuIdArrary(self):
1965         if self.SkuUsageType == SkuClass.SINGLE:
1966             return "{0x0}"
1967         ArrayStrList = []
1968         for skuname in self.AvailableSkuIds:
1969             if skuname == "COMMON":
1970                 continue
1971             while skuname != "DEFAULT":
1972                 ArrayStrList.append(hex(int(self.AvailableSkuIds[skuname])))
1973                 skuname = self.GetNextSkuId(skuname)
1974             ArrayStrList.append("0x0")
1975         return "{{{myList}}}".format(myList=",".join(ArrayStrList))
1976
1977     @property
1978     def AvailableSkuIdSet(self):
1979         return self.AvailableSkuIds
1980
1981     @property
1982     def SystemSkuId(self):
1983         if self.SkuUsageType == SkuClass.SINGLE:
1984             if len(self.SkuIdSet) == 1:
1985                 return self.SkuIdSet[0]
1986             else:
1987                 return self.SkuIdSet[0] if self.SkuIdSet[0] != 'DEFAULT' else self.SkuIdSet[1]
1988         else:
1989             return 'DEFAULT'
1990
1991 #
1992 # Pack a registry format GUID
1993 #
1994 def PackRegistryFormatGuid(Guid):
1995     return PackGUID(Guid.split('-'))
1996
1997 ##  Get the integer value from string like "14U" or integer like 2
1998 #
1999 #   @param      Input   The object that may be either a integer value or a string
2000 #
2001 #   @retval     Value    The integer value that the input represents
2002 #
2003 def GetIntegerValue(Input):
2004     if type(Input) in (int, long):
2005         return Input
2006     String = Input
2007     if String.endswith("U"):
2008         String = String[:-1]
2009     if String.endswith("ULL"):
2010         String = String[:-3]
2011     if String.endswith("LL"):
2012         String = String[:-2]
2013
2014     if String.startswith("0x") or String.startswith("0X"):
2015         return int(String, 16)
2016     elif String == '':
2017         return 0
2018     else:
2019         return int(String)
2020
2021 #
2022 # Pack a GUID (registry format) list into a buffer and return it
2023 #
2024 def PackGUID(Guid):
2025     return pack(PACK_PATTERN_GUID,
2026                 int(Guid[0], 16),
2027                 int(Guid[1], 16),
2028                 int(Guid[2], 16),
2029                 int(Guid[3][-4:-2], 16),
2030                 int(Guid[3][-2:], 16),
2031                 int(Guid[4][-12:-10], 16),
2032                 int(Guid[4][-10:-8], 16),
2033                 int(Guid[4][-8:-6], 16),
2034                 int(Guid[4][-6:-4], 16),
2035                 int(Guid[4][-4:-2], 16),
2036                 int(Guid[4][-2:], 16)
2037                 )
2038
2039 #
2040 # Pack a GUID (byte) list into a buffer and return it
2041 #
2042 def PackByteFormatGUID(Guid):
2043     return pack(PACK_PATTERN_GUID,
2044                 Guid[0],
2045                 Guid[1],
2046                 Guid[2],
2047                 Guid[3],
2048                 Guid[4],
2049                 Guid[5],
2050                 Guid[6],
2051                 Guid[7],
2052                 Guid[8],
2053                 Guid[9],
2054                 Guid[10],
2055                 )
2056
2057 ## DeepCopy dict/OrderedDict recusively
2058 #
2059 #   @param      ori_dict    a nested dict or ordereddict
2060 #
2061 #   @retval     new dict or orderdict
2062 #
2063 def CopyDict(ori_dict):
2064     dict_type = ori_dict.__class__
2065     if dict_type not in (dict,OrderedDict):
2066         return ori_dict
2067     new_dict = dict_type()
2068     for key in ori_dict:
2069         if isinstance(ori_dict[key],(dict,OrderedDict)):
2070             new_dict[key] = CopyDict(ori_dict[key])
2071         else:
2072             new_dict[key] = ori_dict[key]
2073     return new_dict
2074
2075 #
2076 # Remove the c/c++ comments: // and /* */
2077 #
2078 def RemoveCComments(ctext):
2079     return re.sub('//.*?\n|/\*.*?\*/', '\n', ctext, flags=re.S)
2080 ##
2081 #
2082 # This acts like the main() function for the script, unless it is 'import'ed into another
2083 # script.
2084 #
2085 if __name__ == '__main__':
2086     pass
2087