BaseTools/Source/Python/Common/Misc.py

   1 ## @file
   2 # Common routines used by all tools
   3 #
   4 # Copyright (c) 2007 - 2019, Intel Corporation. All rights reserved.<BR>
   5 # This program and the accompanying materials
   6 # are licensed and made available under the terms and conditions of the BSD License
   7 # which accompanies this distribution.  The full text of the license may be found at
   8 # http://opensource.org/licenses/bsd-license.php
   9 #
  10 # THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS,
  11 # WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED.
  12 #
  13
  14 ##
  15 # Import Modules
  16 #
  17 from __future__ import absolute_import
  18 import Common.LongFilePathOs as os
  19 import sys
  20 import string
  21 import threading
  22 import time
  23 import re
  24 import pickle
  25 import array
  26 import shutil
  27 from random import sample
  28 from struct import pack
  29 from UserDict import IterableUserDict
  30 from UserList import UserList
  31
  32 from Common import EdkLogger as EdkLogger
  33 from Common import GlobalData as GlobalData
  34 from .DataType import *
  35 from .BuildToolError import *
  36 from CommonDataClass.DataClass import *
  37 from .Parsing import GetSplitValueList
  38 from Common.LongFilePathSupport import OpenLongFilePath as open
  39 from Common.MultipleWorkspace import MultipleWorkspace as mws
  40 import uuid
  41 from CommonDataClass.Exceptions import BadExpression
  42 from Common.caching import cached_property
  43 import subprocess
  44 from collections import OrderedDict
  45 ## Regular expression used to find out place holders in string template
  46 gPlaceholderPattern = re.compile("\$\{([^$()\s]+)\}", re.MULTILINE | re.UNICODE)
  47
  48 ## regular expressions for map file processing
  49 startPatternGeneral = re.compile("^Start[' ']+Length[' ']+Name[' ']+Class")
  50 addressPatternGeneral = re.compile("^Address[' ']+Publics by Value[' ']+Rva\+Base")
  51 valuePatternGcc = re.compile('^([\w_\.]+) +([\da-fA-Fx]+) +([\da-fA-Fx]+)$')
  52 pcdPatternGcc = re.compile('^([\da-fA-Fx]+) +([\da-fA-Fx]+)')
  53 secReGeneral = re.compile('^([\da-fA-F]+):([\da-fA-F]+) +([\da-fA-F]+)[Hh]? +([.\w\$]+) +(\w+)', re.UNICODE)
  54
  55 StructPattern = re.compile(r'[_a-zA-Z][0-9A-Za-z_]*$')
  56
  57 ## Dictionary used to store file time stamp for quick re-access
  58 gFileTimeStampCache = {}    # {file path : file time stamp}
  59
  60 ## Dictionary used to store dependencies of files
  61 gDependencyDatabase = {}    # arch : {file path : [dependent files list]}
  62
  63 #
  64 # If a module is built more than once with different PCDs or library classes
  65 # a temporary INF file with same content is created, the temporary file is removed
  66 # when build exits.
  67 #
  68 _TempInfs = []
  69
  70 def GetVariableOffset(mapfilepath, efifilepath, varnames):
  71     """ Parse map file to get variable offset in current EFI file
  72     @param mapfilepath    Map file absolution path
  73     @param efifilepath:   EFI binary file full path
  74     @param varnames       iteratable container whose elements are variable names to be searched
  75
  76     @return List whos elements are tuple with variable name and raw offset
  77     """
  78     lines = []
  79     try:
  80         f = open(mapfilepath, 'r')
  81         lines = f.readlines()
  82         f.close()
  83     except:
  84         return None
  85
  86     if len(lines) == 0: return None
  87     firstline = lines[0].strip()
  88     if (firstline.startswith("Archive member included ") and
  89         firstline.endswith(" file (symbol)")):
  90         return _parseForGCC(lines, efifilepath, varnames)
  91     if firstline.startswith("# Path:"):
  92         return _parseForXcode(lines, efifilepath, varnames)
  93     return _parseGeneral(lines, efifilepath, varnames)
  94
  95 def _parseForXcode(lines, efifilepath, varnames):
  96     status = 0
  97     ret = []
  98     for line in lines:
  99         line = line.strip()
 100         if status == 0 and line == "# Symbols:":
 101             status = 1
 102             continue
 103         if status == 1 and len(line) != 0:
 104             for varname in varnames:
 105                 if varname in line:
 106                     # cannot pregenerate this RegEx since it uses varname from varnames.
 107                     m = re.match('^([\da-fA-FxX]+)([\s\S]*)([_]*%s)$' % varname, line)
 108                     if m is not None:
 109                         ret.append((varname, m.group(1)))
 110     return ret
 111
 112 def _parseForGCC(lines, efifilepath, varnames):
 113     """ Parse map file generated by GCC linker """
 114     status = 0
 115     sections = []
 116     varoffset = []
 117     for index, line in enumerate(lines):
 118         line = line.strip()
 119         # status machine transection
 120         if status == 0 and line == "Memory Configuration":
 121             status = 1
 122             continue
 123         elif status == 1 and line == 'Linker script and memory map':
 124             status = 2
 125             continue
 126         elif status ==2 and line == 'START GROUP':
 127             status = 3
 128             continue
 129
 130         # status handler
 131         if status == 3:
 132             m = valuePatternGcc.match(line)
 133             if m is not None:
 134                 sections.append(m.groups(0))
 135             for varname in varnames:
 136                 Str = ''
 137                 m = re.match("^.data.(%s)" % varname, line)
 138                 if m is not None:
 139                     m = re.match(".data.(%s)$" % varname, line)
 140                     if m is not None:
 141                         Str = lines[index + 1]
 142                     else:
 143                         Str = line[len(".data.%s" % varname):]
 144                     if Str:
 145                         m = pcdPatternGcc.match(Str.strip())
 146                         if m is not None:
 147                             varoffset.append((varname, int(m.groups(0)[0], 16), int(sections[-1][1], 16), sections[-1][0]))
 148
 149     if not varoffset:
 150         return []
 151     # get section information from efi file
 152     efisecs = PeImageClass(efifilepath).SectionHeaderList
 153     if efisecs is None or len(efisecs) == 0:
 154         return []
 155     #redirection
 156     redirection = 0
 157     for efisec in efisecs:
 158         for section in sections:
 159             if section[0].strip() == efisec[0].strip() and section[0].strip() == '.text':
 160                 redirection = int(section[1], 16) - efisec[1]
 161
 162     ret = []
 163     for var in varoffset:
 164         for efisec in efisecs:
 165             if var[1] >= efisec[1] and var[1] < efisec[1]+efisec[3]:
 166                 ret.append((var[0], hex(efisec[2] + var[1] - efisec[1] - redirection)))
 167     return ret
 168
 169 def _parseGeneral(lines, efifilepath, varnames):
 170     status = 0    #0 - beginning of file; 1 - PE section definition; 2 - symbol table
 171     secs  = []    # key = section name
 172     varoffset = []
 173     symRe = re.compile('^([\da-fA-F]+):([\da-fA-F]+) +([\.:\\\\\w\?@\$]+) +([\da-fA-F]+)', re.UNICODE)
 174
 175     for line in lines:
 176         line = line.strip()
 177         if startPatternGeneral.match(line):
 178             status = 1
 179             continue
 180         if addressPatternGeneral.match(line):
 181             status = 2
 182             continue
 183         if line.startswith("entry point at"):
 184             status = 3
 185             continue
 186         if status == 1 and len(line) != 0:
 187             m =  secReGeneral.match(line)
 188             assert m is not None, "Fail to parse the section in map file , line is %s" % line
 189             sec_no, sec_start, sec_length, sec_name, sec_class = m.groups(0)
 190             secs.append([int(sec_no, 16), int(sec_start, 16), int(sec_length, 16), sec_name, sec_class])
 191         if status == 2 and len(line) != 0:
 192             for varname in varnames:
 193                 m = symRe.match(line)
 194                 assert m is not None, "Fail to parse the symbol in map file, line is %s" % line
 195                 sec_no, sym_offset, sym_name, vir_addr = m.groups(0)
 196                 sec_no     = int(sec_no,     16)
 197                 sym_offset = int(sym_offset, 16)
 198                 vir_addr   = int(vir_addr,   16)
 199                 # cannot pregenerate this RegEx since it uses varname from varnames.
 200                 m2 = re.match('^[_]*(%s)' % varname, sym_name)
 201                 if m2 is not None:
 202                     # fond a binary pcd entry in map file
 203                     for sec in secs:
 204                         if sec[0] == sec_no and (sym_offset >= sec[1] and sym_offset < sec[1] + sec[2]):
 205                             varoffset.append([varname, sec[3], sym_offset, vir_addr, sec_no])
 206
 207     if not varoffset: return []
 208
 209     # get section information from efi file
 210     efisecs = PeImageClass(efifilepath).SectionHeaderList
 211     if efisecs is None or len(efisecs) == 0:
 212         return []
 213
 214     ret = []
 215     for var in varoffset:
 216         index = 0
 217         for efisec in efisecs:
 218             index = index + 1
 219             if var[1].strip() == efisec[0].strip():
 220                 ret.append((var[0], hex(efisec[2] + var[2])))
 221             elif var[4] == index:
 222                 ret.append((var[0], hex(efisec[2] + var[2])))
 223
 224     return ret
 225
 226 ## Routine to process duplicated INF
 227 #
 228 #  This function is called by following two cases:
 229 #  Case 1 in DSC:
 230 #    [components.arch]
 231 #    Pkg/module/module.inf
 232 #    Pkg/module/module.inf {
 233 #      <Defines>
 234 #        FILE_GUID = 0D1B936F-68F3-4589-AFCC-FB8B7AEBC836
 235 #    }
 236 #  Case 2 in FDF:
 237 #    INF Pkg/module/module.inf
 238 #    INF FILE_GUID = 0D1B936F-68F3-4589-AFCC-FB8B7AEBC836 Pkg/module/module.inf
 239 #
 240 #  This function copies Pkg/module/module.inf to
 241 #  Conf/.cache/0D1B936F-68F3-4589-AFCC-FB8B7AEBC836module.inf
 242 #
 243 #  @param Path     Original PathClass object
 244 #  @param BaseName New file base name
 245 #
 246 #  @retval         return the new PathClass object
 247 #
 248 def ProcessDuplicatedInf(Path, BaseName, Workspace):
 249     Filename = os.path.split(Path.File)[1]
 250     if '.' in Filename:
 251         Filename = BaseName + Path.BaseName + Filename[Filename.rfind('.'):]
 252     else:
 253         Filename = BaseName + Path.BaseName
 254
 255     #
 256     # If -N is specified on command line, cache is disabled
 257     # The directory has to be created
 258     #
 259     DbDir = os.path.split(GlobalData.gDatabasePath)[0]
 260     if not os.path.exists(DbDir):
 261         os.makedirs(DbDir)
 262     #
 263     # A temporary INF is copied to database path which must have write permission
 264     # The temporary will be removed at the end of build
 265     # In case of name conflict, the file name is
 266     # FILE_GUIDBaseName (0D1B936F-68F3-4589-AFCC-FB8B7AEBC836module.inf)
 267     #
 268     TempFullPath = os.path.join(DbDir,
 269                                 Filename)
 270     RtPath = PathClass(Path.File, Workspace)
 271     #
 272     # Modify the full path to temporary path, keep other unchanged
 273     #
 274     # To build same module more than once, the module path with FILE_GUID overridden has
 275     # the file name FILE_GUIDmodule.inf, but the relative path (self.MetaFile.File) is the real path
 276     # in DSC which is used as relative path by C files and other files in INF.
 277     # A trick was used: all module paths are PathClass instances, after the initialization
 278     # of PathClass, the PathClass.Path is overridden by the temporary INF path.
 279     #
 280     # The reason for creating a temporary INF is:
 281     # Platform.Modules which is the base to create ModuleAutoGen objects is a dictionary,
 282     # the key is the full path of INF, the value is an object to save overridden library instances, PCDs.
 283     # A different key for the same module is needed to create different output directory,
 284     # retrieve overridden PCDs, library instances.
 285     #
 286     # The BaseName is the FILE_GUID which is also the output directory name.
 287     #
 288     #
 289     RtPath.Path = TempFullPath
 290     RtPath.BaseName = BaseName
 291     #
 292     # If file exists, compare contents
 293     #
 294     if os.path.exists(TempFullPath):
 295         with open(str(Path), 'rb') as f1, open(TempFullPath, 'rb') as f2:
 296             if f1.read() == f2.read():
 297                 return RtPath
 298     _TempInfs.append(TempFullPath)
 299     shutil.copy2(str(Path), TempFullPath)
 300     return RtPath
 301
 302 ## Remove temporary created INFs whose paths were saved in _TempInfs
 303 #
 304 def ClearDuplicatedInf():
 305     while _TempInfs:
 306         File = _TempInfs.pop()
 307         if os.path.exists(File):
 308             os.remove(File)
 309
 310 ## Convert GUID string in xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx style to C structure style
 311 #
 312 #   @param      Guid    The GUID string
 313 #
 314 #   @retval     string  The GUID string in C structure style
 315 #
 316 def GuidStringToGuidStructureString(Guid):
 317     GuidList = Guid.split('-')
 318     Result = '{'
 319     for Index in range(0, 3, 1):
 320         Result = Result + '0x' + GuidList[Index] + ', '
 321     Result = Result + '{0x' + GuidList[3][0:2] + ', 0x' + GuidList[3][2:4]
 322     for Index in range(0, 12, 2):
 323         Result = Result + ', 0x' + GuidList[4][Index:Index + 2]
 324     Result += '}}'
 325     return Result
 326
 327 ## Convert GUID structure in byte array to xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
 328 #
 329 #   @param      GuidValue   The GUID value in byte array
 330 #
 331 #   @retval     string      The GUID value in xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx format
 332 #
 333 def GuidStructureByteArrayToGuidString(GuidValue):
 334     guidValueString = GuidValue.lower().replace("{", "").replace("}", "").replace(" ", "").replace(";", "")
 335     guidValueList = guidValueString.split(",")
 336     if len(guidValueList) != 16:
 337         return ''
 338         #EdkLogger.error(None, None, "Invalid GUID value string %s" % GuidValue)
 339     try:
 340         return "%02x%02x%02x%02x-%02x%02x-%02x%02x-%02x%02x-%02x%02x%02x%02x%02x%02x" % (
 341                 int(guidValueList[3], 16),
 342                 int(guidValueList[2], 16),
 343                 int(guidValueList[1], 16),
 344                 int(guidValueList[0], 16),
 345                 int(guidValueList[5], 16),
 346                 int(guidValueList[4], 16),
 347                 int(guidValueList[7], 16),
 348                 int(guidValueList[6], 16),
 349                 int(guidValueList[8], 16),
 350                 int(guidValueList[9], 16),
 351                 int(guidValueList[10], 16),
 352                 int(guidValueList[11], 16),
 353                 int(guidValueList[12], 16),
 354                 int(guidValueList[13], 16),
 355                 int(guidValueList[14], 16),
 356                 int(guidValueList[15], 16)
 357                 )
 358     except:
 359         return ''
 360
 361 ## Convert GUID string in C structure style to xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
 362 #
 363 #   @param      GuidValue   The GUID value in C structure format
 364 #
 365 #   @retval     string      The GUID value in xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx format
 366 #
 367 def GuidStructureStringToGuidString(GuidValue):
 368     if not GlobalData.gGuidCFormatPattern.match(GuidValue):
 369         return ''
 370     guidValueString = GuidValue.lower().replace("{", "").replace("}", "").replace(" ", "").replace(";", "")
 371     guidValueList = guidValueString.split(",")
 372     if len(guidValueList) != 11:
 373         return ''
 374         #EdkLogger.error(None, None, "Invalid GUID value string %s" % GuidValue)
 375     try:
 376         return "%08x-%04x-%04x-%02x%02x-%02x%02x%02x%02x%02x%02x" % (
 377                 int(guidValueList[0], 16),
 378                 int(guidValueList[1], 16),
 379                 int(guidValueList[2], 16),
 380                 int(guidValueList[3], 16),
 381                 int(guidValueList[4], 16),
 382                 int(guidValueList[5], 16),
 383                 int(guidValueList[6], 16),
 384                 int(guidValueList[7], 16),
 385                 int(guidValueList[8], 16),
 386                 int(guidValueList[9], 16),
 387                 int(guidValueList[10], 16)
 388                 )
 389     except:
 390         return ''
 391
 392 ## Convert GUID string in C structure style to xxxxxxxx_xxxx_xxxx_xxxx_xxxxxxxxxxxx
 393 #
 394 #   @param      GuidValue   The GUID value in C structure format
 395 #
 396 #   @retval     string      The GUID value in xxxxxxxx_xxxx_xxxx_xxxx_xxxxxxxxxxxx format
 397 #
 398 def GuidStructureStringToGuidValueName(GuidValue):
 399     guidValueString = GuidValue.lower().replace("{", "").replace("}", "").replace(" ", "")
 400     guidValueList = guidValueString.split(",")
 401     if len(guidValueList) != 11:
 402         EdkLogger.error(None, FORMAT_INVALID, "Invalid GUID value string [%s]" % GuidValue)
 403     return "%08x_%04x_%04x_%02x%02x_%02x%02x%02x%02x%02x%02x" % (
 404             int(guidValueList[0], 16),
 405             int(guidValueList[1], 16),
 406             int(guidValueList[2], 16),
 407             int(guidValueList[3], 16),
 408             int(guidValueList[4], 16),
 409             int(guidValueList[5], 16),
 410             int(guidValueList[6], 16),
 411             int(guidValueList[7], 16),
 412             int(guidValueList[8], 16),
 413             int(guidValueList[9], 16),
 414             int(guidValueList[10], 16)
 415             )
 416
 417 ## Create directories
 418 #
 419 #   @param      Directory   The directory name
 420 #
 421 def CreateDirectory(Directory):
 422     if Directory is None or Directory.strip() == "":
 423         return True
 424     try:
 425         if not os.access(Directory, os.F_OK):
 426             os.makedirs(Directory)
 427     except:
 428         return False
 429     return True
 430
 431 ## Remove directories, including files and sub-directories in it
 432 #
 433 #   @param      Directory   The directory name
 434 #
 435 def RemoveDirectory(Directory, Recursively=False):
 436     if Directory is None or Directory.strip() == "" or not os.path.exists(Directory):
 437         return
 438     if Recursively:
 439         CurrentDirectory = os.getcwd()
 440         os.chdir(Directory)
 441         for File in os.listdir("."):
 442             if os.path.isdir(File):
 443                 RemoveDirectory(File, Recursively)
 444             else:
 445                 os.remove(File)
 446         os.chdir(CurrentDirectory)
 447     os.rmdir(Directory)
 448
 449 ## Store content in file
 450 #
 451 #  This method is used to save file only when its content is changed. This is
 452 #  quite useful for "make" system to decide what will be re-built and what won't.
 453 #
 454 #   @param      File            The path of file
 455 #   @param      Content         The new content of the file
 456 #   @param      IsBinaryFile    The flag indicating if the file is binary file or not
 457 #
 458 #   @retval     True            If the file content is changed and the file is renewed
 459 #   @retval     False           If the file content is the same
 460 #
 461 def SaveFileOnChange(File, Content, IsBinaryFile=True):
 462     if not IsBinaryFile:
 463         Content = Content.replace("\n", os.linesep)
 464
 465     if os.path.exists(File):
 466         try:
 467             if Content == open(File, "rb").read():
 468                 return False
 469         except:
 470             EdkLogger.error(None, FILE_OPEN_FAILURE, ExtraData=File)
 471
 472     DirName = os.path.dirname(File)
 473     if not CreateDirectory(DirName):
 474         EdkLogger.error(None, FILE_CREATE_FAILURE, "Could not create directory %s" % DirName)
 475     else:
 476         if DirName == '':
 477             DirName = os.getcwd()
 478         if not os.access(DirName, os.W_OK):
 479             EdkLogger.error(None, PERMISSION_FAILURE, "Do not have write permission on directory %s" % DirName)
 480
 481     try:
 482         Fd = open(File, "wb")
 483         Fd.write(Content)
 484         Fd.close()
 485     except IOError as X:
 486         EdkLogger.error(None, FILE_CREATE_FAILURE, ExtraData='IOError %s' % X)
 487
 488     return True
 489
 490 ## Retrieve and cache the real path name in file system
 491 #
 492 #   @param      Root    The root directory of path relative to
 493 #
 494 #   @retval     str     The path string if the path exists
 495 #   @retval     None    If path doesn't exist
 496 #
 497 class DirCache:
 498     _CACHE_ = set()
 499     _UPPER_CACHE_ = {}
 500
 501     def __init__(self, Root):
 502         self._Root = Root
 503         for F in os.listdir(Root):
 504             self._CACHE_.add(F)
 505             self._UPPER_CACHE_[F.upper()] = F
 506
 507     # =[] operator
 508     def __getitem__(self, Path):
 509         Path = Path[len(os.path.commonprefix([Path, self._Root])):]
 510         if not Path:
 511             return self._Root
 512         if Path and Path[0] == os.path.sep:
 513             Path = Path[1:]
 514         if Path in self._CACHE_:
 515             return os.path.join(self._Root, Path)
 516         UpperPath = Path.upper()
 517         if UpperPath in self._UPPER_CACHE_:
 518             return os.path.join(self._Root, self._UPPER_CACHE_[UpperPath])
 519
 520         IndexList = []
 521         LastSepIndex = -1
 522         SepIndex = Path.find(os.path.sep)
 523         while SepIndex > -1:
 524             Parent = UpperPath[:SepIndex]
 525             if Parent not in self._UPPER_CACHE_:
 526                 break
 527             LastSepIndex = SepIndex
 528             SepIndex = Path.find(os.path.sep, LastSepIndex + 1)
 529
 530         if LastSepIndex == -1:
 531             return None
 532
 533         Cwd = os.getcwd()
 534         os.chdir(self._Root)
 535         SepIndex = LastSepIndex
 536         while SepIndex > -1:
 537             Parent = Path[:SepIndex]
 538             ParentKey = UpperPath[:SepIndex]
 539             if ParentKey not in self._UPPER_CACHE_:
 540                 os.chdir(Cwd)
 541                 return None
 542
 543             if Parent in self._CACHE_:
 544                 ParentDir = Parent
 545             else:
 546                 ParentDir = self._UPPER_CACHE_[ParentKey]
 547             for F in os.listdir(ParentDir):
 548                 Dir = os.path.join(ParentDir, F)
 549                 self._CACHE_.add(Dir)
 550                 self._UPPER_CACHE_[Dir.upper()] = Dir
 551
 552             SepIndex = Path.find(os.path.sep, SepIndex + 1)
 553
 554         os.chdir(Cwd)
 555         if Path in self._CACHE_:
 556             return os.path.join(self._Root, Path)
 557         elif UpperPath in self._UPPER_CACHE_:
 558             return os.path.join(self._Root, self._UPPER_CACHE_[UpperPath])
 559         return None
 560
 561 def RealPath(File, Dir='', OverrideDir=''):
 562     NewFile = os.path.normpath(os.path.join(Dir, File))
 563     NewFile = GlobalData.gAllFiles[NewFile]
 564     if not NewFile and OverrideDir:
 565         NewFile = os.path.normpath(os.path.join(OverrideDir, File))
 566         NewFile = GlobalData.gAllFiles[NewFile]
 567     return NewFile
 568
 569 ## Get GUID value from given packages
 570 #
 571 #   @param      CName           The CName of the GUID
 572 #   @param      PackageList     List of packages looking-up in
 573 #   @param      Inffile         The driver file
 574 #
 575 #   @retval     GuidValue   if the CName is found in any given package
 576 #   @retval     None        if the CName is not found in all given packages
 577 #
 578 def GuidValue(CName, PackageList, Inffile = None):
 579     for P in PackageList:
 580         GuidKeys = P.Guids.keys()
 581         if Inffile and P._PrivateGuids:
 582             if not Inffile.startswith(P.MetaFile.Dir):
 583                 GuidKeys = [x for x in P.Guids if x not in P._PrivateGuids]
 584         if CName in GuidKeys:
 585             return P.Guids[CName]
 586     return None
 587
 588 ## A string template class
 589 #
 590 #  This class implements a template for string replacement. A string template
 591 #  looks like following
 592 #
 593 #       ${BEGIN} other_string ${placeholder_name} other_string ${END}
 594 #
 595 #  The string between ${BEGIN} and ${END} will be repeated as many times as the
 596 #  length of "placeholder_name", which is a list passed through a dict. The
 597 #  "placeholder_name" is the key name of the dict. The ${BEGIN} and ${END} can
 598 #  be not used and, in this case, the "placeholder_name" must not a list and it
 599 #  will just be replaced once.
 600 #
 601 class TemplateString(object):
 602     _REPEAT_START_FLAG = "BEGIN"
 603     _REPEAT_END_FLAG = "END"
 604
 605     class Section(object):
 606         _LIST_TYPES = [type([]), type(set()), type((0,))]
 607
 608         def __init__(self, TemplateSection, PlaceHolderList):
 609             self._Template = TemplateSection
 610             self._PlaceHolderList = []
 611
 612             # Split the section into sub-sections according to the position of placeholders
 613             if PlaceHolderList:
 614                 self._SubSectionList = []
 615                 SubSectionStart = 0
 616                 #
 617                 # The placeholders passed in must be in the format of
 618                 #
 619                 #   PlaceHolderName, PlaceHolderStartPoint, PlaceHolderEndPoint
 620                 #
 621                 for PlaceHolder, Start, End in PlaceHolderList:
 622                     self._SubSectionList.append(TemplateSection[SubSectionStart:Start])
 623                     self._SubSectionList.append(TemplateSection[Start:End])
 624                     self._PlaceHolderList.append(PlaceHolder)
 625                     SubSectionStart = End
 626                 if SubSectionStart < len(TemplateSection):
 627                     self._SubSectionList.append(TemplateSection[SubSectionStart:])
 628             else:
 629                 self._SubSectionList = [TemplateSection]
 630
 631         def __str__(self):
 632             return self._Template + " : " + str(self._PlaceHolderList)
 633
 634         def Instantiate(self, PlaceHolderValues):
 635             RepeatTime = -1
 636             RepeatPlaceHolders = {}
 637             NonRepeatPlaceHolders = {}
 638
 639             for PlaceHolder in self._PlaceHolderList:
 640                 if PlaceHolder not in PlaceHolderValues:
 641                     continue
 642                 Value = PlaceHolderValues[PlaceHolder]
 643                 if type(Value) in self._LIST_TYPES:
 644                     if RepeatTime < 0:
 645                         RepeatTime = len(Value)
 646                     elif RepeatTime != len(Value):
 647                         EdkLogger.error(
 648                                     "TemplateString",
 649                                     PARAMETER_INVALID,
 650                                     "${%s} has different repeat time from others!" % PlaceHolder,
 651                                     ExtraData=str(self._Template)
 652                                     )
 653                     RepeatPlaceHolders["${%s}" % PlaceHolder] = Value
 654                 else:
 655                     NonRepeatPlaceHolders["${%s}" % PlaceHolder] = Value
 656
 657             if NonRepeatPlaceHolders:
 658                 StringList = []
 659                 for S in self._SubSectionList:
 660                     if S not in NonRepeatPlaceHolders:
 661                         StringList.append(S)
 662                     else:
 663                         StringList.append(str(NonRepeatPlaceHolders[S]))
 664             else:
 665                 StringList = self._SubSectionList
 666
 667             if RepeatPlaceHolders:
 668                 TempStringList = []
 669                 for Index in range(RepeatTime):
 670                     for S in StringList:
 671                         if S not in RepeatPlaceHolders:
 672                             TempStringList.append(S)
 673                         else:
 674                             TempStringList.append(str(RepeatPlaceHolders[S][Index]))
 675                 StringList = TempStringList
 676
 677             return "".join(StringList)
 678
 679     ## Constructor
 680     def __init__(self, Template=None):
 681         self.String = []
 682         self.IsBinary = False
 683         self._Template = Template
 684         self._TemplateSectionList = self._Parse(Template)
 685
 686     ## str() operator
 687     #
 688     #   @retval     string  The string replaced
 689     #
 690     def __str__(self):
 691         return "".join(self.String)
 692
 693     ## Split the template string into fragments per the ${BEGIN} and ${END} flags
 694     #
 695     #   @retval     list    A list of TemplateString.Section objects
 696     #
 697     def _Parse(self, Template):
 698         SectionStart = 0
 699         SearchFrom = 0
 700         MatchEnd = 0
 701         PlaceHolderList = []
 702         TemplateSectionList = []
 703         while Template:
 704             MatchObj = gPlaceholderPattern.search(Template, SearchFrom)
 705             if not MatchObj:
 706                 if MatchEnd <= len(Template):
 707                     TemplateSection = TemplateString.Section(Template[SectionStart:], PlaceHolderList)
 708                     TemplateSectionList.append(TemplateSection)
 709                 break
 710
 711             MatchString = MatchObj.group(1)
 712             MatchStart = MatchObj.start()
 713             MatchEnd = MatchObj.end()
 714
 715             if MatchString == self._REPEAT_START_FLAG:
 716                 if MatchStart > SectionStart:
 717                     TemplateSection = TemplateString.Section(Template[SectionStart:MatchStart], PlaceHolderList)
 718                     TemplateSectionList.append(TemplateSection)
 719                 SectionStart = MatchEnd
 720                 PlaceHolderList = []
 721             elif MatchString == self._REPEAT_END_FLAG:
 722                 TemplateSection = TemplateString.Section(Template[SectionStart:MatchStart], PlaceHolderList)
 723                 TemplateSectionList.append(TemplateSection)
 724                 SectionStart = MatchEnd
 725                 PlaceHolderList = []
 726             else:
 727                 PlaceHolderList.append((MatchString, MatchStart - SectionStart, MatchEnd - SectionStart))
 728             SearchFrom = MatchEnd
 729         return TemplateSectionList
 730
 731     ## Replace the string template with dictionary of placeholders and append it to previous one
 732     #
 733     #   @param      AppendString    The string template to append
 734     #   @param      Dictionary      The placeholder dictionaries
 735     #
 736     def Append(self, AppendString, Dictionary=None):
 737         if Dictionary:
 738             SectionList = self._Parse(AppendString)
 739             self.String.append( "".join(S.Instantiate(Dictionary) for S in SectionList))
 740         else:
 741             if isinstance(AppendString,list):
 742                 self.String.extend(AppendString)
 743             else:
 744                 self.String.append(AppendString)
 745
 746     ## Replace the string template with dictionary of placeholders
 747     #
 748     #   @param      Dictionary      The placeholder dictionaries
 749     #
 750     #   @retval     str             The string replaced with placeholder values
 751     #
 752     def Replace(self, Dictionary=None):
 753         return "".join(S.Instantiate(Dictionary) for S in self._TemplateSectionList)
 754
 755 ## Progress indicator class
 756 #
 757 #  This class makes use of thread to print progress on console.
 758 #
 759 class Progressor:
 760     # for avoiding deadloop
 761     _StopFlag = None
 762     _ProgressThread = None
 763     _CheckInterval = 0.25
 764
 765     ## Constructor
 766     #
 767     #   @param      OpenMessage     The string printed before progress charaters
 768     #   @param      CloseMessage    The string printed after progress charaters
 769     #   @param      ProgressChar    The charater used to indicate the progress
 770     #   @param      Interval        The interval in seconds between two progress charaters
 771     #
 772     def __init__(self, OpenMessage="", CloseMessage="", ProgressChar='.', Interval=1.0):
 773         self.PromptMessage = OpenMessage
 774         self.CodaMessage = CloseMessage
 775         self.ProgressChar = ProgressChar
 776         self.Interval = Interval
 777         if Progressor._StopFlag is None:
 778             Progressor._StopFlag = threading.Event()
 779
 780     ## Start to print progress charater
 781     #
 782     #   @param      OpenMessage     The string printed before progress charaters
 783     #
 784     def Start(self, OpenMessage=None):
 785         if OpenMessage is not None:
 786             self.PromptMessage = OpenMessage
 787         Progressor._StopFlag.clear()
 788         if Progressor._ProgressThread is None:
 789             Progressor._ProgressThread = threading.Thread(target=self._ProgressThreadEntry)
 790             Progressor._ProgressThread.setDaemon(False)
 791             Progressor._ProgressThread.start()
 792
 793     ## Stop printing progress charater
 794     #
 795     #   @param      CloseMessage    The string printed after progress charaters
 796     #
 797     def Stop(self, CloseMessage=None):
 798         OriginalCodaMessage = self.CodaMessage
 799         if CloseMessage is not None:
 800             self.CodaMessage = CloseMessage
 801         self.Abort()
 802         self.CodaMessage = OriginalCodaMessage
 803
 804     ## Thread entry method
 805     def _ProgressThreadEntry(self):
 806         sys.stdout.write(self.PromptMessage + " ")
 807         sys.stdout.flush()
 808         TimeUp = 0.0
 809         while not Progressor._StopFlag.isSet():
 810             if TimeUp <= 0.0:
 811                 sys.stdout.write(self.ProgressChar)
 812                 sys.stdout.flush()
 813                 TimeUp = self.Interval
 814             time.sleep(self._CheckInterval)
 815             TimeUp -= self._CheckInterval
 816         sys.stdout.write(" " + self.CodaMessage + "\n")
 817         sys.stdout.flush()
 818
 819     ## Abort the progress display
 820     @staticmethod
 821     def Abort():
 822         if Progressor._StopFlag is not None:
 823             Progressor._StopFlag.set()
 824         if Progressor._ProgressThread is not None:
 825             Progressor._ProgressThread.join()
 826             Progressor._ProgressThread = None
 827
 828 ## A dict which can access its keys and/or values orderly
 829 #
 830 #  The class implements a new kind of dict which its keys or values can be
 831 #  accessed in the order they are added into the dict. It guarantees the order
 832 #  by making use of an internal list to keep a copy of keys.
 833 #
 834 class sdict(IterableUserDict):
 835     ## Constructor
 836     def __init__(self):
 837         IterableUserDict.__init__(self)
 838         self._key_list = []
 839
 840     ## [] operator
 841     def __setitem__(self, key, value):
 842         if key not in self._key_list:
 843             self._key_list.append(key)
 844         IterableUserDict.__setitem__(self, key, value)
 845
 846     ## del operator
 847     def __delitem__(self, key):
 848         self._key_list.remove(key)
 849         IterableUserDict.__delitem__(self, key)
 850
 851     ## used in "for k in dict" loop to ensure the correct order
 852     def __iter__(self):
 853         return self.iterkeys()
 854
 855     ## len() support
 856     def __len__(self):
 857         return len(self._key_list)
 858
 859     ## "in" test support
 860     def __contains__(self, key):
 861         return key in self._key_list
 862
 863     ## indexof support
 864     def index(self, key):
 865         return self._key_list.index(key)
 866
 867     ## insert support
 868     def insert(self, key, newkey, newvalue, order):
 869         index = self._key_list.index(key)
 870         if order == 'BEFORE':
 871             self._key_list.insert(index, newkey)
 872             IterableUserDict.__setitem__(self, newkey, newvalue)
 873         elif order == 'AFTER':
 874             self._key_list.insert(index + 1, newkey)
 875             IterableUserDict.__setitem__(self, newkey, newvalue)
 876
 877     ## append support
 878     def append(self, sdict):
 879         for key in sdict:
 880             if key not in self._key_list:
 881                 self._key_list.append(key)
 882             IterableUserDict.__setitem__(self, key, sdict[key])
 883
 884     def has_key(self, key):
 885         return key in self._key_list
 886
 887     ## Empty the dict
 888     def clear(self):
 889         self._key_list = []
 890         IterableUserDict.clear(self)
 891
 892     ## Return a copy of keys
 893     def keys(self):
 894         keys = []
 895         for key in self._key_list:
 896             keys.append(key)
 897         return keys
 898
 899     ## Return a copy of values
 900     def values(self):
 901         values = []
 902         for key in self._key_list:
 903             values.append(self[key])
 904         return values
 905
 906     ## Return a copy of (key, value) list
 907     def items(self):
 908         items = []
 909         for key in self._key_list:
 910             items.append((key, self[key]))
 911         return items
 912
 913     ## Iteration support
 914     def iteritems(self):
 915         return iter(self.items())
 916
 917     ## Keys interation support
 918     def iterkeys(self):
 919         return iter(self.keys())
 920
 921     ## Values interation support
 922     def itervalues(self):
 923         return iter(self.values())
 924
 925     ## Return value related to a key, and remove the (key, value) from the dict
 926     def pop(self, key, *dv):
 927         value = None
 928         if key in self._key_list:
 929             value = self[key]
 930             self.__delitem__(key)
 931         elif len(dv) != 0 :
 932             value = kv[0]
 933         return value
 934
 935     ## Return (key, value) pair, and remove the (key, value) from the dict
 936     def popitem(self):
 937         key = self._key_list[-1]
 938         value = self[key]
 939         self.__delitem__(key)
 940         return key, value
 941
 942     def update(self, dict=None, **kwargs):
 943         if dict is not None:
 944             for k, v in dict.items():
 945                 self[k] = v
 946         if len(kwargs):
 947             for k, v in kwargs.items():
 948                 self[k] = v
 949
 950 ## Dictionary with restricted keys
 951 #
 952 class rdict(dict):
 953     ## Constructor
 954     def __init__(self, KeyList):
 955         for Key in KeyList:
 956             dict.__setitem__(self, Key, "")
 957
 958     ## []= operator
 959     def __setitem__(self, key, value):
 960         if key not in self:
 961             EdkLogger.error("RestrictedDict", ATTRIBUTE_SET_FAILURE, "Key [%s] is not allowed" % key,
 962                             ExtraData=", ".join(dict.keys(self)))
 963         dict.__setitem__(self, key, value)
 964
 965     ## =[] operator
 966     def __getitem__(self, key):
 967         if key not in self:
 968             return ""
 969         return dict.__getitem__(self, key)
 970
 971     ## del operator
 972     def __delitem__(self, key):
 973         EdkLogger.error("RestrictedDict", ATTRIBUTE_ACCESS_DENIED, ExtraData="del")
 974
 975     ## Empty the dict
 976     def clear(self):
 977         for Key in self:
 978             self.__setitem__(Key, "")
 979
 980     ## Return value related to a key, and remove the (key, value) from the dict
 981     def pop(self, key, *dv):
 982         EdkLogger.error("RestrictedDict", ATTRIBUTE_ACCESS_DENIED, ExtraData="pop")
 983
 984     ## Return (key, value) pair, and remove the (key, value) from the dict
 985     def popitem(self):
 986         EdkLogger.error("RestrictedDict", ATTRIBUTE_ACCESS_DENIED, ExtraData="popitem")
 987
 988 ## Dictionary using prioritized list as key
 989 #
 990 class tdict:
 991     _ListType = type([])
 992     _TupleType = type(())
 993     _Wildcard = 'COMMON'
 994     _ValidWildcardList = ['COMMON', 'DEFAULT', 'ALL', TAB_STAR, 'PLATFORM']
 995
 996     def __init__(self, _Single_=False, _Level_=2):
 997         self._Level_ = _Level_
 998         self.data = {}
 999         self._Single_ = _Single_
1000
1001     # =[] operator
1002     def __getitem__(self, key):
1003         KeyType = type(key)
1004         RestKeys = None
1005         if KeyType == self._ListType or KeyType == self._TupleType:
1006             FirstKey = key[0]
1007             if len(key) > 1:
1008                 RestKeys = key[1:]
1009             elif self._Level_ > 1:
1010                 RestKeys = [self._Wildcard for i in range(0, self._Level_ - 1)]
1011         else:
1012             FirstKey = key
1013             if self._Level_ > 1:
1014                 RestKeys = [self._Wildcard for i in range(0, self._Level_ - 1)]
1015
1016         if FirstKey is None or str(FirstKey).upper() in self._ValidWildcardList:
1017             FirstKey = self._Wildcard
1018
1019         if self._Single_:
1020             return self._GetSingleValue(FirstKey, RestKeys)
1021         else:
1022             return self._GetAllValues(FirstKey, RestKeys)
1023
1024     def _GetSingleValue(self, FirstKey, RestKeys):
1025         Value = None
1026         #print "%s-%s" % (FirstKey, self._Level_) ,
1027         if self._Level_ > 1:
1028             if FirstKey == self._Wildcard:
1029                 if FirstKey in self.data:
1030                     Value = self.data[FirstKey][RestKeys]
1031                 if Value is None:
1032                     for Key in self.data:
1033                         Value = self.data[Key][RestKeys]
1034                         if Value is not None: break
1035             else:
1036                 if FirstKey in self.data:
1037                     Value = self.data[FirstKey][RestKeys]
1038                 if Value is None and self._Wildcard in self.data:
1039                     #print "Value=None"
1040                     Value = self.data[self._Wildcard][RestKeys]
1041         else:
1042             if FirstKey == self._Wildcard:
1043                 if FirstKey in self.data:
1044                     Value = self.data[FirstKey]
1045                 if Value is None:
1046                     for Key in self.data:
1047                         Value = self.data[Key]
1048                         if Value is not None: break
1049             else:
1050                 if FirstKey in self.data:
1051                     Value = self.data[FirstKey]
1052                 elif self._Wildcard in self.data:
1053                     Value = self.data[self._Wildcard]
1054         return Value
1055
1056     def _GetAllValues(self, FirstKey, RestKeys):
1057         Value = []
1058         if self._Level_ > 1:
1059             if FirstKey == self._Wildcard:
1060                 for Key in self.data:
1061                     Value += self.data[Key][RestKeys]
1062             else:
1063                 if FirstKey in self.data:
1064                     Value += self.data[FirstKey][RestKeys]
1065                 if self._Wildcard in self.data:
1066                     Value += self.data[self._Wildcard][RestKeys]
1067         else:
1068             if FirstKey == self._Wildcard:
1069                 for Key in self.data:
1070                     Value.append(self.data[Key])
1071             else:
1072                 if FirstKey in self.data:
1073                     Value.append(self.data[FirstKey])
1074                 if self._Wildcard in self.data:
1075                     Value.append(self.data[self._Wildcard])
1076         return Value
1077
1078     ## []= operator
1079     def __setitem__(self, key, value):
1080         KeyType = type(key)
1081         RestKeys = None
1082         if KeyType == self._ListType or KeyType == self._TupleType:
1083             FirstKey = key[0]
1084             if len(key) > 1:
1085                 RestKeys = key[1:]
1086             else:
1087                 RestKeys = [self._Wildcard for i in range(0, self._Level_ - 1)]
1088         else:
1089             FirstKey = key
1090             if self._Level_ > 1:
1091                 RestKeys = [self._Wildcard for i in range(0, self._Level_ - 1)]
1092
1093         if FirstKey in self._ValidWildcardList:
1094             FirstKey = self._Wildcard
1095
1096         if FirstKey not in self.data and self._Level_ > 0:
1097             self.data[FirstKey] = tdict(self._Single_, self._Level_ - 1)
1098
1099         if self._Level_ > 1:
1100             self.data[FirstKey][RestKeys] = value
1101         else:
1102             self.data[FirstKey] = value
1103
1104     def SetGreedyMode(self):
1105         self._Single_ = False
1106         if self._Level_ > 1:
1107             for Key in self.data:
1108                 self.data[Key].SetGreedyMode()
1109
1110     def SetSingleMode(self):
1111         self._Single_ = True
1112         if self._Level_ > 1:
1113             for Key in self.data:
1114                 self.data[Key].SetSingleMode()
1115
1116     def GetKeys(self, KeyIndex=0):
1117         assert KeyIndex >= 0
1118         if KeyIndex == 0:
1119             return set(self.data.keys())
1120         else:
1121             keys = set()
1122             for Key in self.data:
1123                 keys |= self.data[Key].GetKeys(KeyIndex - 1)
1124             return keys
1125
1126 def AnalyzePcdExpression(Setting):
1127     RanStr = ''.join(sample(string.ascii_letters + string.digits, 8))
1128     Setting = Setting.replace('\\\\', RanStr).strip()
1129     # There might be escaped quote in a string: \", \\\" , \', \\\'
1130     Data = Setting
1131     # There might be '|' in string and in ( ... | ... ), replace it with '-'
1132     NewStr = ''
1133     InSingleQuoteStr = False
1134     InDoubleQuoteStr = False
1135     Pair = 0
1136     for Index, ch in enumerate(Data):
1137         if ch == '"' and not InSingleQuoteStr:
1138             if Data[Index - 1] != '\\':
1139                 InDoubleQuoteStr = not InDoubleQuoteStr
1140         elif ch == "'" and not InDoubleQuoteStr:
1141             if Data[Index - 1] != '\\':
1142                 InSingleQuoteStr = not InSingleQuoteStr
1143         elif ch == '(' and not (InSingleQuoteStr or InDoubleQuoteStr):
1144             Pair += 1
1145         elif ch == ')' and not (InSingleQuoteStr or InDoubleQuoteStr):
1146             Pair -= 1
1147
1148         if (Pair > 0 or InSingleQuoteStr or InDoubleQuoteStr) and ch == TAB_VALUE_SPLIT:
1149             NewStr += '-'
1150         else:
1151             NewStr += ch
1152     FieldList = []
1153     StartPos = 0
1154     while True:
1155         Pos = NewStr.find(TAB_VALUE_SPLIT, StartPos)
1156         if Pos < 0:
1157             FieldList.append(Setting[StartPos:].strip())
1158             break
1159         FieldList.append(Setting[StartPos:Pos].strip())
1160         StartPos = Pos + 1
1161     for i, ch in enumerate(FieldList):
1162         if RanStr in ch:
1163             FieldList[i] = ch.replace(RanStr,'\\\\')
1164     return FieldList
1165
1166 def ParseFieldValue (Value):
1167     def ParseDevPathValue (Value):
1168         if '\\' in Value:
1169             Value.replace('\\', '/').replace(' ', '')
1170
1171         Cmd = 'DevicePath ' + '"' + Value + '"'
1172         try:
1173             p = subprocess.Popen(Cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
1174             out, err = p.communicate()
1175         except Exception as X:
1176             raise BadExpression("DevicePath: %s" % (str(X)) )
1177         finally:
1178             subprocess._cleanup()
1179             p.stdout.close()
1180             p.stderr.close()
1181         if err:
1182             raise BadExpression("DevicePath: %s" % str(err))
1183         Size = len(out.split())
1184         out = ','.join(out.split())
1185         return '{' + out + '}', Size
1186
1187     if "{CODE(" in Value:
1188         return Value, len(Value.split(","))
1189     if isinstance(Value, type(0)):
1190         return Value, (Value.bit_length() + 7) / 8
1191     if not isinstance(Value, type('')):
1192         raise BadExpression('Type %s is %s' %(Value, type(Value)))
1193     Value = Value.strip()
1194     if Value.startswith(TAB_UINT8) and Value.endswith(')'):
1195         Value, Size = ParseFieldValue(Value.split('(', 1)[1][:-1])
1196         if Size > 1:
1197             raise BadExpression('Value (%s) Size larger than %d' %(Value, Size))
1198         return Value, 1
1199     if Value.startswith(TAB_UINT16) and Value.endswith(')'):
1200         Value, Size = ParseFieldValue(Value.split('(', 1)[1][:-1])
1201         if Size > 2:
1202             raise BadExpression('Value (%s) Size larger than %d' %(Value, Size))
1203         return Value, 2
1204     if Value.startswith(TAB_UINT32) and Value.endswith(')'):
1205         Value, Size = ParseFieldValue(Value.split('(', 1)[1][:-1])
1206         if Size > 4:
1207             raise BadExpression('Value (%s) Size larger than %d' %(Value, Size))
1208         return Value, 4
1209     if Value.startswith(TAB_UINT64) and Value.endswith(')'):
1210         Value, Size = ParseFieldValue(Value.split('(', 1)[1][:-1])
1211         if Size > 8:
1212             raise BadExpression('Value (%s) Size larger than %d' % (Value, Size))
1213         return Value, 8
1214     if Value.startswith(TAB_GUID) and Value.endswith(')'):
1215         Value = Value.split('(', 1)[1][:-1].strip()
1216         if Value[0] == '{' and Value[-1] == '}':
1217             TmpValue = GuidStructureStringToGuidString(Value)
1218             if not TmpValue:
1219                 raise BadExpression("Invalid GUID value string %s" % Value)
1220             Value = TmpValue
1221         if Value[0] == '"' and Value[-1] == '"':
1222             Value = Value[1:-1]
1223         try:
1224             Value = "'" + uuid.UUID(Value).get_bytes_le() + "'"
1225         except ValueError as Message:
1226             raise BadExpression(Message)
1227         Value, Size = ParseFieldValue(Value)
1228         return Value, 16
1229     if Value.startswith('L"') and Value.endswith('"'):
1230         # Unicode String
1231         # translate escape character
1232         Value = Value[1:]
1233         try:
1234             Value = eval(Value)
1235         except:
1236             Value = Value[1:-1]
1237         List = list(Value)
1238         List.reverse()
1239         Value = 0
1240         for Char in List:
1241             Value = (Value << 16) | ord(Char)
1242         return Value, (len(List) + 1) * 2
1243     if Value.startswith('"') and Value.endswith('"'):
1244         # ASCII String
1245         # translate escape character
1246         try:
1247             Value = eval(Value)
1248         except:
1249             Value = Value[1:-1]
1250         List = list(Value)
1251         List.reverse()
1252         Value = 0
1253         for Char in List:
1254             Value = (Value << 8) | ord(Char)
1255         return Value, len(List) + 1
1256     if Value.startswith("L'") and Value.endswith("'"):
1257         # Unicode Character Constant
1258         # translate escape character
1259         Value = Value[1:]
1260         try:
1261             Value = eval(Value)
1262         except:
1263             Value = Value[1:-1]
1264         List = list(Value)
1265         if len(List) == 0:
1266             raise BadExpression('Length %s is %s' % (Value, len(List)))
1267         List.reverse()
1268         Value = 0
1269         for Char in List:
1270             Value = (Value << 16) | ord(Char)
1271         return Value, len(List) * 2
1272     if Value.startswith("'") and Value.endswith("'"):
1273         # Character constant
1274         # translate escape character
1275         try:
1276             Value = eval(Value)
1277         except:
1278             Value = Value[1:-1]
1279         List = list(Value)
1280         if len(List) == 0:
1281             raise BadExpression('Length %s is %s' % (Value, len(List)))
1282         List.reverse()
1283         Value = 0
1284         for Char in List:
1285             Value = (Value << 8) | ord(Char)
1286         return Value, len(List)
1287     if Value.startswith('{') and Value.endswith('}'):
1288         # Byte array
1289         Value = Value[1:-1]
1290         List = [Item.strip() for Item in Value.split(',')]
1291         List.reverse()
1292         Value = 0
1293         RetSize = 0
1294         for Item in List:
1295             ItemValue, Size = ParseFieldValue(Item)
1296             RetSize += Size
1297             for I in range(Size):
1298                 Value = (Value << 8) | ((ItemValue >> 8 * I) & 0xff)
1299         return Value, RetSize
1300     if Value.startswith('DEVICE_PATH(') and Value.endswith(')'):
1301         Value = Value.replace("DEVICE_PATH(", '').rstrip(')')
1302         Value = Value.strip().strip('"')
1303         return ParseDevPathValue(Value)
1304     if Value.lower().startswith('0x'):
1305         try:
1306             Value = int(Value, 16)
1307         except:
1308             raise BadExpression("invalid hex value: %s" % Value)
1309         if Value == 0:
1310             return 0, 1
1311         return Value, (Value.bit_length() + 7) / 8
1312     if Value[0].isdigit():
1313         Value = int(Value, 10)
1314         if Value == 0:
1315             return 0, 1
1316         return Value, (Value.bit_length() + 7) / 8
1317     if Value.lower() == 'true':
1318         return 1, 1
1319     if Value.lower() == 'false':
1320         return 0, 1
1321     return Value, 1
1322
1323 ## AnalyzeDscPcd
1324 #
1325 #  Analyze DSC PCD value, since there is no data type info in DSC
1326 #  This function is used to match functions (AnalyzePcdData) used for retrieving PCD value from database
1327 #  1. Feature flag: TokenSpace.PcdCName|PcdValue
1328 #  2. Fix and Patch:TokenSpace.PcdCName|PcdValue[|VOID*[|MaxSize]]
1329 #  3. Dynamic default:
1330 #     TokenSpace.PcdCName|PcdValue[|VOID*[|MaxSize]]
1331 #     TokenSpace.PcdCName|PcdValue
1332 #  4. Dynamic VPD:
1333 #     TokenSpace.PcdCName|VpdOffset[|VpdValue]
1334 #     TokenSpace.PcdCName|VpdOffset[|MaxSize[|VpdValue]]
1335 #  5. Dynamic HII:
1336 #     TokenSpace.PcdCName|HiiString|VariableGuid|VariableOffset[|HiiValue]
1337 #  PCD value needs to be located in such kind of string, and the PCD value might be an expression in which
1338 #    there might have "|" operator, also in string value.
1339 #
1340 #  @param Setting: String contain information described above with "TokenSpace.PcdCName|" stripped
1341 #  @param PcdType: PCD type: feature, fixed, dynamic default VPD HII
1342 #  @param DataType: The datum type of PCD: VOID*, UNIT, BOOL
1343 #  @retval:
1344 #    ValueList: A List contain fields described above
1345 #    IsValid:   True if conforming EBNF, otherwise False
1346 #    Index:     The index where PcdValue is in ValueList
1347 #
1348 def AnalyzeDscPcd(Setting, PcdType, DataType=''):
1349     FieldList = AnalyzePcdExpression(Setting)
1350
1351     IsValid = True
1352     if PcdType in (MODEL_PCD_FIXED_AT_BUILD, MODEL_PCD_PATCHABLE_IN_MODULE, MODEL_PCD_DYNAMIC_DEFAULT, MODEL_PCD_DYNAMIC_EX_DEFAULT):
1353         Value = FieldList[0]
1354         Size = ''
1355         if len(FieldList) > 1 and FieldList[1]:
1356             DataType = FieldList[1]
1357             if FieldList[1] != TAB_VOID and StructPattern.match(FieldList[1]) is None:
1358                 IsValid = False
1359         if len(FieldList) > 2:
1360             Size = FieldList[2]
1361         if IsValid:
1362             if DataType == "":
1363                 IsValid = (len(FieldList) <= 1)
1364             else:
1365                 IsValid = (len(FieldList) <= 3)
1366
1367         if Size:
1368             try:
1369                 int(Size, 16) if Size.upper().startswith("0X") else int(Size)
1370             except:
1371                 IsValid = False
1372                 Size = -1
1373         return [str(Value), DataType, str(Size)], IsValid, 0
1374     elif PcdType == MODEL_PCD_FEATURE_FLAG:
1375         Value = FieldList[0]
1376         Size = ''
1377         IsValid = (len(FieldList) <= 1)
1378         return [Value, DataType, str(Size)], IsValid, 0
1379     elif PcdType in (MODEL_PCD_DYNAMIC_VPD, MODEL_PCD_DYNAMIC_EX_VPD):
1380         VpdOffset = FieldList[0]
1381         Value = Size = ''
1382         if not DataType == TAB_VOID:
1383             if len(FieldList) > 1:
1384                 Value = FieldList[1]
1385         else:
1386             if len(FieldList) > 1:
1387                 Size = FieldList[1]
1388             if len(FieldList) > 2:
1389                 Value = FieldList[2]
1390         if DataType == "":
1391             IsValid = (len(FieldList) <= 1)
1392         else:
1393             IsValid = (len(FieldList) <= 3)
1394         if Size:
1395             try:
1396                 int(Size, 16) if Size.upper().startswith("0X") else int(Size)
1397             except:
1398                 IsValid = False
1399                 Size = -1
1400         return [VpdOffset, str(Size), Value], IsValid, 2
1401     elif PcdType in (MODEL_PCD_DYNAMIC_HII, MODEL_PCD_DYNAMIC_EX_HII):
1402         IsValid = (3 <= len(FieldList) <= 5)
1403         HiiString = FieldList[0]
1404         Guid = Offset = Value = Attribute = ''
1405         if len(FieldList) > 1:
1406             Guid = FieldList[1]
1407         if len(FieldList) > 2:
1408             Offset = FieldList[2]
1409         if len(FieldList) > 3:
1410             Value = FieldList[3]
1411         if len(FieldList) > 4:
1412             Attribute = FieldList[4]
1413         return [HiiString, Guid, Offset, Value, Attribute], IsValid, 3
1414     return [], False, 0
1415
1416 ## AnalyzePcdData
1417 #
1418 #  Analyze the pcd Value, Datum type and TokenNumber.
1419 #  Used to avoid split issue while the value string contain "|" character
1420 #
1421 #  @param[in] Setting:  A String contain value/datum type/token number information;
1422 #
1423 #  @retval   ValueList: A List contain value, datum type and toke number.
1424 #
1425 def AnalyzePcdData(Setting):
1426     ValueList = ['', '', '']
1427
1428     ValueRe = re.compile(r'^\s*L?\".*\|.*\"')
1429     PtrValue = ValueRe.findall(Setting)
1430
1431     ValueUpdateFlag = False
1432
1433     if len(PtrValue) >= 1:
1434         Setting = re.sub(ValueRe, '', Setting)
1435         ValueUpdateFlag = True
1436
1437     TokenList = Setting.split(TAB_VALUE_SPLIT)
1438     ValueList[0:len(TokenList)] = TokenList
1439
1440     if ValueUpdateFlag:
1441         ValueList[0] = PtrValue[0]
1442
1443     return ValueList
1444
1445 ## check format of PCD value against its the datum type
1446 #
1447 # For PCD value setting
1448 #
1449 def CheckPcdDatum(Type, Value):
1450     if Type == TAB_VOID:
1451         ValueRe = re.compile(r'\s*L?\".*\"\s*$')
1452         if not (((Value.startswith('L"') or Value.startswith('"')) and Value.endswith('"'))
1453                 or (Value.startswith('{') and Value.endswith('}')) or (Value.startswith("L'") or Value.startswith("'") and Value.endswith("'"))
1454                ):
1455             return False, "Invalid value [%s] of type [%s]; must be in the form of {...} for array"\
1456                           ", \"...\" or \'...\' for string, L\"...\" or L\'...\' for unicode string" % (Value, Type)
1457         elif ValueRe.match(Value):
1458             # Check the chars in UnicodeString or CString is printable
1459             if Value.startswith("L"):
1460                 Value = Value[2:-1]
1461             else:
1462                 Value = Value[1:-1]
1463             Printset = set(string.printable)
1464             Printset.remove(TAB_PRINTCHAR_VT)
1465             Printset.add(TAB_PRINTCHAR_BS)
1466             Printset.add(TAB_PRINTCHAR_NUL)
1467             if not set(Value).issubset(Printset):
1468                 PrintList = sorted(Printset)
1469                 return False, "Invalid PCD string value of type [%s]; must be printable chars %s." % (Type, PrintList)
1470     elif Type == 'BOOLEAN':
1471         if Value not in ['TRUE', 'True', 'true', '0x1', '0x01', '1', 'FALSE', 'False', 'false', '0x0', '0x00', '0']:
1472             return False, "Invalid value [%s] of type [%s]; must be one of TRUE, True, true, 0x1, 0x01, 1"\
1473                           ", FALSE, False, false, 0x0, 0x00, 0" % (Value, Type)
1474     elif Type in [TAB_UINT8, TAB_UINT16, TAB_UINT32, TAB_UINT64]:
1475         if Value and int(Value, 0) < 0:
1476             return False, "PCD can't be set to negative value[%s] for datum type [%s]" % (Value, Type)
1477         try:
1478             Value = long(Value, 0)
1479             if Value > MAX_VAL_TYPE[Type]:
1480                 return False, "Too large PCD value[%s] for datum type [%s]" % (Value, Type)
1481         except:
1482             return False, "Invalid value [%s] of type [%s];"\
1483                           " must be a hexadecimal, decimal or octal in C language format." % (Value, Type)
1484     else:
1485         return True, "StructurePcd"
1486
1487     return True, ""
1488
1489 def CommonPath(PathList):
1490     P1 = min(PathList).split(os.path.sep)
1491     P2 = max(PathList).split(os.path.sep)
1492     for Index in xrange(min(len(P1), len(P2))):
1493         if P1[Index] != P2[Index]:
1494             return os.path.sep.join(P1[:Index])
1495     return os.path.sep.join(P1)
1496
1497 class PathClass(object):
1498     def __init__(self, File='', Root='', AlterRoot='', Type='', IsBinary=False,
1499                  Arch='COMMON', ToolChainFamily='', Target='', TagName='', ToolCode=''):
1500         self.Arch = Arch
1501         self.File = str(File)
1502         if os.path.isabs(self.File):
1503             self.Root = ''
1504             self.AlterRoot = ''
1505         else:
1506             self.Root = str(Root)
1507             self.AlterRoot = str(AlterRoot)
1508
1509         # Remove any '.' and '..' in path
1510         if self.Root:
1511             self.Root = mws.getWs(self.Root, self.File)
1512             self.Path = os.path.normpath(os.path.join(self.Root, self.File))
1513             self.Root = os.path.normpath(CommonPath([self.Root, self.Path]))
1514             # eliminate the side-effect of 'C:'
1515             if self.Root[-1] == ':':
1516                 self.Root += os.path.sep
1517             # file path should not start with path separator
1518             if self.Root[-1] == os.path.sep:
1519                 self.File = self.Path[len(self.Root):]
1520             else:
1521                 self.File = self.Path[len(self.Root) + 1:]
1522         else:
1523             self.Path = os.path.normpath(self.File)
1524
1525         self.SubDir, self.Name = os.path.split(self.File)
1526         self.BaseName, self.Ext = os.path.splitext(self.Name)
1527
1528         if self.Root:
1529             if self.SubDir:
1530                 self.Dir = os.path.join(self.Root, self.SubDir)
1531             else:
1532                 self.Dir = self.Root
1533         else:
1534             self.Dir = self.SubDir
1535
1536         if IsBinary:
1537             self.Type = Type
1538         else:
1539             self.Type = self.Ext.lower()
1540
1541         self.IsBinary = IsBinary
1542         self.Target = Target
1543         self.TagName = TagName
1544         self.ToolCode = ToolCode
1545         self.ToolChainFamily = ToolChainFamily
1546
1547     ## Convert the object of this class to a string
1548     #
1549     #  Convert member Path of the class to a string
1550     #
1551     #  @retval string Formatted String
1552     #
1553     def __str__(self):
1554         return self.Path
1555
1556     ## Override __eq__ function
1557     #
1558     # Check whether PathClass are the same
1559     #
1560     # @retval False The two PathClass are different
1561     # @retval True  The two PathClass are the same
1562     #
1563     def __eq__(self, Other):
1564         return self.Path == str(Other)
1565
1566     ## Override __cmp__ function
1567     #
1568     # Customize the comparsion operation of two PathClass
1569     #
1570     # @retval 0     The two PathClass are different
1571     # @retval -1    The first PathClass is less than the second PathClass
1572     # @retval 1     The first PathClass is Bigger than the second PathClass
1573     def __cmp__(self, Other):
1574         OtherKey = str(Other)
1575
1576         SelfKey = self.Path
1577         if SelfKey == OtherKey:
1578             return 0
1579         elif SelfKey > OtherKey:
1580             return 1
1581         else:
1582             return -1
1583
1584     ## Override __hash__ function
1585     #
1586     # Use Path as key in hash table
1587     #
1588     # @retval string Key for hash table
1589     #
1590     def __hash__(self):
1591         return hash(self.Path)
1592
1593     @cached_property
1594     def Key(self):
1595         return self.Path.upper()
1596
1597     @property
1598     def TimeStamp(self):
1599         return os.stat(self.Path)[8]
1600
1601     def Validate(self, Type='', CaseSensitive=True):
1602         def RealPath2(File, Dir='', OverrideDir=''):
1603             NewFile = None
1604             if OverrideDir:
1605                 NewFile = GlobalData.gAllFiles[os.path.normpath(os.path.join(OverrideDir, File))]
1606                 if NewFile:
1607                     if OverrideDir[-1] == os.path.sep:
1608                         return NewFile[len(OverrideDir):], NewFile[0:len(OverrideDir)]
1609                     else:
1610                         return NewFile[len(OverrideDir) + 1:], NewFile[0:len(OverrideDir)]
1611             if GlobalData.gAllFiles:
1612                 NewFile = GlobalData.gAllFiles[os.path.normpath(os.path.join(Dir, File))]
1613             if not NewFile:
1614                 NewFile = os.path.normpath(os.path.join(Dir, File))
1615                 if not os.path.exists(NewFile):
1616                     return None, None
1617             if NewFile:
1618                 if Dir:
1619                     if Dir[-1] == os.path.sep:
1620                         return NewFile[len(Dir):], NewFile[0:len(Dir)]
1621                     else:
1622                         return NewFile[len(Dir) + 1:], NewFile[0:len(Dir)]
1623                 else:
1624                     return NewFile, ''
1625
1626             return None, None
1627
1628         if GlobalData.gCaseInsensitive:
1629             CaseSensitive = False
1630         if Type and Type.lower() != self.Type:
1631             return FILE_TYPE_MISMATCH, '%s (expect %s but got %s)' % (self.File, Type, self.Type)
1632
1633         RealFile, RealRoot = RealPath2(self.File, self.Root, self.AlterRoot)
1634         if not RealRoot and not RealFile:
1635             RealFile = self.File
1636             if self.AlterRoot:
1637                 RealFile = os.path.join(self.AlterRoot, self.File)
1638             elif self.Root:
1639                 RealFile = os.path.join(self.Root, self.File)
1640             if len (mws.getPkgPath()) == 0:
1641                 return FILE_NOT_FOUND, os.path.join(self.AlterRoot, RealFile)
1642             else:
1643                 return FILE_NOT_FOUND, "%s is not found in packages path:\n\t%s" % (self.File, '\n\t'.join(mws.getPkgPath()))
1644
1645         ErrorCode = 0
1646         ErrorInfo = ''
1647         if RealRoot != self.Root or RealFile != self.File:
1648             if CaseSensitive and (RealFile != self.File or (RealRoot != self.Root and RealRoot != self.AlterRoot)):
1649                 ErrorCode = FILE_CASE_MISMATCH
1650                 ErrorInfo = self.File + '\n\t' + RealFile + " [in file system]"
1651
1652             self.SubDir, self.Name = os.path.split(RealFile)
1653             self.BaseName, self.Ext = os.path.splitext(self.Name)
1654             if self.SubDir:
1655                 self.Dir = os.path.join(RealRoot, self.SubDir)
1656             else:
1657                 self.Dir = RealRoot
1658             self.File = RealFile
1659             self.Root = RealRoot
1660             self.Path = os.path.join(RealRoot, RealFile)
1661         return ErrorCode, ErrorInfo
1662
1663 ## Parse PE image to get the required PE informaion.
1664 #
1665 class PeImageClass():
1666     ## Constructor
1667     #
1668     #   @param  File FilePath of PeImage
1669     #
1670     def __init__(self, PeFile):
1671         self.FileName   = PeFile
1672         self.IsValid    = False
1673         self.Size       = 0
1674         self.EntryPoint = 0
1675         self.SectionAlignment  = 0
1676         self.SectionHeaderList = []
1677         self.ErrorInfo = ''
1678         try:
1679             PeObject = open(PeFile, 'rb')
1680         except:
1681             self.ErrorInfo = self.FileName + ' can not be found\n'
1682             return
1683         # Read DOS header
1684         ByteArray = array.array('B')
1685         ByteArray.fromfile(PeObject, 0x3E)
1686         ByteList = ByteArray.tolist()
1687         # DOS signature should be 'MZ'
1688         if self._ByteListToStr (ByteList[0x0:0x2]) != 'MZ':
1689             self.ErrorInfo = self.FileName + ' has no valid DOS signature MZ'
1690             return
1691
1692         # Read 4 byte PE Signature
1693         PeOffset = self._ByteListToInt(ByteList[0x3C:0x3E])
1694         PeObject.seek(PeOffset)
1695         ByteArray = array.array('B')
1696         ByteArray.fromfile(PeObject, 4)
1697         # PE signature should be 'PE\0\0'
1698         if ByteArray.tostring() != 'PE\0\0':
1699             self.ErrorInfo = self.FileName + ' has no valid PE signature PE00'
1700             return
1701
1702         # Read PE file header
1703         ByteArray = array.array('B')
1704         ByteArray.fromfile(PeObject, 0x14)
1705         ByteList = ByteArray.tolist()
1706         SecNumber = self._ByteListToInt(ByteList[0x2:0x4])
1707         if SecNumber == 0:
1708             self.ErrorInfo = self.FileName + ' has no section header'
1709             return
1710
1711         # Read PE optional header
1712         OptionalHeaderSize = self._ByteListToInt(ByteArray[0x10:0x12])
1713         ByteArray = array.array('B')
1714         ByteArray.fromfile(PeObject, OptionalHeaderSize)
1715         ByteList = ByteArray.tolist()
1716         self.EntryPoint       = self._ByteListToInt(ByteList[0x10:0x14])
1717         self.SectionAlignment = self._ByteListToInt(ByteList[0x20:0x24])
1718         self.Size             = self._ByteListToInt(ByteList[0x38:0x3C])
1719
1720         # Read each Section Header
1721         for Index in range(SecNumber):
1722             ByteArray = array.array('B')
1723             ByteArray.fromfile(PeObject, 0x28)
1724             ByteList = ByteArray.tolist()
1725             SecName  = self._ByteListToStr(ByteList[0:8])
1726             SecVirtualSize = self._ByteListToInt(ByteList[8:12])
1727             SecRawAddress  = self._ByteListToInt(ByteList[20:24])
1728             SecVirtualAddress = self._ByteListToInt(ByteList[12:16])
1729             self.SectionHeaderList.append((SecName, SecVirtualAddress, SecRawAddress, SecVirtualSize))
1730         self.IsValid = True
1731         PeObject.close()
1732
1733     def _ByteListToStr(self, ByteList):
1734         String = ''
1735         for index in range(len(ByteList)):
1736             if ByteList[index] == 0:
1737                 break
1738             String += chr(ByteList[index])
1739         return String
1740
1741     def _ByteListToInt(self, ByteList):
1742         Value = 0
1743         for index in range(len(ByteList) - 1, -1, -1):
1744             Value = (Value << 8) | int(ByteList[index])
1745         return Value
1746
1747 class DefaultStore():
1748     def __init__(self, DefaultStores ):
1749
1750         self.DefaultStores = DefaultStores
1751     def DefaultStoreID(self, DefaultStoreName):
1752         for key, value in self.DefaultStores.items():
1753             if value == DefaultStoreName:
1754                 return key
1755         return None
1756     def GetDefaultDefault(self):
1757         if not self.DefaultStores or "0" in self.DefaultStores:
1758             return "0", TAB_DEFAULT_STORES_DEFAULT
1759         else:
1760             minvalue = min(int(value_str) for value_str in self.DefaultStores)
1761             return (str(minvalue), self.DefaultStores[str(minvalue)])
1762     def GetMin(self, DefaultSIdList):
1763         if not DefaultSIdList:
1764             return TAB_DEFAULT_STORES_DEFAULT
1765         storeidset = {storeid for storeid, storename in self.DefaultStores.values() if storename in DefaultSIdList}
1766         if not storeidset:
1767             return ""
1768         minid = min(storeidset )
1769         for sid, name in self.DefaultStores.values():
1770             if sid == minid:
1771                 return name
1772
1773 class SkuClass():
1774     DEFAULT = 0
1775     SINGLE = 1
1776     MULTIPLE =2
1777
1778     def __init__(self,SkuIdentifier='', SkuIds=None):
1779         if SkuIds is None:
1780             SkuIds = {}
1781
1782         for SkuName in SkuIds:
1783             SkuId = SkuIds[SkuName][0]
1784             skuid_num = int(SkuId, 16) if SkuId.upper().startswith("0X") else int(SkuId)
1785             if skuid_num > 0xFFFFFFFFFFFFFFFF:
1786                 EdkLogger.error("build", PARAMETER_INVALID,
1787                             ExtraData = "SKU-ID [%s] value %s exceeds the max value of UINT64"
1788                                       % (SkuName, SkuId))
1789
1790         self.AvailableSkuIds = sdict()
1791         self.SkuIdSet = []
1792         self.SkuIdNumberSet = []
1793         self.SkuData = SkuIds
1794         self._SkuInherit = {}
1795         self._SkuIdentifier = SkuIdentifier
1796         if SkuIdentifier == '' or SkuIdentifier is None:
1797             self.SkuIdSet = ['DEFAULT']
1798             self.SkuIdNumberSet = ['0U']
1799         elif SkuIdentifier == 'ALL':
1800             self.SkuIdSet = SkuIds.keys()
1801             self.SkuIdNumberSet = [num[0].strip() + 'U' for num in SkuIds.values()]
1802         else:
1803             r = SkuIdentifier.split('|')
1804             self.SkuIdSet=[(r[k].strip()).upper() for k in range(len(r))]
1805             k = None
1806             try:
1807                 self.SkuIdNumberSet = [SkuIds[k][0].strip() + 'U' for k in self.SkuIdSet]
1808             except Exception:
1809                 EdkLogger.error("build", PARAMETER_INVALID,
1810                             ExtraData = "SKU-ID [%s] is not supported by the platform. [Valid SKU-ID: %s]"
1811                                       % (k, " | ".join(SkuIds.keys())))
1812         for each in self.SkuIdSet:
1813             if each in SkuIds:
1814                 self.AvailableSkuIds[each] = SkuIds[each][0]
1815             else:
1816                 EdkLogger.error("build", PARAMETER_INVALID,
1817                             ExtraData="SKU-ID [%s] is not supported by the platform. [Valid SKU-ID: %s]"
1818                                       % (each, " | ".join(SkuIds.keys())))
1819         if self.SkuUsageType != SkuClass.SINGLE:
1820             self.AvailableSkuIds.update({'DEFAULT':0, 'COMMON':0})
1821         if self.SkuIdSet:
1822             GlobalData.gSkuids = (self.SkuIdSet)
1823             if 'COMMON' in GlobalData.gSkuids:
1824                 GlobalData.gSkuids.remove('COMMON')
1825             if self.SkuUsageType == self.SINGLE:
1826                 if len(GlobalData.gSkuids) != 1:
1827                     if 'DEFAULT' in GlobalData.gSkuids:
1828                         GlobalData.gSkuids.remove('DEFAULT')
1829             if GlobalData.gSkuids:
1830                 GlobalData.gSkuids.sort()
1831
1832     def GetNextSkuId(self, skuname):
1833         if not self._SkuInherit:
1834             self._SkuInherit = {}
1835             for item in self.SkuData.values():
1836                 self._SkuInherit[item[1]]=item[2] if item[2] else "DEFAULT"
1837         return self._SkuInherit.get(skuname, "DEFAULT")
1838
1839     def GetSkuChain(self, sku):
1840         if sku == "DEFAULT":
1841             return ["DEFAULT"]
1842         skulist = [sku]
1843         nextsku = sku
1844         while True:
1845             nextsku = self.GetNextSkuId(nextsku)
1846             skulist.append(nextsku)
1847             if nextsku == "DEFAULT":
1848                 break
1849         skulist.reverse()
1850         return skulist
1851     def SkuOverrideOrder(self):
1852         skuorderset = []
1853         for skuname in self.SkuIdSet:
1854             skuorderset.append(self.GetSkuChain(skuname))
1855
1856         skuorder = []
1857         for index in range(max(len(item) for item in skuorderset)):
1858             for subset in skuorderset:
1859                 if index > len(subset)-1:
1860                     continue
1861                 if subset[index] in skuorder:
1862                     continue
1863                 skuorder.append(subset[index])
1864
1865         return skuorder
1866
1867     @property
1868     def SkuUsageType(self):
1869         if self._SkuIdentifier.upper() == "ALL":
1870             return SkuClass.MULTIPLE
1871
1872         if len(self.SkuIdSet) == 1:
1873             if self.SkuIdSet[0] == 'DEFAULT':
1874                 return SkuClass.DEFAULT
1875             return SkuClass.SINGLE
1876         if len(self.SkuIdSet) == 2 and 'DEFAULT' in self.SkuIdSet:
1877             return SkuClass.SINGLE
1878         return SkuClass.MULTIPLE
1879
1880     def DumpSkuIdArrary(self):
1881         if self.SkuUsageType == SkuClass.SINGLE:
1882             return "{0x0}"
1883         ArrayStrList = []
1884         for skuname in self.AvailableSkuIds:
1885             if skuname == "COMMON":
1886                 continue
1887             while skuname != "DEFAULT":
1888                 ArrayStrList.append(hex(int(self.AvailableSkuIds[skuname])))
1889                 skuname = self.GetNextSkuId(skuname)
1890             ArrayStrList.append("0x0")
1891         return "{{{myList}}}".format(myList=",".join(ArrayStrList))
1892
1893     @property
1894     def AvailableSkuIdSet(self):
1895         return self.AvailableSkuIds
1896
1897     @property
1898     def SystemSkuId(self):
1899         if self.SkuUsageType == SkuClass.SINGLE:
1900             if len(self.SkuIdSet) == 1:
1901                 return self.SkuIdSet[0]
1902             else:
1903                 return self.SkuIdSet[0] if self.SkuIdSet[0] != 'DEFAULT' else self.SkuIdSet[1]
1904         else:
1905             return 'DEFAULT'
1906
1907 ##  Get the integer value from string like "14U" or integer like 2
1908 #
1909 #   @param      Input   The object that may be either a integer value or a string
1910 #
1911 #   @retval     Value    The integer value that the input represents
1912 #
1913 def GetIntegerValue(Input):
1914     if type(Input) in (int, long):
1915         return Input
1916     String = Input
1917     if String.endswith("U"):
1918         String = String[:-1]
1919     if String.endswith("ULL"):
1920         String = String[:-3]
1921     if String.endswith("LL"):
1922         String = String[:-2]
1923
1924     if String.startswith("0x") or String.startswith("0X"):
1925         return int(String, 16)
1926     elif String == '':
1927         return 0
1928     else:
1929         return int(String)
1930
1931 #
1932 # Pack a GUID (registry format) list into a buffer and return it
1933 #
1934 def PackGUID(Guid):
1935     return pack(PACK_PATTERN_GUID,
1936                 int(Guid[0], 16),
1937                 int(Guid[1], 16),
1938                 int(Guid[2], 16),
1939                 int(Guid[3][-4:-2], 16),
1940                 int(Guid[3][-2:], 16),
1941                 int(Guid[4][-12:-10], 16),
1942                 int(Guid[4][-10:-8], 16),
1943                 int(Guid[4][-8:-6], 16),
1944                 int(Guid[4][-6:-4], 16),
1945                 int(Guid[4][-4:-2], 16),
1946                 int(Guid[4][-2:], 16)
1947                 )
1948
1949 #
1950 # Pack a GUID (byte) list into a buffer and return it
1951 #
1952 def PackByteFormatGUID(Guid):
1953     return pack(PACK_PATTERN_GUID,
1954                 Guid[0],
1955                 Guid[1],
1956                 Guid[2],
1957                 Guid[3],
1958                 Guid[4],
1959                 Guid[5],
1960                 Guid[6],
1961                 Guid[7],
1962                 Guid[8],
1963                 Guid[9],
1964                 Guid[10],
1965                 )
1966
1967 ## DeepCopy dict/OrderedDict recusively
1968 #
1969 #   @param      ori_dict    a nested dict or ordereddict
1970 #
1971 #   @retval     new dict or orderdict
1972 #
1973 def CopyDict(ori_dict):
1974     dict_type = ori_dict.__class__
1975     if dict_type not in (dict,OrderedDict):
1976         return ori_dict
1977     new_dict = dict_type()
1978     for key in ori_dict:
1979         if isinstance(ori_dict[key],(dict,OrderedDict)):
1980             new_dict[key] = CopyDict(ori_dict[key])
1981         else:
1982             new_dict[key] = ori_dict[key]
1983     return new_dict
1984
1985 #
1986 # Remove the c/c++ comments: // and /* */
1987 #
1988 def RemoveCComments(ctext):
1989     return re.sub('//.*?\n|/\*.*?\*/', '\n', ctext, flags=re.S)
1990 ##
1991 #
1992 # This acts like the main() function for the script, unless it is 'import'ed into another
1993 # script.
1994 #
1995 if __name__ == '__main__':
1996     pass
1997