BaseTools/Source/Python/Common/Misc.py

   1 ## @file
   2 # Common routines used by all tools
   3 #
   4 # Copyright (c) 2007 - 2018, Intel Corporation. All rights reserved.<BR>
   5 # This program and the accompanying materials
   6 # are licensed and made available under the terms and conditions of the BSD License
   7 # which accompanies this distribution.  The full text of the license may be found at
   8 # http://opensource.org/licenses/bsd-license.php
   9 #
  10 # THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS,
  11 # WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED.
  12 #
  13
  14 ##
  15 # Import Modules
  16 #
  17 import Common.LongFilePathOs as os
  18 import sys
  19 import string
  20 import threading
  21 import time
  22 import re
  23 import pickle
  24 import array
  25 import shutil
  26 from struct import pack
  27 from collections import UserDict as IterableUserDict
  28 from collections import OrderedDict
  29
  30 from Common import EdkLogger as EdkLogger
  31 from Common import GlobalData as GlobalData
  32 from .DataType import *
  33 from .BuildToolError import *
  34 from CommonDataClass.DataClass import *
  35 from .Parsing import GetSplitValueList
  36 from Common.LongFilePathSupport import OpenLongFilePath as open
  37 from Common.MultipleWorkspace import MultipleWorkspace as mws
  38 import uuid
  39 from CommonDataClass.Exceptions import BadExpression
  40 from Common.caching import cached_property
  41 import subprocess
  42 ## Regular expression used to find out place holders in string template
  43 gPlaceholderPattern = re.compile("\$\{([^$()\s]+)\}", re.MULTILINE | re.UNICODE)
  44
  45 ## regular expressions for map file processing
  46 startPatternGeneral = re.compile("^Start[' ']+Length[' ']+Name[' ']+Class")
  47 addressPatternGeneral = re.compile("^Address[' ']+Publics by Value[' ']+Rva\+Base")
  48 valuePatternGcc = re.compile('^([\w_\.]+) +([\da-fA-Fx]+) +([\da-fA-Fx]+)$')
  49 pcdPatternGcc = re.compile('^([\da-fA-Fx]+) +([\da-fA-Fx]+)')
  50 secReGeneral = re.compile('^([\da-fA-F]+):([\da-fA-F]+) +([\da-fA-F]+)[Hh]? +([.\w\$]+) +(\w+)', re.UNICODE)
  51
  52 ## Dictionary used to store file time stamp for quick re-access
  53 gFileTimeStampCache = {}    # {file path : file time stamp}
  54
  55 ## Dictionary used to store dependencies of files
  56 gDependencyDatabase = {}    # arch : {file path : [dependent files list]}
  57
  58 #
  59 # If a module is built more than once with different PCDs or library classes
  60 # a temporary INF file with same content is created, the temporary file is removed
  61 # when build exits.
  62 #
  63 _TempInfs = []
  64
  65 def GetVariableOffset(mapfilepath, efifilepath, varnames):
  66     """ Parse map file to get variable offset in current EFI file
  67     @param mapfilepath    Map file absolution path
  68     @param efifilepath:   EFI binary file full path
  69     @param varnames       iteratable container whose elements are variable names to be searched
  70
  71     @return List whos elements are tuple with variable name and raw offset
  72     """
  73     lines = []
  74     try:
  75         f = open(mapfilepath, 'r')
  76         lines = f.readlines()
  77         f.close()
  78     except:
  79         return None
  80
  81     if len(lines) == 0: return None
  82     firstline = lines[0].strip()
  83     if (firstline.startswith("Archive member included ") and
  84         firstline.endswith(" file (symbol)")):
  85         return _parseForGCC(lines, efifilepath, varnames)
  86     if firstline.startswith("# Path:"):
  87         return _parseForXcode(lines, efifilepath, varnames)
  88     return _parseGeneral(lines, efifilepath, varnames)
  89
  90 def _parseForXcode(lines, efifilepath, varnames):
  91     status = 0
  92     ret = []
  93     for line in lines:
  94         line = line.strip()
  95         if status == 0 and line == "# Symbols:":
  96             status = 1
  97             continue
  98         if status == 1 and len(line) != 0:
  99             for varname in varnames:
 100                 if varname in line:
 101                     # cannot pregenerate this RegEx since it uses varname from varnames.
 102                     m = re.match('^([\da-fA-FxX]+)([\s\S]*)([_]*%s)$' % varname, line)
 103                     if m is not None:
 104                         ret.append((varname, m.group(1)))
 105     return ret
 106
 107 def _parseForGCC(lines, efifilepath, varnames):
 108     """ Parse map file generated by GCC linker """
 109     status = 0
 110     sections = []
 111     varoffset = []
 112     for index, line in enumerate(lines):
 113         line = line.strip()
 114         # status machine transection
 115         if status == 0 and line == "Memory Configuration":
 116             status = 1
 117             continue
 118         elif status == 1 and line == 'Linker script and memory map':
 119             status = 2
 120             continue
 121         elif status ==2 and line == 'START GROUP':
 122             status = 3
 123             continue
 124
 125         # status handler
 126         if status == 3:
 127             m = valuePatternGcc.match(line)
 128             if m is not None:
 129                 sections.append(m.groups(0))
 130             for varname in varnames:
 131                 Str = ''
 132                 m = re.match("^.data.(%s)" % varname, line)
 133                 if m is not None:
 134                     m = re.match(".data.(%s)$" % varname, line)
 135                     if m is not None:
 136                         Str = lines[index + 1]
 137                     else:
 138                         Str = line[len(".data.%s" % varname):]
 139                     if Str:
 140                         m = pcdPatternGcc.match(Str.strip())
 141                         if m is not None:
 142                             varoffset.append((varname, int(m.groups(0)[0], 16), int(sections[-1][1], 16), sections[-1][0]))
 143
 144     if not varoffset:
 145         return []
 146     # get section information from efi file
 147     efisecs = PeImageClass(efifilepath).SectionHeaderList
 148     if efisecs is None or len(efisecs) == 0:
 149         return []
 150     #redirection
 151     redirection = 0
 152     for efisec in efisecs:
 153         for section in sections:
 154             if section[0].strip() == efisec[0].strip() and section[0].strip() == '.text':
 155                 redirection = int(section[1], 16) - efisec[1]
 156
 157     ret = []
 158     for var in varoffset:
 159         for efisec in efisecs:
 160             if var[1] >= efisec[1] and var[1] < efisec[1]+efisec[3]:
 161                 ret.append((var[0], hex(efisec[2] + var[1] - efisec[1] - redirection)))
 162     return ret
 163
 164 def _parseGeneral(lines, efifilepath, varnames):
 165     status = 0    #0 - beginning of file; 1 - PE section definition; 2 - symbol table
 166     secs  = []    # key = section name
 167     varoffset = []
 168     symRe = re.compile('^([\da-fA-F]+):([\da-fA-F]+) +([\.:\\\\\w\?@\$]+) +([\da-fA-F]+)', re.UNICODE)
 169
 170     for line in lines:
 171         line = line.strip()
 172         if startPatternGeneral.match(line):
 173             status = 1
 174             continue
 175         if addressPatternGeneral.match(line):
 176             status = 2
 177             continue
 178         if line.startswith("entry point at"):
 179             status = 3
 180             continue
 181         if status == 1 and len(line) != 0:
 182             m =  secReGeneral.match(line)
 183             assert m is not None, "Fail to parse the section in map file , line is %s" % line
 184             sec_no, sec_start, sec_length, sec_name, sec_class = m.groups(0)
 185             secs.append([int(sec_no, 16), int(sec_start, 16), int(sec_length, 16), sec_name, sec_class])
 186         if status == 2 and len(line) != 0:
 187             for varname in varnames:
 188                 m = symRe.match(line)
 189                 assert m is not None, "Fail to parse the symbol in map file, line is %s" % line
 190                 sec_no, sym_offset, sym_name, vir_addr = m.groups(0)
 191                 sec_no     = int(sec_no,     16)
 192                 sym_offset = int(sym_offset, 16)
 193                 vir_addr   = int(vir_addr,   16)
 194                 # cannot pregenerate this RegEx since it uses varname from varnames.
 195                 m2 = re.match('^[_]*(%s)' % varname, sym_name)
 196                 if m2 is not None:
 197                     # fond a binary pcd entry in map file
 198                     for sec in secs:
 199                         if sec[0] == sec_no and (sym_offset >= sec[1] and sym_offset < sec[1] + sec[2]):
 200                             varoffset.append([varname, sec[3], sym_offset, vir_addr, sec_no])
 201
 202     if not varoffset: return []
 203
 204     # get section information from efi file
 205     efisecs = PeImageClass(efifilepath).SectionHeaderList
 206     if efisecs is None or len(efisecs) == 0:
 207         return []
 208
 209     ret = []
 210     for var in varoffset:
 211         index = 0
 212         for efisec in efisecs:
 213             index = index + 1
 214             if var[1].strip() == efisec[0].strip():
 215                 ret.append((var[0], hex(efisec[2] + var[2])))
 216             elif var[4] == index:
 217                 ret.append((var[0], hex(efisec[2] + var[2])))
 218
 219     return ret
 220
 221 ## Routine to process duplicated INF
 222 #
 223 #  This function is called by following two cases:
 224 #  Case 1 in DSC:
 225 #    [components.arch]
 226 #    Pkg/module/module.inf
 227 #    Pkg/module/module.inf {
 228 #      <Defines>
 229 #        FILE_GUID = 0D1B936F-68F3-4589-AFCC-FB8B7AEBC836
 230 #    }
 231 #  Case 2 in FDF:
 232 #    INF Pkg/module/module.inf
 233 #    INF FILE_GUID = 0D1B936F-68F3-4589-AFCC-FB8B7AEBC836 Pkg/module/module.inf
 234 #
 235 #  This function copies Pkg/module/module.inf to
 236 #  Conf/.cache/0D1B936F-68F3-4589-AFCC-FB8B7AEBC836module.inf
 237 #
 238 #  @param Path     Original PathClass object
 239 #  @param BaseName New file base name
 240 #
 241 #  @retval         return the new PathClass object
 242 #
 243 def ProcessDuplicatedInf(Path, BaseName, Workspace):
 244     Filename = os.path.split(Path.File)[1]
 245     if '.' in Filename:
 246         Filename = BaseName + Path.BaseName + Filename[Filename.rfind('.'):]
 247     else:
 248         Filename = BaseName + Path.BaseName
 249
 250     #
 251     # If -N is specified on command line, cache is disabled
 252     # The directory has to be created
 253     #
 254     DbDir = os.path.split(GlobalData.gDatabasePath)[0]
 255     if not os.path.exists(DbDir):
 256         os.makedirs(DbDir)
 257     #
 258     # A temporary INF is copied to database path which must have write permission
 259     # The temporary will be removed at the end of build
 260     # In case of name conflict, the file name is
 261     # FILE_GUIDBaseName (0D1B936F-68F3-4589-AFCC-FB8B7AEBC836module.inf)
 262     #
 263     TempFullPath = os.path.join(DbDir,
 264                                 Filename)
 265     RtPath = PathClass(Path.File, Workspace)
 266     #
 267     # Modify the full path to temporary path, keep other unchanged
 268     #
 269     # To build same module more than once, the module path with FILE_GUID overridden has
 270     # the file name FILE_GUIDmodule.inf, but the relative path (self.MetaFile.File) is the real path
 271     # in DSC which is used as relative path by C files and other files in INF.
 272     # A trick was used: all module paths are PathClass instances, after the initialization
 273     # of PathClass, the PathClass.Path is overridden by the temporary INF path.
 274     #
 275     # The reason for creating a temporary INF is:
 276     # Platform.Modules which is the base to create ModuleAutoGen objects is a dictionary,
 277     # the key is the full path of INF, the value is an object to save overridden library instances, PCDs.
 278     # A different key for the same module is needed to create different output directory,
 279     # retrieve overridden PCDs, library instances.
 280     #
 281     # The BaseName is the FILE_GUID which is also the output directory name.
 282     #
 283     #
 284     RtPath.Path = TempFullPath
 285     RtPath.BaseName = BaseName
 286     #
 287     # If file exists, compare contents
 288     #
 289     if os.path.exists(TempFullPath):
 290         with open(str(Path), 'rb') as f1, open(TempFullPath, 'rb') as f2:
 291             if f1.read() == f2.read():
 292                 return RtPath
 293     _TempInfs.append(TempFullPath)
 294     shutil.copy2(str(Path), TempFullPath)
 295     return RtPath
 296
 297 ## Remove temporary created INFs whose paths were saved in _TempInfs
 298 #
 299 def ClearDuplicatedInf():
 300     while _TempInfs:
 301         File = _TempInfs.pop()
 302         if os.path.exists(File):
 303             os.remove(File)
 304
 305 ## Convert GUID string in xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx style to C structure style
 306 #
 307 #   @param      Guid    The GUID string
 308 #
 309 #   @retval     string  The GUID string in C structure style
 310 #
 311 def GuidStringToGuidStructureString(Guid):
 312     GuidList = Guid.split('-')
 313     Result = '{'
 314     for Index in range(0, 3, 1):
 315         Result = Result + '0x' + GuidList[Index] + ', '
 316     Result = Result + '{0x' + GuidList[3][0:2] + ', 0x' + GuidList[3][2:4]
 317     for Index in range(0, 12, 2):
 318         Result = Result + ', 0x' + GuidList[4][Index:Index + 2]
 319     Result += '}}'
 320     return Result
 321
 322 ## Convert GUID structure in byte array to xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
 323 #
 324 #   @param      GuidValue   The GUID value in byte array
 325 #
 326 #   @retval     string      The GUID value in xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx format
 327 #
 328 def GuidStructureByteArrayToGuidString(GuidValue):
 329     guidValueString = GuidValue.lower().replace("{", "").replace("}", "").replace(" ", "").replace(";", "")
 330     guidValueList = guidValueString.split(",")
 331     if len(guidValueList) != 16:
 332         return ''
 333         #EdkLogger.error(None, None, "Invalid GUID value string %s" % GuidValue)
 334     try:
 335         return "%02x%02x%02x%02x-%02x%02x-%02x%02x-%02x%02x-%02x%02x%02x%02x%02x%02x" % (
 336                 int(guidValueList[3], 16),
 337                 int(guidValueList[2], 16),
 338                 int(guidValueList[1], 16),
 339                 int(guidValueList[0], 16),
 340                 int(guidValueList[5], 16),
 341                 int(guidValueList[4], 16),
 342                 int(guidValueList[7], 16),
 343                 int(guidValueList[6], 16),
 344                 int(guidValueList[8], 16),
 345                 int(guidValueList[9], 16),
 346                 int(guidValueList[10], 16),
 347                 int(guidValueList[11], 16),
 348                 int(guidValueList[12], 16),
 349                 int(guidValueList[13], 16),
 350                 int(guidValueList[14], 16),
 351                 int(guidValueList[15], 16)
 352                 )
 353     except:
 354         return ''
 355
 356 ## Convert GUID string in C structure style to xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
 357 #
 358 #   @param      GuidValue   The GUID value in C structure format
 359 #
 360 #   @retval     string      The GUID value in xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx format
 361 #
 362 def GuidStructureStringToGuidString(GuidValue):
 363     if not GlobalData.gGuidCFormatPattern.match(GuidValue):
 364         return ''
 365     guidValueString = GuidValue.lower().replace("{", "").replace("}", "").replace(" ", "").replace(";", "")
 366     guidValueList = guidValueString.split(",")
 367     if len(guidValueList) != 11:
 368         return ''
 369         #EdkLogger.error(None, None, "Invalid GUID value string %s" % GuidValue)
 370     try:
 371         return "%08x-%04x-%04x-%02x%02x-%02x%02x%02x%02x%02x%02x" % (
 372                 int(guidValueList[0], 16),
 373                 int(guidValueList[1], 16),
 374                 int(guidValueList[2], 16),
 375                 int(guidValueList[3], 16),
 376                 int(guidValueList[4], 16),
 377                 int(guidValueList[5], 16),
 378                 int(guidValueList[6], 16),
 379                 int(guidValueList[7], 16),
 380                 int(guidValueList[8], 16),
 381                 int(guidValueList[9], 16),
 382                 int(guidValueList[10], 16)
 383                 )
 384     except:
 385         return ''
 386
 387 ## Convert GUID string in C structure style to xxxxxxxx_xxxx_xxxx_xxxx_xxxxxxxxxxxx
 388 #
 389 #   @param      GuidValue   The GUID value in C structure format
 390 #
 391 #   @retval     string      The GUID value in xxxxxxxx_xxxx_xxxx_xxxx_xxxxxxxxxxxx format
 392 #
 393 def GuidStructureStringToGuidValueName(GuidValue):
 394     guidValueString = GuidValue.lower().replace("{", "").replace("}", "").replace(" ", "")
 395     guidValueList = guidValueString.split(",")
 396     if len(guidValueList) != 11:
 397         EdkLogger.error(None, FORMAT_INVALID, "Invalid GUID value string [%s]" % GuidValue)
 398     return "%08x_%04x_%04x_%02x%02x_%02x%02x%02x%02x%02x%02x" % (
 399             int(guidValueList[0], 16),
 400             int(guidValueList[1], 16),
 401             int(guidValueList[2], 16),
 402             int(guidValueList[3], 16),
 403             int(guidValueList[4], 16),
 404             int(guidValueList[5], 16),
 405             int(guidValueList[6], 16),
 406             int(guidValueList[7], 16),
 407             int(guidValueList[8], 16),
 408             int(guidValueList[9], 16),
 409             int(guidValueList[10], 16)
 410             )
 411
 412 ## Create directories
 413 #
 414 #   @param      Directory   The directory name
 415 #
 416 def CreateDirectory(Directory):
 417     if Directory is None or Directory.strip() == "":
 418         return True
 419     try:
 420         if not os.access(Directory, os.F_OK):
 421             os.makedirs(Directory)
 422     except:
 423         return False
 424     return True
 425
 426 ## Remove directories, including files and sub-directories in it
 427 #
 428 #   @param      Directory   The directory name
 429 #
 430 def RemoveDirectory(Directory, Recursively=False):
 431     if Directory is None or Directory.strip() == "" or not os.path.exists(Directory):
 432         return
 433     if Recursively:
 434         CurrentDirectory = os.getcwd()
 435         os.chdir(Directory)
 436         for File in os.listdir("."):
 437             if os.path.isdir(File):
 438                 RemoveDirectory(File, Recursively)
 439             else:
 440                 os.remove(File)
 441         os.chdir(CurrentDirectory)
 442     os.rmdir(Directory)
 443
 444 ## Store content in file
 445 #
 446 #  This method is used to save file only when its content is changed. This is
 447 #  quite useful for "make" system to decide what will be re-built and what won't.
 448 #
 449 #   @param      File            The path of file
 450 #   @param      Content         The new content of the file
 451 #   @param      IsBinaryFile    The flag indicating if the file is binary file or not
 452 #
 453 #   @retval     True            If the file content is changed and the file is renewed
 454 #   @retval     False           If the file content is the same
 455 #
 456 def SaveFileOnChange(File, Content, IsBinaryFile=True):
 457     if os.path.exists(File):
 458         try:
 459             if isinstance(Content, bytes):
 460                 with open(File, "rb") as f:
 461                     if Content == f.read():
 462                         return False
 463             else:
 464                 with open(File, "r") as f:
 465                     if Content == f.read():
 466                         return False
 467         except:
 468             EdkLogger.error(None, FILE_OPEN_FAILURE, ExtraData=File)
 469
 470     DirName = os.path.dirname(File)
 471     if not CreateDirectory(DirName):
 472         EdkLogger.error(None, FILE_CREATE_FAILURE, "Could not create directory %s" % DirName)
 473     else:
 474         if DirName == '':
 475             DirName = os.getcwd()
 476         if not os.access(DirName, os.W_OK):
 477             EdkLogger.error(None, PERMISSION_FAILURE, "Do not have write permission on directory %s" % DirName)
 478
 479     try:
 480         if GlobalData.gIsWindows:
 481             try:
 482                 from .PyUtility import SaveFileToDisk
 483                 if not SaveFileToDisk(File, Content):
 484                     EdkLogger.error(None, FILE_CREATE_FAILURE, ExtraData=File)
 485             except:
 486                 if isinstance(Content, bytes):
 487                     with open(File, "wb") as Fd:
 488                         Fd.write(Content)
 489                 else:
 490                     with open(File, "w") as Fd:
 491                         Fd.write(Content)
 492         else:
 493             if isinstance(Content, bytes):
 494                 with open(File, "wb") as Fd:
 495                     Fd.write(Content)
 496             else:
 497                 with open(File, "w") as Fd:
 498                     Fd.write(Content)
 499     except IOError as X:
 500         EdkLogger.error(None, FILE_CREATE_FAILURE, ExtraData='IOError %s' % X)
 501
 502     return True
 503
 504 ## Make a Python object persistent on file system
 505 #
 506 #   @param      Data    The object to be stored in file
 507 #   @param      File    The path of file to store the object
 508 #
 509 def DataDump(Data, File):
 510     Fd = None
 511     try:
 512         Fd = open(File, 'wb')
 513         pickle.dump(Data, Fd, pickle.HIGHEST_PROTOCOL)
 514     except:
 515         EdkLogger.error("", FILE_OPEN_FAILURE, ExtraData=File, RaiseError=False)
 516     finally:
 517         if Fd is not None:
 518             Fd.close()
 519
 520 ## Restore a Python object from a file
 521 #
 522 #   @param      File    The path of file stored the object
 523 #
 524 #   @retval     object  A python object
 525 #   @retval     None    If failure in file operation
 526 #
 527 def DataRestore(File):
 528     Data = None
 529     Fd = None
 530     try:
 531         Fd = open(File, 'rb')
 532         Data = pickle.load(Fd)
 533     except Exception as e:
 534         EdkLogger.verbose("Failed to load [%s]\n\t%s" % (File, str(e)))
 535         Data = None
 536     finally:
 537         if Fd is not None:
 538             Fd.close()
 539     return Data
 540
 541 ## Retrieve and cache the real path name in file system
 542 #
 543 #   @param      Root    The root directory of path relative to
 544 #
 545 #   @retval     str     The path string if the path exists
 546 #   @retval     None    If path doesn't exist
 547 #
 548 class DirCache:
 549     _CACHE_ = set()
 550     _UPPER_CACHE_ = {}
 551
 552     def __init__(self, Root):
 553         self._Root = Root
 554         for F in os.listdir(Root):
 555             self._CACHE_.add(F)
 556             self._UPPER_CACHE_[F.upper()] = F
 557
 558     # =[] operator
 559     def __getitem__(self, Path):
 560         Path = Path[len(os.path.commonprefix([Path, self._Root])):]
 561         if not Path:
 562             return self._Root
 563         if Path and Path[0] == os.path.sep:
 564             Path = Path[1:]
 565         if Path in self._CACHE_:
 566             return os.path.join(self._Root, Path)
 567         UpperPath = Path.upper()
 568         if UpperPath in self._UPPER_CACHE_:
 569             return os.path.join(self._Root, self._UPPER_CACHE_[UpperPath])
 570
 571         IndexList = []
 572         LastSepIndex = -1
 573         SepIndex = Path.find(os.path.sep)
 574         while SepIndex > -1:
 575             Parent = UpperPath[:SepIndex]
 576             if Parent not in self._UPPER_CACHE_:
 577                 break
 578             LastSepIndex = SepIndex
 579             SepIndex = Path.find(os.path.sep, LastSepIndex + 1)
 580
 581         if LastSepIndex == -1:
 582             return None
 583
 584         Cwd = os.getcwd()
 585         os.chdir(self._Root)
 586         SepIndex = LastSepIndex
 587         while SepIndex > -1:
 588             Parent = Path[:SepIndex]
 589             ParentKey = UpperPath[:SepIndex]
 590             if ParentKey not in self._UPPER_CACHE_:
 591                 os.chdir(Cwd)
 592                 return None
 593
 594             if Parent in self._CACHE_:
 595                 ParentDir = Parent
 596             else:
 597                 ParentDir = self._UPPER_CACHE_[ParentKey]
 598             for F in os.listdir(ParentDir):
 599                 Dir = os.path.join(ParentDir, F)
 600                 self._CACHE_.add(Dir)
 601                 self._UPPER_CACHE_[Dir.upper()] = Dir
 602
 603             SepIndex = Path.find(os.path.sep, SepIndex + 1)
 604
 605         os.chdir(Cwd)
 606         if Path in self._CACHE_:
 607             return os.path.join(self._Root, Path)
 608         elif UpperPath in self._UPPER_CACHE_:
 609             return os.path.join(self._Root, self._UPPER_CACHE_[UpperPath])
 610         return None
 611
 612 def RealPath(File, Dir='', OverrideDir=''):
 613     NewFile = os.path.normpath(os.path.join(Dir, File))
 614     NewFile = GlobalData.gAllFiles[NewFile]
 615     if not NewFile and OverrideDir:
 616         NewFile = os.path.normpath(os.path.join(OverrideDir, File))
 617         NewFile = GlobalData.gAllFiles[NewFile]
 618     return NewFile
 619
 620 def RealPath2(File, Dir='', OverrideDir=''):
 621     NewFile = None
 622     if OverrideDir:
 623         NewFile = GlobalData.gAllFiles[os.path.normpath(os.path.join(OverrideDir, File))]
 624         if NewFile:
 625             if OverrideDir[-1] == os.path.sep:
 626                 return NewFile[len(OverrideDir):], NewFile[0:len(OverrideDir)]
 627             else:
 628                 return NewFile[len(OverrideDir) + 1:], NewFile[0:len(OverrideDir)]
 629     if GlobalData.gAllFiles:
 630         NewFile = GlobalData.gAllFiles[os.path.normpath(os.path.join(Dir, File))]
 631     if not NewFile:
 632         NewFile = os.path.normpath(os.path.join(Dir, File))
 633         if not os.path.exists(NewFile):
 634             return None, None
 635     if NewFile:
 636         if Dir:
 637             if Dir[-1] == os.path.sep:
 638                 return NewFile[len(Dir):], NewFile[0:len(Dir)]
 639             else:
 640                 return NewFile[len(Dir) + 1:], NewFile[0:len(Dir)]
 641         else:
 642             return NewFile, ''
 643
 644     return None, None
 645
 646 ## Get GUID value from given packages
 647 #
 648 #   @param      CName           The CName of the GUID
 649 #   @param      PackageList     List of packages looking-up in
 650 #   @param      Inffile         The driver file
 651 #
 652 #   @retval     GuidValue   if the CName is found in any given package
 653 #   @retval     None        if the CName is not found in all given packages
 654 #
 655 def GuidValue(CName, PackageList, Inffile = None):
 656     for P in PackageList:
 657         GuidKeys = list(P.Guids.keys())
 658         if Inffile and P._PrivateGuids:
 659             if not Inffile.startswith(P.MetaFile.Dir):
 660                 GuidKeys = [x for x in P.Guids if x not in P._PrivateGuids]
 661         if CName in GuidKeys:
 662             return P.Guids[CName]
 663     return None
 664
 665 ## Get Protocol value from given packages
 666 #
 667 #   @param      CName           The CName of the GUID
 668 #   @param      PackageList     List of packages looking-up in
 669 #   @param      Inffile         The driver file
 670 #
 671 #   @retval     GuidValue   if the CName is found in any given package
 672 #   @retval     None        if the CName is not found in all given packages
 673 #
 674 def ProtocolValue(CName, PackageList, Inffile = None):
 675     for P in PackageList:
 676         ProtocolKeys = list(P.Protocols.keys())
 677         if Inffile and P._PrivateProtocols:
 678             if not Inffile.startswith(P.MetaFile.Dir):
 679                 ProtocolKeys = [x for x in P.Protocols if x not in P._PrivateProtocols]
 680         if CName in ProtocolKeys:
 681             return P.Protocols[CName]
 682     return None
 683
 684 ## Get PPI value from given packages
 685 #
 686 #   @param      CName           The CName of the GUID
 687 #   @param      PackageList     List of packages looking-up in
 688 #   @param      Inffile         The driver file
 689 #
 690 #   @retval     GuidValue   if the CName is found in any given package
 691 #   @retval     None        if the CName is not found in all given packages
 692 #
 693 def PpiValue(CName, PackageList, Inffile = None):
 694     for P in PackageList:
 695         PpiKeys = list(P.Ppis.keys())
 696         if Inffile and P._PrivatePpis:
 697             if not Inffile.startswith(P.MetaFile.Dir):
 698                 PpiKeys = [x for x in P.Ppis if x not in P._PrivatePpis]
 699         if CName in PpiKeys:
 700             return P.Ppis[CName]
 701     return None
 702
 703 ## A string template class
 704 #
 705 #  This class implements a template for string replacement. A string template
 706 #  looks like following
 707 #
 708 #       ${BEGIN} other_string ${placeholder_name} other_string ${END}
 709 #
 710 #  The string between ${BEGIN} and ${END} will be repeated as many times as the
 711 #  length of "placeholder_name", which is a list passed through a dict. The
 712 #  "placeholder_name" is the key name of the dict. The ${BEGIN} and ${END} can
 713 #  be not used and, in this case, the "placeholder_name" must not a list and it
 714 #  will just be replaced once.
 715 #
 716 class TemplateString(object):
 717     _REPEAT_START_FLAG = "BEGIN"
 718     _REPEAT_END_FLAG = "END"
 719
 720     class Section(object):
 721         _LIST_TYPES = [type([]), type(set()), type((0,))]
 722
 723         def __init__(self, TemplateSection, PlaceHolderList):
 724             self._Template = TemplateSection
 725             self._PlaceHolderList = []
 726
 727             # Split the section into sub-sections according to the position of placeholders
 728             if PlaceHolderList:
 729                 self._SubSectionList = []
 730                 SubSectionStart = 0
 731                 #
 732                 # The placeholders passed in must be in the format of
 733                 #
 734                 #   PlaceHolderName, PlaceHolderStartPoint, PlaceHolderEndPoint
 735                 #
 736                 for PlaceHolder, Start, End in PlaceHolderList:
 737                     self._SubSectionList.append(TemplateSection[SubSectionStart:Start])
 738                     self._SubSectionList.append(TemplateSection[Start:End])
 739                     self._PlaceHolderList.append(PlaceHolder)
 740                     SubSectionStart = End
 741                 if SubSectionStart < len(TemplateSection):
 742                     self._SubSectionList.append(TemplateSection[SubSectionStart:])
 743             else:
 744                 self._SubSectionList = [TemplateSection]
 745
 746         def __str__(self):
 747             return self._Template + " : " + str(self._PlaceHolderList)
 748
 749         def Instantiate(self, PlaceHolderValues):
 750             RepeatTime = -1
 751             RepeatPlaceHolders = {}
 752             NonRepeatPlaceHolders = {}
 753
 754             for PlaceHolder in self._PlaceHolderList:
 755                 if PlaceHolder not in PlaceHolderValues:
 756                     continue
 757                 Value = PlaceHolderValues[PlaceHolder]
 758                 if type(Value) in self._LIST_TYPES:
 759                     if RepeatTime < 0:
 760                         RepeatTime = len(Value)
 761                     elif RepeatTime != len(Value):
 762                         EdkLogger.error(
 763                                     "TemplateString",
 764                                     PARAMETER_INVALID,
 765                                     "${%s} has different repeat time from others!" % PlaceHolder,
 766                                     ExtraData=str(self._Template)
 767                                     )
 768                     RepeatPlaceHolders["${%s}" % PlaceHolder] = Value
 769                 else:
 770                     NonRepeatPlaceHolders["${%s}" % PlaceHolder] = Value
 771
 772             if NonRepeatPlaceHolders:
 773                 StringList = []
 774                 for S in self._SubSectionList:
 775                     if S not in NonRepeatPlaceHolders:
 776                         StringList.append(S)
 777                     else:
 778                         StringList.append(str(NonRepeatPlaceHolders[S]))
 779             else:
 780                 StringList = self._SubSectionList
 781
 782             if RepeatPlaceHolders:
 783                 TempStringList = []
 784                 for Index in range(RepeatTime):
 785                     for S in StringList:
 786                         if S not in RepeatPlaceHolders:
 787                             TempStringList.append(S)
 788                         else:
 789                             TempStringList.append(str(RepeatPlaceHolders[S][Index]))
 790                 StringList = TempStringList
 791
 792             return "".join(StringList)
 793
 794     ## Constructor
 795     def __init__(self, Template=None):
 796         self.String = ''
 797         self.IsBinary = False
 798         self._Template = Template
 799         self._TemplateSectionList = self._Parse(Template)
 800
 801     ## str() operator
 802     #
 803     #   @retval     string  The string replaced
 804     #
 805     def __str__(self):
 806         return self.String
 807
 808     ## Split the template string into fragments per the ${BEGIN} and ${END} flags
 809     #
 810     #   @retval     list    A list of TemplateString.Section objects
 811     #
 812     def _Parse(self, Template):
 813         SectionStart = 0
 814         SearchFrom = 0
 815         MatchEnd = 0
 816         PlaceHolderList = []
 817         TemplateSectionList = []
 818         while Template:
 819             MatchObj = gPlaceholderPattern.search(Template, SearchFrom)
 820             if not MatchObj:
 821                 if MatchEnd <= len(Template):
 822                     TemplateSection = TemplateString.Section(Template[SectionStart:], PlaceHolderList)
 823                     TemplateSectionList.append(TemplateSection)
 824                 break
 825
 826             MatchString = MatchObj.group(1)
 827             MatchStart = MatchObj.start()
 828             MatchEnd = MatchObj.end()
 829
 830             if MatchString == self._REPEAT_START_FLAG:
 831                 if MatchStart > SectionStart:
 832                     TemplateSection = TemplateString.Section(Template[SectionStart:MatchStart], PlaceHolderList)
 833                     TemplateSectionList.append(TemplateSection)
 834                 SectionStart = MatchEnd
 835                 PlaceHolderList = []
 836             elif MatchString == self._REPEAT_END_FLAG:
 837                 TemplateSection = TemplateString.Section(Template[SectionStart:MatchStart], PlaceHolderList)
 838                 TemplateSectionList.append(TemplateSection)
 839                 SectionStart = MatchEnd
 840                 PlaceHolderList = []
 841             else:
 842                 PlaceHolderList.append((MatchString, MatchStart - SectionStart, MatchEnd - SectionStart))
 843             SearchFrom = MatchEnd
 844         return TemplateSectionList
 845
 846     ## Replace the string template with dictionary of placeholders and append it to previous one
 847     #
 848     #   @param      AppendString    The string template to append
 849     #   @param      Dictionary      The placeholder dictionaries
 850     #
 851     def Append(self, AppendString, Dictionary=None):
 852         if Dictionary:
 853             SectionList = self._Parse(AppendString)
 854             self.String += "".join(S.Instantiate(Dictionary) for S in SectionList)
 855         else:
 856             self.String += AppendString
 857
 858     ## Replace the string template with dictionary of placeholders
 859     #
 860     #   @param      Dictionary      The placeholder dictionaries
 861     #
 862     #   @retval     str             The string replaced with placeholder values
 863     #
 864     def Replace(self, Dictionary=None):
 865         return "".join(S.Instantiate(Dictionary) for S in self._TemplateSectionList)
 866
 867 ## Progress indicator class
 868 #
 869 #  This class makes use of thread to print progress on console.
 870 #
 871 class Progressor:
 872     # for avoiding deadloop
 873     _StopFlag = None
 874     _ProgressThread = None
 875     _CheckInterval = 0.25
 876
 877     ## Constructor
 878     #
 879     #   @param      OpenMessage     The string printed before progress charaters
 880     #   @param      CloseMessage    The string printed after progress charaters
 881     #   @param      ProgressChar    The charater used to indicate the progress
 882     #   @param      Interval        The interval in seconds between two progress charaters
 883     #
 884     def __init__(self, OpenMessage="", CloseMessage="", ProgressChar='.', Interval=1.0):
 885         self.PromptMessage = OpenMessage
 886         self.CodaMessage = CloseMessage
 887         self.ProgressChar = ProgressChar
 888         self.Interval = Interval
 889         if Progressor._StopFlag is None:
 890             Progressor._StopFlag = threading.Event()
 891
 892     ## Start to print progress charater
 893     #
 894     #   @param      OpenMessage     The string printed before progress charaters
 895     #
 896     def Start(self, OpenMessage=None):
 897         if OpenMessage is not None:
 898             self.PromptMessage = OpenMessage
 899         Progressor._StopFlag.clear()
 900         if Progressor._ProgressThread is None:
 901             Progressor._ProgressThread = threading.Thread(target=self._ProgressThreadEntry)
 902             Progressor._ProgressThread.setDaemon(False)
 903             Progressor._ProgressThread.start()
 904
 905     ## Stop printing progress charater
 906     #
 907     #   @param      CloseMessage    The string printed after progress charaters
 908     #
 909     def Stop(self, CloseMessage=None):
 910         OriginalCodaMessage = self.CodaMessage
 911         if CloseMessage is not None:
 912             self.CodaMessage = CloseMessage
 913         self.Abort()
 914         self.CodaMessage = OriginalCodaMessage
 915
 916     ## Thread entry method
 917     def _ProgressThreadEntry(self):
 918         sys.stdout.write(self.PromptMessage + " ")
 919         sys.stdout.flush()
 920         TimeUp = 0.0
 921         while not Progressor._StopFlag.isSet():
 922             if TimeUp <= 0.0:
 923                 sys.stdout.write(self.ProgressChar)
 924                 sys.stdout.flush()
 925                 TimeUp = self.Interval
 926             time.sleep(self._CheckInterval)
 927             TimeUp -= self._CheckInterval
 928         sys.stdout.write(" " + self.CodaMessage + "\n")
 929         sys.stdout.flush()
 930
 931     ## Abort the progress display
 932     @staticmethod
 933     def Abort():
 934         if Progressor._StopFlag is not None:
 935             Progressor._StopFlag.set()
 936         if Progressor._ProgressThread is not None:
 937             Progressor._ProgressThread.join()
 938             Progressor._ProgressThread = None
 939
 940 ## A dict which can access its keys and/or values orderly
 941 #
 942 #  The class implements a new kind of dict which its keys or values can be
 943 #  accessed in the order they are added into the dict. It guarantees the order
 944 #  by making use of an internal list to keep a copy of keys.
 945 #
 946 class sdict(IterableUserDict):
 947     ## Constructor
 948     def __init__(self):
 949         IterableUserDict.__init__(self)
 950         self._key_list = []
 951
 952     ## [] operator
 953     def __setitem__(self, key, value):
 954         if key not in self._key_list:
 955             self._key_list.append(key)
 956         IterableUserDict.__setitem__(self, key, value)
 957
 958     ## del operator
 959     def __delitem__(self, key):
 960         self._key_list.remove(key)
 961         IterableUserDict.__delitem__(self, key)
 962
 963     ## used in "for k in dict" loop to ensure the correct order
 964     def __iter__(self):
 965         return self.iterkeys()
 966
 967     ## len() support
 968     def __len__(self):
 969         return len(self._key_list)
 970
 971     ## "in" test support
 972     def __contains__(self, key):
 973         return key in self._key_list
 974
 975     ## indexof support
 976     def index(self, key):
 977         return self._key_list.index(key)
 978
 979     ## insert support
 980     def insert(self, key, newkey, newvalue, order):
 981         index = self._key_list.index(key)
 982         if order == 'BEFORE':
 983             self._key_list.insert(index, newkey)
 984             IterableUserDict.__setitem__(self, newkey, newvalue)
 985         elif order == 'AFTER':
 986             self._key_list.insert(index + 1, newkey)
 987             IterableUserDict.__setitem__(self, newkey, newvalue)
 988
 989     ## append support
 990     def append(self, sdict):
 991         for key in sdict:
 992             if key not in self._key_list:
 993                 self._key_list.append(key)
 994             IterableUserDict.__setitem__(self, key, sdict[key])
 995
 996     def has_key(self, key):
 997         return key in self._key_list
 998
 999     ## Empty the dict
1000     def clear(self):
1001         self._key_list = []
1002         IterableUserDict.clear(self)
1003
1004     ## Return a copy of keys
1005     def keys(self):
1006         keys = []
1007         for key in self._key_list:
1008             keys.append(key)
1009         return keys
1010
1011     ## Return a copy of values
1012     def values(self):
1013         values = []
1014         for key in self._key_list:
1015             values.append(self[key])
1016         return values
1017
1018     ## Return a copy of (key, value) list
1019     def items(self):
1020         items = []
1021         for key in self._key_list:
1022             items.append((key, self[key]))
1023         return items
1024
1025     ## Iteration support
1026     def iteritems(self):
1027         return iter(self.items())
1028
1029     ## Keys interation support
1030     def iterkeys(self):
1031         return self.keys()
1032
1033     ## Values interation support
1034     def itervalues(self):
1035         return self.values()
1036
1037     ## Return value related to a key, and remove the (key, value) from the dict
1038     def pop(self, key, *dv):
1039         value = None
1040         if key in self._key_list:
1041             value = self[key]
1042             self.__delitem__(key)
1043         elif len(dv) != 0 :
1044             value = dv[0]
1045         return value
1046
1047     ## Return (key, value) pair, and remove the (key, value) from the dict
1048     def popitem(self):
1049         key = self._key_list[-1]
1050         value = self[key]
1051         self.__delitem__(key)
1052         return key, value
1053
1054     def update(self, dict=None, **kwargs):
1055         if dict is not None:
1056             for k, v in dict.items():
1057                 self[k] = v
1058         if len(kwargs):
1059             for k, v in kwargs.items():
1060                 self[k] = v
1061
1062 ## Dictionary with restricted keys
1063 #
1064 class rdict(dict):
1065     ## Constructor
1066     def __init__(self, KeyList):
1067         for Key in KeyList:
1068             dict.__setitem__(self, Key, "")
1069
1070     ## []= operator
1071     def __setitem__(self, key, value):
1072         if key not in self:
1073             EdkLogger.error("RestrictedDict", ATTRIBUTE_SET_FAILURE, "Key [%s] is not allowed" % key,
1074                             ExtraData=", ".join(dict.keys(self)))
1075         dict.__setitem__(self, key, value)
1076
1077     ## =[] operator
1078     def __getitem__(self, key):
1079         if key not in self:
1080             return ""
1081         return dict.__getitem__(self, key)
1082
1083     ## del operator
1084     def __delitem__(self, key):
1085         EdkLogger.error("RestrictedDict", ATTRIBUTE_ACCESS_DENIED, ExtraData="del")
1086
1087     ## Empty the dict
1088     def clear(self):
1089         for Key in self:
1090             self.__setitem__(Key, "")
1091
1092     ## Return value related to a key, and remove the (key, value) from the dict
1093     def pop(self, key, *dv):
1094         EdkLogger.error("RestrictedDict", ATTRIBUTE_ACCESS_DENIED, ExtraData="pop")
1095
1096     ## Return (key, value) pair, and remove the (key, value) from the dict
1097     def popitem(self):
1098         EdkLogger.error("RestrictedDict", ATTRIBUTE_ACCESS_DENIED, ExtraData="popitem")
1099
1100 ## Dictionary using prioritized list as key
1101 #
1102 class tdict:
1103     _ListType = type([])
1104     _TupleType = type(())
1105     _Wildcard = 'COMMON'
1106     _ValidWildcardList = ['COMMON', 'DEFAULT', 'ALL', '*', 'PLATFORM']
1107
1108     def __init__(self, _Single_=False, _Level_=2):
1109         self._Level_ = _Level_
1110         self.data = {}
1111         self._Single_ = _Single_
1112
1113     # =[] operator
1114     def __getitem__(self, key):
1115         KeyType = type(key)
1116         RestKeys = None
1117         if KeyType == self._ListType or KeyType == self._TupleType:
1118             FirstKey = key[0]
1119             if len(key) > 1:
1120                 RestKeys = key[1:]
1121             elif self._Level_ > 1:
1122                 RestKeys = [self._Wildcard for i in range(0, self._Level_ - 1)]
1123         else:
1124             FirstKey = key
1125             if self._Level_ > 1:
1126                 RestKeys = [self._Wildcard for i in range(0, self._Level_ - 1)]
1127
1128         if FirstKey is None or str(FirstKey).upper() in self._ValidWildcardList:
1129             FirstKey = self._Wildcard
1130
1131         if self._Single_:
1132             return self._GetSingleValue(FirstKey, RestKeys)
1133         else:
1134             return self._GetAllValues(FirstKey, RestKeys)
1135
1136     def _GetSingleValue(self, FirstKey, RestKeys):
1137         Value = None
1138         #print "%s-%s" % (FirstKey, self._Level_) ,
1139         if self._Level_ > 1:
1140             if FirstKey == self._Wildcard:
1141                 if FirstKey in self.data:
1142                     Value = self.data[FirstKey][RestKeys]
1143                 if Value is None:
1144                     for Key in self.data:
1145                         Value = self.data[Key][RestKeys]
1146                         if Value is not None: break
1147             else:
1148                 if FirstKey in self.data:
1149                     Value = self.data[FirstKey][RestKeys]
1150                 if Value is None and self._Wildcard in self.data:
1151                     #print "Value=None"
1152                     Value = self.data[self._Wildcard][RestKeys]
1153         else:
1154             if FirstKey == self._Wildcard:
1155                 if FirstKey in self.data:
1156                     Value = self.data[FirstKey]
1157                 if Value is None:
1158                     for Key in self.data:
1159                         Value = self.data[Key]
1160                         if Value is not None: break
1161             else:
1162                 if FirstKey in self.data:
1163                     Value = self.data[FirstKey]
1164                 elif self._Wildcard in self.data:
1165                     Value = self.data[self._Wildcard]
1166         return Value
1167
1168     def _GetAllValues(self, FirstKey, RestKeys):
1169         Value = []
1170         if self._Level_ > 1:
1171             if FirstKey == self._Wildcard:
1172                 for Key in self.data:
1173                     Value += self.data[Key][RestKeys]
1174             else:
1175                 if FirstKey in self.data:
1176                     Value += self.data[FirstKey][RestKeys]
1177                 if self._Wildcard in self.data:
1178                     Value += self.data[self._Wildcard][RestKeys]
1179         else:
1180             if FirstKey == self._Wildcard:
1181                 for Key in self.data:
1182                     Value.append(self.data[Key])
1183             else:
1184                 if FirstKey in self.data:
1185                     Value.append(self.data[FirstKey])
1186                 if self._Wildcard in self.data:
1187                     Value.append(self.data[self._Wildcard])
1188         return Value
1189
1190     ## []= operator
1191     def __setitem__(self, key, value):
1192         KeyType = type(key)
1193         RestKeys = None
1194         if KeyType == self._ListType or KeyType == self._TupleType:
1195             FirstKey = key[0]
1196             if len(key) > 1:
1197                 RestKeys = key[1:]
1198             else:
1199                 RestKeys = [self._Wildcard for i in range(0, self._Level_ - 1)]
1200         else:
1201             FirstKey = key
1202             if self._Level_ > 1:
1203                 RestKeys = [self._Wildcard for i in range(0, self._Level_ - 1)]
1204
1205         if FirstKey in self._ValidWildcardList:
1206             FirstKey = self._Wildcard
1207
1208         if FirstKey not in self.data and self._Level_ > 0:
1209             self.data[FirstKey] = tdict(self._Single_, self._Level_ - 1)
1210
1211         if self._Level_ > 1:
1212             self.data[FirstKey][RestKeys] = value
1213         else:
1214             self.data[FirstKey] = value
1215
1216     def SetGreedyMode(self):
1217         self._Single_ = False
1218         if self._Level_ > 1:
1219             for Key in self.data:
1220                 self.data[Key].SetGreedyMode()
1221
1222     def SetSingleMode(self):
1223         self._Single_ = True
1224         if self._Level_ > 1:
1225             for Key in self.data:
1226                 self.data[Key].SetSingleMode()
1227
1228     def GetKeys(self, KeyIndex=0):
1229         assert KeyIndex >= 0
1230         if KeyIndex == 0:
1231             return set(self.data.keys())
1232         else:
1233             keys = set()
1234             for Key in self.data:
1235                 keys |= self.data[Key].GetKeys(KeyIndex - 1)
1236             return keys
1237
1238 def IsFieldValueAnArray (Value):
1239     Value = Value.strip()
1240     if Value.startswith(TAB_GUID) and Value.endswith(')'):
1241         return True
1242     if Value.startswith('L"') and Value.endswith('"')  and len(list(Value[2:-1])) > 1:
1243         return True
1244     if Value[0] == '"' and Value[-1] == '"' and len(list(Value[1:-1])) > 1:
1245         return True
1246     if Value[0] == '{' and Value[-1] == '}':
1247         return True
1248     if Value.startswith("L'") and Value.endswith("'") and len(list(Value[2:-1])) > 1:
1249         return True
1250     if Value[0] == "'" and Value[-1] == "'" and len(list(Value[1:-1])) > 1:
1251         return True
1252     return False
1253
1254 def AnalyzePcdExpression(Setting):
1255     Setting = Setting.strip()
1256     # There might be escaped quote in a string: \", \\\" , \', \\\'
1257     Data = Setting
1258     # There might be '|' in string and in ( ... | ... ), replace it with '-'
1259     NewStr = ''
1260     InSingleQuoteStr = False
1261     InDoubleQuoteStr = False
1262     Pair = 0
1263     for Index, ch in enumerate(Data):
1264         if ch == '"' and not InSingleQuoteStr:
1265             if Data[Index - 1] != '\\':
1266                 InDoubleQuoteStr = not InDoubleQuoteStr
1267         elif ch == "'" and not InDoubleQuoteStr:
1268             if Data[Index - 1] != '\\':
1269                 InSingleQuoteStr = not InSingleQuoteStr
1270         elif ch == '(' and not (InSingleQuoteStr or InDoubleQuoteStr):
1271             Pair += 1
1272         elif ch == ')' and not (InSingleQuoteStr or InDoubleQuoteStr):
1273             Pair -= 1
1274
1275         if (Pair > 0 or InSingleQuoteStr or InDoubleQuoteStr) and ch == TAB_VALUE_SPLIT:
1276             NewStr += '-'
1277         else:
1278             NewStr += ch
1279     FieldList = []
1280     StartPos = 0
1281     while True:
1282         Pos = NewStr.find(TAB_VALUE_SPLIT, StartPos)
1283         if Pos < 0:
1284             FieldList.append(Setting[StartPos:].strip())
1285             break
1286         FieldList.append(Setting[StartPos:Pos].strip())
1287         StartPos = Pos + 1
1288
1289     return FieldList
1290
1291 def ParseDevPathValue (Value):
1292     if '\\' in Value:
1293         Value.replace('\\', '/').replace(' ', '')
1294
1295     Cmd = 'DevicePath ' + '"' + Value + '"'
1296     try:
1297         p = subprocess.Popen(Cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
1298         out, err = p.communicate()
1299     except Exception as X:
1300         raise BadExpression("DevicePath: %s" % (str(X)) )
1301     finally:
1302         subprocess._cleanup()
1303         p.stdout.close()
1304         p.stderr.close()
1305     if err:
1306         raise BadExpression("DevicePath: %s" % str(err))
1307     Size = len(out.split())
1308     out = ','.join(out.decode(encoding='utf-8', errors='ignore').split())
1309     return '{' + out + '}', Size
1310
1311 def ParseFieldValue (Value):
1312     if isinstance(Value, type(0)):
1313         return Value, (Value.bit_length() + 7) // 8
1314     if not isinstance(Value, type('')):
1315         raise BadExpression('Type %s is %s' %(Value, type(Value)))
1316     Value = Value.strip()
1317     if Value.startswith(TAB_UINT8) and Value.endswith(')'):
1318         Value, Size = ParseFieldValue(Value.split('(', 1)[1][:-1])
1319         if Size > 1:
1320             raise BadExpression('Value (%s) Size larger than %d' %(Value, Size))
1321         return Value, 1
1322     if Value.startswith(TAB_UINT16) and Value.endswith(')'):
1323         Value, Size = ParseFieldValue(Value.split('(', 1)[1][:-1])
1324         if Size > 2:
1325             raise BadExpression('Value (%s) Size larger than %d' %(Value, Size))
1326         return Value, 2
1327     if Value.startswith(TAB_UINT32) and Value.endswith(')'):
1328         Value, Size = ParseFieldValue(Value.split('(', 1)[1][:-1])
1329         if Size > 4:
1330             raise BadExpression('Value (%s) Size larger than %d' %(Value, Size))
1331         return Value, 4
1332     if Value.startswith(TAB_UINT64) and Value.endswith(')'):
1333         Value, Size = ParseFieldValue(Value.split('(', 1)[1][:-1])
1334         if Size > 8:
1335             raise BadExpression('Value (%s) Size larger than %d' % (Value, Size))
1336         return Value, 8
1337     if Value.startswith(TAB_GUID) and Value.endswith(')'):
1338         Value = Value.split('(', 1)[1][:-1].strip()
1339         if Value[0] == '{' and Value[-1] == '}':
1340             TmpValue = GuidStructureStringToGuidString(Value)
1341             if not TmpValue:
1342                 raise BadExpression("Invalid GUID value string %s" % Value)
1343             Value = TmpValue
1344         if Value[0] == '"' and Value[-1] == '"':
1345             Value = Value[1:-1]
1346         try:
1347             Value = "{" + ','.join([str(i) for i in uuid.UUID(Value).bytes_le]) + "}"
1348         except ValueError as Message:
1349             raise BadExpression(Message)
1350         Value, Size = ParseFieldValue(Value)
1351         return Value, 16
1352     if Value.startswith('L"') and Value.endswith('"'):
1353         # Unicode String
1354         # translate escape character
1355         Value = Value[1:]
1356         try:
1357             Value = eval(Value)
1358         except:
1359             Value = Value[1:-1]
1360         List = list(Value)
1361         List.reverse()
1362         Value = 0
1363         for Char in List:
1364             Value = (Value << 16) | ord(Char)
1365         return Value, (len(List) + 1) * 2
1366     if Value.startswith('"') and Value.endswith('"'):
1367         # ASCII String
1368         # translate escape character
1369         try:
1370             Value = eval(Value)
1371         except:
1372             Value = Value[1:-1]
1373         List = list(Value)
1374         List.reverse()
1375         Value = 0
1376         for Char in List:
1377             Value = (Value << 8) | ord(Char)
1378         return Value, len(List) + 1
1379     if Value.startswith("L'") and Value.endswith("'"):
1380         # Unicode Character Constant
1381         # translate escape character
1382         Value = Value[1:]
1383         try:
1384             Value = eval(Value)
1385         except:
1386             Value = Value[1:-1]
1387         List = list(Value)
1388         if len(List) == 0:
1389             raise BadExpression('Length %s is %s' % (Value, len(List)))
1390         List.reverse()
1391         Value = 0
1392         for Char in List:
1393             Value = (Value << 16) | ord(Char)
1394         return Value, len(List) * 2
1395     if Value.startswith("'") and Value.endswith("'"):
1396         # Character constant
1397         # translate escape character
1398         try:
1399             Value = eval(Value)
1400         except:
1401             Value = Value[1:-1]
1402         List = list(Value)
1403         if len(List) == 0:
1404             raise BadExpression('Length %s is %s' % (Value, len(List)))
1405         List.reverse()
1406         Value = 0
1407         for Char in List:
1408             Value = (Value << 8) | ord(Char)
1409         return Value, len(List)
1410     if Value.startswith('{') and Value.endswith('}'):
1411         # Byte array
1412         Value = Value[1:-1]
1413         List = [Item.strip() for Item in Value.split(',')]
1414         List.reverse()
1415         Value = 0
1416         RetSize = 0
1417         for Item in List:
1418             ItemValue, Size = ParseFieldValue(Item)
1419             RetSize += Size
1420             for I in range(Size):
1421                 Value = (Value << 8) | ((ItemValue >> 8 * I) & 0xff)
1422         return Value, RetSize
1423     if Value.startswith('DEVICE_PATH(') and Value.endswith(')'):
1424         Value = Value.replace("DEVICE_PATH(", '').rstrip(')')
1425         Value = Value.strip().strip('"')
1426         return ParseDevPathValue(Value)
1427     if Value.lower().startswith('0x'):
1428         try:
1429             Value = int(Value, 16)
1430         except:
1431             raise BadExpression("invalid hex value: %s" % Value)
1432         if Value == 0:
1433             return 0, 1
1434         return Value, (Value.bit_length() + 7) // 8
1435     if Value[0].isdigit():
1436         Value = int(Value, 10)
1437         if Value == 0:
1438             return 0, 1
1439         return Value, (Value.bit_length() + 7) // 8
1440     if Value.lower() == 'true':
1441         return 1, 1
1442     if Value.lower() == 'false':
1443         return 0, 1
1444     return Value, 1
1445
1446 ## AnalyzeDscPcd
1447 #
1448 #  Analyze DSC PCD value, since there is no data type info in DSC
1449 #  This function is used to match functions (AnalyzePcdData) used for retrieving PCD value from database
1450 #  1. Feature flag: TokenSpace.PcdCName|PcdValue
1451 #  2. Fix and Patch:TokenSpace.PcdCName|PcdValue[|VOID*[|MaxSize]]
1452 #  3. Dynamic default:
1453 #     TokenSpace.PcdCName|PcdValue[|VOID*[|MaxSize]]
1454 #     TokenSpace.PcdCName|PcdValue
1455 #  4. Dynamic VPD:
1456 #     TokenSpace.PcdCName|VpdOffset[|VpdValue]
1457 #     TokenSpace.PcdCName|VpdOffset[|MaxSize[|VpdValue]]
1458 #  5. Dynamic HII:
1459 #     TokenSpace.PcdCName|HiiString|VariableGuid|VariableOffset[|HiiValue]
1460 #  PCD value needs to be located in such kind of string, and the PCD value might be an expression in which
1461 #    there might have "|" operator, also in string value.
1462 #
1463 #  @param Setting: String contain information described above with "TokenSpace.PcdCName|" stripped
1464 #  @param PcdType: PCD type: feature, fixed, dynamic default VPD HII
1465 #  @param DataType: The datum type of PCD: VOID*, UNIT, BOOL
1466 #  @retval:
1467 #    ValueList: A List contain fields described above
1468 #    IsValid:   True if conforming EBNF, otherwise False
1469 #    Index:     The index where PcdValue is in ValueList
1470 #
1471 def AnalyzeDscPcd(Setting, PcdType, DataType=''):
1472     FieldList = AnalyzePcdExpression(Setting)
1473
1474     IsValid = True
1475     if PcdType in (MODEL_PCD_FIXED_AT_BUILD, MODEL_PCD_PATCHABLE_IN_MODULE, MODEL_PCD_DYNAMIC_DEFAULT, MODEL_PCD_DYNAMIC_EX_DEFAULT):
1476         Value = FieldList[0]
1477         Size = ''
1478         if len(FieldList) > 1 and FieldList[1]:
1479             DataType = FieldList[1]
1480             if FieldList[1] != TAB_VOID:
1481                 IsValid = False
1482         if len(FieldList) > 2:
1483             Size = FieldList[2]
1484         if IsValid:
1485             if DataType == "":
1486                 IsValid = (len(FieldList) <= 1)
1487             else:
1488                 IsValid = (len(FieldList) <= 3)
1489
1490         if Size:
1491             try:
1492                 int(Size, 16) if Size.upper().startswith("0X") else int(Size)
1493             except:
1494                 IsValid = False
1495                 Size = -1
1496         return [str(Value), DataType, str(Size)], IsValid, 0
1497     elif PcdType == MODEL_PCD_FEATURE_FLAG:
1498         Value = FieldList[0]
1499         Size = ''
1500         IsValid = (len(FieldList) <= 1)
1501         return [Value, DataType, str(Size)], IsValid, 0
1502     elif PcdType in (MODEL_PCD_DYNAMIC_VPD, MODEL_PCD_DYNAMIC_EX_VPD):
1503         VpdOffset = FieldList[0]
1504         Value = Size = ''
1505         if not DataType == TAB_VOID:
1506             if len(FieldList) > 1:
1507                 Value = FieldList[1]
1508         else:
1509             if len(FieldList) > 1:
1510                 Size = FieldList[1]
1511             if len(FieldList) > 2:
1512                 Value = FieldList[2]
1513         if DataType == "":
1514             IsValid = (len(FieldList) <= 1)
1515         else:
1516             IsValid = (len(FieldList) <= 3)
1517         if Size:
1518             try:
1519                 int(Size, 16) if Size.upper().startswith("0X") else int(Size)
1520             except:
1521                 IsValid = False
1522                 Size = -1
1523         return [VpdOffset, str(Size), Value], IsValid, 2
1524     elif PcdType in (MODEL_PCD_DYNAMIC_HII, MODEL_PCD_DYNAMIC_EX_HII):
1525         IsValid = (3 <= len(FieldList) <= 5)
1526         HiiString = FieldList[0]
1527         Guid = Offset = Value = Attribute = ''
1528         if len(FieldList) > 1:
1529             Guid = FieldList[1]
1530         if len(FieldList) > 2:
1531             Offset = FieldList[2]
1532         if len(FieldList) > 3:
1533             Value = FieldList[3]
1534             if not Value:
1535                 IsValid = False
1536         if len(FieldList) > 4:
1537             Attribute = FieldList[4]
1538         return [HiiString, Guid, Offset, Value, Attribute], IsValid, 3
1539     return [], False, 0
1540
1541 ## AnalyzePcdData
1542 #
1543 #  Analyze the pcd Value, Datum type and TokenNumber.
1544 #  Used to avoid split issue while the value string contain "|" character
1545 #
1546 #  @param[in] Setting:  A String contain value/datum type/token number information;
1547 #
1548 #  @retval   ValueList: A List contain value, datum type and toke number.
1549 #
1550 def AnalyzePcdData(Setting):
1551     ValueList = ['', '', '']
1552
1553     ValueRe = re.compile(r'^\s*L?\".*\|.*\"')
1554     PtrValue = ValueRe.findall(Setting)
1555
1556     ValueUpdateFlag = False
1557
1558     if len(PtrValue) >= 1:
1559         Setting = re.sub(ValueRe, '', Setting)
1560         ValueUpdateFlag = True
1561
1562     TokenList = Setting.split(TAB_VALUE_SPLIT)
1563     ValueList[0:len(TokenList)] = TokenList
1564
1565     if ValueUpdateFlag:
1566         ValueList[0] = PtrValue[0]
1567
1568     return ValueList
1569
1570 ## check format of PCD value against its the datum type
1571 #
1572 # For PCD value setting
1573 #
1574 def CheckPcdDatum(Type, Value):
1575     if Type == TAB_VOID:
1576         ValueRe = re.compile(r'\s*L?\".*\"\s*$')
1577         if not (((Value.startswith('L"') or Value.startswith('"')) and Value.endswith('"'))
1578                 or (Value.startswith('{') and Value.endswith('}')) or (Value.startswith("L'") or Value.startswith("'") and Value.endswith("'"))
1579                ):
1580             return False, "Invalid value [%s] of type [%s]; must be in the form of {...} for array"\
1581                           ", \"...\" or \'...\' for string, L\"...\" or L\'...\' for unicode string" % (Value, Type)
1582         elif ValueRe.match(Value):
1583             # Check the chars in UnicodeString or CString is printable
1584             if Value.startswith("L"):
1585                 Value = Value[2:-1]
1586             else:
1587                 Value = Value[1:-1]
1588             Printset = set(string.printable)
1589             Printset.remove(TAB_PRINTCHAR_VT)
1590             Printset.add(TAB_PRINTCHAR_BS)
1591             Printset.add(TAB_PRINTCHAR_NUL)
1592             if not set(Value).issubset(Printset):
1593                 PrintList = sorted(Printset)
1594                 return False, "Invalid PCD string value of type [%s]; must be printable chars %s." % (Type, PrintList)
1595     elif Type == 'BOOLEAN':
1596         if Value not in ['TRUE', 'True', 'true', '0x1', '0x01', '1', 'FALSE', 'False', 'false', '0x0', '0x00', '0']:
1597             return False, "Invalid value [%s] of type [%s]; must be one of TRUE, True, true, 0x1, 0x01, 1"\
1598                           ", FALSE, False, false, 0x0, 0x00, 0" % (Value, Type)
1599     elif Type in [TAB_UINT8, TAB_UINT16, TAB_UINT32, TAB_UINT64]:
1600         try:
1601             Val = int(Value, 0)
1602         except:
1603             try:
1604                 Val = int(Value.lstrip('0'))
1605             except:
1606                 return False, "Invalid value [%s] of type [%s];" \
1607                               " must be a hexadecimal, decimal or octal in C language format." % (Value, Type)
1608         if Val > MAX_VAL_TYPE[Type]:
1609             return False, "Too large PCD value[%s] for datum type [%s]" % (Value, Type)
1610         if Val < 0:
1611             return False, "PCD can't be set to negative value[%s] for datum type [%s]" % (Value, Type)
1612
1613     else:
1614         return True, "StructurePcd"
1615
1616     return True, ""
1617
1618 ## Split command line option string to list
1619 #
1620 # subprocess.Popen needs the args to be a sequence. Otherwise there's problem
1621 # in non-windows platform to launch command
1622 #
1623 def SplitOption(OptionString):
1624     OptionList = []
1625     LastChar = " "
1626     OptionStart = 0
1627     QuotationMark = ""
1628     for Index in range(0, len(OptionString)):
1629         CurrentChar = OptionString[Index]
1630         if CurrentChar in ['"', "'"]:
1631             if QuotationMark == CurrentChar:
1632                 QuotationMark = ""
1633             elif QuotationMark == "":
1634                 QuotationMark = CurrentChar
1635             continue
1636         elif QuotationMark:
1637             continue
1638
1639         if CurrentChar in ["/", "-"] and LastChar in [" ", "\t", "\r", "\n"]:
1640             if Index > OptionStart:
1641                 OptionList.append(OptionString[OptionStart:Index - 1])
1642             OptionStart = Index
1643         LastChar = CurrentChar
1644     OptionList.append(OptionString[OptionStart:])
1645     return OptionList
1646
1647 def CommonPath(PathList):
1648     P1 = min(PathList).split(os.path.sep)
1649     P2 = max(PathList).split(os.path.sep)
1650     for Index in range(min(len(P1), len(P2))):
1651         if P1[Index] != P2[Index]:
1652             return os.path.sep.join(P1[:Index])
1653     return os.path.sep.join(P1)
1654
1655 #
1656 # Convert string to C format array
1657 #
1658 def ConvertStringToByteArray(Value):
1659     Value = Value.strip()
1660     if not Value:
1661         return None
1662     if Value[0] == '{':
1663         if not Value.endswith('}'):
1664             return None
1665         Value = Value.replace(' ', '').replace('{', '').replace('}', '')
1666         ValFields = Value.split(',')
1667         try:
1668             for Index in range(len(ValFields)):
1669                 ValFields[Index] = str(int(ValFields[Index], 0))
1670         except ValueError:
1671             return None
1672         Value = '{' + ','.join(ValFields) + '}'
1673         return Value
1674
1675     Unicode = False
1676     if Value.startswith('L"'):
1677         if not Value.endswith('"'):
1678             return None
1679         Value = Value[1:]
1680         Unicode = True
1681     elif not Value.startswith('"') or not Value.endswith('"'):
1682         return None
1683
1684     Value = eval(Value)         # translate escape character
1685     NewValue = '{'
1686     for Index in range(0, len(Value)):
1687         if Unicode:
1688             NewValue = NewValue + str(ord(Value[Index]) % 0x10000) + ','
1689         else:
1690             NewValue = NewValue + str(ord(Value[Index]) % 0x100) + ','
1691     Value = NewValue + '0}'
1692     return Value
1693
1694 class PathClass(object):
1695     def __init__(self, File='', Root='', AlterRoot='', Type='', IsBinary=False,
1696                  Arch='COMMON', ToolChainFamily='', Target='', TagName='', ToolCode=''):
1697         self.Arch = Arch
1698         self.File = str(File)
1699         if os.path.isabs(self.File):
1700             self.Root = ''
1701             self.AlterRoot = ''
1702         else:
1703             self.Root = str(Root)
1704             self.AlterRoot = str(AlterRoot)
1705
1706         # Remove any '.' and '..' in path
1707         if self.Root:
1708             self.Root = mws.getWs(self.Root, self.File)
1709             self.Path = os.path.normpath(os.path.join(self.Root, self.File))
1710             self.Root = os.path.normpath(CommonPath([self.Root, self.Path]))
1711             # eliminate the side-effect of 'C:'
1712             if self.Root[-1] == ':':
1713                 self.Root += os.path.sep
1714             # file path should not start with path separator
1715             if self.Root[-1] == os.path.sep:
1716                 self.File = self.Path[len(self.Root):]
1717             else:
1718                 self.File = self.Path[len(self.Root) + 1:]
1719         else:
1720             self.Path = os.path.normpath(self.File)
1721
1722         self.SubDir, self.Name = os.path.split(self.File)
1723         self.BaseName, self.Ext = os.path.splitext(self.Name)
1724
1725         if self.Root:
1726             if self.SubDir:
1727                 self.Dir = os.path.join(self.Root, self.SubDir)
1728             else:
1729                 self.Dir = self.Root
1730         else:
1731             self.Dir = self.SubDir
1732
1733         if IsBinary:
1734             self.Type = Type
1735         else:
1736             self.Type = self.Ext.lower()
1737
1738         self.IsBinary = IsBinary
1739         self.Target = Target
1740         self.TagName = TagName
1741         self.ToolCode = ToolCode
1742         self.ToolChainFamily = ToolChainFamily
1743
1744     ## Convert the object of this class to a string
1745     #
1746     #  Convert member Path of the class to a string
1747     #
1748     #  @retval string Formatted String
1749     #
1750     def __str__(self):
1751         return self.Path
1752
1753     ## Override __eq__ function
1754     #
1755     # Check whether PathClass are the same
1756     #
1757     # @retval False The two PathClass are different
1758     # @retval True  The two PathClass are the same
1759     #
1760     def __eq__(self, Other):
1761         if isinstance(Other, type(self)):
1762             return self.Path == Other.Path
1763         else:
1764             return self.Path == str(Other)
1765
1766     ## Override __cmp__ function
1767     #
1768     # Customize the comparsion operation of two PathClass
1769     #
1770     # @retval 0     The two PathClass are different
1771     # @retval -1    The first PathClass is less than the second PathClass
1772     # @retval 1     The first PathClass is Bigger than the second PathClass
1773     def __cmp__(self, Other):
1774         if isinstance(Other, type(self)):
1775             OtherKey = Other.Path
1776         else:
1777             OtherKey = str(Other)
1778
1779         SelfKey = self.Path
1780         if SelfKey == OtherKey:
1781             return 0
1782         elif SelfKey > OtherKey:
1783             return 1
1784         else:
1785             return -1
1786
1787     ## Override __hash__ function
1788     #
1789     # Use Path as key in hash table
1790     #
1791     # @retval string Key for hash table
1792     #
1793     def __hash__(self):
1794         return hash(self.Path)
1795
1796     @cached_property
1797     def Key(self):
1798         return self.Path.upper()
1799
1800     @property
1801     def TimeStamp(self):
1802         return os.stat(self.Path)[8]
1803
1804     def Validate(self, Type='', CaseSensitive=True):
1805         if GlobalData.gCaseInsensitive:
1806             CaseSensitive = False
1807         if Type and Type.lower() != self.Type:
1808             return FILE_TYPE_MISMATCH, '%s (expect %s but got %s)' % (self.File, Type, self.Type)
1809
1810         RealFile, RealRoot = RealPath2(self.File, self.Root, self.AlterRoot)
1811         if not RealRoot and not RealFile:
1812             RealFile = self.File
1813             if self.AlterRoot:
1814                 RealFile = os.path.join(self.AlterRoot, self.File)
1815             elif self.Root:
1816                 RealFile = os.path.join(self.Root, self.File)
1817             if len (mws.getPkgPath()) == 0:
1818                 return FILE_NOT_FOUND, os.path.join(self.AlterRoot, RealFile)
1819             else:
1820                 return FILE_NOT_FOUND, "%s is not found in packages path:\n\t%s" % (self.File, '\n\t'.join(mws.getPkgPath()))
1821
1822         ErrorCode = 0
1823         ErrorInfo = ''
1824         if RealRoot != self.Root or RealFile != self.File:
1825             if CaseSensitive and (RealFile != self.File or (RealRoot != self.Root and RealRoot != self.AlterRoot)):
1826                 ErrorCode = FILE_CASE_MISMATCH
1827                 ErrorInfo = self.File + '\n\t' + RealFile + " [in file system]"
1828
1829             self.SubDir, self.Name = os.path.split(RealFile)
1830             self.BaseName, self.Ext = os.path.splitext(self.Name)
1831             if self.SubDir:
1832                 self.Dir = os.path.join(RealRoot, self.SubDir)
1833             else:
1834                 self.Dir = RealRoot
1835             self.File = RealFile
1836             self.Root = RealRoot
1837             self.Path = os.path.join(RealRoot, RealFile)
1838         return ErrorCode, ErrorInfo
1839
1840 ## Parse PE image to get the required PE informaion.
1841 #
1842 class PeImageClass():
1843     ## Constructor
1844     #
1845     #   @param  File FilePath of PeImage
1846     #
1847     def __init__(self, PeFile):
1848         self.FileName   = PeFile
1849         self.IsValid    = False
1850         self.Size       = 0
1851         self.EntryPoint = 0
1852         self.SectionAlignment  = 0
1853         self.SectionHeaderList = []
1854         self.ErrorInfo = ''
1855         try:
1856             PeObject = open(PeFile, 'rb')
1857         except:
1858             self.ErrorInfo = self.FileName + ' can not be found\n'
1859             return
1860         # Read DOS header
1861         ByteArray = array.array('B')
1862         ByteArray.fromfile(PeObject, 0x3E)
1863         ByteList = ByteArray.tolist()
1864         # DOS signature should be 'MZ'
1865         if self._ByteListToStr (ByteList[0x0:0x2]) != 'MZ':
1866             self.ErrorInfo = self.FileName + ' has no valid DOS signature MZ'
1867             return
1868
1869         # Read 4 byte PE Signature
1870         PeOffset = self._ByteListToInt(ByteList[0x3C:0x3E])
1871         PeObject.seek(PeOffset)
1872         ByteArray = array.array('B')
1873         ByteArray.fromfile(PeObject, 4)
1874         # PE signature should be 'PE\0\0'
1875         if ByteArray.tostring() != b'PE\0\0':
1876             self.ErrorInfo = self.FileName + ' has no valid PE signature PE00'
1877             return
1878
1879         # Read PE file header
1880         ByteArray = array.array('B')
1881         ByteArray.fromfile(PeObject, 0x14)
1882         ByteList = ByteArray.tolist()
1883         SecNumber = self._ByteListToInt(ByteList[0x2:0x4])
1884         if SecNumber == 0:
1885             self.ErrorInfo = self.FileName + ' has no section header'
1886             return
1887
1888         # Read PE optional header
1889         OptionalHeaderSize = self._ByteListToInt(ByteArray[0x10:0x12])
1890         ByteArray = array.array('B')
1891         ByteArray.fromfile(PeObject, OptionalHeaderSize)
1892         ByteList = ByteArray.tolist()
1893         self.EntryPoint       = self._ByteListToInt(ByteList[0x10:0x14])
1894         self.SectionAlignment = self._ByteListToInt(ByteList[0x20:0x24])
1895         self.Size             = self._ByteListToInt(ByteList[0x38:0x3C])
1896
1897         # Read each Section Header
1898         for Index in range(SecNumber):
1899             ByteArray = array.array('B')
1900             ByteArray.fromfile(PeObject, 0x28)
1901             ByteList = ByteArray.tolist()
1902             SecName  = self._ByteListToStr(ByteList[0:8])
1903             SecVirtualSize = self._ByteListToInt(ByteList[8:12])
1904             SecRawAddress  = self._ByteListToInt(ByteList[20:24])
1905             SecVirtualAddress = self._ByteListToInt(ByteList[12:16])
1906             self.SectionHeaderList.append((SecName, SecVirtualAddress, SecRawAddress, SecVirtualSize))
1907         self.IsValid = True
1908         PeObject.close()
1909
1910     def _ByteListToStr(self, ByteList):
1911         String = ''
1912         for index in range(len(ByteList)):
1913             if ByteList[index] == 0:
1914                 break
1915             String += chr(ByteList[index])
1916         return String
1917
1918     def _ByteListToInt(self, ByteList):
1919         Value = 0
1920         for index in range(len(ByteList) - 1, -1, -1):
1921             Value = (Value << 8) | int(ByteList[index])
1922         return Value
1923
1924 class DefaultStore():
1925     def __init__(self, DefaultStores ):
1926
1927         self.DefaultStores = DefaultStores
1928     def DefaultStoreID(self, DefaultStoreName):
1929         for key, value in self.DefaultStores.items():
1930             if value == DefaultStoreName:
1931                 return key
1932         return None
1933     def GetDefaultDefault(self):
1934         if not self.DefaultStores or "0" in self.DefaultStores:
1935             return "0", TAB_DEFAULT_STORES_DEFAULT
1936         else:
1937             minvalue = min(int(value_str) for value_str in self.DefaultStores)
1938             return (str(minvalue), self.DefaultStores[str(minvalue)])
1939     def GetMin(self, DefaultSIdList):
1940         if not DefaultSIdList:
1941             return TAB_DEFAULT_STORES_DEFAULT
1942         storeidset = {storeid for storeid, storename in self.DefaultStores.values() if storename in DefaultSIdList}
1943         if not storeidset:
1944             return ""
1945         minid = min(storeidset )
1946         for sid, name in self.DefaultStores.values():
1947             if sid == minid:
1948                 return name
1949
1950 class SkuClass():
1951     DEFAULT = 0
1952     SINGLE = 1
1953     MULTIPLE =2
1954
1955     def __init__(self,SkuIdentifier='', SkuIds=None):
1956         if SkuIds is None:
1957             SkuIds = {}
1958
1959         for SkuName in SkuIds:
1960             SkuId = SkuIds[SkuName][0]
1961             skuid_num = int(SkuId, 16) if SkuId.upper().startswith("0X") else int(SkuId)
1962             if skuid_num > 0xFFFFFFFFFFFFFFFF:
1963                 EdkLogger.error("build", PARAMETER_INVALID,
1964                             ExtraData = "SKU-ID [%s] value %s exceeds the max value of UINT64"
1965                                       % (SkuName, SkuId))
1966
1967         self.AvailableSkuIds = OrderedDict()
1968         self.SkuIdSet = []
1969         self.SkuIdNumberSet = []
1970         self.SkuData = SkuIds
1971         self._SkuInherit = {}
1972         self._SkuIdentifier = SkuIdentifier
1973         if SkuIdentifier == '' or SkuIdentifier is None:
1974             self.SkuIdSet = ['DEFAULT']
1975             self.SkuIdNumberSet = ['0U']
1976         elif SkuIdentifier == 'ALL':
1977             self.SkuIdSet = list(SkuIds.keys())
1978             self.SkuIdNumberSet = [num[0].strip() + 'U' for num in SkuIds.values()]
1979         else:
1980             r = SkuIdentifier.split('|')
1981             self.SkuIdSet=[(r[k].strip()).upper() for k in range(len(r))]
1982             k = None
1983             try:
1984                 self.SkuIdNumberSet = [SkuIds[k][0].strip() + 'U' for k in self.SkuIdSet]
1985             except Exception:
1986                 EdkLogger.error("build", PARAMETER_INVALID,
1987                             ExtraData = "SKU-ID [%s] is not supported by the platform. [Valid SKU-ID: %s]"
1988                                       % (k, " | ".join(SkuIds.keys())))
1989         for each in self.SkuIdSet:
1990             if each in SkuIds:
1991                 self.AvailableSkuIds[each] = SkuIds[each][0]
1992             else:
1993                 EdkLogger.error("build", PARAMETER_INVALID,
1994                             ExtraData="SKU-ID [%s] is not supported by the platform. [Valid SKU-ID: %s]"
1995                                       % (each, " | ".join(SkuIds.keys())))
1996         if self.SkuUsageType != SkuClass.SINGLE:
1997             self.AvailableSkuIds.update({'DEFAULT':0, 'COMMON':0})
1998         if self.SkuIdSet:
1999             GlobalData.gSkuids = (self.SkuIdSet)
2000             if 'COMMON' in GlobalData.gSkuids:
2001                 GlobalData.gSkuids.remove('COMMON')
2002             if self.SkuUsageType == self.SINGLE:
2003                 if len(GlobalData.gSkuids) != 1:
2004                     if 'DEFAULT' in GlobalData.gSkuids:
2005                         GlobalData.gSkuids.remove('DEFAULT')
2006             if GlobalData.gSkuids:
2007                 GlobalData.gSkuids.sort()
2008
2009     def GetNextSkuId(self, skuname):
2010         if not self._SkuInherit:
2011             self._SkuInherit = {}
2012             for item in self.SkuData.values():
2013                 self._SkuInherit[item[1]]=item[2] if item[2] else "DEFAULT"
2014         return self._SkuInherit.get(skuname, "DEFAULT")
2015
2016     def GetSkuChain(self, sku):
2017         if sku == "DEFAULT":
2018             return ["DEFAULT"]
2019         skulist = [sku]
2020         nextsku = sku
2021         while True:
2022             nextsku = self.GetNextSkuId(nextsku)
2023             skulist.append(nextsku)
2024             if nextsku == "DEFAULT":
2025                 break
2026         skulist.reverse()
2027         return skulist
2028     def SkuOverrideOrder(self):
2029         skuorderset = []
2030         for skuname in self.SkuIdSet:
2031             skuorderset.append(self.GetSkuChain(skuname))
2032
2033         skuorder = []
2034         for index in range(max(len(item) for item in skuorderset)):
2035             for subset in skuorderset:
2036                 if index > len(subset)-1:
2037                     continue
2038                 if subset[index] in skuorder:
2039                     continue
2040                 skuorder.append(subset[index])
2041
2042         return skuorder
2043
2044     @property
2045     def SkuUsageType(self):
2046         if self._SkuIdentifier.upper() == "ALL":
2047             return SkuClass.MULTIPLE
2048
2049         if len(self.SkuIdSet) == 1:
2050             if self.SkuIdSet[0] == 'DEFAULT':
2051                 return SkuClass.DEFAULT
2052             return SkuClass.SINGLE
2053         if len(self.SkuIdSet) == 2 and 'DEFAULT' in self.SkuIdSet:
2054             return SkuClass.SINGLE
2055         return SkuClass.MULTIPLE
2056
2057     def DumpSkuIdArrary(self):
2058         if self.SkuUsageType == SkuClass.SINGLE:
2059             return "{0x0}"
2060         ArrayStrList = []
2061         for skuname in self.AvailableSkuIds:
2062             if skuname == "COMMON":
2063                 continue
2064             while skuname != "DEFAULT":
2065                 ArrayStrList.append(hex(int(self.AvailableSkuIds[skuname])))
2066                 skuname = self.GetNextSkuId(skuname)
2067             ArrayStrList.append("0x0")
2068         return "{{{myList}}}".format(myList=",".join(ArrayStrList))
2069
2070     @property
2071     def AvailableSkuIdSet(self):
2072         return self.AvailableSkuIds
2073
2074     @property
2075     def SystemSkuId(self):
2076         if self.SkuUsageType == SkuClass.SINGLE:
2077             if len(self.SkuIdSet) == 1:
2078                 return self.SkuIdSet[0]
2079             else:
2080                 return self.SkuIdSet[0] if self.SkuIdSet[0] != 'DEFAULT' else self.SkuIdSet[1]
2081         else:
2082             return 'DEFAULT'
2083
2084 #
2085 # Pack a registry format GUID
2086 #
2087 def PackRegistryFormatGuid(Guid):
2088     return PackGUID(Guid.split('-'))
2089
2090 ##  Get the integer value from string like "14U" or integer like 2
2091 #
2092 #   @param      Input   The object that may be either a integer value or a string
2093 #
2094 #   @retval     Value    The integer value that the input represents
2095 #
2096 def GetIntegerValue(Input):
2097     if isinstance(Input, int):
2098         return Input
2099     String = Input
2100     if String.endswith("U"):
2101         String = String[:-1]
2102     if String.endswith("ULL"):
2103         String = String[:-3]
2104     if String.endswith("LL"):
2105         String = String[:-2]
2106
2107     if String.startswith("0x") or String.startswith("0X"):
2108         return int(String, 16)
2109     elif String == '':
2110         return 0
2111     else:
2112         return int(String)
2113
2114 #
2115 # Pack a GUID (registry format) list into a buffer and return it
2116 #
2117 def PackGUID(Guid):
2118     return pack(PACK_PATTERN_GUID,
2119                 int(Guid[0], 16),
2120                 int(Guid[1], 16),
2121                 int(Guid[2], 16),
2122                 int(Guid[3][-4:-2], 16),
2123                 int(Guid[3][-2:], 16),
2124                 int(Guid[4][-12:-10], 16),
2125                 int(Guid[4][-10:-8], 16),
2126                 int(Guid[4][-8:-6], 16),
2127                 int(Guid[4][-6:-4], 16),
2128                 int(Guid[4][-4:-2], 16),
2129                 int(Guid[4][-2:], 16)
2130                 )
2131
2132 #
2133 # Pack a GUID (byte) list into a buffer and return it
2134 #
2135 def PackByteFormatGUID(Guid):
2136     return pack(PACK_PATTERN_GUID,
2137                 Guid[0],
2138                 Guid[1],
2139                 Guid[2],
2140                 Guid[3],
2141                 Guid[4],
2142                 Guid[5],
2143                 Guid[6],
2144                 Guid[7],
2145                 Guid[8],
2146                 Guid[9],
2147                 Guid[10],
2148                 )
2149
2150 ##
2151 #
2152 # This acts like the main() function for the script, unless it is 'import'ed into another
2153 # script.
2154 #
2155 if __name__ == '__main__':
2156     pass
2157