BaseTools/Source/Python/Common/Misc.py

   1 ## @file
   2 # Common routines used by all tools
   3 #
   4 # Copyright (c) 2007 - 2018, Intel Corporation. All rights reserved.<BR>
   5 # This program and the accompanying materials
   6 # are licensed and made available under the terms and conditions of the BSD License
   7 # which accompanies this distribution.  The full text of the license may be found at
   8 # http://opensource.org/licenses/bsd-license.php
   9 #
  10 # THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS,
  11 # WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED.
  12 #
  13
  14 ##
  15 # Import Modules
  16 #
  17 import Common.LongFilePathOs as os
  18 import sys
  19 import string
  20 import threading
  21 import time
  22 import re
  23 import pickle
  24 import array
  25 import shutil
  26 from struct import pack
  27 from collections import UserDict as IterableUserDict
  28 from collections import OrderedDict
  29
  30 from Common import EdkLogger as EdkLogger
  31 from Common import GlobalData as GlobalData
  32 from .DataType import *
  33 from .BuildToolError import *
  34 from CommonDataClass.DataClass import *
  35 from .Parsing import GetSplitValueList
  36 from Common.LongFilePathSupport import OpenLongFilePath as open
  37 from Common.MultipleWorkspace import MultipleWorkspace as mws
  38 import uuid
  39 from CommonDataClass.Exceptions import BadExpression
  40 from Common.caching import cached_property
  41 import subprocess
  42 ## Regular expression used to find out place holders in string template
  43 gPlaceholderPattern = re.compile("\$\{([^$()\s]+)\}", re.MULTILINE | re.UNICODE)
  44
  45 ## regular expressions for map file processing
  46 startPatternGeneral = re.compile("^Start[' ']+Length[' ']+Name[' ']+Class")
  47 addressPatternGeneral = re.compile("^Address[' ']+Publics by Value[' ']+Rva\+Base")
  48 valuePatternGcc = re.compile('^([\w_\.]+) +([\da-fA-Fx]+) +([\da-fA-Fx]+)$')
  49 pcdPatternGcc = re.compile('^([\da-fA-Fx]+) +([\da-fA-Fx]+)')
  50 secReGeneral = re.compile('^([\da-fA-F]+):([\da-fA-F]+) +([\da-fA-F]+)[Hh]? +([.\w\$]+) +(\w+)', re.UNICODE)
  51
  52 ## Dictionary used to store file time stamp for quick re-access
  53 gFileTimeStampCache = {}    # {file path : file time stamp}
  54
  55 ## Dictionary used to store dependencies of files
  56 gDependencyDatabase = {}    # arch : {file path : [dependent files list]}
  57
  58 #
  59 # If a module is built more than once with different PCDs or library classes
  60 # a temporary INF file with same content is created, the temporary file is removed
  61 # when build exits.
  62 #
  63 _TempInfs = []
  64
  65 def GetVariableOffset(mapfilepath, efifilepath, varnames):
  66     """ Parse map file to get variable offset in current EFI file
  67     @param mapfilepath    Map file absolution path
  68     @param efifilepath:   EFI binary file full path
  69     @param varnames       iteratable container whose elements are variable names to be searched
  70
  71     @return List whos elements are tuple with variable name and raw offset
  72     """
  73     lines = []
  74     try:
  75         f = open(mapfilepath, 'r')
  76         lines = f.readlines()
  77         f.close()
  78     except:
  79         return None
  80
  81     if len(lines) == 0: return None
  82     firstline = lines[0].strip()
  83     if (firstline.startswith("Archive member included ") and
  84         firstline.endswith(" file (symbol)")):
  85         return _parseForGCC(lines, efifilepath, varnames)
  86     if firstline.startswith("# Path:"):
  87         return _parseForXcode(lines, efifilepath, varnames)
  88     return _parseGeneral(lines, efifilepath, varnames)
  89
  90 def _parseForXcode(lines, efifilepath, varnames):
  91     status = 0
  92     ret = []
  93     for line in lines:
  94         line = line.strip()
  95         if status == 0 and line == "# Symbols:":
  96             status = 1
  97             continue
  98         if status == 1 and len(line) != 0:
  99             for varname in varnames:
 100                 if varname in line:
 101                     # cannot pregenerate this RegEx since it uses varname from varnames.
 102                     m = re.match('^([\da-fA-FxX]+)([\s\S]*)([_]*%s)$' % varname, line)
 103                     if m is not None:
 104                         ret.append((varname, m.group(1)))
 105     return ret
 106
 107 def _parseForGCC(lines, efifilepath, varnames):
 108     """ Parse map file generated by GCC linker """
 109     status = 0
 110     sections = []
 111     varoffset = []
 112     for index, line in enumerate(lines):
 113         line = line.strip()
 114         # status machine transection
 115         if status == 0 and line == "Memory Configuration":
 116             status = 1
 117             continue
 118         elif status == 1 and line == 'Linker script and memory map':
 119             status = 2
 120             continue
 121         elif status ==2 and line == 'START GROUP':
 122             status = 3
 123             continue
 124
 125         # status handler
 126         if status == 3:
 127             m = valuePatternGcc.match(line)
 128             if m is not None:
 129                 sections.append(m.groups(0))
 130             for varname in varnames:
 131                 Str = ''
 132                 m = re.match("^.data.(%s)" % varname, line)
 133                 if m is not None:
 134                     m = re.match(".data.(%s)$" % varname, line)
 135                     if m is not None:
 136                         Str = lines[index + 1]
 137                     else:
 138                         Str = line[len(".data.%s" % varname):]
 139                     if Str:
 140                         m = pcdPatternGcc.match(Str.strip())
 141                         if m is not None:
 142                             varoffset.append((varname, int(m.groups(0)[0], 16), int(sections[-1][1], 16), sections[-1][0]))
 143
 144     if not varoffset:
 145         return []
 146     # get section information from efi file
 147     efisecs = PeImageClass(efifilepath).SectionHeaderList
 148     if efisecs is None or len(efisecs) == 0:
 149         return []
 150     #redirection
 151     redirection = 0
 152     for efisec in efisecs:
 153         for section in sections:
 154             if section[0].strip() == efisec[0].strip() and section[0].strip() == '.text':
 155                 redirection = int(section[1], 16) - efisec[1]
 156
 157     ret = []
 158     for var in varoffset:
 159         for efisec in efisecs:
 160             if var[1] >= efisec[1] and var[1] < efisec[1]+efisec[3]:
 161                 ret.append((var[0], hex(efisec[2] + var[1] - efisec[1] - redirection)))
 162     return ret
 163
 164 def _parseGeneral(lines, efifilepath, varnames):
 165     status = 0    #0 - beginning of file; 1 - PE section definition; 2 - symbol table
 166     secs  = []    # key = section name
 167     varoffset = []
 168     symRe = re.compile('^([\da-fA-F]+):([\da-fA-F]+) +([\.:\\\\\w\?@\$]+) +([\da-fA-F]+)', re.UNICODE)
 169
 170     for line in lines:
 171         line = line.strip()
 172         if startPatternGeneral.match(line):
 173             status = 1
 174             continue
 175         if addressPatternGeneral.match(line):
 176             status = 2
 177             continue
 178         if line.startswith("entry point at"):
 179             status = 3
 180             continue
 181         if status == 1 and len(line) != 0:
 182             m =  secReGeneral.match(line)
 183             assert m is not None, "Fail to parse the section in map file , line is %s" % line
 184             sec_no, sec_start, sec_length, sec_name, sec_class = m.groups(0)
 185             secs.append([int(sec_no, 16), int(sec_start, 16), int(sec_length, 16), sec_name, sec_class])
 186         if status == 2 and len(line) != 0:
 187             for varname in varnames:
 188                 m = symRe.match(line)
 189                 assert m is not None, "Fail to parse the symbol in map file, line is %s" % line
 190                 sec_no, sym_offset, sym_name, vir_addr = m.groups(0)
 191                 sec_no     = int(sec_no,     16)
 192                 sym_offset = int(sym_offset, 16)
 193                 vir_addr   = int(vir_addr,   16)
 194                 # cannot pregenerate this RegEx since it uses varname from varnames.
 195                 m2 = re.match('^[_]*(%s)' % varname, sym_name)
 196                 if m2 is not None:
 197                     # fond a binary pcd entry in map file
 198                     for sec in secs:
 199                         if sec[0] == sec_no and (sym_offset >= sec[1] and sym_offset < sec[1] + sec[2]):
 200                             varoffset.append([varname, sec[3], sym_offset, vir_addr, sec_no])
 201
 202     if not varoffset: return []
 203
 204     # get section information from efi file
 205     efisecs = PeImageClass(efifilepath).SectionHeaderList
 206     if efisecs is None or len(efisecs) == 0:
 207         return []
 208
 209     ret = []
 210     for var in varoffset:
 211         index = 0
 212         for efisec in efisecs:
 213             index = index + 1
 214             if var[1].strip() == efisec[0].strip():
 215                 ret.append((var[0], hex(efisec[2] + var[2])))
 216             elif var[4] == index:
 217                 ret.append((var[0], hex(efisec[2] + var[2])))
 218
 219     return ret
 220
 221 ## Routine to process duplicated INF
 222 #
 223 #  This function is called by following two cases:
 224 #  Case 1 in DSC:
 225 #    [components.arch]
 226 #    Pkg/module/module.inf
 227 #    Pkg/module/module.inf {
 228 #      <Defines>
 229 #        FILE_GUID = 0D1B936F-68F3-4589-AFCC-FB8B7AEBC836
 230 #    }
 231 #  Case 2 in FDF:
 232 #    INF Pkg/module/module.inf
 233 #    INF FILE_GUID = 0D1B936F-68F3-4589-AFCC-FB8B7AEBC836 Pkg/module/module.inf
 234 #
 235 #  This function copies Pkg/module/module.inf to
 236 #  Conf/.cache/0D1B936F-68F3-4589-AFCC-FB8B7AEBC836module.inf
 237 #
 238 #  @param Path     Original PathClass object
 239 #  @param BaseName New file base name
 240 #
 241 #  @retval         return the new PathClass object
 242 #
 243 def ProcessDuplicatedInf(Path, BaseName, Workspace):
 244     Filename = os.path.split(Path.File)[1]
 245     if '.' in Filename:
 246         Filename = BaseName + Path.BaseName + Filename[Filename.rfind('.'):]
 247     else:
 248         Filename = BaseName + Path.BaseName
 249
 250     #
 251     # If -N is specified on command line, cache is disabled
 252     # The directory has to be created
 253     #
 254     DbDir = os.path.split(GlobalData.gDatabasePath)[0]
 255     if not os.path.exists(DbDir):
 256         os.makedirs(DbDir)
 257     #
 258     # A temporary INF is copied to database path which must have write permission
 259     # The temporary will be removed at the end of build
 260     # In case of name conflict, the file name is
 261     # FILE_GUIDBaseName (0D1B936F-68F3-4589-AFCC-FB8B7AEBC836module.inf)
 262     #
 263     TempFullPath = os.path.join(DbDir,
 264                                 Filename)
 265     RtPath = PathClass(Path.File, Workspace)
 266     #
 267     # Modify the full path to temporary path, keep other unchanged
 268     #
 269     # To build same module more than once, the module path with FILE_GUID overridden has
 270     # the file name FILE_GUIDmodule.inf, but the relative path (self.MetaFile.File) is the real path
 271     # in DSC which is used as relative path by C files and other files in INF.
 272     # A trick was used: all module paths are PathClass instances, after the initialization
 273     # of PathClass, the PathClass.Path is overridden by the temporary INF path.
 274     #
 275     # The reason for creating a temporary INF is:
 276     # Platform.Modules which is the base to create ModuleAutoGen objects is a dictionary,
 277     # the key is the full path of INF, the value is an object to save overridden library instances, PCDs.
 278     # A different key for the same module is needed to create different output directory,
 279     # retrieve overridden PCDs, library instances.
 280     #
 281     # The BaseName is the FILE_GUID which is also the output directory name.
 282     #
 283     #
 284     RtPath.Path = TempFullPath
 285     RtPath.BaseName = BaseName
 286     #
 287     # If file exists, compare contents
 288     #
 289     if os.path.exists(TempFullPath):
 290         with open(str(Path), 'rb') as f1, open(TempFullPath, 'rb') as f2:
 291             if f1.read() == f2.read():
 292                 return RtPath
 293     _TempInfs.append(TempFullPath)
 294     shutil.copy2(str(Path), TempFullPath)
 295     return RtPath
 296
 297 ## Remove temporary created INFs whose paths were saved in _TempInfs
 298 #
 299 def ClearDuplicatedInf():
 300     while _TempInfs:
 301         File = _TempInfs.pop()
 302         if os.path.exists(File):
 303             os.remove(File)
 304
 305 ## Convert GUID string in xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx style to C structure style
 306 #
 307 #   @param      Guid    The GUID string
 308 #
 309 #   @retval     string  The GUID string in C structure style
 310 #
 311 def GuidStringToGuidStructureString(Guid):
 312     GuidList = Guid.split('-')
 313     Result = '{'
 314     for Index in range(0, 3, 1):
 315         Result = Result + '0x' + GuidList[Index] + ', '
 316     Result = Result + '{0x' + GuidList[3][0:2] + ', 0x' + GuidList[3][2:4]
 317     for Index in range(0, 12, 2):
 318         Result = Result + ', 0x' + GuidList[4][Index:Index + 2]
 319     Result += '}}'
 320     return Result
 321
 322 ## Convert GUID structure in byte array to xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
 323 #
 324 #   @param      GuidValue   The GUID value in byte array
 325 #
 326 #   @retval     string      The GUID value in xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx format
 327 #
 328 def GuidStructureByteArrayToGuidString(GuidValue):
 329     guidValueString = GuidValue.lower().replace("{", "").replace("}", "").replace(" ", "").replace(";", "")
 330     guidValueList = guidValueString.split(",")
 331     if len(guidValueList) != 16:
 332         return ''
 333         #EdkLogger.error(None, None, "Invalid GUID value string %s" % GuidValue)
 334     try:
 335         return "%02x%02x%02x%02x-%02x%02x-%02x%02x-%02x%02x-%02x%02x%02x%02x%02x%02x" % (
 336                 int(guidValueList[3], 16),
 337                 int(guidValueList[2], 16),
 338                 int(guidValueList[1], 16),
 339                 int(guidValueList[0], 16),
 340                 int(guidValueList[5], 16),
 341                 int(guidValueList[4], 16),
 342                 int(guidValueList[7], 16),
 343                 int(guidValueList[6], 16),
 344                 int(guidValueList[8], 16),
 345                 int(guidValueList[9], 16),
 346                 int(guidValueList[10], 16),
 347                 int(guidValueList[11], 16),
 348                 int(guidValueList[12], 16),
 349                 int(guidValueList[13], 16),
 350                 int(guidValueList[14], 16),
 351                 int(guidValueList[15], 16)
 352                 )
 353     except:
 354         return ''
 355
 356 ## Convert GUID string in C structure style to xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
 357 #
 358 #   @param      GuidValue   The GUID value in C structure format
 359 #
 360 #   @retval     string      The GUID value in xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx format
 361 #
 362 def GuidStructureStringToGuidString(GuidValue):
 363     if not GlobalData.gGuidCFormatPattern.match(GuidValue):
 364         return ''
 365     guidValueString = GuidValue.lower().replace("{", "").replace("}", "").replace(" ", "").replace(";", "")
 366     guidValueList = guidValueString.split(",")
 367     if len(guidValueList) != 11:
 368         return ''
 369         #EdkLogger.error(None, None, "Invalid GUID value string %s" % GuidValue)
 370     try:
 371         return "%08x-%04x-%04x-%02x%02x-%02x%02x%02x%02x%02x%02x" % (
 372                 int(guidValueList[0], 16),
 373                 int(guidValueList[1], 16),
 374                 int(guidValueList[2], 16),
 375                 int(guidValueList[3], 16),
 376                 int(guidValueList[4], 16),
 377                 int(guidValueList[5], 16),
 378                 int(guidValueList[6], 16),
 379                 int(guidValueList[7], 16),
 380                 int(guidValueList[8], 16),
 381                 int(guidValueList[9], 16),
 382                 int(guidValueList[10], 16)
 383                 )
 384     except:
 385         return ''
 386
 387 ## Convert GUID string in C structure style to xxxxxxxx_xxxx_xxxx_xxxx_xxxxxxxxxxxx
 388 #
 389 #   @param      GuidValue   The GUID value in C structure format
 390 #
 391 #   @retval     string      The GUID value in xxxxxxxx_xxxx_xxxx_xxxx_xxxxxxxxxxxx format
 392 #
 393 def GuidStructureStringToGuidValueName(GuidValue):
 394     guidValueString = GuidValue.lower().replace("{", "").replace("}", "").replace(" ", "")
 395     guidValueList = guidValueString.split(",")
 396     if len(guidValueList) != 11:
 397         EdkLogger.error(None, FORMAT_INVALID, "Invalid GUID value string [%s]" % GuidValue)
 398     return "%08x_%04x_%04x_%02x%02x_%02x%02x%02x%02x%02x%02x" % (
 399             int(guidValueList[0], 16),
 400             int(guidValueList[1], 16),
 401             int(guidValueList[2], 16),
 402             int(guidValueList[3], 16),
 403             int(guidValueList[4], 16),
 404             int(guidValueList[5], 16),
 405             int(guidValueList[6], 16),
 406             int(guidValueList[7], 16),
 407             int(guidValueList[8], 16),
 408             int(guidValueList[9], 16),
 409             int(guidValueList[10], 16)
 410             )
 411
 412 ## Create directories
 413 #
 414 #   @param      Directory   The directory name
 415 #
 416 def CreateDirectory(Directory):
 417     if Directory is None or Directory.strip() == "":
 418         return True
 419     try:
 420         if not os.access(Directory, os.F_OK):
 421             os.makedirs(Directory)
 422     except:
 423         return False
 424     return True
 425
 426 ## Remove directories, including files and sub-directories in it
 427 #
 428 #   @param      Directory   The directory name
 429 #
 430 def RemoveDirectory(Directory, Recursively=False):
 431     if Directory is None or Directory.strip() == "" or not os.path.exists(Directory):
 432         return
 433     if Recursively:
 434         CurrentDirectory = os.getcwd()
 435         os.chdir(Directory)
 436         for File in os.listdir("."):
 437             if os.path.isdir(File):
 438                 RemoveDirectory(File, Recursively)
 439             else:
 440                 os.remove(File)
 441         os.chdir(CurrentDirectory)
 442     os.rmdir(Directory)
 443
 444 ## Store content in file
 445 #
 446 #  This method is used to save file only when its content is changed. This is
 447 #  quite useful for "make" system to decide what will be re-built and what won't.
 448 #
 449 #   @param      File            The path of file
 450 #   @param      Content         The new content of the file
 451 #   @param      IsBinaryFile    The flag indicating if the file is binary file or not
 452 #
 453 #   @retval     True            If the file content is changed and the file is renewed
 454 #   @retval     False           If the file content is the same
 455 #
 456 def SaveFileOnChange(File, Content, IsBinaryFile=True):
 457     if os.path.exists(File):
 458         try:
 459             if isinstance(Content, bytes):
 460                 with open(File, "rb") as f:
 461                     if Content == f.read():
 462                         return False
 463             else:
 464                 with open(File, "r") as f:
 465                     if Content == f.read():
 466                         return False
 467         except:
 468             EdkLogger.error(None, FILE_OPEN_FAILURE, ExtraData=File)
 469
 470     DirName = os.path.dirname(File)
 471     if not CreateDirectory(DirName):
 472         EdkLogger.error(None, FILE_CREATE_FAILURE, "Could not create directory %s" % DirName)
 473     else:
 474         if DirName == '':
 475             DirName = os.getcwd()
 476         if not os.access(DirName, os.W_OK):
 477             EdkLogger.error(None, PERMISSION_FAILURE, "Do not have write permission on directory %s" % DirName)
 478
 479     try:
 480         if GlobalData.gIsWindows:
 481             try:
 482                 from .PyUtility import SaveFileToDisk
 483                 if not SaveFileToDisk(File, Content):
 484                     EdkLogger.error(None, FILE_CREATE_FAILURE, ExtraData=File)
 485             except:
 486                 if isinstance(Content, bytes):
 487                     with open(File, "wb") as Fd:
 488                         Fd.write(Content)
 489                 else:
 490                     with open(File, "w") as Fd:
 491                         Fd.write(Content)
 492         else:
 493             if isinstance(Content, bytes):
 494                 with open(File, "wb") as Fd:
 495                     Fd.write(Content)
 496             else:
 497                 with open(File, "w") as Fd:
 498                     Fd.write(Content)
 499     except IOError as X:
 500         EdkLogger.error(None, FILE_CREATE_FAILURE, ExtraData='IOError %s' % X)
 501
 502     return True
 503
 504 ## Make a Python object persistent on file system
 505 #
 506 #   @param      Data    The object to be stored in file
 507 #   @param      File    The path of file to store the object
 508 #
 509 def DataDump(Data, File):
 510     Fd = None
 511     try:
 512         Fd = open(File, 'wb')
 513         pickle.dump(Data, Fd, pickle.HIGHEST_PROTOCOL)
 514     except:
 515         EdkLogger.error("", FILE_OPEN_FAILURE, ExtraData=File, RaiseError=False)
 516     finally:
 517         if Fd is not None:
 518             Fd.close()
 519
 520 ## Restore a Python object from a file
 521 #
 522 #   @param      File    The path of file stored the object
 523 #
 524 #   @retval     object  A python object
 525 #   @retval     None    If failure in file operation
 526 #
 527 def DataRestore(File):
 528     Data = None
 529     Fd = None
 530     try:
 531         Fd = open(File, 'rb')
 532         Data = pickle.load(Fd)
 533     except Exception as e:
 534         EdkLogger.verbose("Failed to load [%s]\n\t%s" % (File, str(e)))
 535         Data = None
 536     finally:
 537         if Fd is not None:
 538             Fd.close()
 539     return Data
 540
 541 ## Retrieve and cache the real path name in file system
 542 #
 543 #   @param      Root    The root directory of path relative to
 544 #
 545 #   @retval     str     The path string if the path exists
 546 #   @retval     None    If path doesn't exist
 547 #
 548 class DirCache:
 549     _CACHE_ = set()
 550     _UPPER_CACHE_ = {}
 551
 552     def __init__(self, Root):
 553         self._Root = Root
 554         for F in os.listdir(Root):
 555             self._CACHE_.add(F)
 556             self._UPPER_CACHE_[F.upper()] = F
 557
 558     # =[] operator
 559     def __getitem__(self, Path):
 560         Path = Path[len(os.path.commonprefix([Path, self._Root])):]
 561         if not Path:
 562             return self._Root
 563         if Path and Path[0] == os.path.sep:
 564             Path = Path[1:]
 565         if Path in self._CACHE_:
 566             return os.path.join(self._Root, Path)
 567         UpperPath = Path.upper()
 568         if UpperPath in self._UPPER_CACHE_:
 569             return os.path.join(self._Root, self._UPPER_CACHE_[UpperPath])
 570
 571         IndexList = []
 572         LastSepIndex = -1
 573         SepIndex = Path.find(os.path.sep)
 574         while SepIndex > -1:
 575             Parent = UpperPath[:SepIndex]
 576             if Parent not in self._UPPER_CACHE_:
 577                 break
 578             LastSepIndex = SepIndex
 579             SepIndex = Path.find(os.path.sep, LastSepIndex + 1)
 580
 581         if LastSepIndex == -1:
 582             return None
 583
 584         Cwd = os.getcwd()
 585         os.chdir(self._Root)
 586         SepIndex = LastSepIndex
 587         while SepIndex > -1:
 588             Parent = Path[:SepIndex]
 589             ParentKey = UpperPath[:SepIndex]
 590             if ParentKey not in self._UPPER_CACHE_:
 591                 os.chdir(Cwd)
 592                 return None
 593
 594             if Parent in self._CACHE_:
 595                 ParentDir = Parent
 596             else:
 597                 ParentDir = self._UPPER_CACHE_[ParentKey]
 598             for F in os.listdir(ParentDir):
 599                 Dir = os.path.join(ParentDir, F)
 600                 self._CACHE_.add(Dir)
 601                 self._UPPER_CACHE_[Dir.upper()] = Dir
 602
 603             SepIndex = Path.find(os.path.sep, SepIndex + 1)
 604
 605         os.chdir(Cwd)
 606         if Path in self._CACHE_:
 607             return os.path.join(self._Root, Path)
 608         elif UpperPath in self._UPPER_CACHE_:
 609             return os.path.join(self._Root, self._UPPER_CACHE_[UpperPath])
 610         return None
 611
 612 def RealPath(File, Dir='', OverrideDir=''):
 613     NewFile = os.path.normpath(os.path.join(Dir, File))
 614     NewFile = GlobalData.gAllFiles[NewFile]
 615     if not NewFile and OverrideDir:
 616         NewFile = os.path.normpath(os.path.join(OverrideDir, File))
 617         NewFile = GlobalData.gAllFiles[NewFile]
 618     return NewFile
 619
 620 def RealPath2(File, Dir='', OverrideDir=''):
 621     NewFile = None
 622     if OverrideDir:
 623         NewFile = GlobalData.gAllFiles[os.path.normpath(os.path.join(OverrideDir, File))]
 624         if NewFile:
 625             if OverrideDir[-1] == os.path.sep:
 626                 return NewFile[len(OverrideDir):], NewFile[0:len(OverrideDir)]
 627             else:
 628                 return NewFile[len(OverrideDir) + 1:], NewFile[0:len(OverrideDir)]
 629     if GlobalData.gAllFiles:
 630         NewFile = GlobalData.gAllFiles[os.path.normpath(os.path.join(Dir, File))]
 631     if not NewFile:
 632         NewFile = os.path.normpath(os.path.join(Dir, File))
 633         if not os.path.exists(NewFile):
 634             return None, None
 635     if NewFile:
 636         if Dir:
 637             if Dir[-1] == os.path.sep:
 638                 return NewFile[len(Dir):], NewFile[0:len(Dir)]
 639             else:
 640                 return NewFile[len(Dir) + 1:], NewFile[0:len(Dir)]
 641         else:
 642             return NewFile, ''
 643
 644     return None, None
 645
 646 ## Get GUID value from given packages
 647 #
 648 #   @param      CName           The CName of the GUID
 649 #   @param      PackageList     List of packages looking-up in
 650 #   @param      Inffile         The driver file
 651 #
 652 #   @retval     GuidValue   if the CName is found in any given package
 653 #   @retval     None        if the CName is not found in all given packages
 654 #
 655 def GuidValue(CName, PackageList, Inffile = None):
 656     for P in PackageList:
 657         GuidKeys = list(P.Guids.keys())
 658         if Inffile and P._PrivateGuids:
 659             if not Inffile.startswith(P.MetaFile.Dir):
 660                 GuidKeys = [x for x in P.Guids if x not in P._PrivateGuids]
 661         if CName in GuidKeys:
 662             return P.Guids[CName]
 663     return None
 664
 665 ## Get Protocol value from given packages
 666 #
 667 #   @param      CName           The CName of the GUID
 668 #   @param      PackageList     List of packages looking-up in
 669 #   @param      Inffile         The driver file
 670 #
 671 #   @retval     GuidValue   if the CName is found in any given package
 672 #   @retval     None        if the CName is not found in all given packages
 673 #
 674 def ProtocolValue(CName, PackageList, Inffile = None):
 675     for P in PackageList:
 676         ProtocolKeys = list(P.Protocols.keys())
 677         if Inffile and P._PrivateProtocols:
 678             if not Inffile.startswith(P.MetaFile.Dir):
 679                 ProtocolKeys = [x for x in P.Protocols if x not in P._PrivateProtocols]
 680         if CName in ProtocolKeys:
 681             return P.Protocols[CName]
 682     return None
 683
 684 ## Get PPI value from given packages
 685 #
 686 #   @param      CName           The CName of the GUID
 687 #   @param      PackageList     List of packages looking-up in
 688 #   @param      Inffile         The driver file
 689 #
 690 #   @retval     GuidValue   if the CName is found in any given package
 691 #   @retval     None        if the CName is not found in all given packages
 692 #
 693 def PpiValue(CName, PackageList, Inffile = None):
 694     for P in PackageList:
 695         PpiKeys = list(P.Ppis.keys())
 696         if Inffile and P._PrivatePpis:
 697             if not Inffile.startswith(P.MetaFile.Dir):
 698                 PpiKeys = [x for x in P.Ppis if x not in P._PrivatePpis]
 699         if CName in PpiKeys:
 700             return P.Ppis[CName]
 701     return None
 702
 703 ## A string template class
 704 #
 705 #  This class implements a template for string replacement. A string template
 706 #  looks like following
 707 #
 708 #       ${BEGIN} other_string ${placeholder_name} other_string ${END}
 709 #
 710 #  The string between ${BEGIN} and ${END} will be repeated as many times as the
 711 #  length of "placeholder_name", which is a list passed through a dict. The
 712 #  "placeholder_name" is the key name of the dict. The ${BEGIN} and ${END} can
 713 #  be not used and, in this case, the "placeholder_name" must not a list and it
 714 #  will just be replaced once.
 715 #
 716 class TemplateString(object):
 717     _REPEAT_START_FLAG = "BEGIN"
 718     _REPEAT_END_FLAG = "END"
 719
 720     class Section(object):
 721         _LIST_TYPES = [type([]), type(set()), type((0,))]
 722
 723         def __init__(self, TemplateSection, PlaceHolderList):
 724             self._Template = TemplateSection
 725             self._PlaceHolderList = []
 726
 727             # Split the section into sub-sections according to the position of placeholders
 728             if PlaceHolderList:
 729                 self._SubSectionList = []
 730                 SubSectionStart = 0
 731                 #
 732                 # The placeholders passed in must be in the format of
 733                 #
 734                 #   PlaceHolderName, PlaceHolderStartPoint, PlaceHolderEndPoint
 735                 #
 736                 for PlaceHolder, Start, End in PlaceHolderList:
 737                     self._SubSectionList.append(TemplateSection[SubSectionStart:Start])
 738                     self._SubSectionList.append(TemplateSection[Start:End])
 739                     self._PlaceHolderList.append(PlaceHolder)
 740                     SubSectionStart = End
 741                 if SubSectionStart < len(TemplateSection):
 742                     self._SubSectionList.append(TemplateSection[SubSectionStart:])
 743             else:
 744                 self._SubSectionList = [TemplateSection]
 745
 746         def __str__(self):
 747             return self._Template + " : " + str(self._PlaceHolderList)
 748
 749         def Instantiate(self, PlaceHolderValues):
 750             RepeatTime = -1
 751             RepeatPlaceHolders = {}
 752             NonRepeatPlaceHolders = {}
 753
 754             for PlaceHolder in self._PlaceHolderList:
 755                 if PlaceHolder not in PlaceHolderValues:
 756                     continue
 757                 Value = PlaceHolderValues[PlaceHolder]
 758                 if type(Value) in self._LIST_TYPES:
 759                     if RepeatTime < 0:
 760                         RepeatTime = len(Value)
 761                     elif RepeatTime != len(Value):
 762                         EdkLogger.error(
 763                                     "TemplateString",
 764                                     PARAMETER_INVALID,
 765                                     "${%s} has different repeat time from others!" % PlaceHolder,
 766                                     ExtraData=str(self._Template)
 767                                     )
 768                     RepeatPlaceHolders["${%s}" % PlaceHolder] = Value
 769                 else:
 770                     NonRepeatPlaceHolders["${%s}" % PlaceHolder] = Value
 771
 772             if NonRepeatPlaceHolders:
 773                 StringList = []
 774                 for S in self._SubSectionList:
 775                     if S not in NonRepeatPlaceHolders:
 776                         StringList.append(S)
 777                     else:
 778                         StringList.append(str(NonRepeatPlaceHolders[S]))
 779             else:
 780                 StringList = self._SubSectionList
 781
 782             if RepeatPlaceHolders:
 783                 TempStringList = []
 784                 for Index in range(RepeatTime):
 785                     for S in StringList:
 786                         if S not in RepeatPlaceHolders:
 787                             TempStringList.append(S)
 788                         else:
 789                             TempStringList.append(str(RepeatPlaceHolders[S][Index]))
 790                 StringList = TempStringList
 791
 792             return "".join(StringList)
 793
 794     ## Constructor
 795     def __init__(self, Template=None):
 796         self.String = ''
 797         self.IsBinary = False
 798         self._Template = Template
 799         self._TemplateSectionList = self._Parse(Template)
 800
 801     ## str() operator
 802     #
 803     #   @retval     string  The string replaced
 804     #
 805     def __str__(self):
 806         return self.String
 807
 808     ## Split the template string into fragments per the ${BEGIN} and ${END} flags
 809     #
 810     #   @retval     list    A list of TemplateString.Section objects
 811     #
 812     def _Parse(self, Template):
 813         SectionStart = 0
 814         SearchFrom = 0
 815         MatchEnd = 0
 816         PlaceHolderList = []
 817         TemplateSectionList = []
 818         while Template:
 819             MatchObj = gPlaceholderPattern.search(Template, SearchFrom)
 820             if not MatchObj:
 821                 if MatchEnd <= len(Template):
 822                     TemplateSection = TemplateString.Section(Template[SectionStart:], PlaceHolderList)
 823                     TemplateSectionList.append(TemplateSection)
 824                 break
 825
 826             MatchString = MatchObj.group(1)
 827             MatchStart = MatchObj.start()
 828             MatchEnd = MatchObj.end()
 829
 830             if MatchString == self._REPEAT_START_FLAG:
 831                 if MatchStart > SectionStart:
 832                     TemplateSection = TemplateString.Section(Template[SectionStart:MatchStart], PlaceHolderList)
 833                     TemplateSectionList.append(TemplateSection)
 834                 SectionStart = MatchEnd
 835                 PlaceHolderList = []
 836             elif MatchString == self._REPEAT_END_FLAG:
 837                 TemplateSection = TemplateString.Section(Template[SectionStart:MatchStart], PlaceHolderList)
 838                 TemplateSectionList.append(TemplateSection)
 839                 SectionStart = MatchEnd
 840                 PlaceHolderList = []
 841             else:
 842                 PlaceHolderList.append((MatchString, MatchStart - SectionStart, MatchEnd - SectionStart))
 843             SearchFrom = MatchEnd
 844         return TemplateSectionList
 845
 846     ## Replace the string template with dictionary of placeholders and append it to previous one
 847     #
 848     #   @param      AppendString    The string template to append
 849     #   @param      Dictionary      The placeholder dictionaries
 850     #
 851     def Append(self, AppendString, Dictionary=None):
 852         if Dictionary:
 853             SectionList = self._Parse(AppendString)
 854             self.String += "".join(S.Instantiate(Dictionary) for S in SectionList)
 855         else:
 856             self.String += AppendString
 857
 858     ## Replace the string template with dictionary of placeholders
 859     #
 860     #   @param      Dictionary      The placeholder dictionaries
 861     #
 862     #   @retval     str             The string replaced with placeholder values
 863     #
 864     def Replace(self, Dictionary=None):
 865         return "".join(S.Instantiate(Dictionary) for S in self._TemplateSectionList)
 866
 867 ## Progress indicator class
 868 #
 869 #  This class makes use of thread to print progress on console.
 870 #
 871 class Progressor:
 872     # for avoiding deadloop
 873     _StopFlag = None
 874     _ProgressThread = None
 875     _CheckInterval = 0.25
 876
 877     ## Constructor
 878     #
 879     #   @param      OpenMessage     The string printed before progress charaters
 880     #   @param      CloseMessage    The string printed after progress charaters
 881     #   @param      ProgressChar    The charater used to indicate the progress
 882     #   @param      Interval        The interval in seconds between two progress charaters
 883     #
 884     def __init__(self, OpenMessage="", CloseMessage="", ProgressChar='.', Interval=1.0):
 885         self.PromptMessage = OpenMessage
 886         self.CodaMessage = CloseMessage
 887         self.ProgressChar = ProgressChar
 888         self.Interval = Interval
 889         if Progressor._StopFlag is None:
 890             Progressor._StopFlag = threading.Event()
 891
 892     ## Start to print progress charater
 893     #
 894     #   @param      OpenMessage     The string printed before progress charaters
 895     #
 896     def Start(self, OpenMessage=None):
 897         if OpenMessage is not None:
 898             self.PromptMessage = OpenMessage
 899         Progressor._StopFlag.clear()
 900         if Progressor._ProgressThread is None:
 901             Progressor._ProgressThread = threading.Thread(target=self._ProgressThreadEntry)
 902             Progressor._ProgressThread.setDaemon(False)
 903             Progressor._ProgressThread.start()
 904
 905     ## Stop printing progress charater
 906     #
 907     #   @param      CloseMessage    The string printed after progress charaters
 908     #
 909     def Stop(self, CloseMessage=None):
 910         OriginalCodaMessage = self.CodaMessage
 911         if CloseMessage is not None:
 912             self.CodaMessage = CloseMessage
 913         self.Abort()
 914         self.CodaMessage = OriginalCodaMessage
 915
 916     ## Thread entry method
 917     def _ProgressThreadEntry(self):
 918         sys.stdout.write(self.PromptMessage + " ")
 919         sys.stdout.flush()
 920         TimeUp = 0.0
 921         while not Progressor._StopFlag.isSet():
 922             if TimeUp <= 0.0:
 923                 sys.stdout.write(self.ProgressChar)
 924                 sys.stdout.flush()
 925                 TimeUp = self.Interval
 926             time.sleep(self._CheckInterval)
 927             TimeUp -= self._CheckInterval
 928         sys.stdout.write(" " + self.CodaMessage + "\n")
 929         sys.stdout.flush()
 930
 931     ## Abort the progress display
 932     @staticmethod
 933     def Abort():
 934         if Progressor._StopFlag is not None:
 935             Progressor._StopFlag.set()
 936         if Progressor._ProgressThread is not None:
 937             Progressor._ProgressThread.join()
 938             Progressor._ProgressThread = None
 939
 940 ## A dict which can access its keys and/or values orderly
 941 #
 942 #  The class implements a new kind of dict which its keys or values can be
 943 #  accessed in the order they are added into the dict. It guarantees the order
 944 #  by making use of an internal list to keep a copy of keys.
 945 #
 946 class sdict(IterableUserDict):
 947     ## Constructor
 948     def __init__(self):
 949         IterableUserDict.__init__(self)
 950         self._key_list = []
 951
 952     ## [] operator
 953     def __setitem__(self, key, value):
 954         if key not in self._key_list:
 955             self._key_list.append(key)
 956         IterableUserDict.__setitem__(self, key, value)
 957
 958     ## del operator
 959     def __delitem__(self, key):
 960         self._key_list.remove(key)
 961         IterableUserDict.__delitem__(self, key)
 962
 963     ## used in "for k in dict" loop to ensure the correct order
 964     def __iter__(self):
 965         return self.iterkeys()
 966
 967     ## len() support
 968     def __len__(self):
 969         return len(self._key_list)
 970
 971     ## "in" test support
 972     def __contains__(self, key):
 973         return key in self._key_list
 974
 975     ## indexof support
 976     def index(self, key):
 977         return self._key_list.index(key)
 978
 979     ## insert support
 980     def insert(self, key, newkey, newvalue, order):
 981         index = self._key_list.index(key)
 982         if order == 'BEFORE':
 983             self._key_list.insert(index, newkey)
 984             IterableUserDict.__setitem__(self, newkey, newvalue)
 985         elif order == 'AFTER':
 986             self._key_list.insert(index + 1, newkey)
 987             IterableUserDict.__setitem__(self, newkey, newvalue)
 988
 989     ## append support
 990     def append(self, sdict):
 991         for key in sdict:
 992             if key not in self._key_list:
 993                 self._key_list.append(key)
 994             IterableUserDict.__setitem__(self, key, sdict[key])
 995
 996     def has_key(self, key):
 997         return key in self._key_list
 998
 999     ## Empty the dict
1000     def clear(self):
1001         self._key_list = []
1002         IterableUserDict.clear(self)
1003
1004     ## Return a copy of keys
1005     def keys(self):
1006         keys = []
1007         for key in self._key_list:
1008             keys.append(key)
1009         return keys
1010
1011     ## Return a copy of values
1012     def values(self):
1013         values = []
1014         for key in self._key_list:
1015             values.append(self[key])
1016         return values
1017
1018     ## Return a copy of (key, value) list
1019     def items(self):
1020         items = []
1021         for key in self._key_list:
1022             items.append((key, self[key]))
1023         return items
1024
1025     ## Iteration support
1026     def iteritems(self):
1027         return iter(self.items())
1028
1029     ## Keys interation support
1030     def iterkeys(self):
1031         return self.keys()
1032
1033     ## Values interation support
1034     def itervalues(self):
1035         return self.values()
1036
1037     ## Return value related to a key, and remove the (key, value) from the dict
1038     def pop(self, key, *dv):
1039         value = None
1040         if key in self._key_list:
1041             value = self[key]
1042             self.__delitem__(key)
1043         elif len(dv) != 0 :
1044             value = dv[0]
1045         return value
1046
1047     ## Return (key, value) pair, and remove the (key, value) from the dict
1048     def popitem(self):
1049         key = self._key_list[-1]
1050         value = self[key]
1051         self.__delitem__(key)
1052         return key, value
1053
1054     def update(self, dict=None, **kwargs):
1055         if dict is not None:
1056             for k, v in dict.items():
1057                 self[k] = v
1058         if len(kwargs):
1059             for k, v in kwargs.items():
1060                 self[k] = v
1061
1062 ## Dictionary with restricted keys
1063 #
1064 class rdict(dict):
1065     ## Constructor
1066     def __init__(self, KeyList):
1067         for Key in KeyList:
1068             dict.__setitem__(self, Key, "")
1069
1070     ## []= operator
1071     def __setitem__(self, key, value):
1072         if key not in self:
1073             EdkLogger.error("RestrictedDict", ATTRIBUTE_SET_FAILURE, "Key [%s] is not allowed" % key,
1074                             ExtraData=", ".join(dict.keys(self)))
1075         dict.__setitem__(self, key, value)
1076
1077     ## =[] operator
1078     def __getitem__(self, key):
1079         if key not in self:
1080             return ""
1081         return dict.__getitem__(self, key)
1082
1083     ## del operator
1084     def __delitem__(self, key):
1085         EdkLogger.error("RestrictedDict", ATTRIBUTE_ACCESS_DENIED, ExtraData="del")
1086
1087     ## Empty the dict
1088     def clear(self):
1089         for Key in self:
1090             self.__setitem__(Key, "")
1091
1092     ## Return value related to a key, and remove the (key, value) from the dict
1093     def pop(self, key, *dv):
1094         EdkLogger.error("RestrictedDict", ATTRIBUTE_ACCESS_DENIED, ExtraData="pop")
1095
1096     ## Return (key, value) pair, and remove the (key, value) from the dict
1097     def popitem(self):
1098         EdkLogger.error("RestrictedDict", ATTRIBUTE_ACCESS_DENIED, ExtraData="popitem")
1099
1100 ## Dictionary using prioritized list as key
1101 #
1102 class tdict:
1103     _ListType = type([])
1104     _TupleType = type(())
1105     _Wildcard = 'COMMON'
1106     _ValidWildcardList = ['COMMON', 'DEFAULT', 'ALL', '*', 'PLATFORM']
1107
1108     def __init__(self, _Single_=False, _Level_=2):
1109         self._Level_ = _Level_
1110         self.data = {}
1111         self._Single_ = _Single_
1112
1113     # =[] operator
1114     def __getitem__(self, key):
1115         KeyType = type(key)
1116         RestKeys = None
1117         if KeyType == self._ListType or KeyType == self._TupleType:
1118             FirstKey = key[0]
1119             if len(key) > 1:
1120                 RestKeys = key[1:]
1121             elif self._Level_ > 1:
1122                 RestKeys = [self._Wildcard for i in range(0, self._Level_ - 1)]
1123         else:
1124             FirstKey = key
1125             if self._Level_ > 1:
1126                 RestKeys = [self._Wildcard for i in range(0, self._Level_ - 1)]
1127
1128         if FirstKey is None or str(FirstKey).upper() in self._ValidWildcardList:
1129             FirstKey = self._Wildcard
1130
1131         if self._Single_:
1132             return self._GetSingleValue(FirstKey, RestKeys)
1133         else:
1134             return self._GetAllValues(FirstKey, RestKeys)
1135
1136     def _GetSingleValue(self, FirstKey, RestKeys):
1137         Value = None
1138         #print "%s-%s" % (FirstKey, self._Level_) ,
1139         if self._Level_ > 1:
1140             if FirstKey == self._Wildcard:
1141                 if FirstKey in self.data:
1142                     Value = self.data[FirstKey][RestKeys]
1143                 if Value is None:
1144                     for Key in self.data:
1145                         Value = self.data[Key][RestKeys]
1146                         if Value is not None: break
1147             else:
1148                 if FirstKey in self.data:
1149                     Value = self.data[FirstKey][RestKeys]
1150                 if Value is None and self._Wildcard in self.data:
1151                     #print "Value=None"
1152                     Value = self.data[self._Wildcard][RestKeys]
1153         else:
1154             if FirstKey == self._Wildcard:
1155                 if FirstKey in self.data:
1156                     Value = self.data[FirstKey]
1157                 if Value is None:
1158                     for Key in self.data:
1159                         Value = self.data[Key]
1160                         if Value is not None: break
1161             else:
1162                 if FirstKey in self.data:
1163                     Value = self.data[FirstKey]
1164                 elif self._Wildcard in self.data:
1165                     Value = self.data[self._Wildcard]
1166         return Value
1167
1168     def _GetAllValues(self, FirstKey, RestKeys):
1169         Value = []
1170         if self._Level_ > 1:
1171             if FirstKey == self._Wildcard:
1172                 for Key in self.data:
1173                     Value += self.data[Key][RestKeys]
1174             else:
1175                 if FirstKey in self.data:
1176                     Value += self.data[FirstKey][RestKeys]
1177                 if self._Wildcard in self.data:
1178                     Value += self.data[self._Wildcard][RestKeys]
1179         else:
1180             if FirstKey == self._Wildcard:
1181                 for Key in self.data:
1182                     Value.append(self.data[Key])
1183             else:
1184                 if FirstKey in self.data:
1185                     Value.append(self.data[FirstKey])
1186                 if self._Wildcard in self.data:
1187                     Value.append(self.data[self._Wildcard])
1188         return Value
1189
1190     ## []= operator
1191     def __setitem__(self, key, value):
1192         KeyType = type(key)
1193         RestKeys = None
1194         if KeyType == self._ListType or KeyType == self._TupleType:
1195             FirstKey = key[0]
1196             if len(key) > 1:
1197                 RestKeys = key[1:]
1198             else:
1199                 RestKeys = [self._Wildcard for i in range(0, self._Level_ - 1)]
1200         else:
1201             FirstKey = key
1202             if self._Level_ > 1:
1203                 RestKeys = [self._Wildcard for i in range(0, self._Level_ - 1)]
1204
1205         if FirstKey in self._ValidWildcardList:
1206             FirstKey = self._Wildcard
1207
1208         if FirstKey not in self.data and self._Level_ > 0:
1209             self.data[FirstKey] = tdict(self._Single_, self._Level_ - 1)
1210
1211         if self._Level_ > 1:
1212             self.data[FirstKey][RestKeys] = value
1213         else:
1214             self.data[FirstKey] = value
1215
1216     def SetGreedyMode(self):
1217         self._Single_ = False
1218         if self._Level_ > 1:
1219             for Key in self.data:
1220                 self.data[Key].SetGreedyMode()
1221
1222     def SetSingleMode(self):
1223         self._Single_ = True
1224         if self._Level_ > 1:
1225             for Key in self.data:
1226                 self.data[Key].SetSingleMode()
1227
1228     def GetKeys(self, KeyIndex=0):
1229         assert KeyIndex >= 0
1230         if KeyIndex == 0:
1231             return set(self.data.keys())
1232         else:
1233             keys = set()
1234             for Key in self.data:
1235                 keys |= self.data[Key].GetKeys(KeyIndex - 1)
1236             return keys
1237
1238 def IsFieldValueAnArray (Value):
1239     Value = Value.strip()
1240     if Value.startswith(TAB_GUID) and Value.endswith(')'):
1241         return True
1242     if Value.startswith('L"') and Value.endswith('"')  and len(list(Value[2:-1])) > 1:
1243         return True
1244     if Value[0] == '"' and Value[-1] == '"' and len(list(Value[1:-1])) > 1:
1245         return True
1246     if Value[0] == '{' and Value[-1] == '}':
1247         return True
1248     if Value.startswith("L'") and Value.endswith("'") and len(list(Value[2:-1])) > 1:
1249         return True
1250     if Value[0] == "'" and Value[-1] == "'" and len(list(Value[1:-1])) > 1:
1251         return True
1252     return False
1253
1254 def AnalyzePcdExpression(Setting):
1255     Setting = Setting.strip()
1256     # There might be escaped quote in a string: \", \\\" , \', \\\'
1257     Data = Setting
1258     # There might be '|' in string and in ( ... | ... ), replace it with '-'
1259     NewStr = ''
1260     InSingleQuoteStr = False
1261     InDoubleQuoteStr = False
1262     Pair = 0
1263     for Index, ch in enumerate(Data):
1264         if ch == '"' and not InSingleQuoteStr:
1265             if Data[Index - 1] != '\\':
1266                 InDoubleQuoteStr = not InDoubleQuoteStr
1267         elif ch == "'" and not InDoubleQuoteStr:
1268             if Data[Index - 1] != '\\':
1269                 InSingleQuoteStr = not InSingleQuoteStr
1270         elif ch == '(' and not (InSingleQuoteStr or InDoubleQuoteStr):
1271             Pair += 1
1272         elif ch == ')' and not (InSingleQuoteStr or InDoubleQuoteStr):
1273             Pair -= 1
1274
1275         if (Pair > 0 or InSingleQuoteStr or InDoubleQuoteStr) and ch == TAB_VALUE_SPLIT:
1276             NewStr += '-'
1277         else:
1278             NewStr += ch
1279     FieldList = []
1280     StartPos = 0
1281     while True:
1282         Pos = NewStr.find(TAB_VALUE_SPLIT, StartPos)
1283         if Pos < 0:
1284             FieldList.append(Setting[StartPos:].strip())
1285             break
1286         FieldList.append(Setting[StartPos:Pos].strip())
1287         StartPos = Pos + 1
1288
1289     return FieldList
1290
1291 def ParseDevPathValue (Value):
1292     if '\\' in Value:
1293         Value.replace('\\', '/').replace(' ', '')
1294
1295     Cmd = 'DevicePath ' + '"' + Value + '"'
1296     try:
1297         p = subprocess.Popen(Cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
1298         out, err = p.communicate()
1299     except Exception as X:
1300         raise BadExpression("DevicePath: %s" % (str(X)) )
1301     finally:
1302         subprocess._cleanup()
1303         p.stdout.close()
1304         p.stderr.close()
1305     if err:
1306         raise BadExpression("DevicePath: %s" % str(err))
1307     Size = len(out.split())
1308     out = ','.join(out.decode(encoding='utf-8', errors='ignore').split())
1309     return '{' + out + '}', Size
1310
1311 def ParseFieldValue (Value):
1312     if isinstance(Value, type(0)):
1313         return Value, (Value.bit_length() + 7) / 8
1314     if not isinstance(Value, type('')):
1315         raise BadExpression('Type %s is %s' %(Value, type(Value)))
1316     Value = Value.strip()
1317     if Value.startswith(TAB_UINT8) and Value.endswith(')'):
1318         Value, Size = ParseFieldValue(Value.split('(', 1)[1][:-1])
1319         if Size > 1:
1320             raise BadExpression('Value (%s) Size larger than %d' %(Value, Size))
1321         return Value, 1
1322     if Value.startswith(TAB_UINT16) and Value.endswith(')'):
1323         Value, Size = ParseFieldValue(Value.split('(', 1)[1][:-1])
1324         if Size > 2:
1325             raise BadExpression('Value (%s) Size larger than %d' %(Value, Size))
1326         return Value, 2
1327     if Value.startswith(TAB_UINT32) and Value.endswith(')'):
1328         Value, Size = ParseFieldValue(Value.split('(', 1)[1][:-1])
1329         if Size > 4:
1330             raise BadExpression('Value (%s) Size larger than %d' %(Value, Size))
1331         return Value, 4
1332     if Value.startswith(TAB_UINT64) and Value.endswith(')'):
1333         Value, Size = ParseFieldValue(Value.split('(', 1)[1][:-1])
1334         if Size > 8:
1335             raise BadExpression('Value (%s) Size larger than %d' % (Value, Size))
1336         return Value, 8
1337     if Value.startswith(TAB_GUID) and Value.endswith(')'):
1338         Value = Value.split('(', 1)[1][:-1].strip()
1339         if Value[0] == '{' and Value[-1] == '}':
1340             TmpValue = GuidStructureStringToGuidString(Value)
1341             if not TmpValue:
1342                 raise BadExpression("Invalid GUID value string %s" % Value)
1343             Value = TmpValue
1344         if Value[0] == '"' and Value[-1] == '"':
1345             Value = Value[1:-1]
1346         try:
1347             Value = "{" + ','.join([str(i) for i in uuid.UUID(Value).bytes_le]) + "}"
1348         except ValueError as Message:
1349             raise BadExpression(Message)
1350         Value, Size = ParseFieldValue(Value)
1351         return Value, 16
1352     if Value.startswith('L"') and Value.endswith('"'):
1353         # Unicode String
1354         # translate escape character
1355         Value = Value[1:]
1356         try:
1357             Value = eval(Value)
1358         except:
1359             Value = Value[1:-1]
1360         List = list(Value)
1361         List.reverse()
1362         Value = 0
1363         for Char in List:
1364             Value = (Value << 16) | ord(Char)
1365         return Value, (len(List) + 1) * 2
1366     if Value.startswith('"') and Value.endswith('"'):
1367         # ASCII String
1368         # translate escape character
1369         try:
1370             Value = eval(Value)
1371         except:
1372             Value = Value[1:-1]
1373         List = list(Value)
1374         List.reverse()
1375         Value = 0
1376         for Char in List:
1377             Value = (Value << 8) | ord(Char)
1378         return Value, len(List) + 1
1379     if Value.startswith("L'") and Value.endswith("'"):
1380         # Unicode Character Constant
1381         # translate escape character
1382         Value = Value[1:]
1383         try:
1384             Value = eval(Value)
1385         except:
1386             Value = Value[1:-1]
1387         List = list(Value)
1388         if len(List) == 0:
1389             raise BadExpression('Length %s is %s' % (Value, len(List)))
1390         List.reverse()
1391         Value = 0
1392         for Char in List:
1393             Value = (Value << 16) | ord(Char)
1394         return Value, len(List) * 2
1395     if Value.startswith("'") and Value.endswith("'"):
1396         # Character constant
1397         # translate escape character
1398         try:
1399             Value = eval(Value)
1400         except:
1401             Value = Value[1:-1]
1402         List = list(Value)
1403         if len(List) == 0:
1404             raise BadExpression('Length %s is %s' % (Value, len(List)))
1405         List.reverse()
1406         Value = 0
1407         for Char in List:
1408             Value = (Value << 8) | ord(Char)
1409         return Value, len(List)
1410     if Value.startswith('{') and Value.endswith('}'):
1411         # Byte array
1412         Value = Value[1:-1]
1413         List = [Item.strip() for Item in Value.split(',')]
1414         List.reverse()
1415         Value = 0
1416         RetSize = 0
1417         for Item in List:
1418             ItemValue, Size = ParseFieldValue(Item)
1419             RetSize += Size
1420             for I in range(Size):
1421                 Value = (Value << 8) | ((ItemValue >> 8 * I) & 0xff)
1422         return Value, RetSize
1423     if Value.startswith('DEVICE_PATH(') and Value.endswith(')'):
1424         Value = Value.replace("DEVICE_PATH(", '').rstrip(')')
1425         Value = Value.strip().strip('"')
1426         return ParseDevPathValue(Value)
1427     if Value.lower().startswith('0x'):
1428         try:
1429             Value = int(Value, 16)
1430         except:
1431             raise BadExpression("invalid hex value: %s" % Value)
1432         if Value == 0:
1433             return 0, 1
1434         return Value, (Value.bit_length() + 7) / 8
1435     if Value[0].isdigit():
1436         Value = int(Value, 10)
1437         if Value == 0:
1438             return 0, 1
1439         return Value, (Value.bit_length() + 7) / 8
1440     if Value.lower() == 'true':
1441         return 1, 1
1442     if Value.lower() == 'false':
1443         return 0, 1
1444     return Value, 1
1445
1446 ## AnalyzeDscPcd
1447 #
1448 #  Analyze DSC PCD value, since there is no data type info in DSC
1449 #  This function is used to match functions (AnalyzePcdData) used for retrieving PCD value from database
1450 #  1. Feature flag: TokenSpace.PcdCName|PcdValue
1451 #  2. Fix and Patch:TokenSpace.PcdCName|PcdValue[|VOID*[|MaxSize]]
1452 #  3. Dynamic default:
1453 #     TokenSpace.PcdCName|PcdValue[|VOID*[|MaxSize]]
1454 #     TokenSpace.PcdCName|PcdValue
1455 #  4. Dynamic VPD:
1456 #     TokenSpace.PcdCName|VpdOffset[|VpdValue]
1457 #     TokenSpace.PcdCName|VpdOffset[|MaxSize[|VpdValue]]
1458 #  5. Dynamic HII:
1459 #     TokenSpace.PcdCName|HiiString|VariableGuid|VariableOffset[|HiiValue]
1460 #  PCD value needs to be located in such kind of string, and the PCD value might be an expression in which
1461 #    there might have "|" operator, also in string value.
1462 #
1463 #  @param Setting: String contain information described above with "TokenSpace.PcdCName|" stripped
1464 #  @param PcdType: PCD type: feature, fixed, dynamic default VPD HII
1465 #  @param DataType: The datum type of PCD: VOID*, UNIT, BOOL
1466 #  @retval:
1467 #    ValueList: A List contain fields described above
1468 #    IsValid:   True if conforming EBNF, otherwise False
1469 #    Index:     The index where PcdValue is in ValueList
1470 #
1471 def AnalyzeDscPcd(Setting, PcdType, DataType=''):
1472     FieldList = AnalyzePcdExpression(Setting)
1473
1474     IsValid = True
1475     if PcdType in (MODEL_PCD_FIXED_AT_BUILD, MODEL_PCD_PATCHABLE_IN_MODULE, MODEL_PCD_DYNAMIC_DEFAULT, MODEL_PCD_DYNAMIC_EX_DEFAULT):
1476         Value = FieldList[0]
1477         Size = ''
1478         if len(FieldList) > 1 and FieldList[1]:
1479             DataType = FieldList[1]
1480             if FieldList[1] != TAB_VOID:
1481                 IsValid = False
1482         if len(FieldList) > 2:
1483             Size = FieldList[2]
1484         if IsValid:
1485             if DataType == "":
1486                 IsValid = (len(FieldList) <= 1)
1487             else:
1488                 IsValid = (len(FieldList) <= 3)
1489
1490         if Size:
1491             try:
1492                 int(Size, 16) if Size.upper().startswith("0X") else int(Size)
1493             except:
1494                 IsValid = False
1495                 Size = -1
1496         return [str(Value), DataType, str(Size)], IsValid, 0
1497     elif PcdType == MODEL_PCD_FEATURE_FLAG:
1498         Value = FieldList[0]
1499         Size = ''
1500         IsValid = (len(FieldList) <= 1)
1501         return [Value, DataType, str(Size)], IsValid, 0
1502     elif PcdType in (MODEL_PCD_DYNAMIC_VPD, MODEL_PCD_DYNAMIC_EX_VPD):
1503         VpdOffset = FieldList[0]
1504         Value = Size = ''
1505         if not DataType == TAB_VOID:
1506             if len(FieldList) > 1:
1507                 Value = FieldList[1]
1508         else:
1509             if len(FieldList) > 1:
1510                 Size = FieldList[1]
1511             if len(FieldList) > 2:
1512                 Value = FieldList[2]
1513         if DataType == "":
1514             IsValid = (len(FieldList) <= 1)
1515         else:
1516             IsValid = (len(FieldList) <= 3)
1517         if Size:
1518             try:
1519                 int(Size, 16) if Size.upper().startswith("0X") else int(Size)
1520             except:
1521                 IsValid = False
1522                 Size = -1
1523         return [VpdOffset, str(Size), Value], IsValid, 2
1524     elif PcdType in (MODEL_PCD_DYNAMIC_HII, MODEL_PCD_DYNAMIC_EX_HII):
1525         IsValid = (3 <= len(FieldList) <= 5)
1526         HiiString = FieldList[0]
1527         Guid = Offset = Value = Attribute = ''
1528         if len(FieldList) > 1:
1529             Guid = FieldList[1]
1530         if len(FieldList) > 2:
1531             Offset = FieldList[2]
1532         if len(FieldList) > 3:
1533             Value = FieldList[3]
1534             if not Value:
1535                 IsValid = False
1536         if len(FieldList) > 4:
1537             Attribute = FieldList[4]
1538         return [HiiString, Guid, Offset, Value, Attribute], IsValid, 3
1539     return [], False, 0
1540
1541 ## AnalyzePcdData
1542 #
1543 #  Analyze the pcd Value, Datum type and TokenNumber.
1544 #  Used to avoid split issue while the value string contain "|" character
1545 #
1546 #  @param[in] Setting:  A String contain value/datum type/token number information;
1547 #
1548 #  @retval   ValueList: A List contain value, datum type and toke number.
1549 #
1550 def AnalyzePcdData(Setting):
1551     ValueList = ['', '', '']
1552
1553     ValueRe = re.compile(r'^\s*L?\".*\|.*\"')
1554     PtrValue = ValueRe.findall(Setting)
1555
1556     ValueUpdateFlag = False
1557
1558     if len(PtrValue) >= 1:
1559         Setting = re.sub(ValueRe, '', Setting)
1560         ValueUpdateFlag = True
1561
1562     TokenList = Setting.split(TAB_VALUE_SPLIT)
1563     ValueList[0:len(TokenList)] = TokenList
1564
1565     if ValueUpdateFlag:
1566         ValueList[0] = PtrValue[0]
1567
1568     return ValueList
1569
1570 ## check format of PCD value against its the datum type
1571 #
1572 # For PCD value setting
1573 #
1574 def CheckPcdDatum(Type, Value):
1575     if Type == TAB_VOID:
1576         ValueRe = re.compile(r'\s*L?\".*\"\s*$')
1577         if not (((Value.startswith('L"') or Value.startswith('"')) and Value.endswith('"'))
1578                 or (Value.startswith('{') and Value.endswith('}')) or (Value.startswith("L'") or Value.startswith("'") and Value.endswith("'"))
1579                ):
1580             return False, "Invalid value [%s] of type [%s]; must be in the form of {...} for array"\
1581                           ", \"...\" or \'...\' for string, L\"...\" or L\'...\' for unicode string" % (Value, Type)
1582         elif ValueRe.match(Value):
1583             # Check the chars in UnicodeString or CString is printable
1584             if Value.startswith("L"):
1585                 Value = Value[2:-1]
1586             else:
1587                 Value = Value[1:-1]
1588             Printset = set(string.printable)
1589             Printset.remove(TAB_PRINTCHAR_VT)
1590             Printset.add(TAB_PRINTCHAR_BS)
1591             Printset.add(TAB_PRINTCHAR_NUL)
1592             if not set(Value).issubset(Printset):
1593                 PrintList = sorted(Printset)
1594                 return False, "Invalid PCD string value of type [%s]; must be printable chars %s." % (Type, PrintList)
1595     elif Type == 'BOOLEAN':
1596         if Value not in ['TRUE', 'True', 'true', '0x1', '0x01', '1', 'FALSE', 'False', 'false', '0x0', '0x00', '0']:
1597             return False, "Invalid value [%s] of type [%s]; must be one of TRUE, True, true, 0x1, 0x01, 1"\
1598                           ", FALSE, False, false, 0x0, 0x00, 0" % (Value, Type)
1599     elif Type in [TAB_UINT8, TAB_UINT16, TAB_UINT32, TAB_UINT64]:
1600         if Value and int(Value, 0) < 0:
1601             return False, "PCD can't be set to negative value[%s] for datum type [%s]" % (Value, Type)
1602         try:
1603             Value = int(Value, 0)
1604             if Value > MAX_VAL_TYPE[Type]:
1605                 return False, "Too large PCD value[%s] for datum type [%s]" % (Value, Type)
1606         except:
1607             return False, "Invalid value [%s] of type [%s];"\
1608                           " must be a hexadecimal, decimal or octal in C language format." % (Value, Type)
1609     else:
1610         return True, "StructurePcd"
1611
1612     return True, ""
1613
1614 ## Split command line option string to list
1615 #
1616 # subprocess.Popen needs the args to be a sequence. Otherwise there's problem
1617 # in non-windows platform to launch command
1618 #
1619 def SplitOption(OptionString):
1620     OptionList = []
1621     LastChar = " "
1622     OptionStart = 0
1623     QuotationMark = ""
1624     for Index in range(0, len(OptionString)):
1625         CurrentChar = OptionString[Index]
1626         if CurrentChar in ['"', "'"]:
1627             if QuotationMark == CurrentChar:
1628                 QuotationMark = ""
1629             elif QuotationMark == "":
1630                 QuotationMark = CurrentChar
1631             continue
1632         elif QuotationMark:
1633             continue
1634
1635         if CurrentChar in ["/", "-"] and LastChar in [" ", "\t", "\r", "\n"]:
1636             if Index > OptionStart:
1637                 OptionList.append(OptionString[OptionStart:Index - 1])
1638             OptionStart = Index
1639         LastChar = CurrentChar
1640     OptionList.append(OptionString[OptionStart:])
1641     return OptionList
1642
1643 def CommonPath(PathList):
1644     P1 = min(PathList).split(os.path.sep)
1645     P2 = max(PathList).split(os.path.sep)
1646     for Index in range(min(len(P1), len(P2))):
1647         if P1[Index] != P2[Index]:
1648             return os.path.sep.join(P1[:Index])
1649     return os.path.sep.join(P1)
1650
1651 #
1652 # Convert string to C format array
1653 #
1654 def ConvertStringToByteArray(Value):
1655     Value = Value.strip()
1656     if not Value:
1657         return None
1658     if Value[0] == '{':
1659         if not Value.endswith('}'):
1660             return None
1661         Value = Value.replace(' ', '').replace('{', '').replace('}', '')
1662         ValFields = Value.split(',')
1663         try:
1664             for Index in range(len(ValFields)):
1665                 ValFields[Index] = str(int(ValFields[Index], 0))
1666         except ValueError:
1667             return None
1668         Value = '{' + ','.join(ValFields) + '}'
1669         return Value
1670
1671     Unicode = False
1672     if Value.startswith('L"'):
1673         if not Value.endswith('"'):
1674             return None
1675         Value = Value[1:]
1676         Unicode = True
1677     elif not Value.startswith('"') or not Value.endswith('"'):
1678         return None
1679
1680     Value = eval(Value)         # translate escape character
1681     NewValue = '{'
1682     for Index in range(0, len(Value)):
1683         if Unicode:
1684             NewValue = NewValue + str(ord(Value[Index]) % 0x10000) + ','
1685         else:
1686             NewValue = NewValue + str(ord(Value[Index]) % 0x100) + ','
1687     Value = NewValue + '0}'
1688     return Value
1689
1690 class PathClass(object):
1691     def __init__(self, File='', Root='', AlterRoot='', Type='', IsBinary=False,
1692                  Arch='COMMON', ToolChainFamily='', Target='', TagName='', ToolCode=''):
1693         self.Arch = Arch
1694         self.File = str(File)
1695         if os.path.isabs(self.File):
1696             self.Root = ''
1697             self.AlterRoot = ''
1698         else:
1699             self.Root = str(Root)
1700             self.AlterRoot = str(AlterRoot)
1701
1702         # Remove any '.' and '..' in path
1703         if self.Root:
1704             self.Root = mws.getWs(self.Root, self.File)
1705             self.Path = os.path.normpath(os.path.join(self.Root, self.File))
1706             self.Root = os.path.normpath(CommonPath([self.Root, self.Path]))
1707             # eliminate the side-effect of 'C:'
1708             if self.Root[-1] == ':':
1709                 self.Root += os.path.sep
1710             # file path should not start with path separator
1711             if self.Root[-1] == os.path.sep:
1712                 self.File = self.Path[len(self.Root):]
1713             else:
1714                 self.File = self.Path[len(self.Root) + 1:]
1715         else:
1716             self.Path = os.path.normpath(self.File)
1717
1718         self.SubDir, self.Name = os.path.split(self.File)
1719         self.BaseName, self.Ext = os.path.splitext(self.Name)
1720
1721         if self.Root:
1722             if self.SubDir:
1723                 self.Dir = os.path.join(self.Root, self.SubDir)
1724             else:
1725                 self.Dir = self.Root
1726         else:
1727             self.Dir = self.SubDir
1728
1729         if IsBinary:
1730             self.Type = Type
1731         else:
1732             self.Type = self.Ext.lower()
1733
1734         self.IsBinary = IsBinary
1735         self.Target = Target
1736         self.TagName = TagName
1737         self.ToolCode = ToolCode
1738         self.ToolChainFamily = ToolChainFamily
1739
1740     ## Convert the object of this class to a string
1741     #
1742     #  Convert member Path of the class to a string
1743     #
1744     #  @retval string Formatted String
1745     #
1746     def __str__(self):
1747         return self.Path
1748
1749     ## Override __eq__ function
1750     #
1751     # Check whether PathClass are the same
1752     #
1753     # @retval False The two PathClass are different
1754     # @retval True  The two PathClass are the same
1755     #
1756     def __eq__(self, Other):
1757         if isinstance(Other, type(self)):
1758             return self.Path == Other.Path
1759         else:
1760             return self.Path == str(Other)
1761
1762     ## Override __cmp__ function
1763     #
1764     # Customize the comparsion operation of two PathClass
1765     #
1766     # @retval 0     The two PathClass are different
1767     # @retval -1    The first PathClass is less than the second PathClass
1768     # @retval 1     The first PathClass is Bigger than the second PathClass
1769     def __cmp__(self, Other):
1770         if isinstance(Other, type(self)):
1771             OtherKey = Other.Path
1772         else:
1773             OtherKey = str(Other)
1774
1775         SelfKey = self.Path
1776         if SelfKey == OtherKey:
1777             return 0
1778         elif SelfKey > OtherKey:
1779             return 1
1780         else:
1781             return -1
1782
1783     ## Override __hash__ function
1784     #
1785     # Use Path as key in hash table
1786     #
1787     # @retval string Key for hash table
1788     #
1789     def __hash__(self):
1790         return hash(self.Path)
1791
1792     @cached_property
1793     def Key(self):
1794         return self.Path.upper()
1795
1796     @property
1797     def TimeStamp(self):
1798         return os.stat(self.Path)[8]
1799
1800     def Validate(self, Type='', CaseSensitive=True):
1801         if GlobalData.gCaseInsensitive:
1802             CaseSensitive = False
1803         if Type and Type.lower() != self.Type:
1804             return FILE_TYPE_MISMATCH, '%s (expect %s but got %s)' % (self.File, Type, self.Type)
1805
1806         RealFile, RealRoot = RealPath2(self.File, self.Root, self.AlterRoot)
1807         if not RealRoot and not RealFile:
1808             RealFile = self.File
1809             if self.AlterRoot:
1810                 RealFile = os.path.join(self.AlterRoot, self.File)
1811             elif self.Root:
1812                 RealFile = os.path.join(self.Root, self.File)
1813             if len (mws.getPkgPath()) == 0:
1814                 return FILE_NOT_FOUND, os.path.join(self.AlterRoot, RealFile)
1815             else:
1816                 return FILE_NOT_FOUND, "%s is not found in packages path:\n\t%s" % (self.File, '\n\t'.join(mws.getPkgPath()))
1817
1818         ErrorCode = 0
1819         ErrorInfo = ''
1820         if RealRoot != self.Root or RealFile != self.File:
1821             if CaseSensitive and (RealFile != self.File or (RealRoot != self.Root and RealRoot != self.AlterRoot)):
1822                 ErrorCode = FILE_CASE_MISMATCH
1823                 ErrorInfo = self.File + '\n\t' + RealFile + " [in file system]"
1824
1825             self.SubDir, self.Name = os.path.split(RealFile)
1826             self.BaseName, self.Ext = os.path.splitext(self.Name)
1827             if self.SubDir:
1828                 self.Dir = os.path.join(RealRoot, self.SubDir)
1829             else:
1830                 self.Dir = RealRoot
1831             self.File = RealFile
1832             self.Root = RealRoot
1833             self.Path = os.path.join(RealRoot, RealFile)
1834         return ErrorCode, ErrorInfo
1835
1836 ## Parse PE image to get the required PE informaion.
1837 #
1838 class PeImageClass():
1839     ## Constructor
1840     #
1841     #   @param  File FilePath of PeImage
1842     #
1843     def __init__(self, PeFile):
1844         self.FileName   = PeFile
1845         self.IsValid    = False
1846         self.Size       = 0
1847         self.EntryPoint = 0
1848         self.SectionAlignment  = 0
1849         self.SectionHeaderList = []
1850         self.ErrorInfo = ''
1851         try:
1852             PeObject = open(PeFile, 'rb')
1853         except:
1854             self.ErrorInfo = self.FileName + ' can not be found\n'
1855             return
1856         # Read DOS header
1857         ByteArray = array.array('B')
1858         ByteArray.fromfile(PeObject, 0x3E)
1859         ByteList = ByteArray.tolist()
1860         # DOS signature should be 'MZ'
1861         if self._ByteListToStr (ByteList[0x0:0x2]) != 'MZ':
1862             self.ErrorInfo = self.FileName + ' has no valid DOS signature MZ'
1863             return
1864
1865         # Read 4 byte PE Signature
1866         PeOffset = self._ByteListToInt(ByteList[0x3C:0x3E])
1867         PeObject.seek(PeOffset)
1868         ByteArray = array.array('B')
1869         ByteArray.fromfile(PeObject, 4)
1870         # PE signature should be 'PE\0\0'
1871         if ByteArray.tostring() != b'PE\0\0':
1872             self.ErrorInfo = self.FileName + ' has no valid PE signature PE00'
1873             return
1874
1875         # Read PE file header
1876         ByteArray = array.array('B')
1877         ByteArray.fromfile(PeObject, 0x14)
1878         ByteList = ByteArray.tolist()
1879         SecNumber = self._ByteListToInt(ByteList[0x2:0x4])
1880         if SecNumber == 0:
1881             self.ErrorInfo = self.FileName + ' has no section header'
1882             return
1883
1884         # Read PE optional header
1885         OptionalHeaderSize = self._ByteListToInt(ByteArray[0x10:0x12])
1886         ByteArray = array.array('B')
1887         ByteArray.fromfile(PeObject, OptionalHeaderSize)
1888         ByteList = ByteArray.tolist()
1889         self.EntryPoint       = self._ByteListToInt(ByteList[0x10:0x14])
1890         self.SectionAlignment = self._ByteListToInt(ByteList[0x20:0x24])
1891         self.Size             = self._ByteListToInt(ByteList[0x38:0x3C])
1892
1893         # Read each Section Header
1894         for Index in range(SecNumber):
1895             ByteArray = array.array('B')
1896             ByteArray.fromfile(PeObject, 0x28)
1897             ByteList = ByteArray.tolist()
1898             SecName  = self._ByteListToStr(ByteList[0:8])
1899             SecVirtualSize = self._ByteListToInt(ByteList[8:12])
1900             SecRawAddress  = self._ByteListToInt(ByteList[20:24])
1901             SecVirtualAddress = self._ByteListToInt(ByteList[12:16])
1902             self.SectionHeaderList.append((SecName, SecVirtualAddress, SecRawAddress, SecVirtualSize))
1903         self.IsValid = True
1904         PeObject.close()
1905
1906     def _ByteListToStr(self, ByteList):
1907         String = ''
1908         for index in range(len(ByteList)):
1909             if ByteList[index] == 0:
1910                 break
1911             String += chr(ByteList[index])
1912         return String
1913
1914     def _ByteListToInt(self, ByteList):
1915         Value = 0
1916         for index in range(len(ByteList) - 1, -1, -1):
1917             Value = (Value << 8) | int(ByteList[index])
1918         return Value
1919
1920 class DefaultStore():
1921     def __init__(self, DefaultStores ):
1922
1923         self.DefaultStores = DefaultStores
1924     def DefaultStoreID(self, DefaultStoreName):
1925         for key, value in self.DefaultStores.items():
1926             if value == DefaultStoreName:
1927                 return key
1928         return None
1929     def GetDefaultDefault(self):
1930         if not self.DefaultStores or "0" in self.DefaultStores:
1931             return "0", TAB_DEFAULT_STORES_DEFAULT
1932         else:
1933             minvalue = min(int(value_str) for value_str in self.DefaultStores)
1934             return (str(minvalue), self.DefaultStores[str(minvalue)])
1935     def GetMin(self, DefaultSIdList):
1936         if not DefaultSIdList:
1937             return TAB_DEFAULT_STORES_DEFAULT
1938         storeidset = {storeid for storeid, storename in self.DefaultStores.values() if storename in DefaultSIdList}
1939         if not storeidset:
1940             return ""
1941         minid = min(storeidset )
1942         for sid, name in self.DefaultStores.values():
1943             if sid == minid:
1944                 return name
1945
1946 class SkuClass():
1947     DEFAULT = 0
1948     SINGLE = 1
1949     MULTIPLE =2
1950
1951     def __init__(self,SkuIdentifier='', SkuIds=None):
1952         if SkuIds is None:
1953             SkuIds = {}
1954
1955         for SkuName in SkuIds:
1956             SkuId = SkuIds[SkuName][0]
1957             skuid_num = int(SkuId, 16) if SkuId.upper().startswith("0X") else int(SkuId)
1958             if skuid_num > 0xFFFFFFFFFFFFFFFF:
1959                 EdkLogger.error("build", PARAMETER_INVALID,
1960                             ExtraData = "SKU-ID [%s] value %s exceeds the max value of UINT64"
1961                                       % (SkuName, SkuId))
1962
1963         self.AvailableSkuIds = OrderedDict()
1964         self.SkuIdSet = []
1965         self.SkuIdNumberSet = []
1966         self.SkuData = SkuIds
1967         self._SkuInherit = {}
1968         self._SkuIdentifier = SkuIdentifier
1969         if SkuIdentifier == '' or SkuIdentifier is None:
1970             self.SkuIdSet = ['DEFAULT']
1971             self.SkuIdNumberSet = ['0U']
1972         elif SkuIdentifier == 'ALL':
1973             self.SkuIdSet = list(SkuIds.keys())
1974             self.SkuIdNumberSet = [num[0].strip() + 'U' for num in SkuIds.values()]
1975         else:
1976             r = SkuIdentifier.split('|')
1977             self.SkuIdSet=[(r[k].strip()).upper() for k in range(len(r))]
1978             k = None
1979             try:
1980                 self.SkuIdNumberSet = [SkuIds[k][0].strip() + 'U' for k in self.SkuIdSet]
1981             except Exception:
1982                 EdkLogger.error("build", PARAMETER_INVALID,
1983                             ExtraData = "SKU-ID [%s] is not supported by the platform. [Valid SKU-ID: %s]"
1984                                       % (k, " | ".join(SkuIds.keys())))
1985         for each in self.SkuIdSet:
1986             if each in SkuIds:
1987                 self.AvailableSkuIds[each] = SkuIds[each][0]
1988             else:
1989                 EdkLogger.error("build", PARAMETER_INVALID,
1990                             ExtraData="SKU-ID [%s] is not supported by the platform. [Valid SKU-ID: %s]"
1991                                       % (each, " | ".join(SkuIds.keys())))
1992         if self.SkuUsageType != SkuClass.SINGLE:
1993             self.AvailableSkuIds.update({'DEFAULT':0, 'COMMON':0})
1994         if self.SkuIdSet:
1995             GlobalData.gSkuids = (self.SkuIdSet)
1996             if 'COMMON' in GlobalData.gSkuids:
1997                 GlobalData.gSkuids.remove('COMMON')
1998             if self.SkuUsageType == self.SINGLE:
1999                 if len(GlobalData.gSkuids) != 1:
2000                     if 'DEFAULT' in GlobalData.gSkuids:
2001                         GlobalData.gSkuids.remove('DEFAULT')
2002             if GlobalData.gSkuids:
2003                 GlobalData.gSkuids.sort()
2004
2005     def GetNextSkuId(self, skuname):
2006         if not self._SkuInherit:
2007             self._SkuInherit = {}
2008             for item in self.SkuData.values():
2009                 self._SkuInherit[item[1]]=item[2] if item[2] else "DEFAULT"
2010         return self._SkuInherit.get(skuname, "DEFAULT")
2011
2012     def GetSkuChain(self, sku):
2013         if sku == "DEFAULT":
2014             return ["DEFAULT"]
2015         skulist = [sku]
2016         nextsku = sku
2017         while True:
2018             nextsku = self.GetNextSkuId(nextsku)
2019             skulist.append(nextsku)
2020             if nextsku == "DEFAULT":
2021                 break
2022         skulist.reverse()
2023         return skulist
2024     def SkuOverrideOrder(self):
2025         skuorderset = []
2026         for skuname in self.SkuIdSet:
2027             skuorderset.append(self.GetSkuChain(skuname))
2028
2029         skuorder = []
2030         for index in range(max(len(item) for item in skuorderset)):
2031             for subset in skuorderset:
2032                 if index > len(subset)-1:
2033                     continue
2034                 if subset[index] in skuorder:
2035                     continue
2036                 skuorder.append(subset[index])
2037
2038         return skuorder
2039
2040     @property
2041     def SkuUsageType(self):
2042         if self._SkuIdentifier.upper() == "ALL":
2043             return SkuClass.MULTIPLE
2044
2045         if len(self.SkuIdSet) == 1:
2046             if self.SkuIdSet[0] == 'DEFAULT':
2047                 return SkuClass.DEFAULT
2048             return SkuClass.SINGLE
2049         if len(self.SkuIdSet) == 2 and 'DEFAULT' in self.SkuIdSet:
2050             return SkuClass.SINGLE
2051         return SkuClass.MULTIPLE
2052
2053     def DumpSkuIdArrary(self):
2054         if self.SkuUsageType == SkuClass.SINGLE:
2055             return "{0x0}"
2056         ArrayStrList = []
2057         for skuname in self.AvailableSkuIds:
2058             if skuname == "COMMON":
2059                 continue
2060             while skuname != "DEFAULT":
2061                 ArrayStrList.append(hex(int(self.AvailableSkuIds[skuname])))
2062                 skuname = self.GetNextSkuId(skuname)
2063             ArrayStrList.append("0x0")
2064         return "{{{myList}}}".format(myList=",".join(ArrayStrList))
2065
2066     @property
2067     def AvailableSkuIdSet(self):
2068         return self.AvailableSkuIds
2069
2070     @property
2071     def SystemSkuId(self):
2072         if self.SkuUsageType == SkuClass.SINGLE:
2073             if len(self.SkuIdSet) == 1:
2074                 return self.SkuIdSet[0]
2075             else:
2076                 return self.SkuIdSet[0] if self.SkuIdSet[0] != 'DEFAULT' else self.SkuIdSet[1]
2077         else:
2078             return 'DEFAULT'
2079
2080 #
2081 # Pack a registry format GUID
2082 #
2083 def PackRegistryFormatGuid(Guid):
2084     return PackGUID(Guid.split('-'))
2085
2086 ##  Get the integer value from string like "14U" or integer like 2
2087 #
2088 #   @param      Input   The object that may be either a integer value or a string
2089 #
2090 #   @retval     Value    The integer value that the input represents
2091 #
2092 def GetIntegerValue(Input):
2093     if isinstance(Input, int):
2094         return Input
2095     String = Input
2096     if String.endswith("U"):
2097         String = String[:-1]
2098     if String.endswith("ULL"):
2099         String = String[:-3]
2100     if String.endswith("LL"):
2101         String = String[:-2]
2102
2103     if String.startswith("0x") or String.startswith("0X"):
2104         return int(String, 16)
2105     elif String == '':
2106         return 0
2107     else:
2108         return int(String)
2109
2110 #
2111 # Pack a GUID (registry format) list into a buffer and return it
2112 #
2113 def PackGUID(Guid):
2114     return pack(PACK_PATTERN_GUID,
2115                 int(Guid[0], 16),
2116                 int(Guid[1], 16),
2117                 int(Guid[2], 16),
2118                 int(Guid[3][-4:-2], 16),
2119                 int(Guid[3][-2:], 16),
2120                 int(Guid[4][-12:-10], 16),
2121                 int(Guid[4][-10:-8], 16),
2122                 int(Guid[4][-8:-6], 16),
2123                 int(Guid[4][-6:-4], 16),
2124                 int(Guid[4][-4:-2], 16),
2125                 int(Guid[4][-2:], 16)
2126                 )
2127
2128 #
2129 # Pack a GUID (byte) list into a buffer and return it
2130 #
2131 def PackByteFormatGUID(Guid):
2132     return pack(PACK_PATTERN_GUID,
2133                 Guid[0],
2134                 Guid[1],
2135                 Guid[2],
2136                 Guid[3],
2137                 Guid[4],
2138                 Guid[5],
2139                 Guid[6],
2140                 Guid[7],
2141                 Guid[8],
2142                 Guid[9],
2143                 Guid[10],
2144                 )
2145
2146 ##
2147 #
2148 # This acts like the main() function for the script, unless it is 'import'ed into another
2149 # script.
2150 #
2151 if __name__ == '__main__':
2152     pass
2153