BaseTools/Source/Python/Common/Misc.py

   1 ## @file
   2 # Common routines used by all tools
   3 #
   4 # Copyright (c) 2007 - 2018, Intel Corporation. All rights reserved.<BR>
   5 # This program and the accompanying materials
   6 # are licensed and made available under the terms and conditions of the BSD License
   7 # which accompanies this distribution.  The full text of the license may be found at
   8 # http://opensource.org/licenses/bsd-license.php
   9 #
  10 # THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS,
  11 # WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED.
  12 #
  13
  14 ##
  15 # Import Modules
  16 #
  17 import Common.LongFilePathOs as os
  18 import sys
  19 import string
  20 import threading
  21 import time
  22 import re
  23 import pickle
  24 import array
  25 import shutil
  26 from struct import pack
  27 from collections import UserDict as IterableUserDict
  28 from collections import OrderedDict
  29
  30 from Common import EdkLogger as EdkLogger
  31 from Common import GlobalData as GlobalData
  32 from .DataType import *
  33 from .BuildToolError import *
  34 from CommonDataClass.DataClass import *
  35 from .Parsing import GetSplitValueList
  36 from Common.LongFilePathSupport import OpenLongFilePath as open
  37 from Common.MultipleWorkspace import MultipleWorkspace as mws
  38 import uuid
  39 from CommonDataClass.Exceptions import BadExpression
  40 from Common.caching import cached_property
  41 import subprocess
  42 ## Regular expression used to find out place holders in string template
  43 gPlaceholderPattern = re.compile("\$\{([^$()\s]+)\}", re.MULTILINE | re.UNICODE)
  44
  45 ## regular expressions for map file processing
  46 startPatternGeneral = re.compile("^Start[' ']+Length[' ']+Name[' ']+Class")
  47 addressPatternGeneral = re.compile("^Address[' ']+Publics by Value[' ']+Rva\+Base")
  48 valuePatternGcc = re.compile('^([\w_\.]+) +([\da-fA-Fx]+) +([\da-fA-Fx]+)$')
  49 pcdPatternGcc = re.compile('^([\da-fA-Fx]+) +([\da-fA-Fx]+)')
  50 secReGeneral = re.compile('^([\da-fA-F]+):([\da-fA-F]+) +([\da-fA-F]+)[Hh]? +([.\w\$]+) +(\w+)', re.UNICODE)
  51
  52 ## Dictionary used to store file time stamp for quick re-access
  53 gFileTimeStampCache = {}    # {file path : file time stamp}
  54
  55 ## Dictionary used to store dependencies of files
  56 gDependencyDatabase = {}    # arch : {file path : [dependent files list]}
  57
  58 #
  59 # If a module is built more than once with different PCDs or library classes
  60 # a temporary INF file with same content is created, the temporary file is removed
  61 # when build exits.
  62 #
  63 _TempInfs = []
  64
  65 def GetVariableOffset(mapfilepath, efifilepath, varnames):
  66     """ Parse map file to get variable offset in current EFI file
  67     @param mapfilepath    Map file absolution path
  68     @param efifilepath:   EFI binary file full path
  69     @param varnames       iteratable container whose elements are variable names to be searched
  70
  71     @return List whos elements are tuple with variable name and raw offset
  72     """
  73     lines = []
  74     try:
  75         f = open(mapfilepath, 'r')
  76         lines = f.readlines()
  77         f.close()
  78     except:
  79         return None
  80
  81     if len(lines) == 0: return None
  82     firstline = lines[0].strip()
  83     if (firstline.startswith("Archive member included ") and
  84         firstline.endswith(" file (symbol)")):
  85         return _parseForGCC(lines, efifilepath, varnames)
  86     if firstline.startswith("# Path:"):
  87         return _parseForXcode(lines, efifilepath, varnames)
  88     return _parseGeneral(lines, efifilepath, varnames)
  89
  90 def _parseForXcode(lines, efifilepath, varnames):
  91     status = 0
  92     ret = []
  93     for line in lines:
  94         line = line.strip()
  95         if status == 0 and line == "# Symbols:":
  96             status = 1
  97             continue
  98         if status == 1 and len(line) != 0:
  99             for varname in varnames:
 100                 if varname in line:
 101                     # cannot pregenerate this RegEx since it uses varname from varnames.
 102                     m = re.match('^([\da-fA-FxX]+)([\s\S]*)([_]*%s)$' % varname, line)
 103                     if m is not None:
 104                         ret.append((varname, m.group(1)))
 105     return ret
 106
 107 def _parseForGCC(lines, efifilepath, varnames):
 108     """ Parse map file generated by GCC linker """
 109     status = 0
 110     sections = []
 111     varoffset = []
 112     for index, line in enumerate(lines):
 113         line = line.strip()
 114         # status machine transection
 115         if status == 0 and line == "Memory Configuration":
 116             status = 1
 117             continue
 118         elif status == 1 and line == 'Linker script and memory map':
 119             status = 2
 120             continue
 121         elif status ==2 and line == 'START GROUP':
 122             status = 3
 123             continue
 124
 125         # status handler
 126         if status == 3:
 127             m = valuePatternGcc.match(line)
 128             if m is not None:
 129                 sections.append(m.groups(0))
 130             for varname in varnames:
 131                 Str = ''
 132                 m = re.match("^.data.(%s)" % varname, line)
 133                 if m is not None:
 134                     m = re.match(".data.(%s)$" % varname, line)
 135                     if m is not None:
 136                         Str = lines[index + 1]
 137                     else:
 138                         Str = line[len(".data.%s" % varname):]
 139                     if Str:
 140                         m = pcdPatternGcc.match(Str.strip())
 141                         if m is not None:
 142                             varoffset.append((varname, int(m.groups(0)[0], 16), int(sections[-1][1], 16), sections[-1][0]))
 143
 144     if not varoffset:
 145         return []
 146     # get section information from efi file
 147     efisecs = PeImageClass(efifilepath).SectionHeaderList
 148     if efisecs is None or len(efisecs) == 0:
 149         return []
 150     #redirection
 151     redirection = 0
 152     for efisec in efisecs:
 153         for section in sections:
 154             if section[0].strip() == efisec[0].strip() and section[0].strip() == '.text':
 155                 redirection = int(section[1], 16) - efisec[1]
 156
 157     ret = []
 158     for var in varoffset:
 159         for efisec in efisecs:
 160             if var[1] >= efisec[1] and var[1] < efisec[1]+efisec[3]:
 161                 ret.append((var[0], hex(efisec[2] + var[1] - efisec[1] - redirection)))
 162     return ret
 163
 164 def _parseGeneral(lines, efifilepath, varnames):
 165     status = 0    #0 - beginning of file; 1 - PE section definition; 2 - symbol table
 166     secs  = []    # key = section name
 167     varoffset = []
 168     symRe = re.compile('^([\da-fA-F]+):([\da-fA-F]+) +([\.:\\\\\w\?@\$]+) +([\da-fA-F]+)', re.UNICODE)
 169
 170     for line in lines:
 171         line = line.strip()
 172         if startPatternGeneral.match(line):
 173             status = 1
 174             continue
 175         if addressPatternGeneral.match(line):
 176             status = 2
 177             continue
 178         if line.startswith("entry point at"):
 179             status = 3
 180             continue
 181         if status == 1 and len(line) != 0:
 182             m =  secReGeneral.match(line)
 183             assert m is not None, "Fail to parse the section in map file , line is %s" % line
 184             sec_no, sec_start, sec_length, sec_name, sec_class = m.groups(0)
 185             secs.append([int(sec_no, 16), int(sec_start, 16), int(sec_length, 16), sec_name, sec_class])
 186         if status == 2 and len(line) != 0:
 187             for varname in varnames:
 188                 m = symRe.match(line)
 189                 assert m is not None, "Fail to parse the symbol in map file, line is %s" % line
 190                 sec_no, sym_offset, sym_name, vir_addr = m.groups(0)
 191                 sec_no     = int(sec_no,     16)
 192                 sym_offset = int(sym_offset, 16)
 193                 vir_addr   = int(vir_addr,   16)
 194                 # cannot pregenerate this RegEx since it uses varname from varnames.
 195                 m2 = re.match('^[_]*(%s)' % varname, sym_name)
 196                 if m2 is not None:
 197                     # fond a binary pcd entry in map file
 198                     for sec in secs:
 199                         if sec[0] == sec_no and (sym_offset >= sec[1] and sym_offset < sec[1] + sec[2]):
 200                             varoffset.append([varname, sec[3], sym_offset, vir_addr, sec_no])
 201
 202     if not varoffset: return []
 203
 204     # get section information from efi file
 205     efisecs = PeImageClass(efifilepath).SectionHeaderList
 206     if efisecs is None or len(efisecs) == 0:
 207         return []
 208
 209     ret = []
 210     for var in varoffset:
 211         index = 0
 212         for efisec in efisecs:
 213             index = index + 1
 214             if var[1].strip() == efisec[0].strip():
 215                 ret.append((var[0], hex(efisec[2] + var[2])))
 216             elif var[4] == index:
 217                 ret.append((var[0], hex(efisec[2] + var[2])))
 218
 219     return ret
 220
 221 ## Routine to process duplicated INF
 222 #
 223 #  This function is called by following two cases:
 224 #  Case 1 in DSC:
 225 #    [components.arch]
 226 #    Pkg/module/module.inf
 227 #    Pkg/module/module.inf {
 228 #      <Defines>
 229 #        FILE_GUID = 0D1B936F-68F3-4589-AFCC-FB8B7AEBC836
 230 #    }
 231 #  Case 2 in FDF:
 232 #    INF Pkg/module/module.inf
 233 #    INF FILE_GUID = 0D1B936F-68F3-4589-AFCC-FB8B7AEBC836 Pkg/module/module.inf
 234 #
 235 #  This function copies Pkg/module/module.inf to
 236 #  Conf/.cache/0D1B936F-68F3-4589-AFCC-FB8B7AEBC836module.inf
 237 #
 238 #  @param Path     Original PathClass object
 239 #  @param BaseName New file base name
 240 #
 241 #  @retval         return the new PathClass object
 242 #
 243 def ProcessDuplicatedInf(Path, BaseName, Workspace):
 244     Filename = os.path.split(Path.File)[1]
 245     if '.' in Filename:
 246         Filename = BaseName + Path.BaseName + Filename[Filename.rfind('.'):]
 247     else:
 248         Filename = BaseName + Path.BaseName
 249
 250     #
 251     # If -N is specified on command line, cache is disabled
 252     # The directory has to be created
 253     #
 254     DbDir = os.path.split(GlobalData.gDatabasePath)[0]
 255     if not os.path.exists(DbDir):
 256         os.makedirs(DbDir)
 257     #
 258     # A temporary INF is copied to database path which must have write permission
 259     # The temporary will be removed at the end of build
 260     # In case of name conflict, the file name is
 261     # FILE_GUIDBaseName (0D1B936F-68F3-4589-AFCC-FB8B7AEBC836module.inf)
 262     #
 263     TempFullPath = os.path.join(DbDir,
 264                                 Filename)
 265     RtPath = PathClass(Path.File, Workspace)
 266     #
 267     # Modify the full path to temporary path, keep other unchanged
 268     #
 269     # To build same module more than once, the module path with FILE_GUID overridden has
 270     # the file name FILE_GUIDmodule.inf, but the relative path (self.MetaFile.File) is the real path
 271     # in DSC which is used as relative path by C files and other files in INF.
 272     # A trick was used: all module paths are PathClass instances, after the initialization
 273     # of PathClass, the PathClass.Path is overridden by the temporary INF path.
 274     #
 275     # The reason for creating a temporary INF is:
 276     # Platform.Modules which is the base to create ModuleAutoGen objects is a dictionary,
 277     # the key is the full path of INF, the value is an object to save overridden library instances, PCDs.
 278     # A different key for the same module is needed to create different output directory,
 279     # retrieve overridden PCDs, library instances.
 280     #
 281     # The BaseName is the FILE_GUID which is also the output directory name.
 282     #
 283     #
 284     RtPath.Path = TempFullPath
 285     RtPath.BaseName = BaseName
 286     #
 287     # If file exists, compare contents
 288     #
 289     if os.path.exists(TempFullPath):
 290         with open(str(Path), 'rb') as f1, open(TempFullPath, 'rb') as f2:
 291             if f1.read() == f2.read():
 292                 return RtPath
 293     _TempInfs.append(TempFullPath)
 294     shutil.copy2(str(Path), TempFullPath)
 295     return RtPath
 296
 297 ## Remove temporary created INFs whose paths were saved in _TempInfs
 298 #
 299 def ClearDuplicatedInf():
 300     while _TempInfs:
 301         File = _TempInfs.pop()
 302         if os.path.exists(File):
 303             os.remove(File)
 304
 305 ## Convert GUID string in xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx style to C structure style
 306 #
 307 #   @param      Guid    The GUID string
 308 #
 309 #   @retval     string  The GUID string in C structure style
 310 #
 311 def GuidStringToGuidStructureString(Guid):
 312     GuidList = Guid.split('-')
 313     Result = '{'
 314     for Index in range(0, 3, 1):
 315         Result = Result + '0x' + GuidList[Index] + ', '
 316     Result = Result + '{0x' + GuidList[3][0:2] + ', 0x' + GuidList[3][2:4]
 317     for Index in range(0, 12, 2):
 318         Result = Result + ', 0x' + GuidList[4][Index:Index + 2]
 319     Result += '}}'
 320     return Result
 321
 322 ## Convert GUID structure in byte array to xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
 323 #
 324 #   @param      GuidValue   The GUID value in byte array
 325 #
 326 #   @retval     string      The GUID value in xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx format
 327 #
 328 def GuidStructureByteArrayToGuidString(GuidValue):
 329     guidValueString = GuidValue.lower().replace("{", "").replace("}", "").replace(" ", "").replace(";", "")
 330     guidValueList = guidValueString.split(",")
 331     if len(guidValueList) != 16:
 332         return ''
 333         #EdkLogger.error(None, None, "Invalid GUID value string %s" % GuidValue)
 334     try:
 335         return "%02x%02x%02x%02x-%02x%02x-%02x%02x-%02x%02x-%02x%02x%02x%02x%02x%02x" % (
 336                 int(guidValueList[3], 16),
 337                 int(guidValueList[2], 16),
 338                 int(guidValueList[1], 16),
 339                 int(guidValueList[0], 16),
 340                 int(guidValueList[5], 16),
 341                 int(guidValueList[4], 16),
 342                 int(guidValueList[7], 16),
 343                 int(guidValueList[6], 16),
 344                 int(guidValueList[8], 16),
 345                 int(guidValueList[9], 16),
 346                 int(guidValueList[10], 16),
 347                 int(guidValueList[11], 16),
 348                 int(guidValueList[12], 16),
 349                 int(guidValueList[13], 16),
 350                 int(guidValueList[14], 16),
 351                 int(guidValueList[15], 16)
 352                 )
 353     except:
 354         return ''
 355
 356 ## Convert GUID string in C structure style to xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
 357 #
 358 #   @param      GuidValue   The GUID value in C structure format
 359 #
 360 #   @retval     string      The GUID value in xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx format
 361 #
 362 def GuidStructureStringToGuidString(GuidValue):
 363     if not GlobalData.gGuidCFormatPattern.match(GuidValue):
 364         return ''
 365     guidValueString = GuidValue.lower().replace("{", "").replace("}", "").replace(" ", "").replace(";", "")
 366     guidValueList = guidValueString.split(",")
 367     if len(guidValueList) != 11:
 368         return ''
 369         #EdkLogger.error(None, None, "Invalid GUID value string %s" % GuidValue)
 370     try:
 371         return "%08x-%04x-%04x-%02x%02x-%02x%02x%02x%02x%02x%02x" % (
 372                 int(guidValueList[0], 16),
 373                 int(guidValueList[1], 16),
 374                 int(guidValueList[2], 16),
 375                 int(guidValueList[3], 16),
 376                 int(guidValueList[4], 16),
 377                 int(guidValueList[5], 16),
 378                 int(guidValueList[6], 16),
 379                 int(guidValueList[7], 16),
 380                 int(guidValueList[8], 16),
 381                 int(guidValueList[9], 16),
 382                 int(guidValueList[10], 16)
 383                 )
 384     except:
 385         return ''
 386
 387 ## Convert GUID string in C structure style to xxxxxxxx_xxxx_xxxx_xxxx_xxxxxxxxxxxx
 388 #
 389 #   @param      GuidValue   The GUID value in C structure format
 390 #
 391 #   @retval     string      The GUID value in xxxxxxxx_xxxx_xxxx_xxxx_xxxxxxxxxxxx format
 392 #
 393 def GuidStructureStringToGuidValueName(GuidValue):
 394     guidValueString = GuidValue.lower().replace("{", "").replace("}", "").replace(" ", "")
 395     guidValueList = guidValueString.split(",")
 396     if len(guidValueList) != 11:
 397         EdkLogger.error(None, FORMAT_INVALID, "Invalid GUID value string [%s]" % GuidValue)
 398     return "%08x_%04x_%04x_%02x%02x_%02x%02x%02x%02x%02x%02x" % (
 399             int(guidValueList[0], 16),
 400             int(guidValueList[1], 16),
 401             int(guidValueList[2], 16),
 402             int(guidValueList[3], 16),
 403             int(guidValueList[4], 16),
 404             int(guidValueList[5], 16),
 405             int(guidValueList[6], 16),
 406             int(guidValueList[7], 16),
 407             int(guidValueList[8], 16),
 408             int(guidValueList[9], 16),
 409             int(guidValueList[10], 16)
 410             )
 411
 412 ## Create directories
 413 #
 414 #   @param      Directory   The directory name
 415 #
 416 def CreateDirectory(Directory):
 417     if Directory is None or Directory.strip() == "":
 418         return True
 419     try:
 420         if not os.access(Directory, os.F_OK):
 421             os.makedirs(Directory)
 422     except:
 423         return False
 424     return True
 425
 426 ## Remove directories, including files and sub-directories in it
 427 #
 428 #   @param      Directory   The directory name
 429 #
 430 def RemoveDirectory(Directory, Recursively=False):
 431     if Directory is None or Directory.strip() == "" or not os.path.exists(Directory):
 432         return
 433     if Recursively:
 434         CurrentDirectory = os.getcwd()
 435         os.chdir(Directory)
 436         for File in os.listdir("."):
 437             if os.path.isdir(File):
 438                 RemoveDirectory(File, Recursively)
 439             else:
 440                 os.remove(File)
 441         os.chdir(CurrentDirectory)
 442     os.rmdir(Directory)
 443
 444 ## Store content in file
 445 #
 446 #  This method is used to save file only when its content is changed. This is
 447 #  quite useful for "make" system to decide what will be re-built and what won't.
 448 #
 449 #   @param      File            The path of file
 450 #   @param      Content         The new content of the file
 451 #   @param      IsBinaryFile    The flag indicating if the file is binary file or not
 452 #
 453 #   @retval     True            If the file content is changed and the file is renewed
 454 #   @retval     False           If the file content is the same
 455 #
 456 def SaveFileOnChange(File, Content, IsBinaryFile=True):
 457     if os.path.exists(File):
 458         try:
 459             if isinstance(Content, bytes):
 460                 with open(File, "rb") as f:
 461                     if Content == f.read():
 462                         return False
 463             else:
 464                 with open(File, "r") as f:
 465                     if Content == f.read():
 466                         return False
 467         except:
 468             EdkLogger.error(None, FILE_OPEN_FAILURE, ExtraData=File)
 469
 470     DirName = os.path.dirname(File)
 471     if not CreateDirectory(DirName):
 472         EdkLogger.error(None, FILE_CREATE_FAILURE, "Could not create directory %s" % DirName)
 473     else:
 474         if DirName == '':
 475             DirName = os.getcwd()
 476         if not os.access(DirName, os.W_OK):
 477             EdkLogger.error(None, PERMISSION_FAILURE, "Do not have write permission on directory %s" % DirName)
 478
 479     try:
 480         if isinstance(Content, bytes):
 481             with open(File, "wb") as Fd:
 482                 Fd.write(Content)
 483         else:
 484             with open(File, "w") as Fd:
 485                 Fd.write(Content)
 486     except IOError as X:
 487         EdkLogger.error(None, FILE_CREATE_FAILURE, ExtraData='IOError %s' % X)
 488
 489     return True
 490
 491 ## Make a Python object persistent on file system
 492 #
 493 #   @param      Data    The object to be stored in file
 494 #   @param      File    The path of file to store the object
 495 #
 496 def DataDump(Data, File):
 497     Fd = None
 498     try:
 499         Fd = open(File, 'wb')
 500         pickle.dump(Data, Fd, pickle.HIGHEST_PROTOCOL)
 501     except:
 502         EdkLogger.error("", FILE_OPEN_FAILURE, ExtraData=File, RaiseError=False)
 503     finally:
 504         if Fd is not None:
 505             Fd.close()
 506
 507 ## Restore a Python object from a file
 508 #
 509 #   @param      File    The path of file stored the object
 510 #
 511 #   @retval     object  A python object
 512 #   @retval     None    If failure in file operation
 513 #
 514 def DataRestore(File):
 515     Data = None
 516     Fd = None
 517     try:
 518         Fd = open(File, 'rb')
 519         Data = pickle.load(Fd)
 520     except Exception as e:
 521         EdkLogger.verbose("Failed to load [%s]\n\t%s" % (File, str(e)))
 522         Data = None
 523     finally:
 524         if Fd is not None:
 525             Fd.close()
 526     return Data
 527
 528 ## Retrieve and cache the real path name in file system
 529 #
 530 #   @param      Root    The root directory of path relative to
 531 #
 532 #   @retval     str     The path string if the path exists
 533 #   @retval     None    If path doesn't exist
 534 #
 535 class DirCache:
 536     _CACHE_ = set()
 537     _UPPER_CACHE_ = {}
 538
 539     def __init__(self, Root):
 540         self._Root = Root
 541         for F in os.listdir(Root):
 542             self._CACHE_.add(F)
 543             self._UPPER_CACHE_[F.upper()] = F
 544
 545     # =[] operator
 546     def __getitem__(self, Path):
 547         Path = Path[len(os.path.commonprefix([Path, self._Root])):]
 548         if not Path:
 549             return self._Root
 550         if Path and Path[0] == os.path.sep:
 551             Path = Path[1:]
 552         if Path in self._CACHE_:
 553             return os.path.join(self._Root, Path)
 554         UpperPath = Path.upper()
 555         if UpperPath in self._UPPER_CACHE_:
 556             return os.path.join(self._Root, self._UPPER_CACHE_[UpperPath])
 557
 558         IndexList = []
 559         LastSepIndex = -1
 560         SepIndex = Path.find(os.path.sep)
 561         while SepIndex > -1:
 562             Parent = UpperPath[:SepIndex]
 563             if Parent not in self._UPPER_CACHE_:
 564                 break
 565             LastSepIndex = SepIndex
 566             SepIndex = Path.find(os.path.sep, LastSepIndex + 1)
 567
 568         if LastSepIndex == -1:
 569             return None
 570
 571         Cwd = os.getcwd()
 572         os.chdir(self._Root)
 573         SepIndex = LastSepIndex
 574         while SepIndex > -1:
 575             Parent = Path[:SepIndex]
 576             ParentKey = UpperPath[:SepIndex]
 577             if ParentKey not in self._UPPER_CACHE_:
 578                 os.chdir(Cwd)
 579                 return None
 580
 581             if Parent in self._CACHE_:
 582                 ParentDir = Parent
 583             else:
 584                 ParentDir = self._UPPER_CACHE_[ParentKey]
 585             for F in os.listdir(ParentDir):
 586                 Dir = os.path.join(ParentDir, F)
 587                 self._CACHE_.add(Dir)
 588                 self._UPPER_CACHE_[Dir.upper()] = Dir
 589
 590             SepIndex = Path.find(os.path.sep, SepIndex + 1)
 591
 592         os.chdir(Cwd)
 593         if Path in self._CACHE_:
 594             return os.path.join(self._Root, Path)
 595         elif UpperPath in self._UPPER_CACHE_:
 596             return os.path.join(self._Root, self._UPPER_CACHE_[UpperPath])
 597         return None
 598
 599 def RealPath(File, Dir='', OverrideDir=''):
 600     NewFile = os.path.normpath(os.path.join(Dir, File))
 601     NewFile = GlobalData.gAllFiles[NewFile]
 602     if not NewFile and OverrideDir:
 603         NewFile = os.path.normpath(os.path.join(OverrideDir, File))
 604         NewFile = GlobalData.gAllFiles[NewFile]
 605     return NewFile
 606
 607 def RealPath2(File, Dir='', OverrideDir=''):
 608     NewFile = None
 609     if OverrideDir:
 610         NewFile = GlobalData.gAllFiles[os.path.normpath(os.path.join(OverrideDir, File))]
 611         if NewFile:
 612             if OverrideDir[-1] == os.path.sep:
 613                 return NewFile[len(OverrideDir):], NewFile[0:len(OverrideDir)]
 614             else:
 615                 return NewFile[len(OverrideDir) + 1:], NewFile[0:len(OverrideDir)]
 616     if GlobalData.gAllFiles:
 617         NewFile = GlobalData.gAllFiles[os.path.normpath(os.path.join(Dir, File))]
 618     if not NewFile:
 619         NewFile = os.path.normpath(os.path.join(Dir, File))
 620         if not os.path.exists(NewFile):
 621             return None, None
 622     if NewFile:
 623         if Dir:
 624             if Dir[-1] == os.path.sep:
 625                 return NewFile[len(Dir):], NewFile[0:len(Dir)]
 626             else:
 627                 return NewFile[len(Dir) + 1:], NewFile[0:len(Dir)]
 628         else:
 629             return NewFile, ''
 630
 631     return None, None
 632
 633 ## Get GUID value from given packages
 634 #
 635 #   @param      CName           The CName of the GUID
 636 #   @param      PackageList     List of packages looking-up in
 637 #   @param      Inffile         The driver file
 638 #
 639 #   @retval     GuidValue   if the CName is found in any given package
 640 #   @retval     None        if the CName is not found in all given packages
 641 #
 642 def GuidValue(CName, PackageList, Inffile = None):
 643     for P in PackageList:
 644         GuidKeys = list(P.Guids.keys())
 645         if Inffile and P._PrivateGuids:
 646             if not Inffile.startswith(P.MetaFile.Dir):
 647                 GuidKeys = [x for x in P.Guids if x not in P._PrivateGuids]
 648         if CName in GuidKeys:
 649             return P.Guids[CName]
 650     return None
 651
 652 ## Get Protocol value from given packages
 653 #
 654 #   @param      CName           The CName of the GUID
 655 #   @param      PackageList     List of packages looking-up in
 656 #   @param      Inffile         The driver file
 657 #
 658 #   @retval     GuidValue   if the CName is found in any given package
 659 #   @retval     None        if the CName is not found in all given packages
 660 #
 661 def ProtocolValue(CName, PackageList, Inffile = None):
 662     for P in PackageList:
 663         ProtocolKeys = list(P.Protocols.keys())
 664         if Inffile and P._PrivateProtocols:
 665             if not Inffile.startswith(P.MetaFile.Dir):
 666                 ProtocolKeys = [x for x in P.Protocols if x not in P._PrivateProtocols]
 667         if CName in ProtocolKeys:
 668             return P.Protocols[CName]
 669     return None
 670
 671 ## Get PPI value from given packages
 672 #
 673 #   @param      CName           The CName of the GUID
 674 #   @param      PackageList     List of packages looking-up in
 675 #   @param      Inffile         The driver file
 676 #
 677 #   @retval     GuidValue   if the CName is found in any given package
 678 #   @retval     None        if the CName is not found in all given packages
 679 #
 680 def PpiValue(CName, PackageList, Inffile = None):
 681     for P in PackageList:
 682         PpiKeys = list(P.Ppis.keys())
 683         if Inffile and P._PrivatePpis:
 684             if not Inffile.startswith(P.MetaFile.Dir):
 685                 PpiKeys = [x for x in P.Ppis if x not in P._PrivatePpis]
 686         if CName in PpiKeys:
 687             return P.Ppis[CName]
 688     return None
 689
 690 ## A string template class
 691 #
 692 #  This class implements a template for string replacement. A string template
 693 #  looks like following
 694 #
 695 #       ${BEGIN} other_string ${placeholder_name} other_string ${END}
 696 #
 697 #  The string between ${BEGIN} and ${END} will be repeated as many times as the
 698 #  length of "placeholder_name", which is a list passed through a dict. The
 699 #  "placeholder_name" is the key name of the dict. The ${BEGIN} and ${END} can
 700 #  be not used and, in this case, the "placeholder_name" must not a list and it
 701 #  will just be replaced once.
 702 #
 703 class TemplateString(object):
 704     _REPEAT_START_FLAG = "BEGIN"
 705     _REPEAT_END_FLAG = "END"
 706
 707     class Section(object):
 708         _LIST_TYPES = [type([]), type(set()), type((0,))]
 709
 710         def __init__(self, TemplateSection, PlaceHolderList):
 711             self._Template = TemplateSection
 712             self._PlaceHolderList = []
 713
 714             # Split the section into sub-sections according to the position of placeholders
 715             if PlaceHolderList:
 716                 self._SubSectionList = []
 717                 SubSectionStart = 0
 718                 #
 719                 # The placeholders passed in must be in the format of
 720                 #
 721                 #   PlaceHolderName, PlaceHolderStartPoint, PlaceHolderEndPoint
 722                 #
 723                 for PlaceHolder, Start, End in PlaceHolderList:
 724                     self._SubSectionList.append(TemplateSection[SubSectionStart:Start])
 725                     self._SubSectionList.append(TemplateSection[Start:End])
 726                     self._PlaceHolderList.append(PlaceHolder)
 727                     SubSectionStart = End
 728                 if SubSectionStart < len(TemplateSection):
 729                     self._SubSectionList.append(TemplateSection[SubSectionStart:])
 730             else:
 731                 self._SubSectionList = [TemplateSection]
 732
 733         def __str__(self):
 734             return self._Template + " : " + str(self._PlaceHolderList)
 735
 736         def Instantiate(self, PlaceHolderValues):
 737             RepeatTime = -1
 738             RepeatPlaceHolders = {}
 739             NonRepeatPlaceHolders = {}
 740
 741             for PlaceHolder in self._PlaceHolderList:
 742                 if PlaceHolder not in PlaceHolderValues:
 743                     continue
 744                 Value = PlaceHolderValues[PlaceHolder]
 745                 if type(Value) in self._LIST_TYPES:
 746                     if RepeatTime < 0:
 747                         RepeatTime = len(Value)
 748                     elif RepeatTime != len(Value):
 749                         EdkLogger.error(
 750                                     "TemplateString",
 751                                     PARAMETER_INVALID,
 752                                     "${%s} has different repeat time from others!" % PlaceHolder,
 753                                     ExtraData=str(self._Template)
 754                                     )
 755                     RepeatPlaceHolders["${%s}" % PlaceHolder] = Value
 756                 else:
 757                     NonRepeatPlaceHolders["${%s}" % PlaceHolder] = Value
 758
 759             if NonRepeatPlaceHolders:
 760                 StringList = []
 761                 for S in self._SubSectionList:
 762                     if S not in NonRepeatPlaceHolders:
 763                         StringList.append(S)
 764                     else:
 765                         StringList.append(str(NonRepeatPlaceHolders[S]))
 766             else:
 767                 StringList = self._SubSectionList
 768
 769             if RepeatPlaceHolders:
 770                 TempStringList = []
 771                 for Index in range(RepeatTime):
 772                     for S in StringList:
 773                         if S not in RepeatPlaceHolders:
 774                             TempStringList.append(S)
 775                         else:
 776                             TempStringList.append(str(RepeatPlaceHolders[S][Index]))
 777                 StringList = TempStringList
 778
 779             return "".join(StringList)
 780
 781     ## Constructor
 782     def __init__(self, Template=None):
 783         self.String = ''
 784         self.IsBinary = False
 785         self._Template = Template
 786         self._TemplateSectionList = self._Parse(Template)
 787
 788     ## str() operator
 789     #
 790     #   @retval     string  The string replaced
 791     #
 792     def __str__(self):
 793         return self.String
 794
 795     ## Split the template string into fragments per the ${BEGIN} and ${END} flags
 796     #
 797     #   @retval     list    A list of TemplateString.Section objects
 798     #
 799     def _Parse(self, Template):
 800         SectionStart = 0
 801         SearchFrom = 0
 802         MatchEnd = 0
 803         PlaceHolderList = []
 804         TemplateSectionList = []
 805         while Template:
 806             MatchObj = gPlaceholderPattern.search(Template, SearchFrom)
 807             if not MatchObj:
 808                 if MatchEnd <= len(Template):
 809                     TemplateSection = TemplateString.Section(Template[SectionStart:], PlaceHolderList)
 810                     TemplateSectionList.append(TemplateSection)
 811                 break
 812
 813             MatchString = MatchObj.group(1)
 814             MatchStart = MatchObj.start()
 815             MatchEnd = MatchObj.end()
 816
 817             if MatchString == self._REPEAT_START_FLAG:
 818                 if MatchStart > SectionStart:
 819                     TemplateSection = TemplateString.Section(Template[SectionStart:MatchStart], PlaceHolderList)
 820                     TemplateSectionList.append(TemplateSection)
 821                 SectionStart = MatchEnd
 822                 PlaceHolderList = []
 823             elif MatchString == self._REPEAT_END_FLAG:
 824                 TemplateSection = TemplateString.Section(Template[SectionStart:MatchStart], PlaceHolderList)
 825                 TemplateSectionList.append(TemplateSection)
 826                 SectionStart = MatchEnd
 827                 PlaceHolderList = []
 828             else:
 829                 PlaceHolderList.append((MatchString, MatchStart - SectionStart, MatchEnd - SectionStart))
 830             SearchFrom = MatchEnd
 831         return TemplateSectionList
 832
 833     ## Replace the string template with dictionary of placeholders and append it to previous one
 834     #
 835     #   @param      AppendString    The string template to append
 836     #   @param      Dictionary      The placeholder dictionaries
 837     #
 838     def Append(self, AppendString, Dictionary=None):
 839         if Dictionary:
 840             SectionList = self._Parse(AppendString)
 841             self.String += "".join(S.Instantiate(Dictionary) for S in SectionList)
 842         else:
 843             self.String += AppendString
 844
 845     ## Replace the string template with dictionary of placeholders
 846     #
 847     #   @param      Dictionary      The placeholder dictionaries
 848     #
 849     #   @retval     str             The string replaced with placeholder values
 850     #
 851     def Replace(self, Dictionary=None):
 852         return "".join(S.Instantiate(Dictionary) for S in self._TemplateSectionList)
 853
 854 ## Progress indicator class
 855 #
 856 #  This class makes use of thread to print progress on console.
 857 #
 858 class Progressor:
 859     # for avoiding deadloop
 860     _StopFlag = None
 861     _ProgressThread = None
 862     _CheckInterval = 0.25
 863
 864     ## Constructor
 865     #
 866     #   @param      OpenMessage     The string printed before progress charaters
 867     #   @param      CloseMessage    The string printed after progress charaters
 868     #   @param      ProgressChar    The charater used to indicate the progress
 869     #   @param      Interval        The interval in seconds between two progress charaters
 870     #
 871     def __init__(self, OpenMessage="", CloseMessage="", ProgressChar='.', Interval=1.0):
 872         self.PromptMessage = OpenMessage
 873         self.CodaMessage = CloseMessage
 874         self.ProgressChar = ProgressChar
 875         self.Interval = Interval
 876         if Progressor._StopFlag is None:
 877             Progressor._StopFlag = threading.Event()
 878
 879     ## Start to print progress charater
 880     #
 881     #   @param      OpenMessage     The string printed before progress charaters
 882     #
 883     def Start(self, OpenMessage=None):
 884         if OpenMessage is not None:
 885             self.PromptMessage = OpenMessage
 886         Progressor._StopFlag.clear()
 887         if Progressor._ProgressThread is None:
 888             Progressor._ProgressThread = threading.Thread(target=self._ProgressThreadEntry)
 889             Progressor._ProgressThread.setDaemon(False)
 890             Progressor._ProgressThread.start()
 891
 892     ## Stop printing progress charater
 893     #
 894     #   @param      CloseMessage    The string printed after progress charaters
 895     #
 896     def Stop(self, CloseMessage=None):
 897         OriginalCodaMessage = self.CodaMessage
 898         if CloseMessage is not None:
 899             self.CodaMessage = CloseMessage
 900         self.Abort()
 901         self.CodaMessage = OriginalCodaMessage
 902
 903     ## Thread entry method
 904     def _ProgressThreadEntry(self):
 905         sys.stdout.write(self.PromptMessage + " ")
 906         sys.stdout.flush()
 907         TimeUp = 0.0
 908         while not Progressor._StopFlag.isSet():
 909             if TimeUp <= 0.0:
 910                 sys.stdout.write(self.ProgressChar)
 911                 sys.stdout.flush()
 912                 TimeUp = self.Interval
 913             time.sleep(self._CheckInterval)
 914             TimeUp -= self._CheckInterval
 915         sys.stdout.write(" " + self.CodaMessage + "\n")
 916         sys.stdout.flush()
 917
 918     ## Abort the progress display
 919     @staticmethod
 920     def Abort():
 921         if Progressor._StopFlag is not None:
 922             Progressor._StopFlag.set()
 923         if Progressor._ProgressThread is not None:
 924             Progressor._ProgressThread.join()
 925             Progressor._ProgressThread = None
 926
 927 ## A dict which can access its keys and/or values orderly
 928 #
 929 #  The class implements a new kind of dict which its keys or values can be
 930 #  accessed in the order they are added into the dict. It guarantees the order
 931 #  by making use of an internal list to keep a copy of keys.
 932 #
 933 class sdict(IterableUserDict):
 934     ## Constructor
 935     def __init__(self):
 936         IterableUserDict.__init__(self)
 937         self._key_list = []
 938
 939     ## [] operator
 940     def __setitem__(self, key, value):
 941         if key not in self._key_list:
 942             self._key_list.append(key)
 943         IterableUserDict.__setitem__(self, key, value)
 944
 945     ## del operator
 946     def __delitem__(self, key):
 947         self._key_list.remove(key)
 948         IterableUserDict.__delitem__(self, key)
 949
 950     ## used in "for k in dict" loop to ensure the correct order
 951     def __iter__(self):
 952         return self.iterkeys()
 953
 954     ## len() support
 955     def __len__(self):
 956         return len(self._key_list)
 957
 958     ## "in" test support
 959     def __contains__(self, key):
 960         return key in self._key_list
 961
 962     ## indexof support
 963     def index(self, key):
 964         return self._key_list.index(key)
 965
 966     ## insert support
 967     def insert(self, key, newkey, newvalue, order):
 968         index = self._key_list.index(key)
 969         if order == 'BEFORE':
 970             self._key_list.insert(index, newkey)
 971             IterableUserDict.__setitem__(self, newkey, newvalue)
 972         elif order == 'AFTER':
 973             self._key_list.insert(index + 1, newkey)
 974             IterableUserDict.__setitem__(self, newkey, newvalue)
 975
 976     ## append support
 977     def append(self, sdict):
 978         for key in sdict.keys():
 979             if key not in self._key_list:
 980                 self._key_list.append(key)
 981             IterableUserDict.__setitem__(self, key, sdict[key])
 982
 983     def has_key(self, key):
 984         return key in self._key_list
 985
 986     ## Empty the dict
 987     def clear(self):
 988         self._key_list = []
 989         IterableUserDict.clear(self)
 990
 991     ## Return a copy of keys
 992     def keys(self):
 993         keys = []
 994         for key in self._key_list:
 995             keys.append(key)
 996         return keys
 997
 998     ## Return a copy of values
 999     def values(self):
1000         values = []
1001         for key in self._key_list:
1002             values.append(self[key])
1003         return values
1004
1005     ## Return a copy of (key, value) list
1006     def items(self):
1007         items = []
1008         for key in self._key_list:
1009             items.append((key, self[key]))
1010         return items
1011
1012     ## Iteration support
1013     def iteritems(self):
1014         return iter(self.items())
1015
1016     ## Keys interation support
1017     def iterkeys(self):
1018         return self.keys()
1019
1020     ## Values interation support
1021     def itervalues(self):
1022         return self.values()
1023
1024     ## Return value related to a key, and remove the (key, value) from the dict
1025     def pop(self, key, *dv):
1026         value = None
1027         if key in self._key_list:
1028             value = self[key]
1029             self.__delitem__(key)
1030         elif len(dv) != 0 :
1031             value = dv[0]
1032         return value
1033
1034     ## Return (key, value) pair, and remove the (key, value) from the dict
1035     def popitem(self):
1036         key = self._key_list[-1]
1037         value = self[key]
1038         self.__delitem__(key)
1039         return key, value
1040
1041     def update(self, dict=None, **kwargs):
1042         if dict is not None:
1043             for k, v in dict.items():
1044                 self[k] = v
1045         if len(kwargs):
1046             for k, v in kwargs.items():
1047                 self[k] = v
1048
1049 ## Dictionary with restricted keys
1050 #
1051 class rdict(dict):
1052     ## Constructor
1053     def __init__(self, KeyList):
1054         for Key in KeyList:
1055             dict.__setitem__(self, Key, "")
1056
1057     ## []= operator
1058     def __setitem__(self, key, value):
1059         if key not in self:
1060             EdkLogger.error("RestrictedDict", ATTRIBUTE_SET_FAILURE, "Key [%s] is not allowed" % key,
1061                             ExtraData=", ".join(dict.keys(self)))
1062         dict.__setitem__(self, key, value)
1063
1064     ## =[] operator
1065     def __getitem__(self, key):
1066         if key not in self:
1067             return ""
1068         return dict.__getitem__(self, key)
1069
1070     ## del operator
1071     def __delitem__(self, key):
1072         EdkLogger.error("RestrictedDict", ATTRIBUTE_ACCESS_DENIED, ExtraData="del")
1073
1074     ## Empty the dict
1075     def clear(self):
1076         for Key in self:
1077             self.__setitem__(Key, "")
1078
1079     ## Return value related to a key, and remove the (key, value) from the dict
1080     def pop(self, key, *dv):
1081         EdkLogger.error("RestrictedDict", ATTRIBUTE_ACCESS_DENIED, ExtraData="pop")
1082
1083     ## Return (key, value) pair, and remove the (key, value) from the dict
1084     def popitem(self):
1085         EdkLogger.error("RestrictedDict", ATTRIBUTE_ACCESS_DENIED, ExtraData="popitem")
1086
1087 ## Dictionary using prioritized list as key
1088 #
1089 class tdict:
1090     _ListType = type([])
1091     _TupleType = type(())
1092     _Wildcard = 'COMMON'
1093     _ValidWildcardList = ['COMMON', 'DEFAULT', 'ALL', '*', 'PLATFORM']
1094
1095     def __init__(self, _Single_=False, _Level_=2):
1096         self._Level_ = _Level_
1097         self.data = {}
1098         self._Single_ = _Single_
1099
1100     # =[] operator
1101     def __getitem__(self, key):
1102         KeyType = type(key)
1103         RestKeys = None
1104         if KeyType == self._ListType or KeyType == self._TupleType:
1105             FirstKey = key[0]
1106             if len(key) > 1:
1107                 RestKeys = key[1:]
1108             elif self._Level_ > 1:
1109                 RestKeys = [self._Wildcard for i in range(0, self._Level_ - 1)]
1110         else:
1111             FirstKey = key
1112             if self._Level_ > 1:
1113                 RestKeys = [self._Wildcard for i in range(0, self._Level_ - 1)]
1114
1115         if FirstKey is None or str(FirstKey).upper() in self._ValidWildcardList:
1116             FirstKey = self._Wildcard
1117
1118         if self._Single_:
1119             return self._GetSingleValue(FirstKey, RestKeys)
1120         else:
1121             return self._GetAllValues(FirstKey, RestKeys)
1122
1123     def _GetSingleValue(self, FirstKey, RestKeys):
1124         Value = None
1125         #print "%s-%s" % (FirstKey, self._Level_) ,
1126         if self._Level_ > 1:
1127             if FirstKey == self._Wildcard:
1128                 if FirstKey in self.data:
1129                     Value = self.data[FirstKey][RestKeys]
1130                 if Value is None:
1131                     for Key in self.data:
1132                         Value = self.data[Key][RestKeys]
1133                         if Value is not None: break
1134             else:
1135                 if FirstKey in self.data:
1136                     Value = self.data[FirstKey][RestKeys]
1137                 if Value is None and self._Wildcard in self.data:
1138                     #print "Value=None"
1139                     Value = self.data[self._Wildcard][RestKeys]
1140         else:
1141             if FirstKey == self._Wildcard:
1142                 if FirstKey in self.data:
1143                     Value = self.data[FirstKey]
1144                 if Value is None:
1145                     for Key in self.data:
1146                         Value = self.data[Key]
1147                         if Value is not None: break
1148             else:
1149                 if FirstKey in self.data:
1150                     Value = self.data[FirstKey]
1151                 elif self._Wildcard in self.data:
1152                     Value = self.data[self._Wildcard]
1153         return Value
1154
1155     def _GetAllValues(self, FirstKey, RestKeys):
1156         Value = []
1157         if self._Level_ > 1:
1158             if FirstKey == self._Wildcard:
1159                 for Key in self.data:
1160                     Value += self.data[Key][RestKeys]
1161             else:
1162                 if FirstKey in self.data:
1163                     Value += self.data[FirstKey][RestKeys]
1164                 if self._Wildcard in self.data:
1165                     Value += self.data[self._Wildcard][RestKeys]
1166         else:
1167             if FirstKey == self._Wildcard:
1168                 for Key in self.data:
1169                     Value.append(self.data[Key])
1170             else:
1171                 if FirstKey in self.data:
1172                     Value.append(self.data[FirstKey])
1173                 if self._Wildcard in self.data:
1174                     Value.append(self.data[self._Wildcard])
1175         return Value
1176
1177     ## []= operator
1178     def __setitem__(self, key, value):
1179         KeyType = type(key)
1180         RestKeys = None
1181         if KeyType == self._ListType or KeyType == self._TupleType:
1182             FirstKey = key[0]
1183             if len(key) > 1:
1184                 RestKeys = key[1:]
1185             else:
1186                 RestKeys = [self._Wildcard for i in range(0, self._Level_ - 1)]
1187         else:
1188             FirstKey = key
1189             if self._Level_ > 1:
1190                 RestKeys = [self._Wildcard for i in range(0, self._Level_ - 1)]
1191
1192         if FirstKey in self._ValidWildcardList:
1193             FirstKey = self._Wildcard
1194
1195         if FirstKey not in self.data and self._Level_ > 0:
1196             self.data[FirstKey] = tdict(self._Single_, self._Level_ - 1)
1197
1198         if self._Level_ > 1:
1199             self.data[FirstKey][RestKeys] = value
1200         else:
1201             self.data[FirstKey] = value
1202
1203     def SetGreedyMode(self):
1204         self._Single_ = False
1205         if self._Level_ > 1:
1206             for Key in self.data:
1207                 self.data[Key].SetGreedyMode()
1208
1209     def SetSingleMode(self):
1210         self._Single_ = True
1211         if self._Level_ > 1:
1212             for Key in self.data:
1213                 self.data[Key].SetSingleMode()
1214
1215     def GetKeys(self, KeyIndex=0):
1216         assert KeyIndex >= 0
1217         if KeyIndex == 0:
1218             return set(self.data.keys())
1219         else:
1220             keys = set()
1221             for Key in self.data:
1222                 keys |= self.data[Key].GetKeys(KeyIndex - 1)
1223             return keys
1224
1225 def IsFieldValueAnArray (Value):
1226     Value = Value.strip()
1227     if Value.startswith(TAB_GUID) and Value.endswith(')'):
1228         return True
1229     if Value.startswith('L"') and Value.endswith('"')  and len(list(Value[2:-1])) > 1:
1230         return True
1231     if Value[0] == '"' and Value[-1] == '"' and len(list(Value[1:-1])) > 1:
1232         return True
1233     if Value[0] == '{' and Value[-1] == '}':
1234         return True
1235     if Value.startswith("L'") and Value.endswith("'") and len(list(Value[2:-1])) > 1:
1236         return True
1237     if Value[0] == "'" and Value[-1] == "'" and len(list(Value[1:-1])) > 1:
1238         return True
1239     return False
1240
1241 def AnalyzePcdExpression(Setting):
1242     Setting = Setting.strip()
1243     # There might be escaped quote in a string: \", \\\" , \', \\\'
1244     Data = Setting
1245     # There might be '|' in string and in ( ... | ... ), replace it with '-'
1246     NewStr = ''
1247     InSingleQuoteStr = False
1248     InDoubleQuoteStr = False
1249     Pair = 0
1250     for Index, ch in enumerate(Data):
1251         if ch == '"' and not InSingleQuoteStr:
1252             if Data[Index - 1] != '\\':
1253                 InDoubleQuoteStr = not InDoubleQuoteStr
1254         elif ch == "'" and not InDoubleQuoteStr:
1255             if Data[Index - 1] != '\\':
1256                 InSingleQuoteStr = not InSingleQuoteStr
1257         elif ch == '(' and not (InSingleQuoteStr or InDoubleQuoteStr):
1258             Pair += 1
1259         elif ch == ')' and not (InSingleQuoteStr or InDoubleQuoteStr):
1260             Pair -= 1
1261
1262         if (Pair > 0 or InSingleQuoteStr or InDoubleQuoteStr) and ch == TAB_VALUE_SPLIT:
1263             NewStr += '-'
1264         else:
1265             NewStr += ch
1266     FieldList = []
1267     StartPos = 0
1268     while True:
1269         Pos = NewStr.find(TAB_VALUE_SPLIT, StartPos)
1270         if Pos < 0:
1271             FieldList.append(Setting[StartPos:].strip())
1272             break
1273         FieldList.append(Setting[StartPos:Pos].strip())
1274         StartPos = Pos + 1
1275
1276     return FieldList
1277
1278 def ParseDevPathValue (Value):
1279     if '\\' in Value:
1280         Value.replace('\\', '/').replace(' ', '')
1281
1282     Cmd = 'DevicePath ' + '"' + Value + '"'
1283     try:
1284         p = subprocess.Popen(Cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
1285         out, err = p.communicate()
1286     except Exception as X:
1287         raise BadExpression("DevicePath: %s" % (str(X)) )
1288     finally:
1289         subprocess._cleanup()
1290         p.stdout.close()
1291         p.stderr.close()
1292     if err:
1293         raise BadExpression("DevicePath: %s" % str(err))
1294     Size = len(out.split())
1295     out = ','.join(out.decode(encoding='utf-8', errors='ignore').split())
1296     return '{' + out + '}', Size
1297
1298 def ParseFieldValue (Value):
1299     if isinstance(Value, type(0)):
1300         return Value, (Value.bit_length() + 7) // 8
1301     if not isinstance(Value, type('')):
1302         raise BadExpression('Type %s is %s' %(Value, type(Value)))
1303     Value = Value.strip()
1304     if Value.startswith(TAB_UINT8) and Value.endswith(')'):
1305         Value, Size = ParseFieldValue(Value.split('(', 1)[1][:-1])
1306         if Size > 1:
1307             raise BadExpression('Value (%s) Size larger than %d' %(Value, Size))
1308         return Value, 1
1309     if Value.startswith(TAB_UINT16) and Value.endswith(')'):
1310         Value, Size = ParseFieldValue(Value.split('(', 1)[1][:-1])
1311         if Size > 2:
1312             raise BadExpression('Value (%s) Size larger than %d' %(Value, Size))
1313         return Value, 2
1314     if Value.startswith(TAB_UINT32) and Value.endswith(')'):
1315         Value, Size = ParseFieldValue(Value.split('(', 1)[1][:-1])
1316         if Size > 4:
1317             raise BadExpression('Value (%s) Size larger than %d' %(Value, Size))
1318         return Value, 4
1319     if Value.startswith(TAB_UINT64) and Value.endswith(')'):
1320         Value, Size = ParseFieldValue(Value.split('(', 1)[1][:-1])
1321         if Size > 8:
1322             raise BadExpression('Value (%s) Size larger than %d' % (Value, Size))
1323         return Value, 8
1324     if Value.startswith(TAB_GUID) and Value.endswith(')'):
1325         Value = Value.split('(', 1)[1][:-1].strip()
1326         if Value[0] == '{' and Value[-1] == '}':
1327             TmpValue = GuidStructureStringToGuidString(Value)
1328             if not TmpValue:
1329                 raise BadExpression("Invalid GUID value string %s" % Value)
1330             Value = TmpValue
1331         if Value[0] == '"' and Value[-1] == '"':
1332             Value = Value[1:-1]
1333         try:
1334             Value = "{" + ','.join([str(i) for i in uuid.UUID(Value).bytes_le]) + "}"
1335         except ValueError as Message:
1336             raise BadExpression(Message)
1337         Value, Size = ParseFieldValue(Value)
1338         return Value, 16
1339     if Value.startswith('L"') and Value.endswith('"'):
1340         # Unicode String
1341         # translate escape character
1342         Value = Value[1:]
1343         try:
1344             Value = eval(Value)
1345         except:
1346             Value = Value[1:-1]
1347         List = list(Value)
1348         List.reverse()
1349         Value = 0
1350         for Char in List:
1351             Value = (Value << 16) | ord(Char)
1352         return Value, (len(List) + 1) * 2
1353     if Value.startswith('"') and Value.endswith('"'):
1354         # ASCII String
1355         # translate escape character
1356         try:
1357             Value = eval(Value)
1358         except:
1359             Value = Value[1:-1]
1360         List = list(Value)
1361         List.reverse()
1362         Value = 0
1363         for Char in List:
1364             Value = (Value << 8) | ord(Char)
1365         return Value, len(List) + 1
1366     if Value.startswith("L'") and Value.endswith("'"):
1367         # Unicode Character Constant
1368         # translate escape character
1369         Value = Value[1:]
1370         try:
1371             Value = eval(Value)
1372         except:
1373             Value = Value[1:-1]
1374         List = list(Value)
1375         if len(List) == 0:
1376             raise BadExpression('Length %s is %s' % (Value, len(List)))
1377         List.reverse()
1378         Value = 0
1379         for Char in List:
1380             Value = (Value << 16) | ord(Char)
1381         return Value, len(List) * 2
1382     if Value.startswith("'") and Value.endswith("'"):
1383         # Character constant
1384         # translate escape character
1385         try:
1386             Value = eval(Value)
1387         except:
1388             Value = Value[1:-1]
1389         List = list(Value)
1390         if len(List) == 0:
1391             raise BadExpression('Length %s is %s' % (Value, len(List)))
1392         List.reverse()
1393         Value = 0
1394         for Char in List:
1395             Value = (Value << 8) | ord(Char)
1396         return Value, len(List)
1397     if Value.startswith('{') and Value.endswith('}'):
1398         # Byte array
1399         Value = Value[1:-1]
1400         List = [Item.strip() for Item in Value.split(',')]
1401         List.reverse()
1402         Value = 0
1403         RetSize = 0
1404         for Item in List:
1405             ItemValue, Size = ParseFieldValue(Item)
1406             RetSize += Size
1407             for I in range(Size):
1408                 Value = (Value << 8) | ((ItemValue >> 8 * I) & 0xff)
1409         return Value, RetSize
1410     if Value.startswith('DEVICE_PATH(') and Value.endswith(')'):
1411         Value = Value.replace("DEVICE_PATH(", '').rstrip(')')
1412         Value = Value.strip().strip('"')
1413         return ParseDevPathValue(Value)
1414     if Value.lower().startswith('0x'):
1415         try:
1416             Value = int(Value, 16)
1417         except:
1418             raise BadExpression("invalid hex value: %s" % Value)
1419         if Value == 0:
1420             return 0, 1
1421         return Value, (Value.bit_length() + 7) // 8
1422     if Value[0].isdigit():
1423         Value = int(Value, 10)
1424         if Value == 0:
1425             return 0, 1
1426         return Value, (Value.bit_length() + 7) // 8
1427     if Value.lower() == 'true':
1428         return 1, 1
1429     if Value.lower() == 'false':
1430         return 0, 1
1431     return Value, 1
1432
1433 ## AnalyzeDscPcd
1434 #
1435 #  Analyze DSC PCD value, since there is no data type info in DSC
1436 #  This function is used to match functions (AnalyzePcdData) used for retrieving PCD value from database
1437 #  1. Feature flag: TokenSpace.PcdCName|PcdValue
1438 #  2. Fix and Patch:TokenSpace.PcdCName|PcdValue[|VOID*[|MaxSize]]
1439 #  3. Dynamic default:
1440 #     TokenSpace.PcdCName|PcdValue[|VOID*[|MaxSize]]
1441 #     TokenSpace.PcdCName|PcdValue
1442 #  4. Dynamic VPD:
1443 #     TokenSpace.PcdCName|VpdOffset[|VpdValue]
1444 #     TokenSpace.PcdCName|VpdOffset[|MaxSize[|VpdValue]]
1445 #  5. Dynamic HII:
1446 #     TokenSpace.PcdCName|HiiString|VariableGuid|VariableOffset[|HiiValue]
1447 #  PCD value needs to be located in such kind of string, and the PCD value might be an expression in which
1448 #    there might have "|" operator, also in string value.
1449 #
1450 #  @param Setting: String contain information described above with "TokenSpace.PcdCName|" stripped
1451 #  @param PcdType: PCD type: feature, fixed, dynamic default VPD HII
1452 #  @param DataType: The datum type of PCD: VOID*, UNIT, BOOL
1453 #  @retval:
1454 #    ValueList: A List contain fields described above
1455 #    IsValid:   True if conforming EBNF, otherwise False
1456 #    Index:     The index where PcdValue is in ValueList
1457 #
1458 def AnalyzeDscPcd(Setting, PcdType, DataType=''):
1459     FieldList = AnalyzePcdExpression(Setting)
1460
1461     IsValid = True
1462     if PcdType in (MODEL_PCD_FIXED_AT_BUILD, MODEL_PCD_PATCHABLE_IN_MODULE, MODEL_PCD_DYNAMIC_DEFAULT, MODEL_PCD_DYNAMIC_EX_DEFAULT):
1463         Value = FieldList[0]
1464         Size = ''
1465         if len(FieldList) > 1 and FieldList[1]:
1466             DataType = FieldList[1]
1467             if FieldList[1] != TAB_VOID:
1468                 IsValid = False
1469         if len(FieldList) > 2:
1470             Size = FieldList[2]
1471         if IsValid:
1472             if DataType == "":
1473                 IsValid = (len(FieldList) <= 1)
1474             else:
1475                 IsValid = (len(FieldList) <= 3)
1476
1477         if Size:
1478             try:
1479                 int(Size, 16) if Size.upper().startswith("0X") else int(Size)
1480             except:
1481                 IsValid = False
1482                 Size = -1
1483         return [str(Value), DataType, str(Size)], IsValid, 0
1484     elif PcdType == MODEL_PCD_FEATURE_FLAG:
1485         Value = FieldList[0]
1486         Size = ''
1487         IsValid = (len(FieldList) <= 1)
1488         return [Value, DataType, str(Size)], IsValid, 0
1489     elif PcdType in (MODEL_PCD_DYNAMIC_VPD, MODEL_PCD_DYNAMIC_EX_VPD):
1490         VpdOffset = FieldList[0]
1491         Value = Size = ''
1492         if not DataType == TAB_VOID:
1493             if len(FieldList) > 1:
1494                 Value = FieldList[1]
1495         else:
1496             if len(FieldList) > 1:
1497                 Size = FieldList[1]
1498             if len(FieldList) > 2:
1499                 Value = FieldList[2]
1500         if DataType == "":
1501             IsValid = (len(FieldList) <= 1)
1502         else:
1503             IsValid = (len(FieldList) <= 3)
1504         if Size:
1505             try:
1506                 int(Size, 16) if Size.upper().startswith("0X") else int(Size)
1507             except:
1508                 IsValid = False
1509                 Size = -1
1510         return [VpdOffset, str(Size), Value], IsValid, 2
1511     elif PcdType in (MODEL_PCD_DYNAMIC_HII, MODEL_PCD_DYNAMIC_EX_HII):
1512         IsValid = (3 <= len(FieldList) <= 5)
1513         HiiString = FieldList[0]
1514         Guid = Offset = Value = Attribute = ''
1515         if len(FieldList) > 1:
1516             Guid = FieldList[1]
1517         if len(FieldList) > 2:
1518             Offset = FieldList[2]
1519         if len(FieldList) > 3:
1520             Value = FieldList[3]
1521             if not Value:
1522                 IsValid = False
1523         if len(FieldList) > 4:
1524             Attribute = FieldList[4]
1525         return [HiiString, Guid, Offset, Value, Attribute], IsValid, 3
1526     return [], False, 0
1527
1528 ## AnalyzePcdData
1529 #
1530 #  Analyze the pcd Value, Datum type and TokenNumber.
1531 #  Used to avoid split issue while the value string contain "|" character
1532 #
1533 #  @param[in] Setting:  A String contain value/datum type/token number information;
1534 #
1535 #  @retval   ValueList: A List contain value, datum type and toke number.
1536 #
1537 def AnalyzePcdData(Setting):
1538     ValueList = ['', '', '']
1539
1540     ValueRe = re.compile(r'^\s*L?\".*\|.*\"')
1541     PtrValue = ValueRe.findall(Setting)
1542
1543     ValueUpdateFlag = False
1544
1545     if len(PtrValue) >= 1:
1546         Setting = re.sub(ValueRe, '', Setting)
1547         ValueUpdateFlag = True
1548
1549     TokenList = Setting.split(TAB_VALUE_SPLIT)
1550     ValueList[0:len(TokenList)] = TokenList
1551
1552     if ValueUpdateFlag:
1553         ValueList[0] = PtrValue[0]
1554
1555     return ValueList
1556
1557 ## check format of PCD value against its the datum type
1558 #
1559 # For PCD value setting
1560 #
1561 def CheckPcdDatum(Type, Value):
1562     if Type == TAB_VOID:
1563         ValueRe = re.compile(r'\s*L?\".*\"\s*$')
1564         if not (((Value.startswith('L"') or Value.startswith('"')) and Value.endswith('"'))
1565                 or (Value.startswith('{') and Value.endswith('}')) or (Value.startswith("L'") or Value.startswith("'") and Value.endswith("'"))
1566                ):
1567             return False, "Invalid value [%s] of type [%s]; must be in the form of {...} for array"\
1568                           ", \"...\" or \'...\' for string, L\"...\" or L\'...\' for unicode string" % (Value, Type)
1569         elif ValueRe.match(Value):
1570             # Check the chars in UnicodeString or CString is printable
1571             if Value.startswith("L"):
1572                 Value = Value[2:-1]
1573             else:
1574                 Value = Value[1:-1]
1575             Printset = set(string.printable)
1576             Printset.remove(TAB_PRINTCHAR_VT)
1577             Printset.add(TAB_PRINTCHAR_BS)
1578             Printset.add(TAB_PRINTCHAR_NUL)
1579             if not set(Value).issubset(Printset):
1580                 PrintList = sorted(Printset)
1581                 return False, "Invalid PCD string value of type [%s]; must be printable chars %s." % (Type, PrintList)
1582     elif Type == 'BOOLEAN':
1583         if Value not in ['TRUE', 'True', 'true', '0x1', '0x01', '1', 'FALSE', 'False', 'false', '0x0', '0x00', '0']:
1584             return False, "Invalid value [%s] of type [%s]; must be one of TRUE, True, true, 0x1, 0x01, 1"\
1585                           ", FALSE, False, false, 0x0, 0x00, 0" % (Value, Type)
1586     elif Type in [TAB_UINT8, TAB_UINT16, TAB_UINT32, TAB_UINT64]:
1587         try:
1588             Val = int(Value, 0)
1589         except:
1590             try:
1591                 Val = int(Value.lstrip('0'))
1592             except:
1593                 return False, "Invalid value [%s] of type [%s];" \
1594                               " must be a hexadecimal, decimal or octal in C language format." % (Value, Type)
1595         if Val > MAX_VAL_TYPE[Type]:
1596             return False, "Too large PCD value[%s] for datum type [%s]" % (Value, Type)
1597         if Val < 0:
1598             return False, "PCD can't be set to negative value[%s] for datum type [%s]" % (Value, Type)
1599
1600     else:
1601         return True, "StructurePcd"
1602
1603     return True, ""
1604
1605 ## Split command line option string to list
1606 #
1607 # subprocess.Popen needs the args to be a sequence. Otherwise there's problem
1608 # in non-windows platform to launch command
1609 #
1610 def SplitOption(OptionString):
1611     OptionList = []
1612     LastChar = " "
1613     OptionStart = 0
1614     QuotationMark = ""
1615     for Index in range(0, len(OptionString)):
1616         CurrentChar = OptionString[Index]
1617         if CurrentChar in ['"', "'"]:
1618             if QuotationMark == CurrentChar:
1619                 QuotationMark = ""
1620             elif QuotationMark == "":
1621                 QuotationMark = CurrentChar
1622             continue
1623         elif QuotationMark:
1624             continue
1625
1626         if CurrentChar in ["/", "-"] and LastChar in [" ", "\t", "\r", "\n"]:
1627             if Index > OptionStart:
1628                 OptionList.append(OptionString[OptionStart:Index - 1])
1629             OptionStart = Index
1630         LastChar = CurrentChar
1631     OptionList.append(OptionString[OptionStart:])
1632     return OptionList
1633
1634 def CommonPath(PathList):
1635     P1 = min(PathList).split(os.path.sep)
1636     P2 = max(PathList).split(os.path.sep)
1637     for Index in range(min(len(P1), len(P2))):
1638         if P1[Index] != P2[Index]:
1639             return os.path.sep.join(P1[:Index])
1640     return os.path.sep.join(P1)
1641
1642 #
1643 # Convert string to C format array
1644 #
1645 def ConvertStringToByteArray(Value):
1646     Value = Value.strip()
1647     if not Value:
1648         return None
1649     if Value[0] == '{':
1650         if not Value.endswith('}'):
1651             return None
1652         Value = Value.replace(' ', '').replace('{', '').replace('}', '')
1653         ValFields = Value.split(',')
1654         try:
1655             for Index in range(len(ValFields)):
1656                 ValFields[Index] = str(int(ValFields[Index], 0))
1657         except ValueError:
1658             return None
1659         Value = '{' + ','.join(ValFields) + '}'
1660         return Value
1661
1662     Unicode = False
1663     if Value.startswith('L"'):
1664         if not Value.endswith('"'):
1665             return None
1666         Value = Value[1:]
1667         Unicode = True
1668     elif not Value.startswith('"') or not Value.endswith('"'):
1669         return None
1670
1671     Value = eval(Value)         # translate escape character
1672     NewValue = '{'
1673     for Index in range(0, len(Value)):
1674         if Unicode:
1675             NewValue = NewValue + str(ord(Value[Index]) % 0x10000) + ','
1676         else:
1677             NewValue = NewValue + str(ord(Value[Index]) % 0x100) + ','
1678     Value = NewValue + '0}'
1679     return Value
1680
1681 class PathClass(object):
1682     def __init__(self, File='', Root='', AlterRoot='', Type='', IsBinary=False,
1683                  Arch='COMMON', ToolChainFamily='', Target='', TagName='', ToolCode=''):
1684         self.Arch = Arch
1685         self.File = str(File)
1686         if os.path.isabs(self.File):
1687             self.Root = ''
1688             self.AlterRoot = ''
1689         else:
1690             self.Root = str(Root)
1691             self.AlterRoot = str(AlterRoot)
1692
1693         # Remove any '.' and '..' in path
1694         if self.Root:
1695             self.Root = mws.getWs(self.Root, self.File)
1696             self.Path = os.path.normpath(os.path.join(self.Root, self.File))
1697             self.Root = os.path.normpath(CommonPath([self.Root, self.Path]))
1698             # eliminate the side-effect of 'C:'
1699             if self.Root[-1] == ':':
1700                 self.Root += os.path.sep
1701             # file path should not start with path separator
1702             if self.Root[-1] == os.path.sep:
1703                 self.File = self.Path[len(self.Root):]
1704             else:
1705                 self.File = self.Path[len(self.Root) + 1:]
1706         else:
1707             self.Path = os.path.normpath(self.File)
1708
1709         self.SubDir, self.Name = os.path.split(self.File)
1710         self.BaseName, self.Ext = os.path.splitext(self.Name)
1711
1712         if self.Root:
1713             if self.SubDir:
1714                 self.Dir = os.path.join(self.Root, self.SubDir)
1715             else:
1716                 self.Dir = self.Root
1717         else:
1718             self.Dir = self.SubDir
1719
1720         if IsBinary:
1721             self.Type = Type
1722         else:
1723             self.Type = self.Ext.lower()
1724
1725         self.IsBinary = IsBinary
1726         self.Target = Target
1727         self.TagName = TagName
1728         self.ToolCode = ToolCode
1729         self.ToolChainFamily = ToolChainFamily
1730
1731     ## Convert the object of this class to a string
1732     #
1733     #  Convert member Path of the class to a string
1734     #
1735     #  @retval string Formatted String
1736     #
1737     def __str__(self):
1738         return self.Path
1739
1740     ## Override __eq__ function
1741     #
1742     # Check whether PathClass are the same
1743     #
1744     # @retval False The two PathClass are different
1745     # @retval True  The two PathClass are the same
1746     #
1747     def __eq__(self, Other):
1748         if isinstance(Other, type(self)):
1749             return self.Path == Other.Path
1750         else:
1751             return self.Path == str(Other)
1752
1753     ## Override __cmp__ function
1754     #
1755     # Customize the comparsion operation of two PathClass
1756     #
1757     # @retval 0     The two PathClass are different
1758     # @retval -1    The first PathClass is less than the second PathClass
1759     # @retval 1     The first PathClass is Bigger than the second PathClass
1760     def __cmp__(self, Other):
1761         if isinstance(Other, type(self)):
1762             OtherKey = Other.Path
1763         else:
1764             OtherKey = str(Other)
1765
1766         SelfKey = self.Path
1767         if SelfKey == OtherKey:
1768             return 0
1769         elif SelfKey > OtherKey:
1770             return 1
1771         else:
1772             return -1
1773
1774     ## Override __hash__ function
1775     #
1776     # Use Path as key in hash table
1777     #
1778     # @retval string Key for hash table
1779     #
1780     def __hash__(self):
1781         return hash(self.Path)
1782
1783     @cached_property
1784     def Key(self):
1785         return self.Path.upper()
1786
1787     @property
1788     def TimeStamp(self):
1789         return os.stat(self.Path)[8]
1790
1791     def Validate(self, Type='', CaseSensitive=True):
1792         if GlobalData.gCaseInsensitive:
1793             CaseSensitive = False
1794         if Type and Type.lower() != self.Type:
1795             return FILE_TYPE_MISMATCH, '%s (expect %s but got %s)' % (self.File, Type, self.Type)
1796
1797         RealFile, RealRoot = RealPath2(self.File, self.Root, self.AlterRoot)
1798         if not RealRoot and not RealFile:
1799             RealFile = self.File
1800             if self.AlterRoot:
1801                 RealFile = os.path.join(self.AlterRoot, self.File)
1802             elif self.Root:
1803                 RealFile = os.path.join(self.Root, self.File)
1804             if len (mws.getPkgPath()) == 0:
1805                 return FILE_NOT_FOUND, os.path.join(self.AlterRoot, RealFile)
1806             else:
1807                 return FILE_NOT_FOUND, "%s is not found in packages path:\n\t%s" % (self.File, '\n\t'.join(mws.getPkgPath()))
1808
1809         ErrorCode = 0
1810         ErrorInfo = ''
1811         if RealRoot != self.Root or RealFile != self.File:
1812             if CaseSensitive and (RealFile != self.File or (RealRoot != self.Root and RealRoot != self.AlterRoot)):
1813                 ErrorCode = FILE_CASE_MISMATCH
1814                 ErrorInfo = self.File + '\n\t' + RealFile + " [in file system]"
1815
1816             self.SubDir, self.Name = os.path.split(RealFile)
1817             self.BaseName, self.Ext = os.path.splitext(self.Name)
1818             if self.SubDir:
1819                 self.Dir = os.path.join(RealRoot, self.SubDir)
1820             else:
1821                 self.Dir = RealRoot
1822             self.File = RealFile
1823             self.Root = RealRoot
1824             self.Path = os.path.join(RealRoot, RealFile)
1825         return ErrorCode, ErrorInfo
1826
1827 ## Parse PE image to get the required PE informaion.
1828 #
1829 class PeImageClass():
1830     ## Constructor
1831     #
1832     #   @param  File FilePath of PeImage
1833     #
1834     def __init__(self, PeFile):
1835         self.FileName   = PeFile
1836         self.IsValid    = False
1837         self.Size       = 0
1838         self.EntryPoint = 0
1839         self.SectionAlignment  = 0
1840         self.SectionHeaderList = []
1841         self.ErrorInfo = ''
1842         try:
1843             PeObject = open(PeFile, 'rb')
1844         except:
1845             self.ErrorInfo = self.FileName + ' can not be found\n'
1846             return
1847         # Read DOS header
1848         ByteArray = array.array('B')
1849         ByteArray.fromfile(PeObject, 0x3E)
1850         ByteList = ByteArray.tolist()
1851         # DOS signature should be 'MZ'
1852         if self._ByteListToStr (ByteList[0x0:0x2]) != 'MZ':
1853             self.ErrorInfo = self.FileName + ' has no valid DOS signature MZ'
1854             return
1855
1856         # Read 4 byte PE Signature
1857         PeOffset = self._ByteListToInt(ByteList[0x3C:0x3E])
1858         PeObject.seek(PeOffset)
1859         ByteArray = array.array('B')
1860         ByteArray.fromfile(PeObject, 4)
1861         # PE signature should be 'PE\0\0'
1862         if ByteArray.tostring() != b'PE\0\0':
1863             self.ErrorInfo = self.FileName + ' has no valid PE signature PE00'
1864             return
1865
1866         # Read PE file header
1867         ByteArray = array.array('B')
1868         ByteArray.fromfile(PeObject, 0x14)
1869         ByteList = ByteArray.tolist()
1870         SecNumber = self._ByteListToInt(ByteList[0x2:0x4])
1871         if SecNumber == 0:
1872             self.ErrorInfo = self.FileName + ' has no section header'
1873             return
1874
1875         # Read PE optional header
1876         OptionalHeaderSize = self._ByteListToInt(ByteArray[0x10:0x12])
1877         ByteArray = array.array('B')
1878         ByteArray.fromfile(PeObject, OptionalHeaderSize)
1879         ByteList = ByteArray.tolist()
1880         self.EntryPoint       = self._ByteListToInt(ByteList[0x10:0x14])
1881         self.SectionAlignment = self._ByteListToInt(ByteList[0x20:0x24])
1882         self.Size             = self._ByteListToInt(ByteList[0x38:0x3C])
1883
1884         # Read each Section Header
1885         for Index in range(SecNumber):
1886             ByteArray = array.array('B')
1887             ByteArray.fromfile(PeObject, 0x28)
1888             ByteList = ByteArray.tolist()
1889             SecName  = self._ByteListToStr(ByteList[0:8])
1890             SecVirtualSize = self._ByteListToInt(ByteList[8:12])
1891             SecRawAddress  = self._ByteListToInt(ByteList[20:24])
1892             SecVirtualAddress = self._ByteListToInt(ByteList[12:16])
1893             self.SectionHeaderList.append((SecName, SecVirtualAddress, SecRawAddress, SecVirtualSize))
1894         self.IsValid = True
1895         PeObject.close()
1896
1897     def _ByteListToStr(self, ByteList):
1898         String = ''
1899         for index in range(len(ByteList)):
1900             if ByteList[index] == 0:
1901                 break
1902             String += chr(ByteList[index])
1903         return String
1904
1905     def _ByteListToInt(self, ByteList):
1906         Value = 0
1907         for index in range(len(ByteList) - 1, -1, -1):
1908             Value = (Value << 8) | int(ByteList[index])
1909         return Value
1910
1911 class DefaultStore():
1912     def __init__(self, DefaultStores ):
1913
1914         self.DefaultStores = DefaultStores
1915     def DefaultStoreID(self, DefaultStoreName):
1916         for key, value in self.DefaultStores.items():
1917             if value == DefaultStoreName:
1918                 return key
1919         return None
1920     def GetDefaultDefault(self):
1921         if not self.DefaultStores or "0" in self.DefaultStores:
1922             return "0", TAB_DEFAULT_STORES_DEFAULT
1923         else:
1924             minvalue = min(int(value_str) for value_str in self.DefaultStores)
1925             return (str(minvalue), self.DefaultStores[str(minvalue)])
1926     def GetMin(self, DefaultSIdList):
1927         if not DefaultSIdList:
1928             return TAB_DEFAULT_STORES_DEFAULT
1929         storeidset = {storeid for storeid, storename in self.DefaultStores.values() if storename in DefaultSIdList}
1930         if not storeidset:
1931             return ""
1932         minid = min(storeidset )
1933         for sid, name in self.DefaultStores.values():
1934             if sid == minid:
1935                 return name
1936
1937 class SkuClass():
1938     DEFAULT = 0
1939     SINGLE = 1
1940     MULTIPLE =2
1941
1942     def __init__(self,SkuIdentifier='', SkuIds=None):
1943         if SkuIds is None:
1944             SkuIds = {}
1945
1946         for SkuName in SkuIds:
1947             SkuId = SkuIds[SkuName][0]
1948             skuid_num = int(SkuId, 16) if SkuId.upper().startswith("0X") else int(SkuId)
1949             if skuid_num > 0xFFFFFFFFFFFFFFFF:
1950                 EdkLogger.error("build", PARAMETER_INVALID,
1951                             ExtraData = "SKU-ID [%s] value %s exceeds the max value of UINT64"
1952                                       % (SkuName, SkuId))
1953
1954         self.AvailableSkuIds = OrderedDict()
1955         self.SkuIdSet = []
1956         self.SkuIdNumberSet = []
1957         self.SkuData = SkuIds
1958         self._SkuInherit = {}
1959         self._SkuIdentifier = SkuIdentifier
1960         if SkuIdentifier == '' or SkuIdentifier is None:
1961             self.SkuIdSet = ['DEFAULT']
1962             self.SkuIdNumberSet = ['0U']
1963         elif SkuIdentifier == 'ALL':
1964             self.SkuIdSet = list(SkuIds.keys())
1965             self.SkuIdNumberSet = [num[0].strip() + 'U' for num in SkuIds.values()]
1966         else:
1967             r = SkuIdentifier.split('|')
1968             self.SkuIdSet=[(r[k].strip()).upper() for k in range(len(r))]
1969             k = None
1970             try:
1971                 self.SkuIdNumberSet = [SkuIds[k][0].strip() + 'U' for k in self.SkuIdSet]
1972             except Exception:
1973                 EdkLogger.error("build", PARAMETER_INVALID,
1974                             ExtraData = "SKU-ID [%s] is not supported by the platform. [Valid SKU-ID: %s]"
1975                                       % (k, " | ".join(SkuIds.keys())))
1976         for each in self.SkuIdSet:
1977             if each in SkuIds:
1978                 self.AvailableSkuIds[each] = SkuIds[each][0]
1979             else:
1980                 EdkLogger.error("build", PARAMETER_INVALID,
1981                             ExtraData="SKU-ID [%s] is not supported by the platform. [Valid SKU-ID: %s]"
1982                                       % (each, " | ".join(SkuIds.keys())))
1983         if self.SkuUsageType != SkuClass.SINGLE:
1984             self.AvailableSkuIds.update({'DEFAULT':0, 'COMMON':0})
1985         if self.SkuIdSet:
1986             GlobalData.gSkuids = (self.SkuIdSet)
1987             if 'COMMON' in GlobalData.gSkuids:
1988                 GlobalData.gSkuids.remove('COMMON')
1989             if self.SkuUsageType == self.SINGLE:
1990                 if len(GlobalData.gSkuids) != 1:
1991                     if 'DEFAULT' in GlobalData.gSkuids:
1992                         GlobalData.gSkuids.remove('DEFAULT')
1993             if GlobalData.gSkuids:
1994                 GlobalData.gSkuids.sort()
1995
1996     def GetNextSkuId(self, skuname):
1997         if not self._SkuInherit:
1998             self._SkuInherit = {}
1999             for item in self.SkuData.values():
2000                 self._SkuInherit[item[1]]=item[2] if item[2] else "DEFAULT"
2001         return self._SkuInherit.get(skuname, "DEFAULT")
2002
2003     def GetSkuChain(self, sku):
2004         if sku == "DEFAULT":
2005             return ["DEFAULT"]
2006         skulist = [sku]
2007         nextsku = sku
2008         while True:
2009             nextsku = self.GetNextSkuId(nextsku)
2010             skulist.append(nextsku)
2011             if nextsku == "DEFAULT":
2012                 break
2013         skulist.reverse()
2014         return skulist
2015     def SkuOverrideOrder(self):
2016         skuorderset = []
2017         for skuname in self.SkuIdSet:
2018             skuorderset.append(self.GetSkuChain(skuname))
2019
2020         skuorder = []
2021         for index in range(max(len(item) for item in skuorderset)):
2022             for subset in skuorderset:
2023                 if index > len(subset)-1:
2024                     continue
2025                 if subset[index] in skuorder:
2026                     continue
2027                 skuorder.append(subset[index])
2028
2029         return skuorder
2030
2031     @property
2032     def SkuUsageType(self):
2033         if self._SkuIdentifier.upper() == "ALL":
2034             return SkuClass.MULTIPLE
2035
2036         if len(self.SkuIdSet) == 1:
2037             if self.SkuIdSet[0] == 'DEFAULT':
2038                 return SkuClass.DEFAULT
2039             return SkuClass.SINGLE
2040         if len(self.SkuIdSet) == 2 and 'DEFAULT' in self.SkuIdSet:
2041             return SkuClass.SINGLE
2042         return SkuClass.MULTIPLE
2043
2044     def DumpSkuIdArrary(self):
2045         if self.SkuUsageType == SkuClass.SINGLE:
2046             return "{0x0}"
2047         ArrayStrList = []
2048         for skuname in self.AvailableSkuIds:
2049             if skuname == "COMMON":
2050                 continue
2051             while skuname != "DEFAULT":
2052                 ArrayStrList.append(hex(int(self.AvailableSkuIds[skuname])))
2053                 skuname = self.GetNextSkuId(skuname)
2054             ArrayStrList.append("0x0")
2055         return "{{{myList}}}".format(myList=",".join(ArrayStrList))
2056
2057     @property
2058     def AvailableSkuIdSet(self):
2059         return self.AvailableSkuIds
2060
2061     @property
2062     def SystemSkuId(self):
2063         if self.SkuUsageType == SkuClass.SINGLE:
2064             if len(self.SkuIdSet) == 1:
2065                 return self.SkuIdSet[0]
2066             else:
2067                 return self.SkuIdSet[0] if self.SkuIdSet[0] != 'DEFAULT' else self.SkuIdSet[1]
2068         else:
2069             return 'DEFAULT'
2070
2071 #
2072 # Pack a registry format GUID
2073 #
2074 def PackRegistryFormatGuid(Guid):
2075     return PackGUID(Guid.split('-'))
2076
2077 ##  Get the integer value from string like "14U" or integer like 2
2078 #
2079 #   @param      Input   The object that may be either a integer value or a string
2080 #
2081 #   @retval     Value    The integer value that the input represents
2082 #
2083 def GetIntegerValue(Input):
2084     if isinstance(Input, int):
2085         return Input
2086     String = Input
2087     if String.endswith("U"):
2088         String = String[:-1]
2089     if String.endswith("ULL"):
2090         String = String[:-3]
2091     if String.endswith("LL"):
2092         String = String[:-2]
2093
2094     if String.startswith("0x") or String.startswith("0X"):
2095         return int(String, 16)
2096     elif String == '':
2097         return 0
2098     else:
2099         return int(String)
2100
2101 #
2102 # Pack a GUID (registry format) list into a buffer and return it
2103 #
2104 def PackGUID(Guid):
2105     return pack(PACK_PATTERN_GUID,
2106                 int(Guid[0], 16),
2107                 int(Guid[1], 16),
2108                 int(Guid[2], 16),
2109                 int(Guid[3][-4:-2], 16),
2110                 int(Guid[3][-2:], 16),
2111                 int(Guid[4][-12:-10], 16),
2112                 int(Guid[4][-10:-8], 16),
2113                 int(Guid[4][-8:-6], 16),
2114                 int(Guid[4][-6:-4], 16),
2115                 int(Guid[4][-4:-2], 16),
2116                 int(Guid[4][-2:], 16)
2117                 )
2118
2119 #
2120 # Pack a GUID (byte) list into a buffer and return it
2121 #
2122 def PackByteFormatGUID(Guid):
2123     return pack(PACK_PATTERN_GUID,
2124                 Guid[0],
2125                 Guid[1],
2126                 Guid[2],
2127                 Guid[3],
2128                 Guid[4],
2129                 Guid[5],
2130                 Guid[6],
2131                 Guid[7],
2132                 Guid[8],
2133                 Guid[9],
2134                 Guid[10],
2135                 )
2136
2137 ##
2138 #
2139 # This acts like the main() function for the script, unless it is 'import'ed into another
2140 # script.
2141 #
2142 if __name__ == '__main__':
2143     pass
2144