BaseTools/Source/Python/Common/Misc.py

   1 ## @file
   2 # Common routines used by all tools
   3 #
   4 # Copyright (c) 2007 - 2017, Intel Corporation. All rights reserved.<BR>
   5 # This program and the accompanying materials
   6 # are licensed and made available under the terms and conditions of the BSD License
   7 # which accompanies this distribution.  The full text of the license may be found at
   8 # http://opensource.org/licenses/bsd-license.php
   9 #
  10 # THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS,
  11 # WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED.
  12 #
  13
  14 ##
  15 # Import Modules
  16 #
  17 import Common.LongFilePathOs as os
  18 import sys
  19 import string
  20 import thread
  21 import threading
  22 import time
  23 import re
  24 import cPickle
  25 import array
  26 import shutil
  27 from struct import pack
  28 from UserDict import IterableUserDict
  29 from UserList import UserList
  30
  31 from Common import EdkLogger as EdkLogger
  32 from Common import GlobalData as GlobalData
  33 from DataType import *
  34 from BuildToolError import *
  35 from CommonDataClass.DataClass import *
  36 from Parsing import GetSplitValueList
  37 from Common.LongFilePathSupport import OpenLongFilePath as open
  38 from Common.MultipleWorkspace import MultipleWorkspace as mws
  39 import uuid
  40 from CommonDataClass.Exceptions import BadExpression
  41
  42 ## Regular expression used to find out place holders in string template
  43 gPlaceholderPattern = re.compile("\$\{([^$()\s]+)\}", re.MULTILINE | re.UNICODE)
  44
  45 ## Dictionary used to store file time stamp for quick re-access
  46 gFileTimeStampCache = {}    # {file path : file time stamp}
  47
  48 ## Dictionary used to store dependencies of files
  49 gDependencyDatabase = {}    # arch : {file path : [dependent files list]}
  50
  51 def GetVariableOffset(mapfilepath, efifilepath, varnames):
  52     """ Parse map file to get variable offset in current EFI file
  53     @param mapfilepath    Map file absolution path
  54     @param efifilepath:   EFI binary file full path
  55     @param varnames       iteratable container whose elements are variable names to be searched
  56
  57     @return List whos elements are tuple with variable name and raw offset
  58     """
  59     lines = []
  60     try:
  61         f = open(mapfilepath, 'r')
  62         lines = f.readlines()
  63         f.close()
  64     except:
  65         return None
  66
  67     if len(lines) == 0: return None
  68     firstline = lines[0].strip()
  69     if (firstline.startswith("Archive member included ") and
  70         firstline.endswith(" file (symbol)")):
  71         return _parseForGCC(lines, efifilepath, varnames)
  72     if firstline.startswith("# Path:"):
  73         return _parseForXcode(lines, efifilepath, varnames)
  74     return _parseGeneral(lines, efifilepath, varnames)
  75
  76 def _parseForXcode(lines, efifilepath, varnames):
  77     status = 0
  78     ret = []
  79     for index, line in enumerate(lines):
  80         line = line.strip()
  81         if status == 0 and line == "# Symbols:":
  82             status = 1
  83             continue
  84         if status == 1 and len(line) != 0:
  85             for varname in varnames:
  86                 if varname in line:
  87                     m = re.match('^([\da-fA-FxX]+)([\s\S]*)([_]*%s)$' % varname, line)
  88                     if m != None:
  89                         ret.append((varname, m.group(1)))
  90     return ret
  91
  92 def _parseForGCC(lines, efifilepath, varnames):
  93     """ Parse map file generated by GCC linker """
  94     status = 0
  95     sections = []
  96     varoffset = []
  97     for index, line in enumerate(lines):
  98         line = line.strip()
  99         # status machine transection
 100         if status == 0 and line == "Memory Configuration":
 101             status = 1
 102             continue
 103         elif status == 1 and line == 'Linker script and memory map':
 104             status = 2
 105             continue
 106         elif status ==2 and line == 'START GROUP':
 107             status = 3
 108             continue
 109
 110         # status handler
 111         if status == 3:
 112             m = re.match('^([\w_\.]+) +([\da-fA-Fx]+) +([\da-fA-Fx]+)$', line)
 113             if m != None:
 114                 sections.append(m.groups(0))
 115             for varname in varnames:
 116                 Str = ''
 117                 m = re.match("^.data.(%s)" % varname, line)
 118                 if m != None:
 119                     m = re.match(".data.(%s)$" % varname, line)
 120                     if m != None:
 121                         Str = lines[index + 1]
 122                     else:
 123                         Str = line[len(".data.%s" % varname):]
 124                     if Str:
 125                         m = re.match('^([\da-fA-Fx]+) +([\da-fA-Fx]+)', Str.strip())
 126                         if m != None:
 127                             varoffset.append((varname, int(m.groups(0)[0], 16) , int(sections[-1][1], 16), sections[-1][0]))
 128
 129     if not varoffset:
 130         return []
 131     # get section information from efi file
 132     efisecs = PeImageClass(efifilepath).SectionHeaderList
 133     if efisecs == None or len(efisecs) == 0:
 134         return []
 135     #redirection
 136     redirection = 0
 137     for efisec in efisecs:
 138         for section in sections:
 139             if section[0].strip() == efisec[0].strip() and section[0].strip() == '.text':
 140                 redirection = int(section[1], 16) - efisec[1]
 141
 142     ret = []
 143     for var in varoffset:
 144         for efisec in efisecs:
 145             if var[1] >= efisec[1] and var[1] < efisec[1]+efisec[3]:
 146                 ret.append((var[0], hex(efisec[2] + var[1] - efisec[1] - redirection)))
 147     return ret
 148
 149 def _parseGeneral(lines, efifilepath, varnames):
 150     status = 0    #0 - beginning of file; 1 - PE section definition; 2 - symbol table
 151     secs  = []    # key = section name
 152     varoffset = []
 153     secRe = re.compile('^([\da-fA-F]+):([\da-fA-F]+) +([\da-fA-F]+)[Hh]? +([.\w\$]+) +(\w+)', re.UNICODE)
 154     symRe = re.compile('^([\da-fA-F]+):([\da-fA-F]+) +([\.:\\\\\w\?@\$]+) +([\da-fA-F]+)', re.UNICODE)
 155
 156     for line in lines:
 157         line = line.strip()
 158         if re.match("^Start[' ']+Length[' ']+Name[' ']+Class", line):
 159             status = 1
 160             continue
 161         if re.match("^Address[' ']+Publics by Value[' ']+Rva\+Base", line):
 162             status = 2
 163             continue
 164         if re.match("^entry point at", line):
 165             status = 3
 166             continue
 167         if status == 1 and len(line) != 0:
 168             m =  secRe.match(line)
 169             assert m != None, "Fail to parse the section in map file , line is %s" % line
 170             sec_no, sec_start, sec_length, sec_name, sec_class = m.groups(0)
 171             secs.append([int(sec_no, 16), int(sec_start, 16), int(sec_length, 16), sec_name, sec_class])
 172         if status == 2 and len(line) != 0:
 173             for varname in varnames:
 174                 m = symRe.match(line)
 175                 assert m != None, "Fail to parse the symbol in map file, line is %s" % line
 176                 sec_no, sym_offset, sym_name, vir_addr = m.groups(0)
 177                 sec_no     = int(sec_no,     16)
 178                 sym_offset = int(sym_offset, 16)
 179                 vir_addr   = int(vir_addr,   16)
 180                 m2 = re.match('^[_]*(%s)' % varname, sym_name)
 181                 if m2 != None:
 182                     # fond a binary pcd entry in map file
 183                     for sec in secs:
 184                         if sec[0] == sec_no and (sym_offset >= sec[1] and sym_offset < sec[1] + sec[2]):
 185                             varoffset.append([varname, sec[3], sym_offset, vir_addr, sec_no])
 186
 187     if not varoffset: return []
 188
 189     # get section information from efi file
 190     efisecs = PeImageClass(efifilepath).SectionHeaderList
 191     if efisecs == None or len(efisecs) == 0:
 192         return []
 193
 194     ret = []
 195     for var in varoffset:
 196         index = 0
 197         for efisec in efisecs:
 198             index = index + 1
 199             if var[1].strip() == efisec[0].strip():
 200                 ret.append((var[0], hex(efisec[2] + var[2])))
 201             elif var[4] == index:
 202                 ret.append((var[0], hex(efisec[2] + var[2])))
 203
 204     return ret
 205
 206 ## Routine to process duplicated INF
 207 #
 208 #  This function is called by following two cases:
 209 #  Case 1 in DSC:
 210 #    [components.arch]
 211 #    Pkg/module/module.inf
 212 #    Pkg/module/module.inf {
 213 #      <Defines>
 214 #        FILE_GUID = 0D1B936F-68F3-4589-AFCC-FB8B7AEBC836
 215 #    }
 216 #  Case 2 in FDF:
 217 #    INF Pkg/module/module.inf
 218 #    INF FILE_GUID = 0D1B936F-68F3-4589-AFCC-FB8B7AEBC836 Pkg/module/module.inf
 219 #
 220 #  This function copies Pkg/module/module.inf to
 221 #  Conf/.cache/0D1B936F-68F3-4589-AFCC-FB8B7AEBC836module.inf
 222 #
 223 #  @param Path     Original PathClass object
 224 #  @param BaseName New file base name
 225 #
 226 #  @retval         return the new PathClass object
 227 #
 228 def ProcessDuplicatedInf(Path, BaseName, Workspace):
 229     Filename = os.path.split(Path.File)[1]
 230     if '.' in Filename:
 231         Filename = BaseName + Path.BaseName + Filename[Filename.rfind('.'):]
 232     else:
 233         Filename = BaseName + Path.BaseName
 234
 235     #
 236     # If -N is specified on command line, cache is disabled
 237     # The directory has to be created
 238     #
 239     DbDir = os.path.split(GlobalData.gDatabasePath)[0]
 240     if not os.path.exists(DbDir):
 241         os.makedirs(DbDir)
 242     #
 243     # A temporary INF is copied to database path which must have write permission
 244     # The temporary will be removed at the end of build
 245     # In case of name conflict, the file name is
 246     # FILE_GUIDBaseName (0D1B936F-68F3-4589-AFCC-FB8B7AEBC836module.inf)
 247     #
 248     TempFullPath = os.path.join(DbDir,
 249                                 Filename)
 250     RtPath = PathClass(Path.File, Workspace)
 251     #
 252     # Modify the full path to temporary path, keep other unchanged
 253     #
 254     # To build same module more than once, the module path with FILE_GUID overridden has
 255     # the file name FILE_GUIDmodule.inf, but the relative path (self.MetaFile.File) is the real path
 256     # in DSC which is used as relative path by C files and other files in INF.
 257     # A trick was used: all module paths are PathClass instances, after the initialization
 258     # of PathClass, the PathClass.Path is overridden by the temporary INF path.
 259     #
 260     # The reason for creating a temporary INF is:
 261     # Platform.Modules which is the base to create ModuleAutoGen objects is a dictionary,
 262     # the key is the full path of INF, the value is an object to save overridden library instances, PCDs.
 263     # A different key for the same module is needed to create different output directory,
 264     # retrieve overridden PCDs, library instances.
 265     #
 266     # The BaseName is the FILE_GUID which is also the output directory name.
 267     #
 268     #
 269     RtPath.Path = TempFullPath
 270     RtPath.BaseName = BaseName
 271     #
 272     # If file exists, compare contents
 273     #
 274     if os.path.exists(TempFullPath):
 275         with open(str(Path), 'rb') as f1: Src = f1.read()
 276         with open(TempFullPath, 'rb') as f2: Dst = f2.read()
 277         if Src == Dst:
 278             return RtPath
 279     GlobalData.gTempInfs.append(TempFullPath)
 280     shutil.copy2(str(Path), TempFullPath)
 281     return RtPath
 282
 283 ## Remove temporary created INFs whose paths were saved in gTempInfs
 284 #
 285 def ClearDuplicatedInf():
 286     for File in GlobalData.gTempInfs:
 287         if os.path.exists(File):
 288             os.remove(File)
 289
 290 ## callback routine for processing variable option
 291 #
 292 # This function can be used to process variable number of option values. The
 293 # typical usage of it is specify architecure list on command line.
 294 # (e.g. <tool> -a IA32 X64 IPF)
 295 #
 296 # @param  Option        Standard callback function parameter
 297 # @param  OptionString  Standard callback function parameter
 298 # @param  Value         Standard callback function parameter
 299 # @param  Parser        Standard callback function parameter
 300 #
 301 # @retval
 302 #
 303 def ProcessVariableArgument(Option, OptionString, Value, Parser):
 304     assert Value is None
 305     Value = []
 306     RawArgs = Parser.rargs
 307     while RawArgs:
 308         Arg = RawArgs[0]
 309         if (Arg[:2] == "--" and len(Arg) > 2) or \
 310            (Arg[:1] == "-" and len(Arg) > 1 and Arg[1] != "-"):
 311             break
 312         Value.append(Arg)
 313         del RawArgs[0]
 314     setattr(Parser.values, Option.dest, Value)
 315
 316 ## Convert GUID string in xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx style to C structure style
 317 #
 318 #   @param      Guid    The GUID string
 319 #
 320 #   @retval     string  The GUID string in C structure style
 321 #
 322 def GuidStringToGuidStructureString(Guid):
 323     GuidList = Guid.split('-')
 324     Result = '{'
 325     for Index in range(0, 3, 1):
 326         Result = Result + '0x' + GuidList[Index] + ', '
 327     Result = Result + '{0x' + GuidList[3][0:2] + ', 0x' + GuidList[3][2:4]
 328     for Index in range(0, 12, 2):
 329         Result = Result + ', 0x' + GuidList[4][Index:Index + 2]
 330     Result += '}}'
 331     return Result
 332
 333 ## Convert GUID structure in byte array to xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
 334 #
 335 #   @param      GuidValue   The GUID value in byte array
 336 #
 337 #   @retval     string      The GUID value in xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx format
 338 #
 339 def GuidStructureByteArrayToGuidString(GuidValue):
 340     guidValueString = GuidValue.lower().replace("{", "").replace("}", "").replace(" ", "").replace(";", "")
 341     guidValueList = guidValueString.split(",")
 342     if len(guidValueList) != 16:
 343         return ''
 344         #EdkLogger.error(None, None, "Invalid GUID value string %s" % GuidValue)
 345     try:
 346         return "%02x%02x%02x%02x-%02x%02x-%02x%02x-%02x%02x-%02x%02x%02x%02x%02x%02x" % (
 347                 int(guidValueList[3], 16),
 348                 int(guidValueList[2], 16),
 349                 int(guidValueList[1], 16),
 350                 int(guidValueList[0], 16),
 351                 int(guidValueList[5], 16),
 352                 int(guidValueList[4], 16),
 353                 int(guidValueList[7], 16),
 354                 int(guidValueList[6], 16),
 355                 int(guidValueList[8], 16),
 356                 int(guidValueList[9], 16),
 357                 int(guidValueList[10], 16),
 358                 int(guidValueList[11], 16),
 359                 int(guidValueList[12], 16),
 360                 int(guidValueList[13], 16),
 361                 int(guidValueList[14], 16),
 362                 int(guidValueList[15], 16)
 363                 )
 364     except:
 365         return ''
 366
 367 ## Convert GUID string in C structure style to xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
 368 #
 369 #   @param      GuidValue   The GUID value in C structure format
 370 #
 371 #   @retval     string      The GUID value in xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx format
 372 #
 373 def GuidStructureStringToGuidString(GuidValue):
 374     guidValueString = GuidValue.lower().replace("{", "").replace("}", "").replace(" ", "").replace(";", "")
 375     guidValueList = guidValueString.split(",")
 376     if len(guidValueList) != 11:
 377         return ''
 378         #EdkLogger.error(None, None, "Invalid GUID value string %s" % GuidValue)
 379     try:
 380         return "%08x-%04x-%04x-%02x%02x-%02x%02x%02x%02x%02x%02x" % (
 381                 int(guidValueList[0], 16),
 382                 int(guidValueList[1], 16),
 383                 int(guidValueList[2], 16),
 384                 int(guidValueList[3], 16),
 385                 int(guidValueList[4], 16),
 386                 int(guidValueList[5], 16),
 387                 int(guidValueList[6], 16),
 388                 int(guidValueList[7], 16),
 389                 int(guidValueList[8], 16),
 390                 int(guidValueList[9], 16),
 391                 int(guidValueList[10], 16)
 392                 )
 393     except:
 394         return ''
 395
 396 ## Convert GUID string in C structure style to xxxxxxxx_xxxx_xxxx_xxxx_xxxxxxxxxxxx
 397 #
 398 #   @param      GuidValue   The GUID value in C structure format
 399 #
 400 #   @retval     string      The GUID value in xxxxxxxx_xxxx_xxxx_xxxx_xxxxxxxxxxxx format
 401 #
 402 def GuidStructureStringToGuidValueName(GuidValue):
 403     guidValueString = GuidValue.lower().replace("{", "").replace("}", "").replace(" ", "")
 404     guidValueList = guidValueString.split(",")
 405     if len(guidValueList) != 11:
 406         EdkLogger.error(None, FORMAT_INVALID, "Invalid GUID value string [%s]" % GuidValue)
 407     return "%08x_%04x_%04x_%02x%02x_%02x%02x%02x%02x%02x%02x" % (
 408             int(guidValueList[0], 16),
 409             int(guidValueList[1], 16),
 410             int(guidValueList[2], 16),
 411             int(guidValueList[3], 16),
 412             int(guidValueList[4], 16),
 413             int(guidValueList[5], 16),
 414             int(guidValueList[6], 16),
 415             int(guidValueList[7], 16),
 416             int(guidValueList[8], 16),
 417             int(guidValueList[9], 16),
 418             int(guidValueList[10], 16)
 419             )
 420
 421 ## Create directories
 422 #
 423 #   @param      Directory   The directory name
 424 #
 425 def CreateDirectory(Directory):
 426     if Directory == None or Directory.strip() == "":
 427         return True
 428     try:
 429         if not os.access(Directory, os.F_OK):
 430             os.makedirs(Directory)
 431     except:
 432         return False
 433     return True
 434
 435 ## Remove directories, including files and sub-directories in it
 436 #
 437 #   @param      Directory   The directory name
 438 #
 439 def RemoveDirectory(Directory, Recursively=False):
 440     if Directory == None or Directory.strip() == "" or not os.path.exists(Directory):
 441         return
 442     if Recursively:
 443         CurrentDirectory = os.getcwd()
 444         os.chdir(Directory)
 445         for File in os.listdir("."):
 446             if os.path.isdir(File):
 447                 RemoveDirectory(File, Recursively)
 448             else:
 449                 os.remove(File)
 450         os.chdir(CurrentDirectory)
 451     os.rmdir(Directory)
 452
 453 ## Check if given file is changed or not
 454 #
 455 #  This method is used to check if a file is changed or not between two build
 456 #  actions. It makes use a cache to store files timestamp.
 457 #
 458 #   @param      File    The path of file
 459 #
 460 #   @retval     True    If the given file is changed, doesn't exist, or can't be
 461 #                       found in timestamp cache
 462 #   @retval     False   If the given file is changed
 463 #
 464 def IsChanged(File):
 465     if not os.path.exists(File):
 466         return True
 467
 468     FileState = os.stat(File)
 469     TimeStamp = FileState[-2]
 470
 471     if File in gFileTimeStampCache and TimeStamp == gFileTimeStampCache[File]:
 472         FileChanged = False
 473     else:
 474         FileChanged = True
 475         gFileTimeStampCache[File] = TimeStamp
 476
 477     return FileChanged
 478
 479 ## Store content in file
 480 #
 481 #  This method is used to save file only when its content is changed. This is
 482 #  quite useful for "make" system to decide what will be re-built and what won't.
 483 #
 484 #   @param      File            The path of file
 485 #   @param      Content         The new content of the file
 486 #   @param      IsBinaryFile    The flag indicating if the file is binary file or not
 487 #
 488 #   @retval     True            If the file content is changed and the file is renewed
 489 #   @retval     False           If the file content is the same
 490 #
 491 def SaveFileOnChange(File, Content, IsBinaryFile=True):
 492     if not IsBinaryFile:
 493         Content = Content.replace("\n", os.linesep)
 494
 495     if os.path.exists(File):
 496         try:
 497             if Content == open(File, "rb").read():
 498                 return False
 499         except:
 500             EdkLogger.error(None, FILE_OPEN_FAILURE, ExtraData=File)
 501
 502     DirName = os.path.dirname(File)
 503     if not CreateDirectory(DirName):
 504         EdkLogger.error(None, FILE_CREATE_FAILURE, "Could not create directory %s" % DirName)
 505     else:
 506         if DirName == '':
 507             DirName = os.getcwd()
 508         if not os.access(DirName, os.W_OK):
 509             EdkLogger.error(None, PERMISSION_FAILURE, "Do not have write permission on directory %s" % DirName)
 510
 511     try:
 512         if GlobalData.gIsWindows:
 513             try:
 514                 from PyUtility import SaveFileToDisk
 515                 if not SaveFileToDisk(File, Content):
 516                     EdkLogger.error(None, FILE_CREATE_FAILURE, ExtraData=File)
 517             except:
 518                 Fd = open(File, "wb")
 519                 Fd.write(Content)
 520                 Fd.close()
 521         else:
 522             Fd = open(File, "wb")
 523             Fd.write(Content)
 524             Fd.close()
 525     except IOError, X:
 526         EdkLogger.error(None, FILE_CREATE_FAILURE, ExtraData='IOError %s' % X)
 527
 528     return True
 529
 530 ## Make a Python object persistent on file system
 531 #
 532 #   @param      Data    The object to be stored in file
 533 #   @param      File    The path of file to store the object
 534 #
 535 def DataDump(Data, File):
 536     Fd = None
 537     try:
 538         Fd = open(File, 'wb')
 539         cPickle.dump(Data, Fd, cPickle.HIGHEST_PROTOCOL)
 540     except:
 541         EdkLogger.error("", FILE_OPEN_FAILURE, ExtraData=File, RaiseError=False)
 542     finally:
 543         if Fd != None:
 544             Fd.close()
 545
 546 ## Restore a Python object from a file
 547 #
 548 #   @param      File    The path of file stored the object
 549 #
 550 #   @retval     object  A python object
 551 #   @retval     None    If failure in file operation
 552 #
 553 def DataRestore(File):
 554     Data = None
 555     Fd = None
 556     try:
 557         Fd = open(File, 'rb')
 558         Data = cPickle.load(Fd)
 559     except Exception, e:
 560         EdkLogger.verbose("Failed to load [%s]\n\t%s" % (File, str(e)))
 561         Data = None
 562     finally:
 563         if Fd != None:
 564             Fd.close()
 565     return Data
 566
 567 ## Retrieve and cache the real path name in file system
 568 #
 569 #   @param      Root    The root directory of path relative to
 570 #
 571 #   @retval     str     The path string if the path exists
 572 #   @retval     None    If path doesn't exist
 573 #
 574 class DirCache:
 575     _CACHE_ = set()
 576     _UPPER_CACHE_ = {}
 577
 578     def __init__(self, Root):
 579         self._Root = Root
 580         for F in os.listdir(Root):
 581             self._CACHE_.add(F)
 582             self._UPPER_CACHE_[F.upper()] = F
 583
 584     # =[] operator
 585     def __getitem__(self, Path):
 586         Path = Path[len(os.path.commonprefix([Path, self._Root])):]
 587         if not Path:
 588             return self._Root
 589         if Path and Path[0] == os.path.sep:
 590             Path = Path[1:]
 591         if Path in self._CACHE_:
 592             return os.path.join(self._Root, Path)
 593         UpperPath = Path.upper()
 594         if UpperPath in self._UPPER_CACHE_:
 595             return os.path.join(self._Root, self._UPPER_CACHE_[UpperPath])
 596
 597         IndexList = []
 598         LastSepIndex = -1
 599         SepIndex = Path.find(os.path.sep)
 600         while SepIndex > -1:
 601             Parent = UpperPath[:SepIndex]
 602             if Parent not in self._UPPER_CACHE_:
 603                 break
 604             LastSepIndex = SepIndex
 605             SepIndex = Path.find(os.path.sep, LastSepIndex + 1)
 606
 607         if LastSepIndex == -1:
 608             return None
 609
 610         Cwd = os.getcwd()
 611         os.chdir(self._Root)
 612         SepIndex = LastSepIndex
 613         while SepIndex > -1:
 614             Parent = Path[:SepIndex]
 615             ParentKey = UpperPath[:SepIndex]
 616             if ParentKey not in self._UPPER_CACHE_:
 617                 os.chdir(Cwd)
 618                 return None
 619
 620             if Parent in self._CACHE_:
 621                 ParentDir = Parent
 622             else:
 623                 ParentDir = self._UPPER_CACHE_[ParentKey]
 624             for F in os.listdir(ParentDir):
 625                 Dir = os.path.join(ParentDir, F)
 626                 self._CACHE_.add(Dir)
 627                 self._UPPER_CACHE_[Dir.upper()] = Dir
 628
 629             SepIndex = Path.find(os.path.sep, SepIndex + 1)
 630
 631         os.chdir(Cwd)
 632         if Path in self._CACHE_:
 633             return os.path.join(self._Root, Path)
 634         elif UpperPath in self._UPPER_CACHE_:
 635             return os.path.join(self._Root, self._UPPER_CACHE_[UpperPath])
 636         return None
 637
 638 ## Get all files of a directory
 639 #
 640 # @param Root:       Root dir
 641 # @param SkipList :  The files need be skipped
 642 #
 643 # @retval  A list of all files
 644 #
 645 def GetFiles(Root, SkipList=None, FullPath=True):
 646     OriPath = Root
 647     FileList = []
 648     for Root, Dirs, Files in os.walk(Root):
 649         if SkipList:
 650             for Item in SkipList:
 651                 if Item in Dirs:
 652                     Dirs.remove(Item)
 653
 654         for File in Files:
 655             File = os.path.normpath(os.path.join(Root, File))
 656             if not FullPath:
 657                 File = File[len(OriPath) + 1:]
 658             FileList.append(File)
 659
 660     return FileList
 661
 662 ## Check if gvien file exists or not
 663 #
 664 #   @param      File    File name or path to be checked
 665 #   @param      Dir     The directory the file is relative to
 666 #
 667 #   @retval     True    if file exists
 668 #   @retval     False   if file doesn't exists
 669 #
 670 def ValidFile(File, Ext=None):
 671     if Ext != None:
 672         Dummy, FileExt = os.path.splitext(File)
 673         if FileExt.lower() != Ext.lower():
 674             return False
 675     if not os.path.exists(File):
 676         return False
 677     return True
 678
 679 def RealPath(File, Dir='', OverrideDir=''):
 680     NewFile = os.path.normpath(os.path.join(Dir, File))
 681     NewFile = GlobalData.gAllFiles[NewFile]
 682     if not NewFile and OverrideDir:
 683         NewFile = os.path.normpath(os.path.join(OverrideDir, File))
 684         NewFile = GlobalData.gAllFiles[NewFile]
 685     return NewFile
 686
 687 def RealPath2(File, Dir='', OverrideDir=''):
 688     NewFile = None
 689     if OverrideDir:
 690         NewFile = GlobalData.gAllFiles[os.path.normpath(os.path.join(OverrideDir, File))]
 691         if NewFile:
 692             if OverrideDir[-1] == os.path.sep:
 693                 return NewFile[len(OverrideDir):], NewFile[0:len(OverrideDir)]
 694             else:
 695                 return NewFile[len(OverrideDir) + 1:], NewFile[0:len(OverrideDir)]
 696     if GlobalData.gAllFiles:
 697         NewFile = GlobalData.gAllFiles[os.path.normpath(os.path.join(Dir, File))]
 698     if not NewFile:
 699         NewFile = os.path.normpath(os.path.join(Dir, File))
 700         if not os.path.exists(NewFile):
 701             return None, None
 702     if NewFile:
 703         if Dir:
 704             if Dir[-1] == os.path.sep:
 705                 return NewFile[len(Dir):], NewFile[0:len(Dir)]
 706             else:
 707                 return NewFile[len(Dir) + 1:], NewFile[0:len(Dir)]
 708         else:
 709             return NewFile, ''
 710
 711     return None, None
 712
 713 ## Check if gvien file exists or not
 714 #
 715 #
 716 def ValidFile2(AllFiles, File, Ext=None, Workspace='', EfiSource='', EdkSource='', Dir='.', OverrideDir=''):
 717     NewFile = File
 718     if Ext != None:
 719         Dummy, FileExt = os.path.splitext(File)
 720         if FileExt.lower() != Ext.lower():
 721             return False, File
 722
 723     # Replace the Edk macros
 724     if OverrideDir != '' and OverrideDir != None:
 725         if OverrideDir.find('$(EFI_SOURCE)') > -1:
 726             OverrideDir = OverrideDir.replace('$(EFI_SOURCE)', EfiSource)
 727         if OverrideDir.find('$(EDK_SOURCE)') > -1:
 728             OverrideDir = OverrideDir.replace('$(EDK_SOURCE)', EdkSource)
 729
 730     # Replace the default dir to current dir
 731     if Dir == '.':
 732         Dir = os.getcwd()
 733         Dir = Dir[len(Workspace) + 1:]
 734
 735     # First check if File has Edk definition itself
 736     if File.find('$(EFI_SOURCE)') > -1 or File.find('$(EDK_SOURCE)') > -1:
 737         NewFile = File.replace('$(EFI_SOURCE)', EfiSource)
 738         NewFile = NewFile.replace('$(EDK_SOURCE)', EdkSource)
 739         NewFile = AllFiles[os.path.normpath(NewFile)]
 740         if NewFile != None:
 741             return True, NewFile
 742
 743     # Second check the path with override value
 744     if OverrideDir != '' and OverrideDir != None:
 745         NewFile = AllFiles[os.path.normpath(os.path.join(OverrideDir, File))]
 746         if NewFile != None:
 747             return True, NewFile
 748
 749     # Last check the path with normal definitions
 750     File = os.path.join(Dir, File)
 751     NewFile = AllFiles[os.path.normpath(File)]
 752     if NewFile != None:
 753         return True, NewFile
 754
 755     return False, File
 756
 757 ## Check if gvien file exists or not
 758 #
 759 #
 760 def ValidFile3(AllFiles, File, Workspace='', EfiSource='', EdkSource='', Dir='.', OverrideDir=''):
 761     # Replace the Edk macros
 762     if OverrideDir != '' and OverrideDir != None:
 763         if OverrideDir.find('$(EFI_SOURCE)') > -1:
 764             OverrideDir = OverrideDir.replace('$(EFI_SOURCE)', EfiSource)
 765         if OverrideDir.find('$(EDK_SOURCE)') > -1:
 766             OverrideDir = OverrideDir.replace('$(EDK_SOURCE)', EdkSource)
 767
 768     # Replace the default dir to current dir
 769     # Dir is current module dir related to workspace
 770     if Dir == '.':
 771         Dir = os.getcwd()
 772         Dir = Dir[len(Workspace) + 1:]
 773
 774     NewFile = File
 775     RelaPath = AllFiles[os.path.normpath(Dir)]
 776     NewRelaPath = RelaPath
 777
 778     while(True):
 779         # First check if File has Edk definition itself
 780         if File.find('$(EFI_SOURCE)') > -1 or File.find('$(EDK_SOURCE)') > -1:
 781             File = File.replace('$(EFI_SOURCE)', EfiSource)
 782             File = File.replace('$(EDK_SOURCE)', EdkSource)
 783             NewFile = AllFiles[os.path.normpath(File)]
 784             if NewFile != None:
 785                 NewRelaPath = os.path.dirname(NewFile)
 786                 File = os.path.basename(NewFile)
 787                 #NewRelaPath = NewFile[:len(NewFile) - len(File.replace("..\\", '').replace("../", '')) - 1]
 788                 break
 789
 790         # Second check the path with override value
 791         if OverrideDir != '' and OverrideDir != None:
 792             NewFile = AllFiles[os.path.normpath(os.path.join(OverrideDir, File))]
 793             if NewFile != None:
 794                 #NewRelaPath = os.path.dirname(NewFile)
 795                 NewRelaPath = NewFile[:len(NewFile) - len(File.replace("..\\", '').replace("../", '')) - 1]
 796                 break
 797
 798         # Last check the path with normal definitions
 799         NewFile = AllFiles[os.path.normpath(os.path.join(Dir, File))]
 800         if NewFile != None:
 801             break
 802
 803         # No file found
 804         break
 805
 806     return NewRelaPath, RelaPath, File
 807
 808
 809 def GetRelPath(Path1, Path2):
 810     FileName = os.path.basename(Path2)
 811     L1 = os.path.normpath(Path1).split(os.path.normpath('/'))
 812     L2 = os.path.normpath(Path2).split(os.path.normpath('/'))
 813     for Index in range(0, len(L1)):
 814         if L1[Index] != L2[Index]:
 815             FileName = '../' * (len(L1) - Index)
 816             for Index2 in range(Index, len(L2)):
 817                 FileName = os.path.join(FileName, L2[Index2])
 818             break
 819     return os.path.normpath(FileName)
 820
 821
 822 ## Get GUID value from given packages
 823 #
 824 #   @param      CName           The CName of the GUID
 825 #   @param      PackageList     List of packages looking-up in
 826 #   @param      Inffile         The driver file
 827 #
 828 #   @retval     GuidValue   if the CName is found in any given package
 829 #   @retval     None        if the CName is not found in all given packages
 830 #
 831 def GuidValue(CName, PackageList, Inffile = None):
 832     for P in PackageList:
 833         GuidKeys = P.Guids.keys()
 834         if Inffile and P._PrivateGuids:
 835             if not Inffile.startswith(P.MetaFile.Dir):
 836                 GuidKeys = (dict.fromkeys(x for x in P.Guids if x not in P._PrivateGuids)).keys()
 837         if CName in GuidKeys:
 838             return P.Guids[CName]
 839     return None
 840
 841 ## Get Protocol value from given packages
 842 #
 843 #   @param      CName           The CName of the GUID
 844 #   @param      PackageList     List of packages looking-up in
 845 #   @param      Inffile         The driver file
 846 #
 847 #   @retval     GuidValue   if the CName is found in any given package
 848 #   @retval     None        if the CName is not found in all given packages
 849 #
 850 def ProtocolValue(CName, PackageList, Inffile = None):
 851     for P in PackageList:
 852         ProtocolKeys = P.Protocols.keys()
 853         if Inffile and P._PrivateProtocols:
 854             if not Inffile.startswith(P.MetaFile.Dir):
 855                 ProtocolKeys = (dict.fromkeys(x for x in P.Protocols if x not in P._PrivateProtocols)).keys()
 856         if CName in ProtocolKeys:
 857             return P.Protocols[CName]
 858     return None
 859
 860 ## Get PPI value from given packages
 861 #
 862 #   @param      CName           The CName of the GUID
 863 #   @param      PackageList     List of packages looking-up in
 864 #   @param      Inffile         The driver file
 865 #
 866 #   @retval     GuidValue   if the CName is found in any given package
 867 #   @retval     None        if the CName is not found in all given packages
 868 #
 869 def PpiValue(CName, PackageList, Inffile = None):
 870     for P in PackageList:
 871         PpiKeys = P.Ppis.keys()
 872         if Inffile and P._PrivatePpis:
 873             if not Inffile.startswith(P.MetaFile.Dir):
 874                 PpiKeys = (dict.fromkeys(x for x in P.Ppis if x not in P._PrivatePpis)).keys()
 875         if CName in PpiKeys:
 876             return P.Ppis[CName]
 877     return None
 878
 879 ## A string template class
 880 #
 881 #  This class implements a template for string replacement. A string template
 882 #  looks like following
 883 #
 884 #       ${BEGIN} other_string ${placeholder_name} other_string ${END}
 885 #
 886 #  The string between ${BEGIN} and ${END} will be repeated as many times as the
 887 #  length of "placeholder_name", which is a list passed through a dict. The
 888 #  "placeholder_name" is the key name of the dict. The ${BEGIN} and ${END} can
 889 #  be not used and, in this case, the "placeholder_name" must not a list and it
 890 #  will just be replaced once.
 891 #
 892 class TemplateString(object):
 893     _REPEAT_START_FLAG = "BEGIN"
 894     _REPEAT_END_FLAG = "END"
 895
 896     class Section(object):
 897         _LIST_TYPES = [type([]), type(set()), type((0,))]
 898
 899         def __init__(self, TemplateSection, PlaceHolderList):
 900             self._Template = TemplateSection
 901             self._PlaceHolderList = []
 902
 903             # Split the section into sub-sections according to the position of placeholders
 904             if PlaceHolderList:
 905                 self._SubSectionList = []
 906                 SubSectionStart = 0
 907                 #
 908                 # The placeholders passed in must be in the format of
 909                 #
 910                 #   PlaceHolderName, PlaceHolderStartPoint, PlaceHolderEndPoint
 911                 #
 912                 for PlaceHolder, Start, End in PlaceHolderList:
 913                     self._SubSectionList.append(TemplateSection[SubSectionStart:Start])
 914                     self._SubSectionList.append(TemplateSection[Start:End])
 915                     self._PlaceHolderList.append(PlaceHolder)
 916                     SubSectionStart = End
 917                 if SubSectionStart < len(TemplateSection):
 918                     self._SubSectionList.append(TemplateSection[SubSectionStart:])
 919             else:
 920                 self._SubSectionList = [TemplateSection]
 921
 922         def __str__(self):
 923             return self._Template + " : " + str(self._PlaceHolderList)
 924
 925         def Instantiate(self, PlaceHolderValues):
 926             RepeatTime = -1
 927             RepeatPlaceHolders = {}
 928             NonRepeatPlaceHolders = {}
 929
 930             for PlaceHolder in self._PlaceHolderList:
 931                 if PlaceHolder not in PlaceHolderValues:
 932                     continue
 933                 Value = PlaceHolderValues[PlaceHolder]
 934                 if type(Value) in self._LIST_TYPES:
 935                     if RepeatTime < 0:
 936                         RepeatTime = len(Value)
 937                     elif RepeatTime != len(Value):
 938                         EdkLogger.error(
 939                                     "TemplateString",
 940                                     PARAMETER_INVALID,
 941                                     "${%s} has different repeat time from others!" % PlaceHolder,
 942                                     ExtraData=str(self._Template)
 943                                     )
 944                     RepeatPlaceHolders["${%s}" % PlaceHolder] = Value
 945                 else:
 946                     NonRepeatPlaceHolders["${%s}" % PlaceHolder] = Value
 947
 948             if NonRepeatPlaceHolders:
 949                 StringList = []
 950                 for S in self._SubSectionList:
 951                     if S not in NonRepeatPlaceHolders:
 952                         StringList.append(S)
 953                     else:
 954                         StringList.append(str(NonRepeatPlaceHolders[S]))
 955             else:
 956                 StringList = self._SubSectionList
 957
 958             if RepeatPlaceHolders:
 959                 TempStringList = []
 960                 for Index in range(RepeatTime):
 961                     for S in StringList:
 962                         if S not in RepeatPlaceHolders:
 963                             TempStringList.append(S)
 964                         else:
 965                             TempStringList.append(str(RepeatPlaceHolders[S][Index]))
 966                 StringList = TempStringList
 967
 968             return "".join(StringList)
 969
 970     ## Constructor
 971     def __init__(self, Template=None):
 972         self.String = ''
 973         self.IsBinary = False
 974         self._Template = Template
 975         self._TemplateSectionList = self._Parse(Template)
 976
 977     ## str() operator
 978     #
 979     #   @retval     string  The string replaced
 980     #
 981     def __str__(self):
 982         return self.String
 983
 984     ## Split the template string into fragments per the ${BEGIN} and ${END} flags
 985     #
 986     #   @retval     list    A list of TemplateString.Section objects
 987     #
 988     def _Parse(self, Template):
 989         SectionStart = 0
 990         SearchFrom = 0
 991         MatchEnd = 0
 992         PlaceHolderList = []
 993         TemplateSectionList = []
 994         while Template:
 995             MatchObj = gPlaceholderPattern.search(Template, SearchFrom)
 996             if not MatchObj:
 997                 if MatchEnd <= len(Template):
 998                     TemplateSection = TemplateString.Section(Template[SectionStart:], PlaceHolderList)
 999                     TemplateSectionList.append(TemplateSection)
1000                 break
1001
1002             MatchString = MatchObj.group(1)
1003             MatchStart = MatchObj.start()
1004             MatchEnd = MatchObj.end()
1005
1006             if MatchString == self._REPEAT_START_FLAG:
1007                 if MatchStart > SectionStart:
1008                     TemplateSection = TemplateString.Section(Template[SectionStart:MatchStart], PlaceHolderList)
1009                     TemplateSectionList.append(TemplateSection)
1010                 SectionStart = MatchEnd
1011                 PlaceHolderList = []
1012             elif MatchString == self._REPEAT_END_FLAG:
1013                 TemplateSection = TemplateString.Section(Template[SectionStart:MatchStart], PlaceHolderList)
1014                 TemplateSectionList.append(TemplateSection)
1015                 SectionStart = MatchEnd
1016                 PlaceHolderList = []
1017             else:
1018                 PlaceHolderList.append((MatchString, MatchStart - SectionStart, MatchEnd - SectionStart))
1019             SearchFrom = MatchEnd
1020         return TemplateSectionList
1021
1022     ## Replace the string template with dictionary of placeholders and append it to previous one
1023     #
1024     #   @param      AppendString    The string template to append
1025     #   @param      Dictionary      The placeholder dictionaries
1026     #
1027     def Append(self, AppendString, Dictionary=None):
1028         if Dictionary:
1029             SectionList = self._Parse(AppendString)
1030             self.String += "".join([S.Instantiate(Dictionary) for S in SectionList])
1031         else:
1032             self.String += AppendString
1033
1034     ## Replace the string template with dictionary of placeholders
1035     #
1036     #   @param      Dictionary      The placeholder dictionaries
1037     #
1038     #   @retval     str             The string replaced with placeholder values
1039     #
1040     def Replace(self, Dictionary=None):
1041         return "".join([S.Instantiate(Dictionary) for S in self._TemplateSectionList])
1042
1043 ## Progress indicator class
1044 #
1045 #  This class makes use of thread to print progress on console.
1046 #
1047 class Progressor:
1048     # for avoiding deadloop
1049     _StopFlag = None
1050     _ProgressThread = None
1051     _CheckInterval = 0.25
1052
1053     ## Constructor
1054     #
1055     #   @param      OpenMessage     The string printed before progress charaters
1056     #   @param      CloseMessage    The string printed after progress charaters
1057     #   @param      ProgressChar    The charater used to indicate the progress
1058     #   @param      Interval        The interval in seconds between two progress charaters
1059     #
1060     def __init__(self, OpenMessage="", CloseMessage="", ProgressChar='.', Interval=1.0):
1061         self.PromptMessage = OpenMessage
1062         self.CodaMessage = CloseMessage
1063         self.ProgressChar = ProgressChar
1064         self.Interval = Interval
1065         if Progressor._StopFlag == None:
1066             Progressor._StopFlag = threading.Event()
1067
1068     ## Start to print progress charater
1069     #
1070     #   @param      OpenMessage     The string printed before progress charaters
1071     #
1072     def Start(self, OpenMessage=None):
1073         if OpenMessage != None:
1074             self.PromptMessage = OpenMessage
1075         Progressor._StopFlag.clear()
1076         if Progressor._ProgressThread == None:
1077             Progressor._ProgressThread = threading.Thread(target=self._ProgressThreadEntry)
1078             Progressor._ProgressThread.setDaemon(False)
1079             Progressor._ProgressThread.start()
1080
1081     ## Stop printing progress charater
1082     #
1083     #   @param      CloseMessage    The string printed after progress charaters
1084     #
1085     def Stop(self, CloseMessage=None):
1086         OriginalCodaMessage = self.CodaMessage
1087         if CloseMessage != None:
1088             self.CodaMessage = CloseMessage
1089         self.Abort()
1090         self.CodaMessage = OriginalCodaMessage
1091
1092     ## Thread entry method
1093     def _ProgressThreadEntry(self):
1094         sys.stdout.write(self.PromptMessage + " ")
1095         sys.stdout.flush()
1096         TimeUp = 0.0
1097         while not Progressor._StopFlag.isSet():
1098             if TimeUp <= 0.0:
1099                 sys.stdout.write(self.ProgressChar)
1100                 sys.stdout.flush()
1101                 TimeUp = self.Interval
1102             time.sleep(self._CheckInterval)
1103             TimeUp -= self._CheckInterval
1104         sys.stdout.write(" " + self.CodaMessage + "\n")
1105         sys.stdout.flush()
1106
1107     ## Abort the progress display
1108     @staticmethod
1109     def Abort():
1110         if Progressor._StopFlag != None:
1111             Progressor._StopFlag.set()
1112         if Progressor._ProgressThread != None:
1113             Progressor._ProgressThread.join()
1114             Progressor._ProgressThread = None
1115
1116 ## A dict which can access its keys and/or values orderly
1117 #
1118 #  The class implements a new kind of dict which its keys or values can be
1119 #  accessed in the order they are added into the dict. It guarantees the order
1120 #  by making use of an internal list to keep a copy of keys.
1121 #
1122 class sdict(IterableUserDict):
1123     ## Constructor
1124     def __init__(self):
1125         IterableUserDict.__init__(self)
1126         self._key_list = []
1127
1128     ## [] operator
1129     def __setitem__(self, key, value):
1130         if key not in self._key_list:
1131             self._key_list.append(key)
1132         IterableUserDict.__setitem__(self, key, value)
1133
1134     ## del operator
1135     def __delitem__(self, key):
1136         self._key_list.remove(key)
1137         IterableUserDict.__delitem__(self, key)
1138
1139     ## used in "for k in dict" loop to ensure the correct order
1140     def __iter__(self):
1141         return self.iterkeys()
1142
1143     ## len() support
1144     def __len__(self):
1145         return len(self._key_list)
1146
1147     ## "in" test support
1148     def __contains__(self, key):
1149         return key in self._key_list
1150
1151     ## indexof support
1152     def index(self, key):
1153         return self._key_list.index(key)
1154
1155     ## insert support
1156     def insert(self, key, newkey, newvalue, order):
1157         index = self._key_list.index(key)
1158         if order == 'BEFORE':
1159             self._key_list.insert(index, newkey)
1160             IterableUserDict.__setitem__(self, newkey, newvalue)
1161         elif order == 'AFTER':
1162             self._key_list.insert(index + 1, newkey)
1163             IterableUserDict.__setitem__(self, newkey, newvalue)
1164
1165     ## append support
1166     def append(self, sdict):
1167         for key in sdict:
1168             if key not in self._key_list:
1169                 self._key_list.append(key)
1170             IterableUserDict.__setitem__(self, key, sdict[key])
1171
1172     def has_key(self, key):
1173         return key in self._key_list
1174
1175     ## Empty the dict
1176     def clear(self):
1177         self._key_list = []
1178         IterableUserDict.clear(self)
1179
1180     ## Return a copy of keys
1181     def keys(self):
1182         keys = []
1183         for key in self._key_list:
1184             keys.append(key)
1185         return keys
1186
1187     ## Return a copy of values
1188     def values(self):
1189         values = []
1190         for key in self._key_list:
1191             values.append(self[key])
1192         return values
1193
1194     ## Return a copy of (key, value) list
1195     def items(self):
1196         items = []
1197         for key in self._key_list:
1198             items.append((key, self[key]))
1199         return items
1200
1201     ## Iteration support
1202     def iteritems(self):
1203         return iter(self.items())
1204
1205     ## Keys interation support
1206     def iterkeys(self):
1207         return iter(self.keys())
1208
1209     ## Values interation support
1210     def itervalues(self):
1211         return iter(self.values())
1212
1213     ## Return value related to a key, and remove the (key, value) from the dict
1214     def pop(self, key, *dv):
1215         value = None
1216         if key in self._key_list:
1217             value = self[key]
1218             self.__delitem__(key)
1219         elif len(dv) != 0 :
1220             value = kv[0]
1221         return value
1222
1223     ## Return (key, value) pair, and remove the (key, value) from the dict
1224     def popitem(self):
1225         key = self._key_list[-1]
1226         value = self[key]
1227         self.__delitem__(key)
1228         return key, value
1229
1230     def update(self, dict=None, **kwargs):
1231         if dict != None:
1232             for k, v in dict.items():
1233                 self[k] = v
1234         if len(kwargs):
1235             for k, v in kwargs.items():
1236                 self[k] = v
1237
1238 ## Dictionary with restricted keys
1239 #
1240 class rdict(dict):
1241     ## Constructor
1242     def __init__(self, KeyList):
1243         for Key in KeyList:
1244             dict.__setitem__(self, Key, "")
1245
1246     ## []= operator
1247     def __setitem__(self, key, value):
1248         if key not in self:
1249             EdkLogger.error("RestrictedDict", ATTRIBUTE_SET_FAILURE, "Key [%s] is not allowed" % key,
1250                             ExtraData=", ".join(dict.keys(self)))
1251         dict.__setitem__(self, key, value)
1252
1253     ## =[] operator
1254     def __getitem__(self, key):
1255         if key not in self:
1256             return ""
1257         return dict.__getitem__(self, key)
1258
1259     ## del operator
1260     def __delitem__(self, key):
1261         EdkLogger.error("RestrictedDict", ATTRIBUTE_ACCESS_DENIED, ExtraData="del")
1262
1263     ## Empty the dict
1264     def clear(self):
1265         for Key in self:
1266             self.__setitem__(Key, "")
1267
1268     ## Return value related to a key, and remove the (key, value) from the dict
1269     def pop(self, key, *dv):
1270         EdkLogger.error("RestrictedDict", ATTRIBUTE_ACCESS_DENIED, ExtraData="pop")
1271
1272     ## Return (key, value) pair, and remove the (key, value) from the dict
1273     def popitem(self):
1274         EdkLogger.error("RestrictedDict", ATTRIBUTE_ACCESS_DENIED, ExtraData="popitem")
1275
1276 ## Dictionary using prioritized list as key
1277 #
1278 class tdict:
1279     _ListType = type([])
1280     _TupleType = type(())
1281     _Wildcard = 'COMMON'
1282     _ValidWildcardList = ['COMMON', 'DEFAULT', 'ALL', '*', 'PLATFORM']
1283
1284     def __init__(self, _Single_=False, _Level_=2):
1285         self._Level_ = _Level_
1286         self.data = {}
1287         self._Single_ = _Single_
1288
1289     # =[] operator
1290     def __getitem__(self, key):
1291         KeyType = type(key)
1292         RestKeys = None
1293         if KeyType == self._ListType or KeyType == self._TupleType:
1294             FirstKey = key[0]
1295             if len(key) > 1:
1296                 RestKeys = key[1:]
1297             elif self._Level_ > 1:
1298                 RestKeys = [self._Wildcard for i in range(0, self._Level_ - 1)]
1299         else:
1300             FirstKey = key
1301             if self._Level_ > 1:
1302                 RestKeys = [self._Wildcard for i in range(0, self._Level_ - 1)]
1303
1304         if FirstKey == None or str(FirstKey).upper() in self._ValidWildcardList:
1305             FirstKey = self._Wildcard
1306
1307         if self._Single_:
1308             return self._GetSingleValue(FirstKey, RestKeys)
1309         else:
1310             return self._GetAllValues(FirstKey, RestKeys)
1311
1312     def _GetSingleValue(self, FirstKey, RestKeys):
1313         Value = None
1314         #print "%s-%s" % (FirstKey, self._Level_) ,
1315         if self._Level_ > 1:
1316             if FirstKey == self._Wildcard:
1317                 if FirstKey in self.data:
1318                     Value = self.data[FirstKey][RestKeys]
1319                 if Value == None:
1320                     for Key in self.data:
1321                         Value = self.data[Key][RestKeys]
1322                         if Value != None: break
1323             else:
1324                 if FirstKey in self.data:
1325                     Value = self.data[FirstKey][RestKeys]
1326                 if Value == None and self._Wildcard in self.data:
1327                     #print "Value=None"
1328                     Value = self.data[self._Wildcard][RestKeys]
1329         else:
1330             if FirstKey == self._Wildcard:
1331                 if FirstKey in self.data:
1332                     Value = self.data[FirstKey]
1333                 if Value == None:
1334                     for Key in self.data:
1335                         Value = self.data[Key]
1336                         if Value != None: break
1337             else:
1338                 if FirstKey in self.data:
1339                     Value = self.data[FirstKey]
1340                 elif self._Wildcard in self.data:
1341                     Value = self.data[self._Wildcard]
1342         return Value
1343
1344     def _GetAllValues(self, FirstKey, RestKeys):
1345         Value = []
1346         if self._Level_ > 1:
1347             if FirstKey == self._Wildcard:
1348                 for Key in self.data:
1349                     Value += self.data[Key][RestKeys]
1350             else:
1351                 if FirstKey in self.data:
1352                     Value += self.data[FirstKey][RestKeys]
1353                 if self._Wildcard in self.data:
1354                     Value += self.data[self._Wildcard][RestKeys]
1355         else:
1356             if FirstKey == self._Wildcard:
1357                 for Key in self.data:
1358                     Value.append(self.data[Key])
1359             else:
1360                 if FirstKey in self.data:
1361                     Value.append(self.data[FirstKey])
1362                 if self._Wildcard in self.data:
1363                     Value.append(self.data[self._Wildcard])
1364         return Value
1365
1366     ## []= operator
1367     def __setitem__(self, key, value):
1368         KeyType = type(key)
1369         RestKeys = None
1370         if KeyType == self._ListType or KeyType == self._TupleType:
1371             FirstKey = key[0]
1372             if len(key) > 1:
1373                 RestKeys = key[1:]
1374             else:
1375                 RestKeys = [self._Wildcard for i in range(0, self._Level_ - 1)]
1376         else:
1377             FirstKey = key
1378             if self._Level_ > 1:
1379                 RestKeys = [self._Wildcard for i in range(0, self._Level_ - 1)]
1380
1381         if FirstKey in self._ValidWildcardList:
1382             FirstKey = self._Wildcard
1383
1384         if FirstKey not in self.data and self._Level_ > 0:
1385             self.data[FirstKey] = tdict(self._Single_, self._Level_ - 1)
1386
1387         if self._Level_ > 1:
1388             self.data[FirstKey][RestKeys] = value
1389         else:
1390             self.data[FirstKey] = value
1391
1392     def SetGreedyMode(self):
1393         self._Single_ = False
1394         if self._Level_ > 1:
1395             for Key in self.data:
1396                 self.data[Key].SetGreedyMode()
1397
1398     def SetSingleMode(self):
1399         self._Single_ = True
1400         if self._Level_ > 1:
1401             for Key in self.data:
1402                 self.data[Key].SetSingleMode()
1403
1404     def GetKeys(self, KeyIndex=0):
1405         assert KeyIndex >= 0
1406         if KeyIndex == 0:
1407             return set(self.data.keys())
1408         else:
1409             keys = set()
1410             for Key in self.data:
1411                 keys |= self.data[Key].GetKeys(KeyIndex - 1)
1412             return keys
1413
1414 ## Boolean chain list
1415 #
1416 class Blist(UserList):
1417     def __init__(self, initlist=None):
1418         UserList.__init__(self, initlist)
1419     def __setitem__(self, i, item):
1420         if item not in [True, False]:
1421             if item == 0:
1422                 item = False
1423             else:
1424                 item = True
1425         self.data[i] = item
1426     def _GetResult(self):
1427         Value = True
1428         for item in self.data:
1429             Value &= item
1430         return Value
1431     Result = property(_GetResult)
1432
1433 def ParseConsoleLog(Filename):
1434     Opr = open(os.path.normpath(Filename), 'r')
1435     Opw = open(os.path.normpath(Filename + '.New'), 'w+')
1436     for Line in Opr.readlines():
1437         if Line.find('.efi') > -1:
1438             Line = Line[Line.rfind(' ') : Line.rfind('.efi')].strip()
1439             Opw.write('%s\n' % Line)
1440
1441     Opr.close()
1442     Opw.close()
1443
1444 def AnalyzePcdExpression(Setting):
1445     Setting = Setting.strip()
1446     # There might be escaped quote in a string: \", \\\"
1447     Data = Setting.replace('\\\\', '//').replace('\\\"', '\\\'')
1448     # There might be '|' in string and in ( ... | ... ), replace it with '-'
1449     NewStr = ''
1450     InStr = False
1451     Pair = 0
1452     for ch in Data:
1453         if ch == '"':
1454             InStr = not InStr
1455         elif ch == '(' and not InStr:
1456             Pair += 1
1457         elif ch == ')' and not InStr:
1458             Pair -= 1
1459
1460         if (Pair > 0 or InStr) and ch == TAB_VALUE_SPLIT:
1461             NewStr += '-'
1462         else:
1463             NewStr += ch
1464     FieldList = []
1465     StartPos = 0
1466     while True:
1467         Pos = NewStr.find(TAB_VALUE_SPLIT, StartPos)
1468         if Pos < 0:
1469             FieldList.append(Setting[StartPos:].strip())
1470             break
1471         FieldList.append(Setting[StartPos:Pos].strip())
1472         StartPos = Pos + 1
1473
1474     return FieldList
1475
1476 def ParseDevPathValue (Value):
1477     pass
1478
1479 def ParseFieldValue (Value):
1480     if type(Value) == type(0):
1481         return Value, (Value.bit_length() + 7) / 8
1482     if type(Value) <> type(''):
1483         raise BadExpression('Type %s is %s' %(Value, type(Value)))
1484     Value = Value.strip()
1485     if Value.startswith('UINT8') and Value.endswith(')'):
1486         Value, Size = ParseFieldValue(Value.split('(', 1)[1][:-1])
1487         if Size > 1:
1488             raise BadExpression('Value (%s) Size larger than %d' %(Value, Size))
1489         return Value, 1
1490     if Value.startswith('UINT16') and Value.endswith(')'):
1491         Value, Size = ParseFieldValue(Value.split('(', 1)[1][:-1])
1492         if Size > 2:
1493             raise BadExpression('Value (%s) Size larger than %d' %(Value, Size))
1494         return Value, 2
1495     if Value.startswith('UINT32') and Value.endswith(')'):
1496         Value, Size = ParseFieldValue(Value.split('(', 1)[1][:-1])
1497         if Size > 4:
1498             raise BadExpression('Value (%s) Size larger than %d' %(Value, Size))
1499         return Value, 4
1500     if Value.startswith('UINT64') and Value.endswith(')'):
1501         Value, Size = ParseFieldValue(Value.split('(', 1)[1][:-1])
1502         if Size > 8:
1503             raise BadExpression('Value (%s) Size larger than %d' % (Value, Size))
1504         return Value, 8
1505     if Value.startswith('GUID') and Value.endswith(')'):
1506         Value = Value.split('(', 1)[1][:-1].strip()
1507         if Value[0] == '{' and Value[-1] == '}':
1508             Value = Value[1:-1].strip()
1509             Value = Value.split('{', 1)
1510             Value = ['%02x' % int(Item, 16) for Item in (Value[0] + Value[1][:-1]).split(',')]
1511             if len(Value[0]) != 8:
1512                 Value[0] = '%08X' % int(Value[0], 16)
1513             if len(Value[1]) != 4:
1514                 Value[1] = '%04X' % int(Value[1], 16)
1515             if len(Value[2]) != 4:
1516                 Value[2] = '%04X' % int(Value[2], 16)
1517             Value = '-'.join(Value[0:3]) + '-' + ''.join(Value[3:5]) + '-' + ''.join(Value[5:11])
1518         if Value[0] == '"' and Value[-1] == '"':
1519             Value = Value[1:-1]
1520         try:
1521             Value = "'" + uuid.UUID(Value).get_bytes_le() + "'"
1522         except ValueError, Message:
1523             raise BadExpression('%s' % Message)
1524         Value, Size = ParseFieldValue(Value)
1525         return Value, 16
1526     if Value.startswith('L"') and Value.endswith('"'):
1527         # Unicode String
1528         List = list(Value[2:-1])
1529         List.reverse()
1530         Value = 0
1531         for Char in List:
1532             Value = (Value << 16) | ord(Char)
1533         return Value, (len(List) + 1) * 2
1534     if Value.startswith('"') and Value.endswith('"'):
1535         # ASCII String
1536         List = list(Value[1:-1])
1537         List.reverse()
1538         Value = 0
1539         for Char in List:
1540             Value = (Value << 8) | ord(Char)
1541         return Value, len(List) + 1
1542     if Value.startswith("L'") and Value.endswith("'"):
1543         # Unicode Character Constant
1544         List = list(Value[2:-1])
1545         List.reverse()
1546         Value = 0
1547         for Char in List:
1548             Value = (Value << 16) | ord(Char)
1549         return Value, len(List) * 2
1550     if Value.startswith("'") and Value.endswith("'"):
1551         # Character constant
1552         List = list(Value[1:-1])
1553         List.reverse()
1554         Value = 0
1555         for Char in List:
1556             Value = (Value << 8) | ord(Char)
1557         return Value, len(List)
1558     if Value.startswith('{') and Value.endswith('}'):
1559         # Byte array
1560         Value = Value[1:-1]
1561         List = [Item.strip() for Item in Value.split(',')]
1562         List.reverse()
1563         Value = 0
1564         RetSize = 0
1565         for Item in List:
1566             ItemValue, Size = ParseFieldValue(Item)
1567             RetSize += Size
1568             for I in range(Size):
1569                 Value = (Value << 8) | ((ItemValue >> 8 * I) & 0xff)
1570         return Value, RetSize
1571     if Value.startswith('DEVICE_PATH(') and Value.endswith(')'):
1572         Value = Value.split('"')[1]
1573         return ParseDevPathValue(Value)
1574     if Value.lower().startswith('0x'):
1575         Value = int(Value, 16)
1576         if Value == 0:
1577             return 0, 1
1578         return Value, (Value.bit_length() + 7) / 8
1579     if Value[0].isdigit():
1580         Value = int(Value, 10)
1581         if Value == 0:
1582             return 0, 1
1583         return Value, (Value.bit_length() + 7) / 8
1584     if Value.lower() == 'true':
1585         return 1, 1
1586     if Value.lower() == 'false':
1587         return 0, 1
1588     return Value, 1
1589
1590 ## AnalyzeDscPcd
1591 #
1592 #  Analyze DSC PCD value, since there is no data type info in DSC
1593 #  This fuction is used to match functions (AnalyzePcdData, AnalyzeHiiPcdData, AnalyzeVpdPcdData) used for retrieving PCD value from database
1594 #  1. Feature flag: TokenSpace.PcdCName|PcdValue
1595 #  2. Fix and Patch:TokenSpace.PcdCName|PcdValue[|MaxSize]
1596 #  3. Dynamic default:
1597 #     TokenSpace.PcdCName|PcdValue[|VOID*[|MaxSize]]
1598 #     TokenSpace.PcdCName|PcdValue
1599 #  4. Dynamic VPD:
1600 #     TokenSpace.PcdCName|VpdOffset[|VpdValue]
1601 #     TokenSpace.PcdCName|VpdOffset[|MaxSize[|VpdValue]]
1602 #  5. Dynamic HII:
1603 #     TokenSpace.PcdCName|HiiString|VaiableGuid|VariableOffset[|HiiValue]
1604 #  PCD value needs to be located in such kind of string, and the PCD value might be an expression in which
1605 #    there might have "|" operator, also in string value.
1606 #
1607 #  @param Setting: String contain information described above with "TokenSpace.PcdCName|" stripped
1608 #  @param PcdType: PCD type: feature, fixed, dynamic default VPD HII
1609 #  @param DataType: The datum type of PCD: VOID*, UNIT, BOOL
1610 #  @retval:
1611 #    ValueList: A List contain fields described above
1612 #    IsValid:   True if conforming EBNF, otherwise False
1613 #    Index:     The index where PcdValue is in ValueList
1614 #
1615 def AnalyzeDscPcd(Setting, PcdType, DataType=''):
1616     FieldList = AnalyzePcdExpression(Setting)
1617
1618     IsValid = True
1619     if PcdType in (MODEL_PCD_FIXED_AT_BUILD, MODEL_PCD_PATCHABLE_IN_MODULE, MODEL_PCD_FEATURE_FLAG):
1620         Value = FieldList[0]
1621         Size = ''
1622         if len(FieldList) > 1:
1623             if FieldList[1].upper().startswith("0X") or FieldList[1].isdigit():
1624                 Size = FieldList[1]
1625             else:
1626                 DataType = FieldList[1]
1627
1628         if len(FieldList) > 2:
1629             Size = FieldList[2]
1630         if DataType == "":
1631             IsValid = (len(FieldList) <= 1)
1632         else:
1633             IsValid = (len(FieldList) <= 3)
1634 #         Value, Size = ParseFieldValue(Value)
1635         if Size:
1636             try:
1637                 int(Size,16) if Size.upper().startswith("0X") else int(Size)
1638             except:
1639                 IsValid = False
1640                 Size = -1
1641         return [str(Value), '', str(Size)], IsValid, 0
1642     elif PcdType in (MODEL_PCD_DYNAMIC_DEFAULT, MODEL_PCD_DYNAMIC_EX_DEFAULT):
1643         Value = FieldList[0]
1644         Size = Type = ''
1645         if len(FieldList) > 1:
1646             Type = FieldList[1]
1647         else:
1648             Type = DataType
1649         if len(FieldList) > 2:
1650             Size = FieldList[2]
1651         else:
1652             if Type == 'VOID*':
1653                 if Value.startswith("L"):
1654                     Size = str((len(Value)- 3 + 1) * 2)
1655                 elif Value.startswith("{"):
1656                     Size = str(len(Value.split(",")))
1657                 else:
1658                     Size = str(len(Value) -2 + 1 )
1659         if DataType == "":
1660             IsValid = (len(FieldList) <= 1)
1661         else:
1662             IsValid = (len(FieldList) <= 3)
1663
1664         if Size:
1665             try:
1666                 int(Size,16) if Size.upper().startswith("0X") else int(Size)
1667             except:
1668                 IsValid = False
1669                 Size = -1
1670         return [Value, Type, str(Size)], IsValid, 0
1671     elif PcdType in (MODEL_PCD_DYNAMIC_VPD, MODEL_PCD_DYNAMIC_EX_VPD):
1672         VpdOffset = FieldList[0]
1673         Value = Size = ''
1674         if not DataType == 'VOID*':
1675             if len(FieldList) > 1:
1676                 Value = FieldList[1]
1677         else:
1678             if len(FieldList) > 1:
1679                 Size = FieldList[1]
1680             if len(FieldList) > 2:
1681                 Value = FieldList[2]
1682         if DataType == "":
1683             IsValid = (len(FieldList) <= 1)
1684         else:
1685             IsValid = (len(FieldList) <= 3)
1686         if Size:
1687             try:
1688                 int(Size,16) if Size.upper().startswith("0X") else int(Size)
1689             except:
1690                 IsValid = False
1691                 Size = -1
1692         return [VpdOffset, str(Size), Value], IsValid, 2
1693     elif PcdType in (MODEL_PCD_DYNAMIC_HII, MODEL_PCD_DYNAMIC_EX_HII):
1694         HiiString = FieldList[0]
1695         Guid = Offset = Value = Attribute = ''
1696         if len(FieldList) > 1:
1697             Guid = FieldList[1]
1698         if len(FieldList) > 2:
1699             Offset = FieldList[2]
1700         if len(FieldList) > 3:
1701             Value = FieldList[3]
1702         if len(FieldList) > 4:
1703             Attribute = FieldList[4]
1704         IsValid = (3 <= len(FieldList) <= 5)
1705         return [HiiString, Guid, Offset, Value, Attribute], IsValid, 3
1706     return [], False, 0
1707
1708 ## AnalyzePcdData
1709 #
1710 #  Analyze the pcd Value, Datum type and TokenNumber.
1711 #  Used to avoid split issue while the value string contain "|" character
1712 #
1713 #  @param[in] Setting:  A String contain value/datum type/token number information;
1714 #
1715 #  @retval   ValueList: A List contain value, datum type and toke number.
1716 #
1717 def AnalyzePcdData(Setting):
1718     ValueList = ['', '', '']
1719
1720     ValueRe = re.compile(r'^\s*L?\".*\|.*\"')
1721     PtrValue = ValueRe.findall(Setting)
1722
1723     ValueUpdateFlag = False
1724
1725     if len(PtrValue) >= 1:
1726         Setting = re.sub(ValueRe, '', Setting)
1727         ValueUpdateFlag = True
1728
1729     TokenList = Setting.split(TAB_VALUE_SPLIT)
1730     ValueList[0:len(TokenList)] = TokenList
1731
1732     if ValueUpdateFlag:
1733         ValueList[0] = PtrValue[0]
1734
1735     return ValueList
1736
1737 ## AnalyzeHiiPcdData
1738 #
1739 #  Analyze the pcd Value, variable name, variable Guid and variable offset.
1740 #  Used to avoid split issue while the value string contain "|" character
1741 #
1742 #  @param[in] Setting:  A String contain VariableName, VariableGuid, VariableOffset, DefaultValue information;
1743 #
1744 #  @retval   ValueList: A List contaian VariableName, VariableGuid, VariableOffset, DefaultValue.
1745 #
1746 def AnalyzeHiiPcdData(Setting):
1747     ValueList = ['', '', '', '']
1748
1749     TokenList = GetSplitValueList(Setting)
1750     ValueList[0:len(TokenList)] = TokenList
1751
1752     return ValueList
1753
1754 ## AnalyzeVpdPcdData
1755 #
1756 #  Analyze the vpd pcd VpdOffset, MaxDatumSize and InitialValue.
1757 #  Used to avoid split issue while the value string contain "|" character
1758 #
1759 #  @param[in] Setting:  A String contain VpdOffset/MaxDatumSize/InitialValue information;
1760 #
1761 #  @retval   ValueList: A List contain VpdOffset, MaxDatumSize and InitialValue.
1762 #
1763 def AnalyzeVpdPcdData(Setting):
1764     ValueList = ['', '', '']
1765
1766     ValueRe = re.compile(r'\s*L?\".*\|.*\"\s*$')
1767     PtrValue = ValueRe.findall(Setting)
1768
1769     ValueUpdateFlag = False
1770
1771     if len(PtrValue) >= 1:
1772         Setting = re.sub(ValueRe, '', Setting)
1773         ValueUpdateFlag = True
1774
1775     TokenList = Setting.split(TAB_VALUE_SPLIT)
1776     ValueList[0:len(TokenList)] = TokenList
1777
1778     if ValueUpdateFlag:
1779         ValueList[2] = PtrValue[0]
1780
1781     return ValueList
1782
1783 ## check format of PCD value against its the datum type
1784 #
1785 # For PCD value setting
1786 #
1787 def CheckPcdDatum(Type, Value):
1788     if Type == "VOID*":
1789         ValueRe = re.compile(r'\s*L?\".*\"\s*$')
1790         if not (((Value.startswith('L"') or Value.startswith('"')) and Value.endswith('"'))
1791                 or (Value.startswith('{') and Value.endswith('}'))
1792                ):
1793             return False, "Invalid value [%s] of type [%s]; must be in the form of {...} for array"\
1794                           ", or \"...\" for string, or L\"...\" for unicode string" % (Value, Type)
1795         elif ValueRe.match(Value):
1796             # Check the chars in UnicodeString or CString is printable
1797             if Value.startswith("L"):
1798                 Value = Value[2:-1]
1799             else:
1800                 Value = Value[1:-1]
1801             Printset = set(string.printable)
1802             Printset.remove(TAB_PRINTCHAR_VT)
1803             Printset.add(TAB_PRINTCHAR_BS)
1804             Printset.add(TAB_PRINTCHAR_NUL)
1805             if not set(Value).issubset(Printset):
1806                 PrintList = list(Printset)
1807                 PrintList.sort()
1808                 return False, "Invalid PCD string value of type [%s]; must be printable chars %s." % (Type, PrintList)
1809     elif Type == 'BOOLEAN':
1810         if Value not in ['TRUE', 'True', 'true', '0x1', '0x01', '1', 'FALSE', 'False', 'false', '0x0', '0x00', '0']:
1811             return False, "Invalid value [%s] of type [%s]; must be one of TRUE, True, true, 0x1, 0x01, 1"\
1812                           ", FALSE, False, false, 0x0, 0x00, 0" % (Value, Type)
1813     elif Type in [TAB_UINT8, TAB_UINT16, TAB_UINT32, TAB_UINT64]:
1814         try:
1815             Value = long(Value, 0)
1816         except:
1817             return False, "Invalid value [%s] of type [%s];"\
1818                           " must be a hexadecimal, decimal or octal in C language format." % (Value, Type)
1819     else:
1820         return True, "StructurePcd"
1821
1822     return True, ""
1823
1824 ## Split command line option string to list
1825 #
1826 # subprocess.Popen needs the args to be a sequence. Otherwise there's problem
1827 # in non-windows platform to launch command
1828 #
1829 def SplitOption(OptionString):
1830     OptionList = []
1831     LastChar = " "
1832     OptionStart = 0
1833     QuotationMark = ""
1834     for Index in range(0, len(OptionString)):
1835         CurrentChar = OptionString[Index]
1836         if CurrentChar in ['"', "'"]:
1837             if QuotationMark == CurrentChar:
1838                 QuotationMark = ""
1839             elif QuotationMark == "":
1840                 QuotationMark = CurrentChar
1841             continue
1842         elif QuotationMark:
1843             continue
1844
1845         if CurrentChar in ["/", "-"] and LastChar in [" ", "\t", "\r", "\n"]:
1846             if Index > OptionStart:
1847                 OptionList.append(OptionString[OptionStart:Index - 1])
1848             OptionStart = Index
1849         LastChar = CurrentChar
1850     OptionList.append(OptionString[OptionStart:])
1851     return OptionList
1852
1853 def CommonPath(PathList):
1854     P1 = min(PathList).split(os.path.sep)
1855     P2 = max(PathList).split(os.path.sep)
1856     for Index in xrange(min(len(P1), len(P2))):
1857         if P1[Index] != P2[Index]:
1858             return os.path.sep.join(P1[:Index])
1859     return os.path.sep.join(P1)
1860
1861 #
1862 # Convert string to C format array
1863 #
1864 def ConvertStringToByteArray(Value):
1865     Value = Value.strip()
1866     if not Value:
1867         return None
1868     if Value[0] == '{':
1869         if not Value.endswith('}'):
1870             return None
1871         Value = Value.replace(' ', '').replace('{', '').replace('}', '')
1872         ValFields = Value.split(',')
1873         try:
1874             for Index in range(len(ValFields)):
1875                 ValFields[Index] = str(int(ValFields[Index], 0))
1876         except ValueError:
1877             return None
1878         Value = '{' + ','.join(ValFields) + '}'
1879         return Value
1880
1881     Unicode = False
1882     if Value.startswith('L"'):
1883         if not Value.endswith('"'):
1884             return None
1885         Value = Value[1:]
1886         Unicode = True
1887     elif not Value.startswith('"') or not Value.endswith('"'):
1888         return None
1889
1890     Value = eval(Value)         # translate escape character
1891     NewValue = '{'
1892     for Index in range(0,len(Value)):
1893         if Unicode:
1894             NewValue = NewValue + str(ord(Value[Index]) % 0x10000) + ','
1895         else:
1896             NewValue = NewValue + str(ord(Value[Index]) % 0x100) + ','
1897     Value = NewValue + '0}'
1898     return Value
1899
1900 class PathClass(object):
1901     def __init__(self, File='', Root='', AlterRoot='', Type='', IsBinary=False,
1902                  Arch='COMMON', ToolChainFamily='', Target='', TagName='', ToolCode=''):
1903         self.Arch = Arch
1904         self.File = str(File)
1905         if os.path.isabs(self.File):
1906             self.Root = ''
1907             self.AlterRoot = ''
1908         else:
1909             self.Root = str(Root)
1910             self.AlterRoot = str(AlterRoot)
1911
1912         # Remove any '.' and '..' in path
1913         if self.Root:
1914             self.Root = mws.getWs(self.Root, self.File)
1915             self.Path = os.path.normpath(os.path.join(self.Root, self.File))
1916             self.Root = os.path.normpath(CommonPath([self.Root, self.Path]))
1917             # eliminate the side-effect of 'C:'
1918             if self.Root[-1] == ':':
1919                 self.Root += os.path.sep
1920             # file path should not start with path separator
1921             if self.Root[-1] == os.path.sep:
1922                 self.File = self.Path[len(self.Root):]
1923             else:
1924                 self.File = self.Path[len(self.Root) + 1:]
1925         else:
1926             self.Path = os.path.normpath(self.File)
1927
1928         self.SubDir, self.Name = os.path.split(self.File)
1929         self.BaseName, self.Ext = os.path.splitext(self.Name)
1930
1931         if self.Root:
1932             if self.SubDir:
1933                 self.Dir = os.path.join(self.Root, self.SubDir)
1934             else:
1935                 self.Dir = self.Root
1936         else:
1937             self.Dir = self.SubDir
1938
1939         if IsBinary:
1940             self.Type = Type
1941         else:
1942             self.Type = self.Ext.lower()
1943
1944         self.IsBinary = IsBinary
1945         self.Target = Target
1946         self.TagName = TagName
1947         self.ToolCode = ToolCode
1948         self.ToolChainFamily = ToolChainFamily
1949
1950         self._Key = None
1951
1952     ## Convert the object of this class to a string
1953     #
1954     #  Convert member Path of the class to a string
1955     #
1956     #  @retval string Formatted String
1957     #
1958     def __str__(self):
1959         return self.Path
1960
1961     ## Override __eq__ function
1962     #
1963     # Check whether PathClass are the same
1964     #
1965     # @retval False The two PathClass are different
1966     # @retval True  The two PathClass are the same
1967     #
1968     def __eq__(self, Other):
1969         if type(Other) == type(self):
1970             return self.Path == Other.Path
1971         else:
1972             return self.Path == str(Other)
1973
1974     ## Override __cmp__ function
1975     #
1976     # Customize the comparsion operation of two PathClass
1977     #
1978     # @retval 0     The two PathClass are different
1979     # @retval -1    The first PathClass is less than the second PathClass
1980     # @retval 1     The first PathClass is Bigger than the second PathClass
1981     def __cmp__(self, Other):
1982         if type(Other) == type(self):
1983             OtherKey = Other.Path
1984         else:
1985             OtherKey = str(Other)
1986
1987         SelfKey = self.Path
1988         if SelfKey == OtherKey:
1989             return 0
1990         elif SelfKey > OtherKey:
1991             return 1
1992         else:
1993             return -1
1994
1995     ## Override __hash__ function
1996     #
1997     # Use Path as key in hash table
1998     #
1999     # @retval string Key for hash table
2000     #
2001     def __hash__(self):
2002         return hash(self.Path)
2003
2004     def _GetFileKey(self):
2005         if self._Key == None:
2006             self._Key = self.Path.upper()   # + self.ToolChainFamily + self.TagName + self.ToolCode + self.Target
2007         return self._Key
2008
2009     def _GetTimeStamp(self):
2010         return os.stat(self.Path)[8]
2011
2012     def Validate(self, Type='', CaseSensitive=True):
2013         if GlobalData.gCaseInsensitive:
2014             CaseSensitive = False
2015         if Type and Type.lower() != self.Type:
2016             return FILE_TYPE_MISMATCH, '%s (expect %s but got %s)' % (self.File, Type, self.Type)
2017
2018         RealFile, RealRoot = RealPath2(self.File, self.Root, self.AlterRoot)
2019         if not RealRoot and not RealFile:
2020             RealFile = self.File
2021             if self.AlterRoot:
2022                 RealFile = os.path.join(self.AlterRoot, self.File)
2023             elif self.Root:
2024                 RealFile = os.path.join(self.Root, self.File)
2025             if len (mws.getPkgPath()) == 0:
2026                 return FILE_NOT_FOUND, os.path.join(self.AlterRoot, RealFile)
2027             else:
2028                 return FILE_NOT_FOUND, "%s is not found in packages path:\n\t%s" % (self.File, '\n\t'.join(mws.getPkgPath()))
2029
2030         ErrorCode = 0
2031         ErrorInfo = ''
2032         if RealRoot != self.Root or RealFile != self.File:
2033             if CaseSensitive and (RealFile != self.File or (RealRoot != self.Root and RealRoot != self.AlterRoot)):
2034                 ErrorCode = FILE_CASE_MISMATCH
2035                 ErrorInfo = self.File + '\n\t' + RealFile + " [in file system]"
2036
2037             self.SubDir, self.Name = os.path.split(RealFile)
2038             self.BaseName, self.Ext = os.path.splitext(self.Name)
2039             if self.SubDir:
2040                 self.Dir = os.path.join(RealRoot, self.SubDir)
2041             else:
2042                 self.Dir = RealRoot
2043             self.File = RealFile
2044             self.Root = RealRoot
2045             self.Path = os.path.join(RealRoot, RealFile)
2046         return ErrorCode, ErrorInfo
2047
2048     Key = property(_GetFileKey)
2049     TimeStamp = property(_GetTimeStamp)
2050
2051 ## Parse PE image to get the required PE informaion.
2052 #
2053 class PeImageClass():
2054     ## Constructor
2055     #
2056     #   @param  File FilePath of PeImage
2057     #
2058     def __init__(self, PeFile):
2059         self.FileName   = PeFile
2060         self.IsValid    = False
2061         self.Size       = 0
2062         self.EntryPoint = 0
2063         self.SectionAlignment  = 0
2064         self.SectionHeaderList = []
2065         self.ErrorInfo = ''
2066         try:
2067             PeObject = open(PeFile, 'rb')
2068         except:
2069             self.ErrorInfo = self.FileName + ' can not be found\n'
2070             return
2071         # Read DOS header
2072         ByteArray = array.array('B')
2073         ByteArray.fromfile(PeObject, 0x3E)
2074         ByteList = ByteArray.tolist()
2075         # DOS signature should be 'MZ'
2076         if self._ByteListToStr (ByteList[0x0:0x2]) != 'MZ':
2077             self.ErrorInfo = self.FileName + ' has no valid DOS signature MZ'
2078             return
2079
2080         # Read 4 byte PE Signature
2081         PeOffset = self._ByteListToInt(ByteList[0x3C:0x3E])
2082         PeObject.seek(PeOffset)
2083         ByteArray = array.array('B')
2084         ByteArray.fromfile(PeObject, 4)
2085         # PE signature should be 'PE\0\0'
2086         if ByteArray.tostring() != 'PE\0\0':
2087             self.ErrorInfo = self.FileName + ' has no valid PE signature PE00'
2088             return
2089
2090         # Read PE file header
2091         ByteArray = array.array('B')
2092         ByteArray.fromfile(PeObject, 0x14)
2093         ByteList = ByteArray.tolist()
2094         SecNumber = self._ByteListToInt(ByteList[0x2:0x4])
2095         if SecNumber == 0:
2096             self.ErrorInfo = self.FileName + ' has no section header'
2097             return
2098
2099         # Read PE optional header
2100         OptionalHeaderSize = self._ByteListToInt(ByteArray[0x10:0x12])
2101         ByteArray = array.array('B')
2102         ByteArray.fromfile(PeObject, OptionalHeaderSize)
2103         ByteList = ByteArray.tolist()
2104         self.EntryPoint       = self._ByteListToInt(ByteList[0x10:0x14])
2105         self.SectionAlignment = self._ByteListToInt(ByteList[0x20:0x24])
2106         self.Size             = self._ByteListToInt(ByteList[0x38:0x3C])
2107
2108         # Read each Section Header
2109         for Index in range(SecNumber):
2110             ByteArray = array.array('B')
2111             ByteArray.fromfile(PeObject, 0x28)
2112             ByteList = ByteArray.tolist()
2113             SecName  = self._ByteListToStr(ByteList[0:8])
2114             SecVirtualSize = self._ByteListToInt(ByteList[8:12])
2115             SecRawAddress  = self._ByteListToInt(ByteList[20:24])
2116             SecVirtualAddress = self._ByteListToInt(ByteList[12:16])
2117             self.SectionHeaderList.append((SecName, SecVirtualAddress, SecRawAddress, SecVirtualSize))
2118         self.IsValid = True
2119         PeObject.close()
2120
2121     def _ByteListToStr(self, ByteList):
2122         String = ''
2123         for index in range(len(ByteList)):
2124             if ByteList[index] == 0:
2125                 break
2126             String += chr(ByteList[index])
2127         return String
2128
2129     def _ByteListToInt(self, ByteList):
2130         Value = 0
2131         for index in range(len(ByteList) - 1, -1, -1):
2132             Value = (Value << 8) | int(ByteList[index])
2133         return Value
2134
2135 class DefaultStore():
2136     def __init__(self,DefaultStores ):
2137
2138         self.DefaultStores = DefaultStores
2139     def DefaultStoreID(self,DefaultStoreName):
2140         for key,value in self.DefaultStores.items():
2141             if value == DefaultStoreName:
2142                 return key
2143         return None
2144     def GetDefaultDefault(self):
2145         if not self.DefaultStores or "0" in self.DefaultStores:
2146             return "0",TAB_DEFAULT_STORES_DEFAULT
2147         else:
2148             minvalue = min([int(value_str) for value_str in self.DefaultStores.keys()])
2149             return (str(minvalue), self.DefaultStores[str(minvalue)])
2150     def GetMin(self,DefaultSIdList):
2151         if not DefaultSIdList:
2152             return "STANDARD"
2153         storeidset = {storeid for storeid, storename in self.DefaultStores.values() if storename in DefaultSIdList}
2154         if not storeidset:
2155             return ""
2156         minid = min(storeidset )
2157         for sid,name in self.DefaultStores.values():
2158             if sid == minid:
2159                 return name
2160 class SkuClass():
2161
2162     DEFAULT = 0
2163     SINGLE = 1
2164     MULTIPLE =2
2165
2166     def __init__(self,SkuIdentifier='', SkuIds=None):
2167         if SkuIds is None:
2168             SkuIds = {}
2169
2170         for SkuName in SkuIds:
2171             SkuId = SkuIds[SkuName][0]
2172             skuid_num = int(SkuId,16) if SkuId.upper().startswith("0X") else int(SkuId)
2173             if skuid_num > 0xFFFFFFFFFFFFFFFF:
2174                 EdkLogger.error("build", PARAMETER_INVALID,
2175                             ExtraData = "SKU-ID [%s] value %s exceeds the max value of UINT64"
2176                                       % (SkuName, SkuId))
2177
2178         self.AvailableSkuIds = sdict()
2179         self.SkuIdSet = []
2180         self.SkuIdNumberSet = []
2181         self.SkuData = SkuIds
2182         self.__SkuInherit = {}
2183         self.__SkuIdentifier = SkuIdentifier
2184         if SkuIdentifier == '' or SkuIdentifier is None:
2185             self.SkuIdSet = ['DEFAULT']
2186             self.SkuIdNumberSet = ['0U']
2187         elif SkuIdentifier == 'ALL':
2188             self.SkuIdSet = SkuIds.keys()
2189             self.SkuIdNumberSet = [num[0].strip() + 'U' for num in SkuIds.values()]
2190         else:
2191             r = SkuIdentifier.split('|')
2192             self.SkuIdSet=[(r[k].strip()).upper() for k in range(len(r))]
2193             k = None
2194             try:
2195                 self.SkuIdNumberSet = [SkuIds[k][0].strip() + 'U' for k in self.SkuIdSet]
2196             except Exception:
2197                 EdkLogger.error("build", PARAMETER_INVALID,
2198                             ExtraData = "SKU-ID [%s] is not supported by the platform. [Valid SKU-ID: %s]"
2199                                       % (k, " | ".join(SkuIds.keys())))
2200         for each in self.SkuIdSet:
2201             if each in SkuIds:
2202                 self.AvailableSkuIds[each] = SkuIds[each][0]
2203             else:
2204                 EdkLogger.error("build", PARAMETER_INVALID,
2205                             ExtraData="SKU-ID [%s] is not supported by the platform. [Valid SKU-ID: %s]"
2206                                       % (each, " | ".join(SkuIds.keys())))
2207         if self.SkuUsageType != self.SINGLE:
2208             self.AvailableSkuIds.update({'DEFAULT':0, 'COMMON':0})
2209         if self.SkuIdSet:
2210             GlobalData.gSkuids = (self.SkuIdSet)
2211             if 'COMMON' in GlobalData.gSkuids:
2212                 GlobalData.gSkuids.remove('COMMON')
2213             if GlobalData.gSkuids:
2214                 GlobalData.gSkuids.sort()
2215
2216     def GetNextSkuId(self, skuname):
2217         if not self.__SkuInherit:
2218             self.__SkuInherit = {}
2219             for item in self.SkuData.values():
2220                 self.__SkuInherit[item[1]]=item[2] if item[2] else "DEFAULT"
2221         return self.__SkuInherit.get(skuname,"DEFAULT")
2222
2223     def GetSkuChain(self,sku):
2224         skulist = [sku]
2225         nextsku = sku
2226         while 1:
2227             nextsku = self.GetNextSkuId(nextsku)
2228             skulist.append(nextsku)
2229             if nextsku == "DEFAULT":
2230                 break
2231         skulist.reverse()
2232         return skulist
2233     def SkuOverrideOrder(self):
2234         skuorderset = []
2235         for skuname in self.SkuIdSet:
2236             skuorderset.append(self.GetSkuChain(skuname))
2237
2238         skuorder = []
2239         for index in range(max([len(item) for item in skuorderset])):
2240             for subset in skuorderset:
2241                 if index > len(subset)-1:
2242                     continue
2243                 if subset[index] in skuorder:
2244                     continue
2245                 skuorder.append(subset[index])
2246
2247         return skuorder
2248
2249     def __SkuUsageType(self):
2250
2251         if self.__SkuIdentifier.upper() == "ALL":
2252             return SkuClass.MULTIPLE
2253
2254         if len(self.SkuIdSet) == 1:
2255             if self.SkuIdSet[0] == 'DEFAULT':
2256                 return SkuClass.DEFAULT
2257             else:
2258                 return SkuClass.SINGLE
2259         elif len(self.SkuIdSet) == 2:
2260             if 'DEFAULT' in self.SkuIdSet:
2261                 return SkuClass.SINGLE
2262             else:
2263                 return SkuClass.MULTIPLE
2264         else:
2265             return SkuClass.MULTIPLE
2266     def DumpSkuIdArrary(self):
2267
2268         ArrayStrList = []
2269         if self.SkuUsageType == SkuClass.SINGLE:
2270             ArrayStr = "{0x0}"
2271         else:
2272             for skuname in self.AvailableSkuIds:
2273                 if skuname == "COMMON":
2274                     continue
2275                 while skuname != "DEFAULT":
2276                     ArrayStrList.append(hex(int(self.AvailableSkuIds[skuname])))
2277                     skuname = self.GetNextSkuId(skuname)
2278                 ArrayStrList.append("0x0")
2279             ArrayStr = "{" + ",".join(ArrayStrList) +  "}"
2280         return ArrayStr
2281     def __GetAvailableSkuIds(self):
2282         return self.AvailableSkuIds
2283
2284     def __GetSystemSkuID(self):
2285         if self.__SkuUsageType() == SkuClass.SINGLE:
2286             if len(self.SkuIdSet) == 1:
2287                 return self.SkuIdSet[0]
2288             else:
2289                 return self.SkuIdSet[0] if self.SkuIdSet[0] != 'DEFAULT' else self.SkuIdSet[1]
2290         else:
2291             return 'DEFAULT'
2292     def __GetAvailableSkuIdNumber(self):
2293         return self.SkuIdNumberSet
2294     SystemSkuId = property(__GetSystemSkuID)
2295     AvailableSkuIdSet = property(__GetAvailableSkuIds)
2296     SkuUsageType = property(__SkuUsageType)
2297     AvailableSkuIdNumSet = property(__GetAvailableSkuIdNumber)
2298
2299 #
2300 # Pack a registry format GUID
2301 #
2302 def PackRegistryFormatGuid(Guid):
2303     Guid = Guid.split('-')
2304     return pack('=LHHBBBBBBBB',
2305                 int(Guid[0], 16),
2306                 int(Guid[1], 16),
2307                 int(Guid[2], 16),
2308                 int(Guid[3][-4:-2], 16),
2309                 int(Guid[3][-2:], 16),
2310                 int(Guid[4][-12:-10], 16),
2311                 int(Guid[4][-10:-8], 16),
2312                 int(Guid[4][-8:-6], 16),
2313                 int(Guid[4][-6:-4], 16),
2314                 int(Guid[4][-4:-2], 16),
2315                 int(Guid[4][-2:], 16)
2316                 )
2317
2318 def BuildOptionPcdValueFormat(TokenSpaceGuidCName, TokenCName, PcdDatumType, Value):
2319     if PcdDatumType == 'VOID*':
2320         if Value.startswith('L'):
2321             if not Value[1]:
2322                 EdkLogger.error("build", FORMAT_INVALID, 'For Void* type PCD, when specify the Value in the command line, please use the following format: "string", L"string", H"{...}"')
2323             Value = Value[0] + '"' + Value[1:] + '"'
2324         elif Value.startswith('H'):
2325             if not Value[1]:
2326                 EdkLogger.error("build", FORMAT_INVALID, 'For Void* type PCD, when specify the Value in the command line, please use the following format: "string", L"string", H"{...}"')
2327             Value = Value[1:]
2328         else:
2329             if not Value[0]:
2330                 EdkLogger.error("build", FORMAT_INVALID, 'For Void* type PCD, when specify the Value in the command line, please use the following format: "string", L"string", H"{...}"')
2331             Value = '"' + Value + '"'
2332
2333     IsValid, Cause = CheckPcdDatum(PcdDatumType, Value)
2334     if not IsValid:
2335         EdkLogger.error("build", FORMAT_INVALID, Cause, ExtraData="%s.%s" % (TokenSpaceGuidCName, TokenCName))
2336     if PcdDatumType == 'BOOLEAN':
2337         Value = Value.upper()
2338         if Value == 'TRUE' or Value == '1':
2339             Value = '1'
2340         elif Value == 'FALSE' or Value == '0':
2341             Value = '0'
2342     return  Value
2343 ##  Get the integer value from string like "14U" or integer like 2
2344 #
2345 #   @param      Input   The object that may be either a integer value or a string
2346 #
2347 #   @retval     Value    The integer value that the input represents
2348 #
2349 def GetIntegerValue(Input):
2350     if type(Input) in (int, long):
2351         return Input
2352     String = Input
2353     if String.endswith("U"):
2354         String = String[:-1]
2355     if String.endswith("ULL"):
2356         String = String[:-3]
2357     if String.endswith("LL"):
2358         String = String[:-2]
2359
2360     if String.startswith("0x") or String.startswith("0X"):
2361         return int(String, 16)
2362     elif String == '':
2363         return 0
2364     else:
2365         return int(String)
2366
2367 ##
2368 #
2369 # This acts like the main() function for the script, unless it is 'import'ed into another
2370 # script.
2371 #
2372 if __name__ == '__main__':
2373     pass
2374