BaseTools/Source/Python/Common/Misc.py

   1 ## @file
   2 # Common routines used by all tools
   3 #
   4 # Copyright (c) 2007 - 2017, Intel Corporation. All rights reserved.<BR>
   5 # This program and the accompanying materials
   6 # are licensed and made available under the terms and conditions of the BSD License
   7 # which accompanies this distribution.  The full text of the license may be found at
   8 # http://opensource.org/licenses/bsd-license.php
   9 #
  10 # THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS,
  11 # WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED.
  12 #
  13
  14 ##
  15 # Import Modules
  16 #
  17 import Common.LongFilePathOs as os
  18 import sys
  19 import string
  20 import thread
  21 import threading
  22 import time
  23 import re
  24 import cPickle
  25 import array
  26 import shutil
  27 from struct import pack
  28 from UserDict import IterableUserDict
  29 from UserList import UserList
  30
  31 from Common import EdkLogger as EdkLogger
  32 from Common import GlobalData as GlobalData
  33 from DataType import *
  34 from BuildToolError import *
  35 from CommonDataClass.DataClass import *
  36 from Parsing import GetSplitValueList
  37 from Common.LongFilePathSupport import OpenLongFilePath as open
  38 from Common.MultipleWorkspace import MultipleWorkspace as mws
  39 import uuid
  40
  41 ## Regular expression used to find out place holders in string template
  42 gPlaceholderPattern = re.compile("\$\{([^$()\s]+)\}", re.MULTILINE | re.UNICODE)
  43
  44 ## Dictionary used to store file time stamp for quick re-access
  45 gFileTimeStampCache = {}    # {file path : file time stamp}
  46
  47 ## Dictionary used to store dependencies of files
  48 gDependencyDatabase = {}    # arch : {file path : [dependent files list]}
  49
  50 def GetVariableOffset(mapfilepath, efifilepath, varnames):
  51     """ Parse map file to get variable offset in current EFI file
  52     @param mapfilepath    Map file absolution path
  53     @param efifilepath:   EFI binary file full path
  54     @param varnames       iteratable container whose elements are variable names to be searched
  55
  56     @return List whos elements are tuple with variable name and raw offset
  57     """
  58     lines = []
  59     try:
  60         f = open(mapfilepath, 'r')
  61         lines = f.readlines()
  62         f.close()
  63     except:
  64         return None
  65
  66     if len(lines) == 0: return None
  67     firstline = lines[0].strip()
  68     if (firstline.startswith("Archive member included ") and
  69         firstline.endswith(" file (symbol)")):
  70         return _parseForGCC(lines, efifilepath, varnames)
  71     if firstline.startswith("# Path:"):
  72         return _parseForXcode(lines, efifilepath, varnames)
  73     return _parseGeneral(lines, efifilepath, varnames)
  74
  75 def _parseForXcode(lines, efifilepath, varnames):
  76     status = 0
  77     ret = []
  78     for index, line in enumerate(lines):
  79         line = line.strip()
  80         if status == 0 and line == "# Symbols:":
  81             status = 1
  82             continue
  83         if status == 1 and len(line) != 0:
  84             for varname in varnames:
  85                 if varname in line:
  86                     m = re.match('^([\da-fA-FxX]+)([\s\S]*)([_]*%s)$' % varname, line)
  87                     if m != None:
  88                         ret.append((varname, m.group(1)))
  89     return ret
  90
  91 def _parseForGCC(lines, efifilepath, varnames):
  92     """ Parse map file generated by GCC linker """
  93     status = 0
  94     sections = []
  95     varoffset = []
  96     for index, line in enumerate(lines):
  97         line = line.strip()
  98         # status machine transection
  99         if status == 0 and line == "Memory Configuration":
 100             status = 1
 101             continue
 102         elif status == 1 and line == 'Linker script and memory map':
 103             status = 2
 104             continue
 105         elif status ==2 and line == 'START GROUP':
 106             status = 3
 107             continue
 108
 109         # status handler
 110         if status == 3:
 111             m = re.match('^([\w_\.]+) +([\da-fA-Fx]+) +([\da-fA-Fx]+)$', line)
 112             if m != None:
 113                 sections.append(m.groups(0))
 114             for varname in varnames:
 115                 Str = ''
 116                 m = re.match("^.data.(%s)" % varname, line)
 117                 if m != None:
 118                     m = re.match(".data.(%s)$" % varname, line)
 119                     if m != None:
 120                         Str = lines[index + 1]
 121                     else:
 122                         Str = line[len(".data.%s" % varname):]
 123                     if Str:
 124                         m = re.match('^([\da-fA-Fx]+) +([\da-fA-Fx]+)', Str.strip())
 125                         if m != None:
 126                             varoffset.append((varname, int(m.groups(0)[0], 16) , int(sections[-1][1], 16), sections[-1][0]))
 127
 128     if not varoffset:
 129         return []
 130     # get section information from efi file
 131     efisecs = PeImageClass(efifilepath).SectionHeaderList
 132     if efisecs == None or len(efisecs) == 0:
 133         return []
 134     #redirection
 135     redirection = 0
 136     for efisec in efisecs:
 137         for section in sections:
 138             if section[0].strip() == efisec[0].strip() and section[0].strip() == '.text':
 139                 redirection = int(section[1], 16) - efisec[1]
 140
 141     ret = []
 142     for var in varoffset:
 143         for efisec in efisecs:
 144             if var[1] >= efisec[1] and var[1] < efisec[1]+efisec[3]:
 145                 ret.append((var[0], hex(efisec[2] + var[1] - efisec[1] - redirection)))
 146     return ret
 147
 148 def _parseGeneral(lines, efifilepath, varnames):
 149     status = 0    #0 - beginning of file; 1 - PE section definition; 2 - symbol table
 150     secs  = []    # key = section name
 151     varoffset = []
 152     secRe = re.compile('^([\da-fA-F]+):([\da-fA-F]+) +([\da-fA-F]+)[Hh]? +([.\w\$]+) +(\w+)', re.UNICODE)
 153     symRe = re.compile('^([\da-fA-F]+):([\da-fA-F]+) +([\.:\\\\\w\?@\$]+) +([\da-fA-F]+)', re.UNICODE)
 154
 155     for line in lines:
 156         line = line.strip()
 157         if re.match("^Start[' ']+Length[' ']+Name[' ']+Class", line):
 158             status = 1
 159             continue
 160         if re.match("^Address[' ']+Publics by Value[' ']+Rva\+Base", line):
 161             status = 2
 162             continue
 163         if re.match("^entry point at", line):
 164             status = 3
 165             continue
 166         if status == 1 and len(line) != 0:
 167             m =  secRe.match(line)
 168             assert m != None, "Fail to parse the section in map file , line is %s" % line
 169             sec_no, sec_start, sec_length, sec_name, sec_class = m.groups(0)
 170             secs.append([int(sec_no, 16), int(sec_start, 16), int(sec_length, 16), sec_name, sec_class])
 171         if status == 2 and len(line) != 0:
 172             for varname in varnames:
 173                 m = symRe.match(line)
 174                 assert m != None, "Fail to parse the symbol in map file, line is %s" % line
 175                 sec_no, sym_offset, sym_name, vir_addr = m.groups(0)
 176                 sec_no     = int(sec_no,     16)
 177                 sym_offset = int(sym_offset, 16)
 178                 vir_addr   = int(vir_addr,   16)
 179                 m2 = re.match('^[_]*(%s)' % varname, sym_name)
 180                 if m2 != None:
 181                     # fond a binary pcd entry in map file
 182                     for sec in secs:
 183                         if sec[0] == sec_no and (sym_offset >= sec[1] and sym_offset < sec[1] + sec[2]):
 184                             varoffset.append([varname, sec[3], sym_offset, vir_addr, sec_no])
 185
 186     if not varoffset: return []
 187
 188     # get section information from efi file
 189     efisecs = PeImageClass(efifilepath).SectionHeaderList
 190     if efisecs == None or len(efisecs) == 0:
 191         return []
 192
 193     ret = []
 194     for var in varoffset:
 195         index = 0
 196         for efisec in efisecs:
 197             index = index + 1
 198             if var[1].strip() == efisec[0].strip():
 199                 ret.append((var[0], hex(efisec[2] + var[2])))
 200             elif var[4] == index:
 201                 ret.append((var[0], hex(efisec[2] + var[2])))
 202
 203     return ret
 204
 205 ## Routine to process duplicated INF
 206 #
 207 #  This function is called by following two cases:
 208 #  Case 1 in DSC:
 209 #    [components.arch]
 210 #    Pkg/module/module.inf
 211 #    Pkg/module/module.inf {
 212 #      <Defines>
 213 #        FILE_GUID = 0D1B936F-68F3-4589-AFCC-FB8B7AEBC836
 214 #    }
 215 #  Case 2 in FDF:
 216 #    INF Pkg/module/module.inf
 217 #    INF FILE_GUID = 0D1B936F-68F3-4589-AFCC-FB8B7AEBC836 Pkg/module/module.inf
 218 #
 219 #  This function copies Pkg/module/module.inf to
 220 #  Conf/.cache/0D1B936F-68F3-4589-AFCC-FB8B7AEBC836module.inf
 221 #
 222 #  @param Path     Original PathClass object
 223 #  @param BaseName New file base name
 224 #
 225 #  @retval         return the new PathClass object
 226 #
 227 def ProcessDuplicatedInf(Path, BaseName, Workspace):
 228     Filename = os.path.split(Path.File)[1]
 229     if '.' in Filename:
 230         Filename = BaseName + Path.BaseName + Filename[Filename.rfind('.'):]
 231     else:
 232         Filename = BaseName + Path.BaseName
 233
 234     #
 235     # If -N is specified on command line, cache is disabled
 236     # The directory has to be created
 237     #
 238     DbDir = os.path.split(GlobalData.gDatabasePath)[0]
 239     if not os.path.exists(DbDir):
 240         os.makedirs(DbDir)
 241     #
 242     # A temporary INF is copied to database path which must have write permission
 243     # The temporary will be removed at the end of build
 244     # In case of name conflict, the file name is
 245     # FILE_GUIDBaseName (0D1B936F-68F3-4589-AFCC-FB8B7AEBC836module.inf)
 246     #
 247     TempFullPath = os.path.join(DbDir,
 248                                 Filename)
 249     RtPath = PathClass(Path.File, Workspace)
 250     #
 251     # Modify the full path to temporary path, keep other unchanged
 252     #
 253     # To build same module more than once, the module path with FILE_GUID overridden has
 254     # the file name FILE_GUIDmodule.inf, but the relative path (self.MetaFile.File) is the real path
 255     # in DSC which is used as relative path by C files and other files in INF.
 256     # A trick was used: all module paths are PathClass instances, after the initialization
 257     # of PathClass, the PathClass.Path is overridden by the temporary INF path.
 258     #
 259     # The reason for creating a temporary INF is:
 260     # Platform.Modules which is the base to create ModuleAutoGen objects is a dictionary,
 261     # the key is the full path of INF, the value is an object to save overridden library instances, PCDs.
 262     # A different key for the same module is needed to create different output directory,
 263     # retrieve overridden PCDs, library instances.
 264     #
 265     # The BaseName is the FILE_GUID which is also the output directory name.
 266     #
 267     #
 268     RtPath.Path = TempFullPath
 269     RtPath.BaseName = BaseName
 270     #
 271     # If file exists, compare contents
 272     #
 273     if os.path.exists(TempFullPath):
 274         with open(str(Path), 'rb') as f1: Src = f1.read()
 275         with open(TempFullPath, 'rb') as f2: Dst = f2.read()
 276         if Src == Dst:
 277             return RtPath
 278     GlobalData.gTempInfs.append(TempFullPath)
 279     shutil.copy2(str(Path), TempFullPath)
 280     return RtPath
 281
 282 ## Remove temporary created INFs whose paths were saved in gTempInfs
 283 #
 284 def ClearDuplicatedInf():
 285     for File in GlobalData.gTempInfs:
 286         if os.path.exists(File):
 287             os.remove(File)
 288
 289 ## callback routine for processing variable option
 290 #
 291 # This function can be used to process variable number of option values. The
 292 # typical usage of it is specify architecure list on command line.
 293 # (e.g. <tool> -a IA32 X64 IPF)
 294 #
 295 # @param  Option        Standard callback function parameter
 296 # @param  OptionString  Standard callback function parameter
 297 # @param  Value         Standard callback function parameter
 298 # @param  Parser        Standard callback function parameter
 299 #
 300 # @retval
 301 #
 302 def ProcessVariableArgument(Option, OptionString, Value, Parser):
 303     assert Value is None
 304     Value = []
 305     RawArgs = Parser.rargs
 306     while RawArgs:
 307         Arg = RawArgs[0]
 308         if (Arg[:2] == "--" and len(Arg) > 2) or \
 309            (Arg[:1] == "-" and len(Arg) > 1 and Arg[1] != "-"):
 310             break
 311         Value.append(Arg)
 312         del RawArgs[0]
 313     setattr(Parser.values, Option.dest, Value)
 314
 315 ## Convert GUID string in xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx style to C structure style
 316 #
 317 #   @param      Guid    The GUID string
 318 #
 319 #   @retval     string  The GUID string in C structure style
 320 #
 321 def GuidStringToGuidStructureString(Guid):
 322     GuidList = Guid.split('-')
 323     Result = '{'
 324     for Index in range(0, 3, 1):
 325         Result = Result + '0x' + GuidList[Index] + ', '
 326     Result = Result + '{0x' + GuidList[3][0:2] + ', 0x' + GuidList[3][2:4]
 327     for Index in range(0, 12, 2):
 328         Result = Result + ', 0x' + GuidList[4][Index:Index + 2]
 329     Result += '}}'
 330     return Result
 331
 332 ## Convert GUID structure in byte array to xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
 333 #
 334 #   @param      GuidValue   The GUID value in byte array
 335 #
 336 #   @retval     string      The GUID value in xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx format
 337 #
 338 def GuidStructureByteArrayToGuidString(GuidValue):
 339     guidValueString = GuidValue.lower().replace("{", "").replace("}", "").replace(" ", "").replace(";", "")
 340     guidValueList = guidValueString.split(",")
 341     if len(guidValueList) != 16:
 342         return ''
 343         #EdkLogger.error(None, None, "Invalid GUID value string %s" % GuidValue)
 344     try:
 345         return "%02x%02x%02x%02x-%02x%02x-%02x%02x-%02x%02x-%02x%02x%02x%02x%02x%02x" % (
 346                 int(guidValueList[3], 16),
 347                 int(guidValueList[2], 16),
 348                 int(guidValueList[1], 16),
 349                 int(guidValueList[0], 16),
 350                 int(guidValueList[5], 16),
 351                 int(guidValueList[4], 16),
 352                 int(guidValueList[7], 16),
 353                 int(guidValueList[6], 16),
 354                 int(guidValueList[8], 16),
 355                 int(guidValueList[9], 16),
 356                 int(guidValueList[10], 16),
 357                 int(guidValueList[11], 16),
 358                 int(guidValueList[12], 16),
 359                 int(guidValueList[13], 16),
 360                 int(guidValueList[14], 16),
 361                 int(guidValueList[15], 16)
 362                 )
 363     except:
 364         return ''
 365
 366 ## Convert GUID string in C structure style to xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
 367 #
 368 #   @param      GuidValue   The GUID value in C structure format
 369 #
 370 #   @retval     string      The GUID value in xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx format
 371 #
 372 def GuidStructureStringToGuidString(GuidValue):
 373     guidValueString = GuidValue.lower().replace("{", "").replace("}", "").replace(" ", "").replace(";", "")
 374     guidValueList = guidValueString.split(",")
 375     if len(guidValueList) != 11:
 376         return ''
 377         #EdkLogger.error(None, None, "Invalid GUID value string %s" % GuidValue)
 378     try:
 379         return "%08x-%04x-%04x-%02x%02x-%02x%02x%02x%02x%02x%02x" % (
 380                 int(guidValueList[0], 16),
 381                 int(guidValueList[1], 16),
 382                 int(guidValueList[2], 16),
 383                 int(guidValueList[3], 16),
 384                 int(guidValueList[4], 16),
 385                 int(guidValueList[5], 16),
 386                 int(guidValueList[6], 16),
 387                 int(guidValueList[7], 16),
 388                 int(guidValueList[8], 16),
 389                 int(guidValueList[9], 16),
 390                 int(guidValueList[10], 16)
 391                 )
 392     except:
 393         return ''
 394
 395 ## Convert GUID string in C structure style to xxxxxxxx_xxxx_xxxx_xxxx_xxxxxxxxxxxx
 396 #
 397 #   @param      GuidValue   The GUID value in C structure format
 398 #
 399 #   @retval     string      The GUID value in xxxxxxxx_xxxx_xxxx_xxxx_xxxxxxxxxxxx format
 400 #
 401 def GuidStructureStringToGuidValueName(GuidValue):
 402     guidValueString = GuidValue.lower().replace("{", "").replace("}", "").replace(" ", "")
 403     guidValueList = guidValueString.split(",")
 404     if len(guidValueList) != 11:
 405         EdkLogger.error(None, FORMAT_INVALID, "Invalid GUID value string [%s]" % GuidValue)
 406     return "%08x_%04x_%04x_%02x%02x_%02x%02x%02x%02x%02x%02x" % (
 407             int(guidValueList[0], 16),
 408             int(guidValueList[1], 16),
 409             int(guidValueList[2], 16),
 410             int(guidValueList[3], 16),
 411             int(guidValueList[4], 16),
 412             int(guidValueList[5], 16),
 413             int(guidValueList[6], 16),
 414             int(guidValueList[7], 16),
 415             int(guidValueList[8], 16),
 416             int(guidValueList[9], 16),
 417             int(guidValueList[10], 16)
 418             )
 419
 420 ## Create directories
 421 #
 422 #   @param      Directory   The directory name
 423 #
 424 def CreateDirectory(Directory):
 425     if Directory == None or Directory.strip() == "":
 426         return True
 427     try:
 428         if not os.access(Directory, os.F_OK):
 429             os.makedirs(Directory)
 430     except:
 431         return False
 432     return True
 433
 434 ## Remove directories, including files and sub-directories in it
 435 #
 436 #   @param      Directory   The directory name
 437 #
 438 def RemoveDirectory(Directory, Recursively=False):
 439     if Directory == None or Directory.strip() == "" or not os.path.exists(Directory):
 440         return
 441     if Recursively:
 442         CurrentDirectory = os.getcwd()
 443         os.chdir(Directory)
 444         for File in os.listdir("."):
 445             if os.path.isdir(File):
 446                 RemoveDirectory(File, Recursively)
 447             else:
 448                 os.remove(File)
 449         os.chdir(CurrentDirectory)
 450     os.rmdir(Directory)
 451
 452 ## Check if given file is changed or not
 453 #
 454 #  This method is used to check if a file is changed or not between two build
 455 #  actions. It makes use a cache to store files timestamp.
 456 #
 457 #   @param      File    The path of file
 458 #
 459 #   @retval     True    If the given file is changed, doesn't exist, or can't be
 460 #                       found in timestamp cache
 461 #   @retval     False   If the given file is changed
 462 #
 463 def IsChanged(File):
 464     if not os.path.exists(File):
 465         return True
 466
 467     FileState = os.stat(File)
 468     TimeStamp = FileState[-2]
 469
 470     if File in gFileTimeStampCache and TimeStamp == gFileTimeStampCache[File]:
 471         FileChanged = False
 472     else:
 473         FileChanged = True
 474         gFileTimeStampCache[File] = TimeStamp
 475
 476     return FileChanged
 477
 478 ## Store content in file
 479 #
 480 #  This method is used to save file only when its content is changed. This is
 481 #  quite useful for "make" system to decide what will be re-built and what won't.
 482 #
 483 #   @param      File            The path of file
 484 #   @param      Content         The new content of the file
 485 #   @param      IsBinaryFile    The flag indicating if the file is binary file or not
 486 #
 487 #   @retval     True            If the file content is changed and the file is renewed
 488 #   @retval     False           If the file content is the same
 489 #
 490 def SaveFileOnChange(File, Content, IsBinaryFile=True):
 491     if not IsBinaryFile:
 492         Content = Content.replace("\n", os.linesep)
 493
 494     if os.path.exists(File):
 495         try:
 496             if Content == open(File, "rb").read():
 497                 return False
 498         except:
 499             EdkLogger.error(None, FILE_OPEN_FAILURE, ExtraData=File)
 500
 501     DirName = os.path.dirname(File)
 502     if not CreateDirectory(DirName):
 503         EdkLogger.error(None, FILE_CREATE_FAILURE, "Could not create directory %s" % DirName)
 504     else:
 505         if DirName == '':
 506             DirName = os.getcwd()
 507         if not os.access(DirName, os.W_OK):
 508             EdkLogger.error(None, PERMISSION_FAILURE, "Do not have write permission on directory %s" % DirName)
 509
 510     try:
 511         if GlobalData.gIsWindows:
 512             try:
 513                 from PyUtility import SaveFileToDisk
 514                 if not SaveFileToDisk(File, Content):
 515                     EdkLogger.error(None, FILE_CREATE_FAILURE, ExtraData=File)
 516             except:
 517                 Fd = open(File, "wb")
 518                 Fd.write(Content)
 519                 Fd.close()
 520         else:
 521             Fd = open(File, "wb")
 522             Fd.write(Content)
 523             Fd.close()
 524     except IOError, X:
 525         EdkLogger.error(None, FILE_CREATE_FAILURE, ExtraData='IOError %s' % X)
 526
 527     return True
 528
 529 ## Make a Python object persistent on file system
 530 #
 531 #   @param      Data    The object to be stored in file
 532 #   @param      File    The path of file to store the object
 533 #
 534 def DataDump(Data, File):
 535     Fd = None
 536     try:
 537         Fd = open(File, 'wb')
 538         cPickle.dump(Data, Fd, cPickle.HIGHEST_PROTOCOL)
 539     except:
 540         EdkLogger.error("", FILE_OPEN_FAILURE, ExtraData=File, RaiseError=False)
 541     finally:
 542         if Fd != None:
 543             Fd.close()
 544
 545 ## Restore a Python object from a file
 546 #
 547 #   @param      File    The path of file stored the object
 548 #
 549 #   @retval     object  A python object
 550 #   @retval     None    If failure in file operation
 551 #
 552 def DataRestore(File):
 553     Data = None
 554     Fd = None
 555     try:
 556         Fd = open(File, 'rb')
 557         Data = cPickle.load(Fd)
 558     except Exception, e:
 559         EdkLogger.verbose("Failed to load [%s]\n\t%s" % (File, str(e)))
 560         Data = None
 561     finally:
 562         if Fd != None:
 563             Fd.close()
 564     return Data
 565
 566 ## Retrieve and cache the real path name in file system
 567 #
 568 #   @param      Root    The root directory of path relative to
 569 #
 570 #   @retval     str     The path string if the path exists
 571 #   @retval     None    If path doesn't exist
 572 #
 573 class DirCache:
 574     _CACHE_ = set()
 575     _UPPER_CACHE_ = {}
 576
 577     def __init__(self, Root):
 578         self._Root = Root
 579         for F in os.listdir(Root):
 580             self._CACHE_.add(F)
 581             self._UPPER_CACHE_[F.upper()] = F
 582
 583     # =[] operator
 584     def __getitem__(self, Path):
 585         Path = Path[len(os.path.commonprefix([Path, self._Root])):]
 586         if not Path:
 587             return self._Root
 588         if Path and Path[0] == os.path.sep:
 589             Path = Path[1:]
 590         if Path in self._CACHE_:
 591             return os.path.join(self._Root, Path)
 592         UpperPath = Path.upper()
 593         if UpperPath in self._UPPER_CACHE_:
 594             return os.path.join(self._Root, self._UPPER_CACHE_[UpperPath])
 595
 596         IndexList = []
 597         LastSepIndex = -1
 598         SepIndex = Path.find(os.path.sep)
 599         while SepIndex > -1:
 600             Parent = UpperPath[:SepIndex]
 601             if Parent not in self._UPPER_CACHE_:
 602                 break
 603             LastSepIndex = SepIndex
 604             SepIndex = Path.find(os.path.sep, LastSepIndex + 1)
 605
 606         if LastSepIndex == -1:
 607             return None
 608
 609         Cwd = os.getcwd()
 610         os.chdir(self._Root)
 611         SepIndex = LastSepIndex
 612         while SepIndex > -1:
 613             Parent = Path[:SepIndex]
 614             ParentKey = UpperPath[:SepIndex]
 615             if ParentKey not in self._UPPER_CACHE_:
 616                 os.chdir(Cwd)
 617                 return None
 618
 619             if Parent in self._CACHE_:
 620                 ParentDir = Parent
 621             else:
 622                 ParentDir = self._UPPER_CACHE_[ParentKey]
 623             for F in os.listdir(ParentDir):
 624                 Dir = os.path.join(ParentDir, F)
 625                 self._CACHE_.add(Dir)
 626                 self._UPPER_CACHE_[Dir.upper()] = Dir
 627
 628             SepIndex = Path.find(os.path.sep, SepIndex + 1)
 629
 630         os.chdir(Cwd)
 631         if Path in self._CACHE_:
 632             return os.path.join(self._Root, Path)
 633         elif UpperPath in self._UPPER_CACHE_:
 634             return os.path.join(self._Root, self._UPPER_CACHE_[UpperPath])
 635         return None
 636
 637 ## Get all files of a directory
 638 #
 639 # @param Root:       Root dir
 640 # @param SkipList :  The files need be skipped
 641 #
 642 # @retval  A list of all files
 643 #
 644 def GetFiles(Root, SkipList=None, FullPath=True):
 645     OriPath = Root
 646     FileList = []
 647     for Root, Dirs, Files in os.walk(Root):
 648         if SkipList:
 649             for Item in SkipList:
 650                 if Item in Dirs:
 651                     Dirs.remove(Item)
 652
 653         for File in Files:
 654             File = os.path.normpath(os.path.join(Root, File))
 655             if not FullPath:
 656                 File = File[len(OriPath) + 1:]
 657             FileList.append(File)
 658
 659     return FileList
 660
 661 ## Check if gvien file exists or not
 662 #
 663 #   @param      File    File name or path to be checked
 664 #   @param      Dir     The directory the file is relative to
 665 #
 666 #   @retval     True    if file exists
 667 #   @retval     False   if file doesn't exists
 668 #
 669 def ValidFile(File, Ext=None):
 670     if Ext != None:
 671         Dummy, FileExt = os.path.splitext(File)
 672         if FileExt.lower() != Ext.lower():
 673             return False
 674     if not os.path.exists(File):
 675         return False
 676     return True
 677
 678 def RealPath(File, Dir='', OverrideDir=''):
 679     NewFile = os.path.normpath(os.path.join(Dir, File))
 680     NewFile = GlobalData.gAllFiles[NewFile]
 681     if not NewFile and OverrideDir:
 682         NewFile = os.path.normpath(os.path.join(OverrideDir, File))
 683         NewFile = GlobalData.gAllFiles[NewFile]
 684     return NewFile
 685
 686 def RealPath2(File, Dir='', OverrideDir=''):
 687     NewFile = None
 688     if OverrideDir:
 689         NewFile = GlobalData.gAllFiles[os.path.normpath(os.path.join(OverrideDir, File))]
 690         if NewFile:
 691             if OverrideDir[-1] == os.path.sep:
 692                 return NewFile[len(OverrideDir):], NewFile[0:len(OverrideDir)]
 693             else:
 694                 return NewFile[len(OverrideDir) + 1:], NewFile[0:len(OverrideDir)]
 695     if GlobalData.gAllFiles:
 696         NewFile = GlobalData.gAllFiles[os.path.normpath(os.path.join(Dir, File))]
 697     if not NewFile:
 698         NewFile = os.path.normpath(os.path.join(Dir, File))
 699         if not os.path.exists(NewFile):
 700             return None, None
 701     if NewFile:
 702         if Dir:
 703             if Dir[-1] == os.path.sep:
 704                 return NewFile[len(Dir):], NewFile[0:len(Dir)]
 705             else:
 706                 return NewFile[len(Dir) + 1:], NewFile[0:len(Dir)]
 707         else:
 708             return NewFile, ''
 709
 710     return None, None
 711
 712 ## Check if gvien file exists or not
 713 #
 714 #
 715 def ValidFile2(AllFiles, File, Ext=None, Workspace='', EfiSource='', EdkSource='', Dir='.', OverrideDir=''):
 716     NewFile = File
 717     if Ext != None:
 718         Dummy, FileExt = os.path.splitext(File)
 719         if FileExt.lower() != Ext.lower():
 720             return False, File
 721
 722     # Replace the Edk macros
 723     if OverrideDir != '' and OverrideDir != None:
 724         if OverrideDir.find('$(EFI_SOURCE)') > -1:
 725             OverrideDir = OverrideDir.replace('$(EFI_SOURCE)', EfiSource)
 726         if OverrideDir.find('$(EDK_SOURCE)') > -1:
 727             OverrideDir = OverrideDir.replace('$(EDK_SOURCE)', EdkSource)
 728
 729     # Replace the default dir to current dir
 730     if Dir == '.':
 731         Dir = os.getcwd()
 732         Dir = Dir[len(Workspace) + 1:]
 733
 734     # First check if File has Edk definition itself
 735     if File.find('$(EFI_SOURCE)') > -1 or File.find('$(EDK_SOURCE)') > -1:
 736         NewFile = File.replace('$(EFI_SOURCE)', EfiSource)
 737         NewFile = NewFile.replace('$(EDK_SOURCE)', EdkSource)
 738         NewFile = AllFiles[os.path.normpath(NewFile)]
 739         if NewFile != None:
 740             return True, NewFile
 741
 742     # Second check the path with override value
 743     if OverrideDir != '' and OverrideDir != None:
 744         NewFile = AllFiles[os.path.normpath(os.path.join(OverrideDir, File))]
 745         if NewFile != None:
 746             return True, NewFile
 747
 748     # Last check the path with normal definitions
 749     File = os.path.join(Dir, File)
 750     NewFile = AllFiles[os.path.normpath(File)]
 751     if NewFile != None:
 752         return True, NewFile
 753
 754     return False, File
 755
 756 ## Check if gvien file exists or not
 757 #
 758 #
 759 def ValidFile3(AllFiles, File, Workspace='', EfiSource='', EdkSource='', Dir='.', OverrideDir=''):
 760     # Replace the Edk macros
 761     if OverrideDir != '' and OverrideDir != None:
 762         if OverrideDir.find('$(EFI_SOURCE)') > -1:
 763             OverrideDir = OverrideDir.replace('$(EFI_SOURCE)', EfiSource)
 764         if OverrideDir.find('$(EDK_SOURCE)') > -1:
 765             OverrideDir = OverrideDir.replace('$(EDK_SOURCE)', EdkSource)
 766
 767     # Replace the default dir to current dir
 768     # Dir is current module dir related to workspace
 769     if Dir == '.':
 770         Dir = os.getcwd()
 771         Dir = Dir[len(Workspace) + 1:]
 772
 773     NewFile = File
 774     RelaPath = AllFiles[os.path.normpath(Dir)]
 775     NewRelaPath = RelaPath
 776
 777     while(True):
 778         # First check if File has Edk definition itself
 779         if File.find('$(EFI_SOURCE)') > -1 or File.find('$(EDK_SOURCE)') > -1:
 780             File = File.replace('$(EFI_SOURCE)', EfiSource)
 781             File = File.replace('$(EDK_SOURCE)', EdkSource)
 782             NewFile = AllFiles[os.path.normpath(File)]
 783             if NewFile != None:
 784                 NewRelaPath = os.path.dirname(NewFile)
 785                 File = os.path.basename(NewFile)
 786                 #NewRelaPath = NewFile[:len(NewFile) - len(File.replace("..\\", '').replace("../", '')) - 1]
 787                 break
 788
 789         # Second check the path with override value
 790         if OverrideDir != '' and OverrideDir != None:
 791             NewFile = AllFiles[os.path.normpath(os.path.join(OverrideDir, File))]
 792             if NewFile != None:
 793                 #NewRelaPath = os.path.dirname(NewFile)
 794                 NewRelaPath = NewFile[:len(NewFile) - len(File.replace("..\\", '').replace("../", '')) - 1]
 795                 break
 796
 797         # Last check the path with normal definitions
 798         NewFile = AllFiles[os.path.normpath(os.path.join(Dir, File))]
 799         if NewFile != None:
 800             break
 801
 802         # No file found
 803         break
 804
 805     return NewRelaPath, RelaPath, File
 806
 807
 808 def GetRelPath(Path1, Path2):
 809     FileName = os.path.basename(Path2)
 810     L1 = os.path.normpath(Path1).split(os.path.normpath('/'))
 811     L2 = os.path.normpath(Path2).split(os.path.normpath('/'))
 812     for Index in range(0, len(L1)):
 813         if L1[Index] != L2[Index]:
 814             FileName = '../' * (len(L1) - Index)
 815             for Index2 in range(Index, len(L2)):
 816                 FileName = os.path.join(FileName, L2[Index2])
 817             break
 818     return os.path.normpath(FileName)
 819
 820
 821 ## Get GUID value from given packages
 822 #
 823 #   @param      CName           The CName of the GUID
 824 #   @param      PackageList     List of packages looking-up in
 825 #   @param      Inffile         The driver file
 826 #
 827 #   @retval     GuidValue   if the CName is found in any given package
 828 #   @retval     None        if the CName is not found in all given packages
 829 #
 830 def GuidValue(CName, PackageList, Inffile = None):
 831     for P in PackageList:
 832         GuidKeys = P.Guids.keys()
 833         if Inffile and P._PrivateGuids:
 834             if not Inffile.startswith(P.MetaFile.Dir):
 835                 GuidKeys = (dict.fromkeys(x for x in P.Guids if x not in P._PrivateGuids)).keys()
 836         if CName in GuidKeys:
 837             return P.Guids[CName]
 838     return None
 839
 840 ## Get Protocol value from given packages
 841 #
 842 #   @param      CName           The CName of the GUID
 843 #   @param      PackageList     List of packages looking-up in
 844 #   @param      Inffile         The driver file
 845 #
 846 #   @retval     GuidValue   if the CName is found in any given package
 847 #   @retval     None        if the CName is not found in all given packages
 848 #
 849 def ProtocolValue(CName, PackageList, Inffile = None):
 850     for P in PackageList:
 851         ProtocolKeys = P.Protocols.keys()
 852         if Inffile and P._PrivateProtocols:
 853             if not Inffile.startswith(P.MetaFile.Dir):
 854                 ProtocolKeys = (dict.fromkeys(x for x in P.Protocols if x not in P._PrivateProtocols)).keys()
 855         if CName in ProtocolKeys:
 856             return P.Protocols[CName]
 857     return None
 858
 859 ## Get PPI value from given packages
 860 #
 861 #   @param      CName           The CName of the GUID
 862 #   @param      PackageList     List of packages looking-up in
 863 #   @param      Inffile         The driver file
 864 #
 865 #   @retval     GuidValue   if the CName is found in any given package
 866 #   @retval     None        if the CName is not found in all given packages
 867 #
 868 def PpiValue(CName, PackageList, Inffile = None):
 869     for P in PackageList:
 870         PpiKeys = P.Ppis.keys()
 871         if Inffile and P._PrivatePpis:
 872             if not Inffile.startswith(P.MetaFile.Dir):
 873                 PpiKeys = (dict.fromkeys(x for x in P.Ppis if x not in P._PrivatePpis)).keys()
 874         if CName in PpiKeys:
 875             return P.Ppis[CName]
 876     return None
 877
 878 ## A string template class
 879 #
 880 #  This class implements a template for string replacement. A string template
 881 #  looks like following
 882 #
 883 #       ${BEGIN} other_string ${placeholder_name} other_string ${END}
 884 #
 885 #  The string between ${BEGIN} and ${END} will be repeated as many times as the
 886 #  length of "placeholder_name", which is a list passed through a dict. The
 887 #  "placeholder_name" is the key name of the dict. The ${BEGIN} and ${END} can
 888 #  be not used and, in this case, the "placeholder_name" must not a list and it
 889 #  will just be replaced once.
 890 #
 891 class TemplateString(object):
 892     _REPEAT_START_FLAG = "BEGIN"
 893     _REPEAT_END_FLAG = "END"
 894
 895     class Section(object):
 896         _LIST_TYPES = [type([]), type(set()), type((0,))]
 897
 898         def __init__(self, TemplateSection, PlaceHolderList):
 899             self._Template = TemplateSection
 900             self._PlaceHolderList = []
 901
 902             # Split the section into sub-sections according to the position of placeholders
 903             if PlaceHolderList:
 904                 self._SubSectionList = []
 905                 SubSectionStart = 0
 906                 #
 907                 # The placeholders passed in must be in the format of
 908                 #
 909                 #   PlaceHolderName, PlaceHolderStartPoint, PlaceHolderEndPoint
 910                 #
 911                 for PlaceHolder, Start, End in PlaceHolderList:
 912                     self._SubSectionList.append(TemplateSection[SubSectionStart:Start])
 913                     self._SubSectionList.append(TemplateSection[Start:End])
 914                     self._PlaceHolderList.append(PlaceHolder)
 915                     SubSectionStart = End
 916                 if SubSectionStart < len(TemplateSection):
 917                     self._SubSectionList.append(TemplateSection[SubSectionStart:])
 918             else:
 919                 self._SubSectionList = [TemplateSection]
 920
 921         def __str__(self):
 922             return self._Template + " : " + str(self._PlaceHolderList)
 923
 924         def Instantiate(self, PlaceHolderValues):
 925             RepeatTime = -1
 926             RepeatPlaceHolders = {}
 927             NonRepeatPlaceHolders = {}
 928
 929             for PlaceHolder in self._PlaceHolderList:
 930                 if PlaceHolder not in PlaceHolderValues:
 931                     continue
 932                 Value = PlaceHolderValues[PlaceHolder]
 933                 if type(Value) in self._LIST_TYPES:
 934                     if RepeatTime < 0:
 935                         RepeatTime = len(Value)
 936                     elif RepeatTime != len(Value):
 937                         EdkLogger.error(
 938                                     "TemplateString",
 939                                     PARAMETER_INVALID,
 940                                     "${%s} has different repeat time from others!" % PlaceHolder,
 941                                     ExtraData=str(self._Template)
 942                                     )
 943                     RepeatPlaceHolders["${%s}" % PlaceHolder] = Value
 944                 else:
 945                     NonRepeatPlaceHolders["${%s}" % PlaceHolder] = Value
 946
 947             if NonRepeatPlaceHolders:
 948                 StringList = []
 949                 for S in self._SubSectionList:
 950                     if S not in NonRepeatPlaceHolders:
 951                         StringList.append(S)
 952                     else:
 953                         StringList.append(str(NonRepeatPlaceHolders[S]))
 954             else:
 955                 StringList = self._SubSectionList
 956
 957             if RepeatPlaceHolders:
 958                 TempStringList = []
 959                 for Index in range(RepeatTime):
 960                     for S in StringList:
 961                         if S not in RepeatPlaceHolders:
 962                             TempStringList.append(S)
 963                         else:
 964                             TempStringList.append(str(RepeatPlaceHolders[S][Index]))
 965                 StringList = TempStringList
 966
 967             return "".join(StringList)
 968
 969     ## Constructor
 970     def __init__(self, Template=None):
 971         self.String = ''
 972         self.IsBinary = False
 973         self._Template = Template
 974         self._TemplateSectionList = self._Parse(Template)
 975
 976     ## str() operator
 977     #
 978     #   @retval     string  The string replaced
 979     #
 980     def __str__(self):
 981         return self.String
 982
 983     ## Split the template string into fragments per the ${BEGIN} and ${END} flags
 984     #
 985     #   @retval     list    A list of TemplateString.Section objects
 986     #
 987     def _Parse(self, Template):
 988         SectionStart = 0
 989         SearchFrom = 0
 990         MatchEnd = 0
 991         PlaceHolderList = []
 992         TemplateSectionList = []
 993         while Template:
 994             MatchObj = gPlaceholderPattern.search(Template, SearchFrom)
 995             if not MatchObj:
 996                 if MatchEnd <= len(Template):
 997                     TemplateSection = TemplateString.Section(Template[SectionStart:], PlaceHolderList)
 998                     TemplateSectionList.append(TemplateSection)
 999                 break
1000
1001             MatchString = MatchObj.group(1)
1002             MatchStart = MatchObj.start()
1003             MatchEnd = MatchObj.end()
1004
1005             if MatchString == self._REPEAT_START_FLAG:
1006                 if MatchStart > SectionStart:
1007                     TemplateSection = TemplateString.Section(Template[SectionStart:MatchStart], PlaceHolderList)
1008                     TemplateSectionList.append(TemplateSection)
1009                 SectionStart = MatchEnd
1010                 PlaceHolderList = []
1011             elif MatchString == self._REPEAT_END_FLAG:
1012                 TemplateSection = TemplateString.Section(Template[SectionStart:MatchStart], PlaceHolderList)
1013                 TemplateSectionList.append(TemplateSection)
1014                 SectionStart = MatchEnd
1015                 PlaceHolderList = []
1016             else:
1017                 PlaceHolderList.append((MatchString, MatchStart - SectionStart, MatchEnd - SectionStart))
1018             SearchFrom = MatchEnd
1019         return TemplateSectionList
1020
1021     ## Replace the string template with dictionary of placeholders and append it to previous one
1022     #
1023     #   @param      AppendString    The string template to append
1024     #   @param      Dictionary      The placeholder dictionaries
1025     #
1026     def Append(self, AppendString, Dictionary=None):
1027         if Dictionary:
1028             SectionList = self._Parse(AppendString)
1029             self.String += "".join([S.Instantiate(Dictionary) for S in SectionList])
1030         else:
1031             self.String += AppendString
1032
1033     ## Replace the string template with dictionary of placeholders
1034     #
1035     #   @param      Dictionary      The placeholder dictionaries
1036     #
1037     #   @retval     str             The string replaced with placeholder values
1038     #
1039     def Replace(self, Dictionary=None):
1040         return "".join([S.Instantiate(Dictionary) for S in self._TemplateSectionList])
1041
1042 ## Progress indicator class
1043 #
1044 #  This class makes use of thread to print progress on console.
1045 #
1046 class Progressor:
1047     # for avoiding deadloop
1048     _StopFlag = None
1049     _ProgressThread = None
1050     _CheckInterval = 0.25
1051
1052     ## Constructor
1053     #
1054     #   @param      OpenMessage     The string printed before progress charaters
1055     #   @param      CloseMessage    The string printed after progress charaters
1056     #   @param      ProgressChar    The charater used to indicate the progress
1057     #   @param      Interval        The interval in seconds between two progress charaters
1058     #
1059     def __init__(self, OpenMessage="", CloseMessage="", ProgressChar='.', Interval=1.0):
1060         self.PromptMessage = OpenMessage
1061         self.CodaMessage = CloseMessage
1062         self.ProgressChar = ProgressChar
1063         self.Interval = Interval
1064         if Progressor._StopFlag == None:
1065             Progressor._StopFlag = threading.Event()
1066
1067     ## Start to print progress charater
1068     #
1069     #   @param      OpenMessage     The string printed before progress charaters
1070     #
1071     def Start(self, OpenMessage=None):
1072         if OpenMessage != None:
1073             self.PromptMessage = OpenMessage
1074         Progressor._StopFlag.clear()
1075         if Progressor._ProgressThread == None:
1076             Progressor._ProgressThread = threading.Thread(target=self._ProgressThreadEntry)
1077             Progressor._ProgressThread.setDaemon(False)
1078             Progressor._ProgressThread.start()
1079
1080     ## Stop printing progress charater
1081     #
1082     #   @param      CloseMessage    The string printed after progress charaters
1083     #
1084     def Stop(self, CloseMessage=None):
1085         OriginalCodaMessage = self.CodaMessage
1086         if CloseMessage != None:
1087             self.CodaMessage = CloseMessage
1088         self.Abort()
1089         self.CodaMessage = OriginalCodaMessage
1090
1091     ## Thread entry method
1092     def _ProgressThreadEntry(self):
1093         sys.stdout.write(self.PromptMessage + " ")
1094         sys.stdout.flush()
1095         TimeUp = 0.0
1096         while not Progressor._StopFlag.isSet():
1097             if TimeUp <= 0.0:
1098                 sys.stdout.write(self.ProgressChar)
1099                 sys.stdout.flush()
1100                 TimeUp = self.Interval
1101             time.sleep(self._CheckInterval)
1102             TimeUp -= self._CheckInterval
1103         sys.stdout.write(" " + self.CodaMessage + "\n")
1104         sys.stdout.flush()
1105
1106     ## Abort the progress display
1107     @staticmethod
1108     def Abort():
1109         if Progressor._StopFlag != None:
1110             Progressor._StopFlag.set()
1111         if Progressor._ProgressThread != None:
1112             Progressor._ProgressThread.join()
1113             Progressor._ProgressThread = None
1114
1115 ## A dict which can access its keys and/or values orderly
1116 #
1117 #  The class implements a new kind of dict which its keys or values can be
1118 #  accessed in the order they are added into the dict. It guarantees the order
1119 #  by making use of an internal list to keep a copy of keys.
1120 #
1121 class sdict(IterableUserDict):
1122     ## Constructor
1123     def __init__(self):
1124         IterableUserDict.__init__(self)
1125         self._key_list = []
1126
1127     ## [] operator
1128     def __setitem__(self, key, value):
1129         if key not in self._key_list:
1130             self._key_list.append(key)
1131         IterableUserDict.__setitem__(self, key, value)
1132
1133     ## del operator
1134     def __delitem__(self, key):
1135         self._key_list.remove(key)
1136         IterableUserDict.__delitem__(self, key)
1137
1138     ## used in "for k in dict" loop to ensure the correct order
1139     def __iter__(self):
1140         return self.iterkeys()
1141
1142     ## len() support
1143     def __len__(self):
1144         return len(self._key_list)
1145
1146     ## "in" test support
1147     def __contains__(self, key):
1148         return key in self._key_list
1149
1150     ## indexof support
1151     def index(self, key):
1152         return self._key_list.index(key)
1153
1154     ## insert support
1155     def insert(self, key, newkey, newvalue, order):
1156         index = self._key_list.index(key)
1157         if order == 'BEFORE':
1158             self._key_list.insert(index, newkey)
1159             IterableUserDict.__setitem__(self, newkey, newvalue)
1160         elif order == 'AFTER':
1161             self._key_list.insert(index + 1, newkey)
1162             IterableUserDict.__setitem__(self, newkey, newvalue)
1163
1164     ## append support
1165     def append(self, sdict):
1166         for key in sdict:
1167             if key not in self._key_list:
1168                 self._key_list.append(key)
1169             IterableUserDict.__setitem__(self, key, sdict[key])
1170
1171     def has_key(self, key):
1172         return key in self._key_list
1173
1174     ## Empty the dict
1175     def clear(self):
1176         self._key_list = []
1177         IterableUserDict.clear(self)
1178
1179     ## Return a copy of keys
1180     def keys(self):
1181         keys = []
1182         for key in self._key_list:
1183             keys.append(key)
1184         return keys
1185
1186     ## Return a copy of values
1187     def values(self):
1188         values = []
1189         for key in self._key_list:
1190             values.append(self[key])
1191         return values
1192
1193     ## Return a copy of (key, value) list
1194     def items(self):
1195         items = []
1196         for key in self._key_list:
1197             items.append((key, self[key]))
1198         return items
1199
1200     ## Iteration support
1201     def iteritems(self):
1202         return iter(self.items())
1203
1204     ## Keys interation support
1205     def iterkeys(self):
1206         return iter(self.keys())
1207
1208     ## Values interation support
1209     def itervalues(self):
1210         return iter(self.values())
1211
1212     ## Return value related to a key, and remove the (key, value) from the dict
1213     def pop(self, key, *dv):
1214         value = None
1215         if key in self._key_list:
1216             value = self[key]
1217             self.__delitem__(key)
1218         elif len(dv) != 0 :
1219             value = kv[0]
1220         return value
1221
1222     ## Return (key, value) pair, and remove the (key, value) from the dict
1223     def popitem(self):
1224         key = self._key_list[-1]
1225         value = self[key]
1226         self.__delitem__(key)
1227         return key, value
1228
1229     def update(self, dict=None, **kwargs):
1230         if dict != None:
1231             for k, v in dict.items():
1232                 self[k] = v
1233         if len(kwargs):
1234             for k, v in kwargs.items():
1235                 self[k] = v
1236
1237 ## Dictionary with restricted keys
1238 #
1239 class rdict(dict):
1240     ## Constructor
1241     def __init__(self, KeyList):
1242         for Key in KeyList:
1243             dict.__setitem__(self, Key, "")
1244
1245     ## []= operator
1246     def __setitem__(self, key, value):
1247         if key not in self:
1248             EdkLogger.error("RestrictedDict", ATTRIBUTE_SET_FAILURE, "Key [%s] is not allowed" % key,
1249                             ExtraData=", ".join(dict.keys(self)))
1250         dict.__setitem__(self, key, value)
1251
1252     ## =[] operator
1253     def __getitem__(self, key):
1254         if key not in self:
1255             return ""
1256         return dict.__getitem__(self, key)
1257
1258     ## del operator
1259     def __delitem__(self, key):
1260         EdkLogger.error("RestrictedDict", ATTRIBUTE_ACCESS_DENIED, ExtraData="del")
1261
1262     ## Empty the dict
1263     def clear(self):
1264         for Key in self:
1265             self.__setitem__(Key, "")
1266
1267     ## Return value related to a key, and remove the (key, value) from the dict
1268     def pop(self, key, *dv):
1269         EdkLogger.error("RestrictedDict", ATTRIBUTE_ACCESS_DENIED, ExtraData="pop")
1270
1271     ## Return (key, value) pair, and remove the (key, value) from the dict
1272     def popitem(self):
1273         EdkLogger.error("RestrictedDict", ATTRIBUTE_ACCESS_DENIED, ExtraData="popitem")
1274
1275 ## Dictionary using prioritized list as key
1276 #
1277 class tdict:
1278     _ListType = type([])
1279     _TupleType = type(())
1280     _Wildcard = 'COMMON'
1281     _ValidWildcardList = ['COMMON', 'DEFAULT', 'ALL', '*', 'PLATFORM']
1282
1283     def __init__(self, _Single_=False, _Level_=2):
1284         self._Level_ = _Level_
1285         self.data = {}
1286         self._Single_ = _Single_
1287
1288     # =[] operator
1289     def __getitem__(self, key):
1290         KeyType = type(key)
1291         RestKeys = None
1292         if KeyType == self._ListType or KeyType == self._TupleType:
1293             FirstKey = key[0]
1294             if len(key) > 1:
1295                 RestKeys = key[1:]
1296             elif self._Level_ > 1:
1297                 RestKeys = [self._Wildcard for i in range(0, self._Level_ - 1)]
1298         else:
1299             FirstKey = key
1300             if self._Level_ > 1:
1301                 RestKeys = [self._Wildcard for i in range(0, self._Level_ - 1)]
1302
1303         if FirstKey == None or str(FirstKey).upper() in self._ValidWildcardList:
1304             FirstKey = self._Wildcard
1305
1306         if self._Single_:
1307             return self._GetSingleValue(FirstKey, RestKeys)
1308         else:
1309             return self._GetAllValues(FirstKey, RestKeys)
1310
1311     def _GetSingleValue(self, FirstKey, RestKeys):
1312         Value = None
1313         #print "%s-%s" % (FirstKey, self._Level_) ,
1314         if self._Level_ > 1:
1315             if FirstKey == self._Wildcard:
1316                 if FirstKey in self.data:
1317                     Value = self.data[FirstKey][RestKeys]
1318                 if Value == None:
1319                     for Key in self.data:
1320                         Value = self.data[Key][RestKeys]
1321                         if Value != None: break
1322             else:
1323                 if FirstKey in self.data:
1324                     Value = self.data[FirstKey][RestKeys]
1325                 if Value == None and self._Wildcard in self.data:
1326                     #print "Value=None"
1327                     Value = self.data[self._Wildcard][RestKeys]
1328         else:
1329             if FirstKey == self._Wildcard:
1330                 if FirstKey in self.data:
1331                     Value = self.data[FirstKey]
1332                 if Value == None:
1333                     for Key in self.data:
1334                         Value = self.data[Key]
1335                         if Value != None: break
1336             else:
1337                 if FirstKey in self.data:
1338                     Value = self.data[FirstKey]
1339                 elif self._Wildcard in self.data:
1340                     Value = self.data[self._Wildcard]
1341         return Value
1342
1343     def _GetAllValues(self, FirstKey, RestKeys):
1344         Value = []
1345         if self._Level_ > 1:
1346             if FirstKey == self._Wildcard:
1347                 for Key in self.data:
1348                     Value += self.data[Key][RestKeys]
1349             else:
1350                 if FirstKey in self.data:
1351                     Value += self.data[FirstKey][RestKeys]
1352                 if self._Wildcard in self.data:
1353                     Value += self.data[self._Wildcard][RestKeys]
1354         else:
1355             if FirstKey == self._Wildcard:
1356                 for Key in self.data:
1357                     Value.append(self.data[Key])
1358             else:
1359                 if FirstKey in self.data:
1360                     Value.append(self.data[FirstKey])
1361                 if self._Wildcard in self.data:
1362                     Value.append(self.data[self._Wildcard])
1363         return Value
1364
1365     ## []= operator
1366     def __setitem__(self, key, value):
1367         KeyType = type(key)
1368         RestKeys = None
1369         if KeyType == self._ListType or KeyType == self._TupleType:
1370             FirstKey = key[0]
1371             if len(key) > 1:
1372                 RestKeys = key[1:]
1373             else:
1374                 RestKeys = [self._Wildcard for i in range(0, self._Level_ - 1)]
1375         else:
1376             FirstKey = key
1377             if self._Level_ > 1:
1378                 RestKeys = [self._Wildcard for i in range(0, self._Level_ - 1)]
1379
1380         if FirstKey in self._ValidWildcardList:
1381             FirstKey = self._Wildcard
1382
1383         if FirstKey not in self.data and self._Level_ > 0:
1384             self.data[FirstKey] = tdict(self._Single_, self._Level_ - 1)
1385
1386         if self._Level_ > 1:
1387             self.data[FirstKey][RestKeys] = value
1388         else:
1389             self.data[FirstKey] = value
1390
1391     def SetGreedyMode(self):
1392         self._Single_ = False
1393         if self._Level_ > 1:
1394             for Key in self.data:
1395                 self.data[Key].SetGreedyMode()
1396
1397     def SetSingleMode(self):
1398         self._Single_ = True
1399         if self._Level_ > 1:
1400             for Key in self.data:
1401                 self.data[Key].SetSingleMode()
1402
1403     def GetKeys(self, KeyIndex=0):
1404         assert KeyIndex >= 0
1405         if KeyIndex == 0:
1406             return set(self.data.keys())
1407         else:
1408             keys = set()
1409             for Key in self.data:
1410                 keys |= self.data[Key].GetKeys(KeyIndex - 1)
1411             return keys
1412
1413 ## Boolean chain list
1414 #
1415 class Blist(UserList):
1416     def __init__(self, initlist=None):
1417         UserList.__init__(self, initlist)
1418     def __setitem__(self, i, item):
1419         if item not in [True, False]:
1420             if item == 0:
1421                 item = False
1422             else:
1423                 item = True
1424         self.data[i] = item
1425     def _GetResult(self):
1426         Value = True
1427         for item in self.data:
1428             Value &= item
1429         return Value
1430     Result = property(_GetResult)
1431
1432 def ParseConsoleLog(Filename):
1433     Opr = open(os.path.normpath(Filename), 'r')
1434     Opw = open(os.path.normpath(Filename + '.New'), 'w+')
1435     for Line in Opr.readlines():
1436         if Line.find('.efi') > -1:
1437             Line = Line[Line.rfind(' ') : Line.rfind('.efi')].strip()
1438             Opw.write('%s\n' % Line)
1439
1440     Opr.close()
1441     Opw.close()
1442
1443 def AnalyzePcdExpression(Setting):
1444     Setting = Setting.strip()
1445     # There might be escaped quote in a string: \", \\\"
1446     Data = Setting.replace('\\\\', '//').replace('\\\"', '\\\'')
1447     # There might be '|' in string and in ( ... | ... ), replace it with '-'
1448     NewStr = ''
1449     InStr = False
1450     Pair = 0
1451     for ch in Data:
1452         if ch == '"':
1453             InStr = not InStr
1454         elif ch == '(' and not InStr:
1455             Pair += 1
1456         elif ch == ')' and not InStr:
1457             Pair -= 1
1458
1459         if (Pair > 0 or InStr) and ch == TAB_VALUE_SPLIT:
1460             NewStr += '-'
1461         else:
1462             NewStr += ch
1463     FieldList = []
1464     StartPos = 0
1465     while True:
1466         Pos = NewStr.find(TAB_VALUE_SPLIT, StartPos)
1467         if Pos < 0:
1468             FieldList.append(Setting[StartPos:].strip())
1469             break
1470         FieldList.append(Setting[StartPos:Pos].strip())
1471         StartPos = Pos + 1
1472
1473     return FieldList
1474
1475 def ParseFieldValue (Value):
1476   if type(Value) == type(0):
1477     return Value, (Value.bit_length() + 7) / 8
1478   if type(Value) <> type(''):
1479     raise ValueError
1480   Value = Value.strip()
1481   if Value.startswith('UINT8') and Value.endswith(')'):
1482     Value, Size = ParseFieldValue(Value.split('(', 1)[1][:-1])
1483     if Size > 1:
1484       raise ValueError
1485     return Value, 1
1486   if Value.startswith('UINT16') and Value.endswith(')'):
1487     Value, Size = ParseFieldValue(Value.split('(', 1)[1][:-1])
1488     if Size > 2:
1489       raise ValueError
1490     return Value, 2
1491   if Value.startswith('UINT32') and Value.endswith(')'):
1492     Value, Size = ParseFieldValue(Value.split('(', 1)[1][:-1])
1493     if Size > 4:
1494       raise ValueError
1495     return Value, 4
1496   if Value.startswith('UINT64') and Value.endswith(')'):
1497     Value, Size = ParseFieldValue(Value.split('(', 1)[1][:-1])
1498     if Size > 8:
1499       raise ValueError
1500     return Value, 8
1501   if Value.startswith('GUID') and Value.endswith(')'):
1502     Value = Value.split('(', 1)[1][:-1].strip()
1503     if Value[0] == '{' and Value[-1] == '}':
1504       Value = Value[1:-1].strip()
1505       Value = Value.split('{', 1)
1506       Value = [Item.strip()[2:] for Item in (Value[0] + Value[1][:-1]).split(',')]
1507       Value = '-'.join(Value[0:3]) + '-' + ''.join(Value[3:5]) + '-' + ''.join(Value[5:11])
1508     if Value[0] == '"' and Value[-1] == '"':
1509       Value = Value[1:-1]
1510     Value = "'" + uuid.UUID(Value).get_bytes_le() + "'"
1511     Value, Size = ParseFieldValue(Value)
1512     return Value, 16
1513   if Value.startswith('L"') and Value.endswith('"'):
1514     # Unicode String
1515     List = list(Value[2:-1])
1516     List.reverse()
1517     Value = 0
1518     for Char in List:
1519       Value = (Value << 16) | ord(Char)
1520     return Value, (len(List) + 1) * 2
1521   if Value.startswith('"') and Value.endswith('"'):
1522     # ASCII String
1523     List = list(Value[1:-1])
1524     List.reverse()
1525     Value = 0
1526     for Char in List:
1527       Value = (Value << 8) | ord(Char)
1528     return Value, len(List) + 1
1529   if Value.startswith("L'") and Value.endswith("'"):
1530     # Unicode Character Constant
1531     List = list(Value[2:-1])
1532     List.reverse()
1533     Value = 0
1534     for Char in List:
1535       Value = (Value << 16) | ord(Char)
1536     return Value, len(List) * 2
1537   if Value.startswith("'") and Value.endswith("'"):
1538     # Character constant
1539     List = list(Value[1:-1])
1540     List.reverse()
1541     Value = 0
1542     for Char in List:
1543       Value = (Value << 8) | ord(Char)
1544     return Value, len(List)
1545   if Value.startswith('{') and Value.endswith('}'):
1546     # Byte array
1547     Value = Value[1:-1]
1548     List = [Item.strip() for Item in Value.split(',')]
1549     List.reverse()
1550     Value = 0
1551     for Item in List:
1552       ItemValue, Size = ParseFieldValue(Item)
1553       if Size > 1:
1554         raise ValueError
1555       Value = (Value << 8) | ItemValue
1556     return Value, len(List)
1557   if Value.lower().startswith('0x'):
1558     Value = int(Value, 16)
1559     return Value, (Value.bit_length() + 7) / 8
1560   if Value[0].isdigit():
1561     Value = int(Value, 10)
1562     return Value, (Value.bit_length() + 7) / 8
1563   if Value.lower() == 'true':
1564     return 1, 1
1565   if Value.lower() == 'false':
1566     return 0, 1
1567   return Value, 1
1568
1569 ## AnalyzeDscPcd
1570 #
1571 #  Analyze DSC PCD value, since there is no data type info in DSC
1572 #  This fuction is used to match functions (AnalyzePcdData, AnalyzeHiiPcdData, AnalyzeVpdPcdData) used for retrieving PCD value from database
1573 #  1. Feature flag: TokenSpace.PcdCName|PcdValue
1574 #  2. Fix and Patch:TokenSpace.PcdCName|PcdValue[|MaxSize]
1575 #  3. Dynamic default:
1576 #     TokenSpace.PcdCName|PcdValue[|VOID*[|MaxSize]]
1577 #     TokenSpace.PcdCName|PcdValue
1578 #  4. Dynamic VPD:
1579 #     TokenSpace.PcdCName|VpdOffset[|VpdValue]
1580 #     TokenSpace.PcdCName|VpdOffset[|MaxSize[|VpdValue]]
1581 #  5. Dynamic HII:
1582 #     TokenSpace.PcdCName|HiiString|VaiableGuid|VariableOffset[|HiiValue]
1583 #  PCD value needs to be located in such kind of string, and the PCD value might be an expression in which
1584 #    there might have "|" operator, also in string value.
1585 #
1586 #  @param Setting: String contain information described above with "TokenSpace.PcdCName|" stripped
1587 #  @param PcdType: PCD type: feature, fixed, dynamic default VPD HII
1588 #  @param DataType: The datum type of PCD: VOID*, UNIT, BOOL
1589 #  @retval:
1590 #    ValueList: A List contain fields described above
1591 #    IsValid:   True if conforming EBNF, otherwise False
1592 #    Index:     The index where PcdValue is in ValueList
1593 #
1594 def AnalyzeDscPcd(Setting, PcdType, DataType=''):
1595     FieldList = AnalyzePcdExpression(Setting)
1596
1597     IsValid = True
1598     if PcdType in (MODEL_PCD_FIXED_AT_BUILD, MODEL_PCD_PATCHABLE_IN_MODULE, MODEL_PCD_FEATURE_FLAG):
1599         Value = FieldList[0]
1600         Size = ''
1601         if len(FieldList) > 1:
1602             if FieldList[1].upper().startswith("0X") or FieldList[1].isdigit():
1603                 Size = FieldList[1]
1604             else:
1605                 DataType = FieldList[1]
1606
1607         if len(FieldList) > 2:
1608             Size = FieldList[2]
1609         if DataType == "":
1610             IsValid = (len(FieldList) <= 1)
1611         else:
1612             IsValid = (len(FieldList) <= 3)
1613 #         Value, Size = ParseFieldValue(Value)
1614         if Size:
1615             try:
1616                 int(Size,16) if Size.upper().startswith("0X") else int(Size)
1617             except:
1618                 IsValid = False
1619                 Size = -1
1620         return [str(Value), '', str(Size)], IsValid, 0
1621     elif PcdType in (MODEL_PCD_DYNAMIC_DEFAULT, MODEL_PCD_DYNAMIC_EX_DEFAULT):
1622         Value = FieldList[0]
1623         Size = Type = ''
1624         if len(FieldList) > 1:
1625             Type = FieldList[1]
1626         else:
1627             Type = DataType
1628         if len(FieldList) > 2:
1629             Size = FieldList[2]
1630         else:
1631             if Type == 'VOID*':
1632                 if Value.startswith("L"):
1633                     Size = str((len(Value)- 3 + 1) * 2)
1634                 elif Value.startswith("{"):
1635                     Size = str(len(Value.split(",")))
1636                 else:
1637                     Size = str(len(Value) -2 + 1 )
1638         if DataType == "":
1639             IsValid = (len(FieldList) <= 1)
1640         else:
1641             IsValid = (len(FieldList) <= 3)
1642
1643         if Size:
1644             try:
1645                 int(Size,16) if Size.upper().startswith("0X") else int(Size)
1646             except:
1647                 IsValid = False
1648                 Size = -1
1649         return [Value, Type, str(Size)], IsValid, 0
1650     elif PcdType in (MODEL_PCD_DYNAMIC_VPD, MODEL_PCD_DYNAMIC_EX_VPD):
1651         VpdOffset = FieldList[0]
1652         Value = Size = ''
1653         if not DataType == 'VOID*':
1654             if len(FieldList) > 1:
1655                 Value = FieldList[1]
1656         else:
1657             if len(FieldList) > 1:
1658                 Size = FieldList[1]
1659             if len(FieldList) > 2:
1660                 Value = FieldList[2]
1661         if DataType == "":
1662             IsValid = (len(FieldList) <= 1)
1663         else:
1664             IsValid = (len(FieldList) <= 3)
1665         if Size:
1666             try:
1667                 int(Size,16) if Size.upper().startswith("0X") else int(Size)
1668             except:
1669                 IsValid = False
1670                 Size = -1
1671         return [VpdOffset, str(Size), Value], IsValid, 2
1672     elif PcdType in (MODEL_PCD_DYNAMIC_HII, MODEL_PCD_DYNAMIC_EX_HII):
1673         HiiString = FieldList[0]
1674         Guid = Offset = Value = Attribute = ''
1675         if len(FieldList) > 1:
1676             Guid = FieldList[1]
1677         if len(FieldList) > 2:
1678             Offset = FieldList[2]
1679         if len(FieldList) > 3:
1680             Value = FieldList[3]
1681         if len(FieldList) > 4:
1682             Attribute = FieldList[4]
1683         IsValid = (3 <= len(FieldList) <= 5)
1684         return [HiiString, Guid, Offset, Value, Attribute], IsValid, 3
1685     return [], False, 0
1686
1687 ## AnalyzePcdData
1688 #
1689 #  Analyze the pcd Value, Datum type and TokenNumber.
1690 #  Used to avoid split issue while the value string contain "|" character
1691 #
1692 #  @param[in] Setting:  A String contain value/datum type/token number information;
1693 #
1694 #  @retval   ValueList: A List contain value, datum type and toke number.
1695 #
1696 def AnalyzePcdData(Setting):
1697     ValueList = ['', '', '']
1698
1699     ValueRe = re.compile(r'^\s*L?\".*\|.*\"')
1700     PtrValue = ValueRe.findall(Setting)
1701
1702     ValueUpdateFlag = False
1703
1704     if len(PtrValue) >= 1:
1705         Setting = re.sub(ValueRe, '', Setting)
1706         ValueUpdateFlag = True
1707
1708     TokenList = Setting.split(TAB_VALUE_SPLIT)
1709     ValueList[0:len(TokenList)] = TokenList
1710
1711     if ValueUpdateFlag:
1712         ValueList[0] = PtrValue[0]
1713
1714     return ValueList
1715
1716 ## AnalyzeHiiPcdData
1717 #
1718 #  Analyze the pcd Value, variable name, variable Guid and variable offset.
1719 #  Used to avoid split issue while the value string contain "|" character
1720 #
1721 #  @param[in] Setting:  A String contain VariableName, VariableGuid, VariableOffset, DefaultValue information;
1722 #
1723 #  @retval   ValueList: A List contaian VariableName, VariableGuid, VariableOffset, DefaultValue.
1724 #
1725 def AnalyzeHiiPcdData(Setting):
1726     ValueList = ['', '', '', '']
1727
1728     TokenList = GetSplitValueList(Setting)
1729     ValueList[0:len(TokenList)] = TokenList
1730
1731     return ValueList
1732
1733 ## AnalyzeVpdPcdData
1734 #
1735 #  Analyze the vpd pcd VpdOffset, MaxDatumSize and InitialValue.
1736 #  Used to avoid split issue while the value string contain "|" character
1737 #
1738 #  @param[in] Setting:  A String contain VpdOffset/MaxDatumSize/InitialValue information;
1739 #
1740 #  @retval   ValueList: A List contain VpdOffset, MaxDatumSize and InitialValue.
1741 #
1742 def AnalyzeVpdPcdData(Setting):
1743     ValueList = ['', '', '']
1744
1745     ValueRe = re.compile(r'\s*L?\".*\|.*\"\s*$')
1746     PtrValue = ValueRe.findall(Setting)
1747
1748     ValueUpdateFlag = False
1749
1750     if len(PtrValue) >= 1:
1751         Setting = re.sub(ValueRe, '', Setting)
1752         ValueUpdateFlag = True
1753
1754     TokenList = Setting.split(TAB_VALUE_SPLIT)
1755     ValueList[0:len(TokenList)] = TokenList
1756
1757     if ValueUpdateFlag:
1758         ValueList[2] = PtrValue[0]
1759
1760     return ValueList
1761
1762 ## check format of PCD value against its the datum type
1763 #
1764 # For PCD value setting
1765 #
1766 def CheckPcdDatum(Type, Value):
1767     if Type == "VOID*":
1768         ValueRe = re.compile(r'\s*L?\".*\"\s*$')
1769         if not (((Value.startswith('L"') or Value.startswith('"')) and Value.endswith('"'))
1770                 or (Value.startswith('{') and Value.endswith('}'))
1771                ):
1772             return False, "Invalid value [%s] of type [%s]; must be in the form of {...} for array"\
1773                           ", or \"...\" for string, or L\"...\" for unicode string" % (Value, Type)
1774         elif ValueRe.match(Value):
1775             # Check the chars in UnicodeString or CString is printable
1776             if Value.startswith("L"):
1777                 Value = Value[2:-1]
1778             else:
1779                 Value = Value[1:-1]
1780             Printset = set(string.printable)
1781             Printset.remove(TAB_PRINTCHAR_VT)
1782             Printset.add(TAB_PRINTCHAR_BS)
1783             Printset.add(TAB_PRINTCHAR_NUL)
1784             if not set(Value).issubset(Printset):
1785                 PrintList = list(Printset)
1786                 PrintList.sort()
1787                 return False, "Invalid PCD string value of type [%s]; must be printable chars %s." % (Type, PrintList)
1788     elif Type == 'BOOLEAN':
1789         if Value not in ['TRUE', 'True', 'true', '0x1', '0x01', '1', 'FALSE', 'False', 'false', '0x0', '0x00', '0']:
1790             return False, "Invalid value [%s] of type [%s]; must be one of TRUE, True, true, 0x1, 0x01, 1"\
1791                           ", FALSE, False, false, 0x0, 0x00, 0" % (Value, Type)
1792     elif Type in [TAB_UINT8, TAB_UINT16, TAB_UINT32, TAB_UINT64]:
1793         try:
1794             Value = long(Value, 0)
1795         except:
1796             return False, "Invalid value [%s] of type [%s];"\
1797                           " must be a hexadecimal, decimal or octal in C language format." % (Value, Type)
1798     else:
1799         return True, "StructurePcd"
1800
1801     return True, ""
1802
1803 ## Split command line option string to list
1804 #
1805 # subprocess.Popen needs the args to be a sequence. Otherwise there's problem
1806 # in non-windows platform to launch command
1807 #
1808 def SplitOption(OptionString):
1809     OptionList = []
1810     LastChar = " "
1811     OptionStart = 0
1812     QuotationMark = ""
1813     for Index in range(0, len(OptionString)):
1814         CurrentChar = OptionString[Index]
1815         if CurrentChar in ['"', "'"]:
1816             if QuotationMark == CurrentChar:
1817                 QuotationMark = ""
1818             elif QuotationMark == "":
1819                 QuotationMark = CurrentChar
1820             continue
1821         elif QuotationMark:
1822             continue
1823
1824         if CurrentChar in ["/", "-"] and LastChar in [" ", "\t", "\r", "\n"]:
1825             if Index > OptionStart:
1826                 OptionList.append(OptionString[OptionStart:Index - 1])
1827             OptionStart = Index
1828         LastChar = CurrentChar
1829     OptionList.append(OptionString[OptionStart:])
1830     return OptionList
1831
1832 def CommonPath(PathList):
1833     P1 = min(PathList).split(os.path.sep)
1834     P2 = max(PathList).split(os.path.sep)
1835     for Index in xrange(min(len(P1), len(P2))):
1836         if P1[Index] != P2[Index]:
1837             return os.path.sep.join(P1[:Index])
1838     return os.path.sep.join(P1)
1839
1840 #
1841 # Convert string to C format array
1842 #
1843 def ConvertStringToByteArray(Value):
1844     Value = Value.strip()
1845     if not Value:
1846         return None
1847     if Value[0] == '{':
1848         if not Value.endswith('}'):
1849             return None
1850         Value = Value.replace(' ', '').replace('{', '').replace('}', '')
1851         ValFields = Value.split(',')
1852         try:
1853             for Index in range(len(ValFields)):
1854                 ValFields[Index] = str(int(ValFields[Index], 0))
1855         except ValueError:
1856             return None
1857         Value = '{' + ','.join(ValFields) + '}'
1858         return Value
1859
1860     Unicode = False
1861     if Value.startswith('L"'):
1862         if not Value.endswith('"'):
1863             return None
1864         Value = Value[1:]
1865         Unicode = True
1866     elif not Value.startswith('"') or not Value.endswith('"'):
1867         return None
1868
1869     Value = eval(Value)         # translate escape character
1870     NewValue = '{'
1871     for Index in range(0,len(Value)):
1872         if Unicode:
1873             NewValue = NewValue + str(ord(Value[Index]) % 0x10000) + ','
1874         else:
1875             NewValue = NewValue + str(ord(Value[Index]) % 0x100) + ','
1876     Value = NewValue + '0}'
1877     return Value
1878
1879 class PathClass(object):
1880     def __init__(self, File='', Root='', AlterRoot='', Type='', IsBinary=False,
1881                  Arch='COMMON', ToolChainFamily='', Target='', TagName='', ToolCode=''):
1882         self.Arch = Arch
1883         self.File = str(File)
1884         if os.path.isabs(self.File):
1885             self.Root = ''
1886             self.AlterRoot = ''
1887         else:
1888             self.Root = str(Root)
1889             self.AlterRoot = str(AlterRoot)
1890
1891         # Remove any '.' and '..' in path
1892         if self.Root:
1893             self.Root = mws.getWs(self.Root, self.File)
1894             self.Path = os.path.normpath(os.path.join(self.Root, self.File))
1895             self.Root = os.path.normpath(CommonPath([self.Root, self.Path]))
1896             # eliminate the side-effect of 'C:'
1897             if self.Root[-1] == ':':
1898                 self.Root += os.path.sep
1899             # file path should not start with path separator
1900             if self.Root[-1] == os.path.sep:
1901                 self.File = self.Path[len(self.Root):]
1902             else:
1903                 self.File = self.Path[len(self.Root) + 1:]
1904         else:
1905             self.Path = os.path.normpath(self.File)
1906
1907         self.SubDir, self.Name = os.path.split(self.File)
1908         self.BaseName, self.Ext = os.path.splitext(self.Name)
1909
1910         if self.Root:
1911             if self.SubDir:
1912                 self.Dir = os.path.join(self.Root, self.SubDir)
1913             else:
1914                 self.Dir = self.Root
1915         else:
1916             self.Dir = self.SubDir
1917
1918         if IsBinary:
1919             self.Type = Type
1920         else:
1921             self.Type = self.Ext.lower()
1922
1923         self.IsBinary = IsBinary
1924         self.Target = Target
1925         self.TagName = TagName
1926         self.ToolCode = ToolCode
1927         self.ToolChainFamily = ToolChainFamily
1928
1929         self._Key = None
1930
1931     ## Convert the object of this class to a string
1932     #
1933     #  Convert member Path of the class to a string
1934     #
1935     #  @retval string Formatted String
1936     #
1937     def __str__(self):
1938         return self.Path
1939
1940     ## Override __eq__ function
1941     #
1942     # Check whether PathClass are the same
1943     #
1944     # @retval False The two PathClass are different
1945     # @retval True  The two PathClass are the same
1946     #
1947     def __eq__(self, Other):
1948         if type(Other) == type(self):
1949             return self.Path == Other.Path
1950         else:
1951             return self.Path == str(Other)
1952
1953     ## Override __cmp__ function
1954     #
1955     # Customize the comparsion operation of two PathClass
1956     #
1957     # @retval 0     The two PathClass are different
1958     # @retval -1    The first PathClass is less than the second PathClass
1959     # @retval 1     The first PathClass is Bigger than the second PathClass
1960     def __cmp__(self, Other):
1961         if type(Other) == type(self):
1962             OtherKey = Other.Path
1963         else:
1964             OtherKey = str(Other)
1965
1966         SelfKey = self.Path
1967         if SelfKey == OtherKey:
1968             return 0
1969         elif SelfKey > OtherKey:
1970             return 1
1971         else:
1972             return -1
1973
1974     ## Override __hash__ function
1975     #
1976     # Use Path as key in hash table
1977     #
1978     # @retval string Key for hash table
1979     #
1980     def __hash__(self):
1981         return hash(self.Path)
1982
1983     def _GetFileKey(self):
1984         if self._Key == None:
1985             self._Key = self.Path.upper()   # + self.ToolChainFamily + self.TagName + self.ToolCode + self.Target
1986         return self._Key
1987
1988     def _GetTimeStamp(self):
1989         return os.stat(self.Path)[8]
1990
1991     def Validate(self, Type='', CaseSensitive=True):
1992         if GlobalData.gCaseInsensitive:
1993             CaseSensitive = False
1994         if Type and Type.lower() != self.Type:
1995             return FILE_TYPE_MISMATCH, '%s (expect %s but got %s)' % (self.File, Type, self.Type)
1996
1997         RealFile, RealRoot = RealPath2(self.File, self.Root, self.AlterRoot)
1998         if not RealRoot and not RealFile:
1999             RealFile = self.File
2000             if self.AlterRoot:
2001                 RealFile = os.path.join(self.AlterRoot, self.File)
2002             elif self.Root:
2003                 RealFile = os.path.join(self.Root, self.File)
2004             if len (mws.getPkgPath()) == 0:
2005                 return FILE_NOT_FOUND, os.path.join(self.AlterRoot, RealFile)
2006             else:
2007                 return FILE_NOT_FOUND, "%s is not found in packages path:\n\t%s" % (self.File, '\n\t'.join(mws.getPkgPath()))
2008
2009         ErrorCode = 0
2010         ErrorInfo = ''
2011         if RealRoot != self.Root or RealFile != self.File:
2012             if CaseSensitive and (RealFile != self.File or (RealRoot != self.Root and RealRoot != self.AlterRoot)):
2013                 ErrorCode = FILE_CASE_MISMATCH
2014                 ErrorInfo = self.File + '\n\t' + RealFile + " [in file system]"
2015
2016             self.SubDir, self.Name = os.path.split(RealFile)
2017             self.BaseName, self.Ext = os.path.splitext(self.Name)
2018             if self.SubDir:
2019                 self.Dir = os.path.join(RealRoot, self.SubDir)
2020             else:
2021                 self.Dir = RealRoot
2022             self.File = RealFile
2023             self.Root = RealRoot
2024             self.Path = os.path.join(RealRoot, RealFile)
2025         return ErrorCode, ErrorInfo
2026
2027     Key = property(_GetFileKey)
2028     TimeStamp = property(_GetTimeStamp)
2029
2030 ## Parse PE image to get the required PE informaion.
2031 #
2032 class PeImageClass():
2033     ## Constructor
2034     #
2035     #   @param  File FilePath of PeImage
2036     #
2037     def __init__(self, PeFile):
2038         self.FileName   = PeFile
2039         self.IsValid    = False
2040         self.Size       = 0
2041         self.EntryPoint = 0
2042         self.SectionAlignment  = 0
2043         self.SectionHeaderList = []
2044         self.ErrorInfo = ''
2045         try:
2046             PeObject = open(PeFile, 'rb')
2047         except:
2048             self.ErrorInfo = self.FileName + ' can not be found\n'
2049             return
2050         # Read DOS header
2051         ByteArray = array.array('B')
2052         ByteArray.fromfile(PeObject, 0x3E)
2053         ByteList = ByteArray.tolist()
2054         # DOS signature should be 'MZ'
2055         if self._ByteListToStr (ByteList[0x0:0x2]) != 'MZ':
2056             self.ErrorInfo = self.FileName + ' has no valid DOS signature MZ'
2057             return
2058
2059         # Read 4 byte PE Signature
2060         PeOffset = self._ByteListToInt(ByteList[0x3C:0x3E])
2061         PeObject.seek(PeOffset)
2062         ByteArray = array.array('B')
2063         ByteArray.fromfile(PeObject, 4)
2064         # PE signature should be 'PE\0\0'
2065         if ByteArray.tostring() != 'PE\0\0':
2066             self.ErrorInfo = self.FileName + ' has no valid PE signature PE00'
2067             return
2068
2069         # Read PE file header
2070         ByteArray = array.array('B')
2071         ByteArray.fromfile(PeObject, 0x14)
2072         ByteList = ByteArray.tolist()
2073         SecNumber = self._ByteListToInt(ByteList[0x2:0x4])
2074         if SecNumber == 0:
2075             self.ErrorInfo = self.FileName + ' has no section header'
2076             return
2077
2078         # Read PE optional header
2079         OptionalHeaderSize = self._ByteListToInt(ByteArray[0x10:0x12])
2080         ByteArray = array.array('B')
2081         ByteArray.fromfile(PeObject, OptionalHeaderSize)
2082         ByteList = ByteArray.tolist()
2083         self.EntryPoint       = self._ByteListToInt(ByteList[0x10:0x14])
2084         self.SectionAlignment = self._ByteListToInt(ByteList[0x20:0x24])
2085         self.Size             = self._ByteListToInt(ByteList[0x38:0x3C])
2086
2087         # Read each Section Header
2088         for Index in range(SecNumber):
2089             ByteArray = array.array('B')
2090             ByteArray.fromfile(PeObject, 0x28)
2091             ByteList = ByteArray.tolist()
2092             SecName  = self._ByteListToStr(ByteList[0:8])
2093             SecVirtualSize = self._ByteListToInt(ByteList[8:12])
2094             SecRawAddress  = self._ByteListToInt(ByteList[20:24])
2095             SecVirtualAddress = self._ByteListToInt(ByteList[12:16])
2096             self.SectionHeaderList.append((SecName, SecVirtualAddress, SecRawAddress, SecVirtualSize))
2097         self.IsValid = True
2098         PeObject.close()
2099
2100     def _ByteListToStr(self, ByteList):
2101         String = ''
2102         for index in range(len(ByteList)):
2103             if ByteList[index] == 0:
2104                 break
2105             String += chr(ByteList[index])
2106         return String
2107
2108     def _ByteListToInt(self, ByteList):
2109         Value = 0
2110         for index in range(len(ByteList) - 1, -1, -1):
2111             Value = (Value << 8) | int(ByteList[index])
2112         return Value
2113
2114 class DefaultStore():
2115     def __init__(self,DefaultStores ):
2116
2117         self.DefaultStores = DefaultStores
2118     def DefaultStoreID(self,DefaultStoreName):
2119         for key,value in self.DefaultStores.items():
2120             if value == DefaultStoreName:
2121                 return key
2122         return None
2123     def GetDefaultDefault(self):
2124         if not self.DefaultStores or "0" in self.DefaultStores:
2125             return "0",TAB_DEFAULT_STORES_DEFAULT
2126         else:
2127             minvalue = min([int(value_str) for value_str in self.DefaultStores.keys()])
2128             return (str(minvalue), self.DefaultStores[str(minvalue)])
2129     def GetMin(self,DefaultSIdList):
2130         if not DefaultSIdList:
2131             return "STANDARD"
2132         storeidset = {storeid for storeid, storename in self.DefaultStores.values() if storename in DefaultSIdList}
2133         if not storeidset:
2134             return ""
2135         minid = min(storeidset )
2136         for sid,name in self.DefaultStores.values():
2137             if sid == minid:
2138                 return name
2139 class SkuClass():
2140
2141     DEFAULT = 0
2142     SINGLE = 1
2143     MULTIPLE =2
2144
2145     def __init__(self,SkuIdentifier='', SkuIds=None):
2146         if SkuIds is None:
2147             SkuIds = {}
2148
2149         for SkuName in SkuIds:
2150             SkuId = SkuIds[SkuName][0]
2151             skuid_num = int(SkuId,16) if SkuId.upper().startswith("0X") else int(SkuId)
2152             if skuid_num > 0xFFFFFFFFFFFFFFFF:
2153                 EdkLogger.error("build", PARAMETER_INVALID,
2154                             ExtraData = "SKU-ID [%s] value %s exceeds the max value of UINT64"
2155                                       % (SkuName, SkuId))
2156
2157         self.AvailableSkuIds = sdict()
2158         self.SkuIdSet = []
2159         self.SkuIdNumberSet = []
2160         self.SkuData = SkuIds
2161         self.__SkuInherit = {}
2162         self.__SkuIdentifier = SkuIdentifier
2163         if SkuIdentifier == '' or SkuIdentifier is None:
2164             self.SkuIdSet = ['DEFAULT']
2165             self.SkuIdNumberSet = ['0U']
2166         elif SkuIdentifier == 'ALL':
2167             self.SkuIdSet = SkuIds.keys()
2168             self.SkuIdNumberSet = [num[0].strip() + 'U' for num in SkuIds.values()]
2169         else:
2170             r = SkuIdentifier.split('|')
2171             self.SkuIdSet=[(r[k].strip()).upper() for k in range(len(r))]
2172             k = None
2173             try:
2174                 self.SkuIdNumberSet = [SkuIds[k][0].strip() + 'U' for k in self.SkuIdSet]
2175             except Exception:
2176                 EdkLogger.error("build", PARAMETER_INVALID,
2177                             ExtraData = "SKU-ID [%s] is not supported by the platform. [Valid SKU-ID: %s]"
2178                                       % (k, " | ".join(SkuIds.keys())))
2179         for each in self.SkuIdSet:
2180             if each in SkuIds:
2181                 self.AvailableSkuIds[each] = SkuIds[each][0]
2182             else:
2183                 EdkLogger.error("build", PARAMETER_INVALID,
2184                             ExtraData="SKU-ID [%s] is not supported by the platform. [Valid SKU-ID: %s]"
2185                                       % (each, " | ".join(SkuIds.keys())))
2186         if self.SkuUsageType != self.SINGLE:
2187             self.AvailableSkuIds.update({'DEFAULT':0, 'COMMON':0})
2188         if self.SkuIdSet:
2189             GlobalData.gSkuids = (self.SkuIdSet)
2190             if 'COMMON' in GlobalData.gSkuids:
2191                 GlobalData.gSkuids.remove('COMMON')
2192             if GlobalData.gSkuids:
2193                 GlobalData.gSkuids.sort()
2194
2195     def GetNextSkuId(self, skuname):
2196         if not self.__SkuInherit:
2197             self.__SkuInherit = {}
2198             for item in self.SkuData.values():
2199                 self.__SkuInherit[item[1]]=item[2] if item[2] else "DEFAULT"
2200         return self.__SkuInherit.get(skuname,"DEFAULT")
2201
2202     def GetSkuChain(self,sku):
2203         skulist = [sku]
2204         nextsku = sku
2205         while 1:
2206             nextsku = self.GetNextSkuId(nextsku)
2207             skulist.append(nextsku)
2208             if nextsku == "DEFAULT":
2209                 break
2210         skulist.reverse()
2211         return skulist
2212     def SkuOverrideOrder(self):
2213         skuorderset = []
2214         for skuname in self.SkuIdSet:
2215             skuorderset.append(self.GetSkuChain(skuname))
2216
2217         skuorder = []
2218         for index in range(max([len(item) for item in skuorderset])):
2219             for subset in skuorderset:
2220                 if index > len(subset)-1:
2221                     continue
2222                 if subset[index] in skuorder:
2223                     continue
2224                 skuorder.append(subset[index])
2225
2226         return skuorder
2227
2228     def __SkuUsageType(self):
2229
2230         if self.__SkuIdentifier.upper() == "ALL":
2231             return SkuClass.MULTIPLE
2232
2233         if len(self.SkuIdSet) == 1:
2234             if self.SkuIdSet[0] == 'DEFAULT':
2235                 return SkuClass.DEFAULT
2236             else:
2237                 return SkuClass.SINGLE
2238         elif len(self.SkuIdSet) == 2:
2239             if 'DEFAULT' in self.SkuIdSet:
2240                 return SkuClass.SINGLE
2241             else:
2242                 return SkuClass.MULTIPLE
2243         else:
2244             return SkuClass.MULTIPLE
2245     def DumpSkuIdArrary(self):
2246
2247         ArrayStrList = []
2248         if self.SkuUsageType == SkuClass.SINGLE:
2249             ArrayStr = "{0x0}"
2250         else:
2251             for skuname in self.AvailableSkuIds:
2252                 if skuname == "COMMON":
2253                     continue
2254                 while skuname != "DEFAULT":
2255                     ArrayStrList.append(hex(int(self.AvailableSkuIds[skuname])))
2256                     skuname = self.GetNextSkuId(skuname)
2257                 ArrayStrList.append("0x0")
2258             ArrayStr = "{" + ",".join(ArrayStrList) +  "}"
2259         return ArrayStr
2260     def __GetAvailableSkuIds(self):
2261         return self.AvailableSkuIds
2262
2263     def __GetSystemSkuID(self):
2264         if self.__SkuUsageType() == SkuClass.SINGLE:
2265             if len(self.SkuIdSet) == 1:
2266                 return self.SkuIdSet[0]
2267             else:
2268                 return self.SkuIdSet[0] if self.SkuIdSet[0] != 'DEFAULT' else self.SkuIdSet[1]
2269         else:
2270             return 'DEFAULT'
2271     def __GetAvailableSkuIdNumber(self):
2272         return self.SkuIdNumberSet
2273     SystemSkuId = property(__GetSystemSkuID)
2274     AvailableSkuIdSet = property(__GetAvailableSkuIds)
2275     SkuUsageType = property(__SkuUsageType)
2276     AvailableSkuIdNumSet = property(__GetAvailableSkuIdNumber)
2277
2278 #
2279 # Pack a registry format GUID
2280 #
2281 def PackRegistryFormatGuid(Guid):
2282     Guid = Guid.split('-')
2283     return pack('=LHHBBBBBBBB',
2284                 int(Guid[0], 16),
2285                 int(Guid[1], 16),
2286                 int(Guid[2], 16),
2287                 int(Guid[3][-4:-2], 16),
2288                 int(Guid[3][-2:], 16),
2289                 int(Guid[4][-12:-10], 16),
2290                 int(Guid[4][-10:-8], 16),
2291                 int(Guid[4][-8:-6], 16),
2292                 int(Guid[4][-6:-4], 16),
2293                 int(Guid[4][-4:-2], 16),
2294                 int(Guid[4][-2:], 16)
2295                 )
2296
2297 def BuildOptionPcdValueFormat(TokenSpaceGuidCName, TokenCName, PcdDatumType, Value):
2298     if PcdDatumType == 'VOID*':
2299         if Value.startswith('L'):
2300             if not Value[1]:
2301                 EdkLogger.error("build", FORMAT_INVALID, 'For Void* type PCD, when specify the Value in the command line, please use the following format: "string", L"string", H"{...}"')
2302             Value = Value[0] + '"' + Value[1:] + '"'
2303         elif Value.startswith('H'):
2304             if not Value[1]:
2305                 EdkLogger.error("build", FORMAT_INVALID, 'For Void* type PCD, when specify the Value in the command line, please use the following format: "string", L"string", H"{...}"')
2306             Value = Value[1:]
2307         else:
2308             if not Value[0]:
2309                 EdkLogger.error("build", FORMAT_INVALID, 'For Void* type PCD, when specify the Value in the command line, please use the following format: "string", L"string", H"{...}"')
2310             Value = '"' + Value + '"'
2311
2312     IsValid, Cause = CheckPcdDatum(PcdDatumType, Value)
2313     if not IsValid:
2314         EdkLogger.error("build", FORMAT_INVALID, Cause, ExtraData="%s.%s" % (TokenSpaceGuidCName, TokenCName))
2315     if PcdDatumType == 'BOOLEAN':
2316         Value = Value.upper()
2317         if Value == 'TRUE' or Value == '1':
2318             Value = '1'
2319         elif Value == 'FALSE' or Value == '0':
2320             Value = '0'
2321     return  Value
2322 ##  Get the integer value from string like "14U" or integer like 2
2323 #
2324 #   @param      Input   The object that may be either a integer value or a string
2325 #
2326 #   @retval     Value    The integer value that the input represents
2327 #
2328 def GetIntegerValue(Input):
2329     if type(Input) in (int, long):
2330         return Input
2331     String = Input
2332     if String.endswith("U"):
2333         String = String[:-1]
2334     if String.endswith("ULL"):
2335         String = String[:-3]
2336     if String.endswith("LL"):
2337         String = String[:-2]
2338
2339     if String.startswith("0x") or String.startswith("0X"):
2340         return int(String, 16)
2341     elif String == '':
2342         return 0
2343     else:
2344         return int(String)
2345
2346 ##
2347 #
2348 # This acts like the main() function for the script, unless it is 'import'ed into another
2349 # script.
2350 #
2351 if __name__ == '__main__':
2352     pass
2353