BaseTools/Source/Python/UPT/Library/Misc.py

   1 ## @file
   2 # Common routines used by all tools
   3 #
   4 # Copyright (c) 2011 - 2014, Intel Corporation. All rights reserved.<BR>
   5 #
   6 # This program and the accompanying materials are licensed and made available
   7 # under the terms and conditions of the BSD License which accompanies this
   8 # distribution. The full text of the license may be found at
   9 # http://opensource.org/licenses/bsd-license.php
  10 #
  11 # THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS,
  12 # WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED.
  13 #
  14
  15 '''
  16 Misc
  17 '''
  18
  19 ##
  20 # Import Modules
  21 #
  22 import os.path
  23 from os import access
  24 from os import F_OK
  25 from os import makedirs
  26 from os import getcwd
  27 from os import chdir
  28 from os import listdir
  29 from os import remove
  30 from os import rmdir
  31 from os import linesep
  32 from os import walk
  33 from os import environ
  34 import re
  35 from UserDict import IterableUserDict
  36
  37 import Logger.Log as Logger
  38 from Logger import StringTable as ST
  39 from Logger import ToolError
  40 from Library import GlobalData
  41 from Library.DataType import SUP_MODULE_LIST
  42 from Library.DataType import END_OF_LINE
  43 from Library.DataType import TAB_SPLIT
  44 from Library.DataType import TAB_LANGUAGE_EN_US
  45 from Library.DataType import TAB_LANGUAGE_EN
  46 from Library.DataType import TAB_LANGUAGE_EN_X
  47 from Library.DataType import TAB_UNI_FILE_SUFFIXS
  48 from Library.String import GetSplitValueList
  49 from Library.ParserValidate import IsValidHexVersion
  50 from Library.ParserValidate import IsValidPath
  51 from Object.POM.CommonObject import TextObject
  52 from Core.FileHook import __FileHookOpen__
  53
  54 ## Convert GUID string in xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx style to C
  55 # structure style
  56 #
  57 # @param      Guid:    The GUID string
  58 #
  59 def GuidStringToGuidStructureString(Guid):
  60     GuidList = Guid.split('-')
  61     Result = '{'
  62     for Index in range(0, 3, 1):
  63         Result = Result + '0x' + GuidList[Index] + ', '
  64     Result = Result + '{0x' + GuidList[3][0:2] + ', 0x' + GuidList[3][2:4]
  65     for Index in range(0, 12, 2):
  66         Result = Result + ', 0x' + GuidList[4][Index:Index + 2]
  67     Result += '}}'
  68     return Result
  69
  70 ## Check whether GUID string is of format xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
  71 #
  72 # @param      GuidValue:   The GUID value
  73 #
  74 def CheckGuidRegFormat(GuidValue):
  75     ## Regular expression used to find out register format of GUID
  76     #
  77     RegFormatGuidPattern = re.compile("^\s*([0-9a-fA-F]){8}-"
  78                                        "([0-9a-fA-F]){4}-"
  79                                        "([0-9a-fA-F]){4}-"
  80                                        "([0-9a-fA-F]){4}-"
  81                                        "([0-9a-fA-F]){12}\s*$")
  82
  83     if RegFormatGuidPattern.match(GuidValue):
  84         return True
  85     else:
  86         return False
  87
  88
  89 ## Convert GUID string in C structure style to
  90 # xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
  91 #
  92 # @param      GuidValue:   The GUID value in C structure format
  93 #
  94 def GuidStructureStringToGuidString(GuidValue):
  95     GuidValueString = GuidValue.lower().replace("{", "").replace("}", "").\
  96     replace(" ", "").replace(";", "")
  97     GuidValueList = GuidValueString.split(",")
  98     if len(GuidValueList) != 11:
  99         return ''
 100     try:
 101         return "%08x-%04x-%04x-%02x%02x-%02x%02x%02x%02x%02x%02x" % (
 102                 int(GuidValueList[0], 16),
 103                 int(GuidValueList[1], 16),
 104                 int(GuidValueList[2], 16),
 105                 int(GuidValueList[3], 16),
 106                 int(GuidValueList[4], 16),
 107                 int(GuidValueList[5], 16),
 108                 int(GuidValueList[6], 16),
 109                 int(GuidValueList[7], 16),
 110                 int(GuidValueList[8], 16),
 111                 int(GuidValueList[9], 16),
 112                 int(GuidValueList[10], 16)
 113                 )
 114     except BaseException:
 115         return ''
 116
 117 ## Create directories
 118 #
 119 # @param      Directory:   The directory name
 120 #
 121 def CreateDirectory(Directory):
 122     if Directory == None or Directory.strip() == "":
 123         return True
 124     try:
 125         if not access(Directory, F_OK):
 126             makedirs(Directory)
 127     except BaseException:
 128         return False
 129     return True
 130
 131 ## Remove directories, including files and sub-directories in it
 132 #
 133 # @param      Directory:   The directory name
 134 #
 135 def RemoveDirectory(Directory, Recursively=False):
 136     if Directory == None or Directory.strip() == "" or not \
 137     os.path.exists(Directory):
 138         return
 139     if Recursively:
 140         CurrentDirectory = getcwd()
 141         chdir(Directory)
 142         for File in listdir("."):
 143             if os.path.isdir(File):
 144                 RemoveDirectory(File, Recursively)
 145             else:
 146                 remove(File)
 147         chdir(CurrentDirectory)
 148     rmdir(Directory)
 149
 150 ## Store content in file
 151 #
 152 # This method is used to save file only when its content is changed. This is
 153 # quite useful for "make" system to decide what will be re-built and what
 154 # won't.
 155 #
 156 # @param      File:            The path of file
 157 # @param      Content:         The new content of the file
 158 # @param      IsBinaryFile:    The flag indicating if the file is binary file
 159 #                              or not
 160 #
 161 def SaveFileOnChange(File, Content, IsBinaryFile=True):
 162     if not IsBinaryFile:
 163         Content = Content.replace("\n", linesep)
 164
 165     if os.path.exists(File):
 166         try:
 167             if Content == __FileHookOpen__(File, "rb").read():
 168                 return False
 169         except BaseException:
 170             Logger.Error(None, ToolError.FILE_OPEN_FAILURE, ExtraData=File)
 171
 172     CreateDirectory(os.path.dirname(File))
 173     try:
 174         FileFd = __FileHookOpen__(File, "wb")
 175         FileFd.write(Content)
 176         FileFd.close()
 177     except BaseException:
 178         Logger.Error(None, ToolError.FILE_CREATE_FAILURE, ExtraData=File)
 179
 180     return True
 181
 182 ## Get all files of a directory
 183 #
 184 # @param Root:       Root dir
 185 # @param SkipList :  The files need be skipped
 186 #
 187 def GetFiles(Root, SkipList=None, FullPath=True):
 188     OriPath = os.path.normpath(Root)
 189     FileList = []
 190     for Root, Dirs, Files in walk(Root):
 191         if SkipList:
 192             for Item in SkipList:
 193                 if Item in Dirs:
 194                     Dirs.remove(Item)
 195                 if Item in Files:
 196                     Files.remove(Item)
 197         for Dir in Dirs:
 198             if Dir.startswith('.'):
 199                 Dirs.remove(Dir)
 200
 201         for File in Files:
 202             if File.startswith('.'):
 203                 continue
 204             File = os.path.normpath(os.path.join(Root, File))
 205             if not FullPath:
 206                 File = File[len(OriPath) + 1:]
 207             FileList.append(File)
 208
 209     return FileList
 210
 211 ## Get all non-metadata files of a directory
 212 #
 213 # @param Root:       Root Dir
 214 # @param SkipList :  List of path need be skipped
 215 # @param FullPath:  True if the returned file should be full path
 216 # @param PrefixPath: the path that need to be added to the files found
 217 # @return: the list of files found
 218 #
 219 def GetNonMetaDataFiles(Root, SkipList, FullPath, PrefixPath):
 220     FileList = GetFiles(Root, SkipList, FullPath)
 221     NewFileList = []
 222     for File in FileList:
 223         ExtName = os.path.splitext(File)[1]
 224         #
 225         # skip '.dec', '.inf', '.dsc', '.fdf' files
 226         #
 227         if ExtName.lower() not in ['.dec', '.inf', '.dsc', '.fdf']:
 228             NewFileList.append(os.path.normpath(os.path.join(PrefixPath, File)))
 229
 230     return NewFileList
 231
 232 ## Check if given file exists or not
 233 #
 234 # @param      File:    File name or path to be checked
 235 # @param      Dir:     The directory the file is relative to
 236 #
 237 def ValidFile(File, Ext=None):
 238     File = File.replace('\\', '/')
 239     if Ext != None:
 240         FileExt = os.path.splitext(File)[1]
 241         if FileExt.lower() != Ext.lower():
 242             return False
 243     if not os.path.exists(File):
 244         return False
 245     return True
 246
 247 ## RealPath
 248 #
 249 # @param      File:    File name or path to be checked
 250 # @param      Dir:     The directory the file is relative to
 251 # @param      OverrideDir:     The override directory
 252 #
 253 def RealPath(File, Dir='', OverrideDir=''):
 254     NewFile = os.path.normpath(os.path.join(Dir, File))
 255     NewFile = GlobalData.gALL_FILES[NewFile]
 256     if not NewFile and OverrideDir:
 257         NewFile = os.path.normpath(os.path.join(OverrideDir, File))
 258         NewFile = GlobalData.gALL_FILES[NewFile]
 259     return NewFile
 260
 261 ## RealPath2
 262 #
 263 # @param      File:    File name or path to be checked
 264 # @param      Dir:     The directory the file is relative to
 265 # @param      OverrideDir:     The override directory
 266 #
 267 def RealPath2(File, Dir='', OverrideDir=''):
 268     if OverrideDir:
 269         NewFile = GlobalData.gALL_FILES[os.path.normpath(os.path.join\
 270                                                         (OverrideDir, File))]
 271         if NewFile:
 272             if OverrideDir[-1] == os.path.sep:
 273                 return NewFile[len(OverrideDir):], NewFile[0:len(OverrideDir)]
 274             else:
 275                 return NewFile[len(OverrideDir) + 1:], \
 276             NewFile[0:len(OverrideDir)]
 277
 278     NewFile = GlobalData.gALL_FILES[os.path.normpath(os.path.join(Dir, File))]
 279     if NewFile:
 280         if Dir:
 281             if Dir[-1] == os.path.sep:
 282                 return NewFile[len(Dir):], NewFile[0:len(Dir)]
 283             else:
 284                 return NewFile[len(Dir) + 1:], NewFile[0:len(Dir)]
 285         else:
 286             return NewFile, ''
 287
 288     return None, None
 289
 290 ## A dict which can access its keys and/or values orderly
 291 #
 292 #  The class implements a new kind of dict which its keys or values can be
 293 #  accessed in the order they are added into the dict. It guarantees the order
 294 #  by making use of an internal list to keep a copy of keys.
 295 #
 296 class Sdict(IterableUserDict):
 297     ## Constructor
 298     #
 299     def __init__(self):
 300         IterableUserDict.__init__(self)
 301         self._key_list = []
 302
 303     ## [] operator
 304     #
 305     def __setitem__(self, Key, Value):
 306         if Key not in self._key_list:
 307             self._key_list.append(Key)
 308         IterableUserDict.__setitem__(self, Key, Value)
 309
 310     ## del operator
 311     #
 312     def __delitem__(self, Key):
 313         self._key_list.remove(Key)
 314         IterableUserDict.__delitem__(self, Key)
 315
 316     ## used in "for k in dict" loop to ensure the correct order
 317     #
 318     def __iter__(self):
 319         return self.iterkeys()
 320
 321     ## len() support
 322     #
 323     def __len__(self):
 324         return len(self._key_list)
 325
 326     ## "in" test support
 327     #
 328     def __contains__(self, Key):
 329         return Key in self._key_list
 330
 331     ## indexof support
 332     #
 333     def index(self, Key):
 334         return self._key_list.index(Key)
 335
 336     ## insert support
 337     #
 338     def insert(self, Key, Newkey, Newvalue, Order):
 339         Index = self._key_list.index(Key)
 340         if Order == 'BEFORE':
 341             self._key_list.insert(Index, Newkey)
 342             IterableUserDict.__setitem__(self, Newkey, Newvalue)
 343         elif Order == 'AFTER':
 344             self._key_list.insert(Index + 1, Newkey)
 345             IterableUserDict.__setitem__(self, Newkey, Newvalue)
 346
 347     ## append support
 348     #
 349     def append(self, Sdict2):
 350         for Key in Sdict2:
 351             if Key not in self._key_list:
 352                 self._key_list.append(Key)
 353             IterableUserDict.__setitem__(self, Key, Sdict2[Key])
 354     ## hash key
 355     #
 356     def has_key(self, Key):
 357         return Key in self._key_list
 358
 359     ## Empty the dict
 360     #
 361     def clear(self):
 362         self._key_list = []
 363         IterableUserDict.clear(self)
 364
 365     ## Return a copy of keys
 366     #
 367     def keys(self):
 368         Keys = []
 369         for Key in self._key_list:
 370             Keys.append(Key)
 371         return Keys
 372
 373     ## Return a copy of values
 374     #
 375     def values(self):
 376         Values = []
 377         for Key in self._key_list:
 378             Values.append(self[Key])
 379         return Values
 380
 381     ## Return a copy of (key, value) list
 382     #
 383     def items(self):
 384         Items = []
 385         for Key in self._key_list:
 386             Items.append((Key, self[Key]))
 387         return Items
 388
 389     ## Iteration support
 390     #
 391     def iteritems(self):
 392         return iter(self.items())
 393
 394     ## Keys interation support
 395     #
 396     def iterkeys(self):
 397         return iter(self.keys())
 398
 399     ## Values interation support
 400     #
 401     def itervalues(self):
 402         return iter(self.values())
 403
 404     ## Return value related to a key, and remove the (key, value) from the dict
 405     #
 406     def pop(self, Key, *Dv):
 407         Value = None
 408         if Key in self._key_list:
 409             Value = self[Key]
 410             self.__delitem__(Key)
 411         elif len(Dv) != 0 :
 412             Value = Dv[0]
 413         return Value
 414
 415     ## Return (key, value) pair, and remove the (key, value) from the dict
 416     #
 417     def popitem(self):
 418         Key = self._key_list[-1]
 419         Value = self[Key]
 420         self.__delitem__(Key)
 421         return Key, Value
 422     ## update method
 423     #
 424     def update(self, Dict=None, **Kwargs):
 425         if Dict != None:
 426             for Key1, Val1 in Dict.items():
 427                 self[Key1] = Val1
 428         if len(Kwargs):
 429             for Key1, Val1 in Kwargs.items():
 430                 self[Key1] = Val1
 431
 432 ## CommonPath
 433 #
 434 # @param PathList: PathList
 435 #
 436 def CommonPath(PathList):
 437     Path1 = min(PathList).split(os.path.sep)
 438     Path2 = max(PathList).split(os.path.sep)
 439     for Index in xrange(min(len(Path1), len(Path2))):
 440         if Path1[Index] != Path2[Index]:
 441             return os.path.sep.join(Path1[:Index])
 442     return os.path.sep.join(Path1)
 443
 444 ## PathClass
 445 #
 446 class PathClass(object):
 447     def __init__(self, File='', Root='', AlterRoot='', Type='', IsBinary=False,
 448                  Arch='COMMON', ToolChainFamily='', Target='', TagName='', \
 449                  ToolCode=''):
 450         self.Arch = Arch
 451         self.File = str(File)
 452         if os.path.isabs(self.File):
 453             self.Root = ''
 454             self.AlterRoot = ''
 455         else:
 456             self.Root = str(Root)
 457             self.AlterRoot = str(AlterRoot)
 458
 459         #
 460         # Remove any '.' and '..' in path
 461         #
 462         if self.Root:
 463             self.Path = os.path.normpath(os.path.join(self.Root, self.File))
 464             self.Root = os.path.normpath(CommonPath([self.Root, self.Path]))
 465             #
 466             # eliminate the side-effect of 'C:'
 467             #
 468             if self.Root[-1] == ':':
 469                 self.Root += os.path.sep
 470             #
 471             # file path should not start with path separator
 472             #
 473             if self.Root[-1] == os.path.sep:
 474                 self.File = self.Path[len(self.Root):]
 475             else:
 476                 self.File = self.Path[len(self.Root) + 1:]
 477         else:
 478             self.Path = os.path.normpath(self.File)
 479
 480         self.SubDir, self.Name = os.path.split(self.File)
 481         self.BaseName, self.Ext = os.path.splitext(self.Name)
 482
 483         if self.Root:
 484             if self.SubDir:
 485                 self.Dir = os.path.join(self.Root, self.SubDir)
 486             else:
 487                 self.Dir = self.Root
 488         else:
 489             self.Dir = self.SubDir
 490
 491         if IsBinary:
 492             self.Type = Type
 493         else:
 494             self.Type = self.Ext.lower()
 495
 496         self.IsBinary = IsBinary
 497         self.Target = Target
 498         self.TagName = TagName
 499         self.ToolCode = ToolCode
 500         self.ToolChainFamily = ToolChainFamily
 501
 502         self._Key = None
 503
 504     ## Convert the object of this class to a string
 505     #
 506     #  Convert member Path of the class to a string
 507     #
 508     def __str__(self):
 509         return self.Path
 510
 511     ## Override __eq__ function
 512     #
 513     # Check whether PathClass are the same
 514     #
 515     def __eq__(self, Other):
 516         if type(Other) == type(self):
 517             return self.Path == Other.Path
 518         else:
 519             return self.Path == str(Other)
 520
 521     ## Override __hash__ function
 522     #
 523     # Use Path as key in hash table
 524     #
 525     def __hash__(self):
 526         return hash(self.Path)
 527
 528     ## _GetFileKey
 529     #
 530     def _GetFileKey(self):
 531         if self._Key == None:
 532             self._Key = self.Path.upper()
 533         return self._Key
 534     ## Validate
 535     #
 536     def Validate(self, Type='', CaseSensitive=True):
 537         if GlobalData.gCASE_INSENSITIVE:
 538             CaseSensitive = False
 539         if Type and Type.lower() != self.Type:
 540             return ToolError.FILE_TYPE_MISMATCH, '%s (expect %s but got %s)' % \
 541         (self.File, Type, self.Type)
 542
 543         RealFile, RealRoot = RealPath2(self.File, self.Root, self.AlterRoot)
 544         if not RealRoot and not RealFile:
 545             RealFile = self.File
 546             if self.AlterRoot:
 547                 RealFile = os.path.join(self.AlterRoot, self.File)
 548             elif self.Root:
 549                 RealFile = os.path.join(self.Root, self.File)
 550             return ToolError.FILE_NOT_FOUND, os.path.join(self.AlterRoot, RealFile)
 551
 552         ErrorCode = 0
 553         ErrorInfo = ''
 554         if RealRoot != self.Root or RealFile != self.File:
 555             if CaseSensitive and (RealFile != self.File or \
 556                                   (RealRoot != self.Root and RealRoot != \
 557                                    self.AlterRoot)):
 558                 ErrorCode = ToolError.FILE_CASE_MISMATCH
 559                 ErrorInfo = self.File + '\n\t' + RealFile + \
 560                  " [in file system]"
 561
 562             self.SubDir, self.Name = os.path.split(RealFile)
 563             self.BaseName, self.Ext = os.path.splitext(self.Name)
 564             if self.SubDir:
 565                 self.Dir = os.path.join(RealRoot, self.SubDir)
 566             else:
 567                 self.Dir = RealRoot
 568             self.File = RealFile
 569             self.Root = RealRoot
 570             self.Path = os.path.join(RealRoot, RealFile)
 571         return ErrorCode, ErrorInfo
 572
 573     Key = property(_GetFileKey)
 574
 575 ## Get current workspace
 576 #
 577 #  get WORKSPACE from environment variable if present,if not use current working directory as WORKSPACE
 578 #
 579 def GetWorkspace():
 580     #
 581     # check WORKSPACE
 582     #
 583     if "WORKSPACE" in environ:
 584         WorkspaceDir = os.path.normpath(environ["WORKSPACE"])
 585         if not os.path.exists(WorkspaceDir):
 586             Logger.Error("UPT",
 587                          ToolError.UPT_ENVIRON_MISSING_ERROR,
 588                          ST.ERR_WORKSPACE_NOTEXIST,
 589                          ExtraData="%s" % WorkspaceDir)
 590     else:
 591         WorkspaceDir = os.getcwd()
 592
 593     if WorkspaceDir[-1] == ':':
 594         WorkspaceDir += os.sep
 595     return WorkspaceDir
 596
 597 ## Get relative path
 598 #
 599 #  use full path and workspace to get relative path
 600 #  the destination of this function is mainly to resolve the root path issue(like c: or c:\)
 601 #
 602 #  @param Fullpath: a string of fullpath
 603 #  @param Workspace: a string of workspace
 604 #
 605 def GetRelativePath(Fullpath, Workspace):
 606
 607     RelativePath = ''
 608     if Workspace.endswith(os.sep):
 609         RelativePath = Fullpath[Fullpath.upper().find(Workspace.upper())+len(Workspace):]
 610     else:
 611         RelativePath = Fullpath[Fullpath.upper().find(Workspace.upper())+len(Workspace)+1:]
 612
 613     return RelativePath
 614
 615 ## Check whether all module types are in list
 616 #
 617 # check whether all module types (SUP_MODULE_LIST) are in list
 618 #
 619 # @param ModuleList:  a list of ModuleType
 620 #
 621 def IsAllModuleList(ModuleList):
 622     NewModuleList = [Module.upper() for Module in ModuleList]
 623     for Module in SUP_MODULE_LIST:
 624         if Module not in NewModuleList:
 625             return False
 626     else:
 627         return True
 628
 629 ## Dictionary that use comment(GenericComment, TailComment) as value,
 630 # if a new comment which key already in the dic is inserted, then the
 631 # comment will be merged.
 632 # Key is (Statement, SupArch), when TailComment is added, it will ident
 633 # according to Statement
 634 #
 635 class MergeCommentDict(dict):
 636     ## []= operator
 637     #
 638     def __setitem__(self, Key, CommentVal):
 639         GenericComment, TailComment = CommentVal
 640         if Key in self:
 641             OrigVal1, OrigVal2 = dict.__getitem__(self, Key)
 642             Statement = Key[0]
 643             dict.__setitem__(self, Key, (OrigVal1 + GenericComment, OrigVal2 \
 644                                          + len(Statement) * ' ' + TailComment))
 645         else:
 646             dict.__setitem__(self, Key, (GenericComment, TailComment))
 647
 648     ## =[] operator
 649     #
 650     def __getitem__(self, Key):
 651         return dict.__getitem__(self, Key)
 652
 653
 654 ## GenDummyHelpTextObj
 655 #
 656 # @retval HelpTxt:   Generated dummy help text object
 657 #
 658 def GenDummyHelpTextObj():
 659     HelpTxt = TextObject()
 660     HelpTxt.SetLang(TAB_LANGUAGE_EN_US)
 661     HelpTxt.SetString(' ')
 662     return HelpTxt
 663
 664 ## ConvertVersionToDecimal, the minor version should be within 0 - 99
 665 # <HexVersion>          ::=  "0x" <Major> <Minor>
 666 # <Major>               ::=  (a-fA-F0-9){4}
 667 # <Minor>               ::=  (a-fA-F0-9){4}
 668 # <DecVersion>          ::=  (0-65535) ["." (0-99)]
 669 #
 670 # @param StringIn:  The string contains version defined in INF file.
 671 #                   It can be Decimal or Hex
 672 #
 673 def ConvertVersionToDecimal(StringIn):
 674     if IsValidHexVersion(StringIn):
 675         Value = int(StringIn, 16)
 676         Major = Value >> 16
 677         Minor = Value & 0xFFFF
 678         MinorStr = str(Minor)
 679         if len(MinorStr) == 1:
 680             MinorStr = '0' + MinorStr
 681         return str(Major) + '.' + MinorStr
 682     else:
 683         if StringIn.find(TAB_SPLIT) != -1:
 684             return StringIn
 685         elif StringIn:
 686             return StringIn + '.0'
 687         else:
 688             #
 689             # when StringIn is '', return it directly
 690             #
 691             return StringIn
 692
 693 ## GetHelpStringByRemoveHashKey
 694 #
 695 # Remove hash key at the header of string and return the remain.
 696 #
 697 # @param String:  The string need to be processed.
 698 #
 699 def GetHelpStringByRemoveHashKey(String):
 700     ReturnString = ''
 701     PattenRemoveHashKey = re.compile(r"^[#+\s]+", re.DOTALL)
 702     String = String.strip()
 703     if String == '':
 704         return String
 705
 706     LineList = GetSplitValueList(String, END_OF_LINE)
 707     for Line in LineList:
 708         ValueList = PattenRemoveHashKey.split(Line)
 709         if len(ValueList) == 1:
 710             ReturnString += ValueList[0] + END_OF_LINE
 711         else:
 712             ReturnString += ValueList[1] + END_OF_LINE
 713
 714     if ReturnString.endswith('\n') and not ReturnString.endswith('\n\n') and ReturnString != '\n':
 715         ReturnString = ReturnString[:-1]
 716
 717     return ReturnString
 718
 719 ## ConvPathFromAbsToRel
 720 #
 721 # Get relative file path from absolute path.
 722 #
 723 # @param Path:  The string contain file absolute path.
 724 # @param Root:  The string contain the parent path of Path in.
 725 #
 726 #
 727 def ConvPathFromAbsToRel(Path, Root):
 728     Path = os.path.normpath(Path)
 729     Root = os.path.normpath(Root)
 730     FullPath = os.path.normpath(os.path.join(Root, Path))
 731
 732     #
 733     # If Path is absolute path.
 734     # It should be in Root.
 735     #
 736     if os.path.isabs(Path):
 737         return FullPath[FullPath.find(Root) + len(Root) + 1:]
 738
 739     else:
 740         return Path
 741
 742 ## ConvertPath
 743 #
 744 # Convert special characters to '_', '\' to '/'
 745 # return converted path: Test!1.inf -> Test_1.inf
 746 #
 747 # @param Path: Path to be converted
 748 #
 749 def ConvertPath(Path):
 750     RetPath = ''
 751     for Char in Path.strip():
 752         if Char.isalnum() or Char in '.-_/':
 753             RetPath = RetPath + Char
 754         elif Char == '\\':
 755             RetPath = RetPath + '/'
 756         else:
 757             RetPath = RetPath + '_'
 758     return RetPath
 759
 760 ## ConvertSpec
 761 #
 762 # during install, convert the Spec string extract from UPD into INF allowable definition,
 763 # the difference is period is allowed in the former (not the first letter) but not in the latter.
 764 # return converted Spec string
 765 #
 766 # @param SpecStr: SpecStr to be converted
 767 #
 768 def ConvertSpec(SpecStr):
 769     RetStr = ''
 770     for Char in SpecStr:
 771         if Char.isalnum() or Char == '_':
 772             RetStr = RetStr + Char
 773         else:
 774             RetStr = RetStr + '_'
 775
 776     return RetStr
 777
 778
 779 ## IsEqualList
 780 #
 781 # Judge two lists are identical(contain same item).
 782 # The rule is elements in List A are in List B and elements in List B are in List A.
 783 #
 784 # @param ListA, ListB  Lists need to be judged.
 785 #
 786 # @return True  ListA and ListB are identical
 787 # @return False ListA and ListB are different with each other
 788 #
 789 def IsEqualList(ListA, ListB):
 790     if ListA == ListB:
 791         return True
 792
 793     for ItemA in ListA:
 794         if not ItemA in ListB:
 795             return False
 796
 797     for ItemB in ListB:
 798         if not ItemB in ListA:
 799             return False
 800
 801     return True
 802
 803 ## ConvertArchList
 804 #
 805 # Convert item in ArchList if the start character is lower case.
 806 # In UDP spec, Arch is only allowed as: [A-Z]([a-zA-Z0-9])*
 807 #
 808 # @param ArchList The ArchList need to be converted.
 809 #
 810 # @return NewList  The ArchList been converted.
 811 #
 812 def ConvertArchList(ArchList):
 813     NewArchList = []
 814     if not ArchList:
 815         return NewArchList
 816
 817     if type(ArchList) == list:
 818         for Arch in ArchList:
 819             Arch = Arch.upper()
 820             NewArchList.append(Arch)
 821     elif type(ArchList) == str:
 822         ArchList = ArchList.upper()
 823         NewArchList.append(ArchList)
 824
 825     return NewArchList
 826
 827 ## ProcessLineExtender
 828 #
 829 # Process the LineExtender of Line in LineList.
 830 # If one line ends with a line extender, then it will be combined together with next line.
 831 #
 832 # @param LineList The LineList need to be processed.
 833 #
 834 # @return NewList  The ArchList been processed.
 835 #
 836 def ProcessLineExtender(LineList):
 837     NewList = []
 838     Count = 0
 839     while Count < len(LineList):
 840         if LineList[Count].strip().endswith("\\") and Count + 1 < len(LineList):
 841             NewList.append(LineList[Count].strip()[:-2] + LineList[Count + 1])
 842             Count = Count + 1
 843         else:
 844             NewList.append(LineList[Count])
 845
 846         Count = Count + 1
 847
 848     return NewList
 849
 850 ## ProcessEdkComment
 851 #
 852 # Process EDK style comment in LineList: c style /* */ comment or cpp style // comment
 853 #
 854 #
 855 # @param LineList The LineList need to be processed.
 856 #
 857 # @return LineList  The LineList been processed.
 858 # @return FirstPos  Where Edk comment is first found, -1 if not found
 859 #
 860 def ProcessEdkComment(LineList):
 861     FindEdkBlockComment = False
 862     Count = 0
 863     StartPos = -1
 864     EndPos = -1
 865     FirstPos = -1
 866
 867     while(Count < len(LineList)):
 868         Line = LineList[Count].strip()
 869         if Line.startswith("/*"):
 870             #
 871             # handling c style comment
 872             #
 873             StartPos = Count
 874             while Count < len(LineList):
 875                 Line = LineList[Count].strip()
 876                 if Line.endswith("*/"):
 877                     if (Count == StartPos) and Line.strip() == '/*/':
 878                         Count = Count + 1
 879                         continue
 880                     EndPos = Count
 881                     FindEdkBlockComment = True
 882                     break
 883                 Count = Count + 1
 884
 885             if FindEdkBlockComment:
 886                 if FirstPos == -1:
 887                     FirstPos = StartPos
 888                 for Index in xrange(StartPos, EndPos+1):
 889                     LineList[Index] = ''
 890                 FindEdkBlockComment = False
 891         elif Line.find("//") != -1 and not Line.startswith("#"):
 892             #
 893             # handling cpp style comment
 894             #
 895             LineList[Count] = Line.replace("//", '#')
 896             if FirstPos == -1:
 897                 FirstPos = Count
 898
 899         Count = Count + 1
 900
 901     return LineList, FirstPos
 902
 903 ## GetLibInstanceInfo
 904 #
 905 # Get the information from Library Instance INF file.
 906 #
 907 # @param string.  A string start with # and followed by INF file path
 908 # @param WorkSpace. The WorkSpace directory used to combined with INF file path.
 909 #
 910 # @return GUID, Version
 911 def GetLibInstanceInfo(String, WorkSpace, LineNo):
 912
 913     FileGuidString = ""
 914     VerString = ""
 915
 916     OrignalString = String
 917     String = String.strip()
 918     if not String:
 919         return None, None
 920     #
 921     # Remove "#" characters at the beginning
 922     #
 923     String = GetHelpStringByRemoveHashKey(String)
 924     String = String.strip()
 925
 926     #
 927     # Validate file name exist.
 928     #
 929     FullFileName = os.path.normpath(os.path.realpath(os.path.join(WorkSpace, String)))
 930     if not (ValidFile(FullFileName)):
 931         Logger.Error("InfParser",
 932                      ToolError.FORMAT_INVALID,
 933                      ST.ERR_FILELIST_EXIST % (String),
 934                      File=GlobalData.gINF_MODULE_NAME,
 935                      Line=LineNo,
 936                      ExtraData=OrignalString)
 937
 938     #
 939     # Validate file exist/format.
 940     #
 941     if IsValidPath(String, WorkSpace):
 942         IsValidFileFlag = True
 943     else:
 944         Logger.Error("InfParser",
 945                      ToolError.FORMAT_INVALID,
 946                      ST.ERR_INF_PARSER_FILE_NOT_EXIST_OR_NAME_INVALID % (String),
 947                      File=GlobalData.gINF_MODULE_NAME,
 948                      Line=LineNo,
 949                      ExtraData=OrignalString)
 950         return False
 951     if IsValidFileFlag:
 952         FileLinesList = []
 953
 954         try:
 955             FInputfile = open(FullFileName, "rb", 0)
 956             try:
 957                 FileLinesList = FInputfile.readlines()
 958             except BaseException:
 959                 Logger.Error("InfParser",
 960                              ToolError.FILE_READ_FAILURE,
 961                              ST.ERR_FILE_OPEN_FAILURE,
 962                              File=FullFileName)
 963             finally:
 964                 FInputfile.close()
 965         except BaseException:
 966             Logger.Error("InfParser",
 967                          ToolError.FILE_READ_FAILURE,
 968                          ST.ERR_FILE_OPEN_FAILURE,
 969                          File=FullFileName)
 970
 971         ReFileGuidPattern = re.compile("^\s*FILE_GUID\s*=.*$")
 972         ReVerStringPattern = re.compile("^\s*VERSION_STRING\s*=.*$")
 973
 974         FileLinesList = ProcessLineExtender(FileLinesList)
 975
 976         for Line in FileLinesList:
 977             if ReFileGuidPattern.match(Line):
 978                 FileGuidString = Line
 979             if ReVerStringPattern.match(Line):
 980                 VerString = Line
 981
 982         if FileGuidString:
 983             FileGuidString = GetSplitValueList(FileGuidString, '=', 1)[1]
 984         if VerString:
 985             VerString = GetSplitValueList(VerString, '=', 1)[1]
 986
 987         return FileGuidString, VerString
 988
 989 ## GetLocalValue
 990 #
 991 # Generate the local value for INF and DEC file. If Lang attribute not present, then use this value.
 992 # If present, and there is no element without the Lang attribute, and one of the elements has the rfc1766 code is
 993 # "en-x-tianocore", or "en-US" if "en-x-tianocore" was not found, or "en" if "en-US" was not found, or startswith 'en'
 994 # if 'en' was not found, then use this value.
 995 # If multiple entries of a tag exist which have the same language code, use the last entry.
 996 #
 997 # @param ValueList  A list need to be processed.
 998 # @param UseFirstValue: True to use the first value, False to use the last value
 999 #
1000 # @return LocalValue
1001 def GetLocalValue(ValueList, UseFirstValue=False):
1002     Value1 = ''
1003     Value2 = ''
1004     Value3 = ''
1005     Value4 = ''
1006     Value5 = ''
1007     for (Key, Value) in ValueList:
1008         if Key == TAB_LANGUAGE_EN_X:
1009             if UseFirstValue:
1010                 if not Value1:
1011                     Value1 = Value
1012             else:
1013                 Value1 = Value
1014         if Key == TAB_LANGUAGE_EN_US:
1015             if UseFirstValue:
1016                 if not Value2:
1017                     Value2 = Value
1018             else:
1019                 Value2 = Value
1020         if Key == TAB_LANGUAGE_EN:
1021             if UseFirstValue:
1022                 if not Value3:
1023                     Value3 = Value
1024             else:
1025                 Value3 = Value
1026         if Key.startswith(TAB_LANGUAGE_EN):
1027             if UseFirstValue:
1028                 if not Value4:
1029                     Value4 = Value
1030             else:
1031                 Value4 = Value
1032         if Key == '':
1033             if UseFirstValue:
1034                 if not Value5:
1035                     Value5 = Value
1036             else:
1037                 Value5 = Value
1038
1039     if Value1:
1040         return Value1
1041     if Value2:
1042         return Value2
1043     if Value3:
1044         return Value3
1045     if Value4:
1046         return Value4
1047     if Value5:
1048         return Value5
1049
1050     return ''
1051
1052
1053 ## GetCharIndexOutStr
1054 #
1055 # Get comment character index outside a string
1056 #
1057 # @param Line:              The string to be checked
1058 # @param CommentCharacter:  Comment char, used to ignore comment content
1059 #
1060 # @retval Index
1061 #
1062 def GetCharIndexOutStr(CommentCharacter, Line):
1063     #
1064     # remove whitespace
1065     #
1066     Line = Line.strip()
1067
1068     #
1069     # Check whether comment character is in a string
1070     #
1071     InString = False
1072     for Index in range(0, len(Line)):
1073         if Line[Index] == '"':
1074             InString = not InString
1075         elif Line[Index] == CommentCharacter and InString :
1076             pass
1077         elif Line[Index] == CommentCharacter and (Index +1) < len(Line) and Line[Index+1] == CommentCharacter \
1078             and not InString :
1079             return Index
1080     return -1
1081
1082 ## ValidateUNIFilePath
1083 #
1084 # Check the UNI file path
1085 #
1086 # @param FilePath: The UNI file path
1087 #
1088 def ValidateUNIFilePath(Path):
1089     Suffix = Path[Path.rfind(TAB_SPLIT):]
1090
1091     #
1092     # Check if the suffix is one of the '.uni', '.UNI', '.Uni'
1093     #
1094     if Suffix not in TAB_UNI_FILE_SUFFIXS:
1095         Logger.Error("Unicode File Parser",
1096                         ToolError.FORMAT_INVALID,
1097                         Message=ST.ERR_UNI_FILE_SUFFIX_WRONG,
1098                         ExtraData=Path)
1099
1100     #
1101     # Check if '..' in the file name(without suffixe)
1102     #
1103     if (TAB_SPLIT + TAB_SPLIT) in Path:
1104         Logger.Error("Unicode File Parser",
1105                         ToolError.FORMAT_INVALID,
1106                         Message=ST.ERR_UNI_FILE_NAME_INVALID,
1107                         ExtraData=Path)
1108
1109     #
1110     # Check if the file name is valid according to the DEC and INF specification
1111     #
1112     Pattern = '[a-zA-Z0-9_][a-zA-Z0-9_\-\.]*'
1113     FileName = Path.replace(Suffix, '')
1114     InvalidCh = re.sub(Pattern, '', FileName)
1115     if InvalidCh:
1116         Logger.Error("Unicode File Parser",
1117                         ToolError.FORMAT_INVALID,
1118                         Message=ST.ERR_INF_PARSER_FILE_NOT_EXIST_OR_NAME_INVALID,
1119                         ExtraData=Path)
1120