BaseTools/Source/Python/UPT/Library/Misc.py

   1 ## @file
   2 # Common routines used by all tools
   3 #
   4 # Copyright (c) 2011 - 2018, Intel Corporation. All rights reserved.<BR>
   5 #
   6 # This program and the accompanying materials are licensed and made available
   7 # under the terms and conditions of the BSD License which accompanies this
   8 # distribution. The full text of the license may be found at
   9 # http://opensource.org/licenses/bsd-license.php
  10 #
  11 # THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS,
  12 # WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED.
  13 #
  14
  15 '''
  16 Misc
  17 '''
  18
  19 ##
  20 # Import Modules
  21 #
  22 import os.path
  23 from os import access
  24 from os import F_OK
  25 from os import makedirs
  26 from os import getcwd
  27 from os import chdir
  28 from os import listdir
  29 from os import remove
  30 from os import rmdir
  31 from os import linesep
  32 from os import walk
  33 from os import environ
  34 import re
  35 from collections import UserDict as IterableUserDict
  36
  37 import Logger.Log as Logger
  38 from Logger import StringTable as ST
  39 from Logger import ToolError
  40 from Library import GlobalData
  41 from Library.DataType import SUP_MODULE_LIST
  42 from Library.DataType import END_OF_LINE
  43 from Library.DataType import TAB_SPLIT
  44 from Library.DataType import TAB_LANGUAGE_EN_US
  45 from Library.DataType import TAB_LANGUAGE_EN
  46 from Library.DataType import TAB_LANGUAGE_EN_X
  47 from Library.DataType import TAB_UNI_FILE_SUFFIXS
  48 from Library.StringUtils import GetSplitValueList
  49 from Library.ParserValidate import IsValidHexVersion
  50 from Library.ParserValidate import IsValidPath
  51 from Object.POM.CommonObject import TextObject
  52 from Core.FileHook import __FileHookOpen__
  53 from Common.MultipleWorkspace import MultipleWorkspace as mws
  54
  55 ## Convert GUID string in xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx style to C
  56 # structure style
  57 #
  58 # @param      Guid:    The GUID string
  59 #
  60 def GuidStringToGuidStructureString(Guid):
  61     GuidList = Guid.split('-')
  62     Result = '{'
  63     for Index in range(0, 3, 1):
  64         Result = Result + '0x' + GuidList[Index] + ', '
  65     Result = Result + '{0x' + GuidList[3][0:2] + ', 0x' + GuidList[3][2:4]
  66     for Index in range(0, 12, 2):
  67         Result = Result + ', 0x' + GuidList[4][Index:Index + 2]
  68     Result += '}}'
  69     return Result
  70
  71 ## Check whether GUID string is of format xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
  72 #
  73 # @param      GuidValue:   The GUID value
  74 #
  75 def CheckGuidRegFormat(GuidValue):
  76     ## Regular expression used to find out register format of GUID
  77     #
  78     RegFormatGuidPattern = re.compile("^\s*([0-9a-fA-F]){8}-"
  79                                        "([0-9a-fA-F]){4}-"
  80                                        "([0-9a-fA-F]){4}-"
  81                                        "([0-9a-fA-F]){4}-"
  82                                        "([0-9a-fA-F]){12}\s*$")
  83
  84     if RegFormatGuidPattern.match(GuidValue):
  85         return True
  86     else:
  87         return False
  88
  89
  90 ## Convert GUID string in C structure style to
  91 # xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
  92 #
  93 # @param      GuidValue:   The GUID value in C structure format
  94 #
  95 def GuidStructureStringToGuidString(GuidValue):
  96     GuidValueString = GuidValue.lower().replace("{", "").replace("}", "").\
  97     replace(" ", "").replace(";", "")
  98     GuidValueList = GuidValueString.split(",")
  99     if len(GuidValueList) != 11:
 100         return ''
 101     try:
 102         return "%08x-%04x-%04x-%02x%02x-%02x%02x%02x%02x%02x%02x" % (
 103                 int(GuidValueList[0], 16),
 104                 int(GuidValueList[1], 16),
 105                 int(GuidValueList[2], 16),
 106                 int(GuidValueList[3], 16),
 107                 int(GuidValueList[4], 16),
 108                 int(GuidValueList[5], 16),
 109                 int(GuidValueList[6], 16),
 110                 int(GuidValueList[7], 16),
 111                 int(GuidValueList[8], 16),
 112                 int(GuidValueList[9], 16),
 113                 int(GuidValueList[10], 16)
 114                 )
 115     except BaseException:
 116         return ''
 117
 118 ## Create directories
 119 #
 120 # @param      Directory:   The directory name
 121 #
 122 def CreateDirectory(Directory):
 123     if Directory is None or Directory.strip() == "":
 124         return True
 125     try:
 126         if not access(Directory, F_OK):
 127             makedirs(Directory)
 128     except BaseException:
 129         return False
 130     return True
 131
 132 ## Remove directories, including files and sub-directories in it
 133 #
 134 # @param      Directory:   The directory name
 135 #
 136 def RemoveDirectory(Directory, Recursively=False):
 137     if Directory is None or Directory.strip() == "" or not \
 138     os.path.exists(Directory):
 139         return
 140     if Recursively:
 141         CurrentDirectory = getcwd()
 142         chdir(Directory)
 143         for File in listdir("."):
 144             if os.path.isdir(File):
 145                 RemoveDirectory(File, Recursively)
 146             else:
 147                 remove(File)
 148         chdir(CurrentDirectory)
 149     rmdir(Directory)
 150
 151 ## Store content in file
 152 #
 153 # This method is used to save file only when its content is changed. This is
 154 # quite useful for "make" system to decide what will be re-built and what
 155 # won't.
 156 #
 157 # @param      File:            The path of file
 158 # @param      Content:         The new content of the file
 159 # @param      IsBinaryFile:    The flag indicating if the file is binary file
 160 #                              or not
 161 #
 162 def SaveFileOnChange(File, Content, IsBinaryFile=True):
 163     if os.path.exists(File):
 164         try:
 165             if isinstance(Content, bytes):
 166                 if Content == __FileHookOpen__(File, "rb").read():
 167                     return False
 168             else:
 169                 if Content == __FileHookOpen__(File, "r").read():
 170                     return False
 171         except BaseException:
 172             Logger.Error(None, ToolError.FILE_OPEN_FAILURE, ExtraData=File)
 173
 174     CreateDirectory(os.path.dirname(File))
 175     try:
 176         if isinstance(Content, bytes):
 177             FileFd = __FileHookOpen__(File, "wb")
 178         else:
 179             FileFd = __FileHookOpen__(File, "w")
 180         FileFd.write(Content)
 181         FileFd.close()
 182     except BaseException:
 183         Logger.Error(None, ToolError.FILE_CREATE_FAILURE, ExtraData=File)
 184
 185     return True
 186
 187 ## Get all files of a directory
 188 #
 189 # @param Root:       Root dir
 190 # @param SkipList :  The files need be skipped
 191 #
 192 def GetFiles(Root, SkipList=None, FullPath=True):
 193     OriPath = os.path.normpath(Root)
 194     FileList = []
 195     for Root, Dirs, Files in walk(Root):
 196         if SkipList:
 197             for Item in SkipList:
 198                 if Item in Dirs:
 199                     Dirs.remove(Item)
 200                 if Item in Files:
 201                     Files.remove(Item)
 202         for Dir in Dirs:
 203             if Dir.startswith('.'):
 204                 Dirs.remove(Dir)
 205
 206         for File in Files:
 207             if File.startswith('.'):
 208                 continue
 209             File = os.path.normpath(os.path.join(Root, File))
 210             if not FullPath:
 211                 File = File[len(OriPath) + 1:]
 212             FileList.append(File)
 213
 214     return FileList
 215
 216 ## Get all non-metadata files of a directory
 217 #
 218 # @param Root:       Root Dir
 219 # @param SkipList :  List of path need be skipped
 220 # @param FullPath:  True if the returned file should be full path
 221 # @param PrefixPath: the path that need to be added to the files found
 222 # @return: the list of files found
 223 #
 224 def GetNonMetaDataFiles(Root, SkipList, FullPath, PrefixPath):
 225     FileList = GetFiles(Root, SkipList, FullPath)
 226     NewFileList = []
 227     for File in FileList:
 228         ExtName = os.path.splitext(File)[1]
 229         #
 230         # skip '.dec', '.inf', '.dsc', '.fdf' files
 231         #
 232         if ExtName.lower() not in ['.dec', '.inf', '.dsc', '.fdf']:
 233             NewFileList.append(os.path.normpath(os.path.join(PrefixPath, File)))
 234
 235     return NewFileList
 236
 237 ## Check if given file exists or not
 238 #
 239 # @param      File:    File name or path to be checked
 240 # @param      Dir:     The directory the file is relative to
 241 #
 242 def ValidFile(File, Ext=None):
 243     File = File.replace('\\', '/')
 244     if Ext is not None:
 245         FileExt = os.path.splitext(File)[1]
 246         if FileExt.lower() != Ext.lower():
 247             return False
 248     if not os.path.exists(File):
 249         return False
 250     return True
 251
 252 ## RealPath
 253 #
 254 # @param      File:    File name or path to be checked
 255 # @param      Dir:     The directory the file is relative to
 256 # @param      OverrideDir:     The override directory
 257 #
 258 def RealPath(File, Dir='', OverrideDir=''):
 259     NewFile = os.path.normpath(os.path.join(Dir, File))
 260     NewFile = GlobalData.gALL_FILES[NewFile]
 261     if not NewFile and OverrideDir:
 262         NewFile = os.path.normpath(os.path.join(OverrideDir, File))
 263         NewFile = GlobalData.gALL_FILES[NewFile]
 264     return NewFile
 265
 266 ## RealPath2
 267 #
 268 # @param      File:    File name or path to be checked
 269 # @param      Dir:     The directory the file is relative to
 270 # @param      OverrideDir:     The override directory
 271 #
 272 def RealPath2(File, Dir='', OverrideDir=''):
 273     if OverrideDir:
 274         NewFile = GlobalData.gALL_FILES[os.path.normpath(os.path.join\
 275                                                         (OverrideDir, File))]
 276         if NewFile:
 277             if OverrideDir[-1] == os.path.sep:
 278                 return NewFile[len(OverrideDir):], NewFile[0:len(OverrideDir)]
 279             else:
 280                 return NewFile[len(OverrideDir) + 1:], \
 281             NewFile[0:len(OverrideDir)]
 282
 283     NewFile = GlobalData.gALL_FILES[os.path.normpath(os.path.join(Dir, File))]
 284     if NewFile:
 285         if Dir:
 286             if Dir[-1] == os.path.sep:
 287                 return NewFile[len(Dir):], NewFile[0:len(Dir)]
 288             else:
 289                 return NewFile[len(Dir) + 1:], NewFile[0:len(Dir)]
 290         else:
 291             return NewFile, ''
 292
 293     return None, None
 294
 295 ## A dict which can access its keys and/or values orderly
 296 #
 297 #  The class implements a new kind of dict which its keys or values can be
 298 #  accessed in the order they are added into the dict. It guarantees the order
 299 #  by making use of an internal list to keep a copy of keys.
 300 #
 301 class Sdict(IterableUserDict):
 302     ## Constructor
 303     #
 304     def __init__(self):
 305         IterableUserDict.__init__(self)
 306         self._key_list = []
 307
 308     ## [] operator
 309     #
 310     def __setitem__(self, Key, Value):
 311         if Key not in self._key_list:
 312             self._key_list.append(Key)
 313         IterableUserDict.__setitem__(self, Key, Value)
 314
 315     ## del operator
 316     #
 317     def __delitem__(self, Key):
 318         self._key_list.remove(Key)
 319         IterableUserDict.__delitem__(self, Key)
 320
 321     ## used in "for k in dict" loop to ensure the correct order
 322     #
 323     def __iter__(self):
 324         return self.iterkeys()
 325
 326     ## len() support
 327     #
 328     def __len__(self):
 329         return len(self._key_list)
 330
 331     ## "in" test support
 332     #
 333     def __contains__(self, Key):
 334         return Key in self._key_list
 335
 336     ## indexof support
 337     #
 338     def index(self, Key):
 339         return self._key_list.index(Key)
 340
 341     ## insert support
 342     #
 343     def insert(self, Key, Newkey, Newvalue, Order):
 344         Index = self._key_list.index(Key)
 345         if Order == 'BEFORE':
 346             self._key_list.insert(Index, Newkey)
 347             IterableUserDict.__setitem__(self, Newkey, Newvalue)
 348         elif Order == 'AFTER':
 349             self._key_list.insert(Index + 1, Newkey)
 350             IterableUserDict.__setitem__(self, Newkey, Newvalue)
 351
 352     ## append support
 353     #
 354     def append(self, Sdict2):
 355         for Key in Sdict2:
 356             if Key not in self._key_list:
 357                 self._key_list.append(Key)
 358             IterableUserDict.__setitem__(self, Key, Sdict2[Key])
 359     ## hash key
 360     #
 361     def has_key(self, Key):
 362         return Key in self._key_list
 363
 364     ## Empty the dict
 365     #
 366     def clear(self):
 367         self._key_list = []
 368         IterableUserDict.clear(self)
 369
 370     ## Return a copy of keys
 371     #
 372     def keys(self):
 373         Keys = []
 374         for Key in self._key_list:
 375             Keys.append(Key)
 376         return Keys
 377
 378     ## Return a copy of values
 379     #
 380     def values(self):
 381         Values = []
 382         for Key in self._key_list:
 383             Values.append(self[Key])
 384         return Values
 385
 386     ## Return a copy of (key, value) list
 387     #
 388     def items(self):
 389         Items = []
 390         for Key in self._key_list:
 391             Items.append((Key, self[Key]))
 392         return Items
 393
 394     ## Iteration support
 395     #
 396     def iteritems(self):
 397         return iter(self.items())
 398
 399     ## Keys interation support
 400     #
 401     def iterkeys(self):
 402         return iter(self.keys())
 403
 404     ## Values interation support
 405     #
 406     def itervalues(self):
 407         return iter(self.values())
 408
 409     ## Return value related to a key, and remove the (key, value) from the dict
 410     #
 411     def pop(self, Key, *Dv):
 412         Value = None
 413         if Key in self._key_list:
 414             Value = self[Key]
 415             self.__delitem__(Key)
 416         elif len(Dv) != 0 :
 417             Value = Dv[0]
 418         return Value
 419
 420     ## Return (key, value) pair, and remove the (key, value) from the dict
 421     #
 422     def popitem(self):
 423         Key = self._key_list[-1]
 424         Value = self[Key]
 425         self.__delitem__(Key)
 426         return Key, Value
 427     ## update method
 428     #
 429     def update(self, Dict=None, **Kwargs):
 430         if Dict is not None:
 431             for Key1, Val1 in Dict.items():
 432                 self[Key1] = Val1
 433         if len(Kwargs):
 434             for Key1, Val1 in Kwargs.items():
 435                 self[Key1] = Val1
 436
 437 ## CommonPath
 438 #
 439 # @param PathList: PathList
 440 #
 441 def CommonPath(PathList):
 442     Path1 = min(PathList).split(os.path.sep)
 443     Path2 = max(PathList).split(os.path.sep)
 444     for Index in range(min(len(Path1), len(Path2))):
 445         if Path1[Index] != Path2[Index]:
 446             return os.path.sep.join(Path1[:Index])
 447     return os.path.sep.join(Path1)
 448
 449 ## PathClass
 450 #
 451 class PathClass(object):
 452     def __init__(self, File='', Root='', AlterRoot='', Type='', IsBinary=False,
 453                  Arch='COMMON', ToolChainFamily='', Target='', TagName='', \
 454                  ToolCode=''):
 455         self.Arch = Arch
 456         self.File = str(File)
 457         if os.path.isabs(self.File):
 458             self.Root = ''
 459             self.AlterRoot = ''
 460         else:
 461             self.Root = str(Root)
 462             self.AlterRoot = str(AlterRoot)
 463
 464         #
 465         # Remove any '.' and '..' in path
 466         #
 467         if self.Root:
 468             self.Path = os.path.normpath(os.path.join(self.Root, self.File))
 469             self.Root = os.path.normpath(CommonPath([self.Root, self.Path]))
 470             #
 471             # eliminate the side-effect of 'C:'
 472             #
 473             if self.Root[-1] == ':':
 474                 self.Root += os.path.sep
 475             #
 476             # file path should not start with path separator
 477             #
 478             if self.Root[-1] == os.path.sep:
 479                 self.File = self.Path[len(self.Root):]
 480             else:
 481                 self.File = self.Path[len(self.Root) + 1:]
 482         else:
 483             self.Path = os.path.normpath(self.File)
 484
 485         self.SubDir, self.Name = os.path.split(self.File)
 486         self.BaseName, self.Ext = os.path.splitext(self.Name)
 487
 488         if self.Root:
 489             if self.SubDir:
 490                 self.Dir = os.path.join(self.Root, self.SubDir)
 491             else:
 492                 self.Dir = self.Root
 493         else:
 494             self.Dir = self.SubDir
 495
 496         if IsBinary:
 497             self.Type = Type
 498         else:
 499             self.Type = self.Ext.lower()
 500
 501         self.IsBinary = IsBinary
 502         self.Target = Target
 503         self.TagName = TagName
 504         self.ToolCode = ToolCode
 505         self.ToolChainFamily = ToolChainFamily
 506
 507         self._Key = None
 508
 509     ## Convert the object of this class to a string
 510     #
 511     #  Convert member Path of the class to a string
 512     #
 513     def __str__(self):
 514         return self.Path
 515
 516     ## Override __eq__ function
 517     #
 518     # Check whether PathClass are the same
 519     #
 520     def __eq__(self, Other):
 521         if isinstance(Other, type(self)):
 522             return self.Path == Other.Path
 523         else:
 524             return self.Path == str(Other)
 525
 526     ## Override __hash__ function
 527     #
 528     # Use Path as key in hash table
 529     #
 530     def __hash__(self):
 531         return hash(self.Path)
 532
 533     ## _GetFileKey
 534     #
 535     def _GetFileKey(self):
 536         if self._Key is None:
 537             self._Key = self.Path.upper()
 538         return self._Key
 539     ## Validate
 540     #
 541     def Validate(self, Type='', CaseSensitive=True):
 542         if GlobalData.gCASE_INSENSITIVE:
 543             CaseSensitive = False
 544         if Type and Type.lower() != self.Type:
 545             return ToolError.FILE_TYPE_MISMATCH, '%s (expect %s but got %s)' % \
 546         (self.File, Type, self.Type)
 547
 548         RealFile, RealRoot = RealPath2(self.File, self.Root, self.AlterRoot)
 549         if not RealRoot and not RealFile:
 550             RealFile = self.File
 551             if self.AlterRoot:
 552                 RealFile = os.path.join(self.AlterRoot, self.File)
 553             elif self.Root:
 554                 RealFile = os.path.join(self.Root, self.File)
 555             return ToolError.FILE_NOT_FOUND, os.path.join(self.AlterRoot, RealFile)
 556
 557         ErrorCode = 0
 558         ErrorInfo = ''
 559         if RealRoot != self.Root or RealFile != self.File:
 560             if CaseSensitive and (RealFile != self.File or \
 561                                   (RealRoot != self.Root and RealRoot != \
 562                                    self.AlterRoot)):
 563                 ErrorCode = ToolError.FILE_CASE_MISMATCH
 564                 ErrorInfo = self.File + '\n\t' + RealFile + \
 565                  " [in file system]"
 566
 567             self.SubDir, self.Name = os.path.split(RealFile)
 568             self.BaseName, self.Ext = os.path.splitext(self.Name)
 569             if self.SubDir:
 570                 self.Dir = os.path.join(RealRoot, self.SubDir)
 571             else:
 572                 self.Dir = RealRoot
 573             self.File = RealFile
 574             self.Root = RealRoot
 575             self.Path = os.path.join(RealRoot, RealFile)
 576         return ErrorCode, ErrorInfo
 577
 578     Key = property(_GetFileKey)
 579
 580 ## Get current workspace
 581 #
 582 #  get WORKSPACE from environment variable if present,if not use current working directory as WORKSPACE
 583 #
 584 def GetWorkspace():
 585     #
 586     # check WORKSPACE
 587     #
 588     if "WORKSPACE" in environ:
 589         WorkspaceDir = os.path.normpath(environ["WORKSPACE"])
 590         if not os.path.exists(WorkspaceDir):
 591             Logger.Error("UPT",
 592                          ToolError.UPT_ENVIRON_MISSING_ERROR,
 593                          ST.ERR_WORKSPACE_NOTEXIST,
 594                          ExtraData="%s" % WorkspaceDir)
 595     else:
 596         WorkspaceDir = os.getcwd()
 597
 598     if WorkspaceDir[-1] == ':':
 599         WorkspaceDir += os.sep
 600
 601     PackagesPath = os.environ.get("PACKAGES_PATH")
 602     mws.setWs(WorkspaceDir, PackagesPath)
 603
 604     return WorkspaceDir, mws.PACKAGES_PATH
 605
 606 ## Get relative path
 607 #
 608 #  use full path and workspace to get relative path
 609 #  the destination of this function is mainly to resolve the root path issue(like c: or c:\)
 610 #
 611 #  @param Fullpath: a string of fullpath
 612 #  @param Workspace: a string of workspace
 613 #
 614 def GetRelativePath(Fullpath, Workspace):
 615
 616     RelativePath = ''
 617     if Workspace.endswith(os.sep):
 618         RelativePath = Fullpath[Fullpath.upper().find(Workspace.upper())+len(Workspace):]
 619     else:
 620         RelativePath = Fullpath[Fullpath.upper().find(Workspace.upper())+len(Workspace)+1:]
 621
 622     return RelativePath
 623
 624 ## Check whether all module types are in list
 625 #
 626 # check whether all module types (SUP_MODULE_LIST) are in list
 627 #
 628 # @param ModuleList:  a list of ModuleType
 629 #
 630 def IsAllModuleList(ModuleList):
 631     NewModuleList = [Module.upper() for Module in ModuleList]
 632     for Module in SUP_MODULE_LIST:
 633         if Module not in NewModuleList:
 634             return False
 635     else:
 636         return True
 637
 638 ## Dictionary that use comment(GenericComment, TailComment) as value,
 639 # if a new comment which key already in the dic is inserted, then the
 640 # comment will be merged.
 641 # Key is (Statement, SupArch), when TailComment is added, it will ident
 642 # according to Statement
 643 #
 644 class MergeCommentDict(dict):
 645     ## []= operator
 646     #
 647     def __setitem__(self, Key, CommentVal):
 648         GenericComment, TailComment = CommentVal
 649         if Key in self:
 650             OrigVal1, OrigVal2 = dict.__getitem__(self, Key)
 651             Statement = Key[0]
 652             dict.__setitem__(self, Key, (OrigVal1 + GenericComment, OrigVal2 \
 653                                          + len(Statement) * ' ' + TailComment))
 654         else:
 655             dict.__setitem__(self, Key, (GenericComment, TailComment))
 656
 657     ## =[] operator
 658     #
 659     def __getitem__(self, Key):
 660         return dict.__getitem__(self, Key)
 661
 662
 663 ## GenDummyHelpTextObj
 664 #
 665 # @retval HelpTxt:   Generated dummy help text object
 666 #
 667 def GenDummyHelpTextObj():
 668     HelpTxt = TextObject()
 669     HelpTxt.SetLang(TAB_LANGUAGE_EN_US)
 670     HelpTxt.SetString(' ')
 671     return HelpTxt
 672
 673 ## ConvertVersionToDecimal, the minor version should be within 0 - 99
 674 # <HexVersion>          ::=  "0x" <Major> <Minor>
 675 # <Major>               ::=  (a-fA-F0-9){4}
 676 # <Minor>               ::=  (a-fA-F0-9){4}
 677 # <DecVersion>          ::=  (0-65535) ["." (0-99)]
 678 #
 679 # @param StringIn:  The string contains version defined in INF file.
 680 #                   It can be Decimal or Hex
 681 #
 682 def ConvertVersionToDecimal(StringIn):
 683     if IsValidHexVersion(StringIn):
 684         Value = int(StringIn, 16)
 685         Major = Value >> 16
 686         Minor = Value & 0xFFFF
 687         MinorStr = str(Minor)
 688         if len(MinorStr) == 1:
 689             MinorStr = '0' + MinorStr
 690         return str(Major) + '.' + MinorStr
 691     else:
 692         if StringIn.find(TAB_SPLIT) != -1:
 693             return StringIn
 694         elif StringIn:
 695             return StringIn + '.0'
 696         else:
 697             #
 698             # when StringIn is '', return it directly
 699             #
 700             return StringIn
 701
 702 ## GetHelpStringByRemoveHashKey
 703 #
 704 # Remove hash key at the header of string and return the remain.
 705 #
 706 # @param String:  The string need to be processed.
 707 #
 708 def GetHelpStringByRemoveHashKey(String):
 709     ReturnString = ''
 710     PattenRemoveHashKey = re.compile(r"^[#+\s]+", re.DOTALL)
 711     String = String.strip()
 712     if String == '':
 713         return String
 714
 715     LineList = GetSplitValueList(String, END_OF_LINE)
 716     for Line in LineList:
 717         ValueList = PattenRemoveHashKey.split(Line)
 718         if len(ValueList) == 1:
 719             ReturnString += ValueList[0] + END_OF_LINE
 720         else:
 721             ReturnString += ValueList[1] + END_OF_LINE
 722
 723     if ReturnString.endswith('\n') and not ReturnString.endswith('\n\n') and ReturnString != '\n':
 724         ReturnString = ReturnString[:-1]
 725
 726     return ReturnString
 727
 728 ## ConvPathFromAbsToRel
 729 #
 730 # Get relative file path from absolute path.
 731 #
 732 # @param Path:  The string contain file absolute path.
 733 # @param Root:  The string contain the parent path of Path in.
 734 #
 735 #
 736 def ConvPathFromAbsToRel(Path, Root):
 737     Path = os.path.normpath(Path)
 738     Root = os.path.normpath(Root)
 739     FullPath = os.path.normpath(os.path.join(Root, Path))
 740
 741     #
 742     # If Path is absolute path.
 743     # It should be in Root.
 744     #
 745     if os.path.isabs(Path):
 746         return FullPath[FullPath.find(Root) + len(Root) + 1:]
 747
 748     else:
 749         return Path
 750
 751 ## ConvertPath
 752 #
 753 # Convert special characters to '_', '\' to '/'
 754 # return converted path: Test!1.inf -> Test_1.inf
 755 #
 756 # @param Path: Path to be converted
 757 #
 758 def ConvertPath(Path):
 759     RetPath = ''
 760     for Char in Path.strip():
 761         if Char.isalnum() or Char in '.-_/':
 762             RetPath = RetPath + Char
 763         elif Char == '\\':
 764             RetPath = RetPath + '/'
 765         else:
 766             RetPath = RetPath + '_'
 767     return RetPath
 768
 769 ## ConvertSpec
 770 #
 771 # during install, convert the Spec string extract from UPD into INF allowable definition,
 772 # the difference is period is allowed in the former (not the first letter) but not in the latter.
 773 # return converted Spec string
 774 #
 775 # @param SpecStr: SpecStr to be converted
 776 #
 777 def ConvertSpec(SpecStr):
 778     RetStr = ''
 779     for Char in SpecStr:
 780         if Char.isalnum() or Char == '_':
 781             RetStr = RetStr + Char
 782         else:
 783             RetStr = RetStr + '_'
 784
 785     return RetStr
 786
 787
 788 ## IsEqualList
 789 #
 790 # Judge two lists are identical(contain same item).
 791 # The rule is elements in List A are in List B and elements in List B are in List A.
 792 #
 793 # @param ListA, ListB  Lists need to be judged.
 794 #
 795 # @return True  ListA and ListB are identical
 796 # @return False ListA and ListB are different with each other
 797 #
 798 def IsEqualList(ListA, ListB):
 799     if ListA == ListB:
 800         return True
 801
 802     for ItemA in ListA:
 803         if not ItemA in ListB:
 804             return False
 805
 806     for ItemB in ListB:
 807         if not ItemB in ListA:
 808             return False
 809
 810     return True
 811
 812 ## ConvertArchList
 813 #
 814 # Convert item in ArchList if the start character is lower case.
 815 # In UDP spec, Arch is only allowed as: [A-Z]([a-zA-Z0-9])*
 816 #
 817 # @param ArchList The ArchList need to be converted.
 818 #
 819 # @return NewList  The ArchList been converted.
 820 #
 821 def ConvertArchList(ArchList):
 822     NewArchList = []
 823     if not ArchList:
 824         return NewArchList
 825
 826     if isinstance(ArchList, list):
 827         for Arch in ArchList:
 828             Arch = Arch.upper()
 829             NewArchList.append(Arch)
 830     elif isinstance(ArchList, str):
 831         ArchList = ArchList.upper()
 832         NewArchList.append(ArchList)
 833
 834     return NewArchList
 835
 836 ## ProcessLineExtender
 837 #
 838 # Process the LineExtender of Line in LineList.
 839 # If one line ends with a line extender, then it will be combined together with next line.
 840 #
 841 # @param LineList The LineList need to be processed.
 842 #
 843 # @return NewList  The ArchList been processed.
 844 #
 845 def ProcessLineExtender(LineList):
 846     NewList = []
 847     Count = 0
 848     while Count < len(LineList):
 849         if LineList[Count].strip().endswith("\\") and Count + 1 < len(LineList):
 850             NewList.append(LineList[Count].strip()[:-2] + LineList[Count + 1])
 851             Count = Count + 1
 852         else:
 853             NewList.append(LineList[Count])
 854
 855         Count = Count + 1
 856
 857     return NewList
 858
 859 ## ProcessEdkComment
 860 #
 861 # Process EDK style comment in LineList: c style /* */ comment or cpp style // comment
 862 #
 863 #
 864 # @param LineList The LineList need to be processed.
 865 #
 866 # @return LineList  The LineList been processed.
 867 # @return FirstPos  Where Edk comment is first found, -1 if not found
 868 #
 869 def ProcessEdkComment(LineList):
 870     FindEdkBlockComment = False
 871     Count = 0
 872     StartPos = -1
 873     EndPos = -1
 874     FirstPos = -1
 875
 876     while(Count < len(LineList)):
 877         Line = LineList[Count].strip()
 878         if Line.startswith("/*"):
 879             #
 880             # handling c style comment
 881             #
 882             StartPos = Count
 883             while Count < len(LineList):
 884                 Line = LineList[Count].strip()
 885                 if Line.endswith("*/"):
 886                     if (Count == StartPos) and Line.strip() == '/*/':
 887                         Count = Count + 1
 888                         continue
 889                     EndPos = Count
 890                     FindEdkBlockComment = True
 891                     break
 892                 Count = Count + 1
 893
 894             if FindEdkBlockComment:
 895                 if FirstPos == -1:
 896                     FirstPos = StartPos
 897                 for Index in range(StartPos, EndPos+1):
 898                     LineList[Index] = ''
 899                 FindEdkBlockComment = False
 900         elif Line.find("//") != -1 and not Line.startswith("#"):
 901             #
 902             # handling cpp style comment
 903             #
 904             LineList[Count] = Line.replace("//", '#')
 905             if FirstPos == -1:
 906                 FirstPos = Count
 907
 908         Count = Count + 1
 909
 910     return LineList, FirstPos
 911
 912 ## GetLibInstanceInfo
 913 #
 914 # Get the information from Library Instance INF file.
 915 #
 916 # @param string.  A string start with # and followed by INF file path
 917 # @param WorkSpace. The WorkSpace directory used to combined with INF file path.
 918 #
 919 # @return GUID, Version
 920 def GetLibInstanceInfo(String, WorkSpace, LineNo):
 921
 922     FileGuidString = ""
 923     VerString = ""
 924
 925     OrignalString = String
 926     String = String.strip()
 927     if not String:
 928         return None, None
 929     #
 930     # Remove "#" characters at the beginning
 931     #
 932     String = GetHelpStringByRemoveHashKey(String)
 933     String = String.strip()
 934
 935     #
 936     # Validate file name exist.
 937     #
 938     FullFileName = os.path.normpath(os.path.realpath(os.path.join(WorkSpace, String)))
 939     if not (ValidFile(FullFileName)):
 940         Logger.Error("InfParser",
 941                      ToolError.FORMAT_INVALID,
 942                      ST.ERR_FILELIST_EXIST % (String),
 943                      File=GlobalData.gINF_MODULE_NAME,
 944                      Line=LineNo,
 945                      ExtraData=OrignalString)
 946
 947     #
 948     # Validate file exist/format.
 949     #
 950     if IsValidPath(String, WorkSpace):
 951         IsValidFileFlag = True
 952     else:
 953         Logger.Error("InfParser",
 954                      ToolError.FORMAT_INVALID,
 955                      ST.ERR_INF_PARSER_FILE_NOT_EXIST_OR_NAME_INVALID % (String),
 956                      File=GlobalData.gINF_MODULE_NAME,
 957                      Line=LineNo,
 958                      ExtraData=OrignalString)
 959         return False
 960     if IsValidFileFlag:
 961         FileLinesList = []
 962
 963         try:
 964             FInputfile = open(FullFileName, "r")
 965             try:
 966                 FileLinesList = FInputfile.readlines()
 967             except BaseException:
 968                 Logger.Error("InfParser",
 969                              ToolError.FILE_READ_FAILURE,
 970                              ST.ERR_FILE_OPEN_FAILURE,
 971                              File=FullFileName)
 972             finally:
 973                 FInputfile.close()
 974         except BaseException:
 975             Logger.Error("InfParser",
 976                          ToolError.FILE_READ_FAILURE,
 977                          ST.ERR_FILE_OPEN_FAILURE,
 978                          File=FullFileName)
 979
 980         ReFileGuidPattern = re.compile("^\s*FILE_GUID\s*=.*$")
 981         ReVerStringPattern = re.compile("^\s*VERSION_STRING\s*=.*$")
 982
 983         FileLinesList = ProcessLineExtender(FileLinesList)
 984
 985         for Line in FileLinesList:
 986             if ReFileGuidPattern.match(Line):
 987                 FileGuidString = Line
 988             if ReVerStringPattern.match(Line):
 989                 VerString = Line
 990
 991         if FileGuidString:
 992             FileGuidString = GetSplitValueList(FileGuidString, '=', 1)[1]
 993         if VerString:
 994             VerString = GetSplitValueList(VerString, '=', 1)[1]
 995
 996         return FileGuidString, VerString
 997
 998 ## GetLocalValue
 999 #
1000 # Generate the local value for INF and DEC file. If Lang attribute not present, then use this value.
1001 # If present, and there is no element without the Lang attribute, and one of the elements has the rfc1766 code is
1002 # "en-x-tianocore", or "en-US" if "en-x-tianocore" was not found, or "en" if "en-US" was not found, or startswith 'en'
1003 # if 'en' was not found, then use this value.
1004 # If multiple entries of a tag exist which have the same language code, use the last entry.
1005 #
1006 # @param ValueList  A list need to be processed.
1007 # @param UseFirstValue: True to use the first value, False to use the last value
1008 #
1009 # @return LocalValue
1010 def GetLocalValue(ValueList, UseFirstValue=False):
1011     Value1 = ''
1012     Value2 = ''
1013     Value3 = ''
1014     Value4 = ''
1015     Value5 = ''
1016     for (Key, Value) in ValueList:
1017         if Key == TAB_LANGUAGE_EN_X:
1018             if UseFirstValue:
1019                 if not Value1:
1020                     Value1 = Value
1021             else:
1022                 Value1 = Value
1023         if Key == TAB_LANGUAGE_EN_US:
1024             if UseFirstValue:
1025                 if not Value2:
1026                     Value2 = Value
1027             else:
1028                 Value2 = Value
1029         if Key == TAB_LANGUAGE_EN:
1030             if UseFirstValue:
1031                 if not Value3:
1032                     Value3 = Value
1033             else:
1034                 Value3 = Value
1035         if Key.startswith(TAB_LANGUAGE_EN):
1036             if UseFirstValue:
1037                 if not Value4:
1038                     Value4 = Value
1039             else:
1040                 Value4 = Value
1041         if Key == '':
1042             if UseFirstValue:
1043                 if not Value5:
1044                     Value5 = Value
1045             else:
1046                 Value5 = Value
1047
1048     if Value1:
1049         return Value1
1050     if Value2:
1051         return Value2
1052     if Value3:
1053         return Value3
1054     if Value4:
1055         return Value4
1056     if Value5:
1057         return Value5
1058
1059     return ''
1060
1061
1062 ## GetCharIndexOutStr
1063 #
1064 # Get comment character index outside a string
1065 #
1066 # @param Line:              The string to be checked
1067 # @param CommentCharacter:  Comment char, used to ignore comment content
1068 #
1069 # @retval Index
1070 #
1071 def GetCharIndexOutStr(CommentCharacter, Line):
1072     #
1073     # remove whitespace
1074     #
1075     Line = Line.strip()
1076
1077     #
1078     # Check whether comment character is in a string
1079     #
1080     InString = False
1081     for Index in range(0, len(Line)):
1082         if Line[Index] == '"':
1083             InString = not InString
1084         elif Line[Index] == CommentCharacter and InString :
1085             pass
1086         elif Line[Index] == CommentCharacter and (Index +1) < len(Line) and Line[Index+1] == CommentCharacter \
1087             and not InString :
1088             return Index
1089     return -1
1090
1091 ## ValidateUNIFilePath
1092 #
1093 # Check the UNI file path
1094 #
1095 # @param FilePath: The UNI file path
1096 #
1097 def ValidateUNIFilePath(Path):
1098     Suffix = Path[Path.rfind(TAB_SPLIT):]
1099
1100     #
1101     # Check if the suffix is one of the '.uni', '.UNI', '.Uni'
1102     #
1103     if Suffix not in TAB_UNI_FILE_SUFFIXS:
1104         Logger.Error("Unicode File Parser",
1105                         ToolError.FORMAT_INVALID,
1106                         Message=ST.ERR_UNI_FILE_SUFFIX_WRONG,
1107                         ExtraData=Path)
1108
1109     #
1110     # Check if '..' in the file name(without suffixe)
1111     #
1112     if (TAB_SPLIT + TAB_SPLIT) in Path:
1113         Logger.Error("Unicode File Parser",
1114                         ToolError.FORMAT_INVALID,
1115                         Message=ST.ERR_UNI_FILE_NAME_INVALID,
1116                         ExtraData=Path)
1117
1118     #
1119     # Check if the file name is valid according to the DEC and INF specification
1120     #
1121     Pattern = '[a-zA-Z0-9_][a-zA-Z0-9_\-\.]*'
1122     FileName = Path.replace(Suffix, '')
1123     InvalidCh = re.sub(Pattern, '', FileName)
1124     if InvalidCh:
1125         Logger.Error("Unicode File Parser",
1126                         ToolError.FORMAT_INVALID,
1127                         Message=ST.ERR_INF_PARSER_FILE_NOT_EXIST_OR_NAME_INVALID,
1128                         ExtraData=Path)
1129