BaseTools/Source/Python/UPT/Library/Misc.py

   1 ## @file
   2 # Common routines used by all tools
   3 #
   4 # Copyright (c) 2011 - 2018, Intel Corporation. All rights reserved.<BR>
   5 #
   6 # This program and the accompanying materials are licensed and made available
   7 # under the terms and conditions of the BSD License which accompanies this
   8 # distribution. The full text of the license may be found at
   9 # http://opensource.org/licenses/bsd-license.php
  10 #
  11 # THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS,
  12 # WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED.
  13 #
  14
  15 '''
  16 Misc
  17 '''
  18
  19 ##
  20 # Import Modules
  21 #
  22 import os.path
  23 from os import access
  24 from os import F_OK
  25 from os import makedirs
  26 from os import getcwd
  27 from os import chdir
  28 from os import listdir
  29 from os import remove
  30 from os import rmdir
  31 from os import linesep
  32 from os import walk
  33 from os import environ
  34 import re
  35 from UserDict import IterableUserDict
  36
  37 import Logger.Log as Logger
  38 from Logger import StringTable as ST
  39 from Logger import ToolError
  40 from Library import GlobalData
  41 from Library.DataType import SUP_MODULE_LIST
  42 from Library.DataType import END_OF_LINE
  43 from Library.DataType import TAB_SPLIT
  44 from Library.DataType import TAB_LANGUAGE_EN_US
  45 from Library.DataType import TAB_LANGUAGE_EN
  46 from Library.DataType import TAB_LANGUAGE_EN_X
  47 from Library.DataType import TAB_UNI_FILE_SUFFIXS
  48 from Library.StringUtils import GetSplitValueList
  49 from Library.ParserValidate import IsValidHexVersion
  50 from Library.ParserValidate import IsValidPath
  51 from Object.POM.CommonObject import TextObject
  52 from Core.FileHook import __FileHookOpen__
  53 from Common.MultipleWorkspace import MultipleWorkspace as mws
  54
  55 ## Convert GUID string in xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx style to C
  56 # structure style
  57 #
  58 # @param      Guid:    The GUID string
  59 #
  60 def GuidStringToGuidStructureString(Guid):
  61     GuidList = Guid.split('-')
  62     Result = '{'
  63     for Index in range(0, 3, 1):
  64         Result = Result + '0x' + GuidList[Index] + ', '
  65     Result = Result + '{0x' + GuidList[3][0:2] + ', 0x' + GuidList[3][2:4]
  66     for Index in range(0, 12, 2):
  67         Result = Result + ', 0x' + GuidList[4][Index:Index + 2]
  68     Result += '}}'
  69     return Result
  70
  71 ## Check whether GUID string is of format xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
  72 #
  73 # @param      GuidValue:   The GUID value
  74 #
  75 def CheckGuidRegFormat(GuidValue):
  76     ## Regular expression used to find out register format of GUID
  77     #
  78     RegFormatGuidPattern = re.compile("^\s*([0-9a-fA-F]){8}-"
  79                                        "([0-9a-fA-F]){4}-"
  80                                        "([0-9a-fA-F]){4}-"
  81                                        "([0-9a-fA-F]){4}-"
  82                                        "([0-9a-fA-F]){12}\s*$")
  83
  84     if RegFormatGuidPattern.match(GuidValue):
  85         return True
  86     else:
  87         return False
  88
  89
  90 ## Convert GUID string in C structure style to
  91 # xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
  92 #
  93 # @param      GuidValue:   The GUID value in C structure format
  94 #
  95 def GuidStructureStringToGuidString(GuidValue):
  96     GuidValueString = GuidValue.lower().replace("{", "").replace("}", "").\
  97     replace(" ", "").replace(";", "")
  98     GuidValueList = GuidValueString.split(",")
  99     if len(GuidValueList) != 11:
 100         return ''
 101     try:
 102         return "%08x-%04x-%04x-%02x%02x-%02x%02x%02x%02x%02x%02x" % (
 103                 int(GuidValueList[0], 16),
 104                 int(GuidValueList[1], 16),
 105                 int(GuidValueList[2], 16),
 106                 int(GuidValueList[3], 16),
 107                 int(GuidValueList[4], 16),
 108                 int(GuidValueList[5], 16),
 109                 int(GuidValueList[6], 16),
 110                 int(GuidValueList[7], 16),
 111                 int(GuidValueList[8], 16),
 112                 int(GuidValueList[9], 16),
 113                 int(GuidValueList[10], 16)
 114                 )
 115     except BaseException:
 116         return ''
 117
 118 ## Create directories
 119 #
 120 # @param      Directory:   The directory name
 121 #
 122 def CreateDirectory(Directory):
 123     if Directory is None or Directory.strip() == "":
 124         return True
 125     try:
 126         if not access(Directory, F_OK):
 127             makedirs(Directory)
 128     except BaseException:
 129         return False
 130     return True
 131
 132 ## Remove directories, including files and sub-directories in it
 133 #
 134 # @param      Directory:   The directory name
 135 #
 136 def RemoveDirectory(Directory, Recursively=False):
 137     if Directory is None or Directory.strip() == "" or not \
 138     os.path.exists(Directory):
 139         return
 140     if Recursively:
 141         CurrentDirectory = getcwd()
 142         chdir(Directory)
 143         for File in listdir("."):
 144             if os.path.isdir(File):
 145                 RemoveDirectory(File, Recursively)
 146             else:
 147                 remove(File)
 148         chdir(CurrentDirectory)
 149     rmdir(Directory)
 150
 151 ## Store content in file
 152 #
 153 # This method is used to save file only when its content is changed. This is
 154 # quite useful for "make" system to decide what will be re-built and what
 155 # won't.
 156 #
 157 # @param      File:            The path of file
 158 # @param      Content:         The new content of the file
 159 # @param      IsBinaryFile:    The flag indicating if the file is binary file
 160 #                              or not
 161 #
 162 def SaveFileOnChange(File, Content, IsBinaryFile=True):
 163     if not IsBinaryFile:
 164         Content = Content.replace("\n", linesep)
 165
 166     if os.path.exists(File):
 167         try:
 168             if Content == __FileHookOpen__(File, "rb").read():
 169                 return False
 170         except BaseException:
 171             Logger.Error(None, ToolError.FILE_OPEN_FAILURE, ExtraData=File)
 172
 173     CreateDirectory(os.path.dirname(File))
 174     try:
 175         FileFd = __FileHookOpen__(File, "wb")
 176         FileFd.write(Content)
 177         FileFd.close()
 178     except BaseException:
 179         Logger.Error(None, ToolError.FILE_CREATE_FAILURE, ExtraData=File)
 180
 181     return True
 182
 183 ## Get all files of a directory
 184 #
 185 # @param Root:       Root dir
 186 # @param SkipList :  The files need be skipped
 187 #
 188 def GetFiles(Root, SkipList=None, FullPath=True):
 189     OriPath = os.path.normpath(Root)
 190     FileList = []
 191     for Root, Dirs, Files in walk(Root):
 192         if SkipList:
 193             for Item in SkipList:
 194                 if Item in Dirs:
 195                     Dirs.remove(Item)
 196                 if Item in Files:
 197                     Files.remove(Item)
 198         for Dir in Dirs:
 199             if Dir.startswith('.'):
 200                 Dirs.remove(Dir)
 201
 202         for File in Files:
 203             if File.startswith('.'):
 204                 continue
 205             File = os.path.normpath(os.path.join(Root, File))
 206             if not FullPath:
 207                 File = File[len(OriPath) + 1:]
 208             FileList.append(File)
 209
 210     return FileList
 211
 212 ## Get all non-metadata files of a directory
 213 #
 214 # @param Root:       Root Dir
 215 # @param SkipList :  List of path need be skipped
 216 # @param FullPath:  True if the returned file should be full path
 217 # @param PrefixPath: the path that need to be added to the files found
 218 # @return: the list of files found
 219 #
 220 def GetNonMetaDataFiles(Root, SkipList, FullPath, PrefixPath):
 221     FileList = GetFiles(Root, SkipList, FullPath)
 222     NewFileList = []
 223     for File in FileList:
 224         ExtName = os.path.splitext(File)[1]
 225         #
 226         # skip '.dec', '.inf', '.dsc', '.fdf' files
 227         #
 228         if ExtName.lower() not in ['.dec', '.inf', '.dsc', '.fdf']:
 229             NewFileList.append(os.path.normpath(os.path.join(PrefixPath, File)))
 230
 231     return NewFileList
 232
 233 ## Check if given file exists or not
 234 #
 235 # @param      File:    File name or path to be checked
 236 # @param      Dir:     The directory the file is relative to
 237 #
 238 def ValidFile(File, Ext=None):
 239     File = File.replace('\\', '/')
 240     if Ext is not None:
 241         FileExt = os.path.splitext(File)[1]
 242         if FileExt.lower() != Ext.lower():
 243             return False
 244     if not os.path.exists(File):
 245         return False
 246     return True
 247
 248 ## RealPath
 249 #
 250 # @param      File:    File name or path to be checked
 251 # @param      Dir:     The directory the file is relative to
 252 # @param      OverrideDir:     The override directory
 253 #
 254 def RealPath(File, Dir='', OverrideDir=''):
 255     NewFile = os.path.normpath(os.path.join(Dir, File))
 256     NewFile = GlobalData.gALL_FILES[NewFile]
 257     if not NewFile and OverrideDir:
 258         NewFile = os.path.normpath(os.path.join(OverrideDir, File))
 259         NewFile = GlobalData.gALL_FILES[NewFile]
 260     return NewFile
 261
 262 ## RealPath2
 263 #
 264 # @param      File:    File name or path to be checked
 265 # @param      Dir:     The directory the file is relative to
 266 # @param      OverrideDir:     The override directory
 267 #
 268 def RealPath2(File, Dir='', OverrideDir=''):
 269     if OverrideDir:
 270         NewFile = GlobalData.gALL_FILES[os.path.normpath(os.path.join\
 271                                                         (OverrideDir, File))]
 272         if NewFile:
 273             if OverrideDir[-1] == os.path.sep:
 274                 return NewFile[len(OverrideDir):], NewFile[0:len(OverrideDir)]
 275             else:
 276                 return NewFile[len(OverrideDir) + 1:], \
 277             NewFile[0:len(OverrideDir)]
 278
 279     NewFile = GlobalData.gALL_FILES[os.path.normpath(os.path.join(Dir, File))]
 280     if NewFile:
 281         if Dir:
 282             if Dir[-1] == os.path.sep:
 283                 return NewFile[len(Dir):], NewFile[0:len(Dir)]
 284             else:
 285                 return NewFile[len(Dir) + 1:], NewFile[0:len(Dir)]
 286         else:
 287             return NewFile, ''
 288
 289     return None, None
 290
 291 ## A dict which can access its keys and/or values orderly
 292 #
 293 #  The class implements a new kind of dict which its keys or values can be
 294 #  accessed in the order they are added into the dict. It guarantees the order
 295 #  by making use of an internal list to keep a copy of keys.
 296 #
 297 class Sdict(IterableUserDict):
 298     ## Constructor
 299     #
 300     def __init__(self):
 301         IterableUserDict.__init__(self)
 302         self._key_list = []
 303
 304     ## [] operator
 305     #
 306     def __setitem__(self, Key, Value):
 307         if Key not in self._key_list:
 308             self._key_list.append(Key)
 309         IterableUserDict.__setitem__(self, Key, Value)
 310
 311     ## del operator
 312     #
 313     def __delitem__(self, Key):
 314         self._key_list.remove(Key)
 315         IterableUserDict.__delitem__(self, Key)
 316
 317     ## used in "for k in dict" loop to ensure the correct order
 318     #
 319     def __iter__(self):
 320         return self.iterkeys()
 321
 322     ## len() support
 323     #
 324     def __len__(self):
 325         return len(self._key_list)
 326
 327     ## "in" test support
 328     #
 329     def __contains__(self, Key):
 330         return Key in self._key_list
 331
 332     ## indexof support
 333     #
 334     def index(self, Key):
 335         return self._key_list.index(Key)
 336
 337     ## insert support
 338     #
 339     def insert(self, Key, Newkey, Newvalue, Order):
 340         Index = self._key_list.index(Key)
 341         if Order == 'BEFORE':
 342             self._key_list.insert(Index, Newkey)
 343             IterableUserDict.__setitem__(self, Newkey, Newvalue)
 344         elif Order == 'AFTER':
 345             self._key_list.insert(Index + 1, Newkey)
 346             IterableUserDict.__setitem__(self, Newkey, Newvalue)
 347
 348     ## append support
 349     #
 350     def append(self, Sdict2):
 351         for Key in Sdict2:
 352             if Key not in self._key_list:
 353                 self._key_list.append(Key)
 354             IterableUserDict.__setitem__(self, Key, Sdict2[Key])
 355     ## hash key
 356     #
 357     def has_key(self, Key):
 358         return Key in self._key_list
 359
 360     ## Empty the dict
 361     #
 362     def clear(self):
 363         self._key_list = []
 364         IterableUserDict.clear(self)
 365
 366     ## Return a copy of keys
 367     #
 368     def keys(self):
 369         Keys = []
 370         for Key in self._key_list:
 371             Keys.append(Key)
 372         return Keys
 373
 374     ## Return a copy of values
 375     #
 376     def values(self):
 377         Values = []
 378         for Key in self._key_list:
 379             Values.append(self[Key])
 380         return Values
 381
 382     ## Return a copy of (key, value) list
 383     #
 384     def items(self):
 385         Items = []
 386         for Key in self._key_list:
 387             Items.append((Key, self[Key]))
 388         return Items
 389
 390     ## Iteration support
 391     #
 392     def iteritems(self):
 393         return iter(self.items())
 394
 395     ## Keys interation support
 396     #
 397     def iterkeys(self):
 398         return iter(self.keys())
 399
 400     ## Values interation support
 401     #
 402     def itervalues(self):
 403         return iter(self.values())
 404
 405     ## Return value related to a key, and remove the (key, value) from the dict
 406     #
 407     def pop(self, Key, *Dv):
 408         Value = None
 409         if Key in self._key_list:
 410             Value = self[Key]
 411             self.__delitem__(Key)
 412         elif len(Dv) != 0 :
 413             Value = Dv[0]
 414         return Value
 415
 416     ## Return (key, value) pair, and remove the (key, value) from the dict
 417     #
 418     def popitem(self):
 419         Key = self._key_list[-1]
 420         Value = self[Key]
 421         self.__delitem__(Key)
 422         return Key, Value
 423     ## update method
 424     #
 425     def update(self, Dict=None, **Kwargs):
 426         if Dict is not None:
 427             for Key1, Val1 in Dict.items():
 428                 self[Key1] = Val1
 429         if len(Kwargs):
 430             for Key1, Val1 in Kwargs.items():
 431                 self[Key1] = Val1
 432
 433 ## CommonPath
 434 #
 435 # @param PathList: PathList
 436 #
 437 def CommonPath(PathList):
 438     Path1 = min(PathList).split(os.path.sep)
 439     Path2 = max(PathList).split(os.path.sep)
 440     for Index in xrange(min(len(Path1), len(Path2))):
 441         if Path1[Index] != Path2[Index]:
 442             return os.path.sep.join(Path1[:Index])
 443     return os.path.sep.join(Path1)
 444
 445 ## PathClass
 446 #
 447 class PathClass(object):
 448     def __init__(self, File='', Root='', AlterRoot='', Type='', IsBinary=False,
 449                  Arch='COMMON', ToolChainFamily='', Target='', TagName='', \
 450                  ToolCode=''):
 451         self.Arch = Arch
 452         self.File = str(File)
 453         if os.path.isabs(self.File):
 454             self.Root = ''
 455             self.AlterRoot = ''
 456         else:
 457             self.Root = str(Root)
 458             self.AlterRoot = str(AlterRoot)
 459
 460         #
 461         # Remove any '.' and '..' in path
 462         #
 463         if self.Root:
 464             self.Path = os.path.normpath(os.path.join(self.Root, self.File))
 465             self.Root = os.path.normpath(CommonPath([self.Root, self.Path]))
 466             #
 467             # eliminate the side-effect of 'C:'
 468             #
 469             if self.Root[-1] == ':':
 470                 self.Root += os.path.sep
 471             #
 472             # file path should not start with path separator
 473             #
 474             if self.Root[-1] == os.path.sep:
 475                 self.File = self.Path[len(self.Root):]
 476             else:
 477                 self.File = self.Path[len(self.Root) + 1:]
 478         else:
 479             self.Path = os.path.normpath(self.File)
 480
 481         self.SubDir, self.Name = os.path.split(self.File)
 482         self.BaseName, self.Ext = os.path.splitext(self.Name)
 483
 484         if self.Root:
 485             if self.SubDir:
 486                 self.Dir = os.path.join(self.Root, self.SubDir)
 487             else:
 488                 self.Dir = self.Root
 489         else:
 490             self.Dir = self.SubDir
 491
 492         if IsBinary:
 493             self.Type = Type
 494         else:
 495             self.Type = self.Ext.lower()
 496
 497         self.IsBinary = IsBinary
 498         self.Target = Target
 499         self.TagName = TagName
 500         self.ToolCode = ToolCode
 501         self.ToolChainFamily = ToolChainFamily
 502
 503         self._Key = None
 504
 505     ## Convert the object of this class to a string
 506     #
 507     #  Convert member Path of the class to a string
 508     #
 509     def __str__(self):
 510         return self.Path
 511
 512     ## Override __eq__ function
 513     #
 514     # Check whether PathClass are the same
 515     #
 516     def __eq__(self, Other):
 517         if type(Other) == type(self):
 518             return self.Path == Other.Path
 519         else:
 520             return self.Path == str(Other)
 521
 522     ## Override __hash__ function
 523     #
 524     # Use Path as key in hash table
 525     #
 526     def __hash__(self):
 527         return hash(self.Path)
 528
 529     ## _GetFileKey
 530     #
 531     def _GetFileKey(self):
 532         if self._Key is None:
 533             self._Key = self.Path.upper()
 534         return self._Key
 535     ## Validate
 536     #
 537     def Validate(self, Type='', CaseSensitive=True):
 538         if GlobalData.gCASE_INSENSITIVE:
 539             CaseSensitive = False
 540         if Type and Type.lower() != self.Type:
 541             return ToolError.FILE_TYPE_MISMATCH, '%s (expect %s but got %s)' % \
 542         (self.File, Type, self.Type)
 543
 544         RealFile, RealRoot = RealPath2(self.File, self.Root, self.AlterRoot)
 545         if not RealRoot and not RealFile:
 546             RealFile = self.File
 547             if self.AlterRoot:
 548                 RealFile = os.path.join(self.AlterRoot, self.File)
 549             elif self.Root:
 550                 RealFile = os.path.join(self.Root, self.File)
 551             return ToolError.FILE_NOT_FOUND, os.path.join(self.AlterRoot, RealFile)
 552
 553         ErrorCode = 0
 554         ErrorInfo = ''
 555         if RealRoot != self.Root or RealFile != self.File:
 556             if CaseSensitive and (RealFile != self.File or \
 557                                   (RealRoot != self.Root and RealRoot != \
 558                                    self.AlterRoot)):
 559                 ErrorCode = ToolError.FILE_CASE_MISMATCH
 560                 ErrorInfo = self.File + '\n\t' + RealFile + \
 561                  " [in file system]"
 562
 563             self.SubDir, self.Name = os.path.split(RealFile)
 564             self.BaseName, self.Ext = os.path.splitext(self.Name)
 565             if self.SubDir:
 566                 self.Dir = os.path.join(RealRoot, self.SubDir)
 567             else:
 568                 self.Dir = RealRoot
 569             self.File = RealFile
 570             self.Root = RealRoot
 571             self.Path = os.path.join(RealRoot, RealFile)
 572         return ErrorCode, ErrorInfo
 573
 574     Key = property(_GetFileKey)
 575
 576 ## Get current workspace
 577 #
 578 #  get WORKSPACE from environment variable if present,if not use current working directory as WORKSPACE
 579 #
 580 def GetWorkspace():
 581     #
 582     # check WORKSPACE
 583     #
 584     if "WORKSPACE" in environ:
 585         WorkspaceDir = os.path.normpath(environ["WORKSPACE"])
 586         if not os.path.exists(WorkspaceDir):
 587             Logger.Error("UPT",
 588                          ToolError.UPT_ENVIRON_MISSING_ERROR,
 589                          ST.ERR_WORKSPACE_NOTEXIST,
 590                          ExtraData="%s" % WorkspaceDir)
 591     else:
 592         WorkspaceDir = os.getcwd()
 593
 594     if WorkspaceDir[-1] == ':':
 595         WorkspaceDir += os.sep
 596
 597     PackagesPath = os.environ.get("PACKAGES_PATH")
 598     mws.setWs(WorkspaceDir, PackagesPath)
 599
 600     return WorkspaceDir, mws.PACKAGES_PATH
 601
 602 ## Get relative path
 603 #
 604 #  use full path and workspace to get relative path
 605 #  the destination of this function is mainly to resolve the root path issue(like c: or c:\)
 606 #
 607 #  @param Fullpath: a string of fullpath
 608 #  @param Workspace: a string of workspace
 609 #
 610 def GetRelativePath(Fullpath, Workspace):
 611
 612     RelativePath = ''
 613     if Workspace.endswith(os.sep):
 614         RelativePath = Fullpath[Fullpath.upper().find(Workspace.upper())+len(Workspace):]
 615     else:
 616         RelativePath = Fullpath[Fullpath.upper().find(Workspace.upper())+len(Workspace)+1:]
 617
 618     return RelativePath
 619
 620 ## Check whether all module types are in list
 621 #
 622 # check whether all module types (SUP_MODULE_LIST) are in list
 623 #
 624 # @param ModuleList:  a list of ModuleType
 625 #
 626 def IsAllModuleList(ModuleList):
 627     NewModuleList = [Module.upper() for Module in ModuleList]
 628     for Module in SUP_MODULE_LIST:
 629         if Module not in NewModuleList:
 630             return False
 631     else:
 632         return True
 633
 634 ## Dictionary that use comment(GenericComment, TailComment) as value,
 635 # if a new comment which key already in the dic is inserted, then the
 636 # comment will be merged.
 637 # Key is (Statement, SupArch), when TailComment is added, it will ident
 638 # according to Statement
 639 #
 640 class MergeCommentDict(dict):
 641     ## []= operator
 642     #
 643     def __setitem__(self, Key, CommentVal):
 644         GenericComment, TailComment = CommentVal
 645         if Key in self:
 646             OrigVal1, OrigVal2 = dict.__getitem__(self, Key)
 647             Statement = Key[0]
 648             dict.__setitem__(self, Key, (OrigVal1 + GenericComment, OrigVal2 \
 649                                          + len(Statement) * ' ' + TailComment))
 650         else:
 651             dict.__setitem__(self, Key, (GenericComment, TailComment))
 652
 653     ## =[] operator
 654     #
 655     def __getitem__(self, Key):
 656         return dict.__getitem__(self, Key)
 657
 658
 659 ## GenDummyHelpTextObj
 660 #
 661 # @retval HelpTxt:   Generated dummy help text object
 662 #
 663 def GenDummyHelpTextObj():
 664     HelpTxt = TextObject()
 665     HelpTxt.SetLang(TAB_LANGUAGE_EN_US)
 666     HelpTxt.SetString(' ')
 667     return HelpTxt
 668
 669 ## ConvertVersionToDecimal, the minor version should be within 0 - 99
 670 # <HexVersion>          ::=  "0x" <Major> <Minor>
 671 # <Major>               ::=  (a-fA-F0-9){4}
 672 # <Minor>               ::=  (a-fA-F0-9){4}
 673 # <DecVersion>          ::=  (0-65535) ["." (0-99)]
 674 #
 675 # @param StringIn:  The string contains version defined in INF file.
 676 #                   It can be Decimal or Hex
 677 #
 678 def ConvertVersionToDecimal(StringIn):
 679     if IsValidHexVersion(StringIn):
 680         Value = int(StringIn, 16)
 681         Major = Value >> 16
 682         Minor = Value & 0xFFFF
 683         MinorStr = str(Minor)
 684         if len(MinorStr) == 1:
 685             MinorStr = '0' + MinorStr
 686         return str(Major) + '.' + MinorStr
 687     else:
 688         if StringIn.find(TAB_SPLIT) != -1:
 689             return StringIn
 690         elif StringIn:
 691             return StringIn + '.0'
 692         else:
 693             #
 694             # when StringIn is '', return it directly
 695             #
 696             return StringIn
 697
 698 ## GetHelpStringByRemoveHashKey
 699 #
 700 # Remove hash key at the header of string and return the remain.
 701 #
 702 # @param String:  The string need to be processed.
 703 #
 704 def GetHelpStringByRemoveHashKey(String):
 705     ReturnString = ''
 706     PattenRemoveHashKey = re.compile(r"^[#+\s]+", re.DOTALL)
 707     String = String.strip()
 708     if String == '':
 709         return String
 710
 711     LineList = GetSplitValueList(String, END_OF_LINE)
 712     for Line in LineList:
 713         ValueList = PattenRemoveHashKey.split(Line)
 714         if len(ValueList) == 1:
 715             ReturnString += ValueList[0] + END_OF_LINE
 716         else:
 717             ReturnString += ValueList[1] + END_OF_LINE
 718
 719     if ReturnString.endswith('\n') and not ReturnString.endswith('\n\n') and ReturnString != '\n':
 720         ReturnString = ReturnString[:-1]
 721
 722     return ReturnString
 723
 724 ## ConvPathFromAbsToRel
 725 #
 726 # Get relative file path from absolute path.
 727 #
 728 # @param Path:  The string contain file absolute path.
 729 # @param Root:  The string contain the parent path of Path in.
 730 #
 731 #
 732 def ConvPathFromAbsToRel(Path, Root):
 733     Path = os.path.normpath(Path)
 734     Root = os.path.normpath(Root)
 735     FullPath = os.path.normpath(os.path.join(Root, Path))
 736
 737     #
 738     # If Path is absolute path.
 739     # It should be in Root.
 740     #
 741     if os.path.isabs(Path):
 742         return FullPath[FullPath.find(Root) + len(Root) + 1:]
 743
 744     else:
 745         return Path
 746
 747 ## ConvertPath
 748 #
 749 # Convert special characters to '_', '\' to '/'
 750 # return converted path: Test!1.inf -> Test_1.inf
 751 #
 752 # @param Path: Path to be converted
 753 #
 754 def ConvertPath(Path):
 755     RetPath = ''
 756     for Char in Path.strip():
 757         if Char.isalnum() or Char in '.-_/':
 758             RetPath = RetPath + Char
 759         elif Char == '\\':
 760             RetPath = RetPath + '/'
 761         else:
 762             RetPath = RetPath + '_'
 763     return RetPath
 764
 765 ## ConvertSpec
 766 #
 767 # during install, convert the Spec string extract from UPD into INF allowable definition,
 768 # the difference is period is allowed in the former (not the first letter) but not in the latter.
 769 # return converted Spec string
 770 #
 771 # @param SpecStr: SpecStr to be converted
 772 #
 773 def ConvertSpec(SpecStr):
 774     RetStr = ''
 775     for Char in SpecStr:
 776         if Char.isalnum() or Char == '_':
 777             RetStr = RetStr + Char
 778         else:
 779             RetStr = RetStr + '_'
 780
 781     return RetStr
 782
 783
 784 ## IsEqualList
 785 #
 786 # Judge two lists are identical(contain same item).
 787 # The rule is elements in List A are in List B and elements in List B are in List A.
 788 #
 789 # @param ListA, ListB  Lists need to be judged.
 790 #
 791 # @return True  ListA and ListB are identical
 792 # @return False ListA and ListB are different with each other
 793 #
 794 def IsEqualList(ListA, ListB):
 795     if ListA == ListB:
 796         return True
 797
 798     for ItemA in ListA:
 799         if not ItemA in ListB:
 800             return False
 801
 802     for ItemB in ListB:
 803         if not ItemB in ListA:
 804             return False
 805
 806     return True
 807
 808 ## ConvertArchList
 809 #
 810 # Convert item in ArchList if the start character is lower case.
 811 # In UDP spec, Arch is only allowed as: [A-Z]([a-zA-Z0-9])*
 812 #
 813 # @param ArchList The ArchList need to be converted.
 814 #
 815 # @return NewList  The ArchList been converted.
 816 #
 817 def ConvertArchList(ArchList):
 818     NewArchList = []
 819     if not ArchList:
 820         return NewArchList
 821
 822     if type(ArchList) == list:
 823         for Arch in ArchList:
 824             Arch = Arch.upper()
 825             NewArchList.append(Arch)
 826     elif type(ArchList) == str:
 827         ArchList = ArchList.upper()
 828         NewArchList.append(ArchList)
 829
 830     return NewArchList
 831
 832 ## ProcessLineExtender
 833 #
 834 # Process the LineExtender of Line in LineList.
 835 # If one line ends with a line extender, then it will be combined together with next line.
 836 #
 837 # @param LineList The LineList need to be processed.
 838 #
 839 # @return NewList  The ArchList been processed.
 840 #
 841 def ProcessLineExtender(LineList):
 842     NewList = []
 843     Count = 0
 844     while Count < len(LineList):
 845         if LineList[Count].strip().endswith("\\") and Count + 1 < len(LineList):
 846             NewList.append(LineList[Count].strip()[:-2] + LineList[Count + 1])
 847             Count = Count + 1
 848         else:
 849             NewList.append(LineList[Count])
 850
 851         Count = Count + 1
 852
 853     return NewList
 854
 855 ## ProcessEdkComment
 856 #
 857 # Process EDK style comment in LineList: c style /* */ comment or cpp style // comment
 858 #
 859 #
 860 # @param LineList The LineList need to be processed.
 861 #
 862 # @return LineList  The LineList been processed.
 863 # @return FirstPos  Where Edk comment is first found, -1 if not found
 864 #
 865 def ProcessEdkComment(LineList):
 866     FindEdkBlockComment = False
 867     Count = 0
 868     StartPos = -1
 869     EndPos = -1
 870     FirstPos = -1
 871
 872     while(Count < len(LineList)):
 873         Line = LineList[Count].strip()
 874         if Line.startswith("/*"):
 875             #
 876             # handling c style comment
 877             #
 878             StartPos = Count
 879             while Count < len(LineList):
 880                 Line = LineList[Count].strip()
 881                 if Line.endswith("*/"):
 882                     if (Count == StartPos) and Line.strip() == '/*/':
 883                         Count = Count + 1
 884                         continue
 885                     EndPos = Count
 886                     FindEdkBlockComment = True
 887                     break
 888                 Count = Count + 1
 889
 890             if FindEdkBlockComment:
 891                 if FirstPos == -1:
 892                     FirstPos = StartPos
 893                 for Index in xrange(StartPos, EndPos+1):
 894                     LineList[Index] = ''
 895                 FindEdkBlockComment = False
 896         elif Line.find("//") != -1 and not Line.startswith("#"):
 897             #
 898             # handling cpp style comment
 899             #
 900             LineList[Count] = Line.replace("//", '#')
 901             if FirstPos == -1:
 902                 FirstPos = Count
 903
 904         Count = Count + 1
 905
 906     return LineList, FirstPos
 907
 908 ## GetLibInstanceInfo
 909 #
 910 # Get the information from Library Instance INF file.
 911 #
 912 # @param string.  A string start with # and followed by INF file path
 913 # @param WorkSpace. The WorkSpace directory used to combined with INF file path.
 914 #
 915 # @return GUID, Version
 916 def GetLibInstanceInfo(String, WorkSpace, LineNo):
 917
 918     FileGuidString = ""
 919     VerString = ""
 920
 921     OrignalString = String
 922     String = String.strip()
 923     if not String:
 924         return None, None
 925     #
 926     # Remove "#" characters at the beginning
 927     #
 928     String = GetHelpStringByRemoveHashKey(String)
 929     String = String.strip()
 930
 931     #
 932     # Validate file name exist.
 933     #
 934     FullFileName = os.path.normpath(os.path.realpath(os.path.join(WorkSpace, String)))
 935     if not (ValidFile(FullFileName)):
 936         Logger.Error("InfParser",
 937                      ToolError.FORMAT_INVALID,
 938                      ST.ERR_FILELIST_EXIST % (String),
 939                      File=GlobalData.gINF_MODULE_NAME,
 940                      Line=LineNo,
 941                      ExtraData=OrignalString)
 942
 943     #
 944     # Validate file exist/format.
 945     #
 946     if IsValidPath(String, WorkSpace):
 947         IsValidFileFlag = True
 948     else:
 949         Logger.Error("InfParser",
 950                      ToolError.FORMAT_INVALID,
 951                      ST.ERR_INF_PARSER_FILE_NOT_EXIST_OR_NAME_INVALID % (String),
 952                      File=GlobalData.gINF_MODULE_NAME,
 953                      Line=LineNo,
 954                      ExtraData=OrignalString)
 955         return False
 956     if IsValidFileFlag:
 957         FileLinesList = []
 958
 959         try:
 960             FInputfile = open(FullFileName, "rb", 0)
 961             try:
 962                 FileLinesList = FInputfile.readlines()
 963             except BaseException:
 964                 Logger.Error("InfParser",
 965                              ToolError.FILE_READ_FAILURE,
 966                              ST.ERR_FILE_OPEN_FAILURE,
 967                              File=FullFileName)
 968             finally:
 969                 FInputfile.close()
 970         except BaseException:
 971             Logger.Error("InfParser",
 972                          ToolError.FILE_READ_FAILURE,
 973                          ST.ERR_FILE_OPEN_FAILURE,
 974                          File=FullFileName)
 975
 976         ReFileGuidPattern = re.compile("^\s*FILE_GUID\s*=.*$")
 977         ReVerStringPattern = re.compile("^\s*VERSION_STRING\s*=.*$")
 978
 979         FileLinesList = ProcessLineExtender(FileLinesList)
 980
 981         for Line in FileLinesList:
 982             if ReFileGuidPattern.match(Line):
 983                 FileGuidString = Line
 984             if ReVerStringPattern.match(Line):
 985                 VerString = Line
 986
 987         if FileGuidString:
 988             FileGuidString = GetSplitValueList(FileGuidString, '=', 1)[1]
 989         if VerString:
 990             VerString = GetSplitValueList(VerString, '=', 1)[1]
 991
 992         return FileGuidString, VerString
 993
 994 ## GetLocalValue
 995 #
 996 # Generate the local value for INF and DEC file. If Lang attribute not present, then use this value.
 997 # If present, and there is no element without the Lang attribute, and one of the elements has the rfc1766 code is
 998 # "en-x-tianocore", or "en-US" if "en-x-tianocore" was not found, or "en" if "en-US" was not found, or startswith 'en'
 999 # if 'en' was not found, then use this value.
1000 # If multiple entries of a tag exist which have the same language code, use the last entry.
1001 #
1002 # @param ValueList  A list need to be processed.
1003 # @param UseFirstValue: True to use the first value, False to use the last value
1004 #
1005 # @return LocalValue
1006 def GetLocalValue(ValueList, UseFirstValue=False):
1007     Value1 = ''
1008     Value2 = ''
1009     Value3 = ''
1010     Value4 = ''
1011     Value5 = ''
1012     for (Key, Value) in ValueList:
1013         if Key == TAB_LANGUAGE_EN_X:
1014             if UseFirstValue:
1015                 if not Value1:
1016                     Value1 = Value
1017             else:
1018                 Value1 = Value
1019         if Key == TAB_LANGUAGE_EN_US:
1020             if UseFirstValue:
1021                 if not Value2:
1022                     Value2 = Value
1023             else:
1024                 Value2 = Value
1025         if Key == TAB_LANGUAGE_EN:
1026             if UseFirstValue:
1027                 if not Value3:
1028                     Value3 = Value
1029             else:
1030                 Value3 = Value
1031         if Key.startswith(TAB_LANGUAGE_EN):
1032             if UseFirstValue:
1033                 if not Value4:
1034                     Value4 = Value
1035             else:
1036                 Value4 = Value
1037         if Key == '':
1038             if UseFirstValue:
1039                 if not Value5:
1040                     Value5 = Value
1041             else:
1042                 Value5 = Value
1043
1044     if Value1:
1045         return Value1
1046     if Value2:
1047         return Value2
1048     if Value3:
1049         return Value3
1050     if Value4:
1051         return Value4
1052     if Value5:
1053         return Value5
1054
1055     return ''
1056
1057
1058 ## GetCharIndexOutStr
1059 #
1060 # Get comment character index outside a string
1061 #
1062 # @param Line:              The string to be checked
1063 # @param CommentCharacter:  Comment char, used to ignore comment content
1064 #
1065 # @retval Index
1066 #
1067 def GetCharIndexOutStr(CommentCharacter, Line):
1068     #
1069     # remove whitespace
1070     #
1071     Line = Line.strip()
1072
1073     #
1074     # Check whether comment character is in a string
1075     #
1076     InString = False
1077     for Index in range(0, len(Line)):
1078         if Line[Index] == '"':
1079             InString = not InString
1080         elif Line[Index] == CommentCharacter and InString :
1081             pass
1082         elif Line[Index] == CommentCharacter and (Index +1) < len(Line) and Line[Index+1] == CommentCharacter \
1083             and not InString :
1084             return Index
1085     return -1
1086
1087 ## ValidateUNIFilePath
1088 #
1089 # Check the UNI file path
1090 #
1091 # @param FilePath: The UNI file path
1092 #
1093 def ValidateUNIFilePath(Path):
1094     Suffix = Path[Path.rfind(TAB_SPLIT):]
1095
1096     #
1097     # Check if the suffix is one of the '.uni', '.UNI', '.Uni'
1098     #
1099     if Suffix not in TAB_UNI_FILE_SUFFIXS:
1100         Logger.Error("Unicode File Parser",
1101                         ToolError.FORMAT_INVALID,
1102                         Message=ST.ERR_UNI_FILE_SUFFIX_WRONG,
1103                         ExtraData=Path)
1104
1105     #
1106     # Check if '..' in the file name(without suffixe)
1107     #
1108     if (TAB_SPLIT + TAB_SPLIT) in Path:
1109         Logger.Error("Unicode File Parser",
1110                         ToolError.FORMAT_INVALID,
1111                         Message=ST.ERR_UNI_FILE_NAME_INVALID,
1112                         ExtraData=Path)
1113
1114     #
1115     # Check if the file name is valid according to the DEC and INF specification
1116     #
1117     Pattern = '[a-zA-Z0-9_][a-zA-Z0-9_\-\.]*'
1118     FileName = Path.replace(Suffix, '')
1119     InvalidCh = re.sub(Pattern, '', FileName)
1120     if InvalidCh:
1121         Logger.Error("Unicode File Parser",
1122                         ToolError.FORMAT_INVALID,
1123                         Message=ST.ERR_INF_PARSER_FILE_NOT_EXIST_OR_NAME_INVALID,
1124                         ExtraData=Path)
1125