BaseTools/Source/Python/UPT/Library/UniClassObject.py

   1 ## @file
   2 # Collect all defined strings in multiple uni files.
   3 #
   4 # Copyright (c) 2014, Intel Corporation. All rights reserved.<BR>
   5 #
   6 # This program and the accompanying materials are licensed and made available
   7 # under the terms and conditions of the BSD License which accompanies this
   8 # distribution. The full text of the license may be found at
   9 # http://opensource.org/licenses/bsd-license.php
  10 #
  11 # THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS,
  12 # WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED.
  13 #
  14 """
  15 Collect all defined strings in multiple uni files
  16 """
  17
  18 ##
  19 # Import Modules
  20 #
  21 import os, codecs, re
  22 import distutils.util
  23 from Logger import ToolError
  24 from Logger import Log as EdkLogger
  25 from Logger import StringTable as ST
  26 from Library.String import GetLineNo
  27 from Library.Misc import PathClass
  28 from Library.Misc import GetCharIndexOutStr
  29 from Library import DataType as DT
  30
  31 ##
  32 # Static definitions
  33 #
  34 UNICODE_WIDE_CHAR = u'\\wide'
  35 UNICODE_NARROW_CHAR = u'\\narrow'
  36 UNICODE_NON_BREAKING_CHAR = u'\\nbr'
  37 UNICODE_UNICODE_CR = '\r'
  38 UNICODE_UNICODE_LF = '\n'
  39
  40 NARROW_CHAR = u'\uFFF0'
  41 WIDE_CHAR = u'\uFFF1'
  42 NON_BREAKING_CHAR = u'\uFFF2'
  43 CR = u'\u000D'
  44 LF = u'\u000A'
  45 NULL = u'\u0000'
  46 TAB = u'\t'
  47 BACK_SPLASH = u'\\'
  48
  49 gINCLUDE_PATTERN = re.compile("^!include[\s]+([\S]+)[\s]*$", re.MULTILINE | re.UNICODE)
  50
  51 gLANG_CONV_TABLE = {'eng':'en', 'fra':'fr', \
  52                  'aar':'aa', 'abk':'ab', 'ave':'ae', 'afr':'af', 'aka':'ak', 'amh':'am', \
  53                  'arg':'an', 'ara':'ar', 'asm':'as', 'ava':'av', 'aym':'ay', 'aze':'az', \
  54                  'bak':'ba', 'bel':'be', 'bul':'bg', 'bih':'bh', 'bis':'bi', 'bam':'bm', \
  55                  'ben':'bn', 'bod':'bo', 'bre':'br', 'bos':'bs', 'cat':'ca', 'che':'ce', \
  56                  'cha':'ch', 'cos':'co', 'cre':'cr', 'ces':'cs', 'chu':'cu', 'chv':'cv', \
  57                  'cym':'cy', 'dan':'da', 'deu':'de', 'div':'dv', 'dzo':'dz', 'ewe':'ee', \
  58                  'ell':'el', 'epo':'eo', 'spa':'es', 'est':'et', 'eus':'eu', 'fas':'fa', \
  59                  'ful':'ff', 'fin':'fi', 'fij':'fj', 'fao':'fo', 'fry':'fy', 'gle':'ga', \
  60                  'gla':'gd', 'glg':'gl', 'grn':'gn', 'guj':'gu', 'glv':'gv', 'hau':'ha', \
  61                  'heb':'he', 'hin':'hi', 'hmo':'ho', 'hrv':'hr', 'hat':'ht', 'hun':'hu', \
  62                  'hye':'hy', 'her':'hz', 'ina':'ia', 'ind':'id', 'ile':'ie', 'ibo':'ig', \
  63                  'iii':'ii', 'ipk':'ik', 'ido':'io', 'isl':'is', 'ita':'it', 'iku':'iu', \
  64                  'jpn':'ja', 'jav':'jv', 'kat':'ka', 'kon':'kg', 'kik':'ki', 'kua':'kj', \
  65                  'kaz':'kk', 'kal':'kl', 'khm':'km', 'kan':'kn', 'kor':'ko', 'kau':'kr', \
  66                  'kas':'ks', 'kur':'ku', 'kom':'kv', 'cor':'kw', 'kir':'ky', 'lat':'la', \
  67                  'ltz':'lb', 'lug':'lg', 'lim':'li', 'lin':'ln', 'lao':'lo', 'lit':'lt', \
  68                  'lub':'lu', 'lav':'lv', 'mlg':'mg', 'mah':'mh', 'mri':'mi', 'mkd':'mk', \
  69                  'mal':'ml', 'mon':'mn', 'mar':'mr', 'msa':'ms', 'mlt':'mt', 'mya':'my', \
  70                  'nau':'na', 'nob':'nb', 'nde':'nd', 'nep':'ne', 'ndo':'ng', 'nld':'nl', \
  71                  'nno':'nn', 'nor':'no', 'nbl':'nr', 'nav':'nv', 'nya':'ny', 'oci':'oc', \
  72                  'oji':'oj', 'orm':'om', 'ori':'or', 'oss':'os', 'pan':'pa', 'pli':'pi', \
  73                  'pol':'pl', 'pus':'ps', 'por':'pt', 'que':'qu', 'roh':'rm', 'run':'rn', \
  74                  'ron':'ro', 'rus':'ru', 'kin':'rw', 'san':'sa', 'srd':'sc', 'snd':'sd', \
  75                  'sme':'se', 'sag':'sg', 'sin':'si', 'slk':'sk', 'slv':'sl', 'smo':'sm', \
  76                  'sna':'sn', 'som':'so', 'sqi':'sq', 'srp':'sr', 'ssw':'ss', 'sot':'st', \
  77                  'sun':'su', 'swe':'sv', 'swa':'sw', 'tam':'ta', 'tel':'te', 'tgk':'tg', \
  78                  'tha':'th', 'tir':'ti', 'tuk':'tk', 'tgl':'tl', 'tsn':'tn', 'ton':'to', \
  79                  'tur':'tr', 'tso':'ts', 'tat':'tt', 'twi':'tw', 'tah':'ty', 'uig':'ug', \
  80                  'ukr':'uk', 'urd':'ur', 'uzb':'uz', 'ven':'ve', 'vie':'vi', 'vol':'vo', \
  81                  'wln':'wa', 'wol':'wo', 'xho':'xh', 'yid':'yi', 'yor':'yo', 'zha':'za', \
  82                  'zho':'zh', 'zul':'zu'}
  83
  84 ## Convert a python unicode string to a normal string
  85 #
  86 # Convert a python unicode string to a normal string
  87 # UniToStr(u'I am a string') is 'I am a string'
  88 #
  89 # @param Uni:  The python unicode string
  90 #
  91 # @retval:     The formatted normal string
  92 #
  93 def UniToStr(Uni):
  94     return repr(Uni)[2:-1]
  95
  96 ## Convert a unicode string to a Hex list
  97 #
  98 # Convert a unicode string to a Hex list
  99 # UniToHexList('ABC') is ['0x41', '0x00', '0x42', '0x00', '0x43', '0x00']
 100 #
 101 # @param Uni:    The python unicode string
 102 #
 103 # @retval List:  The formatted hex list
 104 #
 105 def UniToHexList(Uni):
 106     List = []
 107     for Item in Uni:
 108         Temp = '%04X' % ord(Item)
 109         List.append('0x' + Temp[2:4])
 110         List.append('0x' + Temp[0:2])
 111     return List
 112
 113 ## Convert special unicode characters
 114 #
 115 # Convert special characters to (c), (r) and (tm).
 116 #
 117 # @param Uni:    The python unicode string
 118 #
 119 # @retval NewUni:  The converted unicode string
 120 #
 121 def ConvertSpecialUnicodes(Uni):
 122     NewUni = Uni
 123     NewUni = NewUni.replace(u'\u00A9', '(c)')
 124     NewUni = NewUni.replace(u'\u00AE', '(r)')
 125     NewUni = NewUni.replace(u'\u2122', '(tm)')
 126     return NewUni
 127
 128 ## GetLanguageCode1766
 129 #
 130 # Check the language code read from .UNI file and convert RFC 4646 codes to RFC 1766 codes
 131 # RFC 1766 language codes supported in compatiblity mode
 132 # RFC 4646 language codes supported in native mode
 133 #
 134 # @param LangName:   Language codes read from .UNI file
 135 #
 136 # @retval LangName:  Valid lanugage code in RFC 1766 format or None
 137 #
 138 def GetLanguageCode1766(LangName, File=None):
 139     length = len(LangName)
 140     if length == 2:
 141         if LangName.isalpha():
 142             for Key in gLANG_CONV_TABLE.keys():
 143                 if gLANG_CONV_TABLE.get(Key) == LangName.lower():
 144                     return Key
 145     elif length == 3:
 146         if LangName.isalpha() and gLANG_CONV_TABLE.get(LangName.lower()):
 147             return LangName
 148         else:
 149             EdkLogger.Error("Unicode File Parser",
 150                              ToolError.FORMAT_INVALID,
 151                              "Invalid RFC 1766 language code : %s" % LangName,
 152                              File)
 153     elif length == 5:
 154         if LangName[0:2].isalpha() and LangName[2] == '-':
 155             for Key in gLANG_CONV_TABLE.keys():
 156                 if gLANG_CONV_TABLE.get(Key) == LangName[0:2].lower():
 157                     return Key
 158     elif length >= 6:
 159         if LangName[0:2].isalpha() and LangName[2] == '-':
 160             for Key in gLANG_CONV_TABLE.keys():
 161                 if gLANG_CONV_TABLE.get(Key) == LangName[0:2].lower():
 162                     return Key
 163         if LangName[0:3].isalpha() and gLANG_CONV_TABLE.get(LangName.lower()) == None and LangName[3] == '-':
 164             for Key in gLANG_CONV_TABLE.keys():
 165                 if Key == LangName[0:3].lower():
 166                     return Key
 167
 168     EdkLogger.Error("Unicode File Parser",
 169                              ToolError.FORMAT_INVALID,
 170                              "Invalid RFC 4646 language code : %s" % LangName,
 171                              File)
 172
 173 ## GetLanguageCode
 174 #
 175 # Check the language code read from .UNI file and convert RFC 1766 codes to RFC 4646 codes if appropriate
 176 # RFC 1766 language codes supported in compatiblity mode
 177 # RFC 4646 language codes supported in native mode
 178 #
 179 # @param LangName:   Language codes read from .UNI file
 180 #
 181 # @retval LangName:  Valid lanugage code in RFC 4646 format or None
 182 #
 183 def GetLanguageCode(LangName, IsCompatibleMode, File):
 184     length = len(LangName)
 185     if IsCompatibleMode:
 186         if length == 3 and LangName.isalpha():
 187             TempLangName = gLANG_CONV_TABLE.get(LangName.lower())
 188             if TempLangName != None:
 189                 return TempLangName
 190             return LangName
 191         else:
 192             EdkLogger.Error("Unicode File Parser",
 193                              ToolError.FORMAT_INVALID,
 194                              "Invalid RFC 1766 language code : %s" % LangName,
 195                              File)
 196     if (LangName[0] == 'X' or LangName[0] == 'x') and LangName[1] == '-':
 197         return LangName
 198     if length == 2:
 199         if LangName.isalpha():
 200             return LangName
 201     elif length == 3:
 202         if LangName.isalpha() and gLANG_CONV_TABLE.get(LangName.lower()) == None:
 203             return LangName
 204     elif length == 5:
 205         if LangName[0:2].isalpha() and LangName[2] == '-':
 206             return LangName
 207     elif length >= 6:
 208         if LangName[0:2].isalpha() and LangName[2] == '-':
 209             return LangName
 210         if LangName[0:3].isalpha() and gLANG_CONV_TABLE.get(LangName.lower()) == None and LangName[3] == '-':
 211             return LangName
 212
 213     EdkLogger.Error("Unicode File Parser",
 214                              ToolError.FORMAT_INVALID,
 215                              "Invalid RFC 4646 language code : %s" % LangName,
 216                              File)
 217
 218 ## FormatUniEntry
 219 #
 220 # Formated the entry in Uni file.
 221 #
 222 # @param StrTokenName    StrTokenName.
 223 # @param TokenValueList  A list need to be processed.
 224 # @param ContainerFile   ContainerFile.
 225 #
 226 # @return formated entry
 227 def FormatUniEntry(StrTokenName, TokenValueList, ContainerFile):
 228     SubContent = ''
 229     PreFormatLength = 40
 230     if len(StrTokenName) > PreFormatLength:
 231         PreFormatLength = len(StrTokenName) + 1
 232     for (Lang, Value) in TokenValueList:
 233         if not Value or Lang == DT.TAB_LANGUAGE_EN_X:
 234             continue
 235         if Lang == '':
 236             Lang = DT.TAB_LANGUAGE_EN_US
 237         if Lang == 'eng':
 238             Lang = DT.TAB_LANGUAGE_EN_US
 239         elif len(Lang.split('-')[0]) == 3:
 240             Lang = GetLanguageCode(Lang.split('-')[0], True, ContainerFile)
 241         else:
 242             Lang = GetLanguageCode(Lang, False, ContainerFile)
 243         ValueList = Value.split('\n')
 244         SubValueContent = ''
 245         for SubValue in ValueList:
 246             if SubValue.strip():
 247                 SubValueContent += \
 248                 ' ' * (PreFormatLength + len('#language en-US    ')) + '\"%s\\n\"' % SubValue.strip() + os.linesep
 249         SubValueContent = SubValueContent[(PreFormatLength + len('#language en-US    ')):SubValueContent.rfind('\\n')] \
 250         + '\"' + os.linesep
 251         SubContent += ' '*PreFormatLength + '#language %-5s ' % Lang + SubValueContent
 252     if SubContent:
 253         SubContent = StrTokenName + ' '*(PreFormatLength - len(StrTokenName)) + SubContent[PreFormatLength:]
 254     return SubContent
 255
 256
 257 ## StringDefClassObject
 258 #
 259 # A structure for language definition
 260 #
 261 class StringDefClassObject(object):
 262     def __init__(self, Name = None, Value = None, Referenced = False, Token = None, UseOtherLangDef = ''):
 263         self.StringName = ''
 264         self.StringNameByteList = []
 265         self.StringValue = ''
 266         self.StringValueByteList = ''
 267         self.Token = 0
 268         self.Referenced = Referenced
 269         self.UseOtherLangDef = UseOtherLangDef
 270         self.Length = 0
 271
 272         if Name != None:
 273             self.StringName = Name
 274             self.StringNameByteList = UniToHexList(Name)
 275         if Value != None:
 276             self.StringValue = Value
 277             self.StringValueByteList = UniToHexList(self.StringValue)
 278             self.Length = len(self.StringValueByteList)
 279         if Token != None:
 280             self.Token = Token
 281
 282     def __str__(self):
 283         return repr(self.StringName) + ' ' + \
 284                repr(self.Token) + ' ' + \
 285                repr(self.Referenced) + ' ' + \
 286                repr(self.StringValue) + ' ' + \
 287                repr(self.UseOtherLangDef)
 288
 289     def UpdateValue(self, Value = None):
 290         if Value != None:
 291             if self.StringValue:
 292                 self.StringValue = self.StringValue  + os.linesep + Value
 293             else:
 294                 self.StringValue = Value
 295             self.StringValueByteList = UniToHexList(self.StringValue)
 296             self.Length = len(self.StringValueByteList)
 297
 298 ## UniFileClassObject
 299 #
 300 # A structure for .uni file definition
 301 #
 302 class UniFileClassObject(object):
 303     def __init__(self, FileList = None, IsCompatibleMode = False, IncludePathList = None):
 304         self.FileList = FileList
 305         self.File = None
 306         self.IncFileList = FileList
 307         self.UniFileHeader = ''
 308         self.Token = 2
 309         self.LanguageDef = []                   #[ [u'LanguageIdentifier', u'PrintableName'], ... ]
 310         self.OrderedStringList = {}             #{ u'LanguageIdentifier' : [StringDefClassObject]  }
 311         self.OrderedStringDict = {}             #{ u'LanguageIdentifier' : {StringName:(IndexInList)}  }
 312         self.OrderedStringListByToken = {}      #{ u'LanguageIdentifier' : {Token: StringDefClassObject} }
 313         self.IsCompatibleMode = IsCompatibleMode
 314         if not IncludePathList:
 315             self.IncludePathList = []
 316         else:
 317             self.IncludePathList = IncludePathList
 318         if len(self.FileList) > 0:
 319             self.LoadUniFiles(FileList)
 320
 321     #
 322     # Get Language definition
 323     #
 324     def GetLangDef(self, File, Line):
 325         Lang = distutils.util.split_quoted((Line.split(u"//")[0]))
 326         if len(Lang) != 3:
 327             try:
 328                 FileIn = codecs.open(File.Path, mode='rb', encoding='utf_16').read()
 329             except UnicodeError, Xstr:
 330                 FileIn = codecs.open(File.Path, mode='rb', encoding='utf_16_le').read()
 331             except:
 332                 EdkLogger.Error("Unicode File Parser",
 333                                 ToolError.FILE_OPEN_FAILURE,
 334                                 "File read failure: %s" % str(Xstr),
 335                                 ExtraData=File)
 336             LineNo = GetLineNo(FileIn, Line, False)
 337             EdkLogger.Error("Unicode File Parser",
 338                              ToolError.PARSER_ERROR,
 339                              "Wrong language definition",
 340                              ExtraData="""%s\n\t*Correct format is like '#langdef en-US "English"'""" % Line,
 341                              File = File, Line = LineNo)
 342         else:
 343             LangName = GetLanguageCode(Lang[1], self.IsCompatibleMode, self.File)
 344             LangPrintName = Lang[2]
 345
 346         IsLangInDef = False
 347         for Item in self.LanguageDef:
 348             if Item[0] == LangName:
 349                 IsLangInDef = True
 350                 break
 351
 352         if not IsLangInDef:
 353             self.LanguageDef.append([LangName, LangPrintName])
 354
 355         #
 356         # Add language string
 357         #
 358         self.AddStringToList(u'$LANGUAGE_NAME', LangName, LangName, 0, True, Index=0)
 359         self.AddStringToList(u'$PRINTABLE_LANGUAGE_NAME', LangName, LangPrintName, 1, True, Index=1)
 360
 361         if not IsLangInDef:
 362             #
 363             # The found STRING tokens will be added into new language string list
 364             # so that the unique STRING identifier is reserved for all languages in the package list.
 365             #
 366             FirstLangName = self.LanguageDef[0][0]
 367             if LangName != FirstLangName:
 368                 for Index in range (2, len (self.OrderedStringList[FirstLangName])):
 369                     Item = self.OrderedStringList[FirstLangName][Index]
 370                     if Item.UseOtherLangDef != '':
 371                         OtherLang = Item.UseOtherLangDef
 372                     else:
 373                         OtherLang = FirstLangName
 374                     self.OrderedStringList[LangName].append (StringDefClassObject(Item.StringName,
 375                                                                                   '',
 376                                                                                   Item.Referenced,
 377                                                                                   Item.Token,
 378                                                                                   OtherLang))
 379                     self.OrderedStringDict[LangName][Item.StringName] = len(self.OrderedStringList[LangName]) - 1
 380         return True
 381
 382     #
 383     # Get String name and value
 384     #
 385     def GetStringObject(self, Item):
 386         Language = ''
 387         Value = ''
 388
 389         Name = Item.split()[1]
 390         # Check the string name is the upper character
 391         if Name != '':
 392             MatchString = re.match('[A-Z0-9_]+', Name, re.UNICODE)
 393             if MatchString == None or MatchString.end(0) != len(Name):
 394                 EdkLogger.Error("Unicode File Parser",
 395                              ToolError.FORMAT_INVALID,
 396                              'The string token name %s in UNI file %s must be upper case character.' %(Name, self.File))
 397         LanguageList = Item.split(u'#language ')
 398         for IndexI in range(len(LanguageList)):
 399             if IndexI == 0:
 400                 continue
 401             else:
 402                 Language = LanguageList[IndexI].split()[0]
 403                 #.replace(u'\r\n', u'')
 404                 Value = \
 405                 LanguageList[IndexI][LanguageList[IndexI].find(u'\"') + len(u'\"') : LanguageList[IndexI].rfind(u'\"')]
 406                 Language = GetLanguageCode(Language, self.IsCompatibleMode, self.File)
 407                 self.AddStringToList(Name, Language, Value)
 408
 409     #
 410     # Get include file list and load them
 411     #
 412     def GetIncludeFile(self, Item, Dir = None):
 413         if Dir:
 414             pass
 415         FileName = Item[Item.find(u'!include ') + len(u'!include ') :Item.find(u' ', len(u'!include '))][1:-1]
 416         self.LoadUniFile(FileName)
 417
 418     #
 419     # Pre-process before parse .uni file
 420     #
 421     def PreProcess(self, File, IsIncludeFile=False):
 422         if not os.path.exists(File.Path) or not os.path.isfile(File.Path):
 423             EdkLogger.Error("Unicode File Parser",
 424                              ToolError.FILE_NOT_FOUND,
 425                              ExtraData=File.Path)
 426
 427         try:
 428             FileIn = codecs.open(File.Path, mode='rb', encoding='utf_16').readlines()
 429         except UnicodeError:
 430             FileIn = codecs.open(File.Path, mode='rb', encoding='utf_16_le').readlines()
 431         except:
 432             EdkLogger.Error("Unicode File Parser", ToolError.FILE_OPEN_FAILURE, ExtraData=File.Path)
 433
 434
 435         #
 436         # get the file header
 437         #
 438         Lines = []
 439         HeaderStart = False
 440         HeaderEnd = False
 441         if not self.UniFileHeader:
 442             FirstGenHeader = True
 443         else:
 444             FirstGenHeader = False
 445         for Line in FileIn:
 446             Line = Line.strip()
 447             if Line == u'':
 448                 continue
 449             if Line.startswith(DT.TAB_COMMENT_EDK1_SPLIT) and (Line.find(DT.TAB_HEADER_COMMENT) > -1) \
 450                 and not HeaderEnd and not HeaderStart:
 451                 HeaderStart = True
 452             if not Line.startswith(DT.TAB_COMMENT_EDK1_SPLIT) and HeaderStart and not HeaderEnd:
 453                 HeaderEnd = True
 454             if Line.startswith(DT.TAB_COMMENT_EDK1_SPLIT) and HeaderStart and not HeaderEnd and FirstGenHeader:
 455                 self.UniFileHeader += Line + os.linesep
 456                 continue
 457
 458         #
 459         # Use unique identifier
 460         #
 461         FindFlag = -1
 462         LineCount = 0
 463         MultiLineFeedExits = False
 464         #
 465         # 0: initial value
 466         # 1: signle String entry exist
 467         # 2: line feed exist under the some signle String entry
 468         #
 469         StringEntryExistsFlag = 0
 470         for Line in FileIn:
 471             Line = FileIn[LineCount]
 472             LineCount += 1
 473             Line = Line.strip()
 474             #
 475             # Ignore comment line and empty line
 476             #
 477             if Line == u'' or Line.startswith(u'//'):
 478                 #
 479                 # Change the single line String entry flag status
 480                 #
 481                 if StringEntryExistsFlag == 1:
 482                     StringEntryExistsFlag = 2
 483                 #
 484                 # If the '#string' line and the '#language' line are not in the same line,
 485                 # there should be only one line feed character betwwen them
 486                 #
 487                 if MultiLineFeedExits:
 488                     EdkLogger.Error("Unicode File Parser", ToolError.FORMAT_INVALID, ExtraData=File.Path)
 489                 continue
 490
 491             MultiLineFeedExits = False
 492             #
 493             # Process comment embeded in string define lines
 494             #
 495             FindFlag = Line.find(u'//')
 496             if FindFlag != -1 and Line.find(u'//') < Line.find(u'"'):
 497                 Line = Line.replace(Line[FindFlag:], u' ')
 498                 if FileIn[LineCount].strip().startswith('#language'):
 499                     Line = Line + FileIn[LineCount]
 500                     FileIn[LineCount-1] = Line
 501                     FileIn[LineCount] = os.linesep
 502                     LineCount -= 1
 503                     for Index in xrange (LineCount + 1, len (FileIn) - 1):
 504                         if (Index == len(FileIn) -1):
 505                             FileIn[Index] = os.linesep
 506                         else:
 507                             FileIn[Index] = FileIn[Index + 1]
 508                     continue
 509             CommIndex = GetCharIndexOutStr(u'/', Line)
 510             if CommIndex > -1:
 511                 if (len(Line) - 1) > CommIndex:
 512                     if Line[CommIndex+1] == u'/':
 513                         Line = Line[:CommIndex].strip()
 514                     else:
 515                         EdkLogger.Error("Unicode File Parser", ToolError.FORMAT_INVALID, ExtraData=File.Path)
 516                 else:
 517                     EdkLogger.Error("Unicode File Parser", ToolError.FORMAT_INVALID, ExtraData=File.Path)
 518
 519             Line = Line.replace(UNICODE_WIDE_CHAR, WIDE_CHAR)
 520             Line = Line.replace(UNICODE_NARROW_CHAR, NARROW_CHAR)
 521             Line = Line.replace(UNICODE_NON_BREAKING_CHAR, NON_BREAKING_CHAR)
 522
 523             Line = Line.replace(u'\\\\', u'\u0006')
 524             Line = Line.replace(u'\\r\\n', CR + LF)
 525             Line = Line.replace(u'\\n', CR + LF)
 526             Line = Line.replace(u'\\r', CR)
 527             Line = Line.replace(u'\\t', u'\t')
 528             Line = Line.replace(u'''\"''', u'''"''')
 529             Line = Line.replace(u'\t', u' ')
 530             Line = Line.replace(u'\u0006', u'\\')
 531
 532             IncList = gINCLUDE_PATTERN.findall(Line)
 533             if len(IncList) == 1:
 534                 for Dir in [File.Dir] + self.IncludePathList:
 535                     IncFile = PathClass(str(IncList[0]), Dir)
 536                     self.IncFileList.append(IncFile)
 537                     if os.path.isfile(IncFile.Path):
 538                         Lines.extend(self.PreProcess(IncFile, True))
 539                         break
 540                 else:
 541                     EdkLogger.Error("Unicode File Parser",
 542                                     ToolError.FILE_NOT_FOUND,
 543                                     Message="Cannot find include file",
 544                                     ExtraData=str(IncList[0]))
 545                 continue
 546
 547             #
 548             # Between Name entry and Language entry can not contain line feed
 549             #
 550             if Line.startswith(u'#string') and Line.find(u'#language') == -1:
 551                 MultiLineFeedExits = True
 552
 553             if Line.startswith(u'#string') and Line.find(u'#language') > 0 and Line.find(u'"') < 0:
 554                 MultiLineFeedExits = True
 555
 556             #
 557             # Between Language entry and String entry can not contain line feed
 558             #
 559             if Line.startswith(u'#language') and len(Line.split()) == 2:
 560                 MultiLineFeedExits = True
 561
 562             #
 563             # Between two String entry, can not contain line feed
 564             #
 565             if Line.startswith(u'"'):
 566                 if StringEntryExistsFlag == 2:
 567                     EdkLogger.Error("Unicode File Parser", ToolError.FORMAT_INVALID,
 568                                     Message=ST.ERR_UNIPARSE_LINEFEED_UP_EXIST % Line, ExtraData=File.Path)
 569
 570                 StringEntryExistsFlag = 1
 571                 if not Line.endswith('"'):
 572                     EdkLogger.Error("Unicode File Parser", ToolError.FORMAT_INVALID, ExtraData=File.Path)
 573             elif Line.startswith(u'#language'):
 574                 if StringEntryExistsFlag == 2:
 575                     EdkLogger.Error("Unicode File Parser", ToolError.FORMAT_INVALID,
 576                                     Message=ST.ERR_UNIPARSE_LINEFEED_UP_EXIST % Line, ExtraData=File.Path)
 577                 StringEntryExistsFlag = 0
 578             else:
 579                 StringEntryExistsFlag = 0
 580
 581             Lines.append(Line)
 582
 583         #
 584         # Convert string def format as below
 585         #
 586         #     #string MY_STRING_1
 587         #     #language eng
 588         #     "My first English string line 1"
 589         #     "My first English string line 2"
 590         #     #string MY_STRING_1
 591         #     #language spa
 592         #     "Mi segunda secuencia 1"
 593         #     "Mi segunda secuencia 2"
 594         #
 595
 596         if not IsIncludeFile and not Lines:
 597             EdkLogger.Error("Unicode File Parser", ToolError.FORMAT_INVALID, \
 598                 Message=ST.ERR_UNIPARSE_NO_SECTION_EXIST, \
 599                 ExtraData=File.Path)
 600
 601         NewLines = []
 602         StrName = u''
 603         ExistStrNameList = []
 604         for Line in Lines:
 605             if StrName and not StrName.split()[1].startswith(DT.TAB_STR_TOKENCNAME + DT.TAB_UNDERLINE_SPLIT):
 606                 EdkLogger.Error("Unicode File Parser", ToolError.FORMAT_INVALID, \
 607                                 Message=ST.ERR_UNIPARSE_STRNAME_FORMAT_ERROR % StrName.split()[1], \
 608                                 ExtraData=File.Path)
 609
 610             if StrName and len(StrName.split()[1].split(DT.TAB_UNDERLINE_SPLIT)) == 4:
 611                 StringTokenList = StrName.split()[1].split(DT.TAB_UNDERLINE_SPLIT)
 612                 if (StringTokenList[3].upper() in [DT.TAB_STR_TOKENPROMPT, DT.TAB_STR_TOKENHELP] and \
 613                     StringTokenList[3] not in [DT.TAB_STR_TOKENPROMPT, DT.TAB_STR_TOKENHELP]) or \
 614                     (StringTokenList[2].upper() == DT.TAB_STR_TOKENERR and StringTokenList[2] != DT.TAB_STR_TOKENERR):
 615                     EdkLogger.Error("Unicode File Parser", ToolError.FORMAT_INVALID, \
 616                                 Message=ST.ERR_UNIPARSE_STRTOKEN_FORMAT_ERROR % StrName.split()[1], \
 617                                 ExtraData=File.Path)
 618
 619             if Line.count(u'#language') > 1:
 620                 EdkLogger.Error("Unicode File Parser", ToolError.FORMAT_INVALID, \
 621                                 Message=ST.ERR_UNIPARSE_SEP_LANGENTRY_LINE % Line, \
 622                                 ExtraData=File.Path)
 623
 624             if Line.startswith(u'//'):
 625                 continue
 626             elif Line.startswith(u'#langdef'):
 627                 if len(Line.split()) == 2:
 628                     NewLines.append(Line)
 629                     continue
 630                 elif len(Line.split()) > 2 and Line.find(u'"') > 0:
 631                     NewLines.append(Line[:Line.find(u'"')].strip())
 632                     NewLines.append(Line[Line.find(u'"'):])
 633                 else:
 634                     EdkLogger.Error("Unicode File Parser", ToolError.FORMAT_INVALID, ExtraData=File.Path)
 635             elif Line.startswith(u'#string'):
 636                 if len(Line.split()) == 2:
 637                     StrName = Line
 638                     if StrName:
 639                         if StrName.split()[1] not in ExistStrNameList:
 640                             ExistStrNameList.append(StrName.split()[1].strip())
 641                         elif StrName.split()[1] in [DT.TAB_INF_ABSTRACT, DT.TAB_INF_DESCRIPTION, \
 642                                                     DT.TAB_INF_BINARY_ABSTRACT, DT.TAB_INF_BINARY_DESCRIPTION, \
 643                                                     DT.TAB_DEC_PACKAGE_ABSTRACT, DT.TAB_DEC_PACKAGE_DESCRIPTION, \
 644                                                     DT.TAB_DEC_BINARY_ABSTRACT, DT.TAB_DEC_BINARY_DESCRIPTION]:
 645                             EdkLogger.Error("Unicode File Parser", ToolError.FORMAT_INVALID, \
 646                                             Message=ST.ERR_UNIPARSE_MULTI_ENTRY_EXIST % StrName.split()[1], \
 647                                             ExtraData=File.Path)
 648                     continue
 649                 elif len(Line.split()) == 4 and Line.find(u'#language') > 0:
 650                     if Line[Line.find(u'#language')-1] != ' ' or \
 651                        Line[Line.find(u'#language')+len(u'#language')] != u' ':
 652                         EdkLogger.Error("Unicode File Parser", ToolError.FORMAT_INVALID, ExtraData=File.Path)
 653
 654                     if Line.find(u'"') > 0:
 655                         EdkLogger.Error("Unicode File Parser", ToolError.FORMAT_INVALID, ExtraData=File.Path)
 656
 657                     StrName = Line.split()[0] + u' ' + Line.split()[1]
 658                     if StrName:
 659                         if StrName.split()[1] not in ExistStrNameList:
 660                             ExistStrNameList.append(StrName.split()[1].strip())
 661                         elif StrName.split()[1] in [DT.TAB_INF_ABSTRACT, DT.TAB_INF_DESCRIPTION, \
 662                                                     DT.TAB_INF_BINARY_ABSTRACT, DT.TAB_INF_BINARY_DESCRIPTION, \
 663                                                     DT.TAB_DEC_PACKAGE_ABSTRACT, DT.TAB_DEC_PACKAGE_DESCRIPTION, \
 664                                                     DT.TAB_DEC_BINARY_ABSTRACT, DT.TAB_DEC_BINARY_DESCRIPTION]:
 665                             EdkLogger.Error("Unicode File Parser", ToolError.FORMAT_INVALID, \
 666                                             Message=ST.ERR_UNIPARSE_MULTI_ENTRY_EXIST % StrName.split()[1], \
 667                                             ExtraData=File.Path)
 668                     if IsIncludeFile:
 669                         if StrName not in NewLines:
 670                             NewLines.append((Line[:Line.find(u'#language')]).strip())
 671                     else:
 672                         NewLines.append((Line[:Line.find(u'#language')]).strip())
 673                     NewLines.append((Line[Line.find(u'#language'):]).strip())
 674                 elif len(Line.split()) > 4 and Line.find(u'#language') > 0 and Line.find(u'"') > 0:
 675                     if Line[Line.find(u'#language')-1] != u' ' or \
 676                        Line[Line.find(u'#language')+len(u'#language')] != u' ':
 677                         EdkLogger.Error("Unicode File Parser", ToolError.FORMAT_INVALID, ExtraData=File.Path)
 678
 679                     if Line[Line.find(u'"')-1] != u' ':
 680                         EdkLogger.Error("Unicode File Parser", ToolError.FORMAT_INVALID, ExtraData=File.Path)
 681
 682                     StrName = Line.split()[0] + u' ' + Line.split()[1]
 683                     if StrName:
 684                         if StrName.split()[1] not in ExistStrNameList:
 685                             ExistStrNameList.append(StrName.split()[1].strip())
 686                         elif StrName.split()[1] in [DT.TAB_INF_ABSTRACT, DT.TAB_INF_DESCRIPTION, \
 687                                                     DT.TAB_INF_BINARY_ABSTRACT, DT.TAB_INF_BINARY_DESCRIPTION, \
 688                                                     DT.TAB_DEC_PACKAGE_ABSTRACT, DT.TAB_DEC_PACKAGE_DESCRIPTION, \
 689                                                     DT.TAB_DEC_BINARY_ABSTRACT, DT.TAB_DEC_BINARY_DESCRIPTION]:
 690                             EdkLogger.Error("Unicode File Parser", ToolError.FORMAT_INVALID, \
 691                                             Message=ST.ERR_UNIPARSE_MULTI_ENTRY_EXIST % StrName.split()[1], \
 692                                             ExtraData=File.Path)
 693                     if IsIncludeFile:
 694                         if StrName not in NewLines:
 695                             NewLines.append((Line[:Line.find(u'#language')]).strip())
 696                     else:
 697                         NewLines.append((Line[:Line.find(u'#language')]).strip())
 698                     NewLines.append((Line[Line.find(u'#language'):Line.find(u'"')]).strip())
 699                     NewLines.append((Line[Line.find(u'"'):]).strip())
 700                 else:
 701                     EdkLogger.Error("Unicode File Parser", ToolError.FORMAT_INVALID, ExtraData=File.Path)
 702             elif Line.startswith(u'#language'):
 703                 if len(Line.split()) == 2:
 704                     if IsIncludeFile:
 705                         if StrName not in NewLines:
 706                             NewLines.append(StrName)
 707                     else:
 708                         NewLines.append(StrName)
 709                     NewLines.append(Line)
 710                 elif len(Line.split()) > 2 and Line.find(u'"') > 0:
 711                     if IsIncludeFile:
 712                         if StrName not in NewLines:
 713                             NewLines.append(StrName)
 714                     else:
 715                         NewLines.append(StrName)
 716                     NewLines.append((Line[:Line.find(u'"')]).strip())
 717                     NewLines.append((Line[Line.find(u'"'):]).strip())
 718                 else:
 719                     EdkLogger.Error("Unicode File Parser", ToolError.FORMAT_INVALID, ExtraData=File.Path)
 720             elif Line.startswith(u'"'):
 721                 if u'#string' in Line  or u'#language' in Line:
 722                     EdkLogger.Error("Unicode File Parser", ToolError.FORMAT_INVALID, ExtraData=File.Path)
 723                 NewLines.append(Line)
 724             else:
 725                 print Line
 726                 EdkLogger.Error("Unicode File Parser", ToolError.FORMAT_INVALID, ExtraData=File.Path)
 727
 728         if StrName and not StrName.split()[1].startswith(u'STR_'):
 729             EdkLogger.Error("Unicode File Parser", ToolError.FORMAT_INVALID, \
 730                                 Message=ST.ERR_UNIPARSE_STRNAME_FORMAT_ERROR % StrName.split()[1], \
 731                                 ExtraData=File.Path)
 732
 733         if StrName and not NewLines:
 734             EdkLogger.Error("Unicode File Parser", ToolError.FORMAT_INVALID, \
 735                             Message=ST.ERR_UNI_MISS_LANGENTRY % StrName, \
 736                             ExtraData=File.Path)
 737
 738         #
 739         # Check Abstract, Description, BinaryAbstract and BinaryDescription order,
 740         # should be Abstract, Description, BinaryAbstract, BinaryDesctiption
 741         AbstractPosition = -1
 742         DescriptionPosition = -1
 743         BinaryAbstractPosition = -1
 744         BinaryDescriptionPosition = -1
 745         for StrName in ExistStrNameList:
 746             if DT.TAB_HEADER_ABSTRACT.upper() in StrName:
 747                 if 'BINARY' in StrName:
 748                     BinaryAbstractPosition = ExistStrNameList.index(StrName)
 749                 else:
 750                     AbstractPosition = ExistStrNameList.index(StrName)
 751             if DT.TAB_HEADER_DESCRIPTION.upper() in StrName:
 752                 if 'BINARY' in StrName:
 753                     BinaryDescriptionPosition = ExistStrNameList.index(StrName)
 754                 else:
 755                     DescriptionPosition = ExistStrNameList.index(StrName)
 756
 757         OrderList = sorted([AbstractPosition, DescriptionPosition])
 758         BinaryOrderList = sorted([BinaryAbstractPosition, BinaryDescriptionPosition])
 759         Min = OrderList[0]
 760         Max = OrderList[1]
 761         BinaryMin = BinaryOrderList[0]
 762         BinaryMax = BinaryOrderList[1]
 763         if BinaryDescriptionPosition > -1:
 764             if not(BinaryDescriptionPosition == BinaryMax and BinaryAbstractPosition == BinaryMin and \
 765                    BinaryMax > Max):
 766                 EdkLogger.Error("Unicode File Parser", ToolError.FORMAT_INVALID, \
 767                                 Message=ST.ERR_UNIPARSE_ENTRY_ORDER_WRONG, \
 768                                 ExtraData=File.Path)
 769         elif BinaryAbstractPosition > -1:
 770             if not(BinaryAbstractPosition > Max):
 771                 EdkLogger.Error("Unicode File Parser", ToolError.FORMAT_INVALID, \
 772                                 Message=ST.ERR_UNIPARSE_ENTRY_ORDER_WRONG, \
 773                                 ExtraData=File.Path)
 774
 775         if  DescriptionPosition > -1:
 776             if not(DescriptionPosition == Max and AbstractPosition == Min and \
 777                    DescriptionPosition > AbstractPosition):
 778                 EdkLogger.Error("Unicode File Parser", ToolError.FORMAT_INVALID, \
 779                                 Message=ST.ERR_UNIPARSE_ENTRY_ORDER_WRONG, \
 780                                 ExtraData=File.Path)
 781
 782         if not self.UniFileHeader:
 783             EdkLogger.Error("Unicode File Parser", ToolError.FORMAT_INVALID,
 784                             Message = ST.ERR_NO_SOURCE_HEADER,
 785                             ExtraData=File.Path)
 786
 787         return NewLines
 788
 789     #
 790     # Load a .uni file
 791     #
 792     def LoadUniFile(self, File = None):
 793         if File == None:
 794             EdkLogger.Error("Unicode File Parser",
 795                             ToolError.PARSER_ERROR,
 796                             Message='No unicode file is given',
 797                             ExtraData=File.Path)
 798
 799         self.File = File
 800
 801         #
 802         # Process special char in file
 803         #
 804         Lines = self.PreProcess(File)
 805
 806         #
 807         # Get Unicode Information
 808         #
 809         for IndexI in range(len(Lines)):
 810             Line = Lines[IndexI]
 811             if (IndexI + 1) < len(Lines):
 812                 SecondLine = Lines[IndexI + 1]
 813             if (IndexI + 2) < len(Lines):
 814                 ThirdLine = Lines[IndexI + 2]
 815
 816             #
 817             # Get Language def information
 818             #
 819             if Line.find(u'#langdef ') >= 0:
 820                 self.GetLangDef(File, Line + u' ' + SecondLine)
 821                 continue
 822
 823             Name = ''
 824             Language = ''
 825             Value = ''
 826             CombineToken = False
 827             #
 828             # Get string def information format as below
 829             #
 830             #     #string MY_STRING_1
 831             #     #language eng
 832             #     "My first English string line 1"
 833             #     "My first English string line 2"
 834             #     #string MY_STRING_1
 835             #     #language spa
 836             #     "Mi segunda secuencia 1"
 837             #     "Mi segunda secuencia 2"
 838             #
 839             if Line.find(u'#string ') >= 0 and Line.find(u'#language ') < 0 and \
 840                 SecondLine.find(u'#string ') < 0 and SecondLine.find(u'#language ') >= 0 and \
 841                 ThirdLine.find(u'#string ') < 0 and ThirdLine.find(u'#language ') < 0:
 842                 if Line.find('"') > 0 or SecondLine.find('"') > 0:
 843                     EdkLogger.Error("Unicode File Parser", ToolError.FORMAT_INVALID,
 844                                 Message=ST.ERR_UNIPARSE_DBLQUOTE_UNMATCHED,
 845                                 ExtraData=File.Path)
 846
 847                 Name = Line[Line.find(u'#string ') + len(u'#string ') : ].strip(' ')
 848                 Language = SecondLine[SecondLine.find(u'#language ') + len(u'#language ') : ].strip(' ')
 849                 for IndexJ in range(IndexI + 2, len(Lines)):
 850                     if Lines[IndexJ].find(u'#string ') < 0 and Lines[IndexJ].find(u'#language ') < 0 and \
 851                     Lines[IndexJ].strip().startswith(u'"') and Lines[IndexJ].strip().endswith(u'"'):
 852                         if Lines[IndexJ][-2] == ' ':
 853                             CombineToken = True
 854                         if CombineToken:
 855                             if Lines[IndexJ].strip()[1:-1].strip():
 856                                 Value = Value + Lines[IndexJ].strip()[1:-1].rstrip() + ' '
 857                             else:
 858                                 Value = Value + Lines[IndexJ].strip()[1:-1]
 859                             CombineToken = False
 860                         else:
 861                             Value = Value + Lines[IndexJ].strip()[1:-1] + os.linesep
 862                     else:
 863                         IndexI = IndexJ
 864                         break
 865                 if Value.endswith(os.linesep):
 866                     Value = Value[: Value.rfind(os.linesep)]
 867                 Language = GetLanguageCode(Language, self.IsCompatibleMode, self.File)
 868                 self.AddStringToList(Name, Language, Value)
 869                 continue
 870
 871     #
 872     # Load multiple .uni files
 873     #
 874     def LoadUniFiles(self, FileList):
 875         if len(FileList) > 0:
 876             for File in FileList:
 877                 FilePath = File.Path.strip()
 878                 if FilePath.endswith('.uni') or FilePath.endswith('.UNI') or FilePath.endswith('.Uni'):
 879                     self.LoadUniFile(File)
 880
 881     #
 882     # Add a string to list
 883     #
 884     def AddStringToList(self, Name, Language, Value, Token = 0, Referenced = False, UseOtherLangDef = '', Index = -1):
 885         for LangNameItem in self.LanguageDef:
 886             if Language == LangNameItem[0]:
 887                 break
 888
 889         if Language not in self.OrderedStringList:
 890             self.OrderedStringList[Language] = []
 891             self.OrderedStringDict[Language] = {}
 892
 893         IsAdded = True
 894         if Name in self.OrderedStringDict[Language]:
 895             IsAdded = False
 896             if Value != None:
 897                 ItemIndexInList = self.OrderedStringDict[Language][Name]
 898                 Item = self.OrderedStringList[Language][ItemIndexInList]
 899                 Item.UpdateValue(Value)
 900                 Item.UseOtherLangDef = ''
 901
 902         if IsAdded:
 903             Token = len(self.OrderedStringList[Language])
 904             if Index == -1:
 905                 self.OrderedStringList[Language].append(StringDefClassObject(Name,
 906                                                                              Value,
 907                                                                              Referenced,
 908                                                                              Token,
 909                                                                              UseOtherLangDef))
 910                 self.OrderedStringDict[Language][Name] = Token
 911                 for LangName in self.LanguageDef:
 912                     #
 913                     # New STRING token will be added into all language string lists.
 914                     # so that the unique STRING identifier is reserved for all languages in the package list.
 915                     #
 916                     if LangName[0] != Language:
 917                         if UseOtherLangDef != '':
 918                             OtherLangDef = UseOtherLangDef
 919                         else:
 920                             OtherLangDef = Language
 921                         self.OrderedStringList[LangName[0]].append(StringDefClassObject(Name,
 922                                                                                         '',
 923                                                                                         Referenced,
 924                                                                                         Token,
 925                                                                                         OtherLangDef))
 926                         self.OrderedStringDict[LangName[0]][Name] = len(self.OrderedStringList[LangName[0]]) - 1
 927             else:
 928                 self.OrderedStringList[Language].insert(Index, StringDefClassObject(Name,
 929                                                                                     Value,
 930                                                                                     Referenced,
 931                                                                                     Token,
 932                                                                                     UseOtherLangDef))
 933                 self.OrderedStringDict[Language][Name] = Index
 934
 935     #
 936     # Set the string as referenced
 937     #
 938     def SetStringReferenced(self, Name):
 939         #
 940         # String stoken are added in the same order in all language string lists.
 941         # So, only update the status of string stoken in first language string list.
 942         #
 943         Lang = self.LanguageDef[0][0]
 944         if Name in self.OrderedStringDict[Lang]:
 945             ItemIndexInList = self.OrderedStringDict[Lang][Name]
 946             Item = self.OrderedStringList[Lang][ItemIndexInList]
 947             Item.Referenced = True
 948
 949     #
 950     # Search the string in language definition by Name
 951     #
 952     def FindStringValue(self, Name, Lang):
 953         if Name in self.OrderedStringDict[Lang]:
 954             ItemIndexInList = self.OrderedStringDict[Lang][Name]
 955             return self.OrderedStringList[Lang][ItemIndexInList]
 956
 957         return None
 958
 959     #
 960     # Search the string in language definition by Token
 961     #
 962     def FindByToken(self, Token, Lang):
 963         for Item in self.OrderedStringList[Lang]:
 964             if Item.Token == Token:
 965                 return Item
 966
 967         return None
 968
 969     #
 970     # Re-order strings and re-generate tokens
 971     #
 972     def ReToken(self):
 973         if len(self.LanguageDef) == 0:
 974             return None
 975         #
 976         # Retoken all language strings according to the status of string stoken in the first language string.
 977         #
 978         FirstLangName = self.LanguageDef[0][0]
 979
 980         # Convert the OrderedStringList to be OrderedStringListByToken in order to faciliate future search by token
 981         for LangNameItem in self.LanguageDef:
 982             self.OrderedStringListByToken[LangNameItem[0]] = {}
 983
 984         #
 985         # Use small token for all referred string stoken.
 986         #
 987         RefToken = 0
 988         for Index in range (0, len (self.OrderedStringList[FirstLangName])):
 989             FirstLangItem = self.OrderedStringList[FirstLangName][Index]
 990             if FirstLangItem.Referenced == True:
 991                 for LangNameItem in self.LanguageDef:
 992                     LangName = LangNameItem[0]
 993                     OtherLangItem = self.OrderedStringList[LangName][Index]
 994                     OtherLangItem.Referenced = True
 995                     OtherLangItem.Token = RefToken
 996                     self.OrderedStringListByToken[LangName][OtherLangItem.Token] = OtherLangItem
 997                 RefToken = RefToken + 1
 998
 999         #
1000         # Use big token for all unreferred string stoken.
1001         #
1002         UnRefToken = 0
1003         for Index in range (0, len (self.OrderedStringList[FirstLangName])):
1004             FirstLangItem = self.OrderedStringList[FirstLangName][Index]
1005             if FirstLangItem.Referenced == False:
1006                 for LangNameItem in self.LanguageDef:
1007                     LangName = LangNameItem[0]
1008                     OtherLangItem = self.OrderedStringList[LangName][Index]
1009                     OtherLangItem.Token = RefToken + UnRefToken
1010                     self.OrderedStringListByToken[LangName][OtherLangItem.Token] = OtherLangItem
1011                 UnRefToken = UnRefToken + 1
1012
1013     #
1014     # Show the instance itself
1015     #
1016     def ShowMe(self):
1017         print self.LanguageDef
1018         #print self.OrderedStringList
1019         for Item in self.OrderedStringList:
1020             print Item
1021             for Member in self.OrderedStringList[Item]:
1022                 print str(Member)
1023
1024     #
1025     # Read content from '!include' UNI file
1026     #
1027     def ReadIncludeUNIfile(self, FilaPath):
1028         if self.File:
1029             pass
1030
1031         if not os.path.exists(FilaPath) or not os.path.isfile(FilaPath):
1032             EdkLogger.Error("Unicode File Parser",
1033                              ToolError.FILE_NOT_FOUND,
1034                              ExtraData=FilaPath)
1035         try:
1036             FileIn = codecs.open(FilaPath, mode='rb', encoding='utf_16').readlines()
1037         except UnicodeError:
1038             FileIn = codecs.open(FilaPath, mode='rb', encoding='utf_16_le').readlines()
1039         except:
1040             EdkLogger.Error("Unicode File Parser", ToolError.FILE_OPEN_FAILURE, ExtraData=FilaPath)
1041         return FileIn
1042