X-Git-Url: https://git.proxmox.com/?p=mirror_edk2.git;a=blobdiff_plain;f=BaseTools%2FSource%2FPython%2FAutoGen%2FUniClassObject.py;h=5b674dfbc2f0a04cf6ee3728b072b04efa97f5d0;hp=de2f93b8eabfdf32dd2900ae169a3932c9a2cd74;hb=018f7b827fa4def3476f76cdf1d6400d4a8e6ebc;hpb=52302d4dee589a5df43a464420c9fe68ba83937d diff --git a/BaseTools/Source/Python/AutoGen/UniClassObject.py b/BaseTools/Source/Python/AutoGen/UniClassObject.py index de2f93b8ea..5b674dfbc2 100644 --- a/BaseTools/Source/Python/AutoGen/UniClassObject.py +++ b/BaseTools/Source/Python/AutoGen/UniClassObject.py @@ -1,5 +1,11 @@ -# Copyright (c) 2007, Intel Corporation -# All rights reserved. This program and the accompanying materials +## @file +# This file is used to collect all defined strings in multiple uni files +# +# +# Copyright (c) 2014 Hewlett-Packard Development Company, L.P.
+# +# Copyright (c) 2007 - 2018, Intel Corporation. All rights reserved.
+# This program and the accompanying materials # are licensed and made available under the terms and conditions of the BSD License # which accompanies this distribution. The full text of the license may be found at # http://opensource.org/licenses/bsd-license.php @@ -7,19 +13,18 @@ # THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS, # WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED. -# -#This file is used to collect all defined strings in multiple uni files -# - ## # Import Modules # -import os, codecs, re +import Common.LongFilePathOs as os, codecs, re +import distutils.util import Common.EdkLogger as EdkLogger +import StringIO from Common.BuildToolError import * from Common.String import GetLineNo from Common.Misc import PathClass - +from Common.LongFilePathSupport import LongFilePath +from Common.GlobalData import * ## # Static definitions # @@ -36,7 +41,7 @@ CR = u'\u000D' LF = u'\u000A' NULL = u'\u0000' TAB = u'\t' -BACK_SPLASH = u'\\' +BACK_SLASH_PLACEHOLDER = u'\u0006' gIncludePattern = re.compile("^#include +[\"<]+([^\"< >]+)[>\"]+$", re.MULTILINE | re.UNICODE) @@ -120,11 +125,13 @@ def GetLanguageCode(LangName, IsCompatibleMode, File): if length == 3 and LangName.isalpha(): TempLangName = LangConvTable.get(LangName.lower()) if TempLangName != None: - return TempLangName + return TempLangName return LangName else: EdkLogger.error("Unicode File Parser", FORMAT_INVALID, "Invalid ISO 639-2 language code : %s" % LangName, File) + if (LangName[0] == 'X' or LangName[0] == 'x') and LangName[1] == '-': + return LangName if length == 2: if LangName.isalpha(): return LangName @@ -142,6 +149,37 @@ def GetLanguageCode(LangName, IsCompatibleMode, File): EdkLogger.error("Unicode File Parser", FORMAT_INVALID, "Invalid RFC 4646 language code : %s" % LangName, File) +## Ucs2Codec +# +# This is only a partial codec implementation. It only supports +# encoding, and is primarily used to check that all the characters are +# valid for UCS-2. +# +class Ucs2Codec(codecs.Codec): + def __init__(self): + self.__utf16 = codecs.lookup('utf-16') + + def encode(self, input, errors='strict'): + for Char in input: + CodePoint = ord(Char) + if CodePoint >= 0xd800 and CodePoint <= 0xdfff: + raise ValueError("Code Point is in range reserved for " + + "UTF-16 surrogate pairs") + elif CodePoint > 0xffff: + raise ValueError("Code Point too large to encode in UCS-2") + return self.__utf16.encode(input) + +TheUcs2Codec = Ucs2Codec() +def Ucs2Search(name): + if name == 'ucs-2': + return codecs.CodecInfo( + name=name, + encode=TheUcs2Codec.encode, + decode=TheUcs2Codec.decode) + else: + return None +codecs.register(Ucs2Search) + ## StringDefClassObject # # A structure for language definition @@ -174,18 +212,26 @@ class StringDefClassObject(object): repr(self.StringValue) + ' ' + \ repr(self.UseOtherLangDef) + def UpdateValue(self, Value = None): + if Value != None: + self.StringValue = Value + u'\x00' # Add a NULL at string tail + self.StringValueByteList = UniToHexList(self.StringValue) + self.Length = len(self.StringValueByteList) + ## UniFileClassObject # # A structure for .uni file definition # class UniFileClassObject(object): - def __init__(self, FileList = [], IsCompatibleMode = False): + def __init__(self, FileList = [], IsCompatibleMode = False, IncludePathList = []): self.FileList = FileList self.Token = 2 self.LanguageDef = [] #[ [u'LanguageIdentifier', u'PrintableName'], ... ] self.OrderedStringList = {} #{ u'LanguageIdentifier' : [StringDefClassObject] } + self.OrderedStringDict = {} #{ u'LanguageIdentifier' : {StringName:(IndexInList)} } + self.OrderedStringListByToken = {} #{ u'LanguageIdentifier' : {Token: StringDefClassObject} } self.IsCompatibleMode = IsCompatibleMode - + self.IncludePathList = IncludePathList if len(self.FileList) > 0: self.LoadUniFiles(FileList) @@ -193,20 +239,20 @@ class UniFileClassObject(object): # Get Language definition # def GetLangDef(self, File, Line): - Lang = Line.split() + Lang = distutils.util.split_quoted((Line.split(u"//")[0])) if len(Lang) != 3: try: - FileIn = codecs.open(File, mode='rb', encoding='utf-16').read() + FileIn = self.OpenUniFile(LongFilePath(File.Path)) except UnicodeError, X: EdkLogger.error("build", FILE_READ_FAILURE, "File read failure: %s" % str(X), ExtraData=File); except: EdkLogger.error("build", FILE_OPEN_FAILURE, ExtraData=File); LineNo = GetLineNo(FileIn, Line, False) EdkLogger.error("Unicode File Parser", PARSER_ERROR, "Wrong language definition", - ExtraData="""%s\n\t*Correct format is like '#langdef eng "English"'""" % Line, File = File, Line = LineNo) + ExtraData="""%s\n\t*Correct format is like '#langdef en-US "English"'""" % Line, File=File, Line=LineNo) else: LangName = GetLanguageCode(Lang[1], self.IsCompatibleMode, self.File) - LangPrintName = Lang[2][1:-1] + LangPrintName = Lang[2] IsLangInDef = False for Item in self.LanguageDef: @@ -223,22 +269,91 @@ class UniFileClassObject(object): self.AddStringToList(u'$LANGUAGE_NAME', LangName, LangName, 0, True, Index=0) self.AddStringToList(u'$PRINTABLE_LANGUAGE_NAME', LangName, LangPrintName, 1, True, Index=1) + if not IsLangInDef: + # + # The found STRING tokens will be added into new language string list + # so that the unique STRING identifier is reserved for all languages in the package list. + # + FirstLangName = self.LanguageDef[0][0] + if LangName != FirstLangName: + for Index in range (2, len (self.OrderedStringList[FirstLangName])): + Item = self.OrderedStringList[FirstLangName][Index] + if Item.UseOtherLangDef != '': + OtherLang = Item.UseOtherLangDef + else: + OtherLang = FirstLangName + self.OrderedStringList[LangName].append (StringDefClassObject(Item.StringName, '', Item.Referenced, Item.Token, OtherLang)) + self.OrderedStringDict[LangName][Item.StringName] = len(self.OrderedStringList[LangName]) - 1 return True + def OpenUniFile(self, FileName): + # + # Read file + # + try: + UniFile = open(FileName, mode='rb') + FileIn = UniFile.read() + UniFile.close() + except: + EdkLogger.Error("build", FILE_OPEN_FAILURE, ExtraData=File) + + # + # Detect Byte Order Mark at beginning of file. Default to UTF-8 + # + Encoding = 'utf-8' + if (FileIn.startswith(codecs.BOM_UTF16_BE) or + FileIn.startswith(codecs.BOM_UTF16_LE)): + Encoding = 'utf-16' + + self.VerifyUcs2Data(FileIn, FileName, Encoding) + + UniFile = StringIO.StringIO(FileIn) + Info = codecs.lookup(Encoding) + (Reader, Writer) = (Info.streamreader, Info.streamwriter) + return codecs.StreamReaderWriter(UniFile, Reader, Writer) + + def VerifyUcs2Data(self, FileIn, FileName, Encoding): + Ucs2Info = codecs.lookup('ucs-2') + # + # Convert to unicode + # + try: + FileDecoded = codecs.decode(FileIn, Encoding) + Ucs2Info.encode(FileDecoded) + except: + UniFile = StringIO.StringIO(FileIn) + Info = codecs.lookup(Encoding) + (Reader, Writer) = (Info.streamreader, Info.streamwriter) + File = codecs.StreamReaderWriter(UniFile, Reader, Writer) + LineNumber = 0 + ErrMsg = lambda Encoding, LineNumber: \ + '%s contains invalid %s characters on line %d.' % \ + (FileName, Encoding, LineNumber) + while True: + LineNumber = LineNumber + 1 + try: + Line = File.readline() + if Line == '': + EdkLogger.error('Unicode File Parser', PARSER_ERROR, + ErrMsg(Encoding, LineNumber)) + Ucs2Info.encode(Line) + except: + EdkLogger.error('Unicode File Parser', PARSER_ERROR, + ErrMsg('UCS-2', LineNumber)) + # # Get String name and value # def GetStringObject(self, Item): - Name = '' Language = '' Value = '' Name = Item.split()[1] - # Check the string name is the upper character - if not self.IsCompatibleMode and Name != '': - MatchString = re.match('[A-Z0-9_]+', Name, re.UNICODE) + # Check the string name + if Name != '': + MatchString = re.match('^[a-zA-Z][a-zA-Z0-9_]*$', Name, re.UNICODE) if MatchString == None or MatchString.end(0) != len(Name): - EdkLogger.error('Unicode File Parser', FORMAT_INVALID, 'The string token name %s defined in UNI file %s contains the invalid lower case character.' %(Name, self.File)) + EdkLogger.error('Unicode File Parser', FORMAT_INVALID, 'The string token name %s defined in UNI file %s contains the invalid character.' % (Name, self.File)) LanguageList = Item.split(u'#language ') for IndexI in range(len(LanguageList)): if IndexI == 0: @@ -256,6 +371,20 @@ class UniFileClassObject(object): FileName = Item[Item.find(u'#include ') + len(u'#include ') :Item.find(u' ', len(u'#include '))][1:-1] self.LoadUniFile(FileName) + def StripComments(self, Line): + Comment = u'//' + CommentPos = Line.find(Comment) + while CommentPos >= 0: + # if there are non matched quotes before the comment header + # then we are in the middle of a string + # but we need to ignore the escaped quotes and backslashes. + if ((Line.count(u'"', 0, CommentPos) - Line.count(u'\\"', 0, CommentPos)) & 1) == 1: + CommentPos = Line.find (Comment, CommentPos + 1) + else: + return Line[:CommentPos].strip() + return Line.strip() + + # # Pre-process before parse .uni file # @@ -263,9 +392,8 @@ class UniFileClassObject(object): if not os.path.exists(File.Path) or not os.path.isfile(File.Path): EdkLogger.error("Unicode File Parser", FILE_NOT_FOUND, ExtraData=File.Path) - Dir = File.Dir try: - FileIn = codecs.open(File.Path, mode='rb', encoding='utf-16').readlines() + FileIn = self.OpenUniFile(LongFilePath(File.Path)) except UnicodeError, X: EdkLogger.error("build", FILE_READ_FAILURE, "File read failure: %s" % str(X), ExtraData=File.Path); except: @@ -277,11 +405,16 @@ class UniFileClassObject(object): # for Line in FileIn: Line = Line.strip() + Line = Line.replace(u'\\\\', BACK_SLASH_PLACEHOLDER) + Line = self.StripComments(Line) + # - # Ignore comment line and empty line + # Ignore empty line # - if Line == u'' or Line.startswith(u'//'): - continue + if len(Line) == 0: + continue + + Line = Line.replace(u'/langdef', u'#langdef') Line = Line.replace(u'/string', u'#string') Line = Line.replace(u'/language', u'#language') @@ -291,22 +424,38 @@ class UniFileClassObject(object): Line = Line.replace(UNICODE_NARROW_CHAR, NARROW_CHAR) Line = Line.replace(UNICODE_NON_BREAKING_CHAR, NON_BREAKING_CHAR) - Line = Line.replace(u'\\\\', u'\u0006') Line = Line.replace(u'\\r\\n', CR + LF) Line = Line.replace(u'\\n', CR + LF) Line = Line.replace(u'\\r', CR) - Line = Line.replace(u'\\t', u'\t') - Line = Line.replace(u'''\"''', u'''"''') + Line = Line.replace(u'\\t', u' ') Line = Line.replace(u'\t', u' ') - Line = Line.replace(u'\u0006', u'\\') - -# if Line.find(u'\\x'): -# hex = Line[Line.find(u'\\x') + 2 : Line.find(u'\\x') + 6] -# hex = "u'\\u" + hex + "'" + Line = Line.replace(u'\\"', u'"') + Line = Line.replace(u"\\'", u"'") + Line = Line.replace(BACK_SLASH_PLACEHOLDER, u'\\') + + StartPos = Line.find(u'\\x') + while (StartPos != -1): + EndPos = Line.find(u'\\', StartPos + 1, StartPos + 7) + if EndPos != -1 and EndPos - StartPos == 6 : + if g4HexChar.match(Line[StartPos + 2 : EndPos], re.UNICODE): + EndStr = Line[EndPos: ] + UniStr = ('\u' + (Line[StartPos + 2 : EndPos])).decode('unicode_escape') + if EndStr.startswith(u'\\x') and len(EndStr) >= 7: + if EndStr[6] == u'\\' and g4HexChar.match(EndStr[2 : 6], re.UNICODE): + Line = Line[0 : StartPos] + UniStr + EndStr + else: + Line = Line[0 : StartPos] + UniStr + EndStr[1:] + StartPos = Line.find(u'\\x', StartPos + 1) IncList = gIncludePattern.findall(Line) if len(IncList) == 1: - Lines.extend(self.PreProcess(PathClass(str(IncList[0]), Dir))) + for Dir in [File.Dir] + self.IncludePathList: + IncFile = PathClass(str(IncList[0]), Dir) + if os.path.isfile(IncFile.Path): + Lines.extend(self.PreProcess(IncFile)) + break + else: + EdkLogger.error("Unicode File Parser", FILE_NOT_FOUND, Message="Cannot find include file", ExtraData=str(IncList[0])) continue Lines.append(Line) @@ -370,11 +519,11 @@ class UniFileClassObject(object): break # Value = Value.replace(u'\r\n', u'') Language = GetLanguageCode(Language, self.IsCompatibleMode, self.File) - # Check the string name is the upper character + # Check the string name if not self.IsCompatibleMode and Name != '': - MatchString = re.match('[A-Z0-9_]+', Name, re.UNICODE) + MatchString = re.match('^[a-zA-Z][a-zA-Z0-9_]*$', Name, re.UNICODE) if MatchString == None or MatchString.end(0) != len(Name): - EdkLogger.error('Unicode File Parser', FORMAT_INVALID, 'The string token name %s defined in UNI file %s contains the invalid lower case character.' %(Name, self.File)) + EdkLogger.error('Unicode File Parser', FORMAT_INVALID, 'The string token name %s defined in UNI file %s contains the invalid character.' % (Name, self.File)) self.AddStringToList(Name, Language, Value) continue @@ -415,37 +564,68 @@ class UniFileClassObject(object): # Add a string to list # def AddStringToList(self, Name, Language, Value, Token = None, Referenced = False, UseOtherLangDef = '', Index = -1): + for LangNameItem in self.LanguageDef: + if Language == LangNameItem[0]: + break + else: + EdkLogger.error('Unicode File Parser', FORMAT_NOT_SUPPORTED, "The language '%s' for %s is not defined in Unicode file %s." \ + % (Language, Name, self.File)) + if Language not in self.OrderedStringList: self.OrderedStringList[Language] = [] - - IsAdded = False - for Item in self.OrderedStringList[Language]: - if Name == Item.StringName: - IsAdded = True - break - if not IsAdded: + self.OrderedStringDict[Language] = {} + + IsAdded = True + if Name in self.OrderedStringDict[Language]: + IsAdded = False + if Value != None: + ItemIndexInList = self.OrderedStringDict[Language][Name] + Item = self.OrderedStringList[Language][ItemIndexInList] + Item.UpdateValue(Value) + Item.UseOtherLangDef = '' + + if IsAdded: Token = len(self.OrderedStringList[Language]) if Index == -1: self.OrderedStringList[Language].append(StringDefClassObject(Name, Value, Referenced, Token, UseOtherLangDef)) + self.OrderedStringDict[Language][Name] = Token + for LangName in self.LanguageDef: + # + # New STRING token will be added into all language string lists. + # so that the unique STRING identifier is reserved for all languages in the package list. + # + if LangName[0] != Language: + if UseOtherLangDef != '': + OtherLangDef = UseOtherLangDef + else: + OtherLangDef = Language + self.OrderedStringList[LangName[0]].append(StringDefClassObject(Name, '', Referenced, Token, OtherLangDef)) + self.OrderedStringDict[LangName[0]][Name] = len(self.OrderedStringList[LangName[0]]) - 1 else: self.OrderedStringList[Language].insert(Index, StringDefClassObject(Name, Value, Referenced, Token, UseOtherLangDef)) + self.OrderedStringDict[Language][Name] = Index # # Set the string as referenced # def SetStringReferenced(self, Name): - for Lang in self.OrderedStringList: - for Item in self.OrderedStringList[Lang]: - if Name == Item.StringName: - Item.Referenced = True - break + # + # String stoken are added in the same order in all language string lists. + # So, only update the status of string stoken in first language string list. + # + Lang = self.LanguageDef[0][0] + if Name in self.OrderedStringDict[Lang]: + ItemIndexInList = self.OrderedStringDict[Lang][Name] + Item = self.OrderedStringList[Lang][ItemIndexInList] + Item.Referenced = True + # # Search the string in language definition by Name # def FindStringValue(self, Name, Lang): - for Item in self.OrderedStringList[Lang]: - if Item.StringName == Name: - return Item + if Name in self.OrderedStringDict[Lang]: + ItemIndexInList = self.OrderedStringDict[Lang][Name] + return self.OrderedStringList[Lang][ItemIndexInList] return None @@ -464,61 +644,42 @@ class UniFileClassObject(object): # def ReToken(self): # - # Search each string to find if it is defined for each language - # Use secondary language value to replace if missing in any one language - # - for IndexI in range(0, len(self.LanguageDef)): - LangKey = self.LanguageDef[IndexI][0] - for Item in self.OrderedStringList[LangKey]: - Name = Item.StringName - Value = Item.StringValue[0:-1] - Referenced = Item.Referenced - Index = self.OrderedStringList[LangKey].index(Item) - for IndexJ in range(0, len(self.LanguageDef)): - LangFind = self.LanguageDef[IndexJ][0] - if self.FindStringValue(Name, LangFind) == None: - EdkLogger.debug(EdkLogger.DEBUG_5, Name) - Token = len(self.OrderedStringList[LangFind]) - self.AddStringToList(Name, LangFind, Value, Token, Referenced, LangKey, Index) - # - # Retoken + # Retoken all language strings according to the status of string stoken in the first language string. # - # First re-token the first language - LangName = self.LanguageDef[0][0] - ReferencedStringList = [] - NotReferencedStringList = [] - Token = 0 + FirstLangName = self.LanguageDef[0][0] + + # Convert the OrderedStringList to be OrderedStringListByToken in order to faciliate future search by token + for LangNameItem in self.LanguageDef: + self.OrderedStringListByToken[LangNameItem[0]] = {} # - # Order UNI token by their String Name + # Use small token for all referred string stoken. # - StringNameList = [] - for Item in self.OrderedStringList[LangName]: - StringNameList.append (Item.StringName) - StringNameList.sort() - - for Name in StringNameList: - Item = self.FindStringValue (Name, LangName) - if Item.Referenced == True: - Item.Token = Token - ReferencedStringList.append(Item) - Token = Token + 1 - else: - NotReferencedStringList.append(Item) - self.OrderedStringList[LangName] = ReferencedStringList - for Index in range(len(NotReferencedStringList)): - NotReferencedStringList[Index].Token = Token + Index - self.OrderedStringList[LangName].append(NotReferencedStringList[Index]) + RefToken = 0 + for Index in range (0, len (self.OrderedStringList[FirstLangName])): + FirstLangItem = self.OrderedStringList[FirstLangName][Index] + if FirstLangItem.Referenced == True: + for LangNameItem in self.LanguageDef: + LangName = LangNameItem[0] + OtherLangItem = self.OrderedStringList[LangName][Index] + OtherLangItem.Referenced = True + OtherLangItem.Token = RefToken + self.OrderedStringListByToken[LangName][OtherLangItem.Token] = OtherLangItem + RefToken = RefToken + 1 # - # Adjust the orders of other languages + # Use big token for all unreferred string stoken. # - for IndexOfLanguage in range(1, len(self.LanguageDef)): - for OrderedString in self.OrderedStringList[LangName]: - for UnOrderedString in self.OrderedStringList[self.LanguageDef[IndexOfLanguage][0]]: - if OrderedString.StringName == UnOrderedString.StringName: - UnOrderedString.Token = OrderedString.Token - break + UnRefToken = 0 + for Index in range (0, len (self.OrderedStringList[FirstLangName])): + FirstLangItem = self.OrderedStringList[FirstLangName][Index] + if FirstLangItem.Referenced == False: + for LangNameItem in self.LanguageDef: + LangName = LangNameItem[0] + OtherLangItem = self.OrderedStringList[LangName][Index] + OtherLangItem.Token = RefToken + UnRefToken + self.OrderedStringListByToken[LangName][OtherLangItem.Token] = OtherLangItem + UnRefToken = UnRefToken + 1 # # Show the instance itself @@ -536,6 +697,6 @@ class UniFileClassObject(object): if __name__ == '__main__': EdkLogger.Initialize() EdkLogger.SetLevel(EdkLogger.DEBUG_0) - a = UniFileClassObject(['C:\\Edk\\Strings.uni', 'C:\\Edk\\Strings2.uni']) + a = UniFileClassObject([PathClass("C:\\Edk\\Strings.uni"), PathClass("C:\\Edk\\Strings2.uni")]) a.ReToken() a.ShowMe()