X-Git-Url: https://git.proxmox.com/?a=blobdiff_plain;f=BaseTools%2FSource%2FPython%2FAutoGen%2FUniClassObject.py;h=856d19cda270cdc64b84812f578860f2a35a4c1f;hb=6f49996cedfb198beb374ea0cb89a06a71ef960c;hp=717c10f266771d5bb55a349bdb2c390bbf5d18ed;hpb=f3decdc362d74c06b6b5028b09eecbaf8db23c12;p=mirror_edk2.git
diff --git a/BaseTools/Source/Python/AutoGen/UniClassObject.py b/BaseTools/Source/Python/AutoGen/UniClassObject.py
index 717c10f266..856d19cda2 100644
--- a/BaseTools/Source/Python/AutoGen/UniClassObject.py
+++ b/BaseTools/Source/Python/AutoGen/UniClassObject.py
@@ -1,5 +1,11 @@
-# Copyright (c) 2007 - 2010, Intel Corporation
-# All rights reserved. This program and the accompanying materials
+## @file
+# This file is used to collect all defined strings in multiple uni files
+#
+#
+# Copyright (c) 2014 Hewlett-Packard Development Company, L.P.
+#
+# Copyright (c) 2007 - 2016, Intel Corporation. All rights reserved.
+# This program and the accompanying materials
# are licensed and made available under the terms and conditions of the BSD License
# which accompanies this distribution. The full text of the license may be found at
# http://opensource.org/licenses/bsd-license.php
@@ -7,19 +13,17 @@
# THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS,
# WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED.
-#
-#This file is used to collect all defined strings in multiple uni files
-#
-
##
# Import Modules
#
-import os, codecs, re
+import Common.LongFilePathOs as os, codecs, re
+import distutils.util
import Common.EdkLogger as EdkLogger
+import StringIO
from Common.BuildToolError import *
from Common.String import GetLineNo
from Common.Misc import PathClass
-
+from Common.LongFilePathSupport import LongFilePath
##
# Static definitions
#
@@ -36,7 +40,7 @@ CR = u'\u000D'
LF = u'\u000A'
NULL = u'\u0000'
TAB = u'\t'
-BACK_SPLASH = u'\\'
+BACK_SLASH_PLACEHOLDER = u'\u0006'
gIncludePattern = re.compile("^#include +[\"<]+([^\"< >]+)[>\"]+$", re.MULTILINE | re.UNICODE)
@@ -120,11 +124,13 @@ def GetLanguageCode(LangName, IsCompatibleMode, File):
if length == 3 and LangName.isalpha():
TempLangName = LangConvTable.get(LangName.lower())
if TempLangName != None:
- return TempLangName
+ return TempLangName
return LangName
else:
EdkLogger.error("Unicode File Parser", FORMAT_INVALID, "Invalid ISO 639-2 language code : %s" % LangName, File)
+ if (LangName[0] == 'X' or LangName[0] == 'x') and LangName[1] == '-':
+ return LangName
if length == 2:
if LangName.isalpha():
return LangName
@@ -142,6 +148,37 @@ def GetLanguageCode(LangName, IsCompatibleMode, File):
EdkLogger.error("Unicode File Parser", FORMAT_INVALID, "Invalid RFC 4646 language code : %s" % LangName, File)
+## Ucs2Codec
+#
+# This is only a partial codec implementation. It only supports
+# encoding, and is primarily used to check that all the characters are
+# valid for UCS-2.
+#
+class Ucs2Codec(codecs.Codec):
+ def __init__(self):
+ self.__utf16 = codecs.lookup('utf-16')
+
+ def encode(self, input, errors='strict'):
+ for Char in input:
+ CodePoint = ord(Char)
+ if CodePoint >= 0xd800 and CodePoint <= 0xdfff:
+ raise ValueError("Code Point is in range reserved for " +
+ "UTF-16 surrogate pairs")
+ elif CodePoint > 0xffff:
+ raise ValueError("Code Point too large to encode in UCS-2")
+ return self.__utf16.encode(input)
+
+TheUcs2Codec = Ucs2Codec()
+def Ucs2Search(name):
+ if name == 'ucs-2':
+ return codecs.CodecInfo(
+ name=name,
+ encode=TheUcs2Codec.encode,
+ decode=TheUcs2Codec.decode)
+ else:
+ return None
+codecs.register(Ucs2Search)
+
## StringDefClassObject
#
# A structure for language definition
@@ -174,18 +211,26 @@ class StringDefClassObject(object):
repr(self.StringValue) + ' ' + \
repr(self.UseOtherLangDef)
+ def UpdateValue(self, Value = None):
+ if Value != None:
+ self.StringValue = Value + u'\x00' # Add a NULL at string tail
+ self.StringValueByteList = UniToHexList(self.StringValue)
+ self.Length = len(self.StringValueByteList)
+
## UniFileClassObject
#
# A structure for .uni file definition
#
class UniFileClassObject(object):
- def __init__(self, FileList = [], IsCompatibleMode = False):
+ def __init__(self, FileList = [], IsCompatibleMode = False, IncludePathList = []):
self.FileList = FileList
self.Token = 2
self.LanguageDef = [] #[ [u'LanguageIdentifier', u'PrintableName'], ... ]
self.OrderedStringList = {} #{ u'LanguageIdentifier' : [StringDefClassObject] }
+ self.OrderedStringDict = {} #{ u'LanguageIdentifier' : {StringName:(IndexInList)} }
+ self.OrderedStringListByToken = {} #{ u'LanguageIdentifier' : {Token: StringDefClassObject} }
self.IsCompatibleMode = IsCompatibleMode
-
+ self.IncludePathList = IncludePathList
if len(self.FileList) > 0:
self.LoadUniFiles(FileList)
@@ -193,20 +238,20 @@ class UniFileClassObject(object):
# Get Language definition
#
def GetLangDef(self, File, Line):
- Lang = Line.split(u"//")[0].split()
+ Lang = distutils.util.split_quoted((Line.split(u"//")[0]))
if len(Lang) != 3:
try:
- FileIn = codecs.open(File, mode='rb', encoding='utf-16').read()
+ FileIn = self.OpenUniFile(LongFilePath(File.Path))
except UnicodeError, X:
EdkLogger.error("build", FILE_READ_FAILURE, "File read failure: %s" % str(X), ExtraData=File);
except:
EdkLogger.error("build", FILE_OPEN_FAILURE, ExtraData=File);
LineNo = GetLineNo(FileIn, Line, False)
EdkLogger.error("Unicode File Parser", PARSER_ERROR, "Wrong language definition",
- ExtraData="""%s\n\t*Correct format is like '#langdef eng "English"'""" % Line, File = File, Line = LineNo)
+ ExtraData="""%s\n\t*Correct format is like '#langdef en-US "English"'""" % Line, File=File, Line=LineNo)
else:
LangName = GetLanguageCode(Lang[1], self.IsCompatibleMode, self.File)
- LangPrintName = Lang[2][1:-1]
+ LangPrintName = Lang[2]
IsLangInDef = False
for Item in self.LanguageDef:
@@ -223,22 +268,91 @@ class UniFileClassObject(object):
self.AddStringToList(u'$LANGUAGE_NAME', LangName, LangName, 0, True, Index=0)
self.AddStringToList(u'$PRINTABLE_LANGUAGE_NAME', LangName, LangPrintName, 1, True, Index=1)
+ if not IsLangInDef:
+ #
+ # The found STRING tokens will be added into new language string list
+ # so that the unique STRING identifier is reserved for all languages in the package list.
+ #
+ FirstLangName = self.LanguageDef[0][0]
+ if LangName != FirstLangName:
+ for Index in range (2, len (self.OrderedStringList[FirstLangName])):
+ Item = self.OrderedStringList[FirstLangName][Index]
+ if Item.UseOtherLangDef != '':
+ OtherLang = Item.UseOtherLangDef
+ else:
+ OtherLang = FirstLangName
+ self.OrderedStringList[LangName].append (StringDefClassObject(Item.StringName, '', Item.Referenced, Item.Token, OtherLang))
+ self.OrderedStringDict[LangName][Item.StringName] = len(self.OrderedStringList[LangName]) - 1
return True
+ def OpenUniFile(self, FileName):
+ #
+ # Read file
+ #
+ try:
+ UniFile = open(FileName, mode='rb')
+ FileIn = UniFile.read()
+ UniFile.close()
+ except:
+ EdkLogger.Error("build", FILE_OPEN_FAILURE, ExtraData=File)
+
+ #
+ # Detect Byte Order Mark at beginning of file. Default to UTF-8
+ #
+ Encoding = 'utf-8'
+ if (FileIn.startswith(codecs.BOM_UTF16_BE) or
+ FileIn.startswith(codecs.BOM_UTF16_LE)):
+ Encoding = 'utf-16'
+
+ self.VerifyUcs2Data(FileIn, FileName, Encoding)
+
+ UniFile = StringIO.StringIO(FileIn)
+ Info = codecs.lookup(Encoding)
+ (Reader, Writer) = (Info.streamreader, Info.streamwriter)
+ return codecs.StreamReaderWriter(UniFile, Reader, Writer)
+
+ def VerifyUcs2Data(self, FileIn, FileName, Encoding):
+ Ucs2Info = codecs.lookup('ucs-2')
+ #
+ # Convert to unicode
+ #
+ try:
+ FileDecoded = codecs.decode(FileIn, Encoding)
+ Ucs2Info.encode(FileDecoded)
+ except:
+ UniFile = StringIO.StringIO(FileIn)
+ Info = codecs.lookup(Encoding)
+ (Reader, Writer) = (Info.streamreader, Info.streamwriter)
+ File = codecs.StreamReaderWriter(UniFile, Reader, Writer)
+ LineNumber = 0
+ ErrMsg = lambda Encoding, LineNumber: \
+ '%s contains invalid %s characters on line %d.' % \
+ (FileName, Encoding, LineNumber)
+ while True:
+ LineNumber = LineNumber + 1
+ try:
+ Line = File.readline()
+ if Line == '':
+ EdkLogger.error('Unicode File Parser', PARSER_ERROR,
+ ErrMsg(Encoding, LineNumber))
+ Ucs2Info.encode(Line)
+ except:
+ EdkLogger.error('Unicode File Parser', PARSER_ERROR,
+ ErrMsg('UCS-2', LineNumber))
+
#
# Get String name and value
#
def GetStringObject(self, Item):
- Name = ''
Language = ''
Value = ''
Name = Item.split()[1]
- # Check the string name is the upper character
- if not self.IsCompatibleMode and Name != '':
- MatchString = re.match('[A-Z0-9_]+', Name, re.UNICODE)
+ # Check the string name
+ if Name != '':
+ MatchString = re.match('^[a-zA-Z][a-zA-Z0-9_]*$', Name, re.UNICODE)
if MatchString == None or MatchString.end(0) != len(Name):
- EdkLogger.error('Unicode File Parser', FORMAT_INVALID, 'The string token name %s defined in UNI file %s contains the invalid lower case character.' %(Name, self.File))
+ EdkLogger.error('Unicode File Parser', FORMAT_INVALID, 'The string token name %s defined in UNI file %s contains the invalid character.' % (Name, self.File))
LanguageList = Item.split(u'#language ')
for IndexI in range(len(LanguageList)):
if IndexI == 0:
@@ -256,6 +370,20 @@ class UniFileClassObject(object):
FileName = Item[Item.find(u'#include ') + len(u'#include ') :Item.find(u' ', len(u'#include '))][1:-1]
self.LoadUniFile(FileName)
+ def StripComments(self, Line):
+ Comment = u'//'
+ CommentPos = Line.find(Comment)
+ while CommentPos >= 0:
+ # if there are non matched quotes before the comment header
+ # then we are in the middle of a string
+ # but we need to ignore the escaped quotes and backslashes.
+ if ((Line.count(u'"', 0, CommentPos) - Line.count(u'\\"', 0, CommentPos)) & 1) == 1:
+ CommentPos = Line.find (Comment, CommentPos + 1)
+ else:
+ return Line[:CommentPos].strip()
+ return Line.strip()
+
+
#
# Pre-process before parse .uni file
#
@@ -263,9 +391,8 @@ class UniFileClassObject(object):
if not os.path.exists(File.Path) or not os.path.isfile(File.Path):
EdkLogger.error("Unicode File Parser", FILE_NOT_FOUND, ExtraData=File.Path)
- Dir = File.Dir
try:
- FileIn = codecs.open(File.Path, mode='rb', encoding='utf-16').readlines()
+ FileIn = self.OpenUniFile(LongFilePath(File.Path))
except UnicodeError, X:
EdkLogger.error("build", FILE_READ_FAILURE, "File read failure: %s" % str(X), ExtraData=File.Path);
except:
@@ -277,11 +404,16 @@ class UniFileClassObject(object):
#
for Line in FileIn:
Line = Line.strip()
+ Line = Line.replace(u'\\\\', BACK_SLASH_PLACEHOLDER)
+ Line = self.StripComments(Line)
+
#
- # Ignore comment line and empty line
+ # Ignore empty line
#
- if Line == u'' or Line.startswith(u'//'):
- continue
+ if len(Line) == 0:
+ continue
+
+
Line = Line.replace(u'/langdef', u'#langdef')
Line = Line.replace(u'/string', u'#string')
Line = Line.replace(u'/language', u'#language')
@@ -291,22 +423,38 @@ class UniFileClassObject(object):
Line = Line.replace(UNICODE_NARROW_CHAR, NARROW_CHAR)
Line = Line.replace(UNICODE_NON_BREAKING_CHAR, NON_BREAKING_CHAR)
- Line = Line.replace(u'\\\\', u'\u0006')
Line = Line.replace(u'\\r\\n', CR + LF)
Line = Line.replace(u'\\n', CR + LF)
Line = Line.replace(u'\\r', CR)
- Line = Line.replace(u'\\t', u'\t')
- Line = Line.replace(u'''\"''', u'''"''')
+ Line = Line.replace(u'\\t', u' ')
Line = Line.replace(u'\t', u' ')
- Line = Line.replace(u'\u0006', u'\\')
-
-# if Line.find(u'\\x'):
-# hex = Line[Line.find(u'\\x') + 2 : Line.find(u'\\x') + 6]
-# hex = "u'\\u" + hex + "'"
+ Line = Line.replace(u'\\"', u'"')
+ Line = Line.replace(u"\\'", u"'")
+ Line = Line.replace(BACK_SLASH_PLACEHOLDER, u'\\')
+
+ StartPos = Line.find(u'\\x')
+ while (StartPos != -1):
+ EndPos = Line.find(u'\\', StartPos + 1, StartPos + 7)
+ if EndPos != -1 and EndPos - StartPos == 6 :
+ if re.match('[a-fA-F0-9]{4}', Line[StartPos + 2 : EndPos], re.UNICODE):
+ EndStr = Line[EndPos: ]
+ UniStr = ('\u' + (Line[StartPos + 2 : EndPos])).decode('unicode_escape')
+ if EndStr.startswith(u'\\x') and len(EndStr) >= 7:
+ if EndStr[6] == u'\\' and re.match('[a-fA-F0-9]{4}', EndStr[2 : 6], re.UNICODE):
+ Line = Line[0 : StartPos] + UniStr + EndStr
+ else:
+ Line = Line[0 : StartPos] + UniStr + EndStr[1:]
+ StartPos = Line.find(u'\\x', StartPos + 1)
IncList = gIncludePattern.findall(Line)
if len(IncList) == 1:
- Lines.extend(self.PreProcess(PathClass(str(IncList[0]), Dir)))
+ for Dir in [File.Dir] + self.IncludePathList:
+ IncFile = PathClass(str(IncList[0]), Dir)
+ if os.path.isfile(IncFile.Path):
+ Lines.extend(self.PreProcess(IncFile))
+ break
+ else:
+ EdkLogger.error("Unicode File Parser", FILE_NOT_FOUND, Message="Cannot find include file", ExtraData=str(IncList[0]))
continue
Lines.append(Line)
@@ -370,11 +518,11 @@ class UniFileClassObject(object):
break
# Value = Value.replace(u'\r\n', u'')
Language = GetLanguageCode(Language, self.IsCompatibleMode, self.File)
- # Check the string name is the upper character
+ # Check the string name
if not self.IsCompatibleMode and Name != '':
- MatchString = re.match('[A-Z0-9_]+', Name, re.UNICODE)
+ MatchString = re.match('^[a-zA-Z][a-zA-Z0-9_]*$', Name, re.UNICODE)
if MatchString == None or MatchString.end(0) != len(Name):
- EdkLogger.error('Unicode File Parser', FORMAT_INVALID, 'The string token name %s defined in UNI file %s contains the invalid lower case character.' %(Name, self.File))
+ EdkLogger.error('Unicode File Parser', FORMAT_INVALID, 'The string token name %s defined in UNI file %s contains the invalid character.' % (Name, self.File))
self.AddStringToList(Name, Language, Value)
continue
@@ -415,37 +563,68 @@ class UniFileClassObject(object):
# Add a string to list
#
def AddStringToList(self, Name, Language, Value, Token = None, Referenced = False, UseOtherLangDef = '', Index = -1):
+ for LangNameItem in self.LanguageDef:
+ if Language == LangNameItem[0]:
+ break
+ else:
+ EdkLogger.error('Unicode File Parser', FORMAT_NOT_SUPPORTED, "The language '%s' for %s is not defined in Unicode file %s." \
+ % (Language, Name, self.File))
+
if Language not in self.OrderedStringList:
self.OrderedStringList[Language] = []
-
- IsAdded = False
- for Item in self.OrderedStringList[Language]:
- if Name == Item.StringName:
- IsAdded = True
- break
- if not IsAdded:
+ self.OrderedStringDict[Language] = {}
+
+ IsAdded = True
+ if Name in self.OrderedStringDict[Language]:
+ IsAdded = False
+ if Value != None:
+ ItemIndexInList = self.OrderedStringDict[Language][Name]
+ Item = self.OrderedStringList[Language][ItemIndexInList]
+ Item.UpdateValue(Value)
+ Item.UseOtherLangDef = ''
+
+ if IsAdded:
Token = len(self.OrderedStringList[Language])
if Index == -1:
self.OrderedStringList[Language].append(StringDefClassObject(Name, Value, Referenced, Token, UseOtherLangDef))
+ self.OrderedStringDict[Language][Name] = Token
+ for LangName in self.LanguageDef:
+ #
+ # New STRING token will be added into all language string lists.
+ # so that the unique STRING identifier is reserved for all languages in the package list.
+ #
+ if LangName[0] != Language:
+ if UseOtherLangDef != '':
+ OtherLangDef = UseOtherLangDef
+ else:
+ OtherLangDef = Language
+ self.OrderedStringList[LangName[0]].append(StringDefClassObject(Name, '', Referenced, Token, OtherLangDef))
+ self.OrderedStringDict[LangName[0]][Name] = len(self.OrderedStringList[LangName[0]]) - 1
else:
self.OrderedStringList[Language].insert(Index, StringDefClassObject(Name, Value, Referenced, Token, UseOtherLangDef))
+ self.OrderedStringDict[Language][Name] = Index
#
# Set the string as referenced
#
def SetStringReferenced(self, Name):
- for Lang in self.OrderedStringList:
- for Item in self.OrderedStringList[Lang]:
- if Name == Item.StringName:
- Item.Referenced = True
- break
+ #
+ # String stoken are added in the same order in all language string lists.
+ # So, only update the status of string stoken in first language string list.
+ #
+ Lang = self.LanguageDef[0][0]
+ if Name in self.OrderedStringDict[Lang]:
+ ItemIndexInList = self.OrderedStringDict[Lang][Name]
+ Item = self.OrderedStringList[Lang][ItemIndexInList]
+ Item.Referenced = True
+
#
# Search the string in language definition by Name
#
def FindStringValue(self, Name, Lang):
- for Item in self.OrderedStringList[Lang]:
- if Item.StringName == Name:
- return Item
+ if Name in self.OrderedStringDict[Lang]:
+ ItemIndexInList = self.OrderedStringDict[Lang][Name]
+ return self.OrderedStringList[Lang][ItemIndexInList]
return None
@@ -464,61 +643,42 @@ class UniFileClassObject(object):
#
def ReToken(self):
#
- # Search each string to find if it is defined for each language
- # Use secondary language value to replace if missing in any one language
- #
- for IndexI in range(0, len(self.LanguageDef)):
- LangKey = self.LanguageDef[IndexI][0]
- for Item in self.OrderedStringList[LangKey]:
- Name = Item.StringName
- Value = Item.StringValue[0:-1]
- Referenced = Item.Referenced
- Index = self.OrderedStringList[LangKey].index(Item)
- for IndexJ in range(0, len(self.LanguageDef)):
- LangFind = self.LanguageDef[IndexJ][0]
- if self.FindStringValue(Name, LangFind) == None:
- EdkLogger.debug(EdkLogger.DEBUG_5, Name)
- Token = len(self.OrderedStringList[LangFind])
- self.AddStringToList(Name, LangFind, Value, Token, Referenced, LangKey, Index)
- #
- # Retoken
+ # Retoken all language strings according to the status of string stoken in the first language string.
#
- # First re-token the first language
- LangName = self.LanguageDef[0][0]
- ReferencedStringList = []
- NotReferencedStringList = []
- Token = 0
+ FirstLangName = self.LanguageDef[0][0]
+
+ # Convert the OrderedStringList to be OrderedStringListByToken in order to faciliate future search by token
+ for LangNameItem in self.LanguageDef:
+ self.OrderedStringListByToken[LangNameItem[0]] = {}
#
- # Order UNI token by their String Name
+ # Use small token for all referred string stoken.
#
- StringNameList = []
- for Item in self.OrderedStringList[LangName]:
- StringNameList.append (Item.StringName)
- StringNameList.sort()
-
- for Name in StringNameList:
- Item = self.FindStringValue (Name, LangName)
- if Item.Referenced == True:
- Item.Token = Token
- ReferencedStringList.append(Item)
- Token = Token + 1
- else:
- NotReferencedStringList.append(Item)
- self.OrderedStringList[LangName] = ReferencedStringList
- for Index in range(len(NotReferencedStringList)):
- NotReferencedStringList[Index].Token = Token + Index
- self.OrderedStringList[LangName].append(NotReferencedStringList[Index])
+ RefToken = 0
+ for Index in range (0, len (self.OrderedStringList[FirstLangName])):
+ FirstLangItem = self.OrderedStringList[FirstLangName][Index]
+ if FirstLangItem.Referenced == True:
+ for LangNameItem in self.LanguageDef:
+ LangName = LangNameItem[0]
+ OtherLangItem = self.OrderedStringList[LangName][Index]
+ OtherLangItem.Referenced = True
+ OtherLangItem.Token = RefToken
+ self.OrderedStringListByToken[LangName][OtherLangItem.Token] = OtherLangItem
+ RefToken = RefToken + 1
#
- # Adjust the orders of other languages
+ # Use big token for all unreferred string stoken.
#
- for IndexOfLanguage in range(1, len(self.LanguageDef)):
- for OrderedString in self.OrderedStringList[LangName]:
- for UnOrderedString in self.OrderedStringList[self.LanguageDef[IndexOfLanguage][0]]:
- if OrderedString.StringName == UnOrderedString.StringName:
- UnOrderedString.Token = OrderedString.Token
- break
+ UnRefToken = 0
+ for Index in range (0, len (self.OrderedStringList[FirstLangName])):
+ FirstLangItem = self.OrderedStringList[FirstLangName][Index]
+ if FirstLangItem.Referenced == False:
+ for LangNameItem in self.LanguageDef:
+ LangName = LangNameItem[0]
+ OtherLangItem = self.OrderedStringList[LangName][Index]
+ OtherLangItem.Token = RefToken + UnRefToken
+ self.OrderedStringListByToken[LangName][OtherLangItem.Token] = OtherLangItem
+ UnRefToken = UnRefToken + 1
#
# Show the instance itself
@@ -536,6 +696,6 @@ class UniFileClassObject(object):
if __name__ == '__main__':
EdkLogger.Initialize()
EdkLogger.SetLevel(EdkLogger.DEBUG_0)
- a = UniFileClassObject(['C:\\Edk\\Strings.uni', 'C:\\Edk\\Strings2.uni'])
+ a = UniFileClassObject([PathClass("C:\\Edk\\Strings.uni"), PathClass("C:\\Edk\\Strings2.uni")])
a.ReToken()
a.ShowMe()