-# Copyright (c) 2007, Intel Corporation\r
-# All rights reserved. This program and the accompanying materials\r
+## @file\r
+# This file is used to collect all defined strings in multiple uni files\r
+#\r
+#\r
+# Copyright (c) 2014 Hewlett-Packard Development Company, L.P.<BR>\r
+#\r
+# Copyright (c) 2007 - 2016, Intel Corporation. All rights reserved.<BR>\r
+# This program and the accompanying materials\r
# are licensed and made available under the terms and conditions of the BSD License\r
# which accompanies this distribution. The full text of the license may be found at\r
# http://opensource.org/licenses/bsd-license.php\r
# THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS,\r
# WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED.\r
\r
-#\r
-#This file is used to collect all defined strings in multiple uni files\r
-#\r
-\r
##\r
# Import Modules\r
#\r
-import os, codecs, re\r
+import Common.LongFilePathOs as os, codecs, re\r
+import distutils.util\r
import Common.EdkLogger as EdkLogger\r
+import StringIO\r
from Common.BuildToolError import *\r
from Common.String import GetLineNo\r
from Common.Misc import PathClass\r
-\r
+from Common.LongFilePathSupport import LongFilePath\r
##\r
# Static definitions\r
#\r
LF = u'\u000A'\r
NULL = u'\u0000'\r
TAB = u'\t'\r
-BACK_SPLASH = u'\\'\r
+BACK_SLASH_PLACEHOLDER = u'\u0006'\r
\r
gIncludePattern = re.compile("^#include +[\"<]+([^\"< >]+)[>\"]+$", re.MULTILINE | re.UNICODE)\r
\r
if length == 3 and LangName.isalpha():\r
TempLangName = LangConvTable.get(LangName.lower())\r
if TempLangName != None:\r
- return TempLangName\r
+ return TempLangName\r
return LangName\r
else:\r
EdkLogger.error("Unicode File Parser", FORMAT_INVALID, "Invalid ISO 639-2 language code : %s" % LangName, File)\r
\r
+ if (LangName[0] == 'X' or LangName[0] == 'x') and LangName[1] == '-':\r
+ return LangName\r
if length == 2:\r
if LangName.isalpha():\r
return LangName\r
\r
EdkLogger.error("Unicode File Parser", FORMAT_INVALID, "Invalid RFC 4646 language code : %s" % LangName, File)\r
\r
+## Ucs2Codec\r
+#\r
+# This is only a partial codec implementation. It only supports\r
+# encoding, and is primarily used to check that all the characters are\r
+# valid for UCS-2.\r
+#\r
+class Ucs2Codec(codecs.Codec):\r
+ def __init__(self):\r
+ self.__utf16 = codecs.lookup('utf-16')\r
+\r
+ def encode(self, input, errors='strict'):\r
+ for Char in input:\r
+ CodePoint = ord(Char)\r
+ if CodePoint >= 0xd800 and CodePoint <= 0xdfff:\r
+ raise ValueError("Code Point is in range reserved for " +\r
+ "UTF-16 surrogate pairs")\r
+ elif CodePoint > 0xffff:\r
+ raise ValueError("Code Point too large to encode in UCS-2")\r
+ return self.__utf16.encode(input)\r
+\r
+TheUcs2Codec = Ucs2Codec()\r
+def Ucs2Search(name):\r
+ if name == 'ucs-2':\r
+ return codecs.CodecInfo(\r
+ name=name,\r
+ encode=TheUcs2Codec.encode,\r
+ decode=TheUcs2Codec.decode)\r
+ else:\r
+ return None\r
+codecs.register(Ucs2Search)\r
+\r
## StringDefClassObject\r
#\r
# A structure for language definition\r
repr(self.StringValue) + ' ' + \\r
repr(self.UseOtherLangDef)\r
\r
+ def UpdateValue(self, Value = None):\r
+ if Value != None:\r
+ self.StringValue = Value + u'\x00' # Add a NULL at string tail\r
+ self.StringValueByteList = UniToHexList(self.StringValue)\r
+ self.Length = len(self.StringValueByteList)\r
+\r
## UniFileClassObject\r
#\r
# A structure for .uni file definition\r
#\r
class UniFileClassObject(object):\r
- def __init__(self, FileList = [], IsCompatibleMode = False):\r
+ def __init__(self, FileList = [], IsCompatibleMode = False, IncludePathList = []):\r
self.FileList = FileList\r
self.Token = 2\r
self.LanguageDef = [] #[ [u'LanguageIdentifier', u'PrintableName'], ... ]\r
self.OrderedStringList = {} #{ u'LanguageIdentifier' : [StringDefClassObject] }\r
+ self.OrderedStringDict = {} #{ u'LanguageIdentifier' : {StringName:(IndexInList)} }\r
+ self.OrderedStringListByToken = {} #{ u'LanguageIdentifier' : {Token: StringDefClassObject} }\r
self.IsCompatibleMode = IsCompatibleMode\r
-\r
+ self.IncludePathList = IncludePathList\r
if len(self.FileList) > 0:\r
self.LoadUniFiles(FileList)\r
\r
# Get Language definition\r
#\r
def GetLangDef(self, File, Line):\r
- Lang = Line.split()\r
+ Lang = distutils.util.split_quoted((Line.split(u"//")[0]))\r
if len(Lang) != 3:\r
try:\r
- FileIn = codecs.open(File, mode='rb', encoding='utf-16').read()\r
+ FileIn = self.OpenUniFile(LongFilePath(File.Path))\r
except UnicodeError, X:\r
EdkLogger.error("build", FILE_READ_FAILURE, "File read failure: %s" % str(X), ExtraData=File);\r
except:\r
EdkLogger.error("build", FILE_OPEN_FAILURE, ExtraData=File);\r
LineNo = GetLineNo(FileIn, Line, False)\r
EdkLogger.error("Unicode File Parser", PARSER_ERROR, "Wrong language definition",\r
- ExtraData="""%s\n\t*Correct format is like '#langdef eng "English"'""" % Line, File = File, Line = LineNo)\r
+ ExtraData="""%s\n\t*Correct format is like '#langdef en-US "English"'""" % Line, File=File, Line=LineNo)\r
else:\r
LangName = GetLanguageCode(Lang[1], self.IsCompatibleMode, self.File)\r
- LangPrintName = Lang[2][1:-1]\r
+ LangPrintName = Lang[2]\r
\r
IsLangInDef = False\r
for Item in self.LanguageDef:\r
self.AddStringToList(u'$LANGUAGE_NAME', LangName, LangName, 0, True, Index=0)\r
self.AddStringToList(u'$PRINTABLE_LANGUAGE_NAME', LangName, LangPrintName, 1, True, Index=1)\r
\r
+ if not IsLangInDef:\r
+ #\r
+ # The found STRING tokens will be added into new language string list\r
+ # so that the unique STRING identifier is reserved for all languages in the package list. \r
+ #\r
+ FirstLangName = self.LanguageDef[0][0]\r
+ if LangName != FirstLangName:\r
+ for Index in range (2, len (self.OrderedStringList[FirstLangName])):\r
+ Item = self.OrderedStringList[FirstLangName][Index]\r
+ if Item.UseOtherLangDef != '':\r
+ OtherLang = Item.UseOtherLangDef\r
+ else:\r
+ OtherLang = FirstLangName\r
+ self.OrderedStringList[LangName].append (StringDefClassObject(Item.StringName, '', Item.Referenced, Item.Token, OtherLang))\r
+ self.OrderedStringDict[LangName][Item.StringName] = len(self.OrderedStringList[LangName]) - 1\r
return True\r
\r
+ def OpenUniFile(self, FileName):\r
+ #\r
+ # Read file\r
+ #\r
+ try:\r
+ UniFile = open(FileName, mode='rb')\r
+ FileIn = UniFile.read()\r
+ UniFile.close()\r
+ except:\r
+ EdkLogger.Error("build", FILE_OPEN_FAILURE, ExtraData=File)\r
+\r
+ #\r
+ # Detect Byte Order Mark at beginning of file. Default to UTF-8\r
+ #\r
+ Encoding = 'utf-8'\r
+ if (FileIn.startswith(codecs.BOM_UTF16_BE) or\r
+ FileIn.startswith(codecs.BOM_UTF16_LE)):\r
+ Encoding = 'utf-16'\r
+\r
+ self.VerifyUcs2Data(FileIn, FileName, Encoding)\r
+\r
+ UniFile = StringIO.StringIO(FileIn)\r
+ Info = codecs.lookup(Encoding)\r
+ (Reader, Writer) = (Info.streamreader, Info.streamwriter)\r
+ return codecs.StreamReaderWriter(UniFile, Reader, Writer)\r
+\r
+ def VerifyUcs2Data(self, FileIn, FileName, Encoding):\r
+ Ucs2Info = codecs.lookup('ucs-2')\r
+ #\r
+ # Convert to unicode\r
+ #\r
+ try:\r
+ FileDecoded = codecs.decode(FileIn, Encoding)\r
+ Ucs2Info.encode(FileDecoded)\r
+ except:\r
+ UniFile = StringIO.StringIO(FileIn)\r
+ Info = codecs.lookup(Encoding)\r
+ (Reader, Writer) = (Info.streamreader, Info.streamwriter)\r
+ File = codecs.StreamReaderWriter(UniFile, Reader, Writer)\r
+ LineNumber = 0\r
+ ErrMsg = lambda Encoding, LineNumber: \\r
+ '%s contains invalid %s characters on line %d.' % \\r
+ (FileName, Encoding, LineNumber)\r
+ while True:\r
+ LineNumber = LineNumber + 1\r
+ try:\r
+ Line = File.readline()\r
+ if Line == '':\r
+ EdkLogger.error('Unicode File Parser', PARSER_ERROR,\r
+ ErrMsg(Encoding, LineNumber))\r
+ Ucs2Info.encode(Line)\r
+ except:\r
+ EdkLogger.error('Unicode File Parser', PARSER_ERROR,\r
+ ErrMsg('UCS-2', LineNumber))\r
+\r
#\r
# Get String name and value\r
#\r
def GetStringObject(self, Item):\r
- Name = ''\r
Language = ''\r
Value = ''\r
\r
Name = Item.split()[1]\r
+ # Check the string name\r
+ if Name != '':\r
+ MatchString = re.match('^[a-zA-Z][a-zA-Z0-9_]*$', Name, re.UNICODE)\r
+ if MatchString == None or MatchString.end(0) != len(Name):\r
+ EdkLogger.error('Unicode File Parser', FORMAT_INVALID, 'The string token name %s defined in UNI file %s contains the invalid character.' % (Name, self.File))\r
LanguageList = Item.split(u'#language ')\r
for IndexI in range(len(LanguageList)):\r
if IndexI == 0:\r
FileName = Item[Item.find(u'#include ') + len(u'#include ') :Item.find(u' ', len(u'#include '))][1:-1]\r
self.LoadUniFile(FileName)\r
\r
+ def StripComments(self, Line):\r
+ Comment = u'//'\r
+ CommentPos = Line.find(Comment)\r
+ while CommentPos >= 0:\r
+ # if there are non matched quotes before the comment header\r
+ # then we are in the middle of a string\r
+ # but we need to ignore the escaped quotes and backslashes.\r
+ if ((Line.count(u'"', 0, CommentPos) - Line.count(u'\\"', 0, CommentPos)) & 1) == 1:\r
+ CommentPos = Line.find (Comment, CommentPos + 1)\r
+ else:\r
+ return Line[:CommentPos].strip()\r
+ return Line.strip()\r
+ \r
+\r
#\r
# Pre-process before parse .uni file\r
#\r
if not os.path.exists(File.Path) or not os.path.isfile(File.Path):\r
EdkLogger.error("Unicode File Parser", FILE_NOT_FOUND, ExtraData=File.Path)\r
\r
- Dir = File.Dir\r
try:\r
- FileIn = codecs.open(File.Path, mode='rb', encoding='utf-16').readlines()\r
+ FileIn = self.OpenUniFile(LongFilePath(File.Path))\r
except UnicodeError, X:\r
EdkLogger.error("build", FILE_READ_FAILURE, "File read failure: %s" % str(X), ExtraData=File.Path);\r
except:\r
#\r
for Line in FileIn:\r
Line = Line.strip()\r
+ Line = Line.replace(u'\\\\', BACK_SLASH_PLACEHOLDER)\r
+ Line = self.StripComments(Line)\r
+\r
#\r
- # Ignore comment line and empty line\r
+ # Ignore empty line\r
#\r
- if Line == u'' or Line.startswith(u'//'):\r
- continue\r
+ if len(Line) == 0: \r
+ continue \r
+ \r
+ \r
Line = Line.replace(u'/langdef', u'#langdef')\r
Line = Line.replace(u'/string', u'#string')\r
Line = Line.replace(u'/language', u'#language')\r
Line = Line.replace(UNICODE_NARROW_CHAR, NARROW_CHAR)\r
Line = Line.replace(UNICODE_NON_BREAKING_CHAR, NON_BREAKING_CHAR)\r
\r
- Line = Line.replace(u'\\\\', u'\u0006')\r
Line = Line.replace(u'\\r\\n', CR + LF)\r
Line = Line.replace(u'\\n', CR + LF)\r
Line = Line.replace(u'\\r', CR)\r
- Line = Line.replace(u'\\t', u'\t')\r
- Line = Line.replace(u'''\"''', u'''"''')\r
+ Line = Line.replace(u'\\t', u' ')\r
Line = Line.replace(u'\t', u' ')\r
- Line = Line.replace(u'\u0006', u'\\')\r
- \r
-# if Line.find(u'\\x'):\r
-# hex = Line[Line.find(u'\\x') + 2 : Line.find(u'\\x') + 6]\r
-# hex = "u'\\u" + hex + "'"\r
+ Line = Line.replace(u'\\"', u'"') \r
+ Line = Line.replace(u"\\'", u"'") \r
+ Line = Line.replace(BACK_SLASH_PLACEHOLDER, u'\\')\r
+\r
+ StartPos = Line.find(u'\\x')\r
+ while (StartPos != -1):\r
+ EndPos = Line.find(u'\\', StartPos + 1, StartPos + 7)\r
+ if EndPos != -1 and EndPos - StartPos == 6 :\r
+ if re.match('[a-fA-F0-9]{4}', Line[StartPos + 2 : EndPos], re.UNICODE):\r
+ EndStr = Line[EndPos: ]\r
+ UniStr = ('\u' + (Line[StartPos + 2 : EndPos])).decode('unicode_escape')\r
+ if EndStr.startswith(u'\\x') and len(EndStr) >= 7:\r
+ if EndStr[6] == u'\\' and re.match('[a-fA-F0-9]{4}', EndStr[2 : 6], re.UNICODE):\r
+ Line = Line[0 : StartPos] + UniStr + EndStr\r
+ else:\r
+ Line = Line[0 : StartPos] + UniStr + EndStr[1:]\r
+ StartPos = Line.find(u'\\x', StartPos + 1)\r
\r
IncList = gIncludePattern.findall(Line)\r
if len(IncList) == 1:\r
- Lines.extend(self.PreProcess(PathClass(str(IncList[0]), Dir)))\r
+ for Dir in [File.Dir] + self.IncludePathList:\r
+ IncFile = PathClass(str(IncList[0]), Dir)\r
+ if os.path.isfile(IncFile.Path):\r
+ Lines.extend(self.PreProcess(IncFile))\r
+ break\r
+ else:\r
+ EdkLogger.error("Unicode File Parser", FILE_NOT_FOUND, Message="Cannot find include file", ExtraData=str(IncList[0]))\r
continue\r
\r
Lines.append(Line)\r
break\r
# Value = Value.replace(u'\r\n', u'')\r
Language = GetLanguageCode(Language, self.IsCompatibleMode, self.File)\r
+ # Check the string name\r
+ if not self.IsCompatibleMode and Name != '':\r
+ MatchString = re.match('^[a-zA-Z][a-zA-Z0-9_]*$', Name, re.UNICODE)\r
+ if MatchString == None or MatchString.end(0) != len(Name):\r
+ EdkLogger.error('Unicode File Parser', FORMAT_INVALID, 'The string token name %s defined in UNI file %s contains the invalid character.' % (Name, self.File))\r
self.AddStringToList(Name, Language, Value)\r
continue\r
\r
#\r
# Load multiple .uni files\r
#\r
- def LoadUniFiles(self, FileList = []):\r
+ def LoadUniFiles(self, FileList):\r
if len(FileList) > 0:\r
- if len(FileList) > 1:\r
- NewList = [];\r
- for File in FileList:\r
- NewList.append (File)\r
- NewList.sort()\r
- for File in NewList:\r
- self.LoadUniFile(File)\r
- else:\r
- for File in FileList:\r
- self.LoadUniFile(File)\r
+ for File in FileList:\r
+ self.LoadUniFile(File)\r
\r
#\r
# Add a string to list\r
#\r
def AddStringToList(self, Name, Language, Value, Token = None, Referenced = False, UseOtherLangDef = '', Index = -1):\r
+ for LangNameItem in self.LanguageDef:\r
+ if Language == LangNameItem[0]:\r
+ break\r
+ else:\r
+ EdkLogger.error('Unicode File Parser', FORMAT_NOT_SUPPORTED, "The language '%s' for %s is not defined in Unicode file %s." \\r
+ % (Language, Name, self.File))\r
+ \r
if Language not in self.OrderedStringList:\r
self.OrderedStringList[Language] = []\r
-\r
- IsAdded = False\r
- for Item in self.OrderedStringList[Language]:\r
- if Name == Item.StringName:\r
- IsAdded = True\r
- break\r
- if not IsAdded:\r
+ self.OrderedStringDict[Language] = {}\r
+\r
+ IsAdded = True\r
+ if Name in self.OrderedStringDict[Language]:\r
+ IsAdded = False\r
+ if Value != None:\r
+ ItemIndexInList = self.OrderedStringDict[Language][Name]\r
+ Item = self.OrderedStringList[Language][ItemIndexInList]\r
+ Item.UpdateValue(Value)\r
+ Item.UseOtherLangDef = ''\r
+\r
+ if IsAdded:\r
Token = len(self.OrderedStringList[Language])\r
if Index == -1:\r
self.OrderedStringList[Language].append(StringDefClassObject(Name, Value, Referenced, Token, UseOtherLangDef))\r
+ self.OrderedStringDict[Language][Name] = Token\r
+ for LangName in self.LanguageDef:\r
+ #\r
+ # New STRING token will be added into all language string lists.\r
+ # so that the unique STRING identifier is reserved for all languages in the package list. \r
+ #\r
+ if LangName[0] != Language:\r
+ if UseOtherLangDef != '':\r
+ OtherLangDef = UseOtherLangDef\r
+ else:\r
+ OtherLangDef = Language\r
+ self.OrderedStringList[LangName[0]].append(StringDefClassObject(Name, '', Referenced, Token, OtherLangDef))\r
+ self.OrderedStringDict[LangName[0]][Name] = len(self.OrderedStringList[LangName[0]]) - 1\r
else:\r
self.OrderedStringList[Language].insert(Index, StringDefClassObject(Name, Value, Referenced, Token, UseOtherLangDef))\r
+ self.OrderedStringDict[Language][Name] = Index\r
\r
#\r
# Set the string as referenced\r
#\r
def SetStringReferenced(self, Name):\r
- for Lang in self.OrderedStringList:\r
- for Item in self.OrderedStringList[Lang]:\r
- if Name == Item.StringName:\r
- Item.Referenced = True\r
- break\r
+ #\r
+ # String stoken are added in the same order in all language string lists.\r
+ # So, only update the status of string stoken in first language string list.\r
+ #\r
+ Lang = self.LanguageDef[0][0]\r
+ if Name in self.OrderedStringDict[Lang]:\r
+ ItemIndexInList = self.OrderedStringDict[Lang][Name]\r
+ Item = self.OrderedStringList[Lang][ItemIndexInList]\r
+ Item.Referenced = True\r
+\r
#\r
# Search the string in language definition by Name\r
#\r
def FindStringValue(self, Name, Lang):\r
- for Item in self.OrderedStringList[Lang]:\r
- if Item.StringName == Name:\r
- return Item\r
+ if Name in self.OrderedStringDict[Lang]:\r
+ ItemIndexInList = self.OrderedStringDict[Lang][Name]\r
+ return self.OrderedStringList[Lang][ItemIndexInList]\r
\r
return None\r
\r
#\r
def ReToken(self):\r
#\r
- # Search each string to find if it is defined for each language\r
- # Use secondary language value to replace if missing in any one language\r
+ # Retoken all language strings according to the status of string stoken in the first language string.\r
#\r
- for IndexI in range(0, len(self.LanguageDef)):\r
- LangKey = self.LanguageDef[IndexI][0]\r
- for Item in self.OrderedStringList[LangKey]:\r
- Name = Item.StringName\r
- Value = Item.StringValue[0:-1]\r
- Referenced = Item.Referenced\r
- Index = self.OrderedStringList[LangKey].index(Item)\r
- for IndexJ in range(0, len(self.LanguageDef)):\r
- LangFind = self.LanguageDef[IndexJ][0]\r
- if self.FindStringValue(Name, LangFind) == None:\r
- EdkLogger.debug(EdkLogger.DEBUG_5, Name)\r
- Token = len(self.OrderedStringList[LangFind])\r
- self.AddStringToList(Name, LangFind, Value, Token, Referenced, LangKey, Index)\r
+ FirstLangName = self.LanguageDef[0][0]\r
+\r
+ # Convert the OrderedStringList to be OrderedStringListByToken in order to faciliate future search by token\r
+ for LangNameItem in self.LanguageDef:\r
+ self.OrderedStringListByToken[LangNameItem[0]] = {}\r
\r
#\r
- # Retoken\r
+ # Use small token for all referred string stoken.\r
#\r
- # First re-token the first language\r
- LangName = self.LanguageDef[0][0]\r
- ReferencedStringList = []\r
- NotReferencedStringList = []\r
- Token = 0\r
- for Item in self.OrderedStringList[LangName]:\r
- if Item.Referenced == True:\r
- Item.Token = Token\r
- ReferencedStringList.append(Item)\r
- Token = Token + 1\r
- else:\r
- NotReferencedStringList.append(Item)\r
- self.OrderedStringList[LangName] = ReferencedStringList\r
- for Index in range(len(NotReferencedStringList)):\r
- NotReferencedStringList[Index].Token = Token + Index\r
- self.OrderedStringList[LangName].append(NotReferencedStringList[Index])\r
+ RefToken = 0\r
+ for Index in range (0, len (self.OrderedStringList[FirstLangName])):\r
+ FirstLangItem = self.OrderedStringList[FirstLangName][Index]\r
+ if FirstLangItem.Referenced == True:\r
+ for LangNameItem in self.LanguageDef:\r
+ LangName = LangNameItem[0]\r
+ OtherLangItem = self.OrderedStringList[LangName][Index]\r
+ OtherLangItem.Referenced = True\r
+ OtherLangItem.Token = RefToken\r
+ self.OrderedStringListByToken[LangName][OtherLangItem.Token] = OtherLangItem\r
+ RefToken = RefToken + 1\r
\r
#\r
- # Adjust the orders of other languages\r
+ # Use big token for all unreferred string stoken.\r
#\r
- for IndexOfLanguage in range(1, len(self.LanguageDef)):\r
- for OrderedString in self.OrderedStringList[LangName]:\r
- for UnOrderedString in self.OrderedStringList[self.LanguageDef[IndexOfLanguage][0]]:\r
- if OrderedString.StringName == UnOrderedString.StringName:\r
- UnOrderedString.Token = OrderedString.Token\r
- break\r
+ UnRefToken = 0\r
+ for Index in range (0, len (self.OrderedStringList[FirstLangName])):\r
+ FirstLangItem = self.OrderedStringList[FirstLangName][Index]\r
+ if FirstLangItem.Referenced == False:\r
+ for LangNameItem in self.LanguageDef:\r
+ LangName = LangNameItem[0]\r
+ OtherLangItem = self.OrderedStringList[LangName][Index]\r
+ OtherLangItem.Token = RefToken + UnRefToken\r
+ self.OrderedStringListByToken[LangName][OtherLangItem.Token] = OtherLangItem\r
+ UnRefToken = UnRefToken + 1\r
\r
#\r
# Show the instance itself\r
if __name__ == '__main__':\r
EdkLogger.Initialize()\r
EdkLogger.SetLevel(EdkLogger.DEBUG_0)\r
- a = UniFileClassObject(['C:\\Edk\\Strings.uni', 'C:\\Edk\\Strings2.uni'])\r
+ a = UniFileClassObject([PathClass("C:\\Edk\\Strings.uni"), PathClass("C:\\Edk\\Strings2.uni")])\r
a.ReToken()\r
a.ShowMe()\r