#\r
# Copyright (c) 2014 Hewlett-Packard Development Company, L.P.<BR>\r
#\r
-# Copyright (c) 2007 - 2014, Intel Corporation. All rights reserved.<BR>\r
+# Copyright (c) 2007 - 2018, Intel Corporation. All rights reserved.<BR>\r
# This program and the accompanying materials\r
# are licensed and made available under the terms and conditions of the BSD License\r
# which accompanies this distribution. The full text of the license may be found at\r
import Common.LongFilePathOs as os, codecs, re\r
import distutils.util\r
import Common.EdkLogger as EdkLogger\r
+import StringIO\r
from Common.BuildToolError import *\r
from Common.String import GetLineNo\r
from Common.Misc import PathClass\r
from Common.LongFilePathSupport import LongFilePath\r
+from Common.GlobalData import *\r
##\r
# Static definitions\r
#\r
\r
EdkLogger.error("Unicode File Parser", FORMAT_INVALID, "Invalid RFC 4646 language code : %s" % LangName, File)\r
\r
+## Ucs2Codec\r
+#\r
+# This is only a partial codec implementation. It only supports\r
+# encoding, and is primarily used to check that all the characters are\r
+# valid for UCS-2.\r
+#\r
+class Ucs2Codec(codecs.Codec):\r
+ def __init__(self):\r
+ self.__utf16 = codecs.lookup('utf-16')\r
+\r
+ def encode(self, input, errors='strict'):\r
+ for Char in input:\r
+ CodePoint = ord(Char)\r
+ if CodePoint >= 0xd800 and CodePoint <= 0xdfff:\r
+ raise ValueError("Code Point is in range reserved for " +\r
+ "UTF-16 surrogate pairs")\r
+ elif CodePoint > 0xffff:\r
+ raise ValueError("Code Point too large to encode in UCS-2")\r
+ return self.__utf16.encode(input)\r
+\r
+TheUcs2Codec = Ucs2Codec()\r
+def Ucs2Search(name):\r
+ if name == 'ucs-2':\r
+ return codecs.CodecInfo(\r
+ name=name,\r
+ encode=TheUcs2Codec.encode,\r
+ decode=TheUcs2Codec.decode)\r
+ else:\r
+ return None\r
+codecs.register(Ucs2Search)\r
+\r
## StringDefClassObject\r
#\r
# A structure for language definition\r
Lang = distutils.util.split_quoted((Line.split(u"//")[0]))\r
if len(Lang) != 3:\r
try:\r
- FileIn = codecs.open(LongFilePath(File.Path), mode='rb', encoding='utf-16').read()\r
+ FileIn = self.OpenUniFile(LongFilePath(File.Path))\r
except UnicodeError, X:\r
EdkLogger.error("build", FILE_READ_FAILURE, "File read failure: %s" % str(X), ExtraData=File);\r
except:\r
EdkLogger.error("build", FILE_OPEN_FAILURE, ExtraData=File);\r
LineNo = GetLineNo(FileIn, Line, False)\r
EdkLogger.error("Unicode File Parser", PARSER_ERROR, "Wrong language definition",\r
- ExtraData="""%s\n\t*Correct format is like '#langdef en-US "English"'""" % Line, File = File, Line = LineNo)\r
+ ExtraData="""%s\n\t*Correct format is like '#langdef en-US "English"'""" % Line, File=File, Line=LineNo)\r
else:\r
LangName = GetLanguageCode(Lang[1], self.IsCompatibleMode, self.File)\r
LangPrintName = Lang[2]\r
self.OrderedStringDict[LangName][Item.StringName] = len(self.OrderedStringList[LangName]) - 1\r
return True\r
\r
+ def OpenUniFile(self, FileName):\r
+ #\r
+ # Read file\r
+ #\r
+ try:\r
+ UniFile = open(FileName, mode='rb')\r
+ FileIn = UniFile.read()\r
+ UniFile.close()\r
+ except:\r
+ EdkLogger.Error("build", FILE_OPEN_FAILURE, ExtraData=File)\r
+\r
+ #\r
+ # Detect Byte Order Mark at beginning of file. Default to UTF-8\r
+ #\r
+ Encoding = 'utf-8'\r
+ if (FileIn.startswith(codecs.BOM_UTF16_BE) or\r
+ FileIn.startswith(codecs.BOM_UTF16_LE)):\r
+ Encoding = 'utf-16'\r
+\r
+ self.VerifyUcs2Data(FileIn, FileName, Encoding)\r
+\r
+ UniFile = StringIO.StringIO(FileIn)\r
+ Info = codecs.lookup(Encoding)\r
+ (Reader, Writer) = (Info.streamreader, Info.streamwriter)\r
+ return codecs.StreamReaderWriter(UniFile, Reader, Writer)\r
+\r
+ def VerifyUcs2Data(self, FileIn, FileName, Encoding):\r
+ Ucs2Info = codecs.lookup('ucs-2')\r
+ #\r
+ # Convert to unicode\r
+ #\r
+ try:\r
+ FileDecoded = codecs.decode(FileIn, Encoding)\r
+ Ucs2Info.encode(FileDecoded)\r
+ except:\r
+ UniFile = StringIO.StringIO(FileIn)\r
+ Info = codecs.lookup(Encoding)\r
+ (Reader, Writer) = (Info.streamreader, Info.streamwriter)\r
+ File = codecs.StreamReaderWriter(UniFile, Reader, Writer)\r
+ LineNumber = 0\r
+ ErrMsg = lambda Encoding, LineNumber: \\r
+ '%s contains invalid %s characters on line %d.' % \\r
+ (FileName, Encoding, LineNumber)\r
+ while True:\r
+ LineNumber = LineNumber + 1\r
+ try:\r
+ Line = File.readline()\r
+ if Line == '':\r
+ EdkLogger.error('Unicode File Parser', PARSER_ERROR,\r
+ ErrMsg(Encoding, LineNumber))\r
+ Ucs2Info.encode(Line)\r
+ except:\r
+ EdkLogger.error('Unicode File Parser', PARSER_ERROR,\r
+ ErrMsg('UCS-2', LineNumber))\r
+\r
#\r
# Get String name and value\r
#\r
Value = ''\r
\r
Name = Item.split()[1]\r
- # Check the string name is the upper character\r
+ # Check the string name\r
if Name != '':\r
- MatchString = re.match('[A-Z0-9_]+', Name, re.UNICODE)\r
+ MatchString = re.match('^[a-zA-Z][a-zA-Z0-9_]*$', Name, re.UNICODE)\r
if MatchString == None or MatchString.end(0) != len(Name):\r
- EdkLogger.error('Unicode File Parser', FORMAT_INVALID, 'The string token name %s defined in UNI file %s contains the invalid lower case character.' %(Name, self.File))\r
+ EdkLogger.error('Unicode File Parser', FORMAT_INVALID, 'The string token name %s defined in UNI file %s contains the invalid character.' % (Name, self.File))\r
LanguageList = Item.split(u'#language ')\r
for IndexI in range(len(LanguageList)):\r
if IndexI == 0:\r
if ((Line.count(u'"', 0, CommentPos) - Line.count(u'\\"', 0, CommentPos)) & 1) == 1:\r
CommentPos = Line.find (Comment, CommentPos + 1)\r
else:\r
- return Line[:CommentPos]\r
- return Line\r
+ return Line[:CommentPos].strip()\r
+ return Line.strip()\r
\r
\r
#\r
EdkLogger.error("Unicode File Parser", FILE_NOT_FOUND, ExtraData=File.Path)\r
\r
try:\r
- FileIn = codecs.open(LongFilePath(File.Path), mode='rb', encoding='utf-16')\r
+ FileIn = self.OpenUniFile(LongFilePath(File.Path))\r
except UnicodeError, X:\r
EdkLogger.error("build", FILE_READ_FAILURE, "File read failure: %s" % str(X), ExtraData=File.Path);\r
except:\r
Line = Line.replace(u"\\'", u"'") \r
Line = Line.replace(BACK_SLASH_PLACEHOLDER, u'\\')\r
\r
-# if Line.find(u'\\x'):\r
-# hex = Line[Line.find(u'\\x') + 2 : Line.find(u'\\x') + 6]\r
-# hex = "u'\\u" + hex + "'"\r
+ StartPos = Line.find(u'\\x')\r
+ while (StartPos != -1):\r
+ EndPos = Line.find(u'\\', StartPos + 1, StartPos + 7)\r
+ if EndPos != -1 and EndPos - StartPos == 6 :\r
+ if g4HexChar.match(Line[StartPos + 2 : EndPos], re.UNICODE):\r
+ EndStr = Line[EndPos: ]\r
+ UniStr = ('\u' + (Line[StartPos + 2 : EndPos])).decode('unicode_escape')\r
+ if EndStr.startswith(u'\\x') and len(EndStr) >= 7:\r
+ if EndStr[6] == u'\\' and g4HexChar.match(EndStr[2 : 6], re.UNICODE):\r
+ Line = Line[0 : StartPos] + UniStr + EndStr\r
+ else:\r
+ Line = Line[0 : StartPos] + UniStr + EndStr[1:]\r
+ StartPos = Line.find(u'\\x', StartPos + 1)\r
\r
IncList = gIncludePattern.findall(Line)\r
if len(IncList) == 1:\r
break\r
# Value = Value.replace(u'\r\n', u'')\r
Language = GetLanguageCode(Language, self.IsCompatibleMode, self.File)\r
- # Check the string name is the upper character\r
+ # Check the string name\r
if not self.IsCompatibleMode and Name != '':\r
- MatchString = re.match('[A-Z0-9_]+', Name, re.UNICODE)\r
+ MatchString = re.match('^[a-zA-Z][a-zA-Z0-9_]*$', Name, re.UNICODE)\r
if MatchString == None or MatchString.end(0) != len(Name):\r
- EdkLogger.error('Unicode File Parser', FORMAT_INVALID, 'The string token name %s defined in UNI file %s contains the invalid lower case character.' %(Name, self.File))\r
+ EdkLogger.error('Unicode File Parser', FORMAT_INVALID, 'The string token name %s defined in UNI file %s contains the invalid character.' % (Name, self.File))\r
self.AddStringToList(Name, Language, Value)\r
continue\r
\r
ItemIndexInList = self.OrderedStringDict[Language][Name]\r
Item = self.OrderedStringList[Language][ItemIndexInList]\r
Item.UpdateValue(Value)\r
- Item.UseOtherLangDef = '' \r
+ Item.UseOtherLangDef = ''\r
\r
if IsAdded:\r
Token = len(self.OrderedStringList[Language])\r
projects / mirror_edk2.git / blobdiff