X-Git-Url: https://git.proxmox.com/?p=mirror_edk2.git;a=blobdiff_plain;f=BaseTools%2FSource%2FPython%2FAutoGen%2FUniClassObject.py;h=06cf3e7d5162b694f38f0a30c295706f19e05534;hp=7b1ce72ea7c0b64b902d15cf6c5ca0704d4dcfcd;hb=5b0671c1e514e534c6d5be9604da33bfc2cd0a24;hpb=4afd3d042215afe68d00b9ab8c32f063a3a1c03f
diff --git a/BaseTools/Source/Python/AutoGen/UniClassObject.py b/BaseTools/Source/Python/AutoGen/UniClassObject.py
index 7b1ce72ea7..06cf3e7d51 100644
--- a/BaseTools/Source/Python/AutoGen/UniClassObject.py
+++ b/BaseTools/Source/Python/AutoGen/UniClassObject.py
@@ -1,4 +1,10 @@
-# Copyright (c) 2007 - 2012, Intel Corporation. All rights reserved.
+## @file
+# This file is used to collect all defined strings in multiple uni files
+#
+#
+# Copyright (c) 2014 Hewlett-Packard Development Company, L.P.
+#
+# Copyright (c) 2007 - 2018, Intel Corporation. All rights reserved.
# This program and the accompanying materials
# are licensed and made available under the terms and conditions of the BSD License
# which accompanies this distribution. The full text of the license may be found at
@@ -7,20 +13,18 @@
# THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS,
# WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED.
-#
-#This file is used to collect all defined strings in multiple uni files
-#
-
##
# Import Modules
#
-import os, codecs, re
+import Common.LongFilePathOs as os, codecs, re
import distutils.util
import Common.EdkLogger as EdkLogger
+import StringIO
from Common.BuildToolError import *
-from Common.String import GetLineNo
+from Common.StringUtils import GetLineNo
from Common.Misc import PathClass
-
+from Common.LongFilePathSupport import LongFilePath
+from Common.GlobalData import *
##
# Static definitions
#
@@ -37,10 +41,7 @@ CR = u'\u000D'
LF = u'\u000A'
NULL = u'\u0000'
TAB = u'\t'
-BACK_SPLASH = u'\\'
-DOBULE_QUOTED_SPLASH = u'\\"'
-SIGLE_QUOTED_SPLASH = u"\\'"
-TAB_BACK_SLASH = u"\\/"
+BACK_SLASH_PLACEHOLDER = u'\u0006'
gIncludePattern = re.compile("^#include +[\"<]+([^\"< >]+)[>\"]+$", re.MULTILINE | re.UNICODE)
@@ -117,13 +118,11 @@ LangConvTable = {'eng':'en', 'fra':'fr', \
# @retval LangName: Valid lanugage code in RFC 4646 format or None
#
def GetLanguageCode(LangName, IsCompatibleMode, File):
- global LangConvTable
-
length = len(LangName)
if IsCompatibleMode:
if length == 3 and LangName.isalpha():
TempLangName = LangConvTable.get(LangName.lower())
- if TempLangName != None:
+ if TempLangName is not None:
return TempLangName
return LangName
else:
@@ -135,7 +134,7 @@ def GetLanguageCode(LangName, IsCompatibleMode, File):
if LangName.isalpha():
return LangName
elif length == 3:
- if LangName.isalpha() and LangConvTable.get(LangName.lower()) == None:
+ if LangName.isalpha() and LangConvTable.get(LangName.lower()) is None:
return LangName
elif length == 5:
if LangName[0:2].isalpha() and LangName[2] == '-':
@@ -143,11 +142,42 @@ def GetLanguageCode(LangName, IsCompatibleMode, File):
elif length >= 6:
if LangName[0:2].isalpha() and LangName[2] == '-':
return LangName
- if LangName[0:3].isalpha() and LangConvTable.get(LangName.lower()) == None and LangName[3] == '-':
+ if LangName[0:3].isalpha() and LangConvTable.get(LangName.lower()) is None and LangName[3] == '-':
return LangName
EdkLogger.error("Unicode File Parser", FORMAT_INVALID, "Invalid RFC 4646 language code : %s" % LangName, File)
+## Ucs2Codec
+#
+# This is only a partial codec implementation. It only supports
+# encoding, and is primarily used to check that all the characters are
+# valid for UCS-2.
+#
+class Ucs2Codec(codecs.Codec):
+ def __init__(self):
+ self.__utf16 = codecs.lookup('utf-16')
+
+ def encode(self, input, errors='strict'):
+ for Char in input:
+ CodePoint = ord(Char)
+ if CodePoint >= 0xd800 and CodePoint <= 0xdfff:
+ raise ValueError("Code Point is in range reserved for " +
+ "UTF-16 surrogate pairs")
+ elif CodePoint > 0xffff:
+ raise ValueError("Code Point too large to encode in UCS-2")
+ return self.__utf16.encode(input)
+
+TheUcs2Codec = Ucs2Codec()
+def Ucs2Search(name):
+ if name == 'ucs-2':
+ return codecs.CodecInfo(
+ name=name,
+ encode=TheUcs2Codec.encode,
+ decode=TheUcs2Codec.decode)
+ else:
+ return None
+codecs.register(Ucs2Search)
+
## StringDefClassObject
#
# A structure for language definition
@@ -163,14 +193,14 @@ class StringDefClassObject(object):
self.UseOtherLangDef = UseOtherLangDef
self.Length = 0
- if Name != None:
+ if Name is not None:
self.StringName = Name
self.StringNameByteList = UniToHexList(Name)
- if Value != None:
+ if Value is not None:
self.StringValue = Value + u'\x00' # Add a NULL at string tail
self.StringValueByteList = UniToHexList(self.StringValue)
self.Length = len(self.StringValueByteList)
- if Token != None:
+ if Token is not None:
self.Token = Token
def __str__(self):
@@ -181,11 +211,24 @@ class StringDefClassObject(object):
repr(self.UseOtherLangDef)
def UpdateValue(self, Value = None):
- if Value != None:
+ if Value is not None:
self.StringValue = Value + u'\x00' # Add a NULL at string tail
self.StringValueByteList = UniToHexList(self.StringValue)
self.Length = len(self.StringValueByteList)
+def StripComments(Line):
+ Comment = u'//'
+ CommentPos = Line.find(Comment)
+ while CommentPos >= 0:
+ # if there are non matched quotes before the comment header
+ # then we are in the middle of a string
+ # but we need to ignore the escaped quotes and backslashes.
+ if ((Line.count(u'"', 0, CommentPos) - Line.count(u'\\"', 0, CommentPos)) & 1) == 1:
+ CommentPos = Line.find (Comment, CommentPos + 1)
+ else:
+ return Line[:CommentPos].strip()
+ return Line.strip()
+
## UniFileClassObject
#
# A structure for .uni file definition
@@ -210,14 +253,14 @@ class UniFileClassObject(object):
Lang = distutils.util.split_quoted((Line.split(u"//")[0]))
if len(Lang) != 3:
try:
- FileIn = codecs.open(File.Path, mode='rb', encoding='utf-16').read()
- except UnicodeError, X:
+ FileIn = UniFileClassObject.OpenUniFile(LongFilePath(File.Path))
+ except UnicodeError as X:
EdkLogger.error("build", FILE_READ_FAILURE, "File read failure: %s" % str(X), ExtraData=File);
except:
EdkLogger.error("build", FILE_OPEN_FAILURE, ExtraData=File);
LineNo = GetLineNo(FileIn, Line, False)
EdkLogger.error("Unicode File Parser", PARSER_ERROR, "Wrong language definition",
- ExtraData="""%s\n\t*Correct format is like '#langdef en-US "English"'""" % Line, File = File, Line = LineNo)
+ ExtraData="""%s\n\t*Correct format is like '#langdef en-US "English"'""" % Line, File=File, Line=LineNo)
else:
LangName = GetLanguageCode(Lang[1], self.IsCompatibleMode, self.File)
LangPrintName = Lang[2]
@@ -254,6 +297,63 @@ class UniFileClassObject(object):
self.OrderedStringDict[LangName][Item.StringName] = len(self.OrderedStringList[LangName]) - 1
return True
+ @staticmethod
+ def OpenUniFile(FileName):
+ #
+ # Read file
+ #
+ try:
+ UniFile = open(FileName, mode='rb')
+ FileIn = UniFile.read()
+ UniFile.close()
+ except:
+ EdkLogger.Error("build", FILE_OPEN_FAILURE, ExtraData=File)
+
+ #
+ # Detect Byte Order Mark at beginning of file. Default to UTF-8
+ #
+ Encoding = 'utf-8'
+ if (FileIn.startswith(codecs.BOM_UTF16_BE) or
+ FileIn.startswith(codecs.BOM_UTF16_LE)):
+ Encoding = 'utf-16'
+
+ UniFileClassObject.VerifyUcs2Data(FileIn, FileName, Encoding)
+
+ UniFile = StringIO.StringIO(FileIn)
+ Info = codecs.lookup(Encoding)
+ (Reader, Writer) = (Info.streamreader, Info.streamwriter)
+ return codecs.StreamReaderWriter(UniFile, Reader, Writer)
+
+ @staticmethod
+ def VerifyUcs2Data(FileIn, FileName, Encoding):
+ Ucs2Info = codecs.lookup('ucs-2')
+ #
+ # Convert to unicode
+ #
+ try:
+ FileDecoded = codecs.decode(FileIn, Encoding)
+ Ucs2Info.encode(FileDecoded)
+ except:
+ UniFile = StringIO.StringIO(FileIn)
+ Info = codecs.lookup(Encoding)
+ (Reader, Writer) = (Info.streamreader, Info.streamwriter)
+ File = codecs.StreamReaderWriter(UniFile, Reader, Writer)
+ LineNumber = 0
+ ErrMsg = lambda Encoding, LineNumber: \
+ '%s contains invalid %s characters on line %d.' % \
+ (FileName, Encoding, LineNumber)
+ while True:
+ LineNumber = LineNumber + 1
+ try:
+ Line = File.readline()
+ if Line == '':
+ EdkLogger.error('Unicode File Parser', PARSER_ERROR,
+ ErrMsg(Encoding, LineNumber))
+ Ucs2Info.encode(Line)
+ except:
+ EdkLogger.error('Unicode File Parser', PARSER_ERROR,
+ ErrMsg('UCS-2', LineNumber))
+
#
# Get String name and value
#
@@ -262,11 +362,11 @@ class UniFileClassObject(object):
Value = ''
Name = Item.split()[1]
- # Check the string name is the upper character
+ # Check the string name
if Name != '':
- MatchString = re.match('[A-Z0-9_]+', Name, re.UNICODE)
- if MatchString == None or MatchString.end(0) != len(Name):
- EdkLogger.error('Unicode File Parser', FORMAT_INVALID, 'The string token name %s defined in UNI file %s contains the invalid lower case character.' %(Name, self.File))
+ MatchString = gIdentifierPattern.match(Name)
+ if MatchString is None:
+ EdkLogger.error('Unicode File Parser', FORMAT_INVALID, 'The string token name %s defined in UNI file %s contains the invalid character.' % (Name, self.File))
LanguageList = Item.split(u'#language ')
for IndexI in range(len(LanguageList)):
if IndexI == 0:
@@ -292,8 +392,8 @@ class UniFileClassObject(object):
EdkLogger.error("Unicode File Parser", FILE_NOT_FOUND, ExtraData=File.Path)
try:
- FileIn = codecs.open(File.Path, mode='rb', encoding='utf-16').readlines()
- except UnicodeError, X:
+ FileIn = UniFileClassObject.OpenUniFile(LongFilePath(File.Path))
+ except UnicodeError as X:
EdkLogger.error("build", FILE_READ_FAILURE, "File read failure: %s" % str(X), ExtraData=File.Path);
except:
EdkLogger.error("build", FILE_OPEN_FAILURE, ExtraData=File.Path);
@@ -302,42 +402,23 @@ class UniFileClassObject(object):
#
# Use unique identifier
#
- FindFlag = -1
- LineCount = 0
for Line in FileIn:
- Line = FileIn[LineCount]
- LineCount += 1
Line = Line.strip()
+ Line = Line.replace(u'\\\\', BACK_SLASH_PLACEHOLDER)
+ Line = StripComments(Line)
+
#
- # Ignore comment line and empty line
+ # Ignore empty line
#
- if Line == u'' or Line.startswith(u'//'):
- continue
+ if len(Line) == 0:
+ continue
- #
- # Process comment embeded in string define lines
- #
- FindFlag = Line.find(u'//')
- if FindFlag != -1:
- Line = Line.replace(Line[FindFlag:], u' ')
- if FileIn[LineCount].strip().startswith('#language'):
- Line = Line + FileIn[LineCount]
- FileIn[LineCount-1] = Line
- FileIn[LineCount] = os.linesep
- LineCount -= 1
- for Index in xrange (LineCount + 1, len (FileIn) - 1):
- if (Index == len(FileIn) -1):
- FileIn[Index] = os.linesep
- else:
- FileIn[Index] = FileIn[Index + 1]
- continue
-
+
Line = Line.replace(u'/langdef', u'#langdef')
Line = Line.replace(u'/string', u'#string')
Line = Line.replace(u'/language', u'#language')
Line = Line.replace(u'/include', u'#include')
- Line = Line.replace(u'\\\\', u'\u0006')
Line = Line.replace(UNICODE_WIDE_CHAR, WIDE_CHAR)
Line = Line.replace(UNICODE_NARROW_CHAR, NARROW_CHAR)
Line = Line.replace(UNICODE_NON_BREAKING_CHAR, NON_BREAKING_CHAR)
@@ -345,17 +426,25 @@ class UniFileClassObject(object):
Line = Line.replace(u'\\r\\n', CR + LF)
Line = Line.replace(u'\\n', CR + LF)
Line = Line.replace(u'\\r', CR)
- Line = Line.replace(u'\\t', u'\t')
- Line = Line.replace(u'''\"''', u'''"''')
+ Line = Line.replace(u'\\t', u' ')
Line = Line.replace(u'\t', u' ')
- Line = Line.replace(u'\u0006', u'\\')
- Line = Line.replace(DOBULE_QUOTED_SPLASH, u'"')
- Line = Line.replace(SIGLE_QUOTED_SPLASH, u"'")
- Line = Line.replace(TAB_BACK_SLASH, u"/")
-
-# if Line.find(u'\\x'):
-# hex = Line[Line.find(u'\\x') + 2 : Line.find(u'\\x') + 6]
-# hex = "u'\\u" + hex + "'"
+ Line = Line.replace(u'\\"', u'"')
+ Line = Line.replace(u"\\'", u"'")
+ Line = Line.replace(BACK_SLASH_PLACEHOLDER, u'\\')
+
+ StartPos = Line.find(u'\\x')
+ while (StartPos != -1):
+ EndPos = Line.find(u'\\', StartPos + 1, StartPos + 7)
+ if EndPos != -1 and EndPos - StartPos == 6 :
+ if g4HexChar.match(Line[StartPos + 2 : EndPos], re.UNICODE):
+ EndStr = Line[EndPos: ]
+ UniStr = ('\u' + (Line[StartPos + 2 : EndPos])).decode('unicode_escape')
+ if EndStr.startswith(u'\\x') and len(EndStr) >= 7:
+ if EndStr[6] == u'\\' and g4HexChar.match(EndStr[2 : 6], re.UNICODE):
+ Line = Line[0 : StartPos] + UniStr + EndStr
+ else:
+ Line = Line[0 : StartPos] + UniStr + EndStr[1:]
+ StartPos = Line.find(u'\\x', StartPos + 1)
IncList = gIncludePattern.findall(Line)
if len(IncList) == 1:
@@ -376,7 +465,7 @@ class UniFileClassObject(object):
# Load a .uni file
#
def LoadUniFile(self, File = None):
- if File == None:
+ if File is None:
EdkLogger.error("Unicode File Parser", PARSER_ERROR, 'No unicode file is given')
self.File = File
#
@@ -429,11 +518,11 @@ class UniFileClassObject(object):
break
# Value = Value.replace(u'\r\n', u'')
Language = GetLanguageCode(Language, self.IsCompatibleMode, self.File)
- # Check the string name is the upper character
+ # Check the string name
if not self.IsCompatibleMode and Name != '':
- MatchString = re.match('[A-Z0-9_]+', Name, re.UNICODE)
- if MatchString == None or MatchString.end(0) != len(Name):
- EdkLogger.error('Unicode File Parser', FORMAT_INVALID, 'The string token name %s defined in UNI file %s contains the invalid lower case character.' %(Name, self.File))
+ MatchString = gIdentifierPattern.match(Name)
+ if MatchString is None:
+ EdkLogger.error('Unicode File Parser', FORMAT_INVALID, 'The string token name %s defined in UNI file %s contains the invalid character.' % (Name, self.File))
self.AddStringToList(Name, Language, Value)
continue
@@ -488,11 +577,11 @@ class UniFileClassObject(object):
IsAdded = True
if Name in self.OrderedStringDict[Language]:
IsAdded = False
- if Value != None:
+ if Value is not None:
ItemIndexInList = self.OrderedStringDict[Language][Name]
Item = self.OrderedStringList[Language][ItemIndexInList]
Item.UpdateValue(Value)
- Item.UseOtherLangDef = ''
+ Item.UseOtherLangDef = ''
if IsAdded:
Token = len(self.OrderedStringList[Language])