From d80e451b187c9d33cbd771253fbd5119670f75c6 Mon Sep 17 00:00:00 2001 From: Jordan Justen Date: Tue, 23 Jun 2015 23:34:19 +0000 Subject: [PATCH] BaseTools/UniClassObject: Verify valid UCS-2 chars in UTF-16 .uni files Supplementary Plane characters can exist in UTF-16 files, but they are not valid UCS-2 characters. For example, refer to this python interpreter code: >>> import codecs >>> codecs.encode(u'\U00010300', 'utf-16') '\xff\xfe\x00\xd8\x00\xdf' Therefore the UCS-4 0x00010300 character is encoded as two 16-bit numbers (0xd800 0xdf00) in a little endian UTF-16 file. For more information, see: http://en.wikipedia.org/wiki/UTF-16#U.2B10000_to_U.2B10FFFF This means that our current BaseTools code could be allowing unsupported UTF-16 characters be used. To fix this, we decode the file using python's utf-16 decode support. Then we verify that each character's code point is 0xffff or less. v3: * Based on Mike Kinney's feedback, we now read the whole file and verify up-front that it contains valid UCS-2 characters. Thanks also to Laszlo Ersek for pointing out the Supplementary Plane characters. v4: * Reject code points in 0xd800-0xdfff range since they are reserved for UTF-16 surrogate pairs. (lersek) Contributed-under: TianoCore Contribution Agreement 1.0 Signed-off-by: Jordan Justen Reviewed-by: Laszlo Ersek Reviewed-by: Michael D Kinney Reviewed-by: Yingke Liu git-svn-id: https://svn.code.sf.net/p/edk2/code/trunk/edk2@17694 6f19259b-4bc3-4df7-8a09-765794883524 --- .../Source/Python/AutoGen/UniClassObject.py | 90 ++++++++++++++++++- 1 file changed, 87 insertions(+), 3 deletions(-) diff --git a/BaseTools/Source/Python/AutoGen/UniClassObject.py b/BaseTools/Source/Python/AutoGen/UniClassObject.py index f900eac1d9..d0cbb1d0bb 100644 --- a/BaseTools/Source/Python/AutoGen/UniClassObject.py +++ b/BaseTools/Source/Python/AutoGen/UniClassObject.py @@ -4,7 +4,7 @@ # # Copyright (c) 2014 Hewlett-Packard Development Company, L.P.
# -# Copyright (c) 2007 - 2014, Intel Corporation. All rights reserved.
+# Copyright (c) 2007 - 2015, Intel Corporation. All rights reserved.
# This program and the accompanying materials # are licensed and made available under the terms and conditions of the BSD License # which accompanies this distribution. The full text of the license may be found at @@ -19,6 +19,7 @@ import Common.LongFilePathOs as os, codecs, re import distutils.util import Common.EdkLogger as EdkLogger +import StringIO from Common.BuildToolError import * from Common.String import GetLineNo from Common.Misc import PathClass @@ -147,6 +148,37 @@ def GetLanguageCode(LangName, IsCompatibleMode, File): EdkLogger.error("Unicode File Parser", FORMAT_INVALID, "Invalid RFC 4646 language code : %s" % LangName, File) +## Ucs2Codec +# +# This is only a partial codec implementation. It only supports +# encoding, and is primarily used to check that all the characters are +# valid for UCS-2. +# +class Ucs2Codec(codecs.Codec): + def __init__(self): + self.__utf16 = codecs.lookup('utf-16') + + def encode(self, input, errors='strict'): + for Char in input: + CodePoint = ord(Char) + if CodePoint >= 0xd800 and CodePoint <= 0xdfff: + raise ValueError("Code Point is in range reserved for " + + "UTF-16 surrogate pairs") + elif CodePoint > 0xffff: + raise ValueError("Code Point too large to encode in UCS-2") + return self.__utf16.encode(input) + +TheUcs2Codec = Ucs2Codec() +def Ucs2Search(name): + if name == 'ucs-2': + return codecs.CodecInfo( + name=name, + encode=TheUcs2Codec.encode, + decode=TheUcs2Codec.decode) + else: + return None +codecs.register(Ucs2Search) + ## StringDefClassObject # # A structure for language definition @@ -209,7 +241,7 @@ class UniFileClassObject(object): Lang = distutils.util.split_quoted((Line.split(u"//")[0])) if len(Lang) != 3: try: - FileIn = codecs.open(LongFilePath(File.Path), mode='rb', encoding='utf-16').read() + FileIn = self.OpenUniFile(LongFilePath(File.Path)) except UnicodeError, X: EdkLogger.error("build", FILE_READ_FAILURE, "File read failure: %s" % str(X), ExtraData=File); except: @@ -253,6 +285,58 @@ class UniFileClassObject(object): self.OrderedStringDict[LangName][Item.StringName] = len(self.OrderedStringList[LangName]) - 1 return True + def OpenUniFile(self, FileName): + # + # Read file + # + try: + UniFile = open(FileName, mode='rb') + FileIn = UniFile.read() + UniFile.close() + except: + EdkLogger.Error("build", FILE_OPEN_FAILURE, ExtraData=File) + + # + # We currently only support UTF-16 + # + Encoding = 'utf-16' + + self.VerifyUcs2Data(FileIn, FileName, Encoding) + + UniFile = StringIO.StringIO(FileIn) + Info = codecs.lookup(Encoding) + (Reader, Writer) = (Info.streamreader, Info.streamwriter) + return codecs.StreamReaderWriter(UniFile, Reader, Writer) + + def VerifyUcs2Data(self, FileIn, FileName, Encoding): + Ucs2Info = codecs.lookup('ucs-2') + # + # Convert to unicode + # + try: + FileDecoded = codecs.decode(FileIn, Encoding) + Ucs2Info.encode(FileDecoded) + except: + UniFile = StringIO.StringIO(FileIn) + Info = codecs.lookup(Encoding) + (Reader, Writer) = (Info.streamreader, Info.streamwriter) + File = codecs.StreamReaderWriter(UniFile, Reader, Writer) + LineNumber = 0 + ErrMsg = lambda Encoding, LineNumber: \ + '%s contains invalid %s characters on line %d.' % \ + (FileName, Encoding, LineNumber) + while True: + LineNumber = LineNumber + 1 + try: + Line = File.readline() + if Line == '': + EdkLogger.error('Unicode File Parser', PARSER_ERROR, + ErrMsg(Encoding, LineNumber)) + Ucs2Info.encode(Line) + except: + EdkLogger.error('Unicode File Parser', PARSER_ERROR, + ErrMsg('UCS-2', LineNumber)) + # # Get String name and value # @@ -305,7 +389,7 @@ class UniFileClassObject(object): EdkLogger.error("Unicode File Parser", FILE_NOT_FOUND, ExtraData=File.Path) try: - FileIn = codecs.open(LongFilePath(File.Path), mode='rb', encoding='utf-16') + FileIn = self.OpenUniFile(LongFilePath(File.Path)) except UnicodeError, X: EdkLogger.error("build", FILE_READ_FAILURE, "File read failure: %s" % str(X), ExtraData=File.Path); except: -- 2.39.2