BaseTools/UniClassObject: Verify valid UCS-2 chars in UTF-16 .uni files

Supplementary Plane characters can exist in UTF-16 files,
but they are not valid UCS-2 characters.

For example, refer to this python interpreter code:
>>> import codecs
>>> codecs.encode(u'\U00010300', 'utf-16')
'\xff\xfe\x00\xd8\x00\xdf'

Therefore the UCS-4 0x00010300 character is encoded as two
16-bit numbers (0xd800 0xdf00) in a little endian UTF-16
file.

For more information, see:
http://en.wikipedia.org/wiki/UTF-16#U.2B10000_to_U.2B10FFFF

This means that our current BaseTools code could be allowing
unsupported UTF-16 characters be used. To fix this, we decode the file
using python's utf-16 decode support. Then we verify that each
character's code point is 0xffff or less.

v3:
 * Based on Mike Kinney's feedback, we now read the whole file and
   verify up-front that it contains valid UCS-2 characters. Thanks
   also to Laszlo Ersek for pointing out the Supplementary Plane
   characters.

v4:
 * Reject code points in 0xd800-0xdfff range since they are reserved
   for UTF-16 surrogate pairs. (lersek)

Contributed-under: TianoCore Contribution Agreement 1.0
Signed-off-by: Jordan Justen <jordan.l.justen@intel.com>
Reviewed-by: Laszlo Ersek <lersek@redhat.com>
Reviewed-by: Michael D Kinney <michael.d.kinney@intel.com>
Reviewed-by: Yingke Liu <yingke.d.liu@intel.com>
git-svn-id: https://svn.code.sf.net/p/edk2/code/trunk/edk2@17694 6f19259b-4bc3-4df7-8a09-765794883524

diff --git a/BaseTools/Source/Python/AutoGen/UniClassObject.py b/BaseTools/Source/Python/AutoGen/UniClassObject.py
index f900eac1d94af4df83093f1d51bb5dde0e11a4c7..d0cbb1d0bb20905a44ff37c4c2ab81ad77d8c96c 100644 (file)
--- a/BaseTools/Source/Python/AutoGen/UniClassObject.py
+++ b/BaseTools/Source/Python/AutoGen/UniClassObject.py
@@ -4,7 +4,7 @@
 #\r
 # Copyright (c) 2014 Hewlett-Packard Development Company, L.P.<BR>\r
 #\r
-# Copyright (c) 2007 - 2014, Intel Corporation. All rights reserved.<BR>\r
+# Copyright (c) 2007 - 2015, Intel Corporation. All rights reserved.<BR>\r
 # This program and the accompanying materials\r
 # are licensed and made available under the terms and conditions of the BSD License\r
 # which accompanies this distribution.  The full text of the license may be found at\r
@@ -19,6 +19,7 @@
 import Common.LongFilePathOs as os, codecs, re\r
 import distutils.util\r
 import Common.EdkLogger as EdkLogger\r
+import StringIO\r
 from Common.BuildToolError import *\r
 from Common.String import GetLineNo\r
 from Common.Misc import PathClass\r
@@ -147,6 +148,37 @@ def GetLanguageCode(LangName, IsCompatibleMode, File):
 \r
     EdkLogger.error("Unicode File Parser", FORMAT_INVALID, "Invalid RFC 4646 language code : %s" % LangName, File)\r
 \r
+## Ucs2Codec\r
+#\r
+# This is only a partial codec implementation. It only supports\r
+# encoding, and is primarily used to check that all the characters are\r
+# valid for UCS-2.\r
+#\r
+class Ucs2Codec(codecs.Codec):\r
+    def __init__(self):\r
+        self.__utf16 = codecs.lookup('utf-16')\r
+\r
+    def encode(self, input, errors='strict'):\r
+        for Char in input:\r
+            CodePoint = ord(Char)\r
+            if CodePoint >= 0xd800 and CodePoint <= 0xdfff:\r
+                raise ValueError("Code Point is in range reserved for " +\r
+                                 "UTF-16 surrogate pairs")\r
+            elif CodePoint > 0xffff:\r
+                raise ValueError("Code Point too large to encode in UCS-2")\r
+        return self.__utf16.encode(input)\r
+\r
+TheUcs2Codec = Ucs2Codec()\r
+def Ucs2Search(name):\r
+    if name == 'ucs-2':\r
+        return codecs.CodecInfo(\r
+            name=name,\r
+            encode=TheUcs2Codec.encode,\r
+            decode=TheUcs2Codec.decode)\r
+    else:\r
+        return None\r
+codecs.register(Ucs2Search)\r
+\r
 ## StringDefClassObject\r
 #\r
 # A structure for language definition\r
@@ -209,7 +241,7 @@ class UniFileClassObject(object):
         Lang = distutils.util.split_quoted((Line.split(u"//")[0]))\r
         if len(Lang) != 3:\r
             try:\r
-                FileIn = codecs.open(LongFilePath(File.Path), mode='rb', encoding='utf-16').read()\r
+                FileIn = self.OpenUniFile(LongFilePath(File.Path))\r
             except UnicodeError, X:\r
                 EdkLogger.error("build", FILE_READ_FAILURE, "File read failure: %s" % str(X), ExtraData=File);\r
             except:\r
@@ -253,6 +285,58 @@ class UniFileClassObject(object):
                     self.OrderedStringDict[LangName][Item.StringName] = len(self.OrderedStringList[LangName]) - 1\r
         return True\r
 \r
+    def OpenUniFile(self, FileName):\r
+        #\r
+        # Read file\r
+        #\r
+        try:\r
+            UniFile = open(FileName, mode='rb')\r
+            FileIn = UniFile.read()\r
+            UniFile.close()\r
+        except:\r
+            EdkLogger.Error("build", FILE_OPEN_FAILURE, ExtraData=File)\r
+\r
+        #\r
+        # We currently only support UTF-16\r
+        #\r
+        Encoding = 'utf-16'\r
+\r
+        self.VerifyUcs2Data(FileIn, FileName, Encoding)\r
+\r
+        UniFile = StringIO.StringIO(FileIn)\r
+        Info = codecs.lookup(Encoding)\r
+        (Reader, Writer) = (Info.streamreader, Info.streamwriter)\r
+        return codecs.StreamReaderWriter(UniFile, Reader, Writer)\r
+\r
+    def VerifyUcs2Data(self, FileIn, FileName, Encoding):\r
+        Ucs2Info = codecs.lookup('ucs-2')\r
+        #\r
+        # Convert to unicode\r
+        #\r
+        try:\r
+            FileDecoded = codecs.decode(FileIn, Encoding)\r
+            Ucs2Info.encode(FileDecoded)\r
+        except:\r
+            UniFile = StringIO.StringIO(FileIn)\r
+            Info = codecs.lookup(Encoding)\r
+            (Reader, Writer) = (Info.streamreader, Info.streamwriter)\r
+            File = codecs.StreamReaderWriter(UniFile, Reader, Writer)\r
+            LineNumber = 0\r
+            ErrMsg = lambda Encoding, LineNumber: \\r
+                     '%s contains invalid %s characters on line %d.' % \\r
+                     (FileName, Encoding, LineNumber)\r
+            while True:\r
+                LineNumber = LineNumber + 1\r
+                try:\r
+                    Line = File.readline()\r
+                    if Line == '':\r
+                        EdkLogger.error('Unicode File Parser', PARSER_ERROR,\r
+                                        ErrMsg(Encoding, LineNumber))\r
+                    Ucs2Info.encode(Line)\r
+                except:\r
+                    EdkLogger.error('Unicode File Parser', PARSER_ERROR,\r
+                                    ErrMsg('UCS-2', LineNumber))\r
+\r
     #\r
     # Get String name and value\r
     #\r
@@ -305,7 +389,7 @@ class UniFileClassObject(object):
             EdkLogger.error("Unicode File Parser", FILE_NOT_FOUND, ExtraData=File.Path)\r
 \r
         try:\r
-            FileIn = codecs.open(LongFilePath(File.Path), mode='rb', encoding='utf-16')\r
+            FileIn = self.OpenUniFile(LongFilePath(File.Path))\r
         except UnicodeError, X:\r
             EdkLogger.error("build", FILE_READ_FAILURE, "File read failure: %s" % str(X), ExtraData=File.Path);\r
         except:\r