BaseTools/Tests: Verify unsupported UTF-8 data is rejected

[mirror_edk2.git] / BaseTools / Tests / CheckUnicodeSourceFiles.py

## @file\r
#  Unit tests for AutoGen.UniClassObject\r
#\r
#  Copyright (c) 2015, Intel Corporation. All rights reserved.<BR>\r
#\r
#  This program and the accompanying materials\r
#  are licensed and made available under the terms and conditions of the BSD License\r
#  which accompanies this distribution.  The full text of the license may be found at\r
#  http://opensource.org/licenses/bsd-license.php\r
#\r
#  THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS,\r
#  WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED.\r
#\r
\r
##\r
# Import Modules\r
#\r
import os\r
import unittest\r
\r
import codecs\r
\r
import TestTools\r
\r
from Common.Misc import PathClass\r
import AutoGen.UniClassObject as BtUni\r
\r
from Common import EdkLogger\r
EdkLogger.InitializeForUnitTest()\r
\r
class Tests(TestTools.BaseToolsTest):\r
\r
    SampleData = u'''\r
        #langdef en-US "English"\r
        #string STR_A #language en-US "STR_A for en-US"\r
    '''\r
\r
    def EncodeToFile(self, encoding, string=None):\r
        if string is None:\r
            string = self.SampleData\r
        if encoding is not None:\r
            data = codecs.encode(string, encoding)\r
        else:\r
            data = string\r
        path = 'input.uni'\r
        self.WriteTmpFile(path, data)\r
        return PathClass(self.GetTmpFilePath(path))\r
\r
    def ErrorFailure(self, error, encoding, shouldPass):\r
        msg = error + ' should '\r
        if shouldPass:\r
            msg += 'not '\r
        msg += 'be generated for '\r
        msg += '%s data in a .uni file' % encoding\r
        self.fail(msg)\r
\r
    def UnicodeErrorFailure(self, encoding, shouldPass):\r
        self.ErrorFailure('UnicodeError', encoding, shouldPass)\r
\r
    def EdkErrorFailure(self, encoding, shouldPass):\r
        self.ErrorFailure('EdkLogger.FatalError', encoding, shouldPass)\r
\r
    def CheckFile(self, encoding, shouldPass, string=None):\r
        path = self.EncodeToFile(encoding, string)\r
        try:\r
            BtUni.UniFileClassObject([path])\r
            if shouldPass:\r
                return\r
        except UnicodeError:\r
            if not shouldPass:\r
                return\r
            else:\r
                self.UnicodeErrorFailure(encoding, shouldPass)\r
        except EdkLogger.FatalError:\r
            if not shouldPass:\r
                return\r
            else:\r
                self.EdkErrorFailure(encoding, shouldPass)\r
        except Exception:\r
            pass\r
\r
        self.EdkErrorFailure(encoding, shouldPass)\r
\r
    def testUtf16InUniFile(self):\r
        self.CheckFile('utf_16', shouldPass=True)\r
\r
    def testSupplementaryPlaneUnicodeCharInUtf16File(self):\r
        #\r
        # Supplementary Plane characters can exist in UTF-16 files,\r
        # but they are not valid UCS-2 characters.\r
        #\r
        # This test makes sure that BaseTools rejects these characters\r
        # if seen in a .uni file.\r
        #\r
        data = u'''\r
            #langdef en-US "English"\r
            #string STR_A #language en-US "CodePoint (\U00010300) > 0xFFFF"\r
        '''\r
\r
        self.CheckFile('utf_16', shouldPass=False, string=data)\r
\r
    def testSurrogatePairUnicodeCharInUtf16File(self):\r
        #\r
        # Surrogate Pair code points are used in UTF-16 files to\r
        # encode the Supplementary Plane characters. But, a Surrogate\r
        # Pair code point which is not followed by another Surrogate\r
        # Pair code point might be interpreted as a single code point\r
        # with the Surrogate Pair code point.\r
        #\r
        # This test makes sure that BaseTools rejects these characters\r
        # if seen in a .uni file.\r
        #\r
        data = codecs.BOM_UTF16_LE + '//\x01\xd8 '\r
\r
        self.CheckFile(encoding=None, shouldPass=False, string=data)\r
\r
    def test32bitUnicodeCharInUtf8File(self):\r
        data = u'''\r
            #langdef en-US "English"\r
            #string STR_A #language en-US "CodePoint (\U00010300) > 0xFFFF"\r
        '''\r
\r
        self.CheckFile('utf_16', shouldPass=False, string=data)\r
\r
    def test32bitUnicodeCharInUtf8File(self):\r
        data = u'''\r
            #langdef en-US "English"\r
            #string STR_A #language en-US "CodePoint (\U00010300) > 0xFFFF"\r
        '''\r
\r
        self.CheckFile('utf_8', shouldPass=False, string=data)\r
\r
    def test32bitUnicodeCharInUtf8Comment(self):\r
        data = u'''\r
            // Even in comments, we reject non-UCS-2 chars: \U00010300\r
            #langdef en-US "English"\r
            #string STR_A #language en-US "A"\r
        '''\r
\r
        self.CheckFile('utf_8', shouldPass=False, string=data)\r
\r
    def testSurrogatePairUnicodeCharInUtf8File(self):\r
        #\r
        # Surrogate Pair code points are used in UTF-16 files to\r
        # encode the Supplementary Plane characters. In UTF-8, it is\r
        # trivial to encode these code points, but they are not valid\r
        # code points for characters, since they are reserved for the\r
        # UTF-16 Surrogate Pairs.\r
        #\r
        # This test makes sure that BaseTools rejects these characters\r
        # if seen in a .uni file.\r
        #\r
        data = '\xed\xa0\x81'\r
\r
        self.CheckFile(encoding=None, shouldPass=False, string=data)\r
\r
    def testSurrogatePairUnicodeCharInUtf8FileWithBom(self):\r
        #\r
        # Same test as testSurrogatePairUnicodeCharInUtf8File, but add\r
        # the UTF-8 BOM\r
        #\r
        data = codecs.BOM_UTF8 + '\xed\xa0\x81'\r
\r
        self.CheckFile(encoding=None, shouldPass=False, string=data)\r
\r
TheTestSuite = TestTools.MakeTheTestSuite(locals())\r
\r
if __name__ == '__main__':\r
    allTests = TheTestSuite()\r
    unittest.TextTestRunner().run(allTests)\r