From 8fb5a0cae32261d8650c9d6bb645c10872b8426f Mon Sep 17 00:00:00 2001 From: Jordan Justen Date: Tue, 23 Jun 2015 23:34:38 +0000 Subject: [PATCH] BaseTools/Tests: Verify unsupported UTF-8 data is rejected Surrogate pair characters can be encoded in UTF-8 files, but they are not valid UCS-2 characters. For example, this python interpreter code: >>> import codecs >>> codecs.encode(u'\ud801', 'utf-8') '\xed\xa0\x81' But, the range of 0xd800 - 0xdfff should be rejected as unicode code points because they are reserved for the surrogate pair usage in UTF-16 files. We test that this case is rejected for UTF-8 with and without the UTF-8 BOM. Contributed-under: TianoCore Contribution Agreement 1.0 Signed-off-by: Jordan Justen Reviewed-by: Laszlo Ersek Reviewed-by: Michael D Kinney Reviewed-by: Yingke Liu git-svn-id: https://svn.code.sf.net/p/edk2/code/trunk/edk2@17698 6f19259b-4bc3-4df7-8a09-765794883524 --- BaseTools/Tests/CheckUnicodeSourceFiles.py | 24 ++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/BaseTools/Tests/CheckUnicodeSourceFiles.py b/BaseTools/Tests/CheckUnicodeSourceFiles.py index 102dc3c1f9..2eeb0f551d 100644 --- a/BaseTools/Tests/CheckUnicodeSourceFiles.py +++ b/BaseTools/Tests/CheckUnicodeSourceFiles.py @@ -139,6 +139,30 @@ class Tests(TestTools.BaseToolsTest): self.CheckFile('utf_8', shouldPass=False, string=data) + def testSurrogatePairUnicodeCharInUtf8File(self): + # + # Surrogate Pair code points are used in UTF-16 files to + # encode the Supplementary Plane characters. In UTF-8, it is + # trivial to encode these code points, but they are not valid + # code points for characters, since they are reserved for the + # UTF-16 Surrogate Pairs. + # + # This test makes sure that BaseTools rejects these characters + # if seen in a .uni file. + # + data = '\xed\xa0\x81' + + self.CheckFile(encoding=None, shouldPass=False, string=data) + + def testSurrogatePairUnicodeCharInUtf8FileWithBom(self): + # + # Same test as testSurrogatePairUnicodeCharInUtf8File, but add + # the UTF-8 BOM + # + data = codecs.BOM_UTF8 + '\xed\xa0\x81' + + self.CheckFile(encoding=None, shouldPass=False, string=data) + TheTestSuite = TestTools.MakeTheTestSuite(locals()) if __name__ == '__main__': -- 2.39.2