]>
git.proxmox.com Git - ceph.git/blob - ceph/src/jaegertracing/opentelemetry-cpp/third_party/nlohmann-json/test/src/unit-unicode1.cpp
3 __| | __| | | | JSON for Modern C++ (test suite)
4 | | |__ | | | | | | version 3.10.5
5 |_____|_____|_____|_|___| https://github.com/nlohmann/json
7 Licensed under the MIT License <http://opensource.org/licenses/MIT>.
8 SPDX-License-Identifier: MIT
9 Copyright (c) 2013-2022 Niels Lohmann <http://nlohmann.me>.
11 Permission is hereby granted, free of charge, to any person obtaining a copy
12 of this software and associated documentation files (the "Software"), to deal
13 in the Software without restriction, including without limitation the rights
14 to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
15 copies of the Software, and to permit persons to whom the Software is
16 furnished to do so, subject to the following conditions:
18 The above copyright notice and this permission notice shall be included in all
19 copies or substantial portions of the Software.
21 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
22 IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
23 FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
24 AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
25 LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
26 OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30 #include "doctest_compatibility.h"
32 // for some reason including this after the json header leads to linker errors with VS 2017...
34 #include <nlohmann/json.hpp>
40 #include <test_data.hpp>
42 TEST_CASE ( "Unicode (1/5)" * doctest :: skip ())
44 SECTION ( " \\ uxxxx sequences" )
46 // create an escaped string from a code point
47 const auto codepoint_to_unicode
= []( std :: size_t cp
)
49 // code points are represented as a six-character sequence: a
50 // reverse solidus, followed by the lowercase letter u, followed
51 // by four hexadecimal digits that encode the character's code
54 ss
<< " \\ u" << std :: setw ( 4 ) << std :: setfill ( '0' ) << std :: hex
<< cp
;
58 SECTION ( "correct sequences" )
60 // generate all UTF-8 code points; in total, 1112064 code points are
61 // generated: 0x1FFFFF code points - 2048 invalid values between
63 for ( std :: size_t cp
= 0 ; cp
<= 0x10FFFF u
; ++ cp
)
65 // string to store the code point as in \uxxxx format
66 std :: string json_text
= " \" " ;
68 // decide whether to use one or two \uxxxx sequences
71 // The Unicode standard permanently reserves these code point
72 // values for UTF-16 encoding of the high and low surrogates, and
73 // they will never be assigned a character, so there should be no
74 // reason to encode them. The official Unicode standard says that
75 // no UTF forms, including UTF-16, can encode these code points.
76 if ( cp
>= 0xD800 u
&& cp
<= 0xDFFF u
)
78 // if we would not skip these code points, we would get a
79 // "missing low surrogate" exception
83 // code points in the Basic Multilingual Plane can be
84 // represented with one \uxxxx sequence
85 json_text
+= codepoint_to_unicode ( cp
);
89 // To escape an extended character that is not in the Basic
90 // Multilingual Plane, the character is represented as a
91 // 12-character sequence, encoding the UTF-16 surrogate pair
92 const auto codepoint1
= 0xd800 u
+ ((( cp
- 0x10000 u
) >> 10 ) & 0x3ff u
);
93 const auto codepoint2
= 0xdc00 u
+ (( cp
- 0x10000 u
) & 0x3ff u
);
94 json_text
+= codepoint_to_unicode ( codepoint1
) + codepoint_to_unicode ( codepoint2
);
100 CHECK_NOTHROW ( _
= json :: parse ( json_text
));
104 SECTION ( "incorrect sequences" )
106 SECTION ( "incorrect surrogate values" )
110 CHECK_THROWS_AS ( _
= json :: parse ( " \"\\ uDC00 \\ uDC00 \" " ), json :: parse_error
&);
111 CHECK_THROWS_WITH ( _
= json :: parse ( " \"\\ uDC00 \\ uDC00 \" " ),
112 "[json.exception.parse_error.101] parse error at line 1, column 7: syntax error while parsing value - invalid string: surrogate U+DC00..U+DFFF must follow U+D800..U+DBFF; last read: ' \"\\ uDC00'" );
114 CHECK_THROWS_AS ( _
= json :: parse ( " \"\\ uD7FF \\ uDC00 \" " ), json :: parse_error
&);
115 CHECK_THROWS_WITH ( _
= json :: parse ( " \"\\ uD7FF \\ uDC00 \" " ),
116 "[json.exception.parse_error.101] parse error at line 1, column 13: syntax error while parsing value - invalid string: surrogate U+DC00..U+DFFF must follow U+D800..U+DBFF; last read: ' \"\\ uD7FF \\ uDC00'" );
118 CHECK_THROWS_AS ( _
= json :: parse ( " \"\\ uD800] \" " ), json :: parse_error
&);
119 CHECK_THROWS_WITH ( _
= json :: parse ( " \"\\ uD800] \" " ),
120 "[json.exception.parse_error.101] parse error at line 1, column 8: syntax error while parsing value - invalid string: surrogate U+D800..U+DBFF must be followed by U+DC00..U+DFFF; last read: ' \"\\ uD800]'" );
122 CHECK_THROWS_AS ( _
= json :: parse ( " \"\\ uD800 \\ v \" " ), json :: parse_error
&);
123 CHECK_THROWS_WITH ( _
= json :: parse ( " \"\\ uD800 \\ v \" " ),
124 "[json.exception.parse_error.101] parse error at line 1, column 9: syntax error while parsing value - invalid string: surrogate U+D800..U+DBFF must be followed by U+DC00..U+DFFF; last read: ' \"\\ uD800 \\ v'" );
126 CHECK_THROWS_AS ( _
= json :: parse ( " \"\\ uD800 \\ u123 \" " ), json :: parse_error
&);
127 CHECK_THROWS_WITH ( _
= json :: parse ( " \"\\ uD800 \\ u123 \" " ),
128 "[json.exception.parse_error.101] parse error at line 1, column 13: syntax error while parsing value - invalid string: ' \\ u' must be followed by 4 hex digits; last read: ' \"\\ uD800 \\ u123 \" '" );
130 CHECK_THROWS_AS ( _
= json :: parse ( " \"\\ uD800 \\ uDBFF \" " ), json :: parse_error
&);
131 CHECK_THROWS_WITH ( _
= json :: parse ( " \"\\ uD800 \\ uDBFF \" " ),
132 "[json.exception.parse_error.101] parse error at line 1, column 13: syntax error while parsing value - invalid string: surrogate U+D800..U+DBFF must be followed by U+DC00..U+DFFF; last read: ' \"\\ uD800 \\ uDBFF'" );
134 CHECK_THROWS_AS ( _
= json :: parse ( " \"\\ uD800 \\ uE000 \" " ), json :: parse_error
&);
135 CHECK_THROWS_WITH ( _
= json :: parse ( " \"\\ uD800 \\ uE000 \" " ),
136 "[json.exception.parse_error.101] parse error at line 1, column 13: syntax error while parsing value - invalid string: surrogate U+D800..U+DBFF must be followed by U+DC00..U+DFFF; last read: ' \"\\ uD800 \\ uE000'" );
141 SECTION ( "incorrect sequences" )
143 SECTION ( "high surrogate without low surrogate" )
145 // D800..DBFF are high surrogates and must be followed by low
146 // surrogates DC00..DFFF; here, nothing follows
147 for ( std :: size_t cp
= 0xD800 u
; cp
<= 0xDBFF u
; ++ cp
)
149 std :: string json_text
= " \" " + codepoint_to_unicode ( cp
) + " \" " ;
151 CHECK_THROWS_AS ( json :: parse ( json_text
), json :: parse_error
&);
155 SECTION ( "high surrogate with wrong low surrogate" )
157 // D800..DBFF are high surrogates and must be followed by low
158 // surrogates DC00..DFFF; here a different sequence follows
159 for ( std :: size_t cp1
= 0xD800 u
; cp1
<= 0xDBFF u
; ++ cp1
)
161 for ( std :: size_t cp2
= 0x0000 u
; cp2
<= 0xFFFF u
; ++ cp2
)
163 if ( 0xDC00 u
<= cp2
&& cp2
<= 0xDFFF u
)
168 std :: string json_text
= " \" " + codepoint_to_unicode ( cp1
) + codepoint_to_unicode ( cp2
) + " \" " ;
170 CHECK_THROWS_AS ( json :: parse ( json_text
), json :: parse_error
&);
175 SECTION ( "low surrogate without high surrogate" )
177 // low surrogates DC00..DFFF must follow high surrogates; here,
179 for ( std :: size_t cp
= 0xDC00 u
; cp
<= 0xDFFF u
; ++ cp
)
181 std :: string json_text
= " \" " + codepoint_to_unicode ( cp
) + " \" " ;
183 CHECK_THROWS_AS ( json :: parse ( json_text
), json :: parse_error
&);
191 SECTION ( "read all unicode characters" )
193 // read a file with all unicode characters stored as single-character
194 // strings in a JSON array
195 std :: ifstream
f ( TEST_DATA_DIRECTORY
"/json_nlohmann_tests/all_unicode.json" );
197 CHECK_NOTHROW ( f
>> j
);
199 // the array has 1112064 + 1 elements (a terminating "null" value)
200 // Note: 1112064 = 0x1FFFFF code points - 2048 invalid values between
201 // 0xD800 and 0xDFFF.
202 CHECK ( j
. size () == 1112065 );
204 SECTION ( "check JSON Pointers" )
206 for ( const auto & s
: j
)
208 // skip non-string JSON values
214 auto ptr
= s
. get
< std :: string
>();
216 // tilde must be followed by 0 or 1
222 // JSON Pointers must begin with "/"
225 CHECK_NOTHROW ( json :: json_pointer ( "/" + ptr
));
227 // check escape/unescape roundtrip
228 auto escaped
= nlohmann :: detail :: escape ( ptr
);
229 nlohmann :: detail :: unescape ( escaped
);
230 CHECK ( escaped
== ptr
);
235 SECTION ( "ignore byte-order-mark" )
237 SECTION ( "in a stream" )
239 // read a file with a UTF-8 BOM
240 std :: ifstream
f ( TEST_DATA_DIRECTORY
"/json_nlohmann_tests/bom.json" );
242 CHECK_NOTHROW ( f
>> j
);
245 SECTION ( "with an iterator" )
247 std :: string i
= " \xef\xbb\xbf { \n \" foo \" : true \n }" ;
249 CHECK_NOTHROW ( _
= json :: parse ( i
. begin (), i
. end ()));
253 SECTION ( "error for incomplete/wrong BOM" )
256 CHECK_THROWS_AS ( _
= json :: parse ( " \xef\xbb " ), json :: parse_error
&);
257 CHECK_THROWS_AS ( _
= json :: parse ( " \xef\xbb\xbb " ), json :: parse_error
&);
263 void roundtrip ( bool success_expected
, const std :: string
& s
);
265 void roundtrip ( bool success_expected
, const std :: string
& s
)
270 // create JSON string value
273 std :: string ps
= std :: string ( " \" " ) + s
+ " \" " ;
275 if ( success_expected
)
277 // serialization succeeds
278 CHECK_NOTHROW ( j
. dump ());
280 // exclude parse test for U+0000
283 // parsing JSON text succeeds
284 CHECK_NOTHROW ( _
= json :: parse ( ps
));
287 // roundtrip succeeds
288 CHECK_NOTHROW ( _
= json :: parse ( j
. dump ()));
290 // after roundtrip, the same string is stored
291 json jr
= json :: parse ( j
. dump ());
292 CHECK ( jr
. get
< std :: string
>() == s
);
296 // serialization fails
297 CHECK_THROWS_AS ( j
. dump (), json :: type_error
&);
299 // parsing JSON text fails
300 CHECK_THROWS_AS ( _
= json :: parse ( ps
), json :: parse_error
&);
305 TEST_CASE ( "Markus Kuhn's UTF-8 decoder capability and stress test" )
307 // Markus Kuhn <http://www.cl.cam.ac.uk/~mgk25/> - 2015-08-28 - CC BY 4.0
308 // http://www.cl.cam.ac.uk/~mgk25/ucs/examples/UTF-8-test.txt
310 SECTION ( "1 Some correct UTF-8 text" )
312 roundtrip ( true , "κόσμε" );
315 SECTION ( "2 Boundary condition test cases" )
317 SECTION ( "2.1 First possible sequence of a certain length" )
319 // 2.1.1 1 byte (U-00000000)
320 roundtrip ( true , std :: string ( "\0" , 1 ));
321 // 2.1.2 2 bytes (U-00000080)
322 roundtrip ( true , " \xc2\x80 " );
323 // 2.1.3 3 bytes (U-00000800)
324 roundtrip ( true , " \xe0\xa0\x80 " );
325 // 2.1.4 4 bytes (U-00010000)
326 roundtrip ( true , " \xf0\x90\x80\x80 " );
328 // 2.1.5 5 bytes (U-00200000)
329 roundtrip ( false , " \xF8\x88\x80\x80\x80 " );
330 // 2.1.6 6 bytes (U-04000000)
331 roundtrip ( false , " \xFC\x84\x80\x80\x80\x80 " );
334 SECTION ( "2.2 Last possible sequence of a certain length" )
336 // 2.2.1 1 byte (U-0000007F)
337 roundtrip ( true , " \x7f " );
338 // 2.2.2 2 bytes (U-000007FF)
339 roundtrip ( true , " \xdf\xbf " );
340 // 2.2.3 3 bytes (U-0000FFFF)
341 roundtrip ( true , " \xef\xbf\xbf " );
343 // 2.2.4 4 bytes (U-001FFFFF)
344 roundtrip ( false , " \xF7\xBF\xBF\xBF " );
345 // 2.2.5 5 bytes (U-03FFFFFF)
346 roundtrip ( false , " \xFB\xBF\xBF\xBF\xBF " );
347 // 2.2.6 6 bytes (U-7FFFFFFF)
348 roundtrip ( false , " \xFD\xBF\xBF\xBF\xBF\xBF " );
351 SECTION ( "2.3 Other boundary conditions" )
353 // 2.3.1 U-0000D7FF = ed 9f bf
354 roundtrip ( true , " \xed\x9f\xbf " );
355 // 2.3.2 U-0000E000 = ee 80 80
356 roundtrip ( true , " \xee\x80\x80 " );
357 // 2.3.3 U-0000FFFD = ef bf bd
358 roundtrip ( true , " \xef\xbf\xbd " );
359 // 2.3.4 U-0010FFFF = f4 8f bf bf
360 roundtrip ( true , " \xf4\x8f\xbf\xbf " );
362 // 2.3.5 U-00110000 = f4 90 80 80
363 roundtrip ( false , " \xf4\x90\x80\x80 " );
367 SECTION ( "3 Malformed sequences" )
369 SECTION ( "3.1 Unexpected continuation bytes" )
371 // Each unexpected continuation byte should be separately signalled as a
372 // malformed sequence of its own.
374 // 3.1.1 First continuation byte 0x80
375 roundtrip ( false , " \x80 " );
376 // 3.1.2 Last continuation byte 0xbf
377 roundtrip ( false , " \xbf " );
379 // 3.1.3 2 continuation bytes
380 roundtrip ( false , " \x80\xbf " );
381 // 3.1.4 3 continuation bytes
382 roundtrip ( false , " \x80\xbf\x80 " );
383 // 3.1.5 4 continuation bytes
384 roundtrip ( false , " \x80\xbf\x80\xbf " );
385 // 3.1.6 5 continuation bytes
386 roundtrip ( false , " \x80\xbf\x80\xbf\x80 " );
387 // 3.1.7 6 continuation bytes
388 roundtrip ( false , " \x80\xbf\x80\xbf\x80\xbf " );
389 // 3.1.8 7 continuation bytes
390 roundtrip ( false , " \x80\xbf\x80\xbf\x80\xbf\x80 " );
392 // 3.1.9 Sequence of all 64 possible continuation bytes (0x80-0xbf)
393 roundtrip ( false , " \x80\x81\x82\x83\x84\x85\x86\x87\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f\x90\x91\x92\x93\x94\x95\x96\x97\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7\xa8\xa9\xaa\xab\xac\xad\xae\xaf\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf " );
396 SECTION ( "3.2 Lonely start characters" )
398 // 3.2.1 All 32 first bytes of 2-byte sequences (0xc0-0xdf)
399 roundtrip ( false , " \xc0 \xc1 \xc2 \xc3 \xc4 \xc5 \xc6 \xc7 \xc8 \xc9 \xca \xcb \xcc \xcd \xce \xcf \xd0 \xd1 \xd2 \xd3 \xd4 \xd5 \xd6 \xd7 \xd8 \xd9 \xda \xdb \xdc \xdd \xde \xdf " );
400 // 3.2.2 All 16 first bytes of 3-byte sequences (0xe0-0xef)
401 roundtrip ( false , " \xe0 \xe1 \xe2 \xe3 \xe4 \xe5 \xe6 \xe7 \xe8 \xe9 \xea \xeb \xec \xed \xee \xef " );
402 // 3.2.3 All 8 first bytes of 4-byte sequences (0xf0-0xf7)
403 roundtrip ( false , " \xf0 \xf1 \xf2 \xf3 \xf4 \xf5 \xf6 \xf7 " );
404 // 3.2.4 All 4 first bytes of 5-byte sequences (0xf8-0xfb)
405 roundtrip ( false , " \xf8 \xf9 \xfa \xfb " );
406 // 3.2.5 All 2 first bytes of 6-byte sequences (0xfc-0xfd)
407 roundtrip ( false , " \xfc \xfd " );
410 SECTION ( "3.3 Sequences with last continuation byte missing" )
412 // All bytes of an incomplete sequence should be signalled as a single
413 // malformed sequence, i.e., you should see only a single replacement
414 // character in each of the next 10 tests. (Characters as in section 2)
416 // 3.3.1 2-byte sequence with last byte missing (U+0000)
417 roundtrip ( false , " \xc0 " );
418 // 3.3.2 3-byte sequence with last byte missing (U+0000)
419 roundtrip ( false , " \xe0\x80 " );
420 // 3.3.3 4-byte sequence with last byte missing (U+0000)
421 roundtrip ( false , " \xf0\x80\x80 " );
422 // 3.3.4 5-byte sequence with last byte missing (U+0000)
423 roundtrip ( false , " \xf8\x80\x80\x80 " );
424 // 3.3.5 6-byte sequence with last byte missing (U+0000)
425 roundtrip ( false , " \xfc\x80\x80\x80\x80 " );
426 // 3.3.6 2-byte sequence with last byte missing (U-000007FF)
427 roundtrip ( false , " \xdf " );
428 // 3.3.7 3-byte sequence with last byte missing (U-0000FFFF)
429 roundtrip ( false , " \xef\xbf " );
430 // 3.3.8 4-byte sequence with last byte missing (U-001FFFFF)
431 roundtrip ( false , " \xf7\xbf\xbf " );
432 // 3.3.9 5-byte sequence with last byte missing (U-03FFFFFF)
433 roundtrip ( false , " \xfb\xbf\xbf\xbf " );
434 // 3.3.10 6-byte sequence with last byte missing (U-7FFFFFFF)
435 roundtrip ( false , " \xfd\xbf\xbf\xbf\xbf " );
438 SECTION ( "3.4 Concatenation of incomplete sequences" )
440 // All the 10 sequences of 3.3 concatenated, you should see 10 malformed
441 // sequences being signalled:
442 roundtrip ( false , " \xc0\xe0\x80\xf0\x80\x80\xf8\x80\x80\x80\xfc\x80\x80\x80\x80\xdf\xef\xbf\xf7\xbf\xbf\xfb\xbf\xbf\xbf\xfd\xbf\xbf\xbf\xbf " );
445 SECTION ( "3.5 Impossible bytes" )
447 // The following two bytes cannot appear in a correct UTF-8 string
450 roundtrip ( false , " \xfe " );
452 roundtrip ( false , " \xff " );
454 roundtrip ( false , " \xfe\xfe\xff\xff " );
458 SECTION ( "4 Overlong sequences" )
460 // The following sequences are not malformed according to the letter of
461 // the Unicode 2.0 standard. However, they are longer then necessary and
462 // a correct UTF-8 encoder is not allowed to produce them. A "safe UTF-8
463 // decoder" should reject them just like malformed sequences for two
464 // reasons: (1) It helps to debug applications if overlong sequences are
465 // not treated as valid representations of characters, because this helps
466 // to spot problems more quickly. (2) Overlong sequences provide
467 // alternative representations of characters, that could maliciously be
468 // used to bypass filters that check only for ASCII characters. For
469 // instance, a 2-byte encoded line feed (LF) would not be caught by a
470 // line counter that counts only 0x0a bytes, but it would still be
471 // processed as a line feed by an unsafe UTF-8 decoder later in the
472 // pipeline. From a security point of view, ASCII compatibility of UTF-8
473 // sequences means also, that ASCII characters are *only* allowed to be
474 // represented by ASCII bytes in the range 0x00-0x7f. To ensure this
475 // aspect of ASCII compatibility, use only "safe UTF-8 decoders" that
476 // reject overlong UTF-8 sequences for which a shorter encoding exists.
478 SECTION ( "4.1 Examples of an overlong ASCII character" )
480 // With a safe UTF-8 decoder, all of the following five overlong
481 // representations of the ASCII character slash ("/") should be rejected
482 // like a malformed UTF-8 sequence, for instance by substituting it with
483 // a replacement character. If you see a slash below, you do not have a
484 // safe UTF-8 decoder!
486 // 4.1.1 U+002F = c0 af
487 roundtrip ( false , " \xc0\xaf " );
488 // 4.1.2 U+002F = e0 80 af
489 roundtrip ( false , " \xe0\x80\xaf " );
490 // 4.1.3 U+002F = f0 80 80 af
491 roundtrip ( false , " \xf0\x80\x80\xaf " );
492 // 4.1.4 U+002F = f8 80 80 80 af
493 roundtrip ( false , " \xf8\x80\x80\x80\xaf " );
494 // 4.1.5 U+002F = fc 80 80 80 80 af
495 roundtrip ( false , " \xfc\x80\x80\x80\x80\xaf " );
498 SECTION ( "4.2 Maximum overlong sequences" )
500 // Below you see the highest Unicode value that is still resulting in an
501 // overlong sequence if represented with the given number of bytes. This
502 // is a boundary test for safe UTF-8 decoders. All five characters should
503 // be rejected like malformed UTF-8 sequences.
505 // 4.2.1 U-0000007F = c1 bf
506 roundtrip ( false , " \xc1\xbf " );
507 // 4.2.2 U-000007FF = e0 9f bf
508 roundtrip ( false , " \xe0\x9f\xbf " );
509 // 4.2.3 U-0000FFFF = f0 8f bf bf
510 roundtrip ( false , " \xf0\x8f\xbf\xbf " );
511 // 4.2.4 U-001FFFFF = f8 87 bf bf bf
512 roundtrip ( false , " \xf8\x87\xbf\xbf\xbf " );
513 // 4.2.5 U-03FFFFFF = fc 83 bf bf bf bf
514 roundtrip ( false , " \xfc\x83\xbf\xbf\xbf\xbf " );
517 SECTION ( "4.3 Overlong representation of the NUL character" )
519 // The following five sequences should also be rejected like malformed
520 // UTF-8 sequences and should not be treated like the ASCII NUL
523 // 4.3.1 U+0000 = c0 80
524 roundtrip ( false , " \xc0\x80 " );
525 // 4.3.2 U+0000 = e0 80 80
526 roundtrip ( false , " \xe0\x80\x80 " );
527 // 4.3.3 U+0000 = f0 80 80 80
528 roundtrip ( false , " \xf0\x80\x80\x80 " );
529 // 4.3.4 U+0000 = f8 80 80 80 80
530 roundtrip ( false , " \xf8\x80\x80\x80\x80 " );
531 // 4.3.5 U+0000 = fc 80 80 80 80 80
532 roundtrip ( false , " \xfc\x80\x80\x80\x80\x80 " );
536 SECTION ( "5 Illegal code positions" )
538 // The following UTF-8 sequences should be rejected like malformed
539 // sequences, because they never represent valid ISO 10646 characters and
540 // a UTF-8 decoder that accepts them might introduce security problems
541 // comparable to overlong UTF-8 sequences.
543 SECTION ( "5.1 Single UTF-16 surrogates" )
545 // 5.1.1 U+D800 = ed a0 80
546 roundtrip ( false , " \xed\xa0\x80 " );
547 // 5.1.2 U+DB7F = ed ad bf
548 roundtrip ( false , " \xed\xad\xbf " );
549 // 5.1.3 U+DB80 = ed ae 80
550 roundtrip ( false , " \xed\xae\x80 " );
551 // 5.1.4 U+DBFF = ed af bf
552 roundtrip ( false , " \xed\xaf\xbf " );
553 // 5.1.5 U+DC00 = ed b0 80
554 roundtrip ( false , " \xed\xb0\x80 " );
555 // 5.1.6 U+DF80 = ed be 80
556 roundtrip ( false , " \xed\xbe\x80 " );
557 // 5.1.7 U+DFFF = ed bf bf
558 roundtrip ( false , " \xed\xbf\xbf " );
561 SECTION ( "5.2 Paired UTF-16 surrogates" )
563 // 5.2.1 U+D800 U+DC00 = ed a0 80 ed b0 80
564 roundtrip ( false , " \xed\xa0\x80\xed\xb0\x80 " );
565 // 5.2.2 U+D800 U+DFFF = ed a0 80 ed bf bf
566 roundtrip ( false , " \xed\xa0\x80\xed\xbf\xbf " );
567 // 5.2.3 U+DB7F U+DC00 = ed ad bf ed b0 80
568 roundtrip ( false , " \xed\xad\xbf\xed\xb0\x80 " );
569 // 5.2.4 U+DB7F U+DFFF = ed ad bf ed bf bf
570 roundtrip ( false , " \xed\xad\xbf\xed\xbf\xbf " );
571 // 5.2.5 U+DB80 U+DC00 = ed ae 80 ed b0 80
572 roundtrip ( false , " \xed\xae\x80\xed\xb0\x80 " );
573 // 5.2.6 U+DB80 U+DFFF = ed ae 80 ed bf bf
574 roundtrip ( false , " \xed\xae\x80\xed\xbf\xbf " );
575 // 5.2.7 U+DBFF U+DC00 = ed af bf ed b0 80
576 roundtrip ( false , " \xed\xaf\xbf\xed\xb0\x80 " );
577 // 5.2.8 U+DBFF U+DFFF = ed af bf ed bf bf
578 roundtrip ( false , " \xed\xaf\xbf\xed\xbf\xbf " );
581 SECTION ( "5.3 Noncharacter code positions" )
583 // The following "noncharacters" are "reserved for internal use" by
584 // applications, and according to older versions of the Unicode Standard
585 // "should never be interchanged". Unicode Corrigendum #9 dropped the
586 // latter restriction. Nevertheless, their presence in incoming UTF-8 data
587 // can remain a potential security risk, depending on what use is made of
588 // these codes subsequently. Examples of such internal use:
590 // - Some file APIs with 16-bit characters may use the integer value -1
591 // = U+FFFF to signal an end-of-file (EOF) or error condition.
593 // - In some UTF-16 receivers, code point U+FFFE might trigger a
594 // byte-swap operation (to convert between UTF-16LE and UTF-16BE).
596 // With such internal use of noncharacters, it may be desirable and safer
597 // to block those code points in UTF-8 decoders, as they should never
598 // occur legitimately in incoming UTF-8 data, and could trigger unsafe
599 // behaviour in subsequent processing.
601 // Particularly problematic noncharacters in 16-bit applications:
603 // 5.3.1 U+FFFE = ef bf be
604 roundtrip ( true , " \xef\xbf\xbe " );
605 // 5.3.2 U+FFFF = ef bf bf
606 roundtrip ( true , " \xef\xbf\xbf " );
608 // 5.3.3 U+FDD0 .. U+FDEF
609 roundtrip ( true , " \xEF\xB7\x90 " );
610 roundtrip ( true , " \xEF\xB7\x91 " );
611 roundtrip ( true , " \xEF\xB7\x92 " );
612 roundtrip ( true , " \xEF\xB7\x93 " );
613 roundtrip ( true , " \xEF\xB7\x94 " );
614 roundtrip ( true , " \xEF\xB7\x95 " );
615 roundtrip ( true , " \xEF\xB7\x96 " );
616 roundtrip ( true , " \xEF\xB7\x97 " );
617 roundtrip ( true , " \xEF\xB7\x98 " );
618 roundtrip ( true , " \xEF\xB7\x99 " );
619 roundtrip ( true , " \xEF\xB7\x9A " );
620 roundtrip ( true , " \xEF\xB7\x9B " );
621 roundtrip ( true , " \xEF\xB7\x9C " );
622 roundtrip ( true , " \xEF\xB7\x9D " );
623 roundtrip ( true , " \xEF\xB7\x9E " );
624 roundtrip ( true , " \xEF\xB7\x9F " );
625 roundtrip ( true , " \xEF\xB7\xA0 " );
626 roundtrip ( true , " \xEF\xB7\xA1 " );
627 roundtrip ( true , " \xEF\xB7\xA2 " );
628 roundtrip ( true , " \xEF\xB7\xA3 " );
629 roundtrip ( true , " \xEF\xB7\xA4 " );
630 roundtrip ( true , " \xEF\xB7\xA5 " );
631 roundtrip ( true , " \xEF\xB7\xA6 " );
632 roundtrip ( true , " \xEF\xB7\xA7 " );
633 roundtrip ( true , " \xEF\xB7\xA8 " );
634 roundtrip ( true , " \xEF\xB7\xA9 " );
635 roundtrip ( true , " \xEF\xB7\xAA " );
636 roundtrip ( true , " \xEF\xB7\xAB " );
637 roundtrip ( true , " \xEF\xB7\xAC " );
638 roundtrip ( true , " \xEF\xB7\xAD " );
639 roundtrip ( true , " \xEF\xB7\xAE " );
640 roundtrip ( true , " \xEF\xB7\xAF " );
642 // 5.3.4 U+nFFFE U+nFFFF (for n = 1..10)
643 roundtrip ( true , " \xF0\x9F\xBF\xBF " );
644 roundtrip ( true , " \xF0\xAF\xBF\xBF " );
645 roundtrip ( true , " \xF0\xBF\xBF\xBF " );
646 roundtrip ( true , " \xF1\x8F\xBF\xBF " );
647 roundtrip ( true , " \xF1\x9F\xBF\xBF " );
648 roundtrip ( true , " \xF1\xAF\xBF\xBF " );
649 roundtrip ( true , " \xF1\xBF\xBF\xBF " );
650 roundtrip ( true , " \xF2\x8F\xBF\xBF " );
651 roundtrip ( true , " \xF2\x9F\xBF\xBF " );
652 roundtrip ( true , " \xF2\xAF\xBF\xBF " );