]>
git.proxmox.com Git - ceph.git/blob - ceph/src/boost/libs/locale/test/test_codepage_converter.cpp
2 // Copyright (c) 2009-2011 Artyom Beilis (Tonkikh)
4 // Distributed under the Boost Software License, Version 1.0. (See
5 // accompanying file LICENSE_1_0.txt or copy at
6 // http://www.boost.org/LICENSE_1_0.txt)
8 #include "test_locale.hpp"
9 #include "test_locale_tools.hpp"
10 #include <boost/locale/util.hpp>
11 #ifdef BOOST_LOCALE_WITH_ICU
12 #include "../src/icu/codecvt.hpp"
14 #if defined(BOOST_LOCALE_WITH_ICONV) && !defined(BOOST_LOCALE_NO_POSIX_BACKEND)
15 #include "../src/posix/codecvt.hpp"
20 char *make2(unsigned v
)
22 static unsigned char buf
[3] = {0};
23 buf
[0] = 0xC0 | (v
>> 6);
24 buf
[1] = 0x80 | (v
& 0x3F );
25 return reinterpret_cast<char*>(buf
);
28 char *make3(unsigned v
)
30 static unsigned char buf
[4] = {0};
31 buf
[0] = 0xE0 | ((v
>> 12) ) ;
32 buf
[1] = 0x80 | ((v
>> 6) & 0x3F );
33 buf
[2] = 0x80 | ((v
>> 0) & 0x3F );
34 return reinterpret_cast<char*>(buf
);
37 char *make4(unsigned v
)
39 static unsigned char buf
[5] = {0};
40 buf
[0] = 0xF0 | ((v
>> 18) ) ;
41 buf
[1] = 0x80 | ((v
>> 12) & 0x3F );
42 buf
[2] = 0x80 | ((v
>> 6) & 0x3F );
43 buf
[3] = 0x80 | ((v
>> 0) & 0x3F );
44 return reinterpret_cast<char*>(buf
);
47 static const unsigned illegal
=0xFFFFFFFF;
48 static const unsigned incomplete
=0xFFFFFFFE;
51 bool test_to(boost::locale::util::base_converter
&cvt
,char const *s
,unsigned codepoint
)
53 size_t len
= strlen(s
);
54 char const *end
= s
+ len
;
55 return cvt
.to_unicode(s
,end
) == codepoint
;
58 bool test_from(boost::locale::util::base_converter
&cvt
,unsigned codepoint
,char const *str
)
61 unsigned res
= cvt
.from_unicode(codepoint
,buf
,buf
+sizeof(buf
));
62 if(res
== boost::locale::util::base_converter::illegal
) {
66 return str
!=0 && strlen(str
) == res
&& memcmp(str
,buf
,res
) == 0;
70 bool test_incomplete(boost::locale::util::base_converter
&cvt
,unsigned codepoint
,int len
)
73 unsigned res
= cvt
.from_unicode(codepoint
,buf
,buf
+len
);
74 return res
== incomplete
;
79 #define TEST_TO(str,codepoint) TEST(test_to(*cvt,str,codepoint))
80 #define TEST_FROM(str,codepoint) TEST(test_from(*cvt,codepoint,str))
81 #define TEST_INC(codepoint,len) TEST(test_incomplete(*cvt,codepoint,len))
83 void test_shiftjis(std::auto_ptr
<boost::locale::util::base_converter
> cvt
)
85 std::cout
<< "- Correct" << std::endl
;
88 TEST_TO("\xCB",0xFF8b); // half width katakana Hi ヒ
89 TEST_TO("\x83\x71",0x30d2); // Full width katakana Hi ヒ
90 TEST_TO("\x82\xd0",0x3072); // Full width hiragana Hi ひ
94 TEST_FROM("\xCB",0xFF8b); // half width katakana Hi ヒ
95 TEST_FROM("\x83\x71",0x30d2); // Full width katakana Hi ヒ
96 TEST_FROM("\x82\xd0",0x3072); // Full width hiragana Hi ひ
98 std::cout
<< "- Illegal/incomplete" << std::endl
;
100 TEST_TO("\xa0",illegal
);
101 TEST_TO("\x82",incomplete
);
102 TEST_TO("\x83\xf0",illegal
);
104 TEST_INC(0x30d2,1); // Full width katakana Hi ヒ
105 TEST_INC(0x3072,1); // Full width hiragana Hi ひ
107 TEST_FROM(0,0x5e9); // Hebrew ש not in ShiftJIS
114 using namespace boost::locale::util
;
116 std::auto_ptr
<base_converter
> cvt
;
118 std::cout
<< "Test UTF-8" << std::endl
;
119 std::cout
<< "- From UTF-8" << std::endl
;
122 cvt
= create_utf8_converter();
124 TEST(cvt
->is_thread_safe());
125 TEST(cvt
->max_len() == 4);
127 std::cout
<< "-- Correct" << std::endl
;
129 TEST_TO("\x7f",0x7f);
130 TEST_TO("\xC2\x80",0x80);
131 TEST_TO("\xdf\xBF",0x7FF);
132 TEST_TO("\xe0\xa0\x80",0x800);
133 TEST_TO("\xef\xbf\xbf",0xFFFF);
134 TEST_TO("\xf0\x90\x80\x80",0x10000);
135 TEST_TO("\xf4\x8f\xbf\xbf",0x10FFFF);
137 std::cout
<< "-- Too big" << std::endl
;
138 TEST_TO("\xf4\x9f\x80\x80",illegal
); // 11 0000
139 TEST_TO("\xfb\xbf\xbf\xbf",illegal
); // 3FF FFFF
140 TEST_TO("\xf8\x90\x80\x80\x80",illegal
); // 400 0000
141 TEST_TO("\xfd\xbf\xbf\xbf\xbf\xbf",illegal
); // 7fff ffff
143 std::cout
<< "-- Invalid trail" << std::endl
;
144 TEST_TO("\xC2\x7F",illegal
);
145 TEST_TO("\xdf\x7F",illegal
);
146 TEST_TO("\xe0\x7F\x80",illegal
);
147 TEST_TO("\xef\xbf\x7F",illegal
);
148 TEST_TO("\xe0\x7F\x80",illegal
);
149 TEST_TO("\xef\xbf\x7F",illegal
);
150 TEST_TO("\xf0\x7F\x80\x80",illegal
);
151 TEST_TO("\xf4\x7f\xbf\xbf",illegal
);
152 TEST_TO("\xf0\x90\x7F\x80",illegal
);
153 TEST_TO("\xf4\x8f\x7F\xbf",illegal
);
154 TEST_TO("\xf0\x90\x80\x7F",illegal
);
155 TEST_TO("\xf4\x8f\xbf\x7F",illegal
);
157 std::cout
<< "-- Invalid length" << std::endl
;
159 /// Test that this actually works
160 TEST_TO(make2(0x80),0x80);
161 TEST_TO(make2(0x7ff),0x7ff);
163 TEST_TO(make3(0x800),0x800);
164 TEST_TO(make3(0xffff),0xffff);
166 TEST_TO(make4(0x10000),0x10000);
167 TEST_TO(make4(0x10ffff),0x10ffff);
169 TEST_TO(make4(0x110000),illegal
);
170 TEST_TO(make4(0x1fffff),illegal
);
172 TEST_TO(make2(0),illegal
);
173 TEST_TO(make3(0),illegal
);
174 TEST_TO(make4(0),illegal
);
175 TEST_TO(make2(0x7f),illegal
);
176 TEST_TO(make3(0x7f),illegal
);
177 TEST_TO(make4(0x7f),illegal
);
179 TEST_TO(make3(0x80),illegal
);
180 TEST_TO(make4(0x80),illegal
);
181 TEST_TO(make3(0x7ff),illegal
);
182 TEST_TO(make4(0x7ff),illegal
);
184 TEST_TO(make4(0x8000),illegal
);
185 TEST_TO(make4(0xffff),illegal
);
187 std::cout
<< "-- Invalid surrogate" << std::endl
;
189 TEST_TO(make3(0xD800),illegal
);
190 TEST_TO(make3(0xDBFF),illegal
);
191 TEST_TO(make3(0xDC00),illegal
);
192 TEST_TO(make3(0xDFFF),illegal
);
194 TEST_TO(make4(0xD800),illegal
);
195 TEST_TO(make4(0xDBFF),illegal
);
196 TEST_TO(make4(0xDC00),illegal
);
197 TEST_TO(make4(0xDFFF),illegal
);
199 std::cout
<<"-- Incomplete" << std::endl
;
201 TEST_TO("\x80",illegal
);
202 TEST_TO("\xC2",incomplete
);
204 TEST_TO("\xdf",incomplete
);
206 TEST_TO("\xe0",incomplete
);
207 TEST_TO("\xe0\xa0",incomplete
);
209 TEST_TO("\xef\xbf",incomplete
);
210 TEST_TO("\xef",incomplete
);
212 TEST_TO("\xf0\x90\x80",incomplete
);
213 TEST_TO("\xf0\x90",incomplete
);
214 TEST_TO("\xf0",incomplete
);
216 TEST_TO("\xf4\x8f\xbf",incomplete
);
217 TEST_TO("\xf4\x8f",incomplete
);
218 TEST_TO("\xf4",incomplete
);
220 std::cout
<< "- To UTF-8" << std::endl
;
222 std::cout
<< "-- Test correct" << std::endl
;
224 TEST_FROM("\x7f",0x7f);
225 TEST_FROM("\xC2\x80",0x80);
226 TEST_FROM("\xdf\xBF",0x7FF);
228 TEST_FROM("\xe0\xa0\x80",0x800);
231 TEST_FROM("\xef\xbf\xbf",0xFFFF);
235 TEST_FROM("\xf0\x90\x80\x80",0x10000);
236 TEST_FROM("\xf4\x8f\xbf\xbf",0x10FFFF);
238 std::cout
<< "-- Test no surrogate " << std::endl
;
245 std::cout
<< "-- Test invalid " << std::endl
;
247 TEST_FROM(0,0x110000);
248 TEST_FROM(0,0x1FFFFF);
251 std::cout
<< "Test windows-1255" << std::endl
;
253 cvt
= create_simple_converter("windows-1255");
256 TEST(cvt
->is_thread_safe());
257 TEST(cvt
->max_len() == 1);
259 std::cout
<< "- From 1255" << std::endl
;
261 TEST_TO("\xa4",0x20aa);
262 TEST_TO("\xe0",0x05d0);
263 TEST_TO("\xc4",0x5b4);
264 TEST_TO("\xfb",illegal
);
265 TEST_TO("\xdd",illegal
);
266 TEST_TO("\xff",illegal
);
267 TEST_TO("\xfe",0x200f);
269 std::cout
<< "- To 1255" << std::endl
;
271 TEST_FROM("\xa4",0x20aa);
272 TEST_FROM("\xe0",0x05d0);
273 TEST_FROM("\xc4",0x5b4);
274 TEST_FROM("\xfe",0x200f);
279 #ifdef BOOST_LOCALE_WITH_ICU
280 std::cout
<< "Testing Shift-JIS using ICU/uconv" << std::endl
;
282 cvt
= boost::locale::impl_icu::create_uconv_converter("Shift-JIS");
287 #if defined(BOOST_LOCALE_WITH_ICONV) && !defined(BOOST_LOCALE_NO_POSIX_BACKEND)
288 std::cout
<< "Testing Shift-JIS using POSIX/iconv" << std::endl
;
290 cvt
= boost::locale::impl_posix::create_iconv_converter("Shift-JIS");
295 std::cout
<< "- Shift-JIS is not supported!" << std::endl
;
300 catch(std::exception
const &e
) {
301 std::cerr
<< "Failed " << e
.what() << std::endl
;
307 // vim: tabstop=4 expandtab shiftwidth=4 softtabstop=4
308 // boostinspect:noascii