1 /////////1/////////2/////////3/////////4/////////5/////////6/////////7/////////8
2 // utf8_codecvt_facet.ipp
4 // Copyright (c) 2001 Ronald Garcia, Indiana University (garcia@osl.iu.edu)
5 // Andrew Lumsdaine, Indiana University (lums@osl.iu.edu).
6 // Use, modification and distribution is subject to the Boost Software
7 // License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at
8 // http://www.boost.org/LICENSE_1_0.txt)
10 // Please see the comments in <boost/detail/utf8_codecvt_facet.hpp> to
11 // learn how this file should be used.
13 #include <boost/detail/utf8_codecvt_facet.hpp>
15 #include <cstdlib> // for multi-byte converson routines
18 #include <boost/limits.hpp>
19 #include <boost/config.hpp>
21 // If we don't have wstring, then Unicode support
22 // is not available anyway, so we don't need to even
23 // compiler this file. This also fixes the problem
24 // with mingw, which can compile this file, but will
25 // generate link error when building DLL.
26 #ifndef BOOST_NO_STD_WSTRING
28 BOOST_UTF8_BEGIN_NAMESPACE
30 /////////1/////////2/////////3/////////4/////////5/////////6/////////7/////////8
31 // implementation for wchar_t
33 utf8_codecvt_facet::utf8_codecvt_facet(
34 std::size_t no_locale_manage
36 std::codecvt<wchar_t, char, std::mbstate_t>(no_locale_manage)
39 // Translate incoming UTF-8 into UCS-4
40 std::codecvt_base::result utf8_codecvt_facet::do_in(
41 std::mbstate_t& /*state*/,
43 const char * from_end,
44 const char * & from_next,
49 // Basic algorithm: The first octet determines how many
50 // octets total make up the UCS-4 character. The remaining
51 // "continuing octets" all begin with "10". To convert, subtract
52 // the amount that specifies the number of octets from the first
53 // octet. Subtract 0x80 (1000 0000) from each continuing octet,
54 // then mash the whole lot together. Note that each continuing
55 // octet only uses 6 bits as unique values, so only shift by
56 // multiples of 6 to combine.
57 while (from != from_end && to != to_end) {
59 // Error checking on the first octet
60 if (invalid_leading_octet(*from)){
63 return std::codecvt_base::error;
66 // The first octet is adjusted by a value dependent upon
67 // the number of "continuing octets" encoding the character
68 const int cont_octet_count = get_cont_octet_count(*from);
69 const wchar_t octet1_modifier_table[] = {
70 0x00, 0xc0, 0xe0, 0xf0, 0xf8, 0xfc
73 // The unsigned char conversion is necessary in case char is
74 // signed (I learned this the hard way)
76 (unsigned char)(*from++) - octet1_modifier_table[cont_octet_count];
79 // 1) At the start of the loop, 'i' continuing characters have been
81 // 2) *from points to the next continuing character to be processed.
83 while(i != cont_octet_count && from != from_end) {
85 // Error checking on continuing characters
86 if (invalid_continuing_octet(*from)) {
89 return std::codecvt_base::error;
92 ucs_result *= (1 << 6);
94 // each continuing character has an extra (10xxxxxx)b attached to
95 // it that must be removed.
96 ucs_result += (unsigned char)(*from++) - 0x80;
100 // If the buffer ends with an incomplete unicode character...
101 if (from == from_end && i != cont_octet_count) {
102 // rewind "from" to before the current character translation
103 from_next = from - (i+1);
105 return std::codecvt_base::partial;
112 // Were we done converting or did we run out of destination space?
113 if(from == from_end) return std::codecvt_base::ok;
114 else return std::codecvt_base::partial;
117 std::codecvt_base::result utf8_codecvt_facet::do_out(
118 std::mbstate_t& /*state*/,
119 const wchar_t * from,
120 const wchar_t * from_end,
121 const wchar_t * & from_next,
127 // RG - consider merging this table with the other one
128 const wchar_t octet1_modifier_table[] = {
129 0x00, 0xc0, 0xe0, 0xf0, 0xf8, 0xfc
132 wchar_t max_wchar = (std::numeric_limits<wchar_t>::max)();
133 while (from != from_end && to != to_end) {
135 // Check for invalid UCS-4 character
136 if (*from > max_wchar) {
139 return std::codecvt_base::error;
142 int cont_octet_count = get_cont_octet_out_count(*from);
144 // RG - comment this formula better
145 int shift_exponent = (cont_octet_count) * 6;
147 // Process the first character
148 *to++ = static_cast<char>(octet1_modifier_table[cont_octet_count] +
149 (unsigned char)(*from / (1 << shift_exponent)));
151 // Process the continuation characters
152 // Invariants: At the start of the loop:
153 // 1) 'i' continuing octets have been generated
154 // 2) '*to' points to the next location to place an octet
155 // 3) shift_exponent is 6 more than needed for the next octet
157 while (i != cont_octet_count && to != to_end) {
159 *to++ = static_cast<char>(0x80 + ((*from / (1 << shift_exponent)) % (1 << 6)));
162 // If we filled up the out buffer before encoding the character
163 if(to == to_end && i != cont_octet_count) {
165 to_next = to - (i+1);
166 return std::codecvt_base::partial;
172 // Were we done or did we run out of destination space
173 if(from == from_end) return std::codecvt_base::ok;
174 else return std::codecvt_base::partial;
177 // How many char objects can I process to get <= max_limit
179 int utf8_codecvt_facet::do_length(
182 const char * from_end,
183 std::size_t max_limit
185 #if BOOST_WORKAROUND(__IBMCPP__, BOOST_TESTED_AT(600))
189 // RG - this code is confusing! I need a better way to express it.
193 // 1) last_octet_count has the size of the last measured character
194 // 2) char_count holds the number of characters shown to fit
195 // within the bounds so far (no greater than max_limit)
196 // 3) from_next points to the octet 'last_octet_count' before the
197 // last measured character.
198 int last_octet_count=0;
199 std::size_t char_count = 0;
200 const char* from_next = from;
201 // Use "<" because the buffer may represent incomplete characters
202 while (from_next+last_octet_count <= from_end && char_count <= max_limit) {
203 from_next += last_octet_count;
204 last_octet_count = (get_octet_count(*from_next));
207 return static_cast<int>(from_next-from);
210 unsigned int utf8_codecvt_facet::get_octet_count(
211 unsigned char lead_octet
213 // if the 0-bit (MSB) is 0, then 1 character
214 if (lead_octet <= 0x7f) return 1;
216 // Otherwise the count number of consecutive 1 bits starting at MSB
217 // assert(0xc0 <= lead_octet && lead_octet <= 0xfd);
219 if (0xc0 <= lead_octet && lead_octet <= 0xdf) return 2;
220 else if (0xe0 <= lead_octet && lead_octet <= 0xef) return 3;
221 else if (0xf0 <= lead_octet && lead_octet <= 0xf7) return 4;
222 else if (0xf8 <= lead_octet && lead_octet <= 0xfb) return 5;
228 template<std::size_t s>
229 int get_cont_octet_out_count_impl(wchar_t word){
240 int get_cont_octet_out_count_impl<4>(wchar_t word){
248 // Note that the following code will generate warnings on some platforms
249 // where wchar_t is defined as UCS2. The warnings are superfluous as the
250 // specialization is never instantitiated with such compilers, but this
251 // can cause problems if warnings are being treated as errors, so we guard
252 // against that. Including <boost/detail/utf8_codecvt_facet.hpp> as we do
253 // should be enough to get WCHAR_MAX defined.
254 #if !defined(WCHAR_MAX)
255 # error WCHAR_MAX not defined!
257 // cope with VC++ 7.1 or earlier having invalid WCHAR_MAX
258 #if defined(_MSC_VER) && _MSC_VER <= 1310 // 7.1 or earlier
260 #elif WCHAR_MAX > 0x10000
262 if (word < 0x10000) {
265 if (word < 0x200000) {
268 if (word < 0x4000000) {
278 } // namespace detail
280 // How many "continuing octets" will be needed for this word
281 // == total octets - 1.
282 int utf8_codecvt_facet::get_cont_octet_out_count(
285 return detail::get_cont_octet_out_count_impl<sizeof(wchar_t)>(word);
287 BOOST_UTF8_END_NAMESPACE