1 // Copyright (c) 2001 Ronald Garcia, Indiana University (garcia@osl.iu.edu)
2 // Andrew Lumsdaine, Indiana University (lums@osl.iu.edu).
3 // Distributed under the Boost Software License, Version 1.0. (See accompany-
4 // ing file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
6 #ifndef BOOST_UTF8_CODECVT_FACET_HPP
7 #define BOOST_UTF8_CODECVT_FACET_HPP
9 // MS compatible compilers support #pragma once
10 #if defined(_MSC_VER) && (_MSC_VER >= 1020)
14 /////////1/////////2/////////3/////////4/////////5/////////6/////////7/////////8
15 // utf8_codecvt_facet.hpp
17 // This header defines class utf8_codecvt_facet, derived from
18 // std::codecvt<wchar_t, char>, which can be used to convert utf8 data in
19 // files into wchar_t strings in the application.
21 // The header is NOT STANDALONE, and is not to be included by the USER.
22 // There are at least two libraries which want to use this functionality, and
23 // we want to avoid code duplication. It would be possible to create utf8
25 // - this requires review process first
26 // - in the case, when linking the a library which uses utf8
27 // (say 'program_options'), user should also link to the utf8 library.
28 // This seems inconvenient, and asking a user to link to an unrevieved
29 // library is strange.
30 // Until the above points are fixed, a library which wants to use utf8 must:
31 // - include this header in one of it's headers or sources
32 // - include the corresponding boost/detail/utf8_codecvt_facet.ipp file in one
34 // - before including either file, the library must define
35 // - BOOST_UTF8_BEGIN_NAMESPACE to the namespace declaration that must be used
36 // - BOOST_UTF8_END_NAMESPACE to the code to close the previous namespace
38 // - BOOST_UTF8_DECL -- to the code which must be used for all 'exportable'
41 // For example, program_options library might contain:
42 // #define BOOST_UTF8_BEGIN_NAMESPACE <backslash character>
43 // namespace boost { namespace program_options {
44 // #define BOOST_UTF8_END_NAMESPACE }}
45 // #define BOOST_UTF8_DECL BOOST_PROGRAM_OPTIONS_DECL
46 // #include <boost/detail/utf8_codecvt_facet.ipp>
48 // Essentially, each library will have its own copy of utf8 code, in
49 // different namespaces.
51 // Note:(Robert Ramey). I have made the following alterations in the original
53 // a) Rendered utf8_codecvt<wchar_t, char> with using templates
54 // b) Move longer functions outside class definition to prevent inlining
55 // and make code smaller
56 // c) added on a derived class to permit translation to/from current
59 // See http://www.boost.org for updates, documentation, and revision history.
61 // archives stored as text - note these ar templated on the basic
62 // stream templates to accommodate wide (and other?) kind of characters
64 // note the fact that on libraries without wide characters, ostream is
65 // is not a specialization of basic_ostream which in fact is not defined
66 // in such cases. So we can't use basic_ostream<OStream::char_type> but rather
67 // use two template parameters
70 // This is an implementation of a std::codecvt facet for translating
71 // from UTF-8 externally to UCS-4. Note that this is not tied to
72 // any specific types in order to allow customization on platforms
73 // where wchar_t is not big enough.
75 // NOTES: The current implementation jumps through some unpleasant hoops in
76 // order to deal with signed character types. As a std::codecvt_base::result,
77 // it is necessary for the ExternType to be convertible to unsigned char.
78 // I chose not to tie the extern_type explicitly to char. But if any combination
79 // of types other than <wchar_t,char_t> is used, then std::codecvt must be
80 // specialized on those types for this to work.
83 #include <cwchar> // for mbstate_t
84 #include <cstddef> // for std::size_t
86 #include <boost/config.hpp>
87 #include <boost/detail/workaround.hpp>
89 #if defined(BOOST_NO_STDC_NAMESPACE)
96 // maximum lenght of a multibyte string
97 #define MB_LENGTH_MAX 8
99 BOOST_UTF8_BEGIN_NAMESPACE
101 //----------------------------------------------------------------------------//
103 // utf8_codecvt_facet //
105 // See utf8_codecvt_facet.ipp for the implementation. //
106 //----------------------------------------------------------------------------//
108 #ifndef BOOST_UTF8_DECL
109 #define BOOST_UTF8_DECL
112 struct BOOST_SYMBOL_VISIBLE utf8_codecvt_facet :
113 public std::codecvt<wchar_t, char, std::mbstate_t>
116 BOOST_UTF8_DECL explicit utf8_codecvt_facet(std::size_t no_locale_manage=0);
117 virtual ~utf8_codecvt_facet(){}
119 BOOST_UTF8_DECL virtual std::codecvt_base::result do_in(
120 std::mbstate_t& state,
122 const char * from_end,
123 const char * & from_next,
129 BOOST_UTF8_DECL virtual std::codecvt_base::result do_out(
130 std::mbstate_t & state,
131 const wchar_t * from,
132 const wchar_t * from_end,
133 const wchar_t* & from_next,
139 bool invalid_continuing_octet(unsigned char octet_1) const {
140 return (octet_1 < 0x80|| 0xbf< octet_1);
143 bool invalid_leading_octet(unsigned char octet_1) const {
144 return (0x7f < octet_1 && octet_1 < 0xc0) ||
148 // continuing octets = octets except for the leading octet
149 static unsigned int get_cont_octet_count(unsigned char lead_octet) {
150 return get_octet_count(lead_octet) - 1;
153 BOOST_UTF8_DECL static unsigned int get_octet_count(unsigned char lead_octet);
155 // How many "continuing octets" will be needed for this word
156 // == total octets - 1.
157 BOOST_UTF8_DECL int get_cont_octet_out_count(wchar_t word) const ;
159 virtual bool do_always_noconv() const BOOST_NOEXCEPT_OR_NOTHROW {
163 // UTF-8 isn't really stateful since we rewind on partial conversions
164 virtual std::codecvt_base::result do_unshift(
174 virtual int do_encoding() const BOOST_NOEXCEPT_OR_NOTHROW {
175 const int variable_byte_external_encoding=0;
176 return variable_byte_external_encoding;
179 // How many char objects can I process to get <= max_limit
181 BOOST_UTF8_DECL virtual int do_length(
182 const std::mbstate_t &,
184 const char * from_end,
185 std::size_t max_limit
187 #if BOOST_WORKAROUND(__IBMCPP__, BOOST_TESTED_AT(600))
191 virtual int do_length(
194 const char * from_end,
195 std::size_t max_limit
197 #if BOOST_WORKAROUND(__IBMCPP__, BOOST_TESTED_AT(600))
202 const_cast<const std::mbstate_t &>(s),
208 // Largest possible value do_length(state,from,from_end,1) could return.
209 virtual int do_max_length() const BOOST_NOEXCEPT_OR_NOTHROW {
210 return 6; // largest UTF-8 encoding of a UCS-4 character
214 BOOST_UTF8_END_NAMESPACE
216 #endif // BOOST_UTF8_CODECVT_FACET_HPP