6 * Use, modification and distribution are subject to the
7 * Boost Software License, Version 1.0. (See accompanying file
8 * LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
13 * LOCATION: see http://www.boost.org for most recent version.
14 * FILE unicode_iterator.hpp
15 * VERSION see <boost/version.hpp>
16 * DESCRIPTION: Iterator adapters for converting between different Unicode encodings.
19 /****************************************************************************
24 1) Read Only, Input Adapters:
25 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
27 template <class BaseIterator, class U8Type = ::boost::uint8_t>
28 class u32_to_u8_iterator;
30 Adapts sequence of UTF-32 code points to "look like" a sequence of UTF-8.
32 template <class BaseIterator, class U32Type = ::boost::uint32_t>
33 class u8_to_u32_iterator;
35 Adapts sequence of UTF-8 code points to "look like" a sequence of UTF-32.
37 template <class BaseIterator, class U16Type = ::boost::uint16_t>
38 class u32_to_u16_iterator;
40 Adapts sequence of UTF-32 code points to "look like" a sequence of UTF-16.
42 template <class BaseIterator, class U32Type = ::boost::uint32_t>
43 class u16_to_u32_iterator;
45 Adapts sequence of UTF-16 code points to "look like" a sequence of UTF-32.
47 2) Single pass output iterator adapters:
49 template <class BaseIterator>
50 class utf8_output_iterator;
52 Accepts UTF-32 code points and forwards them on as UTF-8 code points.
54 template <class BaseIterator>
55 class utf16_output_iterator;
57 Accepts UTF-32 code points and forwards them on as UTF-16 code points.
59 ****************************************************************************/
61 #ifndef BOOST_REGEX_UNICODE_ITERATOR_HPP
62 #define BOOST_REGEX_UNICODE_ITERATOR_HPP
63 #include <boost/cstdint.hpp>
64 #include <boost/assert.hpp>
65 #include <boost/iterator/iterator_facade.hpp>
66 #include <boost/static_assert.hpp>
67 #include <boost/throw_exception.hpp>
69 #ifndef BOOST_NO_STD_LOCALE
73 #include <limits.h> // CHAR_BIT
79 static const ::boost::uint16_t high_surrogate_base = 0xD7C0u;
80 static const ::boost::uint16_t low_surrogate_base = 0xDC00u;
81 static const ::boost::uint32_t ten_bit_mask = 0x3FFu;
83 inline bool is_high_surrogate(::boost::uint16_t v)
85 return (v & 0xFFFFFC00u) == 0xd800u;
87 inline bool is_low_surrogate(::boost::uint16_t v)
89 return (v & 0xFFFFFC00u) == 0xdc00u;
92 inline bool is_surrogate(T v)
94 return (v & 0xFFFFF800u) == 0xd800;
97 inline unsigned utf8_byte_count(boost::uint8_t c)
99 // if the most significant bit with a zero in it is in position
100 // 8-N then there are N bytes in this UTF-8 sequence:
101 boost::uint8_t mask = 0x80u;
108 return (result == 0) ? 1 : ((result > 4) ? 4 : result);
111 inline unsigned utf8_trailing_byte_count(boost::uint8_t c)
113 return utf8_byte_count(c) - 1;
117 #pragma warning(push)
118 #pragma warning(disable:4100)
120 BOOST_NORETURN inline void invalid_utf32_code_point(::boost::uint32_t val)
122 #ifndef BOOST_NO_STD_LOCALE
123 std::stringstream ss;
124 ss << "Invalid UTF-32 code point U+" << std::showbase << std::hex << val << " encountered while trying to encode UTF-16 sequence";
125 std::out_of_range e(ss.str());
127 std::out_of_range e("Invalid UTF-32 code point encountered while trying to encode UTF-16 sequence");
129 boost::throw_exception(e);
136 } // namespace detail
138 template <class BaseIterator, class U16Type = ::boost::uint16_t>
139 class u32_to_u16_iterator
140 : public boost::iterator_facade<u32_to_u16_iterator<BaseIterator, U16Type>, U16Type, std::bidirectional_iterator_tag, const U16Type>
142 typedef boost::iterator_facade<u32_to_u16_iterator<BaseIterator, U16Type>, U16Type, std::bidirectional_iterator_tag, const U16Type> base_type;
144 #if !defined(BOOST_NO_STD_ITERATOR_TRAITS)
145 typedef typename std::iterator_traits<BaseIterator>::value_type base_value_type;
147 BOOST_STATIC_ASSERT(sizeof(base_value_type)*CHAR_BIT == 32);
148 BOOST_STATIC_ASSERT(sizeof(U16Type)*CHAR_BIT == 16);
152 typename base_type::reference
157 return m_values[m_current];
159 bool equal(const u32_to_u16_iterator& that)const
161 if(m_position == that.m_position)
163 // Both m_currents must be equal, or both even
164 // this is the same as saying their sum must be even:
165 return (m_current + that.m_current) & 1u ? false : true;
171 // if we have a pending read then read now, so that we know whether
172 // to skip a position, or move to a low-surrogate:
178 // move to the next surrogate position:
180 // if we've reached the end skip a position:
181 if(m_values[m_current] == 0)
191 // decrementing an iterator always leads to a valid position:
194 m_current = m_values[1] ? 1 : 0;
201 BaseIterator base()const
206 u32_to_u16_iterator() : m_position(), m_current(0)
212 u32_to_u16_iterator(BaseIterator b) : m_position(b), m_current(2)
220 void extract_current()const
222 // begin by checking for a code point out of range:
223 ::boost::uint32_t v = *m_position;
227 detail::invalid_utf32_code_point(*m_position);
228 // split into two surrogates:
229 m_values[0] = static_cast<U16Type>(v >> 10) + detail::high_surrogate_base;
230 m_values[1] = static_cast<U16Type>(v & detail::ten_bit_mask) + detail::low_surrogate_base;
232 BOOST_ASSERT(detail::is_high_surrogate(m_values[0]));
233 BOOST_ASSERT(detail::is_low_surrogate(m_values[1]));
237 // 16-bit code point:
238 m_values[0] = static_cast<U16Type>(*m_position);
241 // value must not be a surrogate:
242 if(detail::is_surrogate(m_values[0]))
243 detail::invalid_utf32_code_point(*m_position);
246 BaseIterator m_position;
247 mutable U16Type m_values[3];
248 mutable unsigned m_current;
251 template <class BaseIterator, class U32Type = ::boost::uint32_t>
252 class u16_to_u32_iterator
253 : public boost::iterator_facade<u16_to_u32_iterator<BaseIterator, U32Type>, U32Type, std::bidirectional_iterator_tag, const U32Type>
255 typedef boost::iterator_facade<u16_to_u32_iterator<BaseIterator, U32Type>, U32Type, std::bidirectional_iterator_tag, const U32Type> base_type;
256 // special values for pending iterator reads:
257 BOOST_STATIC_CONSTANT(U32Type, pending_read = 0xffffffffu);
259 #if !defined(BOOST_NO_STD_ITERATOR_TRAITS)
260 typedef typename std::iterator_traits<BaseIterator>::value_type base_value_type;
262 BOOST_STATIC_ASSERT(sizeof(base_value_type)*CHAR_BIT == 16);
263 BOOST_STATIC_ASSERT(sizeof(U32Type)*CHAR_BIT == 32);
267 typename base_type::reference
270 if(m_value == pending_read)
274 bool equal(const u16_to_u32_iterator& that)const
276 return m_position == that.m_position;
280 // skip high surrogate first if there is one:
281 if(detail::is_high_surrogate(*m_position)) ++m_position;
283 m_value = pending_read;
288 // if we have a low surrogate then go back one more:
289 if(detail::is_low_surrogate(*m_position))
291 m_value = pending_read;
293 BaseIterator base()const
298 u16_to_u32_iterator() : m_position()
300 m_value = pending_read;
302 u16_to_u32_iterator(BaseIterator b) : m_position(b)
304 m_value = pending_read;
307 // Range checked version:
309 u16_to_u32_iterator(BaseIterator b, BaseIterator start, BaseIterator end) : m_position(b)
311 m_value = pending_read;
313 // The range must not start with a low surrogate, or end in a high surrogate,
314 // otherwise we run the risk of running outside the underlying input range.
315 // Likewise b must not be located at a low surrogate.
320 if((b != start) && (b != end))
323 if(detail::is_surrogate(val) && ((val & 0xFC00u) == 0xDC00u))
324 invalid_code_point(val);
327 if(detail::is_surrogate(val) && ((val & 0xFC00u) == 0xDC00u))
328 invalid_code_point(val);
330 if(detail::is_high_surrogate(val))
331 invalid_code_point(val);
335 static void invalid_code_point(::boost::uint16_t val)
337 #ifndef BOOST_NO_STD_LOCALE
338 std::stringstream ss;
339 ss << "Misplaced UTF-16 surrogate U+" << std::showbase << std::hex << val << " encountered while trying to encode UTF-32 sequence";
340 std::out_of_range e(ss.str());
342 std::out_of_range e("Misplaced UTF-16 surrogate encountered while trying to encode UTF-32 sequence");
344 boost::throw_exception(e);
346 void extract_current()const
348 m_value = static_cast<U32Type>(static_cast< ::boost::uint16_t>(*m_position));
349 // if the last value is a high surrogate then adjust m_position and m_value as needed:
350 if(detail::is_high_surrogate(*m_position))
352 // precondition; next value must have be a low-surrogate:
353 BaseIterator next(m_position);
354 ::boost::uint16_t t = *++next;
355 if((t & 0xFC00u) != 0xDC00u)
356 invalid_code_point(t);
357 m_value = (m_value - detail::high_surrogate_base) << 10;
358 m_value |= (static_cast<U32Type>(static_cast< ::boost::uint16_t>(t)) & detail::ten_bit_mask);
360 // postcondition; result must not be a surrogate:
361 if(detail::is_surrogate(m_value))
362 invalid_code_point(static_cast< ::boost::uint16_t>(m_value));
364 BaseIterator m_position;
365 mutable U32Type m_value;
368 template <class BaseIterator, class U8Type = ::boost::uint8_t>
369 class u32_to_u8_iterator
370 : public boost::iterator_facade<u32_to_u8_iterator<BaseIterator, U8Type>, U8Type, std::bidirectional_iterator_tag, const U8Type>
372 typedef boost::iterator_facade<u32_to_u8_iterator<BaseIterator, U8Type>, U8Type, std::bidirectional_iterator_tag, const U8Type> base_type;
374 #if !defined(BOOST_NO_STD_ITERATOR_TRAITS)
375 typedef typename std::iterator_traits<BaseIterator>::value_type base_value_type;
377 BOOST_STATIC_ASSERT(sizeof(base_value_type)*CHAR_BIT == 32);
378 BOOST_STATIC_ASSERT(sizeof(U8Type)*CHAR_BIT == 8);
382 typename base_type::reference
387 return m_values[m_current];
389 bool equal(const u32_to_u8_iterator& that)const
391 if(m_position == that.m_position)
393 // either the m_current's must be equal, or one must be 0 and
394 // the other 4: which means neither must have bits 1 or 2 set:
395 return (m_current == that.m_current)
396 || (((m_current | that.m_current) & 3) == 0);
402 // if we have a pending read then read now, so that we know whether
403 // to skip a position, or move to a low-surrogate:
409 // move to the next surrogate position:
411 // if we've reached the end skip a position:
412 if(m_values[m_current] == 0)
420 if((m_current & 3) == 0)
425 while(m_current && (m_values[m_current] == 0))
431 BaseIterator base()const
436 u32_to_u8_iterator() : m_position(), m_current(0)
444 u32_to_u8_iterator(BaseIterator b) : m_position(b), m_current(4)
454 void extract_current()const
456 boost::uint32_t c = *m_position;
458 detail::invalid_utf32_code_point(c);
461 m_values[0] = static_cast<unsigned char>(c);
462 m_values[1] = static_cast<unsigned char>(0u);
463 m_values[2] = static_cast<unsigned char>(0u);
464 m_values[3] = static_cast<unsigned char>(0u);
468 m_values[0] = static_cast<unsigned char>(0xC0u + (c >> 6));
469 m_values[1] = static_cast<unsigned char>(0x80u + (c & 0x3Fu));
470 m_values[2] = static_cast<unsigned char>(0u);
471 m_values[3] = static_cast<unsigned char>(0u);
473 else if(c < 0x10000u)
475 m_values[0] = static_cast<unsigned char>(0xE0u + (c >> 12));
476 m_values[1] = static_cast<unsigned char>(0x80u + ((c >> 6) & 0x3Fu));
477 m_values[2] = static_cast<unsigned char>(0x80u + (c & 0x3Fu));
478 m_values[3] = static_cast<unsigned char>(0u);
482 m_values[0] = static_cast<unsigned char>(0xF0u + (c >> 18));
483 m_values[1] = static_cast<unsigned char>(0x80u + ((c >> 12) & 0x3Fu));
484 m_values[2] = static_cast<unsigned char>(0x80u + ((c >> 6) & 0x3Fu));
485 m_values[3] = static_cast<unsigned char>(0x80u + (c & 0x3Fu));
489 BaseIterator m_position;
490 mutable U8Type m_values[5];
491 mutable unsigned m_current;
494 template <class BaseIterator, class U32Type = ::boost::uint32_t>
495 class u8_to_u32_iterator
496 : public boost::iterator_facade<u8_to_u32_iterator<BaseIterator, U32Type>, U32Type, std::bidirectional_iterator_tag, const U32Type>
498 typedef boost::iterator_facade<u8_to_u32_iterator<BaseIterator, U32Type>, U32Type, std::bidirectional_iterator_tag, const U32Type> base_type;
499 // special values for pending iterator reads:
500 BOOST_STATIC_CONSTANT(U32Type, pending_read = 0xffffffffu);
502 #if !defined(BOOST_NO_STD_ITERATOR_TRAITS)
503 typedef typename std::iterator_traits<BaseIterator>::value_type base_value_type;
505 BOOST_STATIC_ASSERT(sizeof(base_value_type)*CHAR_BIT == 8);
506 BOOST_STATIC_ASSERT(sizeof(U32Type)*CHAR_BIT == 32);
510 typename base_type::reference
513 if(m_value == pending_read)
517 bool equal(const u8_to_u32_iterator& that)const
519 return m_position == that.m_position;
523 // We must not start with a continuation character:
524 if((static_cast<boost::uint8_t>(*m_position) & 0xC0) == 0x80)
526 // skip high surrogate first if there is one:
527 unsigned c = detail::utf8_byte_count(*m_position);
528 if(m_value == pending_read)
530 // Since we haven't read in a value, we need to validate the code points:
531 for(unsigned i = 0; i < c; ++i)
534 // We must have a continuation byte:
535 if((i != c - 1) && ((static_cast<boost::uint8_t>(*m_position) & 0xC0) != 0x80))
541 std::advance(m_position, c);
543 m_value = pending_read;
547 // Keep backtracking until we don't have a trailing character:
549 while((*--m_position & 0xC0u) == 0x80u) ++count;
550 // now check that the sequence was valid:
551 if(count != detail::utf8_trailing_byte_count(*m_position))
553 m_value = pending_read;
555 BaseIterator base()const
560 u8_to_u32_iterator() : m_position()
562 m_value = pending_read;
564 u8_to_u32_iterator(BaseIterator b) : m_position(b)
566 m_value = pending_read;
569 // Checked constructor:
571 u8_to_u32_iterator(BaseIterator b, BaseIterator start, BaseIterator end) : m_position(b)
573 m_value = pending_read;
575 // We must not start with a continuation character, or end with a
576 // truncated UTF-8 sequence otherwise we run the risk of going past
577 // the start/end of the underlying sequence:
581 unsigned char v = *start;
582 if((v & 0xC0u) == 0x80u)
584 if((b != start) && (b != end) && ((*b & 0xC0u) == 0x80u))
586 BaseIterator pos = end;
591 while((start != pos) && ((v & 0xC0u) == 0x80u));
592 std::ptrdiff_t extra = detail::utf8_byte_count(v);
593 if(std::distance(pos, end) < extra)
598 static void invalid_sequence()
600 std::out_of_range e("Invalid UTF-8 sequence encountered while trying to encode UTF-32 character");
601 boost::throw_exception(e);
603 void extract_current()const
605 m_value = static_cast<U32Type>(static_cast< ::boost::uint8_t>(*m_position));
606 // we must not have a continuation character:
607 if((m_value & 0xC0u) == 0x80u)
609 // see how many extra bytes we have:
610 unsigned extra = detail::utf8_trailing_byte_count(*m_position);
611 // extract the extra bits, 6 from each extra byte:
612 BaseIterator next(m_position);
613 for(unsigned c = 0; c < extra; ++c)
617 // We must have a continuation byte:
618 if((static_cast<boost::uint8_t>(*next) & 0xC0) != 0x80)
620 m_value += static_cast<boost::uint8_t>(*next) & 0x3Fu;
622 // we now need to remove a few of the leftmost bits, but how many depends
623 // upon how many extra bytes we've extracted:
624 static const boost::uint32_t masks[4] =
631 m_value &= masks[extra];
632 // check the result is in range:
633 if(m_value > static_cast<U32Type>(0x10FFFFu))
635 // The result must not be a surrogate:
636 if((m_value >= static_cast<U32Type>(0xD800)) && (m_value <= static_cast<U32Type>(0xDFFF)))
638 // We should not have had an invalidly encoded UTF8 sequence:
639 if((extra > 0) && (m_value <= static_cast<U32Type>(masks[extra - 1])))
642 BaseIterator m_position;
643 mutable U32Type m_value;
646 template <class BaseIterator>
647 class utf16_output_iterator
650 typedef void difference_type;
651 typedef void value_type;
652 typedef boost::uint32_t* pointer;
653 typedef boost::uint32_t& reference;
654 typedef std::output_iterator_tag iterator_category;
656 utf16_output_iterator(const BaseIterator& b)
658 utf16_output_iterator(const utf16_output_iterator& that)
659 : m_position(that.m_position){}
660 utf16_output_iterator& operator=(const utf16_output_iterator& that)
662 m_position = that.m_position;
665 const utf16_output_iterator& operator*()const
669 void operator=(boost::uint32_t val)const
673 utf16_output_iterator& operator++()
677 utf16_output_iterator& operator++(int)
681 BaseIterator base()const
686 void push(boost::uint32_t v)const
690 // begin by checking for a code point out of range:
692 detail::invalid_utf32_code_point(v);
693 // split into two surrogates:
694 *m_position++ = static_cast<boost::uint16_t>(v >> 10) + detail::high_surrogate_base;
695 *m_position++ = static_cast<boost::uint16_t>(v & detail::ten_bit_mask) + detail::low_surrogate_base;
699 // 16-bit code point:
700 // value must not be a surrogate:
701 if(detail::is_surrogate(v))
702 detail::invalid_utf32_code_point(v);
703 *m_position++ = static_cast<boost::uint16_t>(v);
706 mutable BaseIterator m_position;
709 template <class BaseIterator>
710 class utf8_output_iterator
713 typedef void difference_type;
714 typedef void value_type;
715 typedef boost::uint32_t* pointer;
716 typedef boost::uint32_t& reference;
717 typedef std::output_iterator_tag iterator_category;
719 utf8_output_iterator(const BaseIterator& b)
721 utf8_output_iterator(const utf8_output_iterator& that)
722 : m_position(that.m_position){}
723 utf8_output_iterator& operator=(const utf8_output_iterator& that)
725 m_position = that.m_position;
728 const utf8_output_iterator& operator*()const
732 void operator=(boost::uint32_t val)const
736 utf8_output_iterator& operator++()
740 utf8_output_iterator& operator++(int)
744 BaseIterator base()const
749 void push(boost::uint32_t c)const
752 detail::invalid_utf32_code_point(c);
755 *m_position++ = static_cast<unsigned char>(c);
759 *m_position++ = static_cast<unsigned char>(0xC0u + (c >> 6));
760 *m_position++ = static_cast<unsigned char>(0x80u + (c & 0x3Fu));
762 else if(c < 0x10000u)
764 *m_position++ = static_cast<unsigned char>(0xE0u + (c >> 12));
765 *m_position++ = static_cast<unsigned char>(0x80u + ((c >> 6) & 0x3Fu));
766 *m_position++ = static_cast<unsigned char>(0x80u + (c & 0x3Fu));
770 *m_position++ = static_cast<unsigned char>(0xF0u + (c >> 18));
771 *m_position++ = static_cast<unsigned char>(0x80u + ((c >> 12) & 0x3Fu));
772 *m_position++ = static_cast<unsigned char>(0x80u + ((c >> 6) & 0x3Fu));
773 *m_position++ = static_cast<unsigned char>(0x80u + (c & 0x3Fu));
776 mutable BaseIterator m_position;
781 #endif // BOOST_REGEX_UNICODE_ITERATOR_HPP