1 // Copyright Mozilla Foundation. See the COPYRIGHT
2 // file at the top-level directory of this distribution.
4 // Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
5 // https://www.apache.org/licenses/LICENSE-2.0> or the MIT license
6 // <LICENSE-MIT or https://opensource.org/licenses/MIT>, at your
7 // option. This file may not be copied, modified, or distributed
8 // except according to those terms.
11 use crate::handles
::*;
12 use crate::variant
::*;
15 if #[cfg(feature = "simd-accel")] {
17 use packed_simd
::u16x8
;
20 fn shift_upper(unpacked
: u16x8
) -> u16x8
{
21 let highest_ascii
= u16x8
::splat(0x7F);
22 unpacked
+ unpacked
.gt(highest_ascii
).select(u16x8
::splat(0xF700), u16x8
::splat(0)) }
27 pub struct UserDefinedDecoder
;
29 impl UserDefinedDecoder
{
30 pub fn new() -> VariantDecoder
{
31 VariantDecoder
::UserDefined(UserDefinedDecoder
)
34 pub fn max_utf16_buffer_length(&self, byte_length
: usize) -> Option
<usize> {
38 pub fn max_utf8_buffer_length_without_replacement(&self, byte_length
: usize) -> Option
<usize> {
39 byte_length
.checked_mul(3)
42 pub fn max_utf8_buffer_length(&self, byte_length
: usize) -> Option
<usize> {
43 byte_length
.checked_mul(3)
52 // ASCII run not optimized, because binary data expected
53 destination_handle
.write_ascii(b
);
56 destination_handle
.write_upper_bmp(u16::from(b
) + 0xF700);
72 #[cfg(not(feature = "simd-accel"))]
73 pub fn decode_to_utf16_raw(
78 ) -> (DecoderResult
, usize, usize) {
79 let (pending
, length
) = if dst
.len() < src
.len() {
80 (DecoderResult
::OutputFull
, dst
.len())
82 (DecoderResult
::InputEmpty
, src
.len())
84 let src_trim
= &src
[..length
];
85 let dst_trim
= &mut dst
[..length
];
88 .zip(dst_trim
.iter_mut())
89 .for_each(|(from
, to
)| {
95 u16::from(unit
) + 0xF700
99 (pending
, length
, length
)
102 #[cfg(feature = "simd-accel")]
103 pub fn decode_to_utf16_raw(
108 ) -> (DecoderResult
, usize, usize) {
109 let (pending
, length
) = if dst
.len() < src
.len() {
110 (DecoderResult
::OutputFull
, dst
.len())
112 (DecoderResult
::InputEmpty
, src
.len())
114 // Not bothering with alignment
115 let tail_start
= length
& !0xF;
116 let simd_iterations
= length
>> 4;
117 let src_ptr
= src
.as_ptr();
118 let dst_ptr
= dst
.as_mut_ptr();
119 for i
in 0..simd_iterations
{
120 let input
= unsafe { load16_unaligned(src_ptr.add(i * 16)) }
;
121 let (first
, second
) = simd_unpack(input
);
123 store8_unaligned(dst_ptr
.add(i
* 16), shift_upper(first
));
124 store8_unaligned(dst_ptr
.add((i
* 16) + 8), shift_upper(second
));
127 let src_tail
= &src
[tail_start
..length
];
128 let dst_tail
= &mut dst
[tail_start
..length
];
131 .zip(dst_tail
.iter_mut())
132 .for_each(|(from
, to
)| {
138 u16::from(unit
) + 0xF700
142 (pending
, length
, length
)
146 pub struct UserDefinedEncoder
;
148 impl UserDefinedEncoder
{
149 pub fn new(encoding
: &'
static Encoding
) -> Encoder
{
150 Encoder
::new(encoding
, VariantEncoder
::UserDefined(UserDefinedEncoder
))
153 pub fn max_buffer_length_from_utf16_without_replacement(
160 pub fn max_buffer_length_from_utf8_without_replacement(
171 // TODO optimize ASCII run
172 destination_handle
.write_one(c
as u8);
175 if c
< '
\u{F780}'
|| c
> '
\u{F7FF}'
{
177 EncoderResult
::Unmappable(c
),
178 unread_handle
.consumed(),
179 destination_handle
.written(),
182 destination_handle
.write_one((u32::from(c
) - 0xF700) as u8);
196 // Any copyright to the test code below this comment is dedicated to the
197 // Public Domain. http://creativecommons.org/publicdomain/zero/1.0/
199 #[cfg(all(test, feature = "alloc"))]
201 use super::super::testing
::*;
204 fn decode_x_user_defined(bytes
: &[u8], expect
: &str) {
205 decode(X_USER_DEFINED
, bytes
, expect
);
208 fn encode_x_user_defined(string
: &str, expect
: &[u8]) {
209 encode(X_USER_DEFINED
, string
, expect
);
213 fn test_x_user_defined_decode() {
215 decode_x_user_defined(b
"", "");
218 decode_x_user_defined(b
"\x61\x62", "\u{0061}\u{0062}");
220 decode_x_user_defined(b
"\x80\xFF", "\u{F780}\u{F7FF}");
221 decode_x_user_defined(b
"\x80\xFF\x61\x62\x80\xFF\x61\x62\x80\xFF\x61\x62\x80\xFF\x61\x62\x80\xFF\x61\x62", "\u{F780}\u{F7FF}\u{0061}\u{0062}\u{F780}\u{F7FF}\u{0061}\u{0062}\u{F780}\u{F7FF}\u{0061}\u{0062}\u{F780}\u{F7FF}\u{0061}\u{0062}\u{F780}\u{F7FF}\u{0061}\u{0062}");
225 fn test_x_user_defined_encode() {
227 encode_x_user_defined("", b
"");
230 encode_x_user_defined("\u{0061}\u{0062}", b
"\x61\x62");
232 encode_x_user_defined("\u{F780}\u{F7FF}", b
"\x80\xFF");
233 encode_x_user_defined("\u{F77F}\u{F800}", b
"");
237 fn test_x_user_defined_from_two_low_surrogates() {
238 let expectation
= b
"��";
239 let mut output
= [0u8; 40];
240 let mut encoder
= X_USER_DEFINED
.new_encoder();
241 let (result
, read
, written
, had_errors
) =
242 encoder
.encode_from_utf16(&[0xDC00u16, 0xDEDEu16], &mut output
[..], true);
243 assert_eq
!(result
, CoderResult
::InputEmpty
);
245 assert_eq
!(written
, expectation
.len());
247 assert_eq
!(&output
[..written
], expectation
);