4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
22 * Copyright 2008 Sun Microsystems, Inc. All rights reserved.
23 * Use is subject to license terms.
30 * UTF-8 text preparation functions (PSARC/2007/149, PSARC/2007/458).
32 * Man pages: u8_textprep_open(9F), u8_textprep_buf(9F), u8_textprep_close(9F),
33 * u8_textprep_str(9F), u8_strcmp(9F), and u8_validate(9F). See also
34 * the section 3C man pages.
35 * Interface stability: Committed.
38 #include <sys/types.h>
39 #include <sys/strings.h>
40 #include <sys/param.h>
41 #include <sys/sysmacros.h>
42 #include <sys/debug.h>
44 #include <sys/sunddi.h>
45 #include <sys/u8_textprep.h>
46 #include <sys/byteorder.h>
47 #include <sys/errno.h>
48 #include <sys/u8_textprep_data.h>
51 /* The maximum possible number of bytes in a UTF-8 character. */
52 #define U8_MB_CUR_MAX (4)
55 * The maximum number of bytes needed for a UTF-8 character to cover
56 * U+0000 - U+FFFF, i.e., the coding space of now deprecated UCS-2.
58 #define U8_MAX_BYTES_UCS2 (3)
60 /* The maximum possible number of bytes in a Stream-Safe Text. */
61 #define U8_STREAM_SAFE_TEXT_MAX (128)
64 * The maximum number of characters in a combining/conjoining sequence and
65 * the actual upperbound limit of a combining/conjoining sequence.
67 #define U8_MAX_CHARS_A_SEQ (32)
68 #define U8_UPPER_LIMIT_IN_A_SEQ (31)
70 /* The combining class value for Starter. */
71 #define U8_COMBINING_CLASS_STARTER (0)
74 * Some Hangul related macros at below.
76 * The first and the last of Hangul syllables, Hangul Jamo Leading consonants,
77 * Vowels, and optional Trailing consonants in Unicode scalar values.
79 * Please be noted that the U8_HANGUL_JAMO_T_FIRST is 0x11A7 at below not
80 * the actual U+11A8. This is due to that the trailing consonant is optional
81 * and thus we are doing a pre-calculation of subtracting one.
83 * Each of 19 modern leading consonants has total 588 possible syllables since
84 * Hangul has 21 modern vowels and 27 modern trailing consonants plus 1 for
85 * no trailing consonant case, i.e., 21 x 28 = 588.
87 * We also have bunch of Hangul related macros at below. Please bear in mind
88 * that the U8_HANGUL_JAMO_1ST_BYTE can be used to check whether it is
89 * a Hangul Jamo or not but the value does not guarantee that it is a Hangul
90 * Jamo; it just guarantee that it will be most likely.
92 #define U8_HANGUL_SYL_FIRST (0xAC00U)
93 #define U8_HANGUL_SYL_LAST (0xD7A3U)
95 #define U8_HANGUL_JAMO_L_FIRST (0x1100U)
96 #define U8_HANGUL_JAMO_L_LAST (0x1112U)
97 #define U8_HANGUL_JAMO_V_FIRST (0x1161U)
98 #define U8_HANGUL_JAMO_V_LAST (0x1175U)
99 #define U8_HANGUL_JAMO_T_FIRST (0x11A7U)
100 #define U8_HANGUL_JAMO_T_LAST (0x11C2U)
102 #define U8_HANGUL_V_COUNT (21)
103 #define U8_HANGUL_VT_COUNT (588)
104 #define U8_HANGUL_T_COUNT (28)
106 #define U8_HANGUL_JAMO_1ST_BYTE (0xE1U)
108 #define U8_SAVE_HANGUL_AS_UTF8(s, i, j, k, b) \
109 (s)[(i)] = (uchar_t)(0xE0U | ((uint32_t)(b) & 0xF000U) >> 12); \
110 (s)[(j)] = (uchar_t)(0x80U | ((uint32_t)(b) & 0x0FC0U) >> 6); \
111 (s)[(k)] = (uchar_t)(0x80U | ((uint32_t)(b) & 0x003FU));
113 #define U8_HANGUL_JAMO_L(u) \
114 ((u) >= U8_HANGUL_JAMO_L_FIRST && (u) <= U8_HANGUL_JAMO_L_LAST)
116 #define U8_HANGUL_JAMO_V(u) \
117 ((u) >= U8_HANGUL_JAMO_V_FIRST && (u) <= U8_HANGUL_JAMO_V_LAST)
119 #define U8_HANGUL_JAMO_T(u) \
120 ((u) > U8_HANGUL_JAMO_T_FIRST && (u) <= U8_HANGUL_JAMO_T_LAST)
122 #define U8_HANGUL_JAMO(u) \
123 ((u) >= U8_HANGUL_JAMO_L_FIRST && (u) <= U8_HANGUL_JAMO_T_LAST)
125 #define U8_HANGUL_SYLLABLE(u) \
126 ((u) >= U8_HANGUL_SYL_FIRST && (u) <= U8_HANGUL_SYL_LAST)
128 #define U8_HANGUL_COMPOSABLE_L_V(s, u) \
129 ((s) == U8_STATE_HANGUL_L && U8_HANGUL_JAMO_V((u)))
131 #define U8_HANGUL_COMPOSABLE_LV_T(s, u) \
132 ((s) == U8_STATE_HANGUL_LV && U8_HANGUL_JAMO_T((u)))
134 /* The types of decomposition mappings. */
135 #define U8_DECOMP_BOTH (0xF5U)
136 #define U8_DECOMP_CANONICAL (0xF6U)
138 /* The indicator for 16-bit table. */
139 #define U8_16BIT_TABLE_INDICATOR (0x8000U)
141 /* The following are some convenience macros. */
142 #define U8_PUT_3BYTES_INTO_UTF32(u, b1, b2, b3) \
143 (u) = ((((uint32_t)(b1) & 0x0F) << 12) | \
144 (((uint32_t)(b2) & 0x3F) << 6) | \
145 ((uint32_t)(b3) & 0x3F));
147 #define U8_SIMPLE_SWAP(a, b, t) \
152 #define U8_ASCII_TOUPPER(c) \
153 (((c) >= 'a' && (c) <= 'z') ? (c) - 'a' + 'A' : (c))
155 #define U8_ASCII_TOLOWER(c) \
156 (((c) >= 'A' && (c) <= 'Z') ? (c) - 'A' + 'a' : (c))
158 #define U8_ISASCII(c) (((uchar_t)(c)) < 0x80U)
160 * The following macro assumes that the two characters that are to be
161 * swapped are adjacent to each other and 'a' comes before 'b'.
163 * If the assumptions are not met, then, the macro will fail.
165 #define U8_SWAP_COMB_MARKS(a, b) \
166 for (k = 0; k < disp[(a)]; k++) \
167 u8t[k] = u8s[start[(a)] + k]; \
168 for (k = 0; k < disp[(b)]; k++) \
169 u8s[start[(a)] + k] = u8s[start[(b)] + k]; \
170 start[(b)] = start[(a)] + disp[(b)]; \
171 for (k = 0; k < disp[(a)]; k++) \
172 u8s[start[(b)] + k] = u8t[k]; \
173 U8_SIMPLE_SWAP(comb_class[(a)], comb_class[(b)], tc); \
174 U8_SIMPLE_SWAP(disp[(a)], disp[(b)], tc);
176 /* The possible states during normalization. */
179 U8_STATE_HANGUL_L
= 1,
180 U8_STATE_HANGUL_LV
= 2,
181 U8_STATE_HANGUL_LVT
= 3,
182 U8_STATE_HANGUL_V
= 4,
183 U8_STATE_HANGUL_T
= 5,
184 U8_STATE_COMBINING_MARK
= 6
185 } u8_normalization_states_t
;
188 * The three vectors at below are used to check bytes of a given UTF-8
189 * character are valid and not containing any malformed byte values.
191 * We used to have a quite relaxed UTF-8 binary representation but then there
192 * was some security related issues and so the Unicode Consortium defined
193 * and announced the UTF-8 Corrigendum at Unicode 3.1 and then refined it
194 * one more time at the Unicode 3.2. The following three tables are based on
198 #define U8_ILLEGAL_NEXT_BYTE_COMMON(c) ((c) < 0x80 || (c) > 0xBF)
200 #define I_ U8_ILLEGAL_CHAR
201 #define O_ U8_OUT_OF_RANGE_CHAR
203 const int8_t u8_number_of_bytes
[0x100] = {
204 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
205 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
206 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
207 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
208 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
209 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
210 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
211 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
213 /* 80 81 82 83 84 85 86 87 88 89 8A 8B 8C 8D 8E 8F */
214 I_
, I_
, I_
, I_
, I_
, I_
, I_
, I_
, I_
, I_
, I_
, I_
, I_
, I_
, I_
, I_
,
216 /* 90 91 92 93 94 95 96 97 98 99 9A 9B 9C 9D 9E 9F */
217 I_
, I_
, I_
, I_
, I_
, I_
, I_
, I_
, I_
, I_
, I_
, I_
, I_
, I_
, I_
, I_
,
219 /* A0 A1 A2 A3 A4 A5 A6 A7 A8 A9 AA AB AC AD AE AF */
220 I_
, I_
, I_
, I_
, I_
, I_
, I_
, I_
, I_
, I_
, I_
, I_
, I_
, I_
, I_
, I_
,
222 /* B0 B1 B2 B3 B4 B5 B6 B7 B8 B9 BA BB BC BD BE BF */
223 I_
, I_
, I_
, I_
, I_
, I_
, I_
, I_
, I_
, I_
, I_
, I_
, I_
, I_
, I_
, I_
,
225 /* C0 C1 C2 C3 C4 C5 C6 C7 C8 C9 CA CB CC CD CE CF */
226 I_
, I_
, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
228 /* D0 D1 D2 D3 D4 D5 D6 D7 D8 D9 DA DB DC DD DE DF */
229 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
231 /* E0 E1 E2 E3 E4 E5 E6 E7 E8 E9 EA EB EC ED EE EF */
232 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
234 /* F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 FA FB FC FD FE FF */
235 4, 4, 4, 4, 4, O_
, O_
, O_
, O_
, O_
, O_
, O_
, O_
, O_
, O_
, O_
,
241 const uint8_t u8_valid_min_2nd_byte
[0x100] = {
242 0, 0, 0, 0, 0, 0, 0, 0,
243 0, 0, 0, 0, 0, 0, 0, 0,
244 0, 0, 0, 0, 0, 0, 0, 0,
245 0, 0, 0, 0, 0, 0, 0, 0,
246 0, 0, 0, 0, 0, 0, 0, 0,
247 0, 0, 0, 0, 0, 0, 0, 0,
248 0, 0, 0, 0, 0, 0, 0, 0,
249 0, 0, 0, 0, 0, 0, 0, 0,
250 0, 0, 0, 0, 0, 0, 0, 0,
251 0, 0, 0, 0, 0, 0, 0, 0,
252 0, 0, 0, 0, 0, 0, 0, 0,
253 0, 0, 0, 0, 0, 0, 0, 0,
254 0, 0, 0, 0, 0, 0, 0, 0,
255 0, 0, 0, 0, 0, 0, 0, 0,
256 0, 0, 0, 0, 0, 0, 0, 0,
257 0, 0, 0, 0, 0, 0, 0, 0,
258 0, 0, 0, 0, 0, 0, 0, 0,
259 0, 0, 0, 0, 0, 0, 0, 0,
260 0, 0, 0, 0, 0, 0, 0, 0,
261 0, 0, 0, 0, 0, 0, 0, 0,
262 0, 0, 0, 0, 0, 0, 0, 0,
263 0, 0, 0, 0, 0, 0, 0, 0,
264 0, 0, 0, 0, 0, 0, 0, 0,
265 0, 0, 0, 0, 0, 0, 0, 0,
266 /* C0 C1 C2 C3 C4 C5 C6 C7 */
267 0, 0, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80,
268 /* C8 C9 CA CB CC CD CE CF */
269 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80,
270 /* D0 D1 D2 D3 D4 D5 D6 D7 */
271 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80,
272 /* D8 D9 DA DB DC DD DE DF */
273 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80,
274 /* E0 E1 E2 E3 E4 E5 E6 E7 */
275 0xa0, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80,
276 /* E8 E9 EA EB EC ED EE EF */
277 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80,
278 /* F0 F1 F2 F3 F4 F5 F6 F7 */
279 0x90, 0x80, 0x80, 0x80, 0x80, 0, 0, 0,
280 0, 0, 0, 0, 0, 0, 0, 0,
283 const uint8_t u8_valid_max_2nd_byte
[0x100] = {
284 0, 0, 0, 0, 0, 0, 0, 0,
285 0, 0, 0, 0, 0, 0, 0, 0,
286 0, 0, 0, 0, 0, 0, 0, 0,
287 0, 0, 0, 0, 0, 0, 0, 0,
288 0, 0, 0, 0, 0, 0, 0, 0,
289 0, 0, 0, 0, 0, 0, 0, 0,
290 0, 0, 0, 0, 0, 0, 0, 0,
291 0, 0, 0, 0, 0, 0, 0, 0,
292 0, 0, 0, 0, 0, 0, 0, 0,
293 0, 0, 0, 0, 0, 0, 0, 0,
294 0, 0, 0, 0, 0, 0, 0, 0,
295 0, 0, 0, 0, 0, 0, 0, 0,
296 0, 0, 0, 0, 0, 0, 0, 0,
297 0, 0, 0, 0, 0, 0, 0, 0,
298 0, 0, 0, 0, 0, 0, 0, 0,
299 0, 0, 0, 0, 0, 0, 0, 0,
300 0, 0, 0, 0, 0, 0, 0, 0,
301 0, 0, 0, 0, 0, 0, 0, 0,
302 0, 0, 0, 0, 0, 0, 0, 0,
303 0, 0, 0, 0, 0, 0, 0, 0,
304 0, 0, 0, 0, 0, 0, 0, 0,
305 0, 0, 0, 0, 0, 0, 0, 0,
306 0, 0, 0, 0, 0, 0, 0, 0,
307 0, 0, 0, 0, 0, 0, 0, 0,
308 /* C0 C1 C2 C3 C4 C5 C6 C7 */
309 0, 0, 0xbf, 0xbf, 0xbf, 0xbf, 0xbf, 0xbf,
310 /* C8 C9 CA CB CC CD CE CF */
311 0xbf, 0xbf, 0xbf, 0xbf, 0xbf, 0xbf, 0xbf, 0xbf,
312 /* D0 D1 D2 D3 D4 D5 D6 D7 */
313 0xbf, 0xbf, 0xbf, 0xbf, 0xbf, 0xbf, 0xbf, 0xbf,
314 /* D8 D9 DA DB DC DD DE DF */
315 0xbf, 0xbf, 0xbf, 0xbf, 0xbf, 0xbf, 0xbf, 0xbf,
316 /* E0 E1 E2 E3 E4 E5 E6 E7 */
317 0xbf, 0xbf, 0xbf, 0xbf, 0xbf, 0xbf, 0xbf, 0xbf,
318 /* E8 E9 EA EB EC ED EE EF */
319 0xbf, 0xbf, 0xbf, 0xbf, 0xbf, 0x9f, 0xbf, 0xbf,
320 /* F0 F1 F2 F3 F4 F5 F6 F7 */
321 0xbf, 0xbf, 0xbf, 0xbf, 0x8f, 0, 0, 0,
322 0, 0, 0, 0, 0, 0, 0, 0,
327 * The u8_validate() validates on the given UTF-8 character string and
328 * calculate the byte length. It is quite similar to mblen(3C) except that
329 * this will validate against the list of characters if required and
330 * specific to UTF-8 and Unicode.
333 u8_validate(char *u8str
, size_t n
, char **list
, int flag
, int *errnum
)
345 boolean_t no_need_to_validate_entire
;
346 boolean_t check_additional
;
347 boolean_t validate_ucs2_range_only
;
352 ib
= (uchar_t
*)u8str
;
357 no_need_to_validate_entire
= ! (flag
& U8_VALIDATE_ENTIRE
);
358 check_additional
= flag
& U8_VALIDATE_CHECK_ADDITIONAL
;
359 validate_ucs2_range_only
= flag
& U8_VALIDATE_UCS2_RANGE
;
361 while (ib
< ibtail
) {
363 * The first byte of a UTF-8 character tells how many
364 * bytes will follow for the character. If the first byte
365 * is an illegal byte value or out of range value, we just
366 * return -1 with an appropriate error number.
368 sz
= u8_number_of_bytes
[*ib
];
369 if (sz
== U8_ILLEGAL_CHAR
) {
374 if (sz
== U8_OUT_OF_RANGE_CHAR
||
375 (validate_ucs2_range_only
&& sz
> U8_MAX_BYTES_UCS2
)) {
381 * If we don't have enough bytes to check on, that's also
382 * an error. As you can see, we give illegal byte sequence
383 * checking higher priority then EINVAL cases.
385 if ((ibtail
- ib
) < sz
) {
395 * Check on the multi-byte UTF-8 character. For more
396 * details on this, see comment added for the used
397 * data structures at the beginning of the file.
402 for (i
= 1; i
< sz
; i
++) {
404 if (*ib
< u8_valid_min_2nd_byte
[f
] ||
405 *ib
> u8_valid_max_2nd_byte
[f
]) {
410 } else if (U8_ILLEGAL_NEXT_BYTE_COMMON(*ib
)) {
419 if (check_additional
) {
420 for (p
= (uchar_t
**)list
, i
= 0; p
[i
]; i
++) {
424 if (*s1
!= *s2
|| *s2
== '\0')
430 if (s1
>= ib
&& *s2
== '\0') {
437 if (no_need_to_validate_entire
)
445 * The do_case_conv() looks at the mapping tables and returns found
446 * bytes if any. If not found, the input bytes are returned. The function
447 * always terminate the return bytes with a null character assuming that
448 * there are plenty of room to do so.
450 * The case conversions are simple case conversions mapping a character to
451 * another character as specified in the Unicode data. The byte size of
452 * the mapped character could be different from that of the input character.
454 * The return value is the byte length of the returned character excluding
455 * the terminating null byte.
458 do_case_conv(int uv
, uchar_t
*u8s
, uchar_t
*s
, int sz
, boolean_t is_it_toupper
)
471 * At this point, the only possible values for sz are 2, 3, and 4.
472 * The u8s should point to a vector that is well beyond the size of
478 } else if (sz
== 3) {
482 } else if (sz
== 4) {
488 /* This is not possible but just in case as a fallback. */
490 *u8s
= U8_ASCII_TOUPPER(*s
);
492 *u8s
= U8_ASCII_TOLOWER(*s
);
500 * Let's find out if we have a corresponding character.
502 b1
= u8_common_b1_tbl
[uv
][b1
];
503 if (b1
== U8_TBL_ELEMENT_NOT_DEF
)
506 b2
= u8_case_common_b2_tbl
[uv
][b1
][b2
];
507 if (b2
== U8_TBL_ELEMENT_NOT_DEF
)
511 b3_tbl
= u8_toupper_b3_tbl
[uv
][b2
][b3
].tbl_id
;
512 if (b3_tbl
== U8_TBL_ELEMENT_NOT_DEF
)
515 start_id
= u8_toupper_b4_tbl
[uv
][b3_tbl
][b4
];
516 end_id
= u8_toupper_b4_tbl
[uv
][b3_tbl
][b4
+ 1];
518 /* Either there is no match or an error at the table. */
519 if (start_id
>= end_id
|| (end_id
- start_id
) > U8_MB_CUR_MAX
)
522 b3_base
= u8_toupper_b3_tbl
[uv
][b2
][b3
].base
;
524 for (i
= 0; start_id
< end_id
; start_id
++)
525 u8s
[i
++] = u8_toupper_final_tbl
[uv
][b3_base
+ start_id
];
527 b3_tbl
= u8_tolower_b3_tbl
[uv
][b2
][b3
].tbl_id
;
528 if (b3_tbl
== U8_TBL_ELEMENT_NOT_DEF
)
531 start_id
= u8_tolower_b4_tbl
[uv
][b3_tbl
][b4
];
532 end_id
= u8_tolower_b4_tbl
[uv
][b3_tbl
][b4
+ 1];
534 if (start_id
>= end_id
|| (end_id
- start_id
) > U8_MB_CUR_MAX
)
537 b3_base
= u8_tolower_b3_tbl
[uv
][b2
][b3
].base
;
539 for (i
= 0; start_id
< end_id
; start_id
++)
540 u8s
[i
++] = u8_tolower_final_tbl
[uv
][b3_base
+ start_id
];
544 * If i is still zero, that means there is no corresponding character.
555 * The do_case_compare() function compares the two input strings, s1 and s2,
556 * one character at a time doing case conversions if applicable and return
557 * the comparison result as like strcmp().
559 * Since, in empirical sense, most of text data are 7-bit ASCII characters,
560 * we treat the 7-bit ASCII characters as a special case trying to yield
561 * faster processing time.
564 do_case_compare(size_t uv
, uchar_t
*s1
, uchar_t
*s2
, size_t n1
,
565 size_t n2
, boolean_t is_it_toupper
, int *errnum
)
573 uchar_t u8s1
[U8_MB_CUR_MAX
+ 1];
574 uchar_t u8s2
[U8_MB_CUR_MAX
+ 1];
577 while (i1
< n1
&& i2
< n2
) {
579 * Find out what would be the byte length for this UTF-8
580 * character at string s1 and also find out if this is
581 * an illegal start byte or not and if so, issue a proper
582 * error number and yet treat this byte as a character.
584 sz1
= u8_number_of_bytes
[*s1
];
591 * For 7-bit ASCII characters mainly, we do a quick case
592 * conversion right at here.
594 * If we don't have enough bytes for this character, issue
595 * an EINVAL error and use what are available.
597 * If we have enough bytes, find out if there is
598 * a corresponding uppercase character and if so, copy over
599 * the bytes for a comparison later. If there is no
600 * corresponding uppercase character, then, use what we have
601 * for the comparison.
605 u8s1
[0] = U8_ASCII_TOUPPER(*s1
);
607 u8s1
[0] = U8_ASCII_TOLOWER(*s1
);
610 } else if ((i1
+ sz1
) > n1
) {
612 for (j
= 0; (i1
+ j
) < n1
; )
616 (void) do_case_conv(uv
, u8s1
, s1
, sz1
, is_it_toupper
);
620 /* Do the same for the string s2. */
621 sz2
= u8_number_of_bytes
[*s2
];
629 u8s2
[0] = U8_ASCII_TOUPPER(*s2
);
631 u8s2
[0] = U8_ASCII_TOLOWER(*s2
);
634 } else if ((i2
+ sz2
) > n2
) {
636 for (j
= 0; (i2
+ j
) < n2
; )
640 (void) do_case_conv(uv
, u8s2
, s2
, sz2
, is_it_toupper
);
644 /* Now compare the two characters. */
645 if (sz1
== 1 && sz2
== 1) {
651 f
= strcmp((const char *)u8s1
, (const char *)u8s2
);
657 * They were the same. Let's move on to the next
665 * We compared until the end of either or both strings.
667 * If we reached to or went over the ends for the both, that means
670 * If we reached only one of the two ends, that means the other string
671 * has something which then the fact can be used to determine
683 * The combining_class() function checks on the given bytes and find out
684 * the corresponding Unicode combining class value. The return value 0 means
685 * it is a Starter. Any illegal UTF-8 character will also be treated as
689 combining_class(size_t uv
, uchar_t
*s
, size_t sz
)
696 if (sz
== 1 || sz
> 4)
702 } else if (sz
== 3) {
706 } else if (sz
== 4) {
713 b1
= u8_common_b1_tbl
[uv
][b1
];
714 if (b1
== U8_TBL_ELEMENT_NOT_DEF
)
717 b2
= u8_combining_class_b2_tbl
[uv
][b1
][b2
];
718 if (b2
== U8_TBL_ELEMENT_NOT_DEF
)
721 b3
= u8_combining_class_b3_tbl
[uv
][b2
][b3
];
722 if (b3
== U8_TBL_ELEMENT_NOT_DEF
)
725 return (u8_combining_class_b4_tbl
[uv
][b3
][b4
]);
729 * The do_decomp() function finds out a matching decomposition if any
730 * and return. If there is no match, the input bytes are copied and returned.
731 * The function also checks if there is a Hangul, decomposes it if necessary
734 * To save time, a single byte 7-bit ASCII character should be handled by
737 * The function returns the number of bytes returned sans always terminating
738 * the null byte. It will also return a state that will tell if there was
739 * a Hangul character decomposed which then will be used by the caller.
742 do_decomp(size_t uv
, uchar_t
*u8s
, uchar_t
*s
, int sz
,
743 boolean_t canonical_decomposition
, u8_normalization_states_t
*state
)
760 } else if (sz
== 3) {
761 /* Convert it to a Unicode scalar value. */
762 U8_PUT_3BYTES_INTO_UTF32(u1
, s
[0], s
[1], s
[2]);
765 * If this is a Hangul syllable, we decompose it into
766 * a leading consonant, a vowel, and an optional trailing
767 * consonant and then return.
769 if (U8_HANGUL_SYLLABLE(u1
)) {
770 u1
-= U8_HANGUL_SYL_FIRST
;
772 b1
= U8_HANGUL_JAMO_L_FIRST
+ u1
/ U8_HANGUL_VT_COUNT
;
773 b2
= U8_HANGUL_JAMO_V_FIRST
+ (u1
% U8_HANGUL_VT_COUNT
)
775 b3
= u1
% U8_HANGUL_T_COUNT
;
777 U8_SAVE_HANGUL_AS_UTF8(u8s
, 0, 1, 2, b1
);
778 U8_SAVE_HANGUL_AS_UTF8(u8s
, 3, 4, 5, b2
);
780 b3
+= U8_HANGUL_JAMO_T_FIRST
;
781 U8_SAVE_HANGUL_AS_UTF8(u8s
, 6, 7, 8, b3
);
784 *state
= U8_STATE_HANGUL_LVT
;
789 *state
= U8_STATE_HANGUL_LV
;
799 * If this is a Hangul Jamo, we know there is nothing
800 * further that we can decompose.
802 if (U8_HANGUL_JAMO_L(u1
)) {
803 *state
= U8_STATE_HANGUL_L
;
807 if (U8_HANGUL_JAMO_V(u1
)) {
808 if (*state
== U8_STATE_HANGUL_L
)
809 *state
= U8_STATE_HANGUL_LV
;
811 *state
= U8_STATE_HANGUL_V
;
815 if (U8_HANGUL_JAMO_T(u1
)) {
816 if (*state
== U8_STATE_HANGUL_LV
)
817 *state
= U8_STATE_HANGUL_LVT
;
819 *state
= U8_STATE_HANGUL_T
;
822 } else if (sz
== 4) {
830 * This is a fallback and should not happen if the function
831 * was called properly.
835 *state
= U8_STATE_START
;
840 * At this point, this routine does not know what it would get.
841 * The caller should sort it out if the state isn't a Hangul one.
843 *state
= U8_STATE_START
;
845 /* Try to find matching decomposition mapping byte sequence. */
846 b1
= u8_common_b1_tbl
[uv
][b1
];
847 if (b1
== U8_TBL_ELEMENT_NOT_DEF
)
850 b2
= u8_decomp_b2_tbl
[uv
][b1
][b2
];
851 if (b2
== U8_TBL_ELEMENT_NOT_DEF
)
854 b3_tbl
= u8_decomp_b3_tbl
[uv
][b2
][b3
].tbl_id
;
855 if (b3_tbl
== U8_TBL_ELEMENT_NOT_DEF
)
859 * If b3_tbl is bigger than or equal to U8_16BIT_TABLE_INDICATOR
860 * which is 0x8000, this means we couldn't fit the mappings into
861 * the cardinality of a unsigned byte.
863 if (b3_tbl
>= U8_16BIT_TABLE_INDICATOR
) {
864 b3_tbl
-= U8_16BIT_TABLE_INDICATOR
;
865 start_id
= u8_decomp_b4_16bit_tbl
[uv
][b3_tbl
][b4
];
866 end_id
= u8_decomp_b4_16bit_tbl
[uv
][b3_tbl
][b4
+ 1];
868 start_id
= u8_decomp_b4_tbl
[uv
][b3_tbl
][b4
];
869 end_id
= u8_decomp_b4_tbl
[uv
][b3_tbl
][b4
+ 1];
872 /* This also means there wasn't any matching decomposition. */
873 if (start_id
>= end_id
)
877 * The final table for decomposition mappings has three types of
878 * byte sequences depending on whether a mapping is for compatibility
879 * decomposition, canonical decomposition, or both like the following:
881 * (1) Compatibility decomposition mappings:
883 * +---+---+-...-+---+
884 * | B0| B1| ... | Bm|
885 * +---+---+-...-+---+
887 * The first byte, B0, is always less then 0xF5 (U8_DECOMP_BOTH).
889 * (2) Canonical decomposition mappings:
891 * +---+---+---+-...-+---+
892 * | T | b0| b1| ... | bn|
893 * +---+---+---+-...-+---+
895 * where the first byte, T, is 0xF6 (U8_DECOMP_CANONICAL).
899 * +---+---+---+---+-...-+---+---+---+-...-+---+
900 * | T | D | b0| b1| ... | bn| B0| B1| ... | Bm|
901 * +---+---+---+---+-...-+---+---+---+-...-+---+
903 * where T is 0xF5 (U8_DECOMP_BOTH) and D is a displacement
904 * byte, b0 to bn are canonical mapping bytes and B0 to Bm are
905 * compatibility mapping bytes.
907 * Note that compatibility decomposition means doing recursive
908 * decompositions using both compatibility decomposition mappings and
909 * canonical decomposition mappings. On the other hand, canonical
910 * decomposition means doing recursive decompositions using only
911 * canonical decomposition mappings. Since the table we have has gone
912 * through the recursions already, we do not need to do so during
913 * runtime, i.e., the table has been completely flattened out
917 b3_base
= u8_decomp_b3_tbl
[uv
][b2
][b3
].base
;
919 /* Get the type, T, of the byte sequence. */
920 b1
= u8_decomp_final_tbl
[uv
][b3_base
+ start_id
];
923 * If necessary, adjust start_id, end_id, or both. Note that if
924 * this is compatibility decomposition mapping, there is no
927 if (canonical_decomposition
) {
928 /* Is the mapping only for compatibility decomposition? */
929 if (b1
< U8_DECOMP_BOTH
)
934 if (b1
== U8_DECOMP_BOTH
) {
936 u8_decomp_final_tbl
[uv
][b3_base
+ start_id
];
941 * Unless this is a compatibility decomposition mapping,
942 * we adjust the start_id.
944 if (b1
== U8_DECOMP_BOTH
) {
946 start_id
+= u8_decomp_final_tbl
[uv
][b3_base
+ start_id
];
947 } else if (b1
== U8_DECOMP_CANONICAL
) {
952 for (i
= 0; start_id
< end_id
; start_id
++)
953 u8s
[i
++] = u8_decomp_final_tbl
[uv
][b3_base
+ start_id
];
960 * The find_composition_start() function uses the character bytes given and
961 * find out the matching composition mappings if any and return the address
962 * to the composition mappings as explained in the do_composition().
965 find_composition_start(size_t uv
, uchar_t
*s
, size_t sz
)
978 } else if (sz
== 2) {
981 } else if (sz
== 3) {
985 } else if (sz
== 4) {
992 * This is a fallback and should not happen if the function
993 * was called properly.
998 b1
= u8_composition_b1_tbl
[uv
][b1
];
999 if (b1
== U8_TBL_ELEMENT_NOT_DEF
)
1002 b2
= u8_composition_b2_tbl
[uv
][b1
][b2
];
1003 if (b2
== U8_TBL_ELEMENT_NOT_DEF
)
1006 b3_tbl
= u8_composition_b3_tbl
[uv
][b2
][b3
].tbl_id
;
1007 if (b3_tbl
== U8_TBL_ELEMENT_NOT_DEF
)
1010 if (b3_tbl
>= U8_16BIT_TABLE_INDICATOR
) {
1011 b3_tbl
-= U8_16BIT_TABLE_INDICATOR
;
1012 start_id
= u8_composition_b4_16bit_tbl
[uv
][b3_tbl
][b4
];
1013 end_id
= u8_composition_b4_16bit_tbl
[uv
][b3_tbl
][b4
+ 1];
1015 start_id
= u8_composition_b4_tbl
[uv
][b3_tbl
][b4
];
1016 end_id
= u8_composition_b4_tbl
[uv
][b3_tbl
][b4
+ 1];
1019 if (start_id
>= end_id
)
1022 b3_base
= u8_composition_b3_tbl
[uv
][b2
][b3
].base
;
1024 return ((uchar_t
*)&(u8_composition_final_tbl
[uv
][b3_base
+ start_id
]));
1028 * The blocked() function checks on the combining class values of previous
1029 * characters in this sequence and return whether it is blocked or not.
1032 blocked(uchar_t
*comb_class
, size_t last
)
1034 uchar_t my_comb_class
;
1037 my_comb_class
= comb_class
[last
];
1038 for (i
= 1; i
< last
; i
++)
1039 if (comb_class
[i
] >= my_comb_class
||
1040 comb_class
[i
] == U8_COMBINING_CLASS_STARTER
)
1047 * The do_composition() reads the character string pointed by 's' and
1048 * do necessary canonical composition and then copy over the result back to
1051 * The input argument 's' cannot contain more than 32 characters.
1054 do_composition(size_t uv
, uchar_t
*s
, uchar_t
*comb_class
, uchar_t
*start
,
1055 uchar_t
*disp
, size_t last
, uchar_t
**os
, uchar_t
*oslast
)
1057 uchar_t t
[U8_STREAM_SAFE_TEXT_MAX
+ 1];
1058 uchar_t tc
[U8_MB_CUR_MAX
] = { '\0' };
1059 uint8_t saved_marks
[U8_MAX_CHARS_A_SEQ
];
1060 size_t saved_marks_count
;
1074 boolean_t match_not_found
= B_TRUE
;
1077 * This should never happen unless the callers are doing some strange
1078 * and unexpected things.
1080 * The "last" is the index pointing to the last character not last + 1.
1082 if (last
>= U8_MAX_CHARS_A_SEQ
)
1083 last
= U8_UPPER_LIMIT_IN_A_SEQ
;
1085 for (i
= l
= 0; i
<= last
; i
++) {
1087 * The last or any non-Starters at the beginning, we don't
1088 * have any chance to do composition and so we just copy them
1089 * to the temporary buffer.
1091 if (i
>= last
|| comb_class
[i
] != U8_COMBINING_CLASS_STARTER
) {
1095 for (k
= 0; k
< size
; k
++)
1101 * If this could be a start of Hangul Jamos, then, we try to
1104 if (s
[start
[i
]] == U8_HANGUL_JAMO_1ST_BYTE
) {
1105 U8_PUT_3BYTES_INTO_UTF32(u1
, s
[start
[i
]],
1106 s
[start
[i
] + 1], s
[start
[i
] + 2]);
1107 U8_PUT_3BYTES_INTO_UTF32(u2
, s
[start
[i
] + 3],
1108 s
[start
[i
] + 4], s
[start
[i
] + 5]);
1110 if (U8_HANGUL_JAMO_L(u1
) && U8_HANGUL_JAMO_V(u2
)) {
1111 u1
-= U8_HANGUL_JAMO_L_FIRST
;
1112 u2
-= U8_HANGUL_JAMO_V_FIRST
;
1113 u1
= U8_HANGUL_SYL_FIRST
+
1114 (u1
* U8_HANGUL_V_COUNT
+ u2
) *
1119 U8_PUT_3BYTES_INTO_UTF32(u2
,
1120 s
[start
[i
]], s
[start
[i
] + 1],
1123 if (U8_HANGUL_JAMO_T(u2
)) {
1125 U8_HANGUL_JAMO_T_FIRST
;
1130 U8_SAVE_HANGUL_AS_UTF8(t
+ l
, 0, 1, 2, u1
);
1138 * Let's then find out if this Starter has composition
1141 p
= find_composition_start(uv
, s
+ start
[i
], disp
[i
]);
1146 * We have a Starter with composition mapping and the next
1147 * character is a non-Starter. Let's try to find out if
1148 * we can do composition.
1154 saved_marks_count
= 0;
1161 * The next for() loop compares the non-Starter pointed by
1162 * 'q' with the possible (joinable) characters pointed by 'p'.
1164 * The composition final table entry pointed by the 'p'
1165 * looks like the following:
1167 * +---+---+---+-...-+---+---+---+---+-...-+---+---+
1168 * | C | b0| b2| ... | bn| F | B0| B1| ... | Bm| F |
1169 * +---+---+---+-...-+---+---+---+---+-...-+---+---+
1171 * where C is the count byte indicating the number of
1172 * mapping pairs where each pair would be look like
1173 * (b0-bn F, B0-Bm F). The b0-bn are the bytes of the second
1174 * character of a canonical decomposition and the B0-Bm are
1175 * the bytes of a matching composite character. The F is
1176 * a filler byte after each character as the separator.
1179 match_not_found
= B_TRUE
;
1181 for (C
= *p
++; C
> 0; C
--) {
1182 for (k
= 0; k
< size
; p
++, k
++)
1186 /* Have we found it? */
1187 if (k
>= size
&& *p
== U8_TBL_ELEMENT_FILLER
) {
1188 match_not_found
= B_FALSE
;
1192 while (*++p
!= U8_TBL_ELEMENT_FILLER
)
1198 /* We didn't find; skip to the next pair. */
1199 if (*p
!= U8_TBL_ELEMENT_FILLER
)
1200 while (*++p
!= U8_TBL_ELEMENT_FILLER
)
1202 while (*++p
!= U8_TBL_ELEMENT_FILLER
)
1208 * If there was no match, we will need to save the combining
1209 * mark for later appending. After that, if the next one
1210 * is a non-Starter and not blocked, then, we try once
1211 * again to do composition with the next non-Starter.
1213 * If there was no match and this was a Starter, then,
1214 * this is a new start.
1216 * If there was a match and a composition done and we have
1217 * more to check on, then, we retrieve a new composition final
1218 * table entry for the composite and then try to do the
1219 * composition again.
1222 if (match_not_found
) {
1223 if (comb_class
[i
] == U8_COMBINING_CLASS_STARTER
) {
1228 saved_marks
[saved_marks_count
++] = i
;
1233 if (blocked(comb_class
, i
+ 1))
1234 saved_marks
[saved_marks_count
++] = ++i
;
1240 goto TRY_THE_NEXT_MARK
;
1242 } else if (i
< last
) {
1243 p
= find_composition_start(uv
, t
+ saved_l
,
1247 goto TRY_THE_NEXT_MARK
;
1252 * There is no more composition possible.
1254 * If there was no composition what so ever then we copy
1255 * over the original Starter and then append any non-Starters
1256 * remaining at the target string sequentially after that.
1260 p
= s
+ start
[saved_i
];
1261 size
= disp
[saved_i
];
1262 for (j
= 0; j
< size
; j
++)
1266 for (k
= 0; k
< saved_marks_count
; k
++) {
1267 p
= s
+ start
[saved_marks
[k
]];
1268 size
= disp
[saved_marks
[k
]];
1269 for (j
= 0; j
< size
; j
++)
1275 * If the last character is a Starter and if we have a character
1276 * (possibly another Starter) that can be turned into a composite,
1277 * we do so and we do so until there is no more of composition
1280 if (comb_class
[last
] == U8_COMBINING_CLASS_STARTER
) {
1282 saved_l
= l
- disp
[last
];
1284 while (p
< oslast
) {
1285 size
= u8_number_of_bytes
[*p
];
1286 if (size
<= 1 || (p
+ size
) > oslast
)
1291 for (i
= 0; i
< size
; i
++)
1294 q
= find_composition_start(uv
, t
+ saved_l
,
1301 match_not_found
= B_TRUE
;
1303 for (C
= *q
++; C
> 0; C
--) {
1304 for (k
= 0; k
< size
; q
++, k
++)
1308 if (k
>= size
&& *q
== U8_TBL_ELEMENT_FILLER
) {
1309 match_not_found
= B_FALSE
;
1313 while (*++q
!= U8_TBL_ELEMENT_FILLER
) {
1315 * This is practically
1316 * impossible but we don't
1317 * want to take any chances.
1320 U8_STREAM_SAFE_TEXT_MAX
) {
1330 if (*q
!= U8_TBL_ELEMENT_FILLER
)
1331 while (*++q
!= U8_TBL_ELEMENT_FILLER
)
1333 while (*++q
!= U8_TBL_ELEMENT_FILLER
)
1338 if (match_not_found
) {
1348 * Now we copy over the temporary string to the target string.
1349 * Since composition always reduces the number of characters or
1350 * the number of characters stay, we don't need to worry about
1351 * the buffer overflow here.
1353 for (i
= 0; i
< l
; i
++)
1361 * The collect_a_seq() function checks on the given string s, collect
1362 * a sequence of characters at u8s, and return the sequence. While it collects
1363 * a sequence, it also applies case conversion, canonical or compatibility
1364 * decomposition, canonical decomposition, or some or all of them and
1367 * The collected sequence cannot be bigger than 32 characters since if
1368 * it is having more than 31 characters, the sequence will be terminated
1369 * with a U+034F COMBINING GRAPHEME JOINER (CGJ) character and turned into
1370 * a Stream-Safe Text. The collected sequence is always terminated with
1371 * a null byte and the return value is the byte length of the sequence
1372 * including 0. The return value does not include the terminating
1376 collect_a_seq(size_t uv
, uchar_t
*u8s
, uchar_t
**source
, uchar_t
*slast
,
1377 boolean_t is_it_toupper
, boolean_t is_it_tolower
,
1378 boolean_t canonical_decomposition
, boolean_t compatibility_decomposition
,
1379 boolean_t canonical_composition
,
1380 int *errnum
, u8_normalization_states_t
*state
)
1389 uchar_t comb_class
[U8_MAX_CHARS_A_SEQ
];
1390 uchar_t disp
[U8_MAX_CHARS_A_SEQ
];
1391 uchar_t start
[U8_MAX_CHARS_A_SEQ
];
1392 uchar_t u8t
[U8_MB_CUR_MAX
] = { '\0' };
1393 uchar_t uts
[U8_STREAM_SAFE_TEXT_MAX
+ 1];
1400 * Save the source string pointer which we will return a changed
1401 * pointer if we do processing.
1406 * The following is a fallback for just in case callers are not
1407 * checking the string boundaries before the calling.
1416 * As the first thing, let's collect a character and do case
1417 * conversion if necessary.
1420 sz
= u8_number_of_bytes
[*s
];
1435 u8s
[0] = U8_ASCII_TOUPPER(*s
);
1436 else if (is_it_tolower
)
1437 u8s
[0] = U8_ASCII_TOLOWER(*s
);
1442 } else if ((s
+ sz
) > slast
) {
1445 for (i
= 0; s
< slast
; )
1453 if (is_it_toupper
|| is_it_tolower
) {
1454 i
= do_case_conv(uv
, u8s
, s
, sz
, is_it_toupper
);
1458 for (i
= 0; i
< sz
; )
1465 * And then canonical/compatibility decomposition followed by
1466 * an optional canonical composition. Please be noted that
1467 * canonical composition is done only when a decomposition is
1470 if (canonical_decomposition
|| compatibility_decomposition
) {
1472 *state
= U8_STATE_START
;
1482 saved_sz
= do_decomp(uv
, u8s
, u8s
, sz
,
1483 canonical_decomposition
, state
);
1487 for (i
= 0; i
< saved_sz
; ) {
1488 sz
= u8_number_of_bytes
[u8s
[i
]];
1490 comb_class
[last
] = combining_class(uv
,
1500 * Decomposition yields various Hangul related
1501 * states but not on combining marks. We need to
1502 * find out at here by checking on the last
1505 if (*state
== U8_STATE_START
) {
1506 if (comb_class
[last
- 1])
1507 *state
= U8_STATE_COMBINING_MARK
;
1514 sz
= u8_number_of_bytes
[*s
];
1517 * If this is an illegal character, an incomplete
1518 * character, or an 7-bit ASCII Starter character,
1519 * then we have collected a sequence; break and let
1520 * the next call deal with the two cases.
1522 * Note that this is okay only if you are using this
1523 * function with a fixed length string, not on
1524 * a buffer with multiple calls of one chunk at a time.
1528 } else if ((s
+ sz
) > slast
) {
1532 * If the previous character was a Hangul Jamo
1533 * and this character is a Hangul Jamo that
1534 * can be conjoined, we collect the Jamo.
1536 if (*s
== U8_HANGUL_JAMO_1ST_BYTE
) {
1537 U8_PUT_3BYTES_INTO_UTF32(u1
,
1538 *s
, *(s
+ 1), *(s
+ 2));
1540 if (U8_HANGUL_COMPOSABLE_L_V(*state
,
1543 *state
= U8_STATE_HANGUL_LV
;
1544 goto COLLECT_A_HANGUL
;
1547 if (U8_HANGUL_COMPOSABLE_LV_T(*state
,
1550 *state
= U8_STATE_HANGUL_LVT
;
1551 goto COLLECT_A_HANGUL
;
1556 * Regardless of whatever it was, if this is
1557 * a Starter, we don't collect the character
1558 * since that's a new start and we will deal
1559 * with it at the next time.
1561 i
= combining_class(uv
, s
, sz
);
1562 if (i
== U8_COMBINING_CLASS_STARTER
)
1566 * We know the current character is a combining
1567 * mark. If the previous character wasn't
1568 * a Starter (not Hangul) or a combining mark,
1569 * then, we don't collect this combining mark.
1571 if (*state
!= U8_STATE_START
&&
1572 *state
!= U8_STATE_COMBINING_MARK
)
1575 *state
= U8_STATE_COMBINING_MARK
;
1578 * If we collected a Starter and combining
1579 * marks up to 30, i.e., total 31 characters,
1580 * then, we terminate this degenerately long
1581 * combining sequence with a U+034F COMBINING
1582 * GRAPHEME JOINER (CGJ) which is 0xCD 0x8F in
1583 * UTF-8 and turn this into a Stream-Safe
1584 * Text. This will be extremely rare but
1587 * The following will also guarantee that
1588 * we are not writing more than 32 characters
1589 * plus a NULL at u8s[].
1591 if (last
>= U8_UPPER_LIMIT_IN_A_SEQ
) {
1593 *state
= U8_STATE_START
;
1594 comb_class
[last
] = 0;
1595 start
[last
] = saved_sz
;
1599 u8s
[saved_sz
++] = 0xCD;
1600 u8s
[saved_sz
++] = 0x8F;
1606 * Some combining marks also do decompose into
1607 * another combining mark or marks.
1609 if (*state
== U8_STATE_COMBINING_MARK
) {
1612 i
= do_decomp(uv
, uts
, s
, sz
,
1613 canonical_decomposition
, state
);
1614 for (j
= 0; j
< i
; ) {
1615 sz
= u8_number_of_bytes
[uts
[j
]];
1620 start
[last
] = saved_sz
+ j
;
1625 U8_UPPER_LIMIT_IN_A_SEQ
) {
1627 goto TURN_STREAM_SAFE
;
1632 *state
= U8_STATE_COMBINING_MARK
;
1636 for (i
= 0; i
< sz
; i
++)
1637 u8s
[saved_sz
++] = uts
[i
];
1639 comb_class
[last
] = i
;
1640 start
[last
] = saved_sz
;
1644 for (i
= 0; i
< sz
; i
++)
1645 u8s
[saved_sz
++] = *s
++;
1649 * If this is U+0345 COMBINING GREEK
1650 * YPOGEGRAMMENI (0xCD 0x85 in UTF-8), a.k.a.,
1651 * iota subscript, and need to be converted to
1652 * uppercase letter, convert it to U+0399 GREEK
1653 * CAPITAL LETTER IOTA (0xCE 0x99 in UTF-8),
1654 * i.e., convert to capital adscript form as
1655 * specified in the Unicode standard.
1657 * This is the only special case of (ambiguous)
1658 * case conversion at combining marks and
1659 * probably the standard will never have
1660 * anything similar like this in future.
1662 if (is_it_toupper
&& sz
>= 2 &&
1663 u8s
[saved_sz
- 2] == 0xCD &&
1664 u8s
[saved_sz
- 1] == 0x85) {
1665 u8s
[saved_sz
- 2] = 0xCE;
1666 u8s
[saved_sz
- 1] = 0x99;
1672 * Let's try to ensure a canonical ordering for the collected
1673 * combining marks. We do this only if we have collected
1674 * at least one more non-Starter. (The decomposition mapping
1675 * data tables have fully (and recursively) expanded and
1676 * canonically ordered decompositions.)
1678 * The U8_SWAP_COMB_MARKS() convenience macro has some
1679 * assumptions and we are meeting the assumptions.
1682 if (last
>= saved_last
) {
1683 for (i
= 0; i
< last
; i
++)
1684 for (j
= last
; j
> i
; j
--)
1685 if (comb_class
[j
] &&
1686 comb_class
[j
- 1] > comb_class
[j
]) {
1687 U8_SWAP_COMB_MARKS(j
- 1, j
);
1693 if (! canonical_composition
) {
1694 u8s
[saved_sz
] = '\0';
1699 * Now do the canonical composition. Note that we do this
1700 * only after a canonical or compatibility decomposition to
1701 * finish up NFC or NFKC.
1703 sz
= do_composition(uv
, u8s
, comb_class
, start
, disp
, last
,
1709 return ((size_t)sz
);
1713 * The do_norm_compare() function does string comparion based on Unicode
1714 * simple case mappings and Unicode Normalization definitions.
1716 * It does so by collecting a sequence of character at a time and comparing
1717 * the collected sequences from the strings.
1719 * The meanings on the return values are the same as the usual strcmp().
1722 do_norm_compare(size_t uv
, uchar_t
*s1
, uchar_t
*s2
, size_t n1
, size_t n2
,
1723 int flag
, int *errnum
)
1728 uchar_t u8s1
[U8_STREAM_SAFE_TEXT_MAX
+ 1];
1729 uchar_t u8s2
[U8_STREAM_SAFE_TEXT_MAX
+ 1];
1732 boolean_t is_it_toupper
;
1733 boolean_t is_it_tolower
;
1734 boolean_t canonical_decomposition
;
1735 boolean_t compatibility_decomposition
;
1736 boolean_t canonical_composition
;
1737 u8_normalization_states_t state
;
1742 is_it_toupper
= flag
& U8_TEXTPREP_TOUPPER
;
1743 is_it_tolower
= flag
& U8_TEXTPREP_TOLOWER
;
1744 canonical_decomposition
= flag
& U8_CANON_DECOMP
;
1745 compatibility_decomposition
= flag
& U8_COMPAT_DECOMP
;
1746 canonical_composition
= flag
& U8_CANON_COMP
;
1748 while (s1
< s1last
&& s2
< s2last
) {
1750 * If the current character is a 7-bit ASCII and the last
1751 * character, or, if the current character and the next
1752 * character are both some 7-bit ASCII characters then
1753 * we treat the current character as a sequence.
1755 * In any other cases, we need to call collect_a_seq().
1758 if (U8_ISASCII(*s1
) && ((s1
+ 1) >= s1last
||
1759 ((s1
+ 1) < s1last
&& U8_ISASCII(*(s1
+ 1))))) {
1761 u8s1
[0] = U8_ASCII_TOUPPER(*s1
);
1762 else if (is_it_tolower
)
1763 u8s1
[0] = U8_ASCII_TOLOWER(*s1
);
1770 state
= U8_STATE_START
;
1771 sz1
= collect_a_seq(uv
, u8s1
, &s1
, s1last
,
1772 is_it_toupper
, is_it_tolower
,
1773 canonical_decomposition
,
1774 compatibility_decomposition
,
1775 canonical_composition
, errnum
, &state
);
1778 if (U8_ISASCII(*s2
) && ((s2
+ 1) >= s2last
||
1779 ((s2
+ 1) < s2last
&& U8_ISASCII(*(s2
+ 1))))) {
1781 u8s2
[0] = U8_ASCII_TOUPPER(*s2
);
1782 else if (is_it_tolower
)
1783 u8s2
[0] = U8_ASCII_TOLOWER(*s2
);
1790 state
= U8_STATE_START
;
1791 sz2
= collect_a_seq(uv
, u8s2
, &s2
, s2last
,
1792 is_it_toupper
, is_it_tolower
,
1793 canonical_decomposition
,
1794 compatibility_decomposition
,
1795 canonical_composition
, errnum
, &state
);
1799 * Now compare the two characters. If they are the same,
1800 * we move on to the next character sequences.
1802 if (sz1
== 1 && sz2
== 1) {
1808 result
= strcmp((const char *)u8s1
, (const char *)u8s2
);
1815 * We compared until the end of either or both strings.
1817 * If we reached to or went over the ends for the both, that means
1818 * they are the same.
1820 * If we reached only one end, that means the other string has
1821 * something which then can be used to determine the return value.
1832 * The u8_strcmp() function compares two UTF-8 strings quite similar to
1833 * the strcmp(). For the comparison, however, Unicode Normalization specific
1834 * equivalency and Unicode simple case conversion mappings based equivalency
1835 * can be requested and checked against.
1838 u8_strcmp(const char *s1
, const char *s2
, size_t n
, int flag
, size_t uv
,
1848 * Check on the requested Unicode version, case conversion, and
1849 * normalization flag values.
1852 if (uv
> U8_UNICODE_LATEST
) {
1854 uv
= U8_UNICODE_LATEST
;
1858 flag
= U8_STRCMP_CS
;
1860 f
= flag
& (U8_STRCMP_CS
| U8_STRCMP_CI_UPPER
|
1861 U8_STRCMP_CI_LOWER
);
1863 flag
|= U8_STRCMP_CS
;
1864 } else if (f
!= U8_STRCMP_CS
&& f
!= U8_STRCMP_CI_UPPER
&&
1865 f
!= U8_STRCMP_CI_LOWER
) {
1867 flag
= U8_STRCMP_CS
;
1870 f
= flag
& (U8_CANON_DECOMP
| U8_COMPAT_DECOMP
| U8_CANON_COMP
);
1871 if (f
&& f
!= U8_STRCMP_NFD
&& f
!= U8_STRCMP_NFC
&&
1872 f
!= U8_STRCMP_NFKD
&& f
!= U8_STRCMP_NFKC
) {
1874 flag
= U8_STRCMP_CS
;
1878 if (flag
== U8_STRCMP_CS
) {
1879 return (n
== 0 ? strcmp(s1
, s2
) : strncmp(s1
, s2
, n
));
1892 * Simple case conversion can be done much faster and so we do
1893 * them separately here.
1895 if (flag
== U8_STRCMP_CI_UPPER
) {
1896 return (do_case_compare(uv
, (uchar_t
*)s1
, (uchar_t
*)s2
,
1897 n1
, n2
, B_TRUE
, errnum
));
1898 } else if (flag
== U8_STRCMP_CI_LOWER
) {
1899 return (do_case_compare(uv
, (uchar_t
*)s1
, (uchar_t
*)s2
,
1900 n1
, n2
, B_FALSE
, errnum
));
1903 return (do_norm_compare(uv
, (uchar_t
*)s1
, (uchar_t
*)s2
, n1
, n2
,
1908 u8_textprep_str(char *inarray
, size_t *inlen
, char *outarray
, size_t *outlen
,
1909 int flag
, size_t unicode_version
, int *errnum
)
1917 boolean_t do_not_ignore_null
;
1918 boolean_t do_not_ignore_invalid
;
1919 boolean_t is_it_toupper
;
1920 boolean_t is_it_tolower
;
1921 boolean_t canonical_decomposition
;
1922 boolean_t compatibility_decomposition
;
1923 boolean_t canonical_composition
;
1927 uchar_t u8s
[U8_STREAM_SAFE_TEXT_MAX
+ 1];
1928 u8_normalization_states_t state
;
1930 if (unicode_version
> U8_UNICODE_LATEST
) {
1932 return ((size_t)-1);
1935 f
= flag
& (U8_TEXTPREP_TOUPPER
| U8_TEXTPREP_TOLOWER
);
1936 if (f
== (U8_TEXTPREP_TOUPPER
| U8_TEXTPREP_TOLOWER
)) {
1938 return ((size_t)-1);
1941 f
= flag
& (U8_CANON_DECOMP
| U8_COMPAT_DECOMP
| U8_CANON_COMP
);
1942 if (f
&& f
!= U8_TEXTPREP_NFD
&& f
!= U8_TEXTPREP_NFC
&&
1943 f
!= U8_TEXTPREP_NFKD
&& f
!= U8_TEXTPREP_NFKC
) {
1945 return ((size_t)-1);
1948 if (inarray
== NULL
|| *inlen
== 0)
1951 if (outarray
== NULL
) {
1953 return ((size_t)-1);
1956 ib
= (uchar_t
*)inarray
;
1957 ob
= (uchar_t
*)outarray
;
1958 ibtail
= ib
+ *inlen
;
1959 obtail
= ob
+ *outlen
;
1961 do_not_ignore_null
= !(flag
& U8_TEXTPREP_IGNORE_NULL
);
1962 do_not_ignore_invalid
= !(flag
& U8_TEXTPREP_IGNORE_INVALID
);
1963 is_it_toupper
= flag
& U8_TEXTPREP_TOUPPER
;
1964 is_it_tolower
= flag
& U8_TEXTPREP_TOLOWER
;
1969 * If we don't have a normalization flag set, we do the simple case
1970 * conversion based text preparation separately below. Text
1971 * preparation involving Normalization will be done in the false task
1972 * block, again, separately since it will take much more time and
1973 * resource than doing simple case conversions.
1976 while (ib
< ibtail
) {
1977 if (*ib
== '\0' && do_not_ignore_null
)
1980 sz
= u8_number_of_bytes
[*ib
];
1983 if (do_not_ignore_invalid
) {
1985 ret_val
= (size_t)-1;
1996 ret_val
= (size_t)-1;
2001 *ob
= U8_ASCII_TOUPPER(*ib
);
2002 else if (is_it_tolower
)
2003 *ob
= U8_ASCII_TOLOWER(*ib
);
2008 } else if ((ib
+ sz
) > ibtail
) {
2009 if (do_not_ignore_invalid
) {
2011 ret_val
= (size_t)-1;
2015 if ((obtail
- ob
) < (ibtail
- ib
)) {
2017 ret_val
= (size_t)-1;
2022 * We treat the remaining incomplete character
2023 * bytes as a character.
2030 if (is_it_toupper
|| is_it_tolower
) {
2031 i
= do_case_conv(unicode_version
, u8s
,
2032 ib
, sz
, is_it_toupper
);
2034 if ((obtail
- ob
) < i
) {
2036 ret_val
= (size_t)-1;
2042 for (sz
= 0; sz
< i
; sz
++)
2045 if ((obtail
- ob
) < sz
) {
2047 ret_val
= (size_t)-1;
2051 for (i
= 0; i
< sz
; i
++)
2057 canonical_decomposition
= flag
& U8_CANON_DECOMP
;
2058 compatibility_decomposition
= flag
& U8_COMPAT_DECOMP
;
2059 canonical_composition
= flag
& U8_CANON_COMP
;
2061 while (ib
< ibtail
) {
2062 if (*ib
== '\0' && do_not_ignore_null
)
2066 * If the current character is a 7-bit ASCII
2067 * character and it is the last character, or,
2068 * if the current character is a 7-bit ASCII
2069 * character and the next character is also a 7-bit
2070 * ASCII character, then, we copy over this
2071 * character without going through collect_a_seq().
2073 * In any other cases, we need to look further with
2074 * the collect_a_seq() function.
2076 if (U8_ISASCII(*ib
) && ((ib
+ 1) >= ibtail
||
2077 ((ib
+ 1) < ibtail
&& U8_ISASCII(*(ib
+ 1))))) {
2080 ret_val
= (size_t)-1;
2085 *ob
= U8_ASCII_TOUPPER(*ib
);
2086 else if (is_it_tolower
)
2087 *ob
= U8_ASCII_TOLOWER(*ib
);
2094 state
= U8_STATE_START
;
2096 j
= collect_a_seq(unicode_version
, u8s
,
2100 canonical_decomposition
,
2101 compatibility_decomposition
,
2102 canonical_composition
,
2105 if (*errnum
&& do_not_ignore_invalid
) {
2106 ret_val
= (size_t)-1;
2110 if ((obtail
- ob
) < j
) {
2112 ret_val
= (size_t)-1;
2116 for (i
= 0; i
< j
; i
++)
2122 *inlen
= ibtail
- ib
;
2123 *outlen
= obtail
- ob
;
2128 #if defined(_KERNEL)
2140 module_init(unicode_init
);
2141 module_exit(unicode_fini
);
2143 MODULE_DESCRIPTION("Unicode implementation");
2144 MODULE_AUTHOR(ZFS_META_AUTHOR
);
2145 MODULE_LICENSE(ZFS_META_LICENSE
);
2146 MODULE_VERSION(ZFS_META_VERSION
"-" ZFS_META_RELEASE
);
2148 EXPORT_SYMBOL(u8_validate
);
2149 EXPORT_SYMBOL(u8_strcmp
);
2150 EXPORT_SYMBOL(u8_textprep_str
);