4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
22 * Copyright 2008 Sun Microsystems, Inc. All rights reserved.
23 * Use is subject to license terms.
30 * UTF-8 text preparation functions (PSARC/2007/149, PSARC/2007/458).
32 * Man pages: u8_textprep_open(9F), u8_textprep_buf(9F), u8_textprep_close(9F),
33 * u8_textprep_str(9F), u8_strcmp(9F), and u8_validate(9F). See also
34 * the section 3C man pages.
35 * Interface stability: Committed.
38 #include <sys/types.h>
40 #include <sys/param.h>
41 #include <sys/sysmacros.h>
42 #include <sys/systm.h>
43 #include <sys/debug.h>
46 #include <sys/sunddi.h>
48 #include <sys/u8_textprep.h>
51 #include <sys/byteorder.h>
52 #include <sys/errno.h>
53 #include <sys/u8_textprep_data.h>
56 /* The maximum possible number of bytes in a UTF-8 character. */
57 #define U8_MB_CUR_MAX (4)
60 * The maximum number of bytes needed for a UTF-8 character to cover
61 * U+0000 - U+FFFF, i.e., the coding space of now deprecated UCS-2.
63 #define U8_MAX_BYTES_UCS2 (3)
65 /* The maximum possible number of bytes in a Stream-Safe Text. */
66 #define U8_STREAM_SAFE_TEXT_MAX (128)
69 * The maximum number of characters in a combining/conjoining sequence and
70 * the actual upperbound limit of a combining/conjoining sequence.
72 #define U8_MAX_CHARS_A_SEQ (32)
73 #define U8_UPPER_LIMIT_IN_A_SEQ (31)
75 /* The combining class value for Starter. */
76 #define U8_COMBINING_CLASS_STARTER (0)
79 * Some Hangul related macros at below.
81 * The first and the last of Hangul syllables, Hangul Jamo Leading consonants,
82 * Vowels, and optional Trailing consonants in Unicode scalar values.
84 * Please be noted that the U8_HANGUL_JAMO_T_FIRST is 0x11A7 at below not
85 * the actual U+11A8. This is due to that the trailing consonant is optional
86 * and thus we are doing a pre-calculation of subtracting one.
88 * Each of 19 modern leading consonants has total 588 possible syllables since
89 * Hangul has 21 modern vowels and 27 modern trailing consonants plus 1 for
90 * no trailing consonant case, i.e., 21 x 28 = 588.
92 * We also have bunch of Hangul related macros at below. Please bear in mind
93 * that the U8_HANGUL_JAMO_1ST_BYTE can be used to check whether it is
94 * a Hangul Jamo or not but the value does not guarantee that it is a Hangul
95 * Jamo; it just guarantee that it will be most likely.
97 #define U8_HANGUL_SYL_FIRST (0xAC00U)
98 #define U8_HANGUL_SYL_LAST (0xD7A3U)
100 #define U8_HANGUL_JAMO_L_FIRST (0x1100U)
101 #define U8_HANGUL_JAMO_L_LAST (0x1112U)
102 #define U8_HANGUL_JAMO_V_FIRST (0x1161U)
103 #define U8_HANGUL_JAMO_V_LAST (0x1175U)
104 #define U8_HANGUL_JAMO_T_FIRST (0x11A7U)
105 #define U8_HANGUL_JAMO_T_LAST (0x11C2U)
107 #define U8_HANGUL_V_COUNT (21)
108 #define U8_HANGUL_VT_COUNT (588)
109 #define U8_HANGUL_T_COUNT (28)
111 #define U8_HANGUL_JAMO_1ST_BYTE (0xE1U)
113 #define U8_SAVE_HANGUL_AS_UTF8(s, i, j, k, b) \
114 (s)[(i)] = (uchar_t)(0xE0U | ((uint32_t)(b) & 0xF000U) >> 12); \
115 (s)[(j)] = (uchar_t)(0x80U | ((uint32_t)(b) & 0x0FC0U) >> 6); \
116 (s)[(k)] = (uchar_t)(0x80U | ((uint32_t)(b) & 0x003FU));
118 #define U8_HANGUL_JAMO_L(u) \
119 ((u) >= U8_HANGUL_JAMO_L_FIRST && (u) <= U8_HANGUL_JAMO_L_LAST)
121 #define U8_HANGUL_JAMO_V(u) \
122 ((u) >= U8_HANGUL_JAMO_V_FIRST && (u) <= U8_HANGUL_JAMO_V_LAST)
124 #define U8_HANGUL_JAMO_T(u) \
125 ((u) > U8_HANGUL_JAMO_T_FIRST && (u) <= U8_HANGUL_JAMO_T_LAST)
127 #define U8_HANGUL_JAMO(u) \
128 ((u) >= U8_HANGUL_JAMO_L_FIRST && (u) <= U8_HANGUL_JAMO_T_LAST)
130 #define U8_HANGUL_SYLLABLE(u) \
131 ((u) >= U8_HANGUL_SYL_FIRST && (u) <= U8_HANGUL_SYL_LAST)
133 #define U8_HANGUL_COMPOSABLE_L_V(s, u) \
134 ((s) == U8_STATE_HANGUL_L && U8_HANGUL_JAMO_V((u)))
136 #define U8_HANGUL_COMPOSABLE_LV_T(s, u) \
137 ((s) == U8_STATE_HANGUL_LV && U8_HANGUL_JAMO_T((u)))
139 /* The types of decomposition mappings. */
140 #define U8_DECOMP_BOTH (0xF5U)
141 #define U8_DECOMP_CANONICAL (0xF6U)
143 /* The indicator for 16-bit table. */
144 #define U8_16BIT_TABLE_INDICATOR (0x8000U)
146 /* The following are some convenience macros. */
147 #define U8_PUT_3BYTES_INTO_UTF32(u, b1, b2, b3) \
148 (u) = ((((uint32_t)(b1) & 0x0F) << 12) | \
149 (((uint32_t)(b2) & 0x3F) << 6) | \
150 ((uint32_t)(b3) & 0x3F));
152 #define U8_SIMPLE_SWAP(a, b, t) \
157 #define U8_ASCII_TOUPPER(c) \
158 (((c) >= 'a' && (c) <= 'z') ? (c) - 'a' + 'A' : (c))
160 #define U8_ASCII_TOLOWER(c) \
161 (((c) >= 'A' && (c) <= 'Z') ? (c) - 'A' + 'a' : (c))
163 #define U8_ISASCII(c) (((uchar_t)(c)) < 0x80U)
165 * The following macro assumes that the two characters that are to be
166 * swapped are adjacent to each other and 'a' comes before 'b'.
168 * If the assumptions are not met, then, the macro will fail.
170 #define U8_SWAP_COMB_MARKS(a, b) \
171 for (k = 0; k < disp[(a)]; k++) \
172 u8t[k] = u8s[start[(a)] + k]; \
173 for (k = 0; k < disp[(b)]; k++) \
174 u8s[start[(a)] + k] = u8s[start[(b)] + k]; \
175 start[(b)] = start[(a)] + disp[(b)]; \
176 for (k = 0; k < disp[(a)]; k++) \
177 u8s[start[(b)] + k] = u8t[k]; \
178 U8_SIMPLE_SWAP(comb_class[(a)], comb_class[(b)], tc); \
179 U8_SIMPLE_SWAP(disp[(a)], disp[(b)], tc);
181 /* The possible states during normalization. */
184 U8_STATE_HANGUL_L
= 1,
185 U8_STATE_HANGUL_LV
= 2,
186 U8_STATE_HANGUL_LVT
= 3,
187 U8_STATE_HANGUL_V
= 4,
188 U8_STATE_HANGUL_T
= 5,
189 U8_STATE_COMBINING_MARK
= 6
190 } u8_normalization_states_t
;
193 * The three vectors at below are used to check bytes of a given UTF-8
194 * character are valid and not containing any malformed byte values.
196 * We used to have a quite relaxed UTF-8 binary representation but then there
197 * was some security related issues and so the Unicode Consortium defined
198 * and announced the UTF-8 Corrigendum at Unicode 3.1 and then refined it
199 * one more time at the Unicode 3.2. The following three tables are based on
203 #define U8_ILLEGAL_NEXT_BYTE_COMMON(c) ((c) < 0x80 || (c) > 0xBF)
205 #define I_ U8_ILLEGAL_CHAR
206 #define O_ U8_OUT_OF_RANGE_CHAR
208 const int8_t u8_number_of_bytes
[0x100] = {
209 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
210 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
211 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
212 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
213 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
214 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
215 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
216 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
218 /* 80 81 82 83 84 85 86 87 88 89 8A 8B 8C 8D 8E 8F */
219 I_
, I_
, I_
, I_
, I_
, I_
, I_
, I_
, I_
, I_
, I_
, I_
, I_
, I_
, I_
, I_
,
221 /* 90 91 92 93 94 95 96 97 98 99 9A 9B 9C 9D 9E 9F */
222 I_
, I_
, I_
, I_
, I_
, I_
, I_
, I_
, I_
, I_
, I_
, I_
, I_
, I_
, I_
, I_
,
224 /* A0 A1 A2 A3 A4 A5 A6 A7 A8 A9 AA AB AC AD AE AF */
225 I_
, I_
, I_
, I_
, I_
, I_
, I_
, I_
, I_
, I_
, I_
, I_
, I_
, I_
, I_
, I_
,
227 /* B0 B1 B2 B3 B4 B5 B6 B7 B8 B9 BA BB BC BD BE BF */
228 I_
, I_
, I_
, I_
, I_
, I_
, I_
, I_
, I_
, I_
, I_
, I_
, I_
, I_
, I_
, I_
,
230 /* C0 C1 C2 C3 C4 C5 C6 C7 C8 C9 CA CB CC CD CE CF */
231 I_
, I_
, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
233 /* D0 D1 D2 D3 D4 D5 D6 D7 D8 D9 DA DB DC DD DE DF */
234 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
236 /* E0 E1 E2 E3 E4 E5 E6 E7 E8 E9 EA EB EC ED EE EF */
237 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
239 /* F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 FA FB FC FD FE FF */
240 4, 4, 4, 4, 4, O_
, O_
, O_
, O_
, O_
, O_
, O_
, O_
, O_
, O_
, O_
,
246 const uint8_t u8_valid_min_2nd_byte
[0x100] = {
247 0, 0, 0, 0, 0, 0, 0, 0,
248 0, 0, 0, 0, 0, 0, 0, 0,
249 0, 0, 0, 0, 0, 0, 0, 0,
250 0, 0, 0, 0, 0, 0, 0, 0,
251 0, 0, 0, 0, 0, 0, 0, 0,
252 0, 0, 0, 0, 0, 0, 0, 0,
253 0, 0, 0, 0, 0, 0, 0, 0,
254 0, 0, 0, 0, 0, 0, 0, 0,
255 0, 0, 0, 0, 0, 0, 0, 0,
256 0, 0, 0, 0, 0, 0, 0, 0,
257 0, 0, 0, 0, 0, 0, 0, 0,
258 0, 0, 0, 0, 0, 0, 0, 0,
259 0, 0, 0, 0, 0, 0, 0, 0,
260 0, 0, 0, 0, 0, 0, 0, 0,
261 0, 0, 0, 0, 0, 0, 0, 0,
262 0, 0, 0, 0, 0, 0, 0, 0,
263 0, 0, 0, 0, 0, 0, 0, 0,
264 0, 0, 0, 0, 0, 0, 0, 0,
265 0, 0, 0, 0, 0, 0, 0, 0,
266 0, 0, 0, 0, 0, 0, 0, 0,
267 0, 0, 0, 0, 0, 0, 0, 0,
268 0, 0, 0, 0, 0, 0, 0, 0,
269 0, 0, 0, 0, 0, 0, 0, 0,
270 0, 0, 0, 0, 0, 0, 0, 0,
271 /* C0 C1 C2 C3 C4 C5 C6 C7 */
272 0, 0, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80,
273 /* C8 C9 CA CB CC CD CE CF */
274 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80,
275 /* D0 D1 D2 D3 D4 D5 D6 D7 */
276 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80,
277 /* D8 D9 DA DB DC DD DE DF */
278 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80,
279 /* E0 E1 E2 E3 E4 E5 E6 E7 */
280 0xa0, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80,
281 /* E8 E9 EA EB EC ED EE EF */
282 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80,
283 /* F0 F1 F2 F3 F4 F5 F6 F7 */
284 0x90, 0x80, 0x80, 0x80, 0x80, 0, 0, 0,
285 0, 0, 0, 0, 0, 0, 0, 0,
288 const uint8_t u8_valid_max_2nd_byte
[0x100] = {
289 0, 0, 0, 0, 0, 0, 0, 0,
290 0, 0, 0, 0, 0, 0, 0, 0,
291 0, 0, 0, 0, 0, 0, 0, 0,
292 0, 0, 0, 0, 0, 0, 0, 0,
293 0, 0, 0, 0, 0, 0, 0, 0,
294 0, 0, 0, 0, 0, 0, 0, 0,
295 0, 0, 0, 0, 0, 0, 0, 0,
296 0, 0, 0, 0, 0, 0, 0, 0,
297 0, 0, 0, 0, 0, 0, 0, 0,
298 0, 0, 0, 0, 0, 0, 0, 0,
299 0, 0, 0, 0, 0, 0, 0, 0,
300 0, 0, 0, 0, 0, 0, 0, 0,
301 0, 0, 0, 0, 0, 0, 0, 0,
302 0, 0, 0, 0, 0, 0, 0, 0,
303 0, 0, 0, 0, 0, 0, 0, 0,
304 0, 0, 0, 0, 0, 0, 0, 0,
305 0, 0, 0, 0, 0, 0, 0, 0,
306 0, 0, 0, 0, 0, 0, 0, 0,
307 0, 0, 0, 0, 0, 0, 0, 0,
308 0, 0, 0, 0, 0, 0, 0, 0,
309 0, 0, 0, 0, 0, 0, 0, 0,
310 0, 0, 0, 0, 0, 0, 0, 0,
311 0, 0, 0, 0, 0, 0, 0, 0,
312 0, 0, 0, 0, 0, 0, 0, 0,
313 /* C0 C1 C2 C3 C4 C5 C6 C7 */
314 0, 0, 0xbf, 0xbf, 0xbf, 0xbf, 0xbf, 0xbf,
315 /* C8 C9 CA CB CC CD CE CF */
316 0xbf, 0xbf, 0xbf, 0xbf, 0xbf, 0xbf, 0xbf, 0xbf,
317 /* D0 D1 D2 D3 D4 D5 D6 D7 */
318 0xbf, 0xbf, 0xbf, 0xbf, 0xbf, 0xbf, 0xbf, 0xbf,
319 /* D8 D9 DA DB DC DD DE DF */
320 0xbf, 0xbf, 0xbf, 0xbf, 0xbf, 0xbf, 0xbf, 0xbf,
321 /* E0 E1 E2 E3 E4 E5 E6 E7 */
322 0xbf, 0xbf, 0xbf, 0xbf, 0xbf, 0xbf, 0xbf, 0xbf,
323 /* E8 E9 EA EB EC ED EE EF */
324 0xbf, 0xbf, 0xbf, 0xbf, 0xbf, 0x9f, 0xbf, 0xbf,
325 /* F0 F1 F2 F3 F4 F5 F6 F7 */
326 0xbf, 0xbf, 0xbf, 0xbf, 0x8f, 0, 0, 0,
327 0, 0, 0, 0, 0, 0, 0, 0,
332 * The u8_validate() validates on the given UTF-8 character string and
333 * calculate the byte length. It is quite similar to mblen(3C) except that
334 * this will validate against the list of characters if required and
335 * specific to UTF-8 and Unicode.
338 u8_validate(char *u8str
, size_t n
, char **list
, int flag
, int *errnum
)
350 boolean_t no_need_to_validate_entire
;
351 boolean_t check_additional
;
352 boolean_t validate_ucs2_range_only
;
357 ib
= (uchar_t
*)u8str
;
362 no_need_to_validate_entire
= ! (flag
& U8_VALIDATE_ENTIRE
);
363 check_additional
= flag
& U8_VALIDATE_CHECK_ADDITIONAL
;
364 validate_ucs2_range_only
= flag
& U8_VALIDATE_UCS2_RANGE
;
366 while (ib
< ibtail
) {
368 * The first byte of a UTF-8 character tells how many
369 * bytes will follow for the character. If the first byte
370 * is an illegal byte value or out of range value, we just
371 * return -1 with an appropriate error number.
373 sz
= u8_number_of_bytes
[*ib
];
374 if (sz
== U8_ILLEGAL_CHAR
) {
379 if (sz
== U8_OUT_OF_RANGE_CHAR
||
380 (validate_ucs2_range_only
&& sz
> U8_MAX_BYTES_UCS2
)) {
386 * If we don't have enough bytes to check on, that's also
387 * an error. As you can see, we give illegal byte sequence
388 * checking higher priority then EINVAL cases.
390 if ((ibtail
- ib
) < sz
) {
400 * Check on the multi-byte UTF-8 character. For more
401 * details on this, see comment added for the used
402 * data structures at the beginning of the file.
407 for (i
= 1; i
< sz
; i
++) {
409 if (*ib
< u8_valid_min_2nd_byte
[f
] ||
410 *ib
> u8_valid_max_2nd_byte
[f
]) {
415 } else if (U8_ILLEGAL_NEXT_BYTE_COMMON(*ib
)) {
424 if (check_additional
) {
425 for (p
= (uchar_t
**)list
, i
= 0; p
[i
]; i
++) {
429 if (*s1
!= *s2
|| *s2
== '\0')
435 if (s1
>= ib
&& *s2
== '\0') {
442 if (no_need_to_validate_entire
)
450 * The do_case_conv() looks at the mapping tables and returns found
451 * bytes if any. If not found, the input bytes are returned. The function
452 * always terminate the return bytes with a null character assuming that
453 * there are plenty of room to do so.
455 * The case conversions are simple case conversions mapping a character to
456 * another character as specified in the Unicode data. The byte size of
457 * the mapped character could be different from that of the input character.
459 * The return value is the byte length of the returned character excluding
460 * the terminating null byte.
463 do_case_conv(int uv
, uchar_t
*u8s
, uchar_t
*s
, int sz
, boolean_t is_it_toupper
)
476 * At this point, the only possible values for sz are 2, 3, and 4.
477 * The u8s should point to a vector that is well beyond the size of
483 } else if (sz
== 3) {
487 } else if (sz
== 4) {
493 /* This is not possible but just in case as a fallback. */
495 *u8s
= U8_ASCII_TOUPPER(*s
);
497 *u8s
= U8_ASCII_TOLOWER(*s
);
505 * Let's find out if we have a corresponding character.
507 b1
= u8_common_b1_tbl
[uv
][b1
];
508 if (b1
== U8_TBL_ELEMENT_NOT_DEF
)
511 b2
= u8_case_common_b2_tbl
[uv
][b1
][b2
];
512 if (b2
== U8_TBL_ELEMENT_NOT_DEF
)
516 b3_tbl
= u8_toupper_b3_tbl
[uv
][b2
][b3
].tbl_id
;
517 if (b3_tbl
== U8_TBL_ELEMENT_NOT_DEF
)
520 start_id
= u8_toupper_b4_tbl
[uv
][b3_tbl
][b4
];
521 end_id
= u8_toupper_b4_tbl
[uv
][b3_tbl
][b4
+ 1];
523 /* Either there is no match or an error at the table. */
524 if (start_id
>= end_id
|| (end_id
- start_id
) > U8_MB_CUR_MAX
)
527 b3_base
= u8_toupper_b3_tbl
[uv
][b2
][b3
].base
;
529 for (i
= 0; start_id
< end_id
; start_id
++)
530 u8s
[i
++] = u8_toupper_final_tbl
[uv
][b3_base
+ start_id
];
532 b3_tbl
= u8_tolower_b3_tbl
[uv
][b2
][b3
].tbl_id
;
533 if (b3_tbl
== U8_TBL_ELEMENT_NOT_DEF
)
536 start_id
= u8_tolower_b4_tbl
[uv
][b3_tbl
][b4
];
537 end_id
= u8_tolower_b4_tbl
[uv
][b3_tbl
][b4
+ 1];
539 if (start_id
>= end_id
|| (end_id
- start_id
) > U8_MB_CUR_MAX
)
542 b3_base
= u8_tolower_b3_tbl
[uv
][b2
][b3
].base
;
544 for (i
= 0; start_id
< end_id
; start_id
++)
545 u8s
[i
++] = u8_tolower_final_tbl
[uv
][b3_base
+ start_id
];
549 * If i is still zero, that means there is no corresponding character.
560 * The do_case_compare() function compares the two input strings, s1 and s2,
561 * one character at a time doing case conversions if applicable and return
562 * the comparison result as like strcmp().
564 * Since, in empirical sense, most of text data are 7-bit ASCII characters,
565 * we treat the 7-bit ASCII characters as a special case trying to yield
566 * faster processing time.
569 do_case_compare(size_t uv
, uchar_t
*s1
, uchar_t
*s2
, size_t n1
,
570 size_t n2
, boolean_t is_it_toupper
, int *errnum
)
578 uchar_t u8s1
[U8_MB_CUR_MAX
+ 1];
579 uchar_t u8s2
[U8_MB_CUR_MAX
+ 1];
582 while (i1
< n1
&& i2
< n2
) {
584 * Find out what would be the byte length for this UTF-8
585 * character at string s1 and also find out if this is
586 * an illegal start byte or not and if so, issue a proper
587 * error number and yet treat this byte as a character.
589 sz1
= u8_number_of_bytes
[*s1
];
596 * For 7-bit ASCII characters mainly, we do a quick case
597 * conversion right at here.
599 * If we don't have enough bytes for this character, issue
600 * an EINVAL error and use what are available.
602 * If we have enough bytes, find out if there is
603 * a corresponding uppercase character and if so, copy over
604 * the bytes for a comparison later. If there is no
605 * corresponding uppercase character, then, use what we have
606 * for the comparison.
610 u8s1
[0] = U8_ASCII_TOUPPER(*s1
);
612 u8s1
[0] = U8_ASCII_TOLOWER(*s1
);
615 } else if ((i1
+ sz1
) > n1
) {
617 for (j
= 0; (i1
+ j
) < n1
; )
621 (void) do_case_conv(uv
, u8s1
, s1
, sz1
, is_it_toupper
);
625 /* Do the same for the string s2. */
626 sz2
= u8_number_of_bytes
[*s2
];
634 u8s2
[0] = U8_ASCII_TOUPPER(*s2
);
636 u8s2
[0] = U8_ASCII_TOLOWER(*s2
);
639 } else if ((i2
+ sz2
) > n2
) {
641 for (j
= 0; (i2
+ j
) < n2
; )
645 (void) do_case_conv(uv
, u8s2
, s2
, sz2
, is_it_toupper
);
649 /* Now compare the two characters. */
650 if (sz1
== 1 && sz2
== 1) {
656 f
= strcmp((const char *)u8s1
, (const char *)u8s2
);
662 * They were the same. Let's move on to the next
670 * We compared until the end of either or both strings.
672 * If we reached to or went over the ends for the both, that means
675 * If we reached only one of the two ends, that means the other string
676 * has something which then the fact can be used to determine
688 * The combining_class() function checks on the given bytes and find out
689 * the corresponding Unicode combining class value. The return value 0 means
690 * it is a Starter. Any illegal UTF-8 character will also be treated as
694 combining_class(size_t uv
, uchar_t
*s
, size_t sz
)
701 if (sz
== 1 || sz
> 4)
707 } else if (sz
== 3) {
711 } else if (sz
== 4) {
718 b1
= u8_common_b1_tbl
[uv
][b1
];
719 if (b1
== U8_TBL_ELEMENT_NOT_DEF
)
722 b2
= u8_combining_class_b2_tbl
[uv
][b1
][b2
];
723 if (b2
== U8_TBL_ELEMENT_NOT_DEF
)
726 b3
= u8_combining_class_b3_tbl
[uv
][b2
][b3
];
727 if (b3
== U8_TBL_ELEMENT_NOT_DEF
)
730 return (u8_combining_class_b4_tbl
[uv
][b3
][b4
]);
734 * The do_decomp() function finds out a matching decomposition if any
735 * and return. If there is no match, the input bytes are copied and returned.
736 * The function also checks if there is a Hangul, decomposes it if necessary
739 * To save time, a single byte 7-bit ASCII character should be handled by
742 * The function returns the number of bytes returned sans always terminating
743 * the null byte. It will also return a state that will tell if there was
744 * a Hangul character decomposed which then will be used by the caller.
747 do_decomp(size_t uv
, uchar_t
*u8s
, uchar_t
*s
, int sz
,
748 boolean_t canonical_decomposition
, u8_normalization_states_t
*state
)
765 } else if (sz
== 3) {
766 /* Convert it to a Unicode scalar value. */
767 U8_PUT_3BYTES_INTO_UTF32(u1
, s
[0], s
[1], s
[2]);
770 * If this is a Hangul syllable, we decompose it into
771 * a leading consonant, a vowel, and an optional trailing
772 * consonant and then return.
774 if (U8_HANGUL_SYLLABLE(u1
)) {
775 u1
-= U8_HANGUL_SYL_FIRST
;
777 b1
= U8_HANGUL_JAMO_L_FIRST
+ u1
/ U8_HANGUL_VT_COUNT
;
778 b2
= U8_HANGUL_JAMO_V_FIRST
+ (u1
% U8_HANGUL_VT_COUNT
)
780 b3
= u1
% U8_HANGUL_T_COUNT
;
782 U8_SAVE_HANGUL_AS_UTF8(u8s
, 0, 1, 2, b1
);
783 U8_SAVE_HANGUL_AS_UTF8(u8s
, 3, 4, 5, b2
);
785 b3
+= U8_HANGUL_JAMO_T_FIRST
;
786 U8_SAVE_HANGUL_AS_UTF8(u8s
, 6, 7, 8, b3
);
789 *state
= U8_STATE_HANGUL_LVT
;
794 *state
= U8_STATE_HANGUL_LV
;
804 * If this is a Hangul Jamo, we know there is nothing
805 * further that we can decompose.
807 if (U8_HANGUL_JAMO_L(u1
)) {
808 *state
= U8_STATE_HANGUL_L
;
812 if (U8_HANGUL_JAMO_V(u1
)) {
813 if (*state
== U8_STATE_HANGUL_L
)
814 *state
= U8_STATE_HANGUL_LV
;
816 *state
= U8_STATE_HANGUL_V
;
820 if (U8_HANGUL_JAMO_T(u1
)) {
821 if (*state
== U8_STATE_HANGUL_LV
)
822 *state
= U8_STATE_HANGUL_LVT
;
824 *state
= U8_STATE_HANGUL_T
;
827 } else if (sz
== 4) {
835 * This is a fallback and should not happen if the function
836 * was called properly.
840 *state
= U8_STATE_START
;
845 * At this point, this rountine does not know what it would get.
846 * The caller should sort it out if the state isn't a Hangul one.
848 *state
= U8_STATE_START
;
850 /* Try to find matching decomposition mapping byte sequence. */
851 b1
= u8_common_b1_tbl
[uv
][b1
];
852 if (b1
== U8_TBL_ELEMENT_NOT_DEF
)
855 b2
= u8_decomp_b2_tbl
[uv
][b1
][b2
];
856 if (b2
== U8_TBL_ELEMENT_NOT_DEF
)
859 b3_tbl
= u8_decomp_b3_tbl
[uv
][b2
][b3
].tbl_id
;
860 if (b3_tbl
== U8_TBL_ELEMENT_NOT_DEF
)
864 * If b3_tbl is bigger than or equal to U8_16BIT_TABLE_INDICATOR
865 * which is 0x8000, this means we couldn't fit the mappings into
866 * the cardinality of a unsigned byte.
868 if (b3_tbl
>= U8_16BIT_TABLE_INDICATOR
) {
869 b3_tbl
-= U8_16BIT_TABLE_INDICATOR
;
870 start_id
= u8_decomp_b4_16bit_tbl
[uv
][b3_tbl
][b4
];
871 end_id
= u8_decomp_b4_16bit_tbl
[uv
][b3_tbl
][b4
+ 1];
873 start_id
= u8_decomp_b4_tbl
[uv
][b3_tbl
][b4
];
874 end_id
= u8_decomp_b4_tbl
[uv
][b3_tbl
][b4
+ 1];
877 /* This also means there wasn't any matching decomposition. */
878 if (start_id
>= end_id
)
882 * The final table for decomposition mappings has three types of
883 * byte sequences depending on whether a mapping is for compatibility
884 * decomposition, canonical decomposition, or both like the following:
886 * (1) Compatibility decomposition mappings:
888 * +---+---+-...-+---+
889 * | B0| B1| ... | Bm|
890 * +---+---+-...-+---+
892 * The first byte, B0, is always less then 0xF5 (U8_DECOMP_BOTH).
894 * (2) Canonical decomposition mappings:
896 * +---+---+---+-...-+---+
897 * | T | b0| b1| ... | bn|
898 * +---+---+---+-...-+---+
900 * where the first byte, T, is 0xF6 (U8_DECOMP_CANONICAL).
904 * +---+---+---+---+-...-+---+---+---+-...-+---+
905 * | T | D | b0| b1| ... | bn| B0| B1| ... | Bm|
906 * +---+---+---+---+-...-+---+---+---+-...-+---+
908 * where T is 0xF5 (U8_DECOMP_BOTH) and D is a displacement
909 * byte, b0 to bn are canonical mapping bytes and B0 to Bm are
910 * compatibility mapping bytes.
912 * Note that compatibility decomposition means doing recursive
913 * decompositions using both compatibility decomposition mappings and
914 * canonical decomposition mappings. On the other hand, canonical
915 * decomposition means doing recursive decompositions using only
916 * canonical decomposition mappings. Since the table we have has gone
917 * through the recursions already, we do not need to do so during
918 * runtime, i.e., the table has been completely flattened out
922 b3_base
= u8_decomp_b3_tbl
[uv
][b2
][b3
].base
;
924 /* Get the type, T, of the byte sequence. */
925 b1
= u8_decomp_final_tbl
[uv
][b3_base
+ start_id
];
928 * If necessary, adjust start_id, end_id, or both. Note that if
929 * this is compatibility decomposition mapping, there is no
932 if (canonical_decomposition
) {
933 /* Is the mapping only for compatibility decomposition? */
934 if (b1
< U8_DECOMP_BOTH
)
939 if (b1
== U8_DECOMP_BOTH
) {
941 u8_decomp_final_tbl
[uv
][b3_base
+ start_id
];
946 * Unless this is a compatibility decomposition mapping,
947 * we adjust the start_id.
949 if (b1
== U8_DECOMP_BOTH
) {
951 start_id
+= u8_decomp_final_tbl
[uv
][b3_base
+ start_id
];
952 } else if (b1
== U8_DECOMP_CANONICAL
) {
957 for (i
= 0; start_id
< end_id
; start_id
++)
958 u8s
[i
++] = u8_decomp_final_tbl
[uv
][b3_base
+ start_id
];
965 * The find_composition_start() function uses the character bytes given and
966 * find out the matching composition mappings if any and return the address
967 * to the composition mappings as explained in the do_composition().
970 find_composition_start(size_t uv
, uchar_t
*s
, size_t sz
)
983 } else if (sz
== 2) {
986 } else if (sz
== 3) {
990 } else if (sz
== 4) {
997 * This is a fallback and should not happen if the function
998 * was called properly.
1003 b1
= u8_composition_b1_tbl
[uv
][b1
];
1004 if (b1
== U8_TBL_ELEMENT_NOT_DEF
)
1007 b2
= u8_composition_b2_tbl
[uv
][b1
][b2
];
1008 if (b2
== U8_TBL_ELEMENT_NOT_DEF
)
1011 b3_tbl
= u8_composition_b3_tbl
[uv
][b2
][b3
].tbl_id
;
1012 if (b3_tbl
== U8_TBL_ELEMENT_NOT_DEF
)
1015 if (b3_tbl
>= U8_16BIT_TABLE_INDICATOR
) {
1016 b3_tbl
-= U8_16BIT_TABLE_INDICATOR
;
1017 start_id
= u8_composition_b4_16bit_tbl
[uv
][b3_tbl
][b4
];
1018 end_id
= u8_composition_b4_16bit_tbl
[uv
][b3_tbl
][b4
+ 1];
1020 start_id
= u8_composition_b4_tbl
[uv
][b3_tbl
][b4
];
1021 end_id
= u8_composition_b4_tbl
[uv
][b3_tbl
][b4
+ 1];
1024 if (start_id
>= end_id
)
1027 b3_base
= u8_composition_b3_tbl
[uv
][b2
][b3
].base
;
1029 return ((uchar_t
*)&(u8_composition_final_tbl
[uv
][b3_base
+ start_id
]));
1033 * The blocked() function checks on the combining class values of previous
1034 * characters in this sequence and return whether it is blocked or not.
1037 blocked(uchar_t
*comb_class
, size_t last
)
1039 uchar_t my_comb_class
;
1042 my_comb_class
= comb_class
[last
];
1043 for (i
= 1; i
< last
; i
++)
1044 if (comb_class
[i
] >= my_comb_class
||
1045 comb_class
[i
] == U8_COMBINING_CLASS_STARTER
)
1052 * The do_composition() reads the character string pointed by 's' and
1053 * do necessary canonical composition and then copy over the result back to
1056 * The input argument 's' cannot contain more than 32 characters.
1059 do_composition(size_t uv
, uchar_t
*s
, uchar_t
*comb_class
, uchar_t
*start
,
1060 uchar_t
*disp
, size_t last
, uchar_t
**os
, uchar_t
*oslast
)
1062 uchar_t t
[U8_STREAM_SAFE_TEXT_MAX
+ 1];
1063 uchar_t tc
[U8_MB_CUR_MAX
] = { '\0' };
1064 uint8_t saved_marks
[U8_MAX_CHARS_A_SEQ
];
1065 size_t saved_marks_count
;
1079 boolean_t match_not_found
= B_TRUE
;
1082 * This should never happen unless the callers are doing some strange
1083 * and unexpected things.
1085 * The "last" is the index pointing to the last character not last + 1.
1087 if (last
>= U8_MAX_CHARS_A_SEQ
)
1088 last
= U8_UPPER_LIMIT_IN_A_SEQ
;
1090 for (i
= l
= 0; i
<= last
; i
++) {
1092 * The last or any non-Starters at the beginning, we don't
1093 * have any chance to do composition and so we just copy them
1094 * to the temporary buffer.
1096 if (i
>= last
|| comb_class
[i
] != U8_COMBINING_CLASS_STARTER
) {
1100 for (k
= 0; k
< size
; k
++)
1106 * If this could be a start of Hangul Jamos, then, we try to
1109 if (s
[start
[i
]] == U8_HANGUL_JAMO_1ST_BYTE
) {
1110 U8_PUT_3BYTES_INTO_UTF32(u1
, s
[start
[i
]],
1111 s
[start
[i
] + 1], s
[start
[i
] + 2]);
1112 U8_PUT_3BYTES_INTO_UTF32(u2
, s
[start
[i
] + 3],
1113 s
[start
[i
] + 4], s
[start
[i
] + 5]);
1115 if (U8_HANGUL_JAMO_L(u1
) && U8_HANGUL_JAMO_V(u2
)) {
1116 u1
-= U8_HANGUL_JAMO_L_FIRST
;
1117 u2
-= U8_HANGUL_JAMO_V_FIRST
;
1118 u1
= U8_HANGUL_SYL_FIRST
+
1119 (u1
* U8_HANGUL_V_COUNT
+ u2
) *
1124 U8_PUT_3BYTES_INTO_UTF32(u2
,
1125 s
[start
[i
]], s
[start
[i
] + 1],
1128 if (U8_HANGUL_JAMO_T(u2
)) {
1130 U8_HANGUL_JAMO_T_FIRST
;
1135 U8_SAVE_HANGUL_AS_UTF8(t
+ l
, 0, 1, 2, u1
);
1143 * Let's then find out if this Starter has composition
1146 p
= find_composition_start(uv
, s
+ start
[i
], disp
[i
]);
1151 * We have a Starter with composition mapping and the next
1152 * character is a non-Starter. Let's try to find out if
1153 * we can do composition.
1159 saved_marks_count
= 0;
1166 * The next for() loop compares the non-Starter pointed by
1167 * 'q' with the possible (joinable) characters pointed by 'p'.
1169 * The composition final table entry pointed by the 'p'
1170 * looks like the following:
1172 * +---+---+---+-...-+---+---+---+---+-...-+---+---+
1173 * | C | b0| b2| ... | bn| F | B0| B1| ... | Bm| F |
1174 * +---+---+---+-...-+---+---+---+---+-...-+---+---+
1176 * where C is the count byte indicating the number of
1177 * mapping pairs where each pair would be look like
1178 * (b0-bn F, B0-Bm F). The b0-bn are the bytes of the second
1179 * character of a canonical decomposition and the B0-Bm are
1180 * the bytes of a matching composite character. The F is
1181 * a filler byte after each character as the separator.
1184 match_not_found
= B_TRUE
;
1186 for (C
= *p
++; C
> 0; C
--) {
1187 for (k
= 0; k
< size
; p
++, k
++)
1191 /* Have we found it? */
1192 if (k
>= size
&& *p
== U8_TBL_ELEMENT_FILLER
) {
1193 match_not_found
= B_FALSE
;
1197 while (*++p
!= U8_TBL_ELEMENT_FILLER
)
1203 /* We didn't find; skip to the next pair. */
1204 if (*p
!= U8_TBL_ELEMENT_FILLER
)
1205 while (*++p
!= U8_TBL_ELEMENT_FILLER
)
1207 while (*++p
!= U8_TBL_ELEMENT_FILLER
)
1213 * If there was no match, we will need to save the combining
1214 * mark for later appending. After that, if the next one
1215 * is a non-Starter and not blocked, then, we try once
1216 * again to do composition with the next non-Starter.
1218 * If there was no match and this was a Starter, then,
1219 * this is a new start.
1221 * If there was a match and a composition done and we have
1222 * more to check on, then, we retrieve a new composition final
1223 * table entry for the composite and then try to do the
1224 * composition again.
1227 if (match_not_found
) {
1228 if (comb_class
[i
] == U8_COMBINING_CLASS_STARTER
) {
1233 saved_marks
[saved_marks_count
++] = i
;
1238 if (blocked(comb_class
, i
+ 1))
1239 saved_marks
[saved_marks_count
++] = ++i
;
1245 goto TRY_THE_NEXT_MARK
;
1247 } else if (i
< last
) {
1248 p
= find_composition_start(uv
, t
+ saved_l
,
1252 goto TRY_THE_NEXT_MARK
;
1257 * There is no more composition possible.
1259 * If there was no composition what so ever then we copy
1260 * over the original Starter and then append any non-Starters
1261 * remaining at the target string sequentially after that.
1265 p
= s
+ start
[saved_i
];
1266 size
= disp
[saved_i
];
1267 for (j
= 0; j
< size
; j
++)
1271 for (k
= 0; k
< saved_marks_count
; k
++) {
1272 p
= s
+ start
[saved_marks
[k
]];
1273 size
= disp
[saved_marks
[k
]];
1274 for (j
= 0; j
< size
; j
++)
1280 * If the last character is a Starter and if we have a character
1281 * (possibly another Starter) that can be turned into a composite,
1282 * we do so and we do so until there is no more of composition
1285 if (comb_class
[last
] == U8_COMBINING_CLASS_STARTER
) {
1287 saved_l
= l
- disp
[last
];
1289 while (p
< oslast
) {
1290 size
= u8_number_of_bytes
[*p
];
1291 if (size
<= 1 || (p
+ size
) > oslast
)
1296 for (i
= 0; i
< size
; i
++)
1299 q
= find_composition_start(uv
, t
+ saved_l
,
1306 match_not_found
= B_TRUE
;
1308 for (C
= *q
++; C
> 0; C
--) {
1309 for (k
= 0; k
< size
; q
++, k
++)
1313 if (k
>= size
&& *q
== U8_TBL_ELEMENT_FILLER
) {
1314 match_not_found
= B_FALSE
;
1318 while (*++q
!= U8_TBL_ELEMENT_FILLER
) {
1320 * This is practically
1321 * impossible but we don't
1322 * want to take any chances.
1325 U8_STREAM_SAFE_TEXT_MAX
) {
1335 if (*q
!= U8_TBL_ELEMENT_FILLER
)
1336 while (*++q
!= U8_TBL_ELEMENT_FILLER
)
1338 while (*++q
!= U8_TBL_ELEMENT_FILLER
)
1343 if (match_not_found
) {
1353 * Now we copy over the temporary string to the target string.
1354 * Since composition always reduces the number of characters or
1355 * the number of characters stay, we don't need to worry about
1356 * the buffer overflow here.
1358 for (i
= 0; i
< l
; i
++)
1366 * The collect_a_seq() function checks on the given string s, collect
1367 * a sequence of characters at u8s, and return the sequence. While it collects
1368 * a sequence, it also applies case conversion, canonical or compatibility
1369 * decomposition, canonical decomposition, or some or all of them and
1372 * The collected sequence cannot be bigger than 32 characters since if
1373 * it is having more than 31 characters, the sequence will be terminated
1374 * with a U+034F COMBINING GRAPHEME JOINER (CGJ) character and turned into
1375 * a Stream-Safe Text. The collected sequence is always terminated with
1376 * a null byte and the return value is the byte length of the sequence
1377 * including 0. The return value does not include the terminating
1381 collect_a_seq(size_t uv
, uchar_t
*u8s
, uchar_t
**source
, uchar_t
*slast
,
1382 boolean_t is_it_toupper
,
1383 boolean_t is_it_tolower
,
1384 boolean_t canonical_decomposition
,
1385 boolean_t compatibility_decomposition
,
1386 boolean_t canonical_composition
,
1387 int *errnum
, u8_normalization_states_t
*state
)
1396 uchar_t comb_class
[U8_MAX_CHARS_A_SEQ
];
1397 uchar_t disp
[U8_MAX_CHARS_A_SEQ
];
1398 uchar_t start
[U8_MAX_CHARS_A_SEQ
];
1399 uchar_t u8t
[U8_MB_CUR_MAX
] = { '\0' };
1400 uchar_t uts
[U8_STREAM_SAFE_TEXT_MAX
+ 1];
1407 * Save the source string pointer which we will return a changed
1408 * pointer if we do processing.
1413 * The following is a fallback for just in case callers are not
1414 * checking the string boundaries before the calling.
1423 * As the first thing, let's collect a character and do case
1424 * conversion if necessary.
1427 sz
= u8_number_of_bytes
[*s
];
1442 u8s
[0] = U8_ASCII_TOUPPER(*s
);
1443 else if (is_it_tolower
)
1444 u8s
[0] = U8_ASCII_TOLOWER(*s
);
1449 } else if ((s
+ sz
) > slast
) {
1452 for (i
= 0; s
< slast
; )
1460 if (is_it_toupper
|| is_it_tolower
) {
1461 i
= do_case_conv(uv
, u8s
, s
, sz
, is_it_toupper
);
1465 for (i
= 0; i
< sz
; )
1472 * And then canonical/compatibility decomposition followed by
1473 * an optional canonical composition. Please be noted that
1474 * canonical composition is done only when a decomposition is
1477 if (canonical_decomposition
|| compatibility_decomposition
) {
1479 *state
= U8_STATE_START
;
1489 saved_sz
= do_decomp(uv
, u8s
, u8s
, sz
,
1490 canonical_decomposition
, state
);
1494 for (i
= 0; i
< saved_sz
; ) {
1495 sz
= u8_number_of_bytes
[u8s
[i
]];
1497 comb_class
[last
] = combining_class(uv
,
1507 * Decomposition yields various Hangul related
1508 * states but not on combining marks. We need to
1509 * find out at here by checking on the last
1512 if (*state
== U8_STATE_START
) {
1513 if (comb_class
[last
- 1])
1514 *state
= U8_STATE_COMBINING_MARK
;
1521 sz
= u8_number_of_bytes
[*s
];
1524 * If this is an illegal character, an incomplete
1525 * character, or an 7-bit ASCII Starter character,
1526 * then we have collected a sequence; break and let
1527 * the next call deal with the two cases.
1529 * Note that this is okay only if you are using this
1530 * function with a fixed length string, not on
1531 * a buffer with multiple calls of one chunk at a time.
1535 } else if ((s
+ sz
) > slast
) {
1539 * If the previous character was a Hangul Jamo
1540 * and this character is a Hangul Jamo that
1541 * can be conjoined, we collect the Jamo.
1543 if (*s
== U8_HANGUL_JAMO_1ST_BYTE
) {
1544 U8_PUT_3BYTES_INTO_UTF32(u1
,
1545 *s
, *(s
+ 1), *(s
+ 2));
1547 if (U8_HANGUL_COMPOSABLE_L_V(*state
,
1550 *state
= U8_STATE_HANGUL_LV
;
1551 goto COLLECT_A_HANGUL
;
1554 if (U8_HANGUL_COMPOSABLE_LV_T(*state
,
1557 *state
= U8_STATE_HANGUL_LVT
;
1558 goto COLLECT_A_HANGUL
;
1563 * Regardless of whatever it was, if this is
1564 * a Starter, we don't collect the character
1565 * since that's a new start and we will deal
1566 * with it at the next time.
1568 i
= combining_class(uv
, s
, sz
);
1569 if (i
== U8_COMBINING_CLASS_STARTER
)
1573 * We know the current character is a combining
1574 * mark. If the previous character wasn't
1575 * a Starter (not Hangul) or a combining mark,
1576 * then, we don't collect this combining mark.
1578 if (*state
!= U8_STATE_START
&&
1579 *state
!= U8_STATE_COMBINING_MARK
)
1582 *state
= U8_STATE_COMBINING_MARK
;
1585 * If we collected a Starter and combining
1586 * marks up to 30, i.e., total 31 characters,
1587 * then, we terminate this degenerately long
1588 * combining sequence with a U+034F COMBINING
1589 * GRAPHEME JOINER (CGJ) which is 0xCD 0x8F in
1590 * UTF-8 and turn this into a Stream-Safe
1591 * Text. This will be extremely rare but
1594 * The following will also guarantee that
1595 * we are not writing more than 32 characters
1596 * plus a NULL at u8s[].
1598 if (last
>= U8_UPPER_LIMIT_IN_A_SEQ
) {
1600 *state
= U8_STATE_START
;
1601 comb_class
[last
] = 0;
1602 start
[last
] = saved_sz
;
1606 u8s
[saved_sz
++] = 0xCD;
1607 u8s
[saved_sz
++] = 0x8F;
1613 * Some combining marks also do decompose into
1614 * another combining mark or marks.
1616 if (*state
== U8_STATE_COMBINING_MARK
) {
1619 i
= do_decomp(uv
, uts
, s
, sz
,
1620 canonical_decomposition
, state
);
1621 for (j
= 0; j
< i
; ) {
1622 sz
= u8_number_of_bytes
[uts
[j
]];
1627 start
[last
] = saved_sz
+ j
;
1632 U8_UPPER_LIMIT_IN_A_SEQ
) {
1634 goto TURN_STREAM_SAFE
;
1639 *state
= U8_STATE_COMBINING_MARK
;
1643 for (i
= 0; i
< sz
; i
++)
1644 u8s
[saved_sz
++] = uts
[i
];
1646 comb_class
[last
] = i
;
1647 start
[last
] = saved_sz
;
1651 for (i
= 0; i
< sz
; i
++)
1652 u8s
[saved_sz
++] = *s
++;
1656 * If this is U+0345 COMBINING GREEK
1657 * YPOGEGRAMMENI (0xCD 0x85 in UTF-8), a.k.a.,
1658 * iota subscript, and need to be converted to
1659 * uppercase letter, convert it to U+0399 GREEK
1660 * CAPITAL LETTER IOTA (0xCE 0x99 in UTF-8),
1661 * i.e., convert to capital adscript form as
1662 * specified in the Unicode standard.
1664 * This is the only special case of (ambiguous)
1665 * case conversion at combining marks and
1666 * probably the standard will never have
1667 * anything similar like this in future.
1669 if (is_it_toupper
&& sz
>= 2 &&
1670 u8s
[saved_sz
- 2] == 0xCD &&
1671 u8s
[saved_sz
- 1] == 0x85) {
1672 u8s
[saved_sz
- 2] = 0xCE;
1673 u8s
[saved_sz
- 1] = 0x99;
1679 * Let's try to ensure a canonical ordering for the collected
1680 * combining marks. We do this only if we have collected
1681 * at least one more non-Starter. (The decomposition mapping
1682 * data tables have fully (and recursively) expanded and
1683 * canonically ordered decompositions.)
1685 * The U8_SWAP_COMB_MARKS() convenience macro has some
1686 * assumptions and we are meeting the assumptions.
1689 if (last
>= saved_last
) {
1690 for (i
= 0; i
< last
; i
++)
1691 for (j
= last
; j
> i
; j
--)
1692 if (comb_class
[j
] &&
1693 comb_class
[j
- 1] > comb_class
[j
]) {
1694 U8_SWAP_COMB_MARKS(j
- 1, j
);
1700 if (! canonical_composition
) {
1701 u8s
[saved_sz
] = '\0';
1706 * Now do the canonical composition. Note that we do this
1707 * only after a canonical or compatibility decomposition to
1708 * finish up NFC or NFKC.
1710 sz
= do_composition(uv
, u8s
, comb_class
, start
, disp
, last
,
1716 return ((size_t)sz
);
1720 * The do_norm_compare() function does string comparion based on Unicode
1721 * simple case mappings and Unicode Normalization definitions.
1723 * It does so by collecting a sequence of character at a time and comparing
1724 * the collected sequences from the strings.
1726 * The meanings on the return values are the same as the usual strcmp().
1729 do_norm_compare(size_t uv
, uchar_t
*s1
, uchar_t
*s2
, size_t n1
, size_t n2
,
1730 int flag
, int *errnum
)
1735 uchar_t u8s1
[U8_STREAM_SAFE_TEXT_MAX
+ 1];
1736 uchar_t u8s2
[U8_STREAM_SAFE_TEXT_MAX
+ 1];
1739 boolean_t is_it_toupper
;
1740 boolean_t is_it_tolower
;
1741 boolean_t canonical_decomposition
;
1742 boolean_t compatibility_decomposition
;
1743 boolean_t canonical_composition
;
1744 u8_normalization_states_t state
;
1749 is_it_toupper
= flag
& U8_TEXTPREP_TOUPPER
;
1750 is_it_tolower
= flag
& U8_TEXTPREP_TOLOWER
;
1751 canonical_decomposition
= flag
& U8_CANON_DECOMP
;
1752 compatibility_decomposition
= flag
& U8_COMPAT_DECOMP
;
1753 canonical_composition
= flag
& U8_CANON_COMP
;
1755 while (s1
< s1last
&& s2
< s2last
) {
1757 * If the current character is a 7-bit ASCII and the last
1758 * character, or, if the current character and the next
1759 * character are both some 7-bit ASCII characters then
1760 * we treat the current character as a sequence.
1762 * In any other cases, we need to call collect_a_seq().
1765 if (U8_ISASCII(*s1
) && ((s1
+ 1) >= s1last
||
1766 ((s1
+ 1) < s1last
&& U8_ISASCII(*(s1
+ 1))))) {
1768 u8s1
[0] = U8_ASCII_TOUPPER(*s1
);
1769 else if (is_it_tolower
)
1770 u8s1
[0] = U8_ASCII_TOLOWER(*s1
);
1777 state
= U8_STATE_START
;
1778 sz1
= collect_a_seq(uv
, u8s1
, &s1
, s1last
,
1779 is_it_toupper
, is_it_tolower
,
1780 canonical_decomposition
,
1781 compatibility_decomposition
,
1782 canonical_composition
, errnum
, &state
);
1785 if (U8_ISASCII(*s2
) && ((s2
+ 1) >= s2last
||
1786 ((s2
+ 1) < s2last
&& U8_ISASCII(*(s2
+ 1))))) {
1788 u8s2
[0] = U8_ASCII_TOUPPER(*s2
);
1789 else if (is_it_tolower
)
1790 u8s2
[0] = U8_ASCII_TOLOWER(*s2
);
1797 state
= U8_STATE_START
;
1798 sz2
= collect_a_seq(uv
, u8s2
, &s2
, s2last
,
1799 is_it_toupper
, is_it_tolower
,
1800 canonical_decomposition
,
1801 compatibility_decomposition
,
1802 canonical_composition
, errnum
, &state
);
1806 * Now compare the two characters. If they are the same,
1807 * we move on to the next character sequences.
1809 if (sz1
== 1 && sz2
== 1) {
1815 result
= strcmp((const char *)u8s1
, (const char *)u8s2
);
1822 * We compared until the end of either or both strings.
1824 * If we reached to or went over the ends for the both, that means
1825 * they are the same.
1827 * If we reached only one end, that means the other string has
1828 * something which then can be used to determine the return value.
1839 * The u8_strcmp() function compares two UTF-8 strings quite similar to
1840 * the strcmp(). For the comparison, however, Unicode Normalization specific
1841 * equivalency and Unicode simple case conversion mappings based equivalency
1842 * can be requested and checked against.
1845 u8_strcmp(const char *s1
, const char *s2
, size_t n
, int flag
, size_t uv
,
1855 * Check on the requested Unicode version, case conversion, and
1856 * normalization flag values.
1859 if (uv
> U8_UNICODE_LATEST
) {
1861 uv
= U8_UNICODE_LATEST
;
1865 flag
= U8_STRCMP_CS
;
1867 f
= flag
& (U8_STRCMP_CS
| U8_STRCMP_CI_UPPER
|
1868 U8_STRCMP_CI_LOWER
);
1870 flag
|= U8_STRCMP_CS
;
1871 } else if (f
!= U8_STRCMP_CS
&& f
!= U8_STRCMP_CI_UPPER
&&
1872 f
!= U8_STRCMP_CI_LOWER
) {
1874 flag
= U8_STRCMP_CS
;
1877 f
= flag
& (U8_CANON_DECOMP
| U8_COMPAT_DECOMP
| U8_CANON_COMP
);
1878 if (f
&& f
!= U8_STRCMP_NFD
&& f
!= U8_STRCMP_NFC
&&
1879 f
!= U8_STRCMP_NFKD
&& f
!= U8_STRCMP_NFKC
) {
1881 flag
= U8_STRCMP_CS
;
1885 if (flag
== U8_STRCMP_CS
) {
1886 return (n
== 0 ? strcmp(s1
, s2
) : strncmp(s1
, s2
, n
));
1899 * Simple case conversion can be done much faster and so we do
1900 * them separately here.
1902 if (flag
== U8_STRCMP_CI_UPPER
) {
1903 return (do_case_compare(uv
, (uchar_t
*)s1
, (uchar_t
*)s2
,
1904 n1
, n2
, B_TRUE
, errnum
));
1905 } else if (flag
== U8_STRCMP_CI_LOWER
) {
1906 return (do_case_compare(uv
, (uchar_t
*)s1
, (uchar_t
*)s2
,
1907 n1
, n2
, B_FALSE
, errnum
));
1910 return (do_norm_compare(uv
, (uchar_t
*)s1
, (uchar_t
*)s2
, n1
, n2
,
1915 u8_textprep_str(char *inarray
, size_t *inlen
, char *outarray
, size_t *outlen
,
1916 int flag
, size_t unicode_version
, int *errnum
)
1924 boolean_t do_not_ignore_null
;
1925 boolean_t do_not_ignore_invalid
;
1926 boolean_t is_it_toupper
;
1927 boolean_t is_it_tolower
;
1928 boolean_t canonical_decomposition
;
1929 boolean_t compatibility_decomposition
;
1930 boolean_t canonical_composition
;
1934 uchar_t u8s
[U8_STREAM_SAFE_TEXT_MAX
+ 1];
1935 u8_normalization_states_t state
;
1937 if (unicode_version
> U8_UNICODE_LATEST
) {
1939 return ((size_t)-1);
1942 f
= flag
& (U8_TEXTPREP_TOUPPER
| U8_TEXTPREP_TOLOWER
);
1943 if (f
== (U8_TEXTPREP_TOUPPER
| U8_TEXTPREP_TOLOWER
)) {
1945 return ((size_t)-1);
1948 f
= flag
& (U8_CANON_DECOMP
| U8_COMPAT_DECOMP
| U8_CANON_COMP
);
1949 if (f
&& f
!= U8_TEXTPREP_NFD
&& f
!= U8_TEXTPREP_NFC
&&
1950 f
!= U8_TEXTPREP_NFKD
&& f
!= U8_TEXTPREP_NFKC
) {
1952 return ((size_t)-1);
1955 if (inarray
== NULL
|| *inlen
== 0)
1958 if (outarray
== NULL
) {
1960 return ((size_t)-1);
1963 ib
= (uchar_t
*)inarray
;
1964 ob
= (uchar_t
*)outarray
;
1965 ibtail
= ib
+ *inlen
;
1966 obtail
= ob
+ *outlen
;
1968 do_not_ignore_null
= !(flag
& U8_TEXTPREP_IGNORE_NULL
);
1969 do_not_ignore_invalid
= !(flag
& U8_TEXTPREP_IGNORE_INVALID
);
1970 is_it_toupper
= flag
& U8_TEXTPREP_TOUPPER
;
1971 is_it_tolower
= flag
& U8_TEXTPREP_TOLOWER
;
1976 * If we don't have a normalization flag set, we do the simple case
1977 * conversion based text preparation separately below. Text
1978 * preparation involving Normalization will be done in the false task
1979 * block, again, separately since it will take much more time and
1980 * resource than doing simple case conversions.
1983 while (ib
< ibtail
) {
1984 if (*ib
== '\0' && do_not_ignore_null
)
1987 sz
= u8_number_of_bytes
[*ib
];
1990 if (do_not_ignore_invalid
) {
1992 ret_val
= (size_t)-1;
2003 ret_val
= (size_t)-1;
2008 *ob
= U8_ASCII_TOUPPER(*ib
);
2009 else if (is_it_tolower
)
2010 *ob
= U8_ASCII_TOLOWER(*ib
);
2015 } else if ((ib
+ sz
) > ibtail
) {
2016 if (do_not_ignore_invalid
) {
2018 ret_val
= (size_t)-1;
2022 if ((obtail
- ob
) < (ibtail
- ib
)) {
2024 ret_val
= (size_t)-1;
2029 * We treat the remaining incomplete character
2030 * bytes as a character.
2037 if (is_it_toupper
|| is_it_tolower
) {
2038 i
= do_case_conv(unicode_version
, u8s
,
2039 ib
, sz
, is_it_toupper
);
2041 if ((obtail
- ob
) < i
) {
2043 ret_val
= (size_t)-1;
2049 for (sz
= 0; sz
< i
; sz
++)
2052 if ((obtail
- ob
) < sz
) {
2054 ret_val
= (size_t)-1;
2058 for (i
= 0; i
< sz
; i
++)
2064 canonical_decomposition
= flag
& U8_CANON_DECOMP
;
2065 compatibility_decomposition
= flag
& U8_COMPAT_DECOMP
;
2066 canonical_composition
= flag
& U8_CANON_COMP
;
2068 while (ib
< ibtail
) {
2069 if (*ib
== '\0' && do_not_ignore_null
)
2073 * If the current character is a 7-bit ASCII
2074 * character and it is the last character, or,
2075 * if the current character is a 7-bit ASCII
2076 * character and the next character is also a 7-bit
2077 * ASCII character, then, we copy over this
2078 * character without going through collect_a_seq().
2080 * In any other cases, we need to look further with
2081 * the collect_a_seq() function.
2083 if (U8_ISASCII(*ib
) && ((ib
+ 1) >= ibtail
||
2084 ((ib
+ 1) < ibtail
&& U8_ISASCII(*(ib
+ 1))))) {
2087 ret_val
= (size_t)-1;
2092 *ob
= U8_ASCII_TOUPPER(*ib
);
2093 else if (is_it_tolower
)
2094 *ob
= U8_ASCII_TOLOWER(*ib
);
2101 state
= U8_STATE_START
;
2103 j
= collect_a_seq(unicode_version
, u8s
,
2107 canonical_decomposition
,
2108 compatibility_decomposition
,
2109 canonical_composition
,
2112 if (*errnum
&& do_not_ignore_invalid
) {
2113 ret_val
= (size_t)-1;
2117 if ((obtail
- ob
) < j
) {
2119 ret_val
= (size_t)-1;
2123 for (i
= 0; i
< j
; i
++)
2129 *inlen
= ibtail
- ib
;
2130 *outlen
= obtail
- ob
;
2135 #if defined(_KERNEL) && defined(HAVE_SPL)
2137 unicode_init(void) {
2146 module_init(unicode_init
);
2147 module_exit(unicode_fini
);
2149 MODULE_DESCRIPTION("Unicode implementation");
2150 MODULE_AUTHOR(ZFS_META_AUTHOR
);
2151 MODULE_LICENSE(ZFS_META_LICENSE
);
2152 MODULE_VERSION(ZFS_META_VERSION
"-" ZFS_META_RELEASE
);
2154 EXPORT_SYMBOL(u8_validate
);
2155 EXPORT_SYMBOL(u8_strcmp
);
2156 EXPORT_SYMBOL(u8_textprep_str
);