+++ /dev/null
-/*\r
- * _codecs_cn.c: Codecs collection for Mainland Chinese encodings\r
- *\r
- * Written by Hye-Shik Chang <perky@FreeBSD.org>\r
- */\r
-\r
-#include "cjkcodecs.h"\r
-#include "mappings_cn.h"\r
-\r
-/**\r
- * hz is predefined as 100 on AIX. So we undefine it to avoid\r
- * conflict against hz codec's.\r
- */\r
-#ifdef _AIX\r
-#undef hz\r
-#endif\r
-\r
-/* GBK and GB2312 map differently in few codepoints that are listed below:\r
- *\r
- * gb2312 gbk\r
- * A1A4 U+30FB KATAKANA MIDDLE DOT U+00B7 MIDDLE DOT\r
- * A1AA U+2015 HORIZONTAL BAR U+2014 EM DASH\r
- * A844 undefined U+2015 HORIZONTAL BAR\r
- */\r
-\r
-#define GBK_DECODE(dc1, dc2, assi) \\r
- if ((dc1) == 0xa1 && (dc2) == 0xaa) (assi) = 0x2014; \\r
- else if ((dc1) == 0xa8 && (dc2) == 0x44) (assi) = 0x2015; \\r
- else if ((dc1) == 0xa1 && (dc2) == 0xa4) (assi) = 0x00b7; \\r
- else TRYMAP_DEC(gb2312, assi, dc1 ^ 0x80, dc2 ^ 0x80); \\r
- else TRYMAP_DEC(gbkext, assi, dc1, dc2);\r
-\r
-#define GBK_ENCODE(code, assi) \\r
- if ((code) == 0x2014) (assi) = 0xa1aa; \\r
- else if ((code) == 0x2015) (assi) = 0xa844; \\r
- else if ((code) == 0x00b7) (assi) = 0xa1a4; \\r
- else if ((code) != 0x30fb && TRYMAP_ENC_COND(gbcommon, assi, code));\r
-\r
-/*\r
- * GB2312 codec\r
- */\r
-\r
-ENCODER(gb2312)\r
-{\r
- while (inleft > 0) {\r
- Py_UNICODE c = IN1;\r
- DBCHAR code;\r
-\r
- if (c < 0x80) {\r
- WRITE1((unsigned char)c)\r
- NEXT(1, 1)\r
- continue;\r
- }\r
- UCS4INVALID(c)\r
-\r
- REQUIRE_OUTBUF(2)\r
- TRYMAP_ENC(gbcommon, code, c);\r
- else return 1;\r
-\r
- if (code & 0x8000) /* MSB set: GBK */\r
- return 1;\r
-\r
- OUT1((code >> 8) | 0x80)\r
- OUT2((code & 0xFF) | 0x80)\r
- NEXT(1, 2)\r
- }\r
-\r
- return 0;\r
-}\r
-\r
-DECODER(gb2312)\r
-{\r
- while (inleft > 0) {\r
- unsigned char c = **inbuf;\r
-\r
- REQUIRE_OUTBUF(1)\r
-\r
- if (c < 0x80) {\r
- OUT1(c)\r
- NEXT(1, 1)\r
- continue;\r
- }\r
-\r
- REQUIRE_INBUF(2)\r
- TRYMAP_DEC(gb2312, **outbuf, c ^ 0x80, IN2 ^ 0x80) {\r
- NEXT(2, 1)\r
- }\r
- else return 2;\r
- }\r
-\r
- return 0;\r
-}\r
-\r
-\r
-/*\r
- * GBK codec\r
- */\r
-\r
-ENCODER(gbk)\r
-{\r
- while (inleft > 0) {\r
- Py_UNICODE c = IN1;\r
- DBCHAR code;\r
-\r
- if (c < 0x80) {\r
- WRITE1((unsigned char)c)\r
- NEXT(1, 1)\r
- continue;\r
- }\r
- UCS4INVALID(c)\r
-\r
- REQUIRE_OUTBUF(2)\r
-\r
- GBK_ENCODE(c, code)\r
- else return 1;\r
-\r
- OUT1((code >> 8) | 0x80)\r
- if (code & 0x8000)\r
- OUT2((code & 0xFF)) /* MSB set: GBK */\r
- else\r
- OUT2((code & 0xFF) | 0x80) /* MSB unset: GB2312 */\r
- NEXT(1, 2)\r
- }\r
-\r
- return 0;\r
-}\r
-\r
-DECODER(gbk)\r
-{\r
- while (inleft > 0) {\r
- unsigned char c = IN1;\r
-\r
- REQUIRE_OUTBUF(1)\r
-\r
- if (c < 0x80) {\r
- OUT1(c)\r
- NEXT(1, 1)\r
- continue;\r
- }\r
-\r
- REQUIRE_INBUF(2)\r
-\r
- GBK_DECODE(c, IN2, **outbuf)\r
- else return 2;\r
-\r
- NEXT(2, 1)\r
- }\r
-\r
- return 0;\r
-}\r
-\r
-\r
-/*\r
- * GB18030 codec\r
- */\r
-\r
-ENCODER(gb18030)\r
-{\r
- while (inleft > 0) {\r
- ucs4_t c = IN1;\r
- DBCHAR code;\r
-\r
- if (c < 0x80) {\r
- WRITE1(c)\r
- NEXT(1, 1)\r
- continue;\r
- }\r
-\r
- DECODE_SURROGATE(c)\r
- if (c > 0x10FFFF)\r
-#if Py_UNICODE_SIZE == 2\r
- return 2; /* surrogates pair */\r
-#else\r
- return 1;\r
-#endif\r
- else if (c >= 0x10000) {\r
- ucs4_t tc = c - 0x10000;\r
-\r
- REQUIRE_OUTBUF(4)\r
-\r
- OUT4((unsigned char)(tc % 10) + 0x30)\r
- tc /= 10;\r
- OUT3((unsigned char)(tc % 126) + 0x81)\r
- tc /= 126;\r
- OUT2((unsigned char)(tc % 10) + 0x30)\r
- tc /= 10;\r
- OUT1((unsigned char)(tc + 0x90))\r
-\r
-#if Py_UNICODE_SIZE == 2\r
- NEXT(2, 4) /* surrogates pair */\r
-#else\r
- NEXT(1, 4)\r
-#endif\r
- continue;\r
- }\r
-\r
- REQUIRE_OUTBUF(2)\r
-\r
- GBK_ENCODE(c, code)\r
- else TRYMAP_ENC(gb18030ext, code, c);\r
- else {\r
- const struct _gb18030_to_unibmp_ranges *utrrange;\r
-\r
- REQUIRE_OUTBUF(4)\r
-\r
- for (utrrange = gb18030_to_unibmp_ranges;\r
- utrrange->first != 0;\r
- utrrange++)\r
- if (utrrange->first <= c &&\r
- c <= utrrange->last) {\r
- Py_UNICODE tc;\r
-\r
- tc = c - utrrange->first +\r
- utrrange->base;\r
-\r
- OUT4((unsigned char)(tc % 10) + 0x30)\r
- tc /= 10;\r
- OUT3((unsigned char)(tc % 126) + 0x81)\r
- tc /= 126;\r
- OUT2((unsigned char)(tc % 10) + 0x30)\r
- tc /= 10;\r
- OUT1((unsigned char)tc + 0x81)\r
-\r
- NEXT(1, 4)\r
- break;\r
- }\r
-\r
- if (utrrange->first == 0)\r
- return 1;\r
- continue;\r
- }\r
-\r
- OUT1((code >> 8) | 0x80)\r
- if (code & 0x8000)\r
- OUT2((code & 0xFF)) /* MSB set: GBK or GB18030ext */\r
- else\r
- OUT2((code & 0xFF) | 0x80) /* MSB unset: GB2312 */\r
-\r
- NEXT(1, 2)\r
- }\r
-\r
- return 0;\r
-}\r
-\r
-DECODER(gb18030)\r
-{\r
- while (inleft > 0) {\r
- unsigned char c = IN1, c2;\r
-\r
- REQUIRE_OUTBUF(1)\r
-\r
- if (c < 0x80) {\r
- OUT1(c)\r
- NEXT(1, 1)\r
- continue;\r
- }\r
-\r
- REQUIRE_INBUF(2)\r
-\r
- c2 = IN2;\r
- if (c2 >= 0x30 && c2 <= 0x39) { /* 4 bytes seq */\r
- const struct _gb18030_to_unibmp_ranges *utr;\r
- unsigned char c3, c4;\r
- ucs4_t lseq;\r
-\r
- REQUIRE_INBUF(4)\r
- c3 = IN3;\r
- c4 = IN4;\r
- if (c < 0x81 || c3 < 0x81 || c4 < 0x30 || c4 > 0x39)\r
- return 4;\r
- c -= 0x81; c2 -= 0x30;\r
- c3 -= 0x81; c4 -= 0x30;\r
-\r
- if (c < 4) { /* U+0080 - U+FFFF */\r
- lseq = ((ucs4_t)c * 10 + c2) * 1260 +\r
- (ucs4_t)c3 * 10 + c4;\r
- if (lseq < 39420) {\r
- for (utr = gb18030_to_unibmp_ranges;\r
- lseq >= (utr + 1)->base;\r
- utr++) ;\r
- OUT1(utr->first - utr->base + lseq)\r
- NEXT(4, 1)\r
- continue;\r
- }\r
- }\r
- else if (c >= 15) { /* U+10000 - U+10FFFF */\r
- lseq = 0x10000 + (((ucs4_t)c-15) * 10 + c2)\r
- * 1260 + (ucs4_t)c3 * 10 + c4;\r
- if (lseq <= 0x10FFFF) {\r
- WRITEUCS4(lseq);\r
- NEXT_IN(4)\r
- continue;\r
- }\r
- }\r
- return 4;\r
- }\r
-\r
- GBK_DECODE(c, c2, **outbuf)\r
- else TRYMAP_DEC(gb18030ext, **outbuf, c, c2);\r
- else return 2;\r
-\r
- NEXT(2, 1)\r
- }\r
-\r
- return 0;\r
-}\r
-\r
-\r
-/*\r
- * HZ codec\r
- */\r
-\r
-ENCODER_INIT(hz)\r
-{\r
- state->i = 0;\r
- return 0;\r
-}\r
-\r
-ENCODER_RESET(hz)\r
-{\r
- if (state->i != 0) {\r
- WRITE2('~', '}')\r
- state->i = 0;\r
- NEXT_OUT(2)\r
- }\r
- return 0;\r
-}\r
-\r
-ENCODER(hz)\r
-{\r
- while (inleft > 0) {\r
- Py_UNICODE c = IN1;\r
- DBCHAR code;\r
-\r
- if (c < 0x80) {\r
- if (state->i == 0) {\r
- WRITE1((unsigned char)c)\r
- NEXT(1, 1)\r
- }\r
- else {\r
- WRITE3('~', '}', (unsigned char)c)\r
- NEXT(1, 3)\r
- state->i = 0;\r
- }\r
- continue;\r
- }\r
-\r
- UCS4INVALID(c)\r
-\r
- TRYMAP_ENC(gbcommon, code, c);\r
- else return 1;\r
-\r
- if (code & 0x8000) /* MSB set: GBK */\r
- return 1;\r
-\r
- if (state->i == 0) {\r
- WRITE4('~', '{', code >> 8, code & 0xff)\r
- NEXT(1, 4)\r
- state->i = 1;\r
- }\r
- else {\r
- WRITE2(code >> 8, code & 0xff)\r
- NEXT(1, 2)\r
- }\r
- }\r
-\r
- return 0;\r
-}\r
-\r
-DECODER_INIT(hz)\r
-{\r
- state->i = 0;\r
- return 0;\r
-}\r
-\r
-DECODER_RESET(hz)\r
-{\r
- state->i = 0;\r
- return 0;\r
-}\r
-\r
-DECODER(hz)\r
-{\r
- while (inleft > 0) {\r
- unsigned char c = IN1;\r
-\r
- if (c == '~') {\r
- unsigned char c2 = IN2;\r
-\r
- REQUIRE_INBUF(2)\r
- if (c2 == '~') {\r
- WRITE1('~')\r
- NEXT(2, 1)\r
- continue;\r
- }\r
- else if (c2 == '{' && state->i == 0)\r
- state->i = 1; /* set GB */\r
- else if (c2 == '}' && state->i == 1)\r
- state->i = 0; /* set ASCII */\r
- else if (c2 == '\n')\r
- ; /* line-continuation */\r
- else\r
- return 2;\r
- NEXT(2, 0);\r
- continue;\r
- }\r
-\r
- if (c & 0x80)\r
- return 1;\r
-\r
- if (state->i == 0) { /* ASCII mode */\r
- WRITE1(c)\r
- NEXT(1, 1)\r
- }\r
- else { /* GB mode */\r
- REQUIRE_INBUF(2)\r
- REQUIRE_OUTBUF(1)\r
- TRYMAP_DEC(gb2312, **outbuf, c, IN2) {\r
- NEXT(2, 1)\r
- }\r
- else\r
- return 2;\r
- }\r
- }\r
-\r
- return 0;\r
-}\r
-\r
-\r
-BEGIN_MAPPINGS_LIST\r
- MAPPING_DECONLY(gb2312)\r
- MAPPING_DECONLY(gbkext)\r
- MAPPING_ENCONLY(gbcommon)\r
- MAPPING_ENCDEC(gb18030ext)\r
-END_MAPPINGS_LIST\r
-\r
-BEGIN_CODECS_LIST\r
- CODEC_STATELESS(gb2312)\r
- CODEC_STATELESS(gbk)\r
- CODEC_STATELESS(gb18030)\r
- CODEC_STATEFUL(hz)\r
-END_CODECS_LIST\r
-\r
-I_AM_A_MODULE_FOR(cn)\r
projects / mirror_edk2.git / blobdiff