+++ /dev/null
-/*\r
- * multibytecodec.c: Common Multibyte Codec Implementation\r
- *\r
- * Written by Hye-Shik Chang <perky@FreeBSD.org>\r
- */\r
-\r
-#define PY_SSIZE_T_CLEAN\r
-#include "Python.h"\r
-#include "structmember.h"\r
-#include "multibytecodec.h"\r
-\r
-typedef struct {\r
- const Py_UNICODE *inbuf, *inbuf_top, *inbuf_end;\r
- unsigned char *outbuf, *outbuf_end;\r
- PyObject *excobj, *outobj;\r
-} MultibyteEncodeBuffer;\r
-\r
-typedef struct {\r
- const unsigned char *inbuf, *inbuf_top, *inbuf_end;\r
- Py_UNICODE *outbuf, *outbuf_end;\r
- PyObject *excobj, *outobj;\r
-} MultibyteDecodeBuffer;\r
-\r
-PyDoc_STRVAR(MultibyteCodec_Encode__doc__,\r
-"I.encode(unicode[, errors]) -> (string, length consumed)\n\\r
-\n\\r
-Return an encoded string version of `unicode'. errors may be given to\n\\r
-set a different error handling scheme. Default is 'strict' meaning that\n\\r
-encoding errors raise a UnicodeEncodeError. Other possible values are\n\\r
-'ignore', 'replace' and 'xmlcharrefreplace' as well as any other name\n\\r
-registered with codecs.register_error that can handle UnicodeEncodeErrors.");\r
-\r
-PyDoc_STRVAR(MultibyteCodec_Decode__doc__,\r
-"I.decode(string[, errors]) -> (unicodeobject, length consumed)\n\\r
-\n\\r
-Decodes `string' using I, an MultibyteCodec instance. errors may be given\n\\r
-to set a different error handling scheme. Default is 'strict' meaning\n\\r
-that encoding errors raise a UnicodeDecodeError. Other possible values\n\\r
-are 'ignore' and 'replace' as well as any other name registered with\n\\r
-codecs.register_error that is able to handle UnicodeDecodeErrors.");\r
-\r
-static char *codeckwarglist[] = {"input", "errors", NULL};\r
-static char *incnewkwarglist[] = {"errors", NULL};\r
-static char *incrementalkwarglist[] = {"input", "final", NULL};\r
-static char *streamkwarglist[] = {"stream", "errors", NULL};\r
-\r
-static PyObject *multibytecodec_encode(MultibyteCodec *,\r
- MultibyteCodec_State *, const Py_UNICODE **, Py_ssize_t,\r
- PyObject *, int);\r
-\r
-#define MBENC_RESET MBENC_MAX<<1 /* reset after an encoding session */\r
-\r
-static PyObject *\r
-make_tuple(PyObject *object, Py_ssize_t len)\r
-{\r
- PyObject *v, *w;\r
-\r
- if (object == NULL)\r
- return NULL;\r
-\r
- v = PyTuple_New(2);\r
- if (v == NULL) {\r
- Py_DECREF(object);\r
- return NULL;\r
- }\r
- PyTuple_SET_ITEM(v, 0, object);\r
-\r
- w = PyInt_FromSsize_t(len);\r
- if (w == NULL) {\r
- Py_DECREF(v);\r
- return NULL;\r
- }\r
- PyTuple_SET_ITEM(v, 1, w);\r
-\r
- return v;\r
-}\r
-\r
-static PyObject *\r
-internal_error_callback(const char *errors)\r
-{\r
- if (errors == NULL || strcmp(errors, "strict") == 0)\r
- return ERROR_STRICT;\r
- else if (strcmp(errors, "ignore") == 0)\r
- return ERROR_IGNORE;\r
- else if (strcmp(errors, "replace") == 0)\r
- return ERROR_REPLACE;\r
- else\r
- return PyString_FromString(errors);\r
-}\r
-\r
-static PyObject *\r
-call_error_callback(PyObject *errors, PyObject *exc)\r
-{\r
- PyObject *args, *cb, *r;\r
-\r
- assert(PyString_Check(errors));\r
- cb = PyCodec_LookupError(PyString_AS_STRING(errors));\r
- if (cb == NULL)\r
- return NULL;\r
-\r
- args = PyTuple_New(1);\r
- if (args == NULL) {\r
- Py_DECREF(cb);\r
- return NULL;\r
- }\r
-\r
- PyTuple_SET_ITEM(args, 0, exc);\r
- Py_INCREF(exc);\r
-\r
- r = PyObject_CallObject(cb, args);\r
- Py_DECREF(args);\r
- Py_DECREF(cb);\r
- return r;\r
-}\r
-\r
-static PyObject *\r
-codecctx_errors_get(MultibyteStatefulCodecContext *self)\r
-{\r
- const char *errors;\r
-\r
- if (self->errors == ERROR_STRICT)\r
- errors = "strict";\r
- else if (self->errors == ERROR_IGNORE)\r
- errors = "ignore";\r
- else if (self->errors == ERROR_REPLACE)\r
- errors = "replace";\r
- else {\r
- Py_INCREF(self->errors);\r
- return self->errors;\r
- }\r
-\r
- return PyString_FromString(errors);\r
-}\r
-\r
-static int\r
-codecctx_errors_set(MultibyteStatefulCodecContext *self, PyObject *value,\r
- void *closure)\r
-{\r
- PyObject *cb;\r
-\r
- if (!PyString_Check(value)) {\r
- PyErr_SetString(PyExc_TypeError, "errors must be a string");\r
- return -1;\r
- }\r
-\r
- cb = internal_error_callback(PyString_AS_STRING(value));\r
- if (cb == NULL)\r
- return -1;\r
-\r
- ERROR_DECREF(self->errors);\r
- self->errors = cb;\r
- return 0;\r
-}\r
-\r
-/* This getset handlers list is used by all the stateful codec objects */\r
-static PyGetSetDef codecctx_getsets[] = {\r
- {"errors", (getter)codecctx_errors_get,\r
- (setter)codecctx_errors_set,\r
- PyDoc_STR("how to treat errors")},\r
- {NULL,}\r
-};\r
-\r
-static int\r
-expand_encodebuffer(MultibyteEncodeBuffer *buf, Py_ssize_t esize)\r
-{\r
- Py_ssize_t orgpos, orgsize, incsize;\r
-\r
- orgpos = (Py_ssize_t)((char *)buf->outbuf -\r
- PyString_AS_STRING(buf->outobj));\r
- orgsize = PyString_GET_SIZE(buf->outobj);\r
- incsize = (esize < (orgsize >> 1) ? (orgsize >> 1) | 1 : esize);\r
-\r
- if (orgsize > PY_SSIZE_T_MAX - incsize) {\r
- PyErr_NoMemory();\r
- return -1;\r
- }\r
-\r
- if (_PyString_Resize(&buf->outobj, orgsize + incsize) == -1)\r
- return -1;\r
-\r
- buf->outbuf = (unsigned char *)PyString_AS_STRING(buf->outobj) +orgpos;\r
- buf->outbuf_end = (unsigned char *)PyString_AS_STRING(buf->outobj)\r
- + PyString_GET_SIZE(buf->outobj);\r
-\r
- return 0;\r
-}\r
-#define REQUIRE_ENCODEBUFFER(buf, s) do { \\r
- if ((s) < 0 || (s) > (buf)->outbuf_end - (buf)->outbuf) \\r
- if (expand_encodebuffer(buf, s) == -1) \\r
- goto errorexit; \\r
-} while(0)\r
-\r
-static int\r
-expand_decodebuffer(MultibyteDecodeBuffer *buf, Py_ssize_t esize)\r
-{\r
- Py_ssize_t orgpos, orgsize;\r
-\r
- orgpos = (Py_ssize_t)(buf->outbuf - PyUnicode_AS_UNICODE(buf->outobj));\r
- orgsize = PyUnicode_GET_SIZE(buf->outobj);\r
- if (PyUnicode_Resize(&buf->outobj, orgsize + (\r
- esize < (orgsize >> 1) ? (orgsize >> 1) | 1 : esize)) == -1)\r
- return -1;\r
-\r
- buf->outbuf = PyUnicode_AS_UNICODE(buf->outobj) + orgpos;\r
- buf->outbuf_end = PyUnicode_AS_UNICODE(buf->outobj)\r
- + PyUnicode_GET_SIZE(buf->outobj);\r
-\r
- return 0;\r
-}\r
-#define REQUIRE_DECODEBUFFER(buf, s) do { \\r
- if ((s) < 0 || (s) > (buf)->outbuf_end - (buf)->outbuf) \\r
- if (expand_decodebuffer(buf, s) == -1) \\r
- goto errorexit; \\r
-} while(0)\r
-\r
-\r
-/**\r
- * MultibyteCodec object\r
- */\r
-\r
-static int\r
-multibytecodec_encerror(MultibyteCodec *codec,\r
- MultibyteCodec_State *state,\r
- MultibyteEncodeBuffer *buf,\r
- PyObject *errors, Py_ssize_t e)\r
-{\r
- PyObject *retobj = NULL, *retstr = NULL, *tobj;\r
- Py_ssize_t retstrsize, newpos;\r
- Py_ssize_t esize, start, end;\r
- const char *reason;\r
-\r
- if (e > 0) {\r
- reason = "illegal multibyte sequence";\r
- esize = e;\r
- }\r
- else {\r
- switch (e) {\r
- case MBERR_TOOSMALL:\r
- REQUIRE_ENCODEBUFFER(buf, -1);\r
- return 0; /* retry it */\r
- case MBERR_TOOFEW:\r
- reason = "incomplete multibyte sequence";\r
- esize = (Py_ssize_t)(buf->inbuf_end - buf->inbuf);\r
- break;\r
- case MBERR_INTERNAL:\r
- PyErr_SetString(PyExc_RuntimeError,\r
- "internal codec error");\r
- return -1;\r
- default:\r
- PyErr_SetString(PyExc_RuntimeError,\r
- "unknown runtime error");\r
- return -1;\r
- }\r
- }\r
-\r
- if (errors == ERROR_REPLACE) {\r
- const Py_UNICODE replchar = '?', *inbuf = &replchar;\r
- Py_ssize_t r;\r
-\r
- for (;;) {\r
- Py_ssize_t outleft;\r
-\r
- outleft = (Py_ssize_t)(buf->outbuf_end - buf->outbuf);\r
- r = codec->encode(state, codec->config, &inbuf, 1,\r
- &buf->outbuf, outleft, 0);\r
- if (r == MBERR_TOOSMALL) {\r
- REQUIRE_ENCODEBUFFER(buf, -1);\r
- continue;\r
- }\r
- else\r
- break;\r
- }\r
-\r
- if (r != 0) {\r
- REQUIRE_ENCODEBUFFER(buf, 1);\r
- *buf->outbuf++ = '?';\r
- }\r
- }\r
- if (errors == ERROR_IGNORE || errors == ERROR_REPLACE) {\r
- buf->inbuf += esize;\r
- return 0;\r
- }\r
-\r
- start = (Py_ssize_t)(buf->inbuf - buf->inbuf_top);\r
- end = start + esize;\r
-\r
- /* use cached exception object if available */\r
- if (buf->excobj == NULL) {\r
- buf->excobj = PyUnicodeEncodeError_Create(codec->encoding,\r
- buf->inbuf_top,\r
- buf->inbuf_end - buf->inbuf_top,\r
- start, end, reason);\r
- if (buf->excobj == NULL)\r
- goto errorexit;\r
- }\r
- else\r
- if (PyUnicodeEncodeError_SetStart(buf->excobj, start) != 0 ||\r
- PyUnicodeEncodeError_SetEnd(buf->excobj, end) != 0 ||\r
- PyUnicodeEncodeError_SetReason(buf->excobj, reason) != 0)\r
- goto errorexit;\r
-\r
- if (errors == ERROR_STRICT) {\r
- PyCodec_StrictErrors(buf->excobj);\r
- goto errorexit;\r
- }\r
-\r
- retobj = call_error_callback(errors, buf->excobj);\r
- if (retobj == NULL)\r
- goto errorexit;\r
-\r
- if (!PyTuple_Check(retobj) || PyTuple_GET_SIZE(retobj) != 2 ||\r
- !PyUnicode_Check((tobj = PyTuple_GET_ITEM(retobj, 0))) ||\r
- !(PyInt_Check(PyTuple_GET_ITEM(retobj, 1)) ||\r
- PyLong_Check(PyTuple_GET_ITEM(retobj, 1)))) {\r
- PyErr_SetString(PyExc_TypeError,\r
- "encoding error handler must return "\r
- "(unicode, int) tuple");\r
- goto errorexit;\r
- }\r
-\r
- {\r
- const Py_UNICODE *uraw = PyUnicode_AS_UNICODE(tobj);\r
-\r
- retstr = multibytecodec_encode(codec, state, &uraw,\r
- PyUnicode_GET_SIZE(tobj), ERROR_STRICT,\r
- MBENC_FLUSH);\r
- if (retstr == NULL)\r
- goto errorexit;\r
- }\r
-\r
- retstrsize = PyString_GET_SIZE(retstr);\r
- if (retstrsize > 0) {\r
- REQUIRE_ENCODEBUFFER(buf, retstrsize);\r
- memcpy(buf->outbuf, PyString_AS_STRING(retstr), retstrsize);\r
- buf->outbuf += retstrsize;\r
- }\r
-\r
- newpos = PyInt_AsSsize_t(PyTuple_GET_ITEM(retobj, 1));\r
- if (newpos < 0 && !PyErr_Occurred())\r
- newpos += (Py_ssize_t)(buf->inbuf_end - buf->inbuf_top);\r
- if (newpos < 0 || buf->inbuf_top + newpos > buf->inbuf_end) {\r
- PyErr_Clear();\r
- PyErr_Format(PyExc_IndexError,\r
- "position %zd from error handler out of bounds",\r
- newpos);\r
- goto errorexit;\r
- }\r
- buf->inbuf = buf->inbuf_top + newpos;\r
-\r
- Py_DECREF(retobj);\r
- Py_DECREF(retstr);\r
- return 0;\r
-\r
-errorexit:\r
- Py_XDECREF(retobj);\r
- Py_XDECREF(retstr);\r
- return -1;\r
-}\r
-\r
-static int\r
-multibytecodec_decerror(MultibyteCodec *codec,\r
- MultibyteCodec_State *state,\r
- MultibyteDecodeBuffer *buf,\r
- PyObject *errors, Py_ssize_t e)\r
-{\r
- PyObject *retobj = NULL, *retuni = NULL;\r
- Py_ssize_t retunisize, newpos;\r
- const char *reason;\r
- Py_ssize_t esize, start, end;\r
-\r
- if (e > 0) {\r
- reason = "illegal multibyte sequence";\r
- esize = e;\r
- }\r
- else {\r
- switch (e) {\r
- case MBERR_TOOSMALL:\r
- REQUIRE_DECODEBUFFER(buf, -1);\r
- return 0; /* retry it */\r
- case MBERR_TOOFEW:\r
- reason = "incomplete multibyte sequence";\r
- esize = (Py_ssize_t)(buf->inbuf_end - buf->inbuf);\r
- break;\r
- case MBERR_INTERNAL:\r
- PyErr_SetString(PyExc_RuntimeError,\r
- "internal codec error");\r
- return -1;\r
- default:\r
- PyErr_SetString(PyExc_RuntimeError,\r
- "unknown runtime error");\r
- return -1;\r
- }\r
- }\r
-\r
- if (errors == ERROR_REPLACE) {\r
- REQUIRE_DECODEBUFFER(buf, 1);\r
- *buf->outbuf++ = Py_UNICODE_REPLACEMENT_CHARACTER;\r
- }\r
- if (errors == ERROR_IGNORE || errors == ERROR_REPLACE) {\r
- buf->inbuf += esize;\r
- return 0;\r
- }\r
-\r
- start = (Py_ssize_t)(buf->inbuf - buf->inbuf_top);\r
- end = start + esize;\r
-\r
- /* use cached exception object if available */\r
- if (buf->excobj == NULL) {\r
- buf->excobj = PyUnicodeDecodeError_Create(codec->encoding,\r
- (const char *)buf->inbuf_top,\r
- (Py_ssize_t)(buf->inbuf_end - buf->inbuf_top),\r
- start, end, reason);\r
- if (buf->excobj == NULL)\r
- goto errorexit;\r
- }\r
- else\r
- if (PyUnicodeDecodeError_SetStart(buf->excobj, start) ||\r
- PyUnicodeDecodeError_SetEnd(buf->excobj, end) ||\r
- PyUnicodeDecodeError_SetReason(buf->excobj, reason))\r
- goto errorexit;\r
-\r
- if (errors == ERROR_STRICT) {\r
- PyCodec_StrictErrors(buf->excobj);\r
- goto errorexit;\r
- }\r
-\r
- retobj = call_error_callback(errors, buf->excobj);\r
- if (retobj == NULL)\r
- goto errorexit;\r
-\r
- if (!PyTuple_Check(retobj) || PyTuple_GET_SIZE(retobj) != 2 ||\r
- !PyUnicode_Check((retuni = PyTuple_GET_ITEM(retobj, 0))) ||\r
- !(PyInt_Check(PyTuple_GET_ITEM(retobj, 1)) ||\r
- PyLong_Check(PyTuple_GET_ITEM(retobj, 1)))) {\r
- PyErr_SetString(PyExc_TypeError,\r
- "decoding error handler must return "\r
- "(unicode, int) tuple");\r
- goto errorexit;\r
- }\r
-\r
- retunisize = PyUnicode_GET_SIZE(retuni);\r
- if (retunisize > 0) {\r
- REQUIRE_DECODEBUFFER(buf, retunisize);\r
- memcpy((char *)buf->outbuf, PyUnicode_AS_DATA(retuni),\r
- retunisize * Py_UNICODE_SIZE);\r
- buf->outbuf += retunisize;\r
- }\r
-\r
- newpos = PyInt_AsSsize_t(PyTuple_GET_ITEM(retobj, 1));\r
- if (newpos < 0 && !PyErr_Occurred())\r
- newpos += (Py_ssize_t)(buf->inbuf_end - buf->inbuf_top);\r
- if (newpos < 0 || buf->inbuf_top + newpos > buf->inbuf_end) {\r
- PyErr_Clear();\r
- PyErr_Format(PyExc_IndexError,\r
- "position %zd from error handler out of bounds",\r
- newpos);\r
- goto errorexit;\r
- }\r
- buf->inbuf = buf->inbuf_top + newpos;\r
- Py_DECREF(retobj);\r
- return 0;\r
-\r
-errorexit:\r
- Py_XDECREF(retobj);\r
- return -1;\r
-}\r
-\r
-static PyObject *\r
-multibytecodec_encode(MultibyteCodec *codec,\r
- MultibyteCodec_State *state,\r
- const Py_UNICODE **data, Py_ssize_t datalen,\r
- PyObject *errors, int flags)\r
-{\r
- MultibyteEncodeBuffer buf;\r
- Py_ssize_t finalsize, r = 0;\r
-\r
- if (datalen == 0 && !(flags & MBENC_RESET))\r
- return PyString_FromString("");\r
-\r
- buf.excobj = NULL;\r
- buf.inbuf = buf.inbuf_top = *data;\r
- buf.inbuf_end = buf.inbuf_top + datalen;\r
-\r
- if (datalen > (PY_SSIZE_T_MAX - 16) / 2) {\r
- PyErr_NoMemory();\r
- goto errorexit;\r
- }\r
-\r
- buf.outobj = PyString_FromStringAndSize(NULL, datalen * 2 + 16);\r
- if (buf.outobj == NULL)\r
- goto errorexit;\r
- buf.outbuf = (unsigned char *)PyString_AS_STRING(buf.outobj);\r
- buf.outbuf_end = buf.outbuf + PyString_GET_SIZE(buf.outobj);\r
-\r
- while (buf.inbuf < buf.inbuf_end) {\r
- Py_ssize_t inleft, outleft;\r
-\r
- /* we don't reuse inleft and outleft here.\r
- * error callbacks can relocate the cursor anywhere on buffer*/\r
- inleft = (Py_ssize_t)(buf.inbuf_end - buf.inbuf);\r
- outleft = (Py_ssize_t)(buf.outbuf_end - buf.outbuf);\r
- r = codec->encode(state, codec->config, &buf.inbuf, inleft,\r
- &buf.outbuf, outleft, flags);\r
- if ((r == 0) || (r == MBERR_TOOFEW && !(flags & MBENC_FLUSH)))\r
- break;\r
- else if (multibytecodec_encerror(codec, state, &buf, errors,r))\r
- goto errorexit;\r
- else if (r == MBERR_TOOFEW)\r
- break;\r
- }\r
-\r
- if (codec->encreset != NULL && (flags & MBENC_RESET))\r
- for (;;) {\r
- Py_ssize_t outleft;\r
-\r
- outleft = (Py_ssize_t)(buf.outbuf_end - buf.outbuf);\r
- r = codec->encreset(state, codec->config, &buf.outbuf,\r
- outleft);\r
- if (r == 0)\r
- break;\r
- else if (multibytecodec_encerror(codec, state,\r
- &buf, errors, r))\r
- goto errorexit;\r
- }\r
-\r
- finalsize = (Py_ssize_t)((char *)buf.outbuf -\r
- PyString_AS_STRING(buf.outobj));\r
-\r
- if (finalsize != PyString_GET_SIZE(buf.outobj))\r
- if (_PyString_Resize(&buf.outobj, finalsize) == -1)\r
- goto errorexit;\r
-\r
- *data = buf.inbuf;\r
- Py_XDECREF(buf.excobj);\r
- return buf.outobj;\r
-\r
-errorexit:\r
- Py_XDECREF(buf.excobj);\r
- Py_XDECREF(buf.outobj);\r
- return NULL;\r
-}\r
-\r
-static PyObject *\r
-MultibyteCodec_Encode(MultibyteCodecObject *self,\r
- PyObject *args, PyObject *kwargs)\r
-{\r
- MultibyteCodec_State state;\r
- Py_UNICODE *data;\r
- PyObject *errorcb, *r, *arg, *ucvt;\r
- const char *errors = NULL;\r
- Py_ssize_t datalen;\r
-\r
- if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O|z:encode",\r
- codeckwarglist, &arg, &errors))\r
- return NULL;\r
-\r
- if (PyUnicode_Check(arg))\r
- ucvt = NULL;\r
- else {\r
- arg = ucvt = PyObject_Unicode(arg);\r
- if (arg == NULL)\r
- return NULL;\r
- else if (!PyUnicode_Check(arg)) {\r
- PyErr_SetString(PyExc_TypeError,\r
- "couldn't convert the object to unicode.");\r
- Py_DECREF(ucvt);\r
- return NULL;\r
- }\r
- }\r
-\r
- data = PyUnicode_AS_UNICODE(arg);\r
- datalen = PyUnicode_GET_SIZE(arg);\r
-\r
- errorcb = internal_error_callback(errors);\r
- if (errorcb == NULL) {\r
- Py_XDECREF(ucvt);\r
- return NULL;\r
- }\r
-\r
- if (self->codec->encinit != NULL &&\r
- self->codec->encinit(&state, self->codec->config) != 0)\r
- goto errorexit;\r
- r = multibytecodec_encode(self->codec, &state,\r
- (const Py_UNICODE **)&data, datalen, errorcb,\r
- MBENC_FLUSH | MBENC_RESET);\r
- if (r == NULL)\r
- goto errorexit;\r
-\r
- ERROR_DECREF(errorcb);\r
- Py_XDECREF(ucvt);\r
- return make_tuple(r, datalen);\r
-\r
-errorexit:\r
- ERROR_DECREF(errorcb);\r
- Py_XDECREF(ucvt);\r
- return NULL;\r
-}\r
-\r
-static PyObject *\r
-MultibyteCodec_Decode(MultibyteCodecObject *self,\r
- PyObject *args, PyObject *kwargs)\r
-{\r
- MultibyteCodec_State state;\r
- MultibyteDecodeBuffer buf;\r
- PyObject *errorcb;\r
- Py_buffer pdata;\r
- const char *data, *errors = NULL;\r
- Py_ssize_t datalen, finalsize;\r
-\r
- if (!PyArg_ParseTupleAndKeywords(args, kwargs, "s*|z:decode",\r
- codeckwarglist, &pdata, &errors))\r
- return NULL;\r
- data = pdata.buf;\r
- datalen = pdata.len;\r
-\r
- errorcb = internal_error_callback(errors);\r
- if (errorcb == NULL) {\r
- PyBuffer_Release(&pdata);\r
- return NULL;\r
- }\r
-\r
- if (datalen == 0) {\r
- PyBuffer_Release(&pdata);\r
- ERROR_DECREF(errorcb);\r
- return make_tuple(PyUnicode_FromUnicode(NULL, 0), 0);\r
- }\r
-\r
- buf.excobj = NULL;\r
- buf.inbuf = buf.inbuf_top = (unsigned char *)data;\r
- buf.inbuf_end = buf.inbuf_top + datalen;\r
- buf.outobj = PyUnicode_FromUnicode(NULL, datalen);\r
- if (buf.outobj == NULL)\r
- goto errorexit;\r
- buf.outbuf = PyUnicode_AS_UNICODE(buf.outobj);\r
- buf.outbuf_end = buf.outbuf + PyUnicode_GET_SIZE(buf.outobj);\r
-\r
- if (self->codec->decinit != NULL &&\r
- self->codec->decinit(&state, self->codec->config) != 0)\r
- goto errorexit;\r
-\r
- while (buf.inbuf < buf.inbuf_end) {\r
- Py_ssize_t inleft, outleft, r;\r
-\r
- inleft = (Py_ssize_t)(buf.inbuf_end - buf.inbuf);\r
- outleft = (Py_ssize_t)(buf.outbuf_end - buf.outbuf);\r
-\r
- r = self->codec->decode(&state, self->codec->config,\r
- &buf.inbuf, inleft, &buf.outbuf, outleft);\r
- if (r == 0)\r
- break;\r
- else if (multibytecodec_decerror(self->codec, &state,\r
- &buf, errorcb, r))\r
- goto errorexit;\r
- }\r
-\r
- finalsize = (Py_ssize_t)(buf.outbuf -\r
- PyUnicode_AS_UNICODE(buf.outobj));\r
-\r
- if (finalsize != PyUnicode_GET_SIZE(buf.outobj))\r
- if (PyUnicode_Resize(&buf.outobj, finalsize) == -1)\r
- goto errorexit;\r
-\r
- PyBuffer_Release(&pdata);\r
- Py_XDECREF(buf.excobj);\r
- ERROR_DECREF(errorcb);\r
- return make_tuple(buf.outobj, datalen);\r
-\r
-errorexit:\r
- PyBuffer_Release(&pdata);\r
- ERROR_DECREF(errorcb);\r
- Py_XDECREF(buf.excobj);\r
- Py_XDECREF(buf.outobj);\r
-\r
- return NULL;\r
-}\r
-\r
-static struct PyMethodDef multibytecodec_methods[] = {\r
- {"encode", (PyCFunction)MultibyteCodec_Encode,\r
- METH_VARARGS | METH_KEYWORDS,\r
- MultibyteCodec_Encode__doc__},\r
- {"decode", (PyCFunction)MultibyteCodec_Decode,\r
- METH_VARARGS | METH_KEYWORDS,\r
- MultibyteCodec_Decode__doc__},\r
- {NULL, NULL},\r
-};\r
-\r
-static void\r
-multibytecodec_dealloc(MultibyteCodecObject *self)\r
-{\r
- PyObject_Del(self);\r
-}\r
-\r
-static PyTypeObject MultibyteCodec_Type = {\r
- PyVarObject_HEAD_INIT(NULL, 0)\r
- "MultibyteCodec", /* tp_name */\r
- sizeof(MultibyteCodecObject), /* tp_basicsize */\r
- 0, /* tp_itemsize */\r
- /* methods */\r
- (destructor)multibytecodec_dealloc, /* tp_dealloc */\r
- 0, /* tp_print */\r
- 0, /* tp_getattr */\r
- 0, /* tp_setattr */\r
- 0, /* tp_compare */\r
- 0, /* tp_repr */\r
- 0, /* tp_as_number */\r
- 0, /* tp_as_sequence */\r
- 0, /* tp_as_mapping */\r
- 0, /* tp_hash */\r
- 0, /* tp_call */\r
- 0, /* tp_str */\r
- PyObject_GenericGetAttr, /* tp_getattro */\r
- 0, /* tp_setattro */\r
- 0, /* tp_as_buffer */\r
- Py_TPFLAGS_DEFAULT, /* tp_flags */\r
- 0, /* tp_doc */\r
- 0, /* tp_traverse */\r
- 0, /* tp_clear */\r
- 0, /* tp_richcompare */\r
- 0, /* tp_weaklistoffset */\r
- 0, /* tp_iter */\r
- 0, /* tp_iterext */\r
- multibytecodec_methods, /* tp_methods */\r
-};\r
-\r
-\r
-/**\r
- * Utility functions for stateful codec mechanism\r
- */\r
-\r
-#define STATEFUL_DCTX(o) ((MultibyteStatefulDecoderContext *)(o))\r
-#define STATEFUL_ECTX(o) ((MultibyteStatefulEncoderContext *)(o))\r
-\r
-static PyObject *\r
-encoder_encode_stateful(MultibyteStatefulEncoderContext *ctx,\r
- PyObject *unistr, int final)\r
-{\r
- PyObject *ucvt, *r = NULL;\r
- Py_UNICODE *inbuf, *inbuf_end, *inbuf_tmp = NULL;\r
- Py_ssize_t datalen, origpending;\r
-\r
- if (PyUnicode_Check(unistr))\r
- ucvt = NULL;\r
- else {\r
- unistr = ucvt = PyObject_Unicode(unistr);\r
- if (unistr == NULL)\r
- return NULL;\r
- else if (!PyUnicode_Check(unistr)) {\r
- PyErr_SetString(PyExc_TypeError,\r
- "couldn't convert the object to unicode.");\r
- Py_DECREF(ucvt);\r
- return NULL;\r
- }\r
- }\r
-\r
- datalen = PyUnicode_GET_SIZE(unistr);\r
- origpending = ctx->pendingsize;\r
-\r
- if (origpending > 0) {\r
- if (datalen > PY_SSIZE_T_MAX - ctx->pendingsize) {\r
- PyErr_NoMemory();\r
- /* inbuf_tmp == NULL */\r
- goto errorexit;\r
- }\r
- inbuf_tmp = PyMem_New(Py_UNICODE, datalen + ctx->pendingsize);\r
- if (inbuf_tmp == NULL)\r
- goto errorexit;\r
- memcpy(inbuf_tmp, ctx->pending,\r
- Py_UNICODE_SIZE * ctx->pendingsize);\r
- memcpy(inbuf_tmp + ctx->pendingsize,\r
- PyUnicode_AS_UNICODE(unistr),\r
- Py_UNICODE_SIZE * datalen);\r
- datalen += ctx->pendingsize;\r
- ctx->pendingsize = 0;\r
- inbuf = inbuf_tmp;\r
- }\r
- else\r
- inbuf = (Py_UNICODE *)PyUnicode_AS_UNICODE(unistr);\r
-\r
- inbuf_end = inbuf + datalen;\r
-\r
- r = multibytecodec_encode(ctx->codec, &ctx->state,\r
- (const Py_UNICODE **)&inbuf, datalen,\r
- ctx->errors, final ? MBENC_FLUSH | MBENC_RESET : 0);\r
- if (r == NULL) {\r
- /* recover the original pending buffer */\r
- if (origpending > 0)\r
- memcpy(ctx->pending, inbuf_tmp,\r
- Py_UNICODE_SIZE * origpending);\r
- ctx->pendingsize = origpending;\r
- goto errorexit;\r
- }\r
-\r
- if (inbuf < inbuf_end) {\r
- ctx->pendingsize = (Py_ssize_t)(inbuf_end - inbuf);\r
- if (ctx->pendingsize > MAXENCPENDING) {\r
- /* normal codecs can't reach here */\r
- ctx->pendingsize = 0;\r
- PyErr_SetString(PyExc_UnicodeError,\r
- "pending buffer overflow");\r
- goto errorexit;\r
- }\r
- memcpy(ctx->pending, inbuf,\r
- ctx->pendingsize * Py_UNICODE_SIZE);\r
- }\r
-\r
- if (inbuf_tmp != NULL)\r
- PyMem_Del(inbuf_tmp);\r
- Py_XDECREF(ucvt);\r
- return r;\r
-\r
-errorexit:\r
- if (inbuf_tmp != NULL)\r
- PyMem_Del(inbuf_tmp);\r
- Py_XDECREF(r);\r
- Py_XDECREF(ucvt);\r
- return NULL;\r
-}\r
-\r
-static int\r
-decoder_append_pending(MultibyteStatefulDecoderContext *ctx,\r
- MultibyteDecodeBuffer *buf)\r
-{\r
- Py_ssize_t npendings;\r
-\r
- npendings = (Py_ssize_t)(buf->inbuf_end - buf->inbuf);\r
- if (npendings + ctx->pendingsize > MAXDECPENDING ||\r
- npendings > PY_SSIZE_T_MAX - ctx->pendingsize) {\r
- PyErr_SetString(PyExc_UnicodeError, "pending buffer overflow");\r
- return -1;\r
- }\r
- memcpy(ctx->pending + ctx->pendingsize, buf->inbuf, npendings);\r
- ctx->pendingsize += npendings;\r
- return 0;\r
-}\r
-\r
-static int\r
-decoder_prepare_buffer(MultibyteDecodeBuffer *buf, const char *data,\r
- Py_ssize_t size)\r
-{\r
- buf->inbuf = buf->inbuf_top = (const unsigned char *)data;\r
- buf->inbuf_end = buf->inbuf_top + size;\r
- if (buf->outobj == NULL) { /* only if outobj is not allocated yet */\r
- buf->outobj = PyUnicode_FromUnicode(NULL, size);\r
- if (buf->outobj == NULL)\r
- return -1;\r
- buf->outbuf = PyUnicode_AS_UNICODE(buf->outobj);\r
- buf->outbuf_end = buf->outbuf +\r
- PyUnicode_GET_SIZE(buf->outobj);\r
- }\r
-\r
- return 0;\r
-}\r
-\r
-static int\r
-decoder_feed_buffer(MultibyteStatefulDecoderContext *ctx,\r
- MultibyteDecodeBuffer *buf)\r
-{\r
- while (buf->inbuf < buf->inbuf_end) {\r
- Py_ssize_t inleft, outleft;\r
- Py_ssize_t r;\r
-\r
- inleft = (Py_ssize_t)(buf->inbuf_end - buf->inbuf);\r
- outleft = (Py_ssize_t)(buf->outbuf_end - buf->outbuf);\r
-\r
- r = ctx->codec->decode(&ctx->state, ctx->codec->config,\r
- &buf->inbuf, inleft, &buf->outbuf, outleft);\r
- if (r == 0 || r == MBERR_TOOFEW)\r
- break;\r
- else if (multibytecodec_decerror(ctx->codec, &ctx->state,\r
- buf, ctx->errors, r))\r
- return -1;\r
- }\r
- return 0;\r
-}\r
-\r
-\r
-/**\r
- * MultibyteIncrementalEncoder object\r
- */\r
-\r
-static PyObject *\r
-mbiencoder_encode(MultibyteIncrementalEncoderObject *self,\r
- PyObject *args, PyObject *kwargs)\r
-{\r
- PyObject *data;\r
- int final = 0;\r
-\r
- if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O|i:encode",\r
- incrementalkwarglist, &data, &final))\r
- return NULL;\r
-\r
- return encoder_encode_stateful(STATEFUL_ECTX(self), data, final);\r
-}\r
-\r
-static PyObject *\r
-mbiencoder_reset(MultibyteIncrementalEncoderObject *self)\r
-{\r
- if (self->codec->decreset != NULL &&\r
- self->codec->decreset(&self->state, self->codec->config) != 0)\r
- return NULL;\r
- self->pendingsize = 0;\r
-\r
- Py_RETURN_NONE;\r
-}\r
-\r
-static struct PyMethodDef mbiencoder_methods[] = {\r
- {"encode", (PyCFunction)mbiencoder_encode,\r
- METH_VARARGS | METH_KEYWORDS, NULL},\r
- {"reset", (PyCFunction)mbiencoder_reset,\r
- METH_NOARGS, NULL},\r
- {NULL, NULL},\r
-};\r
-\r
-static PyObject *\r
-mbiencoder_new(PyTypeObject *type, PyObject *args, PyObject *kwds)\r
-{\r
- MultibyteIncrementalEncoderObject *self;\r
- PyObject *codec = NULL;\r
- char *errors = NULL;\r
-\r
- if (!PyArg_ParseTupleAndKeywords(args, kwds, "|s:IncrementalEncoder",\r
- incnewkwarglist, &errors))\r
- return NULL;\r
-\r
- self = (MultibyteIncrementalEncoderObject *)type->tp_alloc(type, 0);\r
- if (self == NULL)\r
- return NULL;\r
-\r
- codec = PyObject_GetAttrString((PyObject *)type, "codec");\r
- if (codec == NULL)\r
- goto errorexit;\r
- if (!MultibyteCodec_Check(codec)) {\r
- PyErr_SetString(PyExc_TypeError, "codec is unexpected type");\r
- goto errorexit;\r
- }\r
-\r
- self->codec = ((MultibyteCodecObject *)codec)->codec;\r
- self->pendingsize = 0;\r
- self->errors = internal_error_callback(errors);\r
- if (self->errors == NULL)\r
- goto errorexit;\r
- if (self->codec->encinit != NULL &&\r
- self->codec->encinit(&self->state, self->codec->config) != 0)\r
- goto errorexit;\r
-\r
- Py_DECREF(codec);\r
- return (PyObject *)self;\r
-\r
-errorexit:\r
- Py_XDECREF(self);\r
- Py_XDECREF(codec);\r
- return NULL;\r
-}\r
-\r
-static int\r
-mbiencoder_init(PyObject *self, PyObject *args, PyObject *kwds)\r
-{\r
- return 0;\r
-}\r
-\r
-static int\r
-mbiencoder_traverse(MultibyteIncrementalEncoderObject *self,\r
- visitproc visit, void *arg)\r
-{\r
- if (ERROR_ISCUSTOM(self->errors))\r
- Py_VISIT(self->errors);\r
- return 0;\r
-}\r
-\r
-static void\r
-mbiencoder_dealloc(MultibyteIncrementalEncoderObject *self)\r
-{\r
- PyObject_GC_UnTrack(self);\r
- ERROR_DECREF(self->errors);\r
- Py_TYPE(self)->tp_free(self);\r
-}\r
-\r
-static PyTypeObject MultibyteIncrementalEncoder_Type = {\r
- PyVarObject_HEAD_INIT(NULL, 0)\r
- "MultibyteIncrementalEncoder", /* tp_name */\r
- sizeof(MultibyteIncrementalEncoderObject), /* tp_basicsize */\r
- 0, /* tp_itemsize */\r
- /* methods */\r
- (destructor)mbiencoder_dealloc, /* tp_dealloc */\r
- 0, /* tp_print */\r
- 0, /* tp_getattr */\r
- 0, /* tp_setattr */\r
- 0, /* tp_compare */\r
- 0, /* tp_repr */\r
- 0, /* tp_as_number */\r
- 0, /* tp_as_sequence */\r
- 0, /* tp_as_mapping */\r
- 0, /* tp_hash */\r
- 0, /* tp_call */\r
- 0, /* tp_str */\r
- PyObject_GenericGetAttr, /* tp_getattro */\r
- 0, /* tp_setattro */\r
- 0, /* tp_as_buffer */\r
- Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC\r
- | Py_TPFLAGS_BASETYPE, /* tp_flags */\r
- 0, /* tp_doc */\r
- (traverseproc)mbiencoder_traverse, /* tp_traverse */\r
- 0, /* tp_clear */\r
- 0, /* tp_richcompare */\r
- 0, /* tp_weaklistoffset */\r
- 0, /* tp_iter */\r
- 0, /* tp_iterext */\r
- mbiencoder_methods, /* tp_methods */\r
- 0, /* tp_members */\r
- codecctx_getsets, /* tp_getset */\r
- 0, /* tp_base */\r
- 0, /* tp_dict */\r
- 0, /* tp_descr_get */\r
- 0, /* tp_descr_set */\r
- 0, /* tp_dictoffset */\r
- mbiencoder_init, /* tp_init */\r
- 0, /* tp_alloc */\r
- mbiencoder_new, /* tp_new */\r
-};\r
-\r
-\r
-/**\r
- * MultibyteIncrementalDecoder object\r
- */\r
-\r
-static PyObject *\r
-mbidecoder_decode(MultibyteIncrementalDecoderObject *self,\r
- PyObject *args, PyObject *kwargs)\r
-{\r
- MultibyteDecodeBuffer buf;\r
- char *data, *wdata = NULL;\r
- Py_buffer pdata;\r
- Py_ssize_t wsize, finalsize = 0, size, origpending;\r
- int final = 0;\r
-\r
- if (!PyArg_ParseTupleAndKeywords(args, kwargs, "s*|i:decode",\r
- incrementalkwarglist, &pdata, &final))\r
- return NULL;\r
- data = pdata.buf;\r
- size = pdata.len;\r
-\r
- buf.outobj = buf.excobj = NULL;\r
- origpending = self->pendingsize;\r
-\r
- if (self->pendingsize == 0) {\r
- wsize = size;\r
- wdata = data;\r
- }\r
- else {\r
- if (size > PY_SSIZE_T_MAX - self->pendingsize) {\r
- PyErr_NoMemory();\r
- goto errorexit;\r
- }\r
- wsize = size + self->pendingsize;\r
- wdata = PyMem_Malloc(wsize);\r
- if (wdata == NULL)\r
- goto errorexit;\r
- memcpy(wdata, self->pending, self->pendingsize);\r
- memcpy(wdata + self->pendingsize, data, size);\r
- self->pendingsize = 0;\r
- }\r
-\r
- if (decoder_prepare_buffer(&buf, wdata, wsize) != 0)\r
- goto errorexit;\r
-\r
- if (decoder_feed_buffer(STATEFUL_DCTX(self), &buf))\r
- goto errorexit;\r
-\r
- if (final && buf.inbuf < buf.inbuf_end) {\r
- if (multibytecodec_decerror(self->codec, &self->state,\r
- &buf, self->errors, MBERR_TOOFEW)) {\r
- /* recover the original pending buffer */\r
- memcpy(self->pending, wdata, origpending);\r
- self->pendingsize = origpending;\r
- goto errorexit;\r
- }\r
- }\r
-\r
- if (buf.inbuf < buf.inbuf_end) { /* pending sequence still exists */\r
- if (decoder_append_pending(STATEFUL_DCTX(self), &buf) != 0)\r
- goto errorexit;\r
- }\r
-\r
- finalsize = (Py_ssize_t)(buf.outbuf - PyUnicode_AS_UNICODE(buf.outobj));\r
- if (finalsize != PyUnicode_GET_SIZE(buf.outobj))\r
- if (PyUnicode_Resize(&buf.outobj, finalsize) == -1)\r
- goto errorexit;\r
-\r
- PyBuffer_Release(&pdata);\r
- if (wdata != data)\r
- PyMem_Del(wdata);\r
- Py_XDECREF(buf.excobj);\r
- return buf.outobj;\r
-\r
-errorexit:\r
- PyBuffer_Release(&pdata);\r
- if (wdata != NULL && wdata != data)\r
- PyMem_Del(wdata);\r
- Py_XDECREF(buf.excobj);\r
- Py_XDECREF(buf.outobj);\r
- return NULL;\r
-}\r
-\r
-static PyObject *\r
-mbidecoder_reset(MultibyteIncrementalDecoderObject *self)\r
-{\r
- if (self->codec->decreset != NULL &&\r
- self->codec->decreset(&self->state, self->codec->config) != 0)\r
- return NULL;\r
- self->pendingsize = 0;\r
-\r
- Py_RETURN_NONE;\r
-}\r
-\r
-static struct PyMethodDef mbidecoder_methods[] = {\r
- {"decode", (PyCFunction)mbidecoder_decode,\r
- METH_VARARGS | METH_KEYWORDS, NULL},\r
- {"reset", (PyCFunction)mbidecoder_reset,\r
- METH_NOARGS, NULL},\r
- {NULL, NULL},\r
-};\r
-\r
-static PyObject *\r
-mbidecoder_new(PyTypeObject *type, PyObject *args, PyObject *kwds)\r
-{\r
- MultibyteIncrementalDecoderObject *self;\r
- PyObject *codec = NULL;\r
- char *errors = NULL;\r
-\r
- if (!PyArg_ParseTupleAndKeywords(args, kwds, "|s:IncrementalDecoder",\r
- incnewkwarglist, &errors))\r
- return NULL;\r
-\r
- self = (MultibyteIncrementalDecoderObject *)type->tp_alloc(type, 0);\r
- if (self == NULL)\r
- return NULL;\r
-\r
- codec = PyObject_GetAttrString((PyObject *)type, "codec");\r
- if (codec == NULL)\r
- goto errorexit;\r
- if (!MultibyteCodec_Check(codec)) {\r
- PyErr_SetString(PyExc_TypeError, "codec is unexpected type");\r
- goto errorexit;\r
- }\r
-\r
- self->codec = ((MultibyteCodecObject *)codec)->codec;\r
- self->pendingsize = 0;\r
- self->errors = internal_error_callback(errors);\r
- if (self->errors == NULL)\r
- goto errorexit;\r
- if (self->codec->decinit != NULL &&\r
- self->codec->decinit(&self->state, self->codec->config) != 0)\r
- goto errorexit;\r
-\r
- Py_DECREF(codec);\r
- return (PyObject *)self;\r
-\r
-errorexit:\r
- Py_XDECREF(self);\r
- Py_XDECREF(codec);\r
- return NULL;\r
-}\r
-\r
-static int\r
-mbidecoder_init(PyObject *self, PyObject *args, PyObject *kwds)\r
-{\r
- return 0;\r
-}\r
-\r
-static int\r
-mbidecoder_traverse(MultibyteIncrementalDecoderObject *self,\r
- visitproc visit, void *arg)\r
-{\r
- if (ERROR_ISCUSTOM(self->errors))\r
- Py_VISIT(self->errors);\r
- return 0;\r
-}\r
-\r
-static void\r
-mbidecoder_dealloc(MultibyteIncrementalDecoderObject *self)\r
-{\r
- PyObject_GC_UnTrack(self);\r
- ERROR_DECREF(self->errors);\r
- Py_TYPE(self)->tp_free(self);\r
-}\r
-\r
-static PyTypeObject MultibyteIncrementalDecoder_Type = {\r
- PyVarObject_HEAD_INIT(NULL, 0)\r
- "MultibyteIncrementalDecoder", /* tp_name */\r
- sizeof(MultibyteIncrementalDecoderObject), /* tp_basicsize */\r
- 0, /* tp_itemsize */\r
- /* methods */\r
- (destructor)mbidecoder_dealloc, /* tp_dealloc */\r
- 0, /* tp_print */\r
- 0, /* tp_getattr */\r
- 0, /* tp_setattr */\r
- 0, /* tp_compare */\r
- 0, /* tp_repr */\r
- 0, /* tp_as_number */\r
- 0, /* tp_as_sequence */\r
- 0, /* tp_as_mapping */\r
- 0, /* tp_hash */\r
- 0, /* tp_call */\r
- 0, /* tp_str */\r
- PyObject_GenericGetAttr, /* tp_getattro */\r
- 0, /* tp_setattro */\r
- 0, /* tp_as_buffer */\r
- Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC\r
- | Py_TPFLAGS_BASETYPE, /* tp_flags */\r
- 0, /* tp_doc */\r
- (traverseproc)mbidecoder_traverse, /* tp_traverse */\r
- 0, /* tp_clear */\r
- 0, /* tp_richcompare */\r
- 0, /* tp_weaklistoffset */\r
- 0, /* tp_iter */\r
- 0, /* tp_iterext */\r
- mbidecoder_methods, /* tp_methods */\r
- 0, /* tp_members */\r
- codecctx_getsets, /* tp_getset */\r
- 0, /* tp_base */\r
- 0, /* tp_dict */\r
- 0, /* tp_descr_get */\r
- 0, /* tp_descr_set */\r
- 0, /* tp_dictoffset */\r
- mbidecoder_init, /* tp_init */\r
- 0, /* tp_alloc */\r
- mbidecoder_new, /* tp_new */\r
-};\r
-\r
-\r
-/**\r
- * MultibyteStreamReader object\r
- */\r
-\r
-static PyObject *\r
-mbstreamreader_iread(MultibyteStreamReaderObject *self,\r
- const char *method, Py_ssize_t sizehint)\r
-{\r
- MultibyteDecodeBuffer buf;\r
- PyObject *cres;\r
- Py_ssize_t rsize, finalsize = 0;\r
-\r
- if (sizehint == 0)\r
- return PyUnicode_FromUnicode(NULL, 0);\r
-\r
- buf.outobj = buf.excobj = NULL;\r
- cres = NULL;\r
-\r
- for (;;) {\r
- int endoffile;\r
-\r
- if (sizehint < 0)\r
- cres = PyObject_CallMethod(self->stream,\r
- (char *)method, NULL);\r
- else\r
- cres = PyObject_CallMethod(self->stream,\r
- (char *)method, "i", sizehint);\r
- if (cres == NULL)\r
- goto errorexit;\r
-\r
- if (!PyString_Check(cres)) {\r
- PyErr_SetString(PyExc_TypeError,\r
- "stream function returned a "\r
- "non-string object");\r
- goto errorexit;\r
- }\r
-\r
- endoffile = (PyString_GET_SIZE(cres) == 0);\r
-\r
- if (self->pendingsize > 0) {\r
- PyObject *ctr;\r
- char *ctrdata;\r
-\r
- if (PyString_GET_SIZE(cres) > PY_SSIZE_T_MAX - self->pendingsize) {\r
- PyErr_NoMemory();\r
- goto errorexit;\r
- }\r
- rsize = PyString_GET_SIZE(cres) + self->pendingsize;\r
- ctr = PyString_FromStringAndSize(NULL, rsize);\r
- if (ctr == NULL)\r
- goto errorexit;\r
- ctrdata = PyString_AS_STRING(ctr);\r
- memcpy(ctrdata, self->pending, self->pendingsize);\r
- memcpy(ctrdata + self->pendingsize,\r
- PyString_AS_STRING(cres),\r
- PyString_GET_SIZE(cres));\r
- Py_DECREF(cres);\r
- cres = ctr;\r
- self->pendingsize = 0;\r
- }\r
-\r
- rsize = PyString_GET_SIZE(cres);\r
- if (decoder_prepare_buffer(&buf, PyString_AS_STRING(cres),\r
- rsize) != 0)\r
- goto errorexit;\r
-\r
- if (rsize > 0 && decoder_feed_buffer(\r
- (MultibyteStatefulDecoderContext *)self, &buf))\r
- goto errorexit;\r
-\r
- if (endoffile || sizehint < 0) {\r
- if (buf.inbuf < buf.inbuf_end &&\r
- multibytecodec_decerror(self->codec, &self->state,\r
- &buf, self->errors, MBERR_TOOFEW))\r
- goto errorexit;\r
- }\r
-\r
- if (buf.inbuf < buf.inbuf_end) { /* pending sequence exists */\r
- if (decoder_append_pending(STATEFUL_DCTX(self),\r
- &buf) != 0)\r
- goto errorexit;\r
- }\r
-\r
- finalsize = (Py_ssize_t)(buf.outbuf -\r
- PyUnicode_AS_UNICODE(buf.outobj));\r
- Py_DECREF(cres);\r
- cres = NULL;\r
-\r
- if (sizehint < 0 || finalsize != 0 || rsize == 0)\r
- break;\r
-\r
- sizehint = 1; /* read 1 more byte and retry */\r
- }\r
-\r
- if (finalsize != PyUnicode_GET_SIZE(buf.outobj))\r
- if (PyUnicode_Resize(&buf.outobj, finalsize) == -1)\r
- goto errorexit;\r
-\r
- Py_XDECREF(cres);\r
- Py_XDECREF(buf.excobj);\r
- return buf.outobj;\r
-\r
-errorexit:\r
- Py_XDECREF(cres);\r
- Py_XDECREF(buf.excobj);\r
- Py_XDECREF(buf.outobj);\r
- return NULL;\r
-}\r
-\r
-static PyObject *\r
-mbstreamreader_read(MultibyteStreamReaderObject *self, PyObject *args)\r
-{\r
- PyObject *sizeobj = NULL;\r
- Py_ssize_t size;\r
-\r
- if (!PyArg_UnpackTuple(args, "read", 0, 1, &sizeobj))\r
- return NULL;\r
-\r
- if (sizeobj == Py_None || sizeobj == NULL)\r
- size = -1;\r
- else if (PyInt_Check(sizeobj))\r
- size = PyInt_AsSsize_t(sizeobj);\r
- else {\r
- PyErr_SetString(PyExc_TypeError, "arg 1 must be an integer");\r
- return NULL;\r
- }\r
-\r
- return mbstreamreader_iread(self, "read", size);\r
-}\r
-\r
-static PyObject *\r
-mbstreamreader_readline(MultibyteStreamReaderObject *self, PyObject *args)\r
-{\r
- PyObject *sizeobj = NULL;\r
- Py_ssize_t size;\r
-\r
- if (!PyArg_UnpackTuple(args, "readline", 0, 1, &sizeobj))\r
- return NULL;\r
-\r
- if (sizeobj == Py_None || sizeobj == NULL)\r
- size = -1;\r
- else if (PyInt_Check(sizeobj))\r
- size = PyInt_AsSsize_t(sizeobj);\r
- else {\r
- PyErr_SetString(PyExc_TypeError, "arg 1 must be an integer");\r
- return NULL;\r
- }\r
-\r
- return mbstreamreader_iread(self, "readline", size);\r
-}\r
-\r
-static PyObject *\r
-mbstreamreader_readlines(MultibyteStreamReaderObject *self, PyObject *args)\r
-{\r
- PyObject *sizehintobj = NULL, *r, *sr;\r
- Py_ssize_t sizehint;\r
-\r
- if (!PyArg_UnpackTuple(args, "readlines", 0, 1, &sizehintobj))\r
- return NULL;\r
-\r
- if (sizehintobj == Py_None || sizehintobj == NULL)\r
- sizehint = -1;\r
- else if (PyInt_Check(sizehintobj))\r
- sizehint = PyInt_AsSsize_t(sizehintobj);\r
- else {\r
- PyErr_SetString(PyExc_TypeError, "arg 1 must be an integer");\r
- return NULL;\r
- }\r
-\r
- r = mbstreamreader_iread(self, "read", sizehint);\r
- if (r == NULL)\r
- return NULL;\r
-\r
- sr = PyUnicode_Splitlines(r, 1);\r
- Py_DECREF(r);\r
- return sr;\r
-}\r
-\r
-static PyObject *\r
-mbstreamreader_reset(MultibyteStreamReaderObject *self)\r
-{\r
- if (self->codec->decreset != NULL &&\r
- self->codec->decreset(&self->state, self->codec->config) != 0)\r
- return NULL;\r
- self->pendingsize = 0;\r
-\r
- Py_RETURN_NONE;\r
-}\r
-\r
-static struct PyMethodDef mbstreamreader_methods[] = {\r
- {"read", (PyCFunction)mbstreamreader_read,\r
- METH_VARARGS, NULL},\r
- {"readline", (PyCFunction)mbstreamreader_readline,\r
- METH_VARARGS, NULL},\r
- {"readlines", (PyCFunction)mbstreamreader_readlines,\r
- METH_VARARGS, NULL},\r
- {"reset", (PyCFunction)mbstreamreader_reset,\r
- METH_NOARGS, NULL},\r
- {NULL, NULL},\r
-};\r
-\r
-static PyMemberDef mbstreamreader_members[] = {\r
- {"stream", T_OBJECT,\r
- offsetof(MultibyteStreamReaderObject, stream),\r
- READONLY, NULL},\r
- {NULL,}\r
-};\r
-\r
-static PyObject *\r
-mbstreamreader_new(PyTypeObject *type, PyObject *args, PyObject *kwds)\r
-{\r
- MultibyteStreamReaderObject *self;\r
- PyObject *stream, *codec = NULL;\r
- char *errors = NULL;\r
-\r
- if (!PyArg_ParseTupleAndKeywords(args, kwds, "O|s:StreamReader",\r
- streamkwarglist, &stream, &errors))\r
- return NULL;\r
-\r
- self = (MultibyteStreamReaderObject *)type->tp_alloc(type, 0);\r
- if (self == NULL)\r
- return NULL;\r
-\r
- codec = PyObject_GetAttrString((PyObject *)type, "codec");\r
- if (codec == NULL)\r
- goto errorexit;\r
- if (!MultibyteCodec_Check(codec)) {\r
- PyErr_SetString(PyExc_TypeError, "codec is unexpected type");\r
- goto errorexit;\r
- }\r
-\r
- self->codec = ((MultibyteCodecObject *)codec)->codec;\r
- self->stream = stream;\r
- Py_INCREF(stream);\r
- self->pendingsize = 0;\r
- self->errors = internal_error_callback(errors);\r
- if (self->errors == NULL)\r
- goto errorexit;\r
- if (self->codec->decinit != NULL &&\r
- self->codec->decinit(&self->state, self->codec->config) != 0)\r
- goto errorexit;\r
-\r
- Py_DECREF(codec);\r
- return (PyObject *)self;\r
-\r
-errorexit:\r
- Py_XDECREF(self);\r
- Py_XDECREF(codec);\r
- return NULL;\r
-}\r
-\r
-static int\r
-mbstreamreader_init(PyObject *self, PyObject *args, PyObject *kwds)\r
-{\r
- return 0;\r
-}\r
-\r
-static int\r
-mbstreamreader_traverse(MultibyteStreamReaderObject *self,\r
- visitproc visit, void *arg)\r
-{\r
- if (ERROR_ISCUSTOM(self->errors))\r
- Py_VISIT(self->errors);\r
- Py_VISIT(self->stream);\r
- return 0;\r
-}\r
-\r
-static void\r
-mbstreamreader_dealloc(MultibyteStreamReaderObject *self)\r
-{\r
- PyObject_GC_UnTrack(self);\r
- ERROR_DECREF(self->errors);\r
- Py_XDECREF(self->stream);\r
- Py_TYPE(self)->tp_free(self);\r
-}\r
-\r
-static PyTypeObject MultibyteStreamReader_Type = {\r
- PyVarObject_HEAD_INIT(NULL, 0)\r
- "MultibyteStreamReader", /* tp_name */\r
- sizeof(MultibyteStreamReaderObject), /* tp_basicsize */\r
- 0, /* tp_itemsize */\r
- /* methods */\r
- (destructor)mbstreamreader_dealloc, /* tp_dealloc */\r
- 0, /* tp_print */\r
- 0, /* tp_getattr */\r
- 0, /* tp_setattr */\r
- 0, /* tp_compare */\r
- 0, /* tp_repr */\r
- 0, /* tp_as_number */\r
- 0, /* tp_as_sequence */\r
- 0, /* tp_as_mapping */\r
- 0, /* tp_hash */\r
- 0, /* tp_call */\r
- 0, /* tp_str */\r
- PyObject_GenericGetAttr, /* tp_getattro */\r
- 0, /* tp_setattro */\r
- 0, /* tp_as_buffer */\r
- Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC\r
- | Py_TPFLAGS_BASETYPE, /* tp_flags */\r
- 0, /* tp_doc */\r
- (traverseproc)mbstreamreader_traverse, /* tp_traverse */\r
- 0, /* tp_clear */\r
- 0, /* tp_richcompare */\r
- 0, /* tp_weaklistoffset */\r
- 0, /* tp_iter */\r
- 0, /* tp_iterext */\r
- mbstreamreader_methods, /* tp_methods */\r
- mbstreamreader_members, /* tp_members */\r
- codecctx_getsets, /* tp_getset */\r
- 0, /* tp_base */\r
- 0, /* tp_dict */\r
- 0, /* tp_descr_get */\r
- 0, /* tp_descr_set */\r
- 0, /* tp_dictoffset */\r
- mbstreamreader_init, /* tp_init */\r
- 0, /* tp_alloc */\r
- mbstreamreader_new, /* tp_new */\r
-};\r
-\r
-\r
-/**\r
- * MultibyteStreamWriter object\r
- */\r
-\r
-static int\r
-mbstreamwriter_iwrite(MultibyteStreamWriterObject *self,\r
- PyObject *unistr)\r
-{\r
- PyObject *str, *wr;\r
-\r
- str = encoder_encode_stateful(STATEFUL_ECTX(self), unistr, 0);\r
- if (str == NULL)\r
- return -1;\r
-\r
- wr = PyObject_CallMethod(self->stream, "write", "O", str);\r
- Py_DECREF(str);\r
- if (wr == NULL)\r
- return -1;\r
-\r
- Py_DECREF(wr);\r
- return 0;\r
-}\r
-\r
-static PyObject *\r
-mbstreamwriter_write(MultibyteStreamWriterObject *self, PyObject *strobj)\r
-{\r
- if (mbstreamwriter_iwrite(self, strobj))\r
- return NULL;\r
- else\r
- Py_RETURN_NONE;\r
-}\r
-\r
-static PyObject *\r
-mbstreamwriter_writelines(MultibyteStreamWriterObject *self, PyObject *lines)\r
-{\r
- PyObject *strobj;\r
- int i, r;\r
-\r
- if (!PySequence_Check(lines)) {\r
- PyErr_SetString(PyExc_TypeError,\r
- "arg must be a sequence object");\r
- return NULL;\r
- }\r
-\r
- for (i = 0; i < PySequence_Length(lines); i++) {\r
- /* length can be changed even within this loop */\r
- strobj = PySequence_GetItem(lines, i);\r
- if (strobj == NULL)\r
- return NULL;\r
-\r
- r = mbstreamwriter_iwrite(self, strobj);\r
- Py_DECREF(strobj);\r
- if (r == -1)\r
- return NULL;\r
- }\r
-\r
- Py_RETURN_NONE;\r
-}\r
-\r
-static PyObject *\r
-mbstreamwriter_reset(MultibyteStreamWriterObject *self)\r
-{\r
- const Py_UNICODE *pending;\r
- PyObject *pwrt;\r
-\r
- pending = self->pending;\r
- pwrt = multibytecodec_encode(self->codec, &self->state,\r
- &pending, self->pendingsize, self->errors,\r
- MBENC_FLUSH | MBENC_RESET);\r
- /* some pending buffer can be truncated when UnicodeEncodeError is\r
- * raised on 'strict' mode. but, 'reset' method is designed to\r
- * reset the pending buffer or states so failed string sequence\r
- * ought to be missed */\r
- self->pendingsize = 0;\r
- if (pwrt == NULL)\r
- return NULL;\r
-\r
- if (PyString_Size(pwrt) > 0) {\r
- PyObject *wr;\r
- wr = PyObject_CallMethod(self->stream, "write", "O", pwrt);\r
- if (wr == NULL) {\r
- Py_DECREF(pwrt);\r
- return NULL;\r
- }\r
- }\r
- Py_DECREF(pwrt);\r
-\r
- Py_RETURN_NONE;\r
-}\r
-\r
-static PyObject *\r
-mbstreamwriter_new(PyTypeObject *type, PyObject *args, PyObject *kwds)\r
-{\r
- MultibyteStreamWriterObject *self;\r
- PyObject *stream, *codec = NULL;\r
- char *errors = NULL;\r
-\r
- if (!PyArg_ParseTupleAndKeywords(args, kwds, "O|s:StreamWriter",\r
- streamkwarglist, &stream, &errors))\r
- return NULL;\r
-\r
- self = (MultibyteStreamWriterObject *)type->tp_alloc(type, 0);\r
- if (self == NULL)\r
- return NULL;\r
-\r
- codec = PyObject_GetAttrString((PyObject *)type, "codec");\r
- if (codec == NULL)\r
- goto errorexit;\r
- if (!MultibyteCodec_Check(codec)) {\r
- PyErr_SetString(PyExc_TypeError, "codec is unexpected type");\r
- goto errorexit;\r
- }\r
-\r
- self->codec = ((MultibyteCodecObject *)codec)->codec;\r
- self->stream = stream;\r
- Py_INCREF(stream);\r
- self->pendingsize = 0;\r
- self->errors = internal_error_callback(errors);\r
- if (self->errors == NULL)\r
- goto errorexit;\r
- if (self->codec->encinit != NULL &&\r
- self->codec->encinit(&self->state, self->codec->config) != 0)\r
- goto errorexit;\r
-\r
- Py_DECREF(codec);\r
- return (PyObject *)self;\r
-\r
-errorexit:\r
- Py_XDECREF(self);\r
- Py_XDECREF(codec);\r
- return NULL;\r
-}\r
-\r
-static int\r
-mbstreamwriter_init(PyObject *self, PyObject *args, PyObject *kwds)\r
-{\r
- return 0;\r
-}\r
-\r
-static int\r
-mbstreamwriter_traverse(MultibyteStreamWriterObject *self,\r
- visitproc visit, void *arg)\r
-{\r
- if (ERROR_ISCUSTOM(self->errors))\r
- Py_VISIT(self->errors);\r
- Py_VISIT(self->stream);\r
- return 0;\r
-}\r
-\r
-static void\r
-mbstreamwriter_dealloc(MultibyteStreamWriterObject *self)\r
-{\r
- PyObject_GC_UnTrack(self);\r
- ERROR_DECREF(self->errors);\r
- Py_XDECREF(self->stream);\r
- Py_TYPE(self)->tp_free(self);\r
-}\r
-\r
-static struct PyMethodDef mbstreamwriter_methods[] = {\r
- {"write", (PyCFunction)mbstreamwriter_write,\r
- METH_O, NULL},\r
- {"writelines", (PyCFunction)mbstreamwriter_writelines,\r
- METH_O, NULL},\r
- {"reset", (PyCFunction)mbstreamwriter_reset,\r
- METH_NOARGS, NULL},\r
- {NULL, NULL},\r
-};\r
-\r
-static PyMemberDef mbstreamwriter_members[] = {\r
- {"stream", T_OBJECT,\r
- offsetof(MultibyteStreamWriterObject, stream),\r
- READONLY, NULL},\r
- {NULL,}\r
-};\r
-\r
-static PyTypeObject MultibyteStreamWriter_Type = {\r
- PyVarObject_HEAD_INIT(NULL, 0)\r
- "MultibyteStreamWriter", /* tp_name */\r
- sizeof(MultibyteStreamWriterObject), /* tp_basicsize */\r
- 0, /* tp_itemsize */\r
- /* methods */\r
- (destructor)mbstreamwriter_dealloc, /* tp_dealloc */\r
- 0, /* tp_print */\r
- 0, /* tp_getattr */\r
- 0, /* tp_setattr */\r
- 0, /* tp_compare */\r
- 0, /* tp_repr */\r
- 0, /* tp_as_number */\r
- 0, /* tp_as_sequence */\r
- 0, /* tp_as_mapping */\r
- 0, /* tp_hash */\r
- 0, /* tp_call */\r
- 0, /* tp_str */\r
- PyObject_GenericGetAttr, /* tp_getattro */\r
- 0, /* tp_setattro */\r
- 0, /* tp_as_buffer */\r
- Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC\r
- | Py_TPFLAGS_BASETYPE, /* tp_flags */\r
- 0, /* tp_doc */\r
- (traverseproc)mbstreamwriter_traverse, /* tp_traverse */\r
- 0, /* tp_clear */\r
- 0, /* tp_richcompare */\r
- 0, /* tp_weaklistoffset */\r
- 0, /* tp_iter */\r
- 0, /* tp_iterext */\r
- mbstreamwriter_methods, /* tp_methods */\r
- mbstreamwriter_members, /* tp_members */\r
- codecctx_getsets, /* tp_getset */\r
- 0, /* tp_base */\r
- 0, /* tp_dict */\r
- 0, /* tp_descr_get */\r
- 0, /* tp_descr_set */\r
- 0, /* tp_dictoffset */\r
- mbstreamwriter_init, /* tp_init */\r
- 0, /* tp_alloc */\r
- mbstreamwriter_new, /* tp_new */\r
-};\r
-\r
-\r
-/**\r
- * Exposed factory function\r
- */\r
-\r
-static PyObject *\r
-__create_codec(PyObject *ignore, PyObject *arg)\r
-{\r
- MultibyteCodecObject *self;\r
- MultibyteCodec *codec;\r
-\r
- if (!PyCapsule_IsValid(arg, PyMultibyteCodec_CAPSULE_NAME)) {\r
- PyErr_SetString(PyExc_ValueError, "argument type invalid");\r
- return NULL;\r
- }\r
-\r
- codec = PyCapsule_GetPointer(arg, PyMultibyteCodec_CAPSULE_NAME);\r
- if (codec->codecinit != NULL && codec->codecinit(codec->config) != 0)\r
- return NULL;\r
-\r
- self = PyObject_New(MultibyteCodecObject, &MultibyteCodec_Type);\r
- if (self == NULL)\r
- return NULL;\r
- self->codec = codec;\r
-\r
- return (PyObject *)self;\r
-}\r
-\r
-static struct PyMethodDef __methods[] = {\r
- {"__create_codec", (PyCFunction)__create_codec, METH_O},\r
- {NULL, NULL},\r
-};\r
-\r
-PyMODINIT_FUNC\r
-init_multibytecodec(void)\r
-{\r
- int i;\r
- PyObject *m;\r
- PyTypeObject *typelist[] = {\r
- &MultibyteIncrementalEncoder_Type,\r
- &MultibyteIncrementalDecoder_Type,\r
- &MultibyteStreamReader_Type,\r
- &MultibyteStreamWriter_Type,\r
- NULL\r
- };\r
-\r
- if (PyType_Ready(&MultibyteCodec_Type) < 0)\r
- return;\r
-\r
- m = Py_InitModule("_multibytecodec", __methods);\r
- if (m == NULL)\r
- return;\r
-\r
- for (i = 0; typelist[i] != NULL; i++) {\r
- if (PyType_Ready(typelist[i]) < 0)\r
- return;\r
- Py_INCREF(typelist[i]);\r
- PyModule_AddObject(m, typelist[i]->tp_name,\r
- (PyObject *)typelist[i]);\r
- }\r
-\r
- if (PyErr_Occurred())\r
- Py_FatalError("can't initialize the _multibytecodec module");\r
-}\r
projects / mirror_edk2.git / blobdiff