2 #include "structmember.h" /* for offsetof() */
9 Py_ssize_t string_size
;
12 PyObject
*weakreflist
;
15 #define CHECK_CLOSED(self) \
16 if ((self)->buf == NULL) { \
17 PyErr_SetString(PyExc_ValueError, \
18 "I/O operation on closed file."); \
22 /* Internal routine to get a line from the buffer of a BytesIO
23 object. Returns the length between the current position to the
24 next newline character. */
26 get_line(bytesio
*self
, char **output
)
32 assert(self
->buf
!= NULL
);
34 /* Move to the end of the line, up to the end of the string, s. */
35 str_end
= self
->buf
+ self
->string_size
;
36 for (n
= self
->buf
+ self
->pos
;
37 n
< str_end
&& *n
!= '\n';
40 /* Skip the newline character */
44 /* Get the length from the current position to the end of the line. */
45 len
= n
- (self
->buf
+ self
->pos
);
46 *output
= self
->buf
+ self
->pos
;
49 assert(self
->pos
< PY_SSIZE_T_MAX
- len
);
55 /* Internal routine for changing the size of the buffer of BytesIO objects.
56 The caller should ensure that the 'size' argument is non-negative. Returns
57 0 on success, -1 otherwise. */
59 resize_buffer(bytesio
*self
, size_t size
)
61 /* Here, unsigned types are used to avoid dealing with signed integer
62 overflow, which is undefined in C. */
63 size_t alloc
= self
->buf_size
;
66 assert(self
->buf
!= NULL
);
68 /* For simplicity, stay in the range of the signed type. Anyway, Python
69 doesn't allow strings to be longer than this. */
70 if (size
> PY_SSIZE_T_MAX
)
73 if (size
< alloc
/ 2) {
74 /* Major downsize; resize down to exact size. */
77 else if (size
< alloc
) {
78 /* Within allocated size; quick exit */
81 else if (size
<= alloc
* 1.125) {
82 /* Moderate upsize; overallocate similar to list_resize() */
83 alloc
= size
+ (size
>> 3) + (size
< 9 ? 3 : 6);
86 /* Major upsize; resize up to exact size */
90 if (alloc
> ((size_t)-1) / sizeof(char))
92 new_buf
= (char *)PyMem_Realloc(self
->buf
, alloc
* sizeof(char));
93 if (new_buf
== NULL
) {
97 self
->buf_size
= alloc
;
103 PyErr_SetString(PyExc_OverflowError
,
104 "new buffer size too large");
108 /* Internal routine for writing a string of bytes to the buffer of a BytesIO
109 object. Returns the number of bytes written, or -1 on error. */
111 write_bytes(bytesio
*self
, const char *bytes
, Py_ssize_t len
)
113 assert(self
->buf
!= NULL
);
114 assert(self
->pos
>= 0);
117 if ((size_t)self
->pos
+ len
> self
->buf_size
) {
118 if (resize_buffer(self
, (size_t)self
->pos
+ len
) < 0)
122 if (self
->pos
> self
->string_size
) {
123 /* In case of overseek, pad with null bytes the buffer region between
124 the end of stream and the current position.
127 | |<---used--->|<----------available----------->|
128 | | <--to pad-->|<---to write---> |
131 memset(self
->buf
+ self
->string_size
, '\0',
132 (self
->pos
- self
->string_size
) * sizeof(char));
135 /* Copy the data to the internal buffer, overwriting some of the existing
136 data if self->pos < self->string_size. */
137 memcpy(self
->buf
+ self
->pos
, bytes
, len
);
140 /* Set the new length of the internal string if it has changed. */
141 if (self
->string_size
< self
->pos
) {
142 self
->string_size
= self
->pos
;
149 bytesio_get_closed(bytesio
*self
)
151 if (self
->buf
== NULL
) {
159 PyDoc_STRVAR(readable_doc
,
160 "readable() -> bool. Returns True if the IO object can be read.");
162 PyDoc_STRVAR(writable_doc
,
163 "writable() -> bool. Returns True if the IO object can be written.");
165 PyDoc_STRVAR(seekable_doc
,
166 "seekable() -> bool. Returns True if the IO object can be seeked.");
168 /* Generic getter for the writable, readable and seekable properties */
170 return_not_closed(bytesio
*self
)
176 PyDoc_STRVAR(flush_doc
,
177 "flush() -> None. Does nothing.");
180 bytesio_flush(bytesio
*self
)
186 PyDoc_STRVAR(getval_doc
,
187 "getvalue() -> bytes.\n"
189 "Retrieve the entire contents of the BytesIO object.");
192 bytesio_getvalue(bytesio
*self
)
195 return PyBytes_FromStringAndSize(self
->buf
, self
->string_size
);
198 PyDoc_STRVAR(isatty_doc
,
199 "isatty() -> False.\n"
201 "Always returns False since BytesIO objects are not connected\n"
202 "to a tty-like device.");
205 bytesio_isatty(bytesio
*self
)
211 PyDoc_STRVAR(tell_doc
,
212 "tell() -> current file position, an integer\n");
215 bytesio_tell(bytesio
*self
)
218 return PyLong_FromSsize_t(self
->pos
);
221 PyDoc_STRVAR(read_doc
,
222 "read([size]) -> read at most size bytes, returned as a string.\n"
224 "If the size argument is negative, read until EOF is reached.\n"
225 "Return an empty string at EOF.");
228 bytesio_read(bytesio
*self
, PyObject
*args
)
232 PyObject
*arg
= Py_None
;
236 if (!PyArg_ParseTuple(args
, "|O:read", &arg
))
239 if (PyNumber_Check(arg
)) {
240 size
= PyNumber_AsSsize_t(arg
, PyExc_OverflowError
);
241 if (size
== -1 && PyErr_Occurred())
244 else if (arg
== Py_None
) {
245 /* Read until EOF is reached, by default. */
249 PyErr_Format(PyExc_TypeError
, "integer argument expected, got '%s'",
250 Py_TYPE(arg
)->tp_name
);
254 /* adjust invalid sizes */
255 n
= self
->string_size
- self
->pos
;
256 if (size
< 0 || size
> n
) {
262 assert(self
->buf
!= NULL
);
263 output
= self
->buf
+ self
->pos
;
266 return PyBytes_FromStringAndSize(output
, size
);
270 PyDoc_STRVAR(read1_doc
,
271 "read1(size) -> read at most size bytes, returned as a string.\n"
273 "If the size argument is negative or omitted, read until EOF is reached.\n"
274 "Return an empty string at EOF.");
277 bytesio_read1(bytesio
*self
, PyObject
*n
)
281 arg
= PyTuple_Pack(1, n
);
284 res
= bytesio_read(self
, arg
);
289 PyDoc_STRVAR(readline_doc
,
290 "readline([size]) -> next line from the file, as a string.\n"
292 "Retain newline. A non-negative size argument limits the maximum\n"
293 "number of bytes to return (an incomplete line may be returned then).\n"
294 "Return an empty string at EOF.\n");
297 bytesio_readline(bytesio
*self
, PyObject
*args
)
301 PyObject
*arg
= Py_None
;
305 if (!PyArg_ParseTuple(args
, "|O:readline", &arg
))
308 if (PyNumber_Check(arg
)) {
309 size
= PyNumber_AsSsize_t(arg
, PyExc_OverflowError
);
310 if (size
== -1 && PyErr_Occurred())
313 else if (arg
== Py_None
) {
314 /* No size limit, by default. */
318 PyErr_Format(PyExc_TypeError
, "integer argument expected, got '%s'",
319 Py_TYPE(arg
)->tp_name
);
323 n
= get_line(self
, &output
);
325 if (size
>= 0 && size
< n
) {
331 return PyBytes_FromStringAndSize(output
, n
);
334 PyDoc_STRVAR(readlines_doc
,
335 "readlines([size]) -> list of strings, each a line from the file.\n"
337 "Call readline() repeatedly and return a list of the lines so read.\n"
338 "The optional size argument, if given, is an approximate bound on the\n"
339 "total number of bytes in the lines returned.\n");
342 bytesio_readlines(bytesio
*self
, PyObject
*args
)
344 Py_ssize_t maxsize
, size
, n
;
345 PyObject
*result
, *line
;
347 PyObject
*arg
= Py_None
;
351 if (!PyArg_ParseTuple(args
, "|O:readlines", &arg
))
354 if (PyNumber_Check(arg
)) {
355 maxsize
= PyNumber_AsSsize_t(arg
, PyExc_OverflowError
);
356 if (maxsize
== -1 && PyErr_Occurred())
359 else if (arg
== Py_None
) {
360 /* No size limit, by default. */
364 PyErr_Format(PyExc_TypeError
, "integer argument expected, got '%s'",
365 Py_TYPE(arg
)->tp_name
);
370 result
= PyList_New(0);
374 while ((n
= get_line(self
, &output
)) != 0) {
375 line
= PyBytes_FromStringAndSize(output
, n
);
378 if (PyList_Append(result
, line
) == -1) {
384 if (maxsize
> 0 && size
>= maxsize
)
394 PyDoc_STRVAR(readinto_doc
,
395 "readinto(bytearray) -> int. Read up to len(b) bytes into b.\n"
397 "Returns number of bytes read (0 for EOF), or None if the object\n"
398 "is set not to block as has no data to read.");
401 bytesio_readinto(bytesio
*self
, PyObject
*args
)
408 if (!PyArg_ParseTuple(args
, "w*", &buf
))
412 /* adjust invalid sizes */
413 n
= self
->string_size
- self
->pos
;
420 memcpy(buf
.buf
, self
->buf
+ self
->pos
, len
);
421 assert(self
->pos
+ len
< PY_SSIZE_T_MAX
);
425 PyBuffer_Release(&buf
);
426 return PyLong_FromSsize_t(len
);
429 PyDoc_STRVAR(truncate_doc
,
430 "truncate([size]) -> int. Truncate the file to at most size bytes.\n"
432 "Size defaults to the current file position, as returned by tell().\n"
433 "The current file position is unchanged. Returns the new size.\n");
436 bytesio_truncate(bytesio
*self
, PyObject
*args
)
439 PyObject
*arg
= Py_None
;
443 if (!PyArg_ParseTuple(args
, "|O:truncate", &arg
))
446 if (PyNumber_Check(arg
)) {
447 size
= PyNumber_AsSsize_t(arg
, PyExc_OverflowError
);
448 if (size
== -1 && PyErr_Occurred())
451 else if (arg
== Py_None
) {
452 /* Truncate to current position if no argument is passed. */
456 PyErr_Format(PyExc_TypeError
, "integer argument expected, got '%s'",
457 Py_TYPE(arg
)->tp_name
);
462 PyErr_Format(PyExc_ValueError
,
463 "negative size value %zd", size
);
467 if (size
< self
->string_size
) {
468 self
->string_size
= size
;
469 if (resize_buffer(self
, size
) < 0)
473 return PyLong_FromSsize_t(size
);
477 bytesio_iternext(bytesio
*self
)
484 n
= get_line(self
, &next
);
489 return PyBytes_FromStringAndSize(next
, n
);
492 PyDoc_STRVAR(seek_doc
,
493 "seek(pos, whence=0) -> int. Change stream position.\n"
495 "Seek to byte offset pos relative to position indicated by whence:\n"
496 " 0 Start of stream (the default). pos should be >= 0;\n"
497 " 1 Current position - pos may be negative;\n"
498 " 2 End of stream - pos usually negative.\n"
499 "Returns the new absolute position.");
502 bytesio_seek(bytesio
*self
, PyObject
*args
)
510 if (!PyArg_ParseTuple(args
, "O|i:seek", &posobj
, &mode
))
513 pos
= PyNumber_AsSsize_t(posobj
, PyExc_OverflowError
);
514 if (pos
== -1 && PyErr_Occurred())
517 if (pos
< 0 && mode
== 0) {
518 PyErr_Format(PyExc_ValueError
,
519 "negative seek value %zd", pos
);
523 /* mode 0: offset relative to beginning of the string.
524 mode 1: offset relative to current position.
525 mode 2: offset relative the end of the string. */
527 if (pos
> PY_SSIZE_T_MAX
- self
->pos
) {
528 PyErr_SetString(PyExc_OverflowError
,
529 "new position too large");
534 else if (mode
== 2) {
535 if (pos
> PY_SSIZE_T_MAX
- self
->string_size
) {
536 PyErr_SetString(PyExc_OverflowError
,
537 "new position too large");
540 pos
+= self
->string_size
;
542 else if (mode
!= 0) {
543 PyErr_Format(PyExc_ValueError
,
544 "invalid whence (%i, should be 0, 1 or 2)", mode
);
552 return PyLong_FromSsize_t(self
->pos
);
555 PyDoc_STRVAR(write_doc
,
556 "write(bytes) -> int. Write bytes to file.\n"
558 "Return the number of bytes written.");
561 bytesio_write(bytesio
*self
, PyObject
*obj
)
565 PyObject
*result
= NULL
;
569 if (PyObject_GetBuffer(obj
, &buf
, PyBUF_CONTIG_RO
) < 0)
573 n
= write_bytes(self
, buf
.buf
, buf
.len
);
575 result
= PyLong_FromSsize_t(n
);
577 PyBuffer_Release(&buf
);
581 PyDoc_STRVAR(writelines_doc
,
582 "writelines(sequence_of_strings) -> None. Write strings to the file.\n"
584 "Note that newlines are not added. The sequence can be any iterable\n"
585 "object producing strings. This is equivalent to calling write() for\n"
589 bytesio_writelines(bytesio
*self
, PyObject
*v
)
596 it
= PyObject_GetIter(v
);
600 while ((item
= PyIter_Next(it
)) != NULL
) {
601 ret
= bytesio_write(self
, item
);
611 /* See if PyIter_Next failed */
612 if (PyErr_Occurred())
618 PyDoc_STRVAR(close_doc
,
619 "close() -> None. Disable all I/O operations.");
622 bytesio_close(bytesio
*self
)
624 if (self
->buf
!= NULL
) {
625 PyMem_Free(self
->buf
);
633 Note that only pickle protocol 2 and onward are supported since we use
634 extended __reduce__ API of PEP 307 to make BytesIO instances picklable.
636 Providing support for protocol < 2 would require the __reduce_ex__ method
637 which is notably long-winded when defined properly.
639 For BytesIO, the implementation would similar to one coded for
640 object.__reduce_ex__, but slightly less general. To be more specific, we
641 could call bytesio_getstate directly and avoid checking for the presence of
642 a fallback __reduce__ method. However, we would still need a __newobj__
643 function to use the efficient instance representation of PEP 307.
647 bytesio_getstate(bytesio
*self
)
649 PyObject
*initvalue
= bytesio_getvalue(self
);
653 if (initvalue
== NULL
)
655 if (self
->dict
== NULL
) {
660 dict
= PyDict_Copy(self
->dict
);
665 state
= Py_BuildValue("(OnN)", initvalue
, self
->pos
, dict
);
666 Py_DECREF(initvalue
);
671 bytesio_setstate(bytesio
*self
, PyObject
*state
)
674 PyObject
*position_obj
;
678 assert(state
!= NULL
);
680 /* We allow the state tuple to be longer than 3, because we may need
681 someday to extend the object's state without breaking
682 backward-compatibility. */
683 if (!PyTuple_Check(state
) || Py_SIZE(state
) < 3) {
684 PyErr_Format(PyExc_TypeError
,
685 "%.200s.__setstate__ argument should be 3-tuple, got %.200s",
686 Py_TYPE(self
)->tp_name
, Py_TYPE(state
)->tp_name
);
689 /* Reset the object to its default state. This is only needed to handle
690 the case of repeated calls to __setstate__. */
691 self
->string_size
= 0;
694 /* Set the value of the internal buffer. If state[0] does not support the
695 buffer protocol, bytesio_write will raise the appropriate TypeError. */
696 result
= bytesio_write(self
, PyTuple_GET_ITEM(state
, 0));
701 /* Set carefully the position value. Alternatively, we could use the seek
702 method instead of modifying self->pos directly to better protect the
703 object internal state against errneous (or malicious) inputs. */
704 position_obj
= PyTuple_GET_ITEM(state
, 1);
705 if (!PyIndex_Check(position_obj
)) {
706 PyErr_Format(PyExc_TypeError
,
707 "second item of state must be an integer, not %.200s",
708 Py_TYPE(position_obj
)->tp_name
);
711 pos
= PyNumber_AsSsize_t(position_obj
, PyExc_OverflowError
);
712 if (pos
== -1 && PyErr_Occurred())
715 PyErr_SetString(PyExc_ValueError
,
716 "position value cannot be negative");
721 /* Set the dictionary of the instance variables. */
722 dict
= PyTuple_GET_ITEM(state
, 2);
723 if (dict
!= Py_None
) {
724 if (!PyDict_Check(dict
)) {
725 PyErr_Format(PyExc_TypeError
,
726 "third item of state should be a dict, got a %.200s",
727 Py_TYPE(dict
)->tp_name
);
731 /* Alternatively, we could replace the internal dictionary
732 completely. However, it seems more practical to just update it. */
733 if (PyDict_Update(self
->dict
, dict
) < 0)
746 bytesio_dealloc(bytesio
*self
)
748 _PyObject_GC_UNTRACK(self
);
749 if (self
->buf
!= NULL
) {
750 PyMem_Free(self
->buf
);
753 Py_CLEAR(self
->dict
);
754 if (self
->weakreflist
!= NULL
)
755 PyObject_ClearWeakRefs((PyObject
*) self
);
756 Py_TYPE(self
)->tp_free(self
);
760 bytesio_new(PyTypeObject
*type
, PyObject
*args
, PyObject
*kwds
)
764 assert(type
!= NULL
&& type
->tp_alloc
!= NULL
);
765 self
= (bytesio
*)type
->tp_alloc(type
, 0);
769 /* tp_alloc initializes all the fields to zero. So we don't have to
770 initialize them here. */
772 self
->buf
= (char *)PyMem_Malloc(0);
773 if (self
->buf
== NULL
) {
775 return PyErr_NoMemory();
778 return (PyObject
*)self
;
782 bytesio_init(bytesio
*self
, PyObject
*args
, PyObject
*kwds
)
784 char *kwlist
[] = {"initial_bytes", NULL
};
785 PyObject
*initvalue
= NULL
;
787 if (!PyArg_ParseTupleAndKeywords(args
, kwds
, "|O:BytesIO", kwlist
,
791 /* In case, __init__ is called multiple times. */
792 self
->string_size
= 0;
795 if (initvalue
&& initvalue
!= Py_None
) {
797 res
= bytesio_write(self
, initvalue
);
808 bytesio_sizeof(bytesio
*self
, void *unused
)
812 res
= sizeof(bytesio
);
814 res
+= self
->buf_size
;
815 return PyLong_FromSsize_t(res
);
819 bytesio_traverse(bytesio
*self
, visitproc visit
, void *arg
)
821 Py_VISIT(self
->dict
);
826 bytesio_clear(bytesio
*self
)
828 Py_CLEAR(self
->dict
);
833 static PyGetSetDef bytesio_getsetlist
[] = {
834 {"closed", (getter
)bytesio_get_closed
, NULL
,
835 "True if the file is closed."},
836 {NULL
}, /* sentinel */
839 static struct PyMethodDef bytesio_methods
[] = {
840 {"readable", (PyCFunction
)return_not_closed
, METH_NOARGS
, readable_doc
},
841 {"seekable", (PyCFunction
)return_not_closed
, METH_NOARGS
, seekable_doc
},
842 {"writable", (PyCFunction
)return_not_closed
, METH_NOARGS
, writable_doc
},
843 {"close", (PyCFunction
)bytesio_close
, METH_NOARGS
, close_doc
},
844 {"flush", (PyCFunction
)bytesio_flush
, METH_NOARGS
, flush_doc
},
845 {"isatty", (PyCFunction
)bytesio_isatty
, METH_NOARGS
, isatty_doc
},
846 {"tell", (PyCFunction
)bytesio_tell
, METH_NOARGS
, tell_doc
},
847 {"write", (PyCFunction
)bytesio_write
, METH_O
, write_doc
},
848 {"writelines", (PyCFunction
)bytesio_writelines
, METH_O
, writelines_doc
},
849 {"read1", (PyCFunction
)bytesio_read1
, METH_O
, read1_doc
},
850 {"readinto", (PyCFunction
)bytesio_readinto
, METH_VARARGS
, readinto_doc
},
851 {"readline", (PyCFunction
)bytesio_readline
, METH_VARARGS
, readline_doc
},
852 {"readlines", (PyCFunction
)bytesio_readlines
, METH_VARARGS
, readlines_doc
},
853 {"read", (PyCFunction
)bytesio_read
, METH_VARARGS
, read_doc
},
854 {"getvalue", (PyCFunction
)bytesio_getvalue
, METH_NOARGS
, getval_doc
},
855 {"seek", (PyCFunction
)bytesio_seek
, METH_VARARGS
, seek_doc
},
856 {"truncate", (PyCFunction
)bytesio_truncate
, METH_VARARGS
, truncate_doc
},
857 {"__getstate__", (PyCFunction
)bytesio_getstate
, METH_NOARGS
, NULL
},
858 {"__setstate__", (PyCFunction
)bytesio_setstate
, METH_O
, NULL
},
859 {"__sizeof__", (PyCFunction
)bytesio_sizeof
, METH_NOARGS
, NULL
},
860 {NULL
, NULL
} /* sentinel */
863 PyDoc_STRVAR(bytesio_doc
,
864 "BytesIO([buffer]) -> object\n"
866 "Create a buffered I/O implementation using an in-memory bytes\n"
867 "buffer, ready for reading and writing.");
869 PyTypeObject PyBytesIO_Type
= {
870 PyVarObject_HEAD_INIT(NULL
, 0)
871 "_io.BytesIO", /*tp_name*/
872 sizeof(bytesio
), /*tp_basicsize*/
874 (destructor
)bytesio_dealloc
, /*tp_dealloc*/
881 0, /*tp_as_sequence*/
889 Py_TPFLAGS_DEFAULT
| Py_TPFLAGS_BASETYPE
|
890 Py_TPFLAGS_HAVE_GC
, /*tp_flags*/
891 bytesio_doc
, /*tp_doc*/
892 (traverseproc
)bytesio_traverse
, /*tp_traverse*/
893 (inquiry
)bytesio_clear
, /*tp_clear*/
894 0, /*tp_richcompare*/
895 offsetof(bytesio
, weakreflist
), /*tp_weaklistoffset*/
896 PyObject_SelfIter
, /*tp_iter*/
897 (iternextfunc
)bytesio_iternext
, /*tp_iternext*/
898 bytesio_methods
, /*tp_methods*/
900 bytesio_getsetlist
, /*tp_getset*/
905 offsetof(bytesio
, dict
), /*tp_dictoffset*/
906 (initproc
)bytesio_init
, /*tp_init*/
908 bytesio_new
, /*tp_new*/