2 #include "structmember.h" /* for offsetof() */
9 Py_ssize_t string_size
;
12 PyObject
*weakreflist
;
15 #define CHECK_CLOSED(self) \
16 if ((self)->buf == NULL) { \
17 PyErr_SetString(PyExc_ValueError, \
18 "I/O operation on closed file."); \
22 /* Internal routine to get a line from the buffer of a BytesIO
23 object. Returns the length between the current position to the
24 next newline character. */
26 get_line(bytesio
*self
, char **output
)
32 assert(self
->buf
!= NULL
);
34 /* Move to the end of the line, up to the end of the string, s. */
35 str_end
= self
->buf
+ self
->string_size
;
36 for (n
= self
->buf
+ self
->pos
;
37 n
< str_end
&& *n
!= '\n';
40 /* Skip the newline character */
44 /* Get the length from the current position to the end of the line. */
45 len
= n
- (self
->buf
+ self
->pos
);
46 *output
= self
->buf
+ self
->pos
;
49 assert(self
->pos
< PY_SSIZE_T_MAX
- len
);
55 /* Internal routine for changing the size of the buffer of BytesIO objects.
56 The caller should ensure that the 'size' argument is non-negative. Returns
57 0 on success, -1 otherwise. */
59 resize_buffer(bytesio
*self
, size_t size
)
61 /* Here, unsigned types are used to avoid dealing with signed integer
62 overflow, which is undefined in C. */
63 size_t alloc
= self
->buf_size
;
66 assert(self
->buf
!= NULL
);
68 /* For simplicity, stay in the range of the signed type. Anyway, Python
69 doesn't allow strings to be longer than this. */
70 if (size
> PY_SSIZE_T_MAX
)
73 if (size
< alloc
/ 2) {
74 /* Major downsize; resize down to exact size. */
77 else if (size
< alloc
) {
78 /* Within allocated size; quick exit */
81 else if (size
<= alloc
* 1.125) {
82 /* Moderate upsize; overallocate similar to list_resize() */
83 alloc
= size
+ (size
>> 3) + (size
< 9 ? 3 : 6);
86 /* Major upsize; resize up to exact size */
90 if (alloc
> ((size_t)-1) / sizeof(char))
92 new_buf
= (char *)PyMem_Realloc(self
->buf
, alloc
* sizeof(char));
93 if (new_buf
== NULL
) {
97 self
->buf_size
= alloc
;
103 PyErr_SetString(PyExc_OverflowError
,
104 "new buffer size too large");
108 /* Internal routine for writing a string of bytes to the buffer of a BytesIO
109 object. Returns the number of bytes wrote, or -1 on error. */
111 write_bytes(bytesio
*self
, const char *bytes
, Py_ssize_t len
)
113 assert(self
->buf
!= NULL
);
114 assert(self
->pos
>= 0);
117 if ((size_t)self
->pos
+ len
> self
->buf_size
) {
118 if (resize_buffer(self
, (size_t)self
->pos
+ len
) < 0)
122 if (self
->pos
> self
->string_size
) {
123 /* In case of overseek, pad with null bytes the buffer region between
124 the end of stream and the current position.
127 | |<---used--->|<----------available----------->|
128 | | <--to pad-->|<---to write---> |
131 memset(self
->buf
+ self
->string_size
, '\0',
132 (self
->pos
- self
->string_size
) * sizeof(char));
135 /* Copy the data to the internal buffer, overwriting some of the existing
136 data if self->pos < self->string_size. */
137 memcpy(self
->buf
+ self
->pos
, bytes
, len
);
140 /* Set the new length of the internal string if it has changed. */
141 if (self
->string_size
< self
->pos
) {
142 self
->string_size
= self
->pos
;
149 bytesio_get_closed(bytesio
*self
)
151 if (self
->buf
== NULL
) {
159 /* Generic getter for the writable, readable and seekable properties */
161 return_true(bytesio
*self
)
166 PyDoc_STRVAR(flush_doc
,
167 "flush() -> None. Does nothing.");
170 bytesio_flush(bytesio
*self
)
176 PyDoc_STRVAR(getval_doc
,
177 "getvalue() -> bytes.\n"
179 "Retrieve the entire contents of the BytesIO object.");
182 bytesio_getvalue(bytesio
*self
)
185 return PyBytes_FromStringAndSize(self
->buf
, self
->string_size
);
188 PyDoc_STRVAR(isatty_doc
,
189 "isatty() -> False.\n"
191 "Always returns False since BytesIO objects are not connected\n"
192 "to a tty-like device.");
195 bytesio_isatty(bytesio
*self
)
201 PyDoc_STRVAR(tell_doc
,
202 "tell() -> current file position, an integer\n");
205 bytesio_tell(bytesio
*self
)
208 return PyLong_FromSsize_t(self
->pos
);
211 PyDoc_STRVAR(read_doc
,
212 "read([size]) -> read at most size bytes, returned as a string.\n"
214 "If the size argument is negative, read until EOF is reached.\n"
215 "Return an empty string at EOF.");
218 bytesio_read(bytesio
*self
, PyObject
*args
)
222 PyObject
*arg
= Py_None
;
226 if (!PyArg_ParseTuple(args
, "|O:read", &arg
))
229 if (PyNumber_Check(arg
)) {
230 size
= PyNumber_AsSsize_t(arg
, PyExc_OverflowError
);
231 if (size
== -1 && PyErr_Occurred())
234 else if (arg
== Py_None
) {
235 /* Read until EOF is reached, by default. */
239 PyErr_Format(PyExc_TypeError
, "integer argument expected, got '%s'",
240 Py_TYPE(arg
)->tp_name
);
244 /* adjust invalid sizes */
245 n
= self
->string_size
- self
->pos
;
246 if (size
< 0 || size
> n
) {
252 assert(self
->buf
!= NULL
);
253 output
= self
->buf
+ self
->pos
;
256 return PyBytes_FromStringAndSize(output
, size
);
260 PyDoc_STRVAR(read1_doc
,
261 "read1(size) -> read at most size bytes, returned as a string.\n"
263 "If the size argument is negative or omitted, read until EOF is reached.\n"
264 "Return an empty string at EOF.");
267 bytesio_read1(bytesio
*self
, PyObject
*n
)
271 arg
= PyTuple_Pack(1, n
);
274 res
= bytesio_read(self
, arg
);
279 PyDoc_STRVAR(readline_doc
,
280 "readline([size]) -> next line from the file, as a string.\n"
282 "Retain newline. A non-negative size argument limits the maximum\n"
283 "number of bytes to return (an incomplete line may be returned then).\n"
284 "Return an empty string at EOF.\n");
287 bytesio_readline(bytesio
*self
, PyObject
*args
)
291 PyObject
*arg
= Py_None
;
295 if (!PyArg_ParseTuple(args
, "|O:readline", &arg
))
298 if (PyNumber_Check(arg
)) {
299 size
= PyNumber_AsSsize_t(arg
, PyExc_OverflowError
);
300 if (size
== -1 && PyErr_Occurred())
303 else if (arg
== Py_None
) {
304 /* No size limit, by default. */
308 PyErr_Format(PyExc_TypeError
, "integer argument expected, got '%s'",
309 Py_TYPE(arg
)->tp_name
);
313 n
= get_line(self
, &output
);
315 if (size
>= 0 && size
< n
) {
321 return PyBytes_FromStringAndSize(output
, n
);
324 PyDoc_STRVAR(readlines_doc
,
325 "readlines([size]) -> list of strings, each a line from the file.\n"
327 "Call readline() repeatedly and return a list of the lines so read.\n"
328 "The optional size argument, if given, is an approximate bound on the\n"
329 "total number of bytes in the lines returned.\n");
332 bytesio_readlines(bytesio
*self
, PyObject
*args
)
334 Py_ssize_t maxsize
, size
, n
;
335 PyObject
*result
, *line
;
337 PyObject
*arg
= Py_None
;
341 if (!PyArg_ParseTuple(args
, "|O:readlines", &arg
))
344 if (PyNumber_Check(arg
)) {
345 maxsize
= PyNumber_AsSsize_t(arg
, PyExc_OverflowError
);
346 if (maxsize
== -1 && PyErr_Occurred())
349 else if (arg
== Py_None
) {
350 /* No size limit, by default. */
354 PyErr_Format(PyExc_TypeError
, "integer argument expected, got '%s'",
355 Py_TYPE(arg
)->tp_name
);
360 result
= PyList_New(0);
364 while ((n
= get_line(self
, &output
)) != 0) {
365 line
= PyBytes_FromStringAndSize(output
, n
);
368 if (PyList_Append(result
, line
) == -1) {
374 if (maxsize
> 0 && size
>= maxsize
)
384 PyDoc_STRVAR(readinto_doc
,
385 "readinto(bytearray) -> int. Read up to len(b) bytes into b.\n"
387 "Returns number of bytes read (0 for EOF), or None if the object\n"
388 "is set not to block as has no data to read.");
391 bytesio_readinto(bytesio
*self
, PyObject
*args
)
398 if (!PyArg_ParseTuple(args
, "w*", &buf
))
402 /* adjust invalid sizes */
403 n
= self
->string_size
- self
->pos
;
410 memcpy(buf
.buf
, self
->buf
+ self
->pos
, len
);
411 assert(self
->pos
+ len
< PY_SSIZE_T_MAX
);
415 PyBuffer_Release(&buf
);
416 return PyLong_FromSsize_t(len
);
419 PyDoc_STRVAR(truncate_doc
,
420 "truncate([size]) -> int. Truncate the file to at most size bytes.\n"
422 "Size defaults to the current file position, as returned by tell().\n"
423 "The current file position is unchanged. Returns the new size.\n");
426 bytesio_truncate(bytesio
*self
, PyObject
*args
)
429 PyObject
*arg
= Py_None
;
433 if (!PyArg_ParseTuple(args
, "|O:truncate", &arg
))
436 if (PyNumber_Check(arg
)) {
437 size
= PyNumber_AsSsize_t(arg
, PyExc_OverflowError
);
438 if (size
== -1 && PyErr_Occurred())
441 else if (arg
== Py_None
) {
442 /* Truncate to current position if no argument is passed. */
446 PyErr_Format(PyExc_TypeError
, "integer argument expected, got '%s'",
447 Py_TYPE(arg
)->tp_name
);
452 PyErr_Format(PyExc_ValueError
,
453 "negative size value %zd", size
);
457 if (size
< self
->string_size
) {
458 self
->string_size
= size
;
459 if (resize_buffer(self
, size
) < 0)
463 return PyLong_FromSsize_t(size
);
467 bytesio_iternext(bytesio
*self
)
474 n
= get_line(self
, &next
);
479 return PyBytes_FromStringAndSize(next
, n
);
482 PyDoc_STRVAR(seek_doc
,
483 "seek(pos, whence=0) -> int. Change stream position.\n"
485 "Seek to byte offset pos relative to position indicated by whence:\n"
486 " 0 Start of stream (the default). pos should be >= 0;\n"
487 " 1 Current position - pos may be negative;\n"
488 " 2 End of stream - pos usually negative.\n"
489 "Returns the new absolute position.");
492 bytesio_seek(bytesio
*self
, PyObject
*args
)
500 if (!PyArg_ParseTuple(args
, "O|i:seek", &posobj
, &mode
))
503 pos
= PyNumber_AsSsize_t(posobj
, PyExc_OverflowError
);
504 if (pos
== -1 && PyErr_Occurred())
507 if (pos
< 0 && mode
== 0) {
508 PyErr_Format(PyExc_ValueError
,
509 "negative seek value %zd", pos
);
513 /* mode 0: offset relative to beginning of the string.
514 mode 1: offset relative to current position.
515 mode 2: offset relative the end of the string. */
517 if (pos
> PY_SSIZE_T_MAX
- self
->pos
) {
518 PyErr_SetString(PyExc_OverflowError
,
519 "new position too large");
524 else if (mode
== 2) {
525 if (pos
> PY_SSIZE_T_MAX
- self
->string_size
) {
526 PyErr_SetString(PyExc_OverflowError
,
527 "new position too large");
530 pos
+= self
->string_size
;
532 else if (mode
!= 0) {
533 PyErr_Format(PyExc_ValueError
,
534 "invalid whence (%i, should be 0, 1 or 2)", mode
);
542 return PyLong_FromSsize_t(self
->pos
);
545 PyDoc_STRVAR(write_doc
,
546 "write(bytes) -> int. Write bytes to file.\n"
548 "Return the number of bytes written.");
551 bytesio_write(bytesio
*self
, PyObject
*obj
)
555 PyObject
*result
= NULL
;
559 if (PyObject_GetBuffer(obj
, &buf
, PyBUF_CONTIG_RO
) < 0)
563 n
= write_bytes(self
, buf
.buf
, buf
.len
);
565 result
= PyLong_FromSsize_t(n
);
567 PyBuffer_Release(&buf
);
571 PyDoc_STRVAR(writelines_doc
,
572 "writelines(sequence_of_strings) -> None. Write strings to the file.\n"
574 "Note that newlines are not added. The sequence can be any iterable\n"
575 "object producing strings. This is equivalent to calling write() for\n"
579 bytesio_writelines(bytesio
*self
, PyObject
*v
)
586 it
= PyObject_GetIter(v
);
590 while ((item
= PyIter_Next(it
)) != NULL
) {
591 ret
= bytesio_write(self
, item
);
601 /* See if PyIter_Next failed */
602 if (PyErr_Occurred())
608 PyDoc_STRVAR(close_doc
,
609 "close() -> None. Disable all I/O operations.");
612 bytesio_close(bytesio
*self
)
614 if (self
->buf
!= NULL
) {
615 PyMem_Free(self
->buf
);
623 Note that only pickle protocol 2 and onward are supported since we use
624 extended __reduce__ API of PEP 307 to make BytesIO instances picklable.
626 Providing support for protocol < 2 would require the __reduce_ex__ method
627 which is notably long-winded when defined properly.
629 For BytesIO, the implementation would similar to one coded for
630 object.__reduce_ex__, but slightly less general. To be more specific, we
631 could call bytesio_getstate directly and avoid checking for the presence of
632 a fallback __reduce__ method. However, we would still need a __newobj__
633 function to use the efficient instance representation of PEP 307.
637 bytesio_getstate(bytesio
*self
)
639 PyObject
*initvalue
= bytesio_getvalue(self
);
643 if (initvalue
== NULL
)
645 if (self
->dict
== NULL
) {
650 dict
= PyDict_Copy(self
->dict
);
655 state
= Py_BuildValue("(OnN)", initvalue
, self
->pos
, dict
);
656 Py_DECREF(initvalue
);
661 bytesio_setstate(bytesio
*self
, PyObject
*state
)
664 PyObject
*position_obj
;
668 assert(state
!= NULL
);
670 /* We allow the state tuple to be longer than 3, because we may need
671 someday to extend the object's state without breaking
672 backward-compatibility. */
673 if (!PyTuple_Check(state
) || Py_SIZE(state
) < 3) {
674 PyErr_Format(PyExc_TypeError
,
675 "%.200s.__setstate__ argument should be 3-tuple, got %.200s",
676 Py_TYPE(self
)->tp_name
, Py_TYPE(state
)->tp_name
);
679 /* Reset the object to its default state. This is only needed to handle
680 the case of repeated calls to __setstate__. */
681 self
->string_size
= 0;
684 /* Set the value of the internal buffer. If state[0] does not support the
685 buffer protocol, bytesio_write will raise the appropriate TypeError. */
686 result
= bytesio_write(self
, PyTuple_GET_ITEM(state
, 0));
691 /* Set carefully the position value. Alternatively, we could use the seek
692 method instead of modifying self->pos directly to better protect the
693 object internal state against errneous (or malicious) inputs. */
694 position_obj
= PyTuple_GET_ITEM(state
, 1);
695 if (!PyIndex_Check(position_obj
)) {
696 PyErr_Format(PyExc_TypeError
,
697 "second item of state must be an integer, not %.200s",
698 Py_TYPE(position_obj
)->tp_name
);
701 pos
= PyNumber_AsSsize_t(position_obj
, PyExc_OverflowError
);
702 if (pos
== -1 && PyErr_Occurred())
705 PyErr_SetString(PyExc_ValueError
,
706 "position value cannot be negative");
711 /* Set the dictionary of the instance variables. */
712 dict
= PyTuple_GET_ITEM(state
, 2);
713 if (dict
!= Py_None
) {
714 if (!PyDict_Check(dict
)) {
715 PyErr_Format(PyExc_TypeError
,
716 "third item of state should be a dict, got a %.200s",
717 Py_TYPE(dict
)->tp_name
);
721 /* Alternatively, we could replace the internal dictionary
722 completely. However, it seems more practical to just update it. */
723 if (PyDict_Update(self
->dict
, dict
) < 0)
736 bytesio_dealloc(bytesio
*self
)
738 _PyObject_GC_UNTRACK(self
);
739 if (self
->buf
!= NULL
) {
740 PyMem_Free(self
->buf
);
743 Py_CLEAR(self
->dict
);
744 if (self
->weakreflist
!= NULL
)
745 PyObject_ClearWeakRefs((PyObject
*) self
);
746 Py_TYPE(self
)->tp_free(self
);
750 bytesio_new(PyTypeObject
*type
, PyObject
*args
, PyObject
*kwds
)
754 assert(type
!= NULL
&& type
->tp_alloc
!= NULL
);
755 self
= (bytesio
*)type
->tp_alloc(type
, 0);
759 /* tp_alloc initializes all the fields to zero. So we don't have to
760 initialize them here. */
762 self
->buf
= (char *)PyMem_Malloc(0);
763 if (self
->buf
== NULL
) {
765 return PyErr_NoMemory();
768 return (PyObject
*)self
;
772 bytesio_init(bytesio
*self
, PyObject
*args
, PyObject
*kwds
)
774 char *kwlist
[] = {"initial_bytes", NULL
};
775 PyObject
*initvalue
= NULL
;
777 if (!PyArg_ParseTupleAndKeywords(args
, kwds
, "|O:BytesIO", kwlist
,
781 /* In case, __init__ is called multiple times. */
782 self
->string_size
= 0;
785 if (initvalue
&& initvalue
!= Py_None
) {
787 res
= bytesio_write(self
, initvalue
);
798 bytesio_traverse(bytesio
*self
, visitproc visit
, void *arg
)
800 Py_VISIT(self
->dict
);
805 bytesio_clear(bytesio
*self
)
807 Py_CLEAR(self
->dict
);
812 static PyGetSetDef bytesio_getsetlist
[] = {
813 {"closed", (getter
)bytesio_get_closed
, NULL
,
814 "True if the file is closed."},
815 {NULL
}, /* sentinel */
818 static struct PyMethodDef bytesio_methods
[] = {
819 {"readable", (PyCFunction
)return_true
, METH_NOARGS
, NULL
},
820 {"seekable", (PyCFunction
)return_true
, METH_NOARGS
, NULL
},
821 {"writable", (PyCFunction
)return_true
, METH_NOARGS
, NULL
},
822 {"close", (PyCFunction
)bytesio_close
, METH_NOARGS
, close_doc
},
823 {"flush", (PyCFunction
)bytesio_flush
, METH_NOARGS
, flush_doc
},
824 {"isatty", (PyCFunction
)bytesio_isatty
, METH_NOARGS
, isatty_doc
},
825 {"tell", (PyCFunction
)bytesio_tell
, METH_NOARGS
, tell_doc
},
826 {"write", (PyCFunction
)bytesio_write
, METH_O
, write_doc
},
827 {"writelines", (PyCFunction
)bytesio_writelines
, METH_O
, writelines_doc
},
828 {"read1", (PyCFunction
)bytesio_read1
, METH_O
, read1_doc
},
829 {"readinto", (PyCFunction
)bytesio_readinto
, METH_VARARGS
, readinto_doc
},
830 {"readline", (PyCFunction
)bytesio_readline
, METH_VARARGS
, readline_doc
},
831 {"readlines", (PyCFunction
)bytesio_readlines
, METH_VARARGS
, readlines_doc
},
832 {"read", (PyCFunction
)bytesio_read
, METH_VARARGS
, read_doc
},
833 {"getvalue", (PyCFunction
)bytesio_getvalue
, METH_NOARGS
, getval_doc
},
834 {"seek", (PyCFunction
)bytesio_seek
, METH_VARARGS
, seek_doc
},
835 {"truncate", (PyCFunction
)bytesio_truncate
, METH_VARARGS
, truncate_doc
},
836 {"__getstate__", (PyCFunction
)bytesio_getstate
, METH_NOARGS
, NULL
},
837 {"__setstate__", (PyCFunction
)bytesio_setstate
, METH_O
, NULL
},
838 {NULL
, NULL
} /* sentinel */
841 PyDoc_STRVAR(bytesio_doc
,
842 "BytesIO([buffer]) -> object\n"
844 "Create a buffered I/O implementation using an in-memory bytes\n"
845 "buffer, ready for reading and writing.");
847 PyTypeObject PyBytesIO_Type
= {
848 PyVarObject_HEAD_INIT(NULL
, 0)
849 "_io.BytesIO", /*tp_name*/
850 sizeof(bytesio
), /*tp_basicsize*/
852 (destructor
)bytesio_dealloc
, /*tp_dealloc*/
859 0, /*tp_as_sequence*/
867 Py_TPFLAGS_DEFAULT
| Py_TPFLAGS_BASETYPE
|
868 Py_TPFLAGS_HAVE_GC
, /*tp_flags*/
869 bytesio_doc
, /*tp_doc*/
870 (traverseproc
)bytesio_traverse
, /*tp_traverse*/
871 (inquiry
)bytesio_clear
, /*tp_clear*/
872 0, /*tp_richcompare*/
873 offsetof(bytesio
, weakreflist
), /*tp_weaklistoffset*/
874 PyObject_SelfIter
, /*tp_iter*/
875 (iternextfunc
)bytesio_iternext
, /*tp_iternext*/
876 bytesio_methods
, /*tp_methods*/
878 bytesio_getsetlist
, /*tp_getset*/
883 offsetof(bytesio
, dict
), /*tp_dictoffset*/
884 (initproc
)bytesio_init
, /*tp_init*/
886 bytesio_new
, /*tp_new*/