1 /* Author: Daniel Stutzbach */
3 #define PY_SSIZE_T_CLEAN
5 #ifdef HAVE_SYS_TYPES_H
14 #include <stddef.h> /* For offsetof */
15 #include "_iomodule.h"
18 * Known likely problems:
20 * - Files larger then 2**32-1
21 * - Files with unicode filenames
22 * - Passing numbers greater than 2**32-1 when an integer is expected
23 * - Making it work on Windows and other oddball platforms
27 * - autoconfify header file inclusion
31 /* can simulate truncate with Win32 API functions; see file_truncate */
32 #define HAVE_FTRUNCATE
33 #define WIN32_LEAN_AND_MEAN
38 #define SMALLCHUNK (8*1024)
39 #elif (BUFSIZ >= (2 << 25))
40 #error "unreasonable BUFSIZ > 64MB defined"
42 #define SMALLCHUNK BUFSIZ
48 unsigned int readable
: 1;
49 unsigned int writable
: 1;
50 unsigned int appending
: 1;
51 signed int seekable
: 2; /* -1 means unknown */
52 unsigned int closefd
: 1;
53 PyObject
*weakreflist
;
57 PyTypeObject PyFileIO_Type
;
59 #define PyFileIO_Check(op) (PyObject_TypeCheck((op), &PyFileIO_Type))
62 _PyFileIO_closed(PyObject
*self
)
64 return ((fileio
*)self
)->fd
< 0;
68 portable_lseek(int fd
, PyObject
*posobj
, int whence
);
70 static PyObject
*portable_lseek(int fd
, PyObject
*posobj
, int whence
);
72 /* Returns 0 on success, -1 with exception set on failure. */
74 internal_close(fileio
*self
)
81 /* fd is accessible and someone else may have closed it */
82 if (_PyVerify_fd(fd
)) {
83 Py_BEGIN_ALLOW_THREADS
95 PyErr_SetFromErrno(PyExc_IOError
);
102 fileio_close(fileio
*self
)
105 res
= PyObject_CallMethod((PyObject
*)&PyRawIOBase_Type
,
107 if (!self
->closefd
) {
111 if (internal_close(self
) < 0)
117 fileio_new(PyTypeObject
*type
, PyObject
*args
, PyObject
*kwds
)
121 assert(type
!= NULL
&& type
->tp_alloc
!= NULL
);
123 self
= (fileio
*) type
->tp_alloc(type
, 0);
131 self
->weakreflist
= NULL
;
134 return (PyObject
*) self
;
137 /* On Unix, open will succeed for directories.
138 In Python, there should be no file objects referring to
139 directories, so we need a check. */
142 dircheck(fileio
* self
, PyObject
*nameobj
)
144 #if defined(HAVE_FSTAT) && defined(S_IFDIR) && defined(EISDIR)
148 if (fstat(self
->fd
, &buf
) == 0 && S_ISDIR(buf
.st_mode
)) {
150 PyErr_SetFromErrnoWithFilenameObject(PyExc_IOError
, nameobj
);
160 #if defined(HAVE_FSTAT)
162 if (!_PyVerify_fd(fd
) || (fstat(fd
, &buf
) < 0 && errno
== EBADF
)) {
164 char *msg
= strerror(EBADF
);
165 exc
= PyObject_CallFunction(PyExc_OSError
, "(is)",
167 PyErr_SetObject(PyExc_OSError
, exc
);
177 fileio_init(PyObject
*oself
, PyObject
*args
, PyObject
*kwds
)
179 fileio
*self
= (fileio
*) oself
;
180 static char *kwlist
[] = {"file", "mode", "closefd", NULL
};
181 const char *name
= NULL
;
182 PyObject
*nameobj
, *stringobj
= NULL
;
186 Py_UNICODE
*widename
= NULL
;
189 int rwa
= 0, plus
= 0;
195 assert(PyFileIO_Check(oself
));
198 /* Have to close the existing file first. */
199 if (internal_close(self
) < 0)
206 if (!PyArg_ParseTupleAndKeywords(args
, kwds
, "O|si:fileio",
207 kwlist
, &nameobj
, &mode
, &closefd
))
210 if (PyFloat_Check(nameobj
)) {
211 PyErr_SetString(PyExc_TypeError
,
212 "integer argument expected, got float");
216 fd
= _PyLong_AsInt(nameobj
);
218 if (!PyErr_Occurred()) {
219 PyErr_SetString(PyExc_ValueError
,
220 "negative file descriptor");
227 if (PyUnicode_Check(nameobj
))
228 widename
= PyUnicode_AS_UNICODE(nameobj
);
229 if (widename
== NULL
)
233 if (PyBytes_Check(nameobj
) || PyByteArray_Check(nameobj
)) {
235 if (PyObject_AsCharBuffer(nameobj
, &name
, &namelen
) < 0)
239 PyObject
*u
= PyUnicode_FromObject(nameobj
);
244 stringobj
= PyUnicode_AsEncodedString(
245 u
, Py_FileSystemDefaultEncoding
, NULL
);
247 if (stringobj
== NULL
)
249 if (!PyBytes_Check(stringobj
)) {
250 PyErr_SetString(PyExc_TypeError
,
251 "encoder failed to return bytes");
254 name
= PyBytes_AS_STRING(stringobj
);
264 PyErr_SetString(PyExc_ValueError
,
265 "Must have exactly one of read/write/append "
266 "mode and at most one plus");
277 flags
|= O_CREAT
| O_TRUNC
;
285 flags
|= O_APPEND
| O_CREAT
;
292 self
->readable
= self
->writable
= 1;
296 PyErr_Format(PyExc_ValueError
,
297 "invalid mode: %.200s", mode
);
305 if (self
->readable
&& self
->writable
)
307 else if (self
->readable
)
320 self
->closefd
= closefd
;
325 PyErr_SetString(PyExc_ValueError
,
326 "Cannot use closefd=False with file name");
330 Py_BEGIN_ALLOW_THREADS
333 if (widename
!= NULL
)
334 self
->fd
= _wopen(widename
, flags
, 0666);
337 self
->fd
= open(name
, flags
, 0666);
342 if (widename
!= NULL
)
343 PyErr_SetFromErrnoWithUnicodeFilename(PyExc_IOError
, widename
);
346 PyErr_SetFromErrnoWithFilename(PyExc_IOError
, name
);
350 if (dircheck(self
, nameobj
) < 0)
353 if (PyObject_SetAttrString((PyObject
*)self
, "name", nameobj
) < 0)
356 if (self
->appending
) {
357 /* For consistent behaviour, we explicitly seek to the
358 end of file (otherwise, it might be done only on the
360 PyObject
*pos
= portable_lseek(self
->fd
, NULL
, 2);
380 fileio_traverse(fileio
*self
, visitproc visit
, void *arg
)
382 Py_VISIT(self
->dict
);
387 fileio_clear(fileio
*self
)
389 Py_CLEAR(self
->dict
);
394 fileio_dealloc(fileio
*self
)
396 if (_PyIOBase_finalize((PyObject
*) self
) < 0)
398 _PyObject_GC_UNTRACK(self
);
399 if (self
->weakreflist
!= NULL
)
400 PyObject_ClearWeakRefs((PyObject
*) self
);
401 Py_CLEAR(self
->dict
);
402 Py_TYPE(self
)->tp_free((PyObject
*)self
);
408 PyErr_SetString(PyExc_ValueError
, "I/O operation on closed file");
413 err_mode(char *action
)
415 PyErr_Format(PyExc_ValueError
, "File not open for %s", action
);
420 fileio_fileno(fileio
*self
)
424 return PyInt_FromLong((long) self
->fd
);
428 fileio_readable(fileio
*self
)
432 return PyBool_FromLong((long) self
->readable
);
436 fileio_writable(fileio
*self
)
440 return PyBool_FromLong((long) self
->writable
);
444 fileio_seekable(fileio
*self
)
448 if (self
->seekable
< 0) {
449 PyObject
*pos
= portable_lseek(self
->fd
, NULL
, SEEK_CUR
);
458 return PyBool_FromLong((long) self
->seekable
);
462 fileio_readinto(fileio
*self
, PyObject
*args
)
470 return err_mode("reading");
472 if (!PyArg_ParseTuple(args
, "w*", &pbuf
))
475 if (_PyVerify_fd(self
->fd
)) {
477 Py_BEGIN_ALLOW_THREADS
479 #if defined(MS_WIN64) || defined(MS_WINDOWS)
482 n
= read(self
->fd
, pbuf
.buf
, (int)len
);
484 n
= read(self
->fd
, pbuf
.buf
, len
);
489 PyBuffer_Release(&pbuf
);
493 PyErr_SetFromErrno(PyExc_IOError
);
497 return PyLong_FromSsize_t(n
);
501 new_buffersize(fileio
*self
, size_t currentsize
)
506 if (fstat(self
->fd
, &st
) == 0) {
508 pos
= lseek(self
->fd
, 0L, SEEK_CUR
);
509 /* Files claiming a size smaller than SMALLCHUNK may
510 actually be streaming pseudo-files. In this case, we
511 apply the more aggressive algorithm below.
513 if (end
>= SMALLCHUNK
&& end
>= pos
&& pos
>= 0) {
514 /* Add 1 so if the file were to grow we'd notice. */
515 return currentsize
+ end
- pos
+ 1;
519 /* Expand the buffer by an amount proportional to the current size,
520 giving us amortized linear-time behavior. Use a less-than-double
521 growth factor to avoid excessive allocation. */
522 return currentsize
+ (currentsize
>> 3) + 6;
526 fileio_readall(fileio
*self
)
529 Py_ssize_t total
= 0;
534 if (!_PyVerify_fd(self
->fd
))
535 return PyErr_SetFromErrno(PyExc_IOError
);
537 result
= PyBytes_FromStringAndSize(NULL
, SMALLCHUNK
);
542 size_t newsize
= new_buffersize(self
, total
);
543 if (newsize
> PY_SSIZE_T_MAX
|| newsize
<= 0) {
544 PyErr_SetString(PyExc_OverflowError
,
545 "unbounded read returned more bytes "
546 "than a Python string can hold ");
551 if (PyBytes_GET_SIZE(result
) < (Py_ssize_t
)newsize
) {
552 if (_PyBytes_Resize(&result
, newsize
) < 0)
553 return NULL
; /* result has been freed */
555 Py_BEGIN_ALLOW_THREADS
558 #if defined(MS_WIN64) || defined(MS_WINDOWS)
562 PyBytes_AS_STRING(result
) + total
,
566 PyBytes_AS_STRING(result
) + total
,
573 if (errno
== EINTR
) {
574 if (PyErr_CheckSignals()) {
580 if (errno
== EAGAIN
) {
587 PyErr_SetFromErrno(PyExc_IOError
);
593 if (PyBytes_GET_SIZE(result
) > total
) {
594 if (_PyBytes_Resize(&result
, total
) < 0) {
595 /* This should never happen, but just in case */
603 fileio_read(fileio
*self
, PyObject
*args
)
607 Py_ssize_t size
= -1;
613 return err_mode("reading");
615 if (!PyArg_ParseTuple(args
, "|O&", &_PyIO_ConvertSsize_t
, &size
))
619 return fileio_readall(self
);
622 #if defined(MS_WIN64) || defined(MS_WINDOWS)
626 bytes
= PyBytes_FromStringAndSize(NULL
, size
);
629 ptr
= PyBytes_AS_STRING(bytes
);
631 if (_PyVerify_fd(self
->fd
)) {
632 Py_BEGIN_ALLOW_THREADS
634 #if defined(MS_WIN64) || defined(MS_WINDOWS)
635 n
= read(self
->fd
, ptr
, (int)size
);
637 n
= read(self
->fd
, ptr
, size
);
647 PyErr_SetFromErrno(PyExc_IOError
);
652 if (_PyBytes_Resize(&bytes
, n
) < 0)
656 return (PyObject
*) bytes
;
660 fileio_write(fileio
*self
, PyObject
*args
)
668 return err_mode("writing");
670 if (!PyArg_ParseTuple(args
, "s*", &pbuf
))
673 if (_PyVerify_fd(self
->fd
)) {
674 Py_BEGIN_ALLOW_THREADS
677 #if defined(MS_WIN64) || defined(MS_WINDOWS)
680 n
= write(self
->fd
, pbuf
.buf
, (int)len
);
682 n
= write(self
->fd
, pbuf
.buf
, len
);
688 PyBuffer_Release(&pbuf
);
693 PyErr_SetFromErrno(PyExc_IOError
);
697 return PyLong_FromSsize_t(n
);
700 /* XXX Windows support below is likely incomplete */
702 /* Cribbed from posix_lseek() */
704 portable_lseek(int fd
, PyObject
*posobj
, int whence
)
709 /* Turn 0, 1, 2 into SEEK_{SET,CUR,END} */
712 case 0: whence
= SEEK_SET
; break;
715 case 1: whence
= SEEK_CUR
; break;
718 case 2: whence
= SEEK_END
; break;
721 #endif /* SEEK_SET */
726 if(PyFloat_Check(posobj
)) {
727 PyErr_SetString(PyExc_TypeError
, "an integer is required");
730 #if defined(HAVE_LARGEFILE_SUPPORT)
731 pos
= PyLong_AsLongLong(posobj
);
733 pos
= PyLong_AsLong(posobj
);
735 if (PyErr_Occurred())
739 if (_PyVerify_fd(fd
)) {
740 Py_BEGIN_ALLOW_THREADS
741 #if defined(MS_WIN64) || defined(MS_WINDOWS)
742 res
= _lseeki64(fd
, pos
, whence
);
744 res
= lseek(fd
, pos
, whence
);
750 return PyErr_SetFromErrno(PyExc_IOError
);
752 #if defined(HAVE_LARGEFILE_SUPPORT)
753 return PyLong_FromLongLong(res
);
755 return PyLong_FromLong(res
);
760 fileio_seek(fileio
*self
, PyObject
*args
)
768 if (!PyArg_ParseTuple(args
, "O|i", &posobj
, &whence
))
771 return portable_lseek(self
->fd
, posobj
, whence
);
775 fileio_tell(fileio
*self
, PyObject
*args
)
780 return portable_lseek(self
->fd
, NULL
, 1);
783 #ifdef HAVE_FTRUNCATE
785 fileio_truncate(fileio
*self
, PyObject
*args
)
787 PyObject
*posobj
= NULL
; /* the new size wanted by the user */
798 return err_mode("writing");
800 if (!PyArg_ParseTuple(args
, "|O", &posobj
))
803 if (posobj
== Py_None
|| posobj
== NULL
) {
804 /* Get the current position. */
805 posobj
= portable_lseek(fd
, NULL
, 1);
814 /* MS _chsize doesn't work if newsize doesn't fit in 32 bits,
815 so don't even try using it. */
817 PyObject
*oldposobj
, *tempposobj
;
820 /* we save the file pointer position */
821 oldposobj
= portable_lseek(fd
, NULL
, 1);
822 if (oldposobj
== NULL
) {
827 /* we then move to the truncation position */
828 tempposobj
= portable_lseek(fd
, posobj
, 0);
829 if (tempposobj
== NULL
) {
830 Py_DECREF(oldposobj
);
834 Py_DECREF(tempposobj
);
836 /* Truncate. Note that this may grow the file! */
837 Py_BEGIN_ALLOW_THREADS
839 hFile
= (HANDLE
)_get_osfhandle(fd
);
840 ret
= hFile
== (HANDLE
)-1; /* testing for INVALID_HANDLE value */
842 ret
= SetEndOfFile(hFile
) == 0;
848 /* we restore the file pointer position in any case */
849 tempposobj
= portable_lseek(fd
, oldposobj
, 0);
850 Py_DECREF(oldposobj
);
851 if (tempposobj
== NULL
) {
855 Py_DECREF(tempposobj
);
859 #if defined(HAVE_LARGEFILE_SUPPORT)
860 pos
= PyLong_AsLongLong(posobj
);
862 pos
= PyLong_AsLong(posobj
);
864 if (PyErr_Occurred()){
869 Py_BEGIN_ALLOW_THREADS
871 ret
= ftruncate(fd
, pos
);
874 #endif /* !MS_WINDOWS */
878 PyErr_SetFromErrno(PyExc_IOError
);
884 #endif /* HAVE_FTRUNCATE */
887 mode_string(fileio
*self
)
889 if (self
->appending
) {
895 else if (self
->readable
) {
906 fileio_repr(fileio
*self
)
908 PyObject
*nameobj
, *res
;
911 return PyString_FromFormat("<_io.FileIO [closed]>");
913 nameobj
= PyObject_GetAttrString((PyObject
*) self
, "name");
914 if (nameobj
== NULL
) {
915 if (PyErr_ExceptionMatches(PyExc_AttributeError
))
919 res
= PyString_FromFormat("<_io.FileIO fd=%d mode='%s'>",
920 self
->fd
, mode_string(self
));
923 PyObject
*repr
= PyObject_Repr(nameobj
);
927 res
= PyString_FromFormat("<_io.FileIO name=%s mode='%s'>",
928 PyString_AS_STRING(repr
),
936 fileio_isatty(fileio
*self
)
942 Py_BEGIN_ALLOW_THREADS
943 res
= isatty(self
->fd
);
945 return PyBool_FromLong(res
);
949 PyDoc_STRVAR(fileio_doc
,
950 "file(name: str[, mode: str]) -> file IO object\n"
952 "Open a file. The mode can be 'r' (default), 'w' or 'a' for reading,\n"
953 "writing or appending. The file will be created if it doesn't exist\n"
954 "when opened for writing or appending; it will be truncated when\n"
955 "opened for writing. Add a '+' to the mode to allow simultaneous\n"
956 "reading and writing.");
958 PyDoc_STRVAR(read_doc
,
959 "read(size: int) -> bytes. read at most size bytes, returned as bytes.\n"
961 "Only makes one system call, so less data may be returned than requested\n"
962 "In non-blocking mode, returns None if no data is available.\n"
963 "On end-of-file, returns ''.");
965 PyDoc_STRVAR(readall_doc
,
966 "readall() -> bytes. read all data from the file, returned as bytes.\n"
968 "In non-blocking mode, returns as much as is immediately available,\n"
969 "or None if no data is available. On end-of-file, returns ''.");
971 PyDoc_STRVAR(write_doc
,
972 "write(b: bytes) -> int. Write bytes b to file, return number written.\n"
974 "Only makes one system call, so not all of the data may be written.\n"
975 "The number of bytes actually written is returned. In non-blocking mode,\n"
976 "returns None if the write would block."
979 PyDoc_STRVAR(fileno_doc
,
980 "fileno() -> int. Return the underlying file descriptor (an integer).");
982 PyDoc_STRVAR(seek_doc
,
983 "seek(offset: int[, whence: int]) -> int. Move to new file position\n"
984 "and return the file position.\n"
986 "Argument offset is a byte count. Optional argument whence defaults to\n"
987 "SEEK_SET or 0 (offset from start of file, offset should be >= 0); other values\n"
988 "are SEEK_CUR or 1 (move relative to current position, positive or negative),\n"
989 "and SEEK_END or 2 (move relative to end of file, usually negative, although\n"
990 "many platforms allow seeking beyond the end of a file).\n"
992 "Note that not all file objects are seekable.");
994 #ifdef HAVE_FTRUNCATE
995 PyDoc_STRVAR(truncate_doc
,
996 "truncate([size: int]) -> int. Truncate the file to at most size bytes and\n"
997 "return the truncated size.\n"
999 "Size defaults to the current file position, as returned by tell().\n"
1000 "The current file position is changed to the value of size.");
1003 PyDoc_STRVAR(tell_doc
,
1004 "tell() -> int. Current file position.\n"
1006 "Can raise OSError for non seekable files."
1009 PyDoc_STRVAR(readinto_doc
,
1010 "readinto() -> Same as RawIOBase.readinto().");
1012 PyDoc_STRVAR(close_doc
,
1013 "close() -> None. Close the file.\n"
1015 "A closed file cannot be used for further I/O operations. close() may be\n"
1016 "called more than once without error.");
1018 PyDoc_STRVAR(isatty_doc
,
1019 "isatty() -> bool. True if the file is connected to a TTY device.");
1021 PyDoc_STRVAR(seekable_doc
,
1022 "seekable() -> bool. True if file supports random-access.");
1024 PyDoc_STRVAR(readable_doc
,
1025 "readable() -> bool. True if file was opened in a read mode.");
1027 PyDoc_STRVAR(writable_doc
,
1028 "writable() -> bool. True if file was opened in a write mode.");
1030 static PyMethodDef fileio_methods
[] = {
1031 {"read", (PyCFunction
)fileio_read
, METH_VARARGS
, read_doc
},
1032 {"readall", (PyCFunction
)fileio_readall
, METH_NOARGS
, readall_doc
},
1033 {"readinto", (PyCFunction
)fileio_readinto
, METH_VARARGS
, readinto_doc
},
1034 {"write", (PyCFunction
)fileio_write
, METH_VARARGS
, write_doc
},
1035 {"seek", (PyCFunction
)fileio_seek
, METH_VARARGS
, seek_doc
},
1036 {"tell", (PyCFunction
)fileio_tell
, METH_VARARGS
, tell_doc
},
1037 #ifdef HAVE_FTRUNCATE
1038 {"truncate", (PyCFunction
)fileio_truncate
, METH_VARARGS
, truncate_doc
},
1040 {"close", (PyCFunction
)fileio_close
, METH_NOARGS
, close_doc
},
1041 {"seekable", (PyCFunction
)fileio_seekable
, METH_NOARGS
, seekable_doc
},
1042 {"readable", (PyCFunction
)fileio_readable
, METH_NOARGS
, readable_doc
},
1043 {"writable", (PyCFunction
)fileio_writable
, METH_NOARGS
, writable_doc
},
1044 {"fileno", (PyCFunction
)fileio_fileno
, METH_NOARGS
, fileno_doc
},
1045 {"isatty", (PyCFunction
)fileio_isatty
, METH_NOARGS
, isatty_doc
},
1046 {NULL
, NULL
} /* sentinel */
1049 /* 'closed' and 'mode' are attributes for backwards compatibility reasons. */
1052 get_closed(fileio
*self
, void *closure
)
1054 return PyBool_FromLong((long)(self
->fd
< 0));
1058 get_closefd(fileio
*self
, void *closure
)
1060 return PyBool_FromLong((long)(self
->closefd
));
1064 get_mode(fileio
*self
, void *closure
)
1066 return PyUnicode_FromString(mode_string(self
));
1069 static PyGetSetDef fileio_getsetlist
[] = {
1070 {"closed", (getter
)get_closed
, NULL
, "True if the file is closed"},
1071 {"closefd", (getter
)get_closefd
, NULL
,
1072 "True if the file descriptor will be closed by close()."},
1073 {"mode", (getter
)get_mode
, NULL
, "String giving the file mode"},
1077 PyTypeObject PyFileIO_Type
= {
1078 PyVarObject_HEAD_INIT(NULL
, 0)
1082 (destructor
)fileio_dealloc
, /* tp_dealloc */
1086 0, /* tp_reserved */
1087 (reprfunc
)fileio_repr
, /* tp_repr */
1088 0, /* tp_as_number */
1089 0, /* tp_as_sequence */
1090 0, /* tp_as_mapping */
1094 PyObject_GenericGetAttr
, /* tp_getattro */
1095 0, /* tp_setattro */
1096 0, /* tp_as_buffer */
1097 Py_TPFLAGS_DEFAULT
| Py_TPFLAGS_BASETYPE
1098 | Py_TPFLAGS_HAVE_GC
, /* tp_flags */
1099 fileio_doc
, /* tp_doc */
1100 (traverseproc
)fileio_traverse
, /* tp_traverse */
1101 (inquiry
)fileio_clear
, /* tp_clear */
1102 0, /* tp_richcompare */
1103 offsetof(fileio
, weakreflist
), /* tp_weaklistoffset */
1105 0, /* tp_iternext */
1106 fileio_methods
, /* tp_methods */
1108 fileio_getsetlist
, /* tp_getset */
1111 0, /* tp_descr_get */
1112 0, /* tp_descr_set */
1113 offsetof(fileio
, dict
), /* tp_dictoffset */
1114 fileio_init
, /* tp_init */
1115 PyType_GenericAlloc
, /* tp_alloc */
1116 fileio_new
, /* tp_new */
1117 PyObject_GC_Del
, /* tp_free */