1 /* File object implementation */
3 #define PY_SSIZE_T_CLEAN
5 #include "structmember.h"
7 #ifdef HAVE_SYS_TYPES_H
9 #endif /* HAVE_SYS_TYPES_H */
12 #define fileno _fileno
13 /* can simulate truncate with Win32 API functions; see file_truncate */
14 #define HAVE_FTRUNCATE
15 #define WIN32_LEAN_AND_MEAN
19 #if defined(PYOS_OS2) && defined(PYCC_GCC)
23 #define BUF(v) PyString_AS_STRING((PyStringObject *)v)
29 #ifdef HAVE_GETC_UNLOCKED
30 #define GETC(f) getc_unlocked(f)
31 #define FLOCKFILE(f) flockfile(f)
32 #define FUNLOCKFILE(f) funlockfile(f)
34 #define GETC(f) getc(f)
36 #define FUNLOCKFILE(f)
39 /* Bits in f_newlinetypes */
40 #define NEWLINE_UNKNOWN 0 /* No newline seen, yet */
41 #define NEWLINE_CR 1 /* \r newline seen */
42 #define NEWLINE_LF 2 /* \n newline seen */
43 #define NEWLINE_CRLF 4 /* \r\n newline seen */
46 * These macros release the GIL while preventing the f_close() function being
47 * called in the interval between them. For that purpose, a running total of
48 * the number of currently running unlocked code sections is kept in
49 * the unlocked_count field of the PyFileObject. The close() method raises
50 * an IOError if that field is non-zero. See issue #815646, #595601.
53 #define FILE_BEGIN_ALLOW_THREADS(fobj) \
55 fobj->unlocked_count++; \
56 Py_BEGIN_ALLOW_THREADS
58 #define FILE_END_ALLOW_THREADS(fobj) \
59 Py_END_ALLOW_THREADS \
60 fobj->unlocked_count--; \
61 assert(fobj->unlocked_count >= 0); \
64 #define FILE_ABORT_ALLOW_THREADS(fobj) \
66 fobj->unlocked_count--; \
67 assert(fobj->unlocked_count >= 0);
74 PyFile_AsFile(PyObject
*f
)
76 if (f
== NULL
|| !PyFile_Check(f
))
79 return ((PyFileObject
*)f
)->f_fp
;
82 void PyFile_IncUseCount(PyFileObject
*fobj
)
84 fobj
->unlocked_count
++;
87 void PyFile_DecUseCount(PyFileObject
*fobj
)
89 fobj
->unlocked_count
--;
90 assert(fobj
->unlocked_count
>= 0);
94 PyFile_Name(PyObject
*f
)
96 if (f
== NULL
|| !PyFile_Check(f
))
99 return ((PyFileObject
*)f
)->f_name
;
102 /* This is a safe wrapper around PyObject_Print to print to the FILE
103 of a PyFileObject. PyObject_Print releases the GIL but knows nothing
104 about PyFileObject. */
106 file_PyObject_Print(PyObject
*op
, PyFileObject
*f
, int flags
)
109 PyFile_IncUseCount(f
);
110 result
= PyObject_Print(op
, f
->f_fp
, flags
);
111 PyFile_DecUseCount(f
);
115 /* On Unix, fopen will succeed for directories.
116 In Python, there should be no file objects referring to
117 directories, so we need a check. */
120 dircheck(PyFileObject
* f
)
122 #if defined(HAVE_FSTAT) && defined(S_IFDIR) && defined(EISDIR)
126 if (fstat(fileno(f
->f_fp
), &buf
) == 0 &&
127 S_ISDIR(buf
.st_mode
)) {
128 char *msg
= strerror(EISDIR
);
129 PyObject
*exc
= PyObject_CallFunction(PyExc_IOError
, "(isO)",
130 EISDIR
, msg
, f
->f_name
);
131 PyErr_SetObject(PyExc_IOError
, exc
);
141 fill_file_fields(PyFileObject
*f
, FILE *fp
, PyObject
*name
, char *mode
,
142 int (*close
)(FILE *))
144 assert(name
!= NULL
);
146 assert(PyFile_Check(f
));
147 assert(f
->f_fp
== NULL
);
149 Py_DECREF(f
->f_name
);
150 Py_DECREF(f
->f_mode
);
151 Py_DECREF(f
->f_encoding
);
152 Py_DECREF(f
->f_errors
);
157 f
->f_mode
= PyString_FromString(mode
);
161 f
->f_binary
= strchr(mode
,'b') != NULL
;
163 f
->f_univ_newline
= (strchr(mode
, 'U') != NULL
);
164 f
->f_newlinetypes
= NEWLINE_UNKNOWN
;
167 f
->f_encoding
= Py_None
;
169 f
->f_errors
= Py_None
;
170 f
->readable
= f
->writable
= 0;
171 if (strchr(mode
, 'r') != NULL
|| f
->f_univ_newline
)
173 if (strchr(mode
, 'w') != NULL
|| strchr(mode
, 'a') != NULL
)
175 if (strchr(mode
, '+') != NULL
)
176 f
->readable
= f
->writable
= 1;
178 if (f
->f_mode
== NULL
)
182 return (PyObject
*) f
;
185 #if defined _MSC_VER && _MSC_VER >= 1400 && defined(__STDC_SECURE_LIB__)
186 #define Py_VERIFY_WINNT
187 /* The CRT on windows compiled with Visual Studio 2005 and higher may
188 * assert if given invalid mode strings. This is all fine and well
189 * in static languages like C where the mode string is typcially hard
190 * coded. But in Python, were we pass in the mode string from the user,
191 * we need to verify it first manually
193 static int _PyVerify_Mode_WINNT(const char *mode
)
195 /* See if mode string is valid on Windows to avoid hard assertions */
196 /* remove leading spacese */
202 while(*mode
== ' ') /* strip initial spaces */
204 if (!strchr("rwa", *mode
)) /* must start with one of these */
207 if (*mode
== ' ' || *mode
== 'N') /* ignore spaces and N */
209 s
= "+TD"; /* each of this can appear only once */
210 c
= strchr(s
, *mode
);
213 if (singles
& (1<<idx
))
218 s
= "btcnSR"; /* only one of each letter in the pairs allowed */
219 c
= strchr(s
, *mode
);
221 ptrdiff_t idx
= (s
-c
)/2;
222 if (pairs
& (1<<idx
))
231 return 0; /* found an invalid char */
235 char *e
[] = {"UTF-8", "UTF-16LE", "UNICODE"};
239 if (strncmp(mode
, "ccs", 3))
248 for(encoding
= 0; encoding
<_countof(e
); ++encoding
) {
249 size_t l
= strlen(e
[encoding
]);
250 if (!strncmp(mode
, e
[encoding
], l
)) {
251 mode
+= l
; /* found a valid encoding */
255 if (encoding
== _countof(e
))
258 /* skip trailing spaces */
262 return *mode
== '\0'; /* must be at the end of the string */
266 /* check for known incorrect mode strings - problem is, platforms are
267 free to accept any mode characters they like and are supposed to
268 ignore stuff they don't understand... write or append mode with
269 universal newline support is expressly forbidden by PEP 278.
270 Additionally, remove the 'U' from the mode string as platforms
271 won't know what it is. Non-zero return signals an exception */
273 _PyFile_SanitizeMode(char *mode
)
276 size_t len
= strlen(mode
);
279 PyErr_SetString(PyExc_ValueError
, "empty mode string");
283 upos
= strchr(mode
, 'U');
285 memmove(upos
, upos
+1, len
-(upos
-mode
)); /* incl null char */
287 if (mode
[0] == 'w' || mode
[0] == 'a') {
288 PyErr_Format(PyExc_ValueError
, "universal newline "
289 "mode can only be used with modes "
290 "starting with 'r'");
294 if (mode
[0] != 'r') {
295 memmove(mode
+1, mode
, strlen(mode
)+1);
299 if (!strchr(mode
, 'b')) {
300 memmove(mode
+2, mode
+1, strlen(mode
));
303 } else if (mode
[0] != 'r' && mode
[0] != 'w' && mode
[0] != 'a') {
304 PyErr_Format(PyExc_ValueError
, "mode string must begin with "
305 "one of 'r', 'w', 'a' or 'U', not '%.200s'", mode
);
308 #ifdef Py_VERIFY_WINNT
309 /* additional checks on NT with visual studio 2005 and higher */
310 if (!_PyVerify_Mode_WINNT(mode
)) {
311 PyErr_Format(PyExc_ValueError
, "Invalid mode ('%.50s')", mode
);
319 open_the_file(PyFileObject
*f
, char *name
, char *mode
)
323 assert(PyFile_Check(f
));
325 /* windows ignores the passed name in order to support Unicode */
326 assert(f
->f_name
!= NULL
);
328 assert(name
!= NULL
);
330 assert(mode
!= NULL
);
331 assert(f
->f_fp
== NULL
);
333 /* probably need to replace 'U' by 'rb' */
334 newmode
= PyMem_MALLOC(strlen(mode
) + 3);
339 strcpy(newmode
, mode
);
341 if (_PyFile_SanitizeMode(newmode
)) {
346 /* rexec.py can't stop a user from getting the file() constructor --
347 all they have to do is get *any* file object f, and then do
348 type(f). Here we prevent them from doing damage with it. */
349 if (PyEval_GetRestricted()) {
350 PyErr_SetString(PyExc_IOError
,
351 "file() constructor not accessible in restricted mode");
358 if (PyUnicode_Check(f
->f_name
)) {
360 wmode
= PyUnicode_DecodeASCII(newmode
, strlen(newmode
), NULL
);
361 if (f
->f_name
&& wmode
) {
362 FILE_BEGIN_ALLOW_THREADS(f
)
363 /* PyUnicode_AS_UNICODE OK without thread
364 lock as it is a simple dereference. */
365 f
->f_fp
= _wfopen(PyUnicode_AS_UNICODE(f
->f_name
),
366 PyUnicode_AS_UNICODE(wmode
));
367 FILE_END_ALLOW_THREADS(f
)
372 if (NULL
== f
->f_fp
&& NULL
!= name
) {
373 FILE_BEGIN_ALLOW_THREADS(f
)
374 f
->f_fp
= fopen(name
, newmode
);
375 FILE_END_ALLOW_THREADS(f
)
378 if (f
->f_fp
== NULL
) {
379 #if defined _MSC_VER && (_MSC_VER < 1400 || !defined(__STDC_SECURE_LIB__))
380 /* MSVC 6 (Microsoft) leaves errno at 0 for bad mode strings,
381 * across all Windows flavors. When it sets EINVAL varies
382 * across Windows flavors, the exact conditions aren't
383 * documented, and the answer lies in the OS's implementation
384 * of Win32's CreateFile function (whose source is secret).
385 * Seems the best we can do is map EINVAL to ENOENT.
386 * Starting with Visual Studio .NET 2005, EINVAL is correctly
387 * set by our CRT error handler (set in exceptions.c.)
389 if (errno
== 0) /* bad mode string */
391 else if (errno
== EINVAL
) /* unknown, but not a mode string */
394 /* EINVAL is returned when an invalid filename or
395 * an invalid mode is supplied. */
396 if (errno
== EINVAL
) {
399 PyOS_snprintf(message
, 100,
400 "invalid mode ('%.50s') or filename", mode
);
401 v
= Py_BuildValue("(isO)", errno
, message
, f
->f_name
);
403 PyErr_SetObject(PyExc_IOError
, v
);
408 PyErr_SetFromErrnoWithFilenameObject(PyExc_IOError
, f
->f_name
);
417 return (PyObject
*)f
;
421 close_the_file(PyFileObject
*f
)
424 int (*local_close
)(FILE *);
425 FILE *local_fp
= f
->f_fp
;
426 char *local_setbuf
= f
->f_setbuf
;
427 if (local_fp
!= NULL
) {
428 local_close
= f
->f_close
;
429 if (local_close
!= NULL
&& f
->unlocked_count
> 0) {
430 if (f
->ob_refcnt
> 0) {
431 PyErr_SetString(PyExc_IOError
,
432 "close() called during concurrent "
433 "operation on the same file object.");
435 /* This should not happen unless someone is
436 * carelessly playing with the PyFileObject
437 * struct fields and/or its associated FILE
439 PyErr_SetString(PyExc_SystemError
,
440 "PyFileObject locking error in "
441 "destructor (refcnt <= 0 at close).");
445 /* NULL out the FILE pointer before releasing the GIL, because
446 * it will not be valid anymore after the close() function is
449 if (local_close
!= NULL
) {
450 /* Issue #9295: must temporarily reset f_setbuf so that another
451 thread doesn't free it when running file_close() concurrently.
452 Otherwise this close() will crash when flushing the buffer. */
454 Py_BEGIN_ALLOW_THREADS
456 sts
= (*local_close
)(local_fp
);
458 f
->f_setbuf
= local_setbuf
;
460 return PyErr_SetFromErrno(PyExc_IOError
);
462 return PyInt_FromLong((long)sts
);
469 PyFile_FromFile(FILE *fp
, char *name
, char *mode
, int (*close
)(FILE *))
474 f
= (PyFileObject
*)PyFile_Type
.tp_new(&PyFile_Type
, NULL
, NULL
);
477 o_name
= PyString_FromString(name
);
478 if (o_name
== NULL
) {
479 if (close
!= NULL
&& fp
!= NULL
)
484 if (fill_file_fields(f
, fp
, o_name
, mode
, close
) == NULL
) {
490 return (PyObject
*)f
;
494 PyFile_FromString(char *name
, char *mode
)
496 extern int fclose(FILE *);
499 f
= (PyFileObject
*)PyFile_FromFile((FILE *)NULL
, name
, mode
, fclose
);
501 if (open_the_file(f
, name
, mode
) == NULL
) {
506 return (PyObject
*)f
;
510 PyFile_SetBufSize(PyObject
*f
, int bufsize
)
512 PyFileObject
*file
= (PyFileObject
*)f
;
533 if (type
== _IONBF
) {
534 PyMem_Free(file
->f_setbuf
);
535 file
->f_setbuf
= NULL
;
537 file
->f_setbuf
= (char *)PyMem_Realloc(file
->f_setbuf
,
541 setvbuf(file
->f_fp
, file
->f_setbuf
, type
, bufsize
);
542 #else /* !HAVE_SETVBUF */
543 setbuf(file
->f_fp
, file
->f_setbuf
);
544 #endif /* !HAVE_SETVBUF */
548 /* Set the encoding used to output Unicode strings.
549 Return 1 on success, 0 on failure. */
552 PyFile_SetEncoding(PyObject
*f
, const char *enc
)
554 return PyFile_SetEncodingAndErrors(f
, enc
, NULL
);
558 PyFile_SetEncodingAndErrors(PyObject
*f
, const char *enc
, char* errors
)
560 PyFileObject
*file
= (PyFileObject
*)f
;
561 PyObject
*str
, *oerrors
;
563 assert(PyFile_Check(f
));
564 str
= PyString_FromString(enc
);
568 oerrors
= PyString_FromString(errors
);
577 Py_DECREF(file
->f_encoding
);
578 file
->f_encoding
= str
;
579 Py_DECREF(file
->f_errors
);
580 file
->f_errors
= oerrors
;
587 PyErr_SetString(PyExc_ValueError
, "I/O operation on closed file");
592 err_mode(char *action
)
594 PyErr_Format(PyExc_IOError
, "File not open for %s", action
);
598 /* Refuse regular file I/O if there's data in the iteration-buffer.
599 * Mixing them would cause data to arrive out of order, as the read*
600 * methods don't use the iteration buffer. */
602 err_iterbuffered(void)
604 PyErr_SetString(PyExc_ValueError
,
605 "Mixing iteration and read methods would lose data");
609 static void drop_readahead(PyFileObject
*);
614 file_dealloc(PyFileObject
*f
)
617 if (f
->weakreflist
!= NULL
)
618 PyObject_ClearWeakRefs((PyObject
*) f
);
619 ret
= close_the_file(f
);
621 PySys_WriteStderr("close failed in file object destructor:\n");
627 PyMem_Free(f
->f_setbuf
);
628 Py_XDECREF(f
->f_name
);
629 Py_XDECREF(f
->f_mode
);
630 Py_XDECREF(f
->f_encoding
);
631 Py_XDECREF(f
->f_errors
);
633 Py_TYPE(f
)->tp_free((PyObject
*)f
);
637 file_repr(PyFileObject
*f
)
639 PyObject
*ret
= NULL
;
640 PyObject
*name
= NULL
;
641 if (PyUnicode_Check(f
->f_name
)) {
642 #ifdef Py_USING_UNICODE
643 const char *name_str
;
644 name
= PyUnicode_AsUnicodeEscapeString(f
->f_name
);
645 name_str
= name
? PyString_AsString(name
) : "?";
646 ret
= PyString_FromFormat("<%s file u'%s', mode '%s' at %p>",
647 f
->f_fp
== NULL
? "closed" : "open",
649 PyString_AsString(f
->f_mode
),
655 name
= PyObject_Repr(f
->f_name
);
658 ret
= PyString_FromFormat("<%s file %s, mode '%s' at %p>",
659 f
->f_fp
== NULL
? "closed" : "open",
660 PyString_AsString(name
),
661 PyString_AsString(f
->f_mode
),
669 file_close(PyFileObject
*f
)
671 PyObject
*sts
= close_the_file(f
);
673 PyMem_Free(f
->f_setbuf
);
680 /* Our very own off_t-like type, 64-bit if possible */
681 #if !defined(HAVE_LARGEFILE_SUPPORT)
682 typedef off_t Py_off_t
;
683 #elif SIZEOF_OFF_T >= 8
684 typedef off_t Py_off_t
;
685 #elif SIZEOF_FPOS_T >= 8
686 typedef fpos_t Py_off_t
;
688 #error "Large file support, but neither off_t nor fpos_t is large enough."
692 /* a portable fseek() function
693 return 0 on success, non-zero on failure (with errno set) */
695 _portable_fseek(FILE *fp
, Py_off_t offset
, int whence
)
697 #if !defined(HAVE_LARGEFILE_SUPPORT)
698 return fseek(fp
, offset
, whence
);
699 #elif defined(HAVE_FSEEKO) && SIZEOF_OFF_T >= 8
700 return fseeko(fp
, offset
, whence
);
701 #elif defined(HAVE_FSEEK64)
702 return fseek64(fp
, offset
, whence
);
703 #elif defined(__BEOS__)
704 return _fseek(fp
, offset
, whence
);
705 #elif SIZEOF_FPOS_T >= 8
706 /* lacking a 64-bit capable fseek(), use a 64-bit capable fsetpos()
707 and fgetpos() to implement fseek()*/
713 if (_lseeki64(fileno(fp
), 0, 2) == -1)
716 if (fseek(fp
, 0, SEEK_END
) != 0)
721 if (fgetpos(fp
, &pos
) != 0)
725 /* case SEEK_SET: break; */
727 return fsetpos(fp
, &offset
);
729 #error "Large file support, but no way to fseek."
734 /* a portable ftell() function
735 Return -1 on failure with errno set appropriately, current file
736 position on success */
738 _portable_ftell(FILE* fp
)
740 #if !defined(HAVE_LARGEFILE_SUPPORT)
742 #elif defined(HAVE_FTELLO) && SIZEOF_OFF_T >= 8
744 #elif defined(HAVE_FTELL64)
746 #elif SIZEOF_FPOS_T >= 8
748 if (fgetpos(fp
, &pos
) != 0)
752 #error "Large file support, but no way to ftell."
758 file_seek(PyFileObject
*f
, PyObject
*args
)
763 PyObject
*offobj
, *off_index
;
769 if (!PyArg_ParseTuple(args
, "O|i:seek", &offobj
, &whence
))
771 off_index
= PyNumber_Index(offobj
);
773 if (!PyFloat_Check(offobj
))
775 /* Deprecated in 2.6 */
777 if (PyErr_WarnEx(PyExc_DeprecationWarning
,
778 "integer argument expected, got float",
784 #if !defined(HAVE_LARGEFILE_SUPPORT)
785 offset
= PyInt_AsLong(off_index
);
787 offset
= PyLong_Check(off_index
) ?
788 PyLong_AsLongLong(off_index
) : PyInt_AsLong(off_index
);
790 Py_DECREF(off_index
);
791 if (PyErr_Occurred())
794 FILE_BEGIN_ALLOW_THREADS(f
)
796 ret
= _portable_fseek(f
->f_fp
, offset
, whence
);
797 FILE_END_ALLOW_THREADS(f
)
800 PyErr_SetFromErrno(PyExc_IOError
);
810 #ifdef HAVE_FTRUNCATE
812 file_truncate(PyFileObject
*f
, PyObject
*args
)
815 PyObject
*newsizeobj
= NULL
;
822 return err_mode("writing");
823 if (!PyArg_UnpackTuple(args
, "truncate", 0, 1, &newsizeobj
))
826 /* Get current file position. If the file happens to be open for
827 * update and the last operation was an input operation, C doesn't
828 * define what the later fflush() will do, but we promise truncate()
829 * won't change the current position (and fflush() *does* change it
830 * then at least on Windows). The easiest thing is to capture
831 * current pos now and seek back to it at the end.
833 FILE_BEGIN_ALLOW_THREADS(f
)
835 initialpos
= _portable_ftell(f
->f_fp
);
836 FILE_END_ALLOW_THREADS(f
)
837 if (initialpos
== -1)
840 /* Set newsize to current postion if newsizeobj NULL, else to the
843 if (newsizeobj
!= NULL
) {
844 #if !defined(HAVE_LARGEFILE_SUPPORT)
845 newsize
= PyInt_AsLong(newsizeobj
);
847 newsize
= PyLong_Check(newsizeobj
) ?
848 PyLong_AsLongLong(newsizeobj
) :
849 PyInt_AsLong(newsizeobj
);
851 if (PyErr_Occurred())
854 else /* default to current position */
855 newsize
= initialpos
;
857 /* Flush the stream. We're mixing stream-level I/O with lower-level
858 * I/O, and a flush may be necessary to synch both platform views
859 * of the current file state.
861 FILE_BEGIN_ALLOW_THREADS(f
)
863 ret
= fflush(f
->f_fp
);
864 FILE_END_ALLOW_THREADS(f
)
869 /* MS _chsize doesn't work if newsize doesn't fit in 32 bits,
870 so don't even try using it. */
874 /* Have to move current pos to desired endpoint on Windows. */
875 FILE_BEGIN_ALLOW_THREADS(f
)
877 ret
= _portable_fseek(f
->f_fp
, newsize
, SEEK_SET
) != 0;
878 FILE_END_ALLOW_THREADS(f
)
882 /* Truncate. Note that this may grow the file! */
883 FILE_BEGIN_ALLOW_THREADS(f
)
885 hFile
= (HANDLE
)_get_osfhandle(fileno(f
->f_fp
));
886 ret
= hFile
== (HANDLE
)-1;
888 ret
= SetEndOfFile(hFile
) == 0;
892 FILE_END_ALLOW_THREADS(f
)
897 FILE_BEGIN_ALLOW_THREADS(f
)
899 ret
= ftruncate(fileno(f
->f_fp
), newsize
);
900 FILE_END_ALLOW_THREADS(f
)
903 #endif /* !MS_WINDOWS */
905 /* Restore original file position. */
906 FILE_BEGIN_ALLOW_THREADS(f
)
908 ret
= _portable_fseek(f
->f_fp
, initialpos
, SEEK_SET
) != 0;
909 FILE_END_ALLOW_THREADS(f
)
917 PyErr_SetFromErrno(PyExc_IOError
);
921 #endif /* HAVE_FTRUNCATE */
924 file_tell(PyFileObject
*f
)
930 FILE_BEGIN_ALLOW_THREADS(f
)
932 pos
= _portable_ftell(f
->f_fp
);
933 FILE_END_ALLOW_THREADS(f
)
936 PyErr_SetFromErrno(PyExc_IOError
);
940 if (f
->f_skipnextlf
) {
944 f
->f_newlinetypes
|= NEWLINE_CRLF
;
947 } else if (c
!= EOF
) ungetc(c
, f
->f_fp
);
949 #if !defined(HAVE_LARGEFILE_SUPPORT)
950 return PyInt_FromLong(pos
);
952 return PyLong_FromLongLong(pos
);
957 file_fileno(PyFileObject
*f
)
961 return PyInt_FromLong((long) fileno(f
->f_fp
));
965 file_flush(PyFileObject
*f
)
971 FILE_BEGIN_ALLOW_THREADS(f
)
973 res
= fflush(f
->f_fp
);
974 FILE_END_ALLOW_THREADS(f
)
976 PyErr_SetFromErrno(PyExc_IOError
);
985 file_isatty(PyFileObject
*f
)
990 FILE_BEGIN_ALLOW_THREADS(f
)
991 res
= isatty((int)fileno(f
->f_fp
));
992 FILE_END_ALLOW_THREADS(f
)
993 return PyBool_FromLong(res
);
998 #define SMALLCHUNK 8192
1000 #define SMALLCHUNK BUFSIZ
1004 new_buffersize(PyFileObject
*f
, size_t currentsize
)
1009 if (fstat(fileno(f
->f_fp
), &st
) == 0) {
1011 /* The following is not a bug: we really need to call lseek()
1012 *and* ftell(). The reason is that some stdio libraries
1013 mistakenly flush their buffer when ftell() is called and
1014 the lseek() call it makes fails, thereby throwing away
1015 data that cannot be recovered in any way. To avoid this,
1016 we first test lseek(), and only call ftell() if lseek()
1017 works. We can't use the lseek() value either, because we
1018 need to take the amount of buffered data into account.
1019 (Yet another reason why stdio stinks. :-) */
1020 pos
= lseek(fileno(f
->f_fp
), 0L, SEEK_CUR
);
1022 pos
= ftell(f
->f_fp
);
1026 if (end
> pos
&& pos
>= 0)
1027 return currentsize
+ end
- pos
+ 1;
1028 /* Add 1 so if the file were to grow we'd notice. */
1031 /* Expand the buffer by an amount proportional to the current size,
1032 giving us amortized linear-time behavior. Use a less-than-double
1033 growth factor to avoid excessive allocation. */
1034 return currentsize
+ (currentsize
>> 3) + 6;
1037 #if defined(EWOULDBLOCK) && defined(EAGAIN) && EWOULDBLOCK != EAGAIN
1038 #define BLOCKED_ERRNO(x) ((x) == EWOULDBLOCK || (x) == EAGAIN)
1041 #define BLOCKED_ERRNO(x) ((x) == EWOULDBLOCK)
1044 #define BLOCKED_ERRNO(x) ((x) == EAGAIN)
1046 #define BLOCKED_ERRNO(x) 0
1052 file_read(PyFileObject
*f
, PyObject
*args
)
1054 long bytesrequested
= -1;
1055 size_t bytesread
, buffersize
, chunksize
;
1058 if (f
->f_fp
== NULL
)
1059 return err_closed();
1061 return err_mode("reading");
1062 /* refuse to mix with f.next() */
1063 if (f
->f_buf
!= NULL
&&
1064 (f
->f_bufend
- f
->f_bufptr
) > 0 &&
1065 f
->f_buf
[0] != '\0')
1066 return err_iterbuffered();
1067 if (!PyArg_ParseTuple(args
, "|l:read", &bytesrequested
))
1069 if (bytesrequested
< 0)
1070 buffersize
= new_buffersize(f
, (size_t)0);
1072 buffersize
= bytesrequested
;
1073 if (buffersize
> PY_SSIZE_T_MAX
) {
1074 PyErr_SetString(PyExc_OverflowError
,
1075 "requested number of bytes is more than a Python string can hold");
1078 v
= PyString_FromStringAndSize((char *)NULL
, buffersize
);
1084 FILE_BEGIN_ALLOW_THREADS(f
)
1086 chunksize
= Py_UniversalNewlineFread(BUF(v
) + bytesread
,
1087 buffersize
- bytesread
, f
->f_fp
, (PyObject
*)f
);
1088 interrupted
= ferror(f
->f_fp
) && errno
== EINTR
;
1089 FILE_END_ALLOW_THREADS(f
)
1092 if (PyErr_CheckSignals()) {
1097 if (chunksize
== 0) {
1100 if (!ferror(f
->f_fp
))
1103 /* When in non-blocking mode, data shouldn't
1104 * be discarded if a blocking signal was
1105 * received. That will also happen if
1106 * chunksize != 0, but bytesread < buffersize. */
1107 if (bytesread
> 0 && BLOCKED_ERRNO(errno
))
1109 PyErr_SetFromErrno(PyExc_IOError
);
1113 bytesread
+= chunksize
;
1114 if (bytesread
< buffersize
&& !interrupted
) {
1118 if (bytesrequested
< 0) {
1119 buffersize
= new_buffersize(f
, buffersize
);
1120 if (_PyString_Resize(&v
, buffersize
) < 0)
1123 /* Got what was requested. */
1127 if (bytesread
!= buffersize
&& _PyString_Resize(&v
, bytesread
))
1133 file_readinto(PyFileObject
*f
, PyObject
*args
)
1137 Py_ssize_t ndone
, nnow
;
1140 if (f
->f_fp
== NULL
)
1141 return err_closed();
1143 return err_mode("reading");
1144 /* refuse to mix with f.next() */
1145 if (f
->f_buf
!= NULL
&&
1146 (f
->f_bufend
- f
->f_bufptr
) > 0 &&
1147 f
->f_buf
[0] != '\0')
1148 return err_iterbuffered();
1149 if (!PyArg_ParseTuple(args
, "w*", &pbuf
))
1156 FILE_BEGIN_ALLOW_THREADS(f
)
1158 nnow
= Py_UniversalNewlineFread(ptr
+ndone
, ntodo
, f
->f_fp
,
1160 interrupted
= ferror(f
->f_fp
) && errno
== EINTR
;
1161 FILE_END_ALLOW_THREADS(f
)
1164 if (PyErr_CheckSignals()) {
1165 PyBuffer_Release(&pbuf
);
1172 if (!ferror(f
->f_fp
))
1174 PyErr_SetFromErrno(PyExc_IOError
);
1176 PyBuffer_Release(&pbuf
);
1182 PyBuffer_Release(&pbuf
);
1183 return PyInt_FromSsize_t(ndone
);
1186 /**************************************************************************
1187 Routine to get next line using platform fgets().
1191 + MS threadsafe getc is very slow (multiple layers of function calls before+
1192 after each character, to lock+unlock the stream).
1193 + The stream-locking functions are MS-internal -- can't access them from user
1195 + There's nothing Tim could find in the MS C or platform SDK libraries that
1196 can worm around this.
1197 + MS fgets locks/unlocks only once per line; it's the only hook we have.
1199 So we use fgets for speed(!), despite that it's painful.
1201 MS realloc is also slow.
1203 Reports from other platforms on this method vs getc_unlocked (which MS doesn't
1207 Tru64 Unix getline_via_fgets significantly faster
1209 CAUTION: The C std isn't clear about this: in those cases where fgets
1210 writes something into the buffer, can it write into any position beyond the
1211 required trailing null byte? MSVC 6 fgets does not, and no platform is (yet)
1212 known on which it does; and it would be a strange way to code fgets. Still,
1213 getline_via_fgets may not work correctly if it does. The std test
1214 test_bufio.py should fail if platform fgets() routinely writes beyond the
1215 trailing null byte. #define DONT_USE_FGETS_IN_GETLINE to disable this code.
1216 **************************************************************************/
1218 /* Use this routine if told to, or by default on non-get_unlocked()
1219 * platforms unless told not to. Yikes! Let's spell that out:
1220 * On a platform with getc_unlocked():
1221 * By default, use getc_unlocked().
1222 * If you want to use fgets() instead, #define USE_FGETS_IN_GETLINE.
1223 * On a platform without getc_unlocked():
1224 * By default, use fgets().
1225 * If you don't want to use fgets(), #define DONT_USE_FGETS_IN_GETLINE.
1227 #if !defined(USE_FGETS_IN_GETLINE) && !defined(HAVE_GETC_UNLOCKED)
1228 #define USE_FGETS_IN_GETLINE
1231 #if defined(DONT_USE_FGETS_IN_GETLINE) && defined(USE_FGETS_IN_GETLINE)
1232 #undef USE_FGETS_IN_GETLINE
1235 #ifdef USE_FGETS_IN_GETLINE
1237 getline_via_fgets(PyFileObject
*f
, FILE *fp
)
1239 /* INITBUFSIZE is the maximum line length that lets us get away with the fast
1240 * no-realloc, one-fgets()-call path. Boosting it isn't free, because we have
1241 * to fill this much of the buffer with a known value in order to figure out
1242 * how much of the buffer fgets() overwrites. So if INITBUFSIZE is larger
1243 * than "most" lines, we waste time filling unused buffer slots. 100 is
1244 * surely adequate for most peoples' email archives, chewing over source code,
1245 * etc -- "regular old text files".
1246 * MAXBUFSIZE is the maximum line length that lets us get away with the less
1247 * fast (but still zippy) no-realloc, two-fgets()-call path. See above for
1248 * cautions about boosting that. 300 was chosen because the worst real-life
1249 * text-crunching job reported on Python-Dev was a mail-log crawler where over
1250 * half the lines were 254 chars.
1252 #define INITBUFSIZE 100
1253 #define MAXBUFSIZE 300
1255 char buf
[MAXBUFSIZE
];
1256 PyObject
* v
; /* the string object result */
1257 char* pvfree
; /* address of next free slot */
1258 char* pvend
; /* address one beyond last free slot */
1259 size_t nfree
; /* # of free buffer slots; pvend-pvfree */
1260 size_t total_v_size
; /* total # of slots in buffer */
1261 size_t increment
; /* amount to increment the buffer */
1264 /* Optimize for normal case: avoid _PyString_Resize if at all
1265 * possible via first reading into stack buffer "buf".
1267 total_v_size
= INITBUFSIZE
; /* start small and pray */
1270 FILE_BEGIN_ALLOW_THREADS(f
)
1271 pvend
= buf
+ total_v_size
;
1272 nfree
= pvend
- pvfree
;
1273 memset(pvfree
, '\n', nfree
);
1274 assert(nfree
< INT_MAX
); /* Should be atmost MAXBUFSIZE */
1275 p
= fgets(pvfree
, (int)nfree
, fp
);
1276 FILE_END_ALLOW_THREADS(f
)
1280 if (PyErr_CheckSignals())
1282 v
= PyString_FromStringAndSize(buf
, pvfree
- buf
);
1285 /* fgets read *something* */
1286 p
= memchr(pvfree
, '\n', nfree
);
1288 /* Did the \n come from fgets or from us?
1289 * Since fgets stops at the first \n, and then writes
1290 * \0, if it's from fgets a \0 must be next. But if
1291 * that's so, it could not have come from us, since
1292 * the \n's we filled the buffer with have only more
1293 * \n's to the right.
1295 if (p
+1 < pvend
&& *(p
+1) == '\0') {
1296 /* It's from fgets: we win! In particular,
1297 * we haven't done any mallocs yet, and can
1298 * build the final result on the first try.
1300 ++p
; /* include \n from fgets */
1303 /* Must be from us: fgets didn't fill the
1304 * buffer and didn't find a newline, so it
1305 * must be the last and newline-free line of
1308 assert(p
> pvfree
&& *(p
-1) == '\0');
1309 --p
; /* don't include \0 from fgets */
1311 v
= PyString_FromStringAndSize(buf
, p
- buf
);
1314 /* yuck: fgets overwrote all the newlines, i.e. the entire
1315 * buffer. So this line isn't over yet, or maybe it is but
1316 * we're exactly at EOF. If we haven't already, try using the
1317 * rest of the stack buffer.
1319 assert(*(pvend
-1) == '\0');
1320 if (pvfree
== buf
) {
1321 pvfree
= pvend
- 1; /* overwrite trailing null */
1322 total_v_size
= MAXBUFSIZE
;
1328 /* The stack buffer isn't big enough; malloc a string object and read
1331 total_v_size
= MAXBUFSIZE
<< 1;
1332 v
= PyString_FromStringAndSize((char*)NULL
, (int)total_v_size
);
1335 /* copy over everything except the last null byte */
1336 memcpy(BUF(v
), buf
, MAXBUFSIZE
-1);
1337 pvfree
= BUF(v
) + MAXBUFSIZE
- 1;
1339 /* Keep reading stuff into v; if it ever ends successfully, break
1340 * after setting p one beyond the end of the line. The code here is
1341 * very much like the code above, except reads into v's buffer; see
1342 * the code above for detailed comments about the logic.
1345 FILE_BEGIN_ALLOW_THREADS(f
)
1346 pvend
= BUF(v
) + total_v_size
;
1347 nfree
= pvend
- pvfree
;
1348 memset(pvfree
, '\n', nfree
);
1349 assert(nfree
< INT_MAX
);
1350 p
= fgets(pvfree
, (int)nfree
, fp
);
1351 FILE_END_ALLOW_THREADS(f
)
1355 if (PyErr_CheckSignals()) {
1362 p
= memchr(pvfree
, '\n', nfree
);
1364 if (p
+1 < pvend
&& *(p
+1) == '\0') {
1365 /* \n came from fgets */
1369 /* \n came from us; last line of file, no newline */
1370 assert(p
> pvfree
&& *(p
-1) == '\0');
1374 /* expand buffer and try again */
1375 assert(*(pvend
-1) == '\0');
1376 increment
= total_v_size
>> 2; /* mild exponential growth */
1377 prev_v_size
= total_v_size
;
1378 total_v_size
+= increment
;
1379 /* check for overflow */
1380 if (total_v_size
<= prev_v_size
||
1381 total_v_size
> PY_SSIZE_T_MAX
) {
1382 PyErr_SetString(PyExc_OverflowError
,
1383 "line is longer than a Python string can hold");
1387 if (_PyString_Resize(&v
, (int)total_v_size
) < 0)
1389 /* overwrite the trailing null byte */
1390 pvfree
= BUF(v
) + (prev_v_size
- 1);
1392 if (BUF(v
) + total_v_size
!= p
&& _PyString_Resize(&v
, p
- BUF(v
)))
1398 #endif /* ifdef USE_FGETS_IN_GETLINE */
1400 /* Internal routine to get a line.
1401 Size argument interpretation:
1403 <= 0: read arbitrary line
1407 get_line(PyFileObject
*f
, int n
)
1412 size_t total_v_size
; /* total # of slots in buffer */
1413 size_t used_v_size
; /* # used slots in buffer */
1414 size_t increment
; /* amount to increment the buffer */
1416 int newlinetypes
= f
->f_newlinetypes
;
1417 int skipnextlf
= f
->f_skipnextlf
;
1418 int univ_newline
= f
->f_univ_newline
;
1420 #if defined(USE_FGETS_IN_GETLINE)
1421 if (n
<= 0 && !univ_newline
)
1422 return getline_via_fgets(f
, fp
);
1424 total_v_size
= n
> 0 ? n
: 100;
1425 v
= PyString_FromStringAndSize((char *)NULL
, total_v_size
);
1429 end
= buf
+ total_v_size
;
1432 FILE_BEGIN_ALLOW_THREADS(f
)
1435 c
= 'x'; /* Shut up gcc warning */
1436 while ( buf
!= end
&& (c
= GETC(fp
)) != EOF
) {
1440 /* Seeing a \n here with
1441 * skipnextlf true means we
1444 newlinetypes
|= NEWLINE_CRLF
;
1446 if (c
== EOF
) break;
1448 newlinetypes
|= NEWLINE_CR
;
1454 } else if ( c
== '\n')
1455 newlinetypes
|= NEWLINE_LF
;
1457 if (c
== '\n') break;
1460 if (ferror(fp
) && errno
== EINTR
) {
1462 FILE_ABORT_ALLOW_THREADS(f
)
1463 f
->f_newlinetypes
= newlinetypes
;
1464 f
->f_skipnextlf
= skipnextlf
;
1466 if (PyErr_CheckSignals()) {
1470 /* We executed Python signal handlers and got no exception.
1471 * Now back to reading the line where we left off. */
1476 newlinetypes
|= NEWLINE_CR
;
1478 } else /* If not universal newlines use the normal loop */
1479 while ((c
= GETC(fp
)) != EOF
&&
1480 (*buf
++ = c
) != '\n' &&
1484 FILE_END_ALLOW_THREADS(f
)
1485 f
->f_newlinetypes
= newlinetypes
;
1486 f
->f_skipnextlf
= skipnextlf
;
1491 if (errno
== EINTR
) {
1492 if (PyErr_CheckSignals()) {
1496 /* We executed Python signal handlers and got no exception.
1497 * Now back to reading the line where we left off. */
1501 PyErr_SetFromErrno(PyExc_IOError
);
1507 if (PyErr_CheckSignals()) {
1513 /* Must be because buf == end */
1516 used_v_size
= total_v_size
;
1517 increment
= total_v_size
>> 2; /* mild exponential growth */
1518 total_v_size
+= increment
;
1519 if (total_v_size
> PY_SSIZE_T_MAX
) {
1520 PyErr_SetString(PyExc_OverflowError
,
1521 "line is longer than a Python string can hold");
1525 if (_PyString_Resize(&v
, total_v_size
) < 0)
1527 buf
= BUF(v
) + used_v_size
;
1528 end
= BUF(v
) + total_v_size
;
1531 used_v_size
= buf
- BUF(v
);
1532 if (used_v_size
!= total_v_size
&& _PyString_Resize(&v
, used_v_size
))
1537 /* External C interface */
1540 PyFile_GetLine(PyObject
*f
, int n
)
1545 PyErr_BadInternalCall();
1549 if (PyFile_Check(f
)) {
1550 PyFileObject
*fo
= (PyFileObject
*)f
;
1551 if (fo
->f_fp
== NULL
)
1552 return err_closed();
1554 return err_mode("reading");
1555 /* refuse to mix with f.next() */
1556 if (fo
->f_buf
!= NULL
&&
1557 (fo
->f_bufend
- fo
->f_bufptr
) > 0 &&
1558 fo
->f_buf
[0] != '\0')
1559 return err_iterbuffered();
1560 result
= get_line(fo
, n
);
1566 reader
= PyObject_GetAttrString(f
, "readline");
1570 args
= PyTuple_New(0);
1572 args
= Py_BuildValue("(i)", n
);
1577 result
= PyEval_CallObject(reader
, args
);
1580 if (result
!= NULL
&& !PyString_Check(result
) &&
1581 !PyUnicode_Check(result
)) {
1584 PyErr_SetString(PyExc_TypeError
,
1585 "object.readline() returned non-string");
1589 if (n
< 0 && result
!= NULL
&& PyString_Check(result
)) {
1590 char *s
= PyString_AS_STRING(result
);
1591 Py_ssize_t len
= PyString_GET_SIZE(result
);
1595 PyErr_SetString(PyExc_EOFError
,
1596 "EOF when reading a line");
1598 else if (s
[len
-1] == '\n') {
1599 if (result
->ob_refcnt
== 1) {
1600 if (_PyString_Resize(&result
, len
-1))
1605 v
= PyString_FromStringAndSize(s
, len
-1);
1611 #ifdef Py_USING_UNICODE
1612 if (n
< 0 && result
!= NULL
&& PyUnicode_Check(result
)) {
1613 Py_UNICODE
*s
= PyUnicode_AS_UNICODE(result
);
1614 Py_ssize_t len
= PyUnicode_GET_SIZE(result
);
1618 PyErr_SetString(PyExc_EOFError
,
1619 "EOF when reading a line");
1621 else if (s
[len
-1] == '\n') {
1622 if (result
->ob_refcnt
== 1)
1623 PyUnicode_Resize(&result
, len
-1);
1626 v
= PyUnicode_FromUnicode(s
, len
-1);
1639 file_readline(PyFileObject
*f
, PyObject
*args
)
1643 if (f
->f_fp
== NULL
)
1644 return err_closed();
1646 return err_mode("reading");
1647 /* refuse to mix with f.next() */
1648 if (f
->f_buf
!= NULL
&&
1649 (f
->f_bufend
- f
->f_bufptr
) > 0 &&
1650 f
->f_buf
[0] != '\0')
1651 return err_iterbuffered();
1652 if (!PyArg_ParseTuple(args
, "|i:readline", &n
))
1655 return PyString_FromString("");
1658 return get_line(f
, n
);
1662 file_readlines(PyFileObject
*f
, PyObject
*args
)
1665 PyObject
*list
= NULL
;
1667 char small_buffer
[SMALLCHUNK
];
1668 char *buffer
= small_buffer
;
1669 size_t buffersize
= SMALLCHUNK
;
1670 PyObject
*big_buffer
= NULL
;
1673 size_t totalread
= 0;
1676 int shortread
= 0; /* bool, did the previous read come up short? */
1678 if (f
->f_fp
== NULL
)
1679 return err_closed();
1681 return err_mode("reading");
1682 /* refuse to mix with f.next() */
1683 if (f
->f_buf
!= NULL
&&
1684 (f
->f_bufend
- f
->f_bufptr
) > 0 &&
1685 f
->f_buf
[0] != '\0')
1686 return err_iterbuffered();
1687 if (!PyArg_ParseTuple(args
, "|l:readlines", &sizehint
))
1689 if ((list
= PyList_New(0)) == NULL
)
1695 FILE_BEGIN_ALLOW_THREADS(f
)
1697 nread
= Py_UniversalNewlineFread(buffer
+nfilled
,
1698 buffersize
-nfilled
, f
->f_fp
, (PyObject
*)f
);
1699 FILE_END_ALLOW_THREADS(f
)
1700 shortread
= (nread
< buffersize
-nfilled
);
1704 if (!ferror(f
->f_fp
))
1706 if (errno
== EINTR
) {
1707 if (PyErr_CheckSignals()) {
1714 PyErr_SetFromErrno(PyExc_IOError
);
1719 p
= (char *)memchr(buffer
+nfilled
, '\n', nread
);
1721 /* Need a larger buffer to fit this line */
1724 if (buffersize
> PY_SSIZE_T_MAX
) {
1725 PyErr_SetString(PyExc_OverflowError
,
1726 "line is longer than a Python string can hold");
1729 if (big_buffer
== NULL
) {
1730 /* Create the big buffer */
1731 big_buffer
= PyString_FromStringAndSize(
1733 if (big_buffer
== NULL
)
1735 buffer
= PyString_AS_STRING(big_buffer
);
1736 memcpy(buffer
, small_buffer
, nfilled
);
1739 /* Grow the big buffer */
1740 if ( _PyString_Resize(&big_buffer
, buffersize
) < 0 )
1742 buffer
= PyString_AS_STRING(big_buffer
);
1746 end
= buffer
+nfilled
+nread
;
1749 /* Process complete lines */
1751 line
= PyString_FromStringAndSize(q
, p
-q
);
1754 err
= PyList_Append(list
, line
);
1759 p
= (char *)memchr(q
, '\n', end
-q
);
1760 } while (p
!= NULL
);
1761 /* Move the remaining incomplete line to the start */
1763 memmove(buffer
, q
, nfilled
);
1765 if (totalread
>= (size_t)sizehint
)
1769 /* Partial last line */
1770 line
= PyString_FromStringAndSize(buffer
, nfilled
);
1774 /* Need to complete the last line */
1775 PyObject
*rest
= get_line(f
, 0);
1780 PyString_Concat(&line
, rest
);
1785 err
= PyList_Append(list
, line
);
1792 Py_XDECREF(big_buffer
);
1801 file_write(PyFileObject
*f
, PyObject
*args
)
1806 PyObject
*encoded
= NULL
;
1807 int err_flag
= 0, err
;
1809 if (f
->f_fp
== NULL
)
1810 return err_closed();
1812 return err_mode("writing");
1814 if (!PyArg_ParseTuple(args
, "s*", &pbuf
))
1821 if (!PyArg_ParseTuple(args
, "O", &text
))
1824 if (PyString_Check(text
)) {
1825 s
= PyString_AS_STRING(text
);
1826 n
= PyString_GET_SIZE(text
);
1827 #ifdef Py_USING_UNICODE
1828 } else if (PyUnicode_Check(text
)) {
1829 const char *encoding
, *errors
;
1830 if (f
->f_encoding
!= Py_None
)
1831 encoding
= PyString_AS_STRING(f
->f_encoding
);
1833 encoding
= PyUnicode_GetDefaultEncoding();
1834 if (f
->f_errors
!= Py_None
)
1835 errors
= PyString_AS_STRING(f
->f_errors
);
1838 encoded
= PyUnicode_AsEncodedString(text
, encoding
, errors
);
1839 if (encoded
== NULL
)
1841 s
= PyString_AS_STRING(encoded
);
1842 n
= PyString_GET_SIZE(encoded
);
1845 if (PyObject_AsCharBuffer(text
, &s
, &n
))
1850 FILE_BEGIN_ALLOW_THREADS(f
)
1852 n2
= fwrite(s
, 1, n
, f
->f_fp
);
1853 if (n2
!= n
|| ferror(f
->f_fp
)) {
1857 FILE_END_ALLOW_THREADS(f
)
1858 Py_XDECREF(encoded
);
1860 PyBuffer_Release(&pbuf
);
1863 PyErr_SetFromErrno(PyExc_IOError
);
1872 file_writelines(PyFileObject
*f
, PyObject
*seq
)
1874 #define CHUNKSIZE 1000
1875 PyObject
*list
, *line
;
1876 PyObject
*it
; /* iter(seq) */
1879 Py_ssize_t i
, j
, nwritten
, len
;
1881 assert(seq
!= NULL
);
1882 if (f
->f_fp
== NULL
)
1883 return err_closed();
1885 return err_mode("writing");
1889 islist
= PyList_Check(seq
);
1893 it
= PyObject_GetIter(seq
);
1895 PyErr_SetString(PyExc_TypeError
,
1896 "writelines() requires an iterable argument");
1899 /* From here on, fail by going to error, to reclaim "it". */
1900 list
= PyList_New(CHUNKSIZE
);
1905 /* Strategy: slurp CHUNKSIZE lines into a private list,
1906 checking that they are all strings, then write that list
1907 without holding the interpreter lock, then come back for more. */
1908 for (index
= 0; ; index
+= CHUNKSIZE
) {
1911 list
= PyList_GetSlice(seq
, index
, index
+CHUNKSIZE
);
1914 j
= PyList_GET_SIZE(list
);
1917 for (j
= 0; j
< CHUNKSIZE
; j
++) {
1918 line
= PyIter_Next(it
);
1920 if (PyErr_Occurred())
1924 PyList_SetItem(list
, j
, line
);
1926 /* The iterator might have closed the file on us. */
1927 if (f
->f_fp
== NULL
) {
1935 /* Check that all entries are indeed strings. If not,
1936 apply the same rules as for file.write() and
1937 convert the results to strings. This is slow, but
1938 seems to be the only way since all conversion APIs
1939 could potentially execute Python code. */
1940 for (i
= 0; i
< j
; i
++) {
1941 PyObject
*v
= PyList_GET_ITEM(list
, i
);
1942 if (!PyString_Check(v
)) {
1946 res
= PyObject_AsReadBuffer(v
, (const void**)&buffer
, &len
);
1948 res
= PyObject_AsCharBuffer(v
, &buffer
, &len
);
1951 PyErr_SetString(PyExc_TypeError
,
1952 "writelines() argument must be a sequence of strings");
1955 line
= PyString_FromStringAndSize(buffer
,
1960 PyList_SET_ITEM(list
, i
, line
);
1964 /* Since we are releasing the global lock, the
1965 following code may *not* execute Python code. */
1967 FILE_BEGIN_ALLOW_THREADS(f
)
1969 for (i
= 0; i
< j
; i
++) {
1970 line
= PyList_GET_ITEM(list
, i
);
1971 len
= PyString_GET_SIZE(line
);
1972 nwritten
= fwrite(PyString_AS_STRING(line
),
1974 if (nwritten
!= len
) {
1975 FILE_ABORT_ALLOW_THREADS(f
)
1976 PyErr_SetFromErrno(PyExc_IOError
);
1981 FILE_END_ALLOW_THREADS(f
)
1997 file_self(PyFileObject
*f
)
1999 if (f
->f_fp
== NULL
)
2000 return err_closed();
2002 return (PyObject
*)f
;
2006 file_xreadlines(PyFileObject
*f
)
2008 if (PyErr_WarnPy3k("f.xreadlines() not supported in 3.x, "
2009 "try 'for line in f' instead", 1) < 0)
2011 return file_self(f
);
2015 file_exit(PyObject
*f
, PyObject
*args
)
2017 PyObject
*ret
= PyObject_CallMethod(f
, "close", NULL
);
2019 /* If error occurred, pass through */
2022 /* We cannot return the result of close since a true
2023 * value will be interpreted as "yes, swallow the
2024 * exception if one was raised inside the with block". */
2028 PyDoc_STRVAR(readline_doc
,
2029 "readline([size]) -> next line from the file, as a string.\n"
2031 "Retain newline. A non-negative size argument limits the maximum\n"
2032 "number of bytes to return (an incomplete line may be returned then).\n"
2033 "Return an empty string at EOF.");
2035 PyDoc_STRVAR(read_doc
,
2036 "read([size]) -> read at most size bytes, returned as a string.\n"
2038 "If the size argument is negative or omitted, read until EOF is reached.\n"
2039 "Notice that when in non-blocking mode, less data than what was requested\n"
2040 "may be returned, even if no size parameter was given.");
2042 PyDoc_STRVAR(write_doc
,
2043 "write(str) -> None. Write string str to file.\n"
2045 "Note that due to buffering, flush() or close() may be needed before\n"
2046 "the file on disk reflects the data written.");
2048 PyDoc_STRVAR(fileno_doc
,
2049 "fileno() -> integer \"file descriptor\".\n"
2051 "This is needed for lower-level file interfaces, such os.read().");
2053 PyDoc_STRVAR(seek_doc
,
2054 "seek(offset[, whence]) -> None. Move to new file position.\n"
2056 "Argument offset is a byte count. Optional argument whence defaults to\n"
2057 "0 (offset from start of file, offset should be >= 0); other values are 1\n"
2058 "(move relative to current position, positive or negative), and 2 (move\n"
2059 "relative to end of file, usually negative, although many platforms allow\n"
2060 "seeking beyond the end of a file). If the file is opened in text mode,\n"
2061 "only offsets returned by tell() are legal. Use of other offsets causes\n"
2062 "undefined behavior."
2064 "Note that not all file objects are seekable.");
2066 #ifdef HAVE_FTRUNCATE
2067 PyDoc_STRVAR(truncate_doc
,
2068 "truncate([size]) -> None. Truncate the file to at most size bytes.\n"
2070 "Size defaults to the current file position, as returned by tell().");
2073 PyDoc_STRVAR(tell_doc
,
2074 "tell() -> current file position, an integer (may be a long integer).");
2076 PyDoc_STRVAR(readinto_doc
,
2077 "readinto() -> Undocumented. Don't use this; it may go away.");
2079 PyDoc_STRVAR(readlines_doc
,
2080 "readlines([size]) -> list of strings, each a line from the file.\n"
2082 "Call readline() repeatedly and return a list of the lines so read.\n"
2083 "The optional size argument, if given, is an approximate bound on the\n"
2084 "total number of bytes in the lines returned.");
2086 PyDoc_STRVAR(xreadlines_doc
,
2087 "xreadlines() -> returns self.\n"
2089 "For backward compatibility. File objects now include the performance\n"
2090 "optimizations previously implemented in the xreadlines module.");
2092 PyDoc_STRVAR(writelines_doc
,
2093 "writelines(sequence_of_strings) -> None. Write the strings to the file.\n"
2095 "Note that newlines are not added. The sequence can be any iterable object\n"
2096 "producing strings. This is equivalent to calling write() for each string.");
2098 PyDoc_STRVAR(flush_doc
,
2099 "flush() -> None. Flush the internal I/O buffer.");
2101 PyDoc_STRVAR(close_doc
,
2102 "close() -> None or (perhaps) an integer. Close the file.\n"
2104 "Sets data attribute .closed to True. A closed file cannot be used for\n"
2105 "further I/O operations. close() may be called more than once without\n"
2106 "error. Some kinds of file objects (for example, opened by popen())\n"
2107 "may return an exit status upon closing.");
2109 PyDoc_STRVAR(isatty_doc
,
2110 "isatty() -> true or false. True if the file is connected to a tty device.");
2112 PyDoc_STRVAR(enter_doc
,
2113 "__enter__() -> self.");
2115 PyDoc_STRVAR(exit_doc
,
2116 "__exit__(*excinfo) -> None. Closes the file.");
2118 static PyMethodDef file_methods
[] = {
2119 {"readline", (PyCFunction
)file_readline
, METH_VARARGS
, readline_doc
},
2120 {"read", (PyCFunction
)file_read
, METH_VARARGS
, read_doc
},
2121 {"write", (PyCFunction
)file_write
, METH_VARARGS
, write_doc
},
2122 {"fileno", (PyCFunction
)file_fileno
, METH_NOARGS
, fileno_doc
},
2123 {"seek", (PyCFunction
)file_seek
, METH_VARARGS
, seek_doc
},
2124 #ifdef HAVE_FTRUNCATE
2125 {"truncate", (PyCFunction
)file_truncate
, METH_VARARGS
, truncate_doc
},
2127 {"tell", (PyCFunction
)file_tell
, METH_NOARGS
, tell_doc
},
2128 {"readinto", (PyCFunction
)file_readinto
, METH_VARARGS
, readinto_doc
},
2129 {"readlines", (PyCFunction
)file_readlines
, METH_VARARGS
, readlines_doc
},
2130 {"xreadlines",(PyCFunction
)file_xreadlines
, METH_NOARGS
, xreadlines_doc
},
2131 {"writelines",(PyCFunction
)file_writelines
, METH_O
, writelines_doc
},
2132 {"flush", (PyCFunction
)file_flush
, METH_NOARGS
, flush_doc
},
2133 {"close", (PyCFunction
)file_close
, METH_NOARGS
, close_doc
},
2134 {"isatty", (PyCFunction
)file_isatty
, METH_NOARGS
, isatty_doc
},
2135 {"__enter__", (PyCFunction
)file_self
, METH_NOARGS
, enter_doc
},
2136 {"__exit__", (PyCFunction
)file_exit
, METH_VARARGS
, exit_doc
},
2137 {NULL
, NULL
} /* sentinel */
2140 #define OFF(x) offsetof(PyFileObject, x)
2142 static PyMemberDef file_memberlist
[] = {
2143 {"mode", T_OBJECT
, OFF(f_mode
), RO
,
2144 "file mode ('r', 'U', 'w', 'a', possibly with 'b' or '+' added)"},
2145 {"name", T_OBJECT
, OFF(f_name
), RO
,
2147 {"encoding", T_OBJECT
, OFF(f_encoding
), RO
,
2149 {"errors", T_OBJECT
, OFF(f_errors
), RO
,
2150 "Unicode error handler"},
2151 /* getattr(f, "closed") is implemented without this table */
2152 {NULL
} /* Sentinel */
2156 get_closed(PyFileObject
*f
, void *closure
)
2158 return PyBool_FromLong((long)(f
->f_fp
== 0));
2161 get_newlines(PyFileObject
*f
, void *closure
)
2163 switch (f
->f_newlinetypes
) {
2164 case NEWLINE_UNKNOWN
:
2168 return PyString_FromString("\r");
2170 return PyString_FromString("\n");
2171 case NEWLINE_CR
|NEWLINE_LF
:
2172 return Py_BuildValue("(ss)", "\r", "\n");
2174 return PyString_FromString("\r\n");
2175 case NEWLINE_CR
|NEWLINE_CRLF
:
2176 return Py_BuildValue("(ss)", "\r", "\r\n");
2177 case NEWLINE_LF
|NEWLINE_CRLF
:
2178 return Py_BuildValue("(ss)", "\n", "\r\n");
2179 case NEWLINE_CR
|NEWLINE_LF
|NEWLINE_CRLF
:
2180 return Py_BuildValue("(sss)", "\r", "\n", "\r\n");
2182 PyErr_Format(PyExc_SystemError
,
2183 "Unknown newlines value 0x%x\n",
2190 get_softspace(PyFileObject
*f
, void *closure
)
2192 if (PyErr_WarnPy3k("file.softspace not supported in 3.x", 1) < 0)
2194 return PyInt_FromLong(f
->f_softspace
);
2198 set_softspace(PyFileObject
*f
, PyObject
*value
)
2201 if (PyErr_WarnPy3k("file.softspace not supported in 3.x", 1) < 0)
2204 if (value
== NULL
) {
2205 PyErr_SetString(PyExc_TypeError
,
2206 "can't delete softspace attribute");
2210 new = PyInt_AsLong(value
);
2211 if (new == -1 && PyErr_Occurred())
2213 f
->f_softspace
= new;
2217 static PyGetSetDef file_getsetlist
[] = {
2218 {"closed", (getter
)get_closed
, NULL
, "True if the file is closed"},
2219 {"newlines", (getter
)get_newlines
, NULL
,
2220 "end-of-line convention used in this file"},
2221 {"softspace", (getter
)get_softspace
, (setter
)set_softspace
,
2222 "flag indicating that a space needs to be printed; used by print"},
2227 drop_readahead(PyFileObject
*f
)
2229 if (f
->f_buf
!= NULL
) {
2230 PyMem_Free(f
->f_buf
);
2235 /* Make sure that file has a readahead buffer with at least one byte
2236 (unless at EOF) and no more than bufsize. Returns negative value on
2237 error, will set MemoryError if bufsize bytes cannot be allocated. */
2239 readahead(PyFileObject
*f
, Py_ssize_t bufsize
)
2241 Py_ssize_t chunksize
;
2243 if (f
->f_buf
!= NULL
) {
2244 if( (f
->f_bufend
- f
->f_bufptr
) >= 1)
2249 if ((f
->f_buf
= (char *)PyMem_Malloc(bufsize
)) == NULL
) {
2253 FILE_BEGIN_ALLOW_THREADS(f
)
2255 chunksize
= Py_UniversalNewlineFread(
2256 f
->f_buf
, bufsize
, f
->f_fp
, (PyObject
*)f
);
2257 FILE_END_ALLOW_THREADS(f
)
2258 if (chunksize
== 0) {
2259 if (ferror(f
->f_fp
)) {
2260 PyErr_SetFromErrno(PyExc_IOError
);
2266 f
->f_bufptr
= f
->f_buf
;
2267 f
->f_bufend
= f
->f_buf
+ chunksize
;
2271 /* Used by file_iternext. The returned string will start with 'skip'
2272 uninitialized bytes followed by the remainder of the line. Don't be
2273 horrified by the recursive call: maximum recursion depth is limited by
2274 logarithmic buffer growth to about 50 even when reading a 1gb line. */
2276 static PyStringObject
*
2277 readahead_get_line_skip(PyFileObject
*f
, Py_ssize_t skip
, Py_ssize_t bufsize
)
2284 if (f
->f_buf
== NULL
)
2285 if (readahead(f
, bufsize
) < 0)
2288 len
= f
->f_bufend
- f
->f_bufptr
;
2290 return (PyStringObject
*)
2291 PyString_FromStringAndSize(NULL
, skip
);
2292 bufptr
= (char *)memchr(f
->f_bufptr
, '\n', len
);
2293 if (bufptr
!= NULL
) {
2294 bufptr
++; /* Count the '\n' */
2295 len
= bufptr
- f
->f_bufptr
;
2296 s
= (PyStringObject
*)
2297 PyString_FromStringAndSize(NULL
, skip
+ len
);
2300 memcpy(PyString_AS_STRING(s
) + skip
, f
->f_bufptr
, len
);
2301 f
->f_bufptr
= bufptr
;
2302 if (bufptr
== f
->f_bufend
)
2305 bufptr
= f
->f_bufptr
;
2307 f
->f_buf
= NULL
; /* Force new readahead buffer */
2308 assert(len
<= PY_SSIZE_T_MAX
- skip
);
2309 s
= readahead_get_line_skip(f
, skip
+ len
, bufsize
+ (bufsize
>>2));
2314 memcpy(PyString_AS_STRING(s
) + skip
, bufptr
, len
);
2320 /* A larger buffer size may actually decrease performance. */
2321 #define READAHEAD_BUFSIZE 8192
2324 file_iternext(PyFileObject
*f
)
2328 if (f
->f_fp
== NULL
)
2329 return err_closed();
2331 return err_mode("reading");
2333 l
= readahead_get_line_skip(f
, 0, READAHEAD_BUFSIZE
);
2334 if (l
== NULL
|| PyString_GET_SIZE(l
) == 0) {
2338 return (PyObject
*)l
;
2343 file_new(PyTypeObject
*type
, PyObject
*args
, PyObject
*kwds
)
2346 static PyObject
*not_yet_string
;
2348 assert(type
!= NULL
&& type
->tp_alloc
!= NULL
);
2350 if (not_yet_string
== NULL
) {
2351 not_yet_string
= PyString_InternFromString("<uninitialized file>");
2352 if (not_yet_string
== NULL
)
2356 self
= type
->tp_alloc(type
, 0);
2358 /* Always fill in the name and mode, so that nobody else
2359 needs to special-case NULLs there. */
2360 Py_INCREF(not_yet_string
);
2361 ((PyFileObject
*)self
)->f_name
= not_yet_string
;
2362 Py_INCREF(not_yet_string
);
2363 ((PyFileObject
*)self
)->f_mode
= not_yet_string
;
2365 ((PyFileObject
*)self
)->f_encoding
= Py_None
;
2367 ((PyFileObject
*)self
)->f_errors
= Py_None
;
2368 ((PyFileObject
*)self
)->weakreflist
= NULL
;
2369 ((PyFileObject
*)self
)->unlocked_count
= 0;
2375 file_init(PyObject
*self
, PyObject
*args
, PyObject
*kwds
)
2377 PyFileObject
*foself
= (PyFileObject
*)self
;
2379 static char *kwlist
[] = {"name", "mode", "buffering", 0};
2383 int wideargument
= 0;
2388 assert(PyFile_Check(self
));
2389 if (foself
->f_fp
!= NULL
) {
2390 /* Have to close the existing file first. */
2391 PyObject
*closeresult
= file_close(foself
);
2392 if (closeresult
== NULL
)
2394 Py_DECREF(closeresult
);
2398 if (PyArg_ParseTupleAndKeywords(args
, kwds
, "U|si:file",
2399 kwlist
, &po
, &mode
, &bufsize
)) {
2401 if (fill_file_fields(foself
, NULL
, po
, mode
,
2405 /* Drop the argument parsing error as narrow
2406 strings are also valid. */
2411 if (!wideargument
) {
2414 if (!PyArg_ParseTupleAndKeywords(args
, kwds
, "et|si:file", kwlist
,
2415 Py_FileSystemDefaultEncoding
,
2420 /* We parse again to get the name as a PyObject */
2421 if (!PyArg_ParseTupleAndKeywords(args
, kwds
, "O|si:file",
2422 kwlist
, &o_name
, &mode
,
2426 if (fill_file_fields(foself
, NULL
, o_name
, mode
,
2430 if (open_the_file(foself
, name
, mode
) == NULL
)
2432 foself
->f_setbuf
= NULL
;
2433 PyFile_SetBufSize(self
, bufsize
);
2440 PyMem_Free(name
); /* free the encoded string */
2444 PyDoc_VAR(file_doc
) =
2446 "file(name[, mode[, buffering]]) -> file object\n"
2448 "Open a file. The mode can be 'r', 'w' or 'a' for reading (default),\n"
2449 "writing or appending. The file will be created if it doesn't exist\n"
2450 "when opened for writing or appending; it will be truncated when\n"
2451 "opened for writing. Add a 'b' to the mode for binary files.\n"
2452 "Add a '+' to the mode to allow simultaneous reading and writing.\n"
2453 "If the buffering argument is given, 0 means unbuffered, 1 means line\n"
2454 "buffered, and larger numbers specify the buffer size. The preferred way\n"
2455 "to open a file is with the builtin open() function.\n"
2458 "Add a 'U' to mode to open the file for input with universal newline\n"
2459 "support. Any line ending in the input file will be seen as a '\\n'\n"
2460 "in Python. Also, a file so opened gains the attribute 'newlines';\n"
2461 "the value for this attribute is one of None (no newline read yet),\n"
2462 "'\\r', '\\n', '\\r\\n' or a tuple containing all the newline types seen.\n"
2464 "'U' cannot be combined with 'w' or '+' mode.\n"
2467 PyTypeObject PyFile_Type
= {
2468 PyVarObject_HEAD_INIT(&PyType_Type
, 0)
2470 sizeof(PyFileObject
),
2472 (destructor
)file_dealloc
, /* tp_dealloc */
2477 (reprfunc
)file_repr
, /* tp_repr */
2478 0, /* tp_as_number */
2479 0, /* tp_as_sequence */
2480 0, /* tp_as_mapping */
2484 PyObject_GenericGetAttr
, /* tp_getattro */
2485 /* softspace is writable: we must supply tp_setattro */
2486 PyObject_GenericSetAttr
, /* tp_setattro */
2487 0, /* tp_as_buffer */
2488 Py_TPFLAGS_DEFAULT
| Py_TPFLAGS_BASETYPE
| Py_TPFLAGS_HAVE_WEAKREFS
, /* tp_flags */
2489 file_doc
, /* tp_doc */
2490 0, /* tp_traverse */
2492 0, /* tp_richcompare */
2493 offsetof(PyFileObject
, weakreflist
), /* tp_weaklistoffset */
2494 (getiterfunc
)file_self
, /* tp_iter */
2495 (iternextfunc
)file_iternext
, /* tp_iternext */
2496 file_methods
, /* tp_methods */
2497 file_memberlist
, /* tp_members */
2498 file_getsetlist
, /* tp_getset */
2501 0, /* tp_descr_get */
2502 0, /* tp_descr_set */
2503 0, /* tp_dictoffset */
2504 file_init
, /* tp_init */
2505 PyType_GenericAlloc
, /* tp_alloc */
2506 file_new
, /* tp_new */
2507 PyObject_Del
, /* tp_free */
2510 /* Interface for the 'soft space' between print items. */
2513 PyFile_SoftSpace(PyObject
*f
, int newflag
)
2519 else if (PyFile_Check(f
)) {
2520 oldflag
= ((PyFileObject
*)f
)->f_softspace
;
2521 ((PyFileObject
*)f
)->f_softspace
= newflag
;
2525 v
= PyObject_GetAttrString(f
, "softspace");
2530 oldflag
= PyInt_AsLong(v
);
2531 assert(oldflag
< INT_MAX
);
2534 v
= PyInt_FromLong((long)newflag
);
2538 if (PyObject_SetAttrString(f
, "softspace", v
) != 0)
2543 return (int)oldflag
;
2546 /* Interfaces to write objects/strings to file-like objects */
2549 PyFile_WriteObject(PyObject
*v
, PyObject
*f
, int flags
)
2551 PyObject
*writer
, *value
, *args
, *result
;
2553 PyErr_SetString(PyExc_TypeError
, "writeobject with NULL file");
2556 else if (PyFile_Check(f
)) {
2557 PyFileObject
*fobj
= (PyFileObject
*) f
;
2558 #ifdef Py_USING_UNICODE
2559 PyObject
*enc
= fobj
->f_encoding
;
2562 if (fobj
->f_fp
== NULL
) {
2566 #ifdef Py_USING_UNICODE
2567 if ((flags
& Py_PRINT_RAW
) &&
2568 PyUnicode_Check(v
) && enc
!= Py_None
) {
2569 char *cenc
= PyString_AS_STRING(enc
);
2570 char *errors
= fobj
->f_errors
== Py_None
?
2571 "strict" : PyString_AS_STRING(fobj
->f_errors
);
2572 value
= PyUnicode_AsEncodedString(v
, cenc
, errors
);
2579 result
= file_PyObject_Print(value
, fobj
, flags
);
2583 return file_PyObject_Print(v
, fobj
, flags
);
2586 writer
= PyObject_GetAttrString(f
, "write");
2589 if (flags
& Py_PRINT_RAW
) {
2590 if (PyUnicode_Check(v
)) {
2594 value
= PyObject_Str(v
);
2597 value
= PyObject_Repr(v
);
2598 if (value
== NULL
) {
2602 args
= PyTuple_Pack(1, value
);
2608 result
= PyEval_CallObject(writer
, args
);
2619 PyFile_WriteString(const char *s
, PyObject
*f
)
2623 /* Should be caused by a pre-existing error */
2624 if (!PyErr_Occurred())
2625 PyErr_SetString(PyExc_SystemError
,
2626 "null file for PyFile_WriteString");
2629 else if (PyFile_Check(f
)) {
2630 PyFileObject
*fobj
= (PyFileObject
*) f
;
2631 FILE *fp
= PyFile_AsFile(f
);
2636 FILE_BEGIN_ALLOW_THREADS(fobj
)
2638 FILE_END_ALLOW_THREADS(fobj
)
2641 else if (!PyErr_Occurred()) {
2642 PyObject
*v
= PyString_FromString(s
);
2646 err
= PyFile_WriteObject(v
, f
, Py_PRINT_RAW
);
2654 /* Try to get a file-descriptor from a Python object. If the object
2655 is an integer or long integer, its value is returned. If not, the
2656 object's fileno() method is called if it exists; the method must return
2657 an integer or long integer, which is returned as the file descriptor value.
2658 -1 is returned on failure.
2661 int PyObject_AsFileDescriptor(PyObject
*o
)
2666 if (PyInt_Check(o
)) {
2667 fd
= _PyInt_AsInt(o
);
2669 else if (PyLong_Check(o
)) {
2670 fd
= _PyLong_AsInt(o
);
2672 else if ((meth
= PyObject_GetAttrString(o
, "fileno")) != NULL
)
2674 PyObject
*fno
= PyEval_CallObject(meth
, NULL
);
2679 if (PyInt_Check(fno
)) {
2680 fd
= _PyInt_AsInt(fno
);
2683 else if (PyLong_Check(fno
)) {
2684 fd
= _PyLong_AsInt(fno
);
2688 PyErr_SetString(PyExc_TypeError
,
2689 "fileno() returned a non-integer");
2695 PyErr_SetString(PyExc_TypeError
,
2696 "argument must be an int, or have a fileno() method.");
2701 PyErr_Format(PyExc_ValueError
,
2702 "file descriptor cannot be a negative integer (%i)",
2709 /* From here on we need access to the real fgets and fread */
2714 ** Py_UniversalNewlineFgets is an fgets variation that understands
2715 ** all of \r, \n and \r\n conventions.
2716 ** The stream should be opened in binary mode.
2717 ** If fobj is NULL the routine always does newline conversion, and
2718 ** it may peek one char ahead to gobble the second char in \r\n.
2719 ** If fobj is non-NULL it must be a PyFileObject. In this case there
2720 ** is no readahead but in stead a flag is used to skip a following
2721 ** \n on the next read. Also, if the file is open in binary mode
2722 ** the whole conversion is skipped. Finally, the routine keeps track of
2723 ** the different types of newlines seen.
2724 ** Note that we need no error handling: fgets() treats error and eof
2728 Py_UniversalNewlineFgets(char *buf
, int n
, FILE *stream
, PyObject
*fobj
)
2732 int newlinetypes
= 0;
2734 int univ_newline
= 1;
2737 if (!PyFile_Check(fobj
)) {
2738 errno
= ENXIO
; /* What can you do... */
2741 univ_newline
= ((PyFileObject
*)fobj
)->f_univ_newline
;
2742 if ( !univ_newline
)
2743 return fgets(buf
, n
, stream
);
2744 newlinetypes
= ((PyFileObject
*)fobj
)->f_newlinetypes
;
2745 skipnextlf
= ((PyFileObject
*)fobj
)->f_skipnextlf
;
2748 c
= 'x'; /* Shut up gcc warning */
2749 while (--n
> 0 && (c
= GETC(stream
)) != EOF
) {
2753 /* Seeing a \n here with skipnextlf true
2754 ** means we saw a \r before.
2756 newlinetypes
|= NEWLINE_CRLF
;
2758 if (c
== EOF
) break;
2761 ** Note that c == EOF also brings us here,
2762 ** so we're okay if the last char in the file
2765 newlinetypes
|= NEWLINE_CR
;
2769 /* A \r is translated into a \n, and we skip
2770 ** an adjacent \n, if any. We don't set the
2771 ** newlinetypes flag until we've seen the next char.
2775 } else if ( c
== '\n') {
2776 newlinetypes
|= NEWLINE_LF
;
2779 if (c
== '\n') break;
2781 if ( c
== EOF
&& skipnextlf
)
2782 newlinetypes
|= NEWLINE_CR
;
2783 FUNLOCKFILE(stream
);
2786 ((PyFileObject
*)fobj
)->f_newlinetypes
= newlinetypes
;
2787 ((PyFileObject
*)fobj
)->f_skipnextlf
= skipnextlf
;
2788 } else if ( skipnextlf
) {
2789 /* If we have no file object we cannot save the
2790 ** skipnextlf flag. We have to readahead, which
2791 ** will cause a pause if we're reading from an
2792 ** interactive stream, but that is very unlikely
2793 ** unless we're doing something silly like
2794 ** execfile("/dev/tty").
2806 ** Py_UniversalNewlineFread is an fread variation that understands
2807 ** all of \r, \n and \r\n conventions.
2808 ** The stream should be opened in binary mode.
2809 ** fobj must be a PyFileObject. In this case there
2810 ** is no readahead but in stead a flag is used to skip a following
2811 ** \n on the next read. Also, if the file is open in binary mode
2812 ** the whole conversion is skipped. Finally, the routine keeps track of
2813 ** the different types of newlines seen.
2816 Py_UniversalNewlineFread(char *buf
, size_t n
,
2817 FILE *stream
, PyObject
*fobj
)
2820 PyFileObject
*f
= (PyFileObject
*)fobj
;
2821 int newlinetypes
, skipnextlf
;
2823 assert(buf
!= NULL
);
2824 assert(stream
!= NULL
);
2826 if (!fobj
|| !PyFile_Check(fobj
)) {
2827 errno
= ENXIO
; /* What can you do... */
2830 if (!f
->f_univ_newline
)
2831 return fread(buf
, 1, n
, stream
);
2832 newlinetypes
= f
->f_newlinetypes
;
2833 skipnextlf
= f
->f_skipnextlf
;
2834 /* Invariant: n is the number of bytes remaining to be filled
2842 nread
= fread(dst
, 1, n
, stream
);
2847 n
-= nread
; /* assuming 1 byte out for each in; will adjust */
2848 shortread
= n
!= 0; /* true iff EOF or error */
2852 /* Save as LF and set flag to skip next LF. */
2856 else if (skipnextlf
&& c
== '\n') {
2857 /* Skip LF, and remember we saw CR LF. */
2859 newlinetypes
|= NEWLINE_CRLF
;
2863 /* Normal char to be stored in buffer. Also
2864 * update the newlinetypes flag if either this
2865 * is an LF or the previous char was a CR.
2868 newlinetypes
|= NEWLINE_LF
;
2869 else if (skipnextlf
)
2870 newlinetypes
|= NEWLINE_CR
;
2876 /* If this is EOF, update type flags. */
2877 if (skipnextlf
&& feof(stream
))
2878 newlinetypes
|= NEWLINE_CR
;
2882 f
->f_newlinetypes
= newlinetypes
;
2883 f
->f_skipnextlf
= skipnextlf
;