2 /* Write Python objects to files and read them back.
3 This is intended for writing and reading compiled Python code only;
4 a true persistent storage facility would be much harder, since
5 it would have to take circular links and sharing into account. */
7 #define PY_SSIZE_T_CLEAN
10 #include "longintrepr.h"
14 #define ABS(x) ((x) < 0 ? -(x) : (x))
16 /* High water mark to determine when the marshalled object is dangerously deep
17 * and risks coring the interpreter. When the object stack gets this deep,
18 * raise an exception instead of continuing.
20 #define MAX_MARSHAL_STACK_DEPTH 2000
24 #define TYPE_FALSE 'F'
26 #define TYPE_STOPITER 'S'
27 #define TYPE_ELLIPSIS '.'
29 #define TYPE_INT64 'I'
30 #define TYPE_FLOAT 'f'
31 #define TYPE_BINARY_FLOAT 'g'
32 #define TYPE_COMPLEX 'x'
33 #define TYPE_BINARY_COMPLEX 'y'
35 #define TYPE_STRING 's'
36 #define TYPE_INTERNED 't'
37 #define TYPE_STRINGREF 'R'
38 #define TYPE_TUPLE '('
42 #define TYPE_UNICODE 'u'
43 #define TYPE_UNKNOWN '?'
45 #define TYPE_FROZENSET '>'
48 #define WFERR_UNMARSHALLABLE 1
49 #define WFERR_NESTEDTOODEEP 2
50 #define WFERR_NOMEMORY 3
54 int error
; /* see WFERR_* values */
56 /* If fp == NULL, the following are valid: */
60 PyObject
*strings
; /* dict on marshal, list on unmarshal */
64 #define w_byte(c, p) if (((p)->fp)) putc((c), (p)->fp); \
65 else if ((p)->ptr != (p)->end) *(p)->ptr++ = (c); \
69 w_more(int c
, WFILE
*p
)
71 Py_ssize_t size
, newsize
;
73 return; /* An error already occurred */
74 size
= PyString_Size(p
->str
);
75 newsize
= size
+ size
+ 1024;
76 if (newsize
> 32*1024*1024) {
77 newsize
= size
+ (size
>> 3); /* 12.5% overallocation */
79 if (_PyString_Resize(&p
->str
, newsize
) != 0) {
80 p
->ptr
= p
->end
= NULL
;
83 p
->ptr
= PyString_AS_STRING((PyStringObject
*)p
->str
) + size
;
85 PyString_AS_STRING((PyStringObject
*)p
->str
) + newsize
;
86 *p
->ptr
++ = Py_SAFE_DOWNCAST(c
, int, char);
91 w_string(const char *s
, Py_ssize_t n
, WFILE
*p
)
94 fwrite(s
, 1, n
, p
->fp
);
105 w_short(int x
, WFILE
*p
)
107 w_byte((char)( x
& 0xff), p
);
108 w_byte((char)((x
>> 8) & 0xff), p
);
112 w_long(long x
, WFILE
*p
)
114 w_byte((char)( x
& 0xff), p
);
115 w_byte((char)((x
>> 8) & 0xff), p
);
116 w_byte((char)((x
>>16) & 0xff), p
);
117 w_byte((char)((x
>>24) & 0xff), p
);
122 w_long64(long x
, WFILE
*p
)
129 #define SIZE32_MAX 0x7FFFFFFF
131 #if SIZEOF_SIZE_T > 4
132 # define W_SIZE(n, p) do { \
133 if ((n) > SIZE32_MAX) { \
135 (p)->error = WFERR_UNMARSHALLABLE; \
138 w_long((long)(n), p); \
141 # define W_SIZE w_long
145 w_pstring(const char *s
, Py_ssize_t n
, WFILE
*p
)
151 /* We assume that Python longs are stored internally in base some power of
152 2**15; for the sake of portability we'll always read and write them in base
155 #define PyLong_MARSHAL_SHIFT 15
156 #define PyLong_MARSHAL_BASE ((short)1 << PyLong_MARSHAL_SHIFT)
157 #define PyLong_MARSHAL_MASK (PyLong_MARSHAL_BASE - 1)
158 #if PyLong_SHIFT % PyLong_MARSHAL_SHIFT != 0
159 #error "PyLong_SHIFT must be a multiple of PyLong_MARSHAL_SHIFT"
161 #define PyLong_MARSHAL_RATIO (PyLong_SHIFT / PyLong_MARSHAL_SHIFT)
164 w_PyLong(const PyLongObject
*ob
, WFILE
*p
)
166 Py_ssize_t i
, j
, n
, l
;
169 w_byte(TYPE_LONG
, p
);
170 if (Py_SIZE(ob
) == 0) {
175 /* set l to number of base PyLong_MARSHAL_BASE digits */
176 n
= ABS(Py_SIZE(ob
));
177 l
= (n
-1) * PyLong_MARSHAL_RATIO
;
178 d
= ob
->ob_digit
[n
-1];
179 assert(d
!= 0); /* a PyLong is always normalized */
181 d
>>= PyLong_MARSHAL_SHIFT
;
184 if (l
> SIZE32_MAX
) {
186 p
->error
= WFERR_UNMARSHALLABLE
;
189 w_long((long)(Py_SIZE(ob
) > 0 ? l
: -l
), p
);
191 for (i
=0; i
< n
-1; i
++) {
193 for (j
=0; j
< PyLong_MARSHAL_RATIO
; j
++) {
194 w_short(d
& PyLong_MARSHAL_MASK
, p
);
195 d
>>= PyLong_MARSHAL_SHIFT
;
199 d
= ob
->ob_digit
[n
-1];
201 w_short(d
& PyLong_MARSHAL_MASK
, p
);
202 d
>>= PyLong_MARSHAL_SHIFT
;
207 w_object(PyObject
*v
, WFILE
*p
)
213 if (p
->depth
> MAX_MARSHAL_STACK_DEPTH
) {
214 p
->error
= WFERR_NESTEDTOODEEP
;
216 else if (v
== NULL
) {
217 w_byte(TYPE_NULL
, p
);
219 else if (v
== Py_None
) {
220 w_byte(TYPE_NONE
, p
);
222 else if (v
== PyExc_StopIteration
) {
223 w_byte(TYPE_STOPITER
, p
);
225 else if (v
== Py_Ellipsis
) {
226 w_byte(TYPE_ELLIPSIS
, p
);
228 else if (v
== Py_False
) {
229 w_byte(TYPE_FALSE
, p
);
231 else if (v
== Py_True
) {
232 w_byte(TYPE_TRUE
, p
);
234 else if (PyInt_CheckExact(v
)) {
235 long x
= PyInt_AS_LONG((PyIntObject
*)v
);
237 long y
= Py_ARITHMETIC_RIGHT_SHIFT(long, x
, 31);
239 w_byte(TYPE_INT64
, p
);
249 else if (PyLong_CheckExact(v
)) {
250 PyLongObject
*ob
= (PyLongObject
*)v
;
253 else if (PyFloat_CheckExact(v
)) {
254 if (p
->version
> 1) {
255 unsigned char buf
[8];
256 if (_PyFloat_Pack8(PyFloat_AsDouble(v
),
258 p
->error
= WFERR_UNMARSHALLABLE
;
261 w_byte(TYPE_BINARY_FLOAT
, p
);
262 w_string((char*)buf
, 8, p
);
265 char *buf
= PyOS_double_to_string(PyFloat_AS_DOUBLE(v
),
268 p
->error
= WFERR_NOMEMORY
;
272 w_byte(TYPE_FLOAT
, p
);
278 #ifndef WITHOUT_COMPLEX
279 else if (PyComplex_CheckExact(v
)) {
280 if (p
->version
> 1) {
281 unsigned char buf
[8];
282 if (_PyFloat_Pack8(PyComplex_RealAsDouble(v
),
284 p
->error
= WFERR_UNMARSHALLABLE
;
287 w_byte(TYPE_BINARY_COMPLEX
, p
);
288 w_string((char*)buf
, 8, p
);
289 if (_PyFloat_Pack8(PyComplex_ImagAsDouble(v
),
291 p
->error
= WFERR_UNMARSHALLABLE
;
294 w_string((char*)buf
, 8, p
);
298 w_byte(TYPE_COMPLEX
, p
);
299 buf
= PyOS_double_to_string(PyComplex_RealAsDouble(v
),
302 p
->error
= WFERR_NOMEMORY
;
309 buf
= PyOS_double_to_string(PyComplex_ImagAsDouble(v
),
312 p
->error
= WFERR_NOMEMORY
;
322 else if (PyString_CheckExact(v
)) {
323 if (p
->strings
&& PyString_CHECK_INTERNED(v
)) {
324 PyObject
*o
= PyDict_GetItem(p
->strings
, v
);
326 long w
= PyInt_AsLong(o
);
327 w_byte(TYPE_STRINGREF
, p
);
333 o
= PyInt_FromSsize_t(PyDict_Size(p
->strings
));
335 PyDict_SetItem(p
->strings
, v
, o
) >= 0;
339 p
->error
= WFERR_UNMARSHALLABLE
;
342 w_byte(TYPE_INTERNED
, p
);
346 w_byte(TYPE_STRING
, p
);
348 w_pstring(PyBytes_AS_STRING(v
), PyString_GET_SIZE(v
), p
);
350 #ifdef Py_USING_UNICODE
351 else if (PyUnicode_CheckExact(v
)) {
353 utf8
= PyUnicode_AsUTF8String(v
);
356 p
->error
= WFERR_UNMARSHALLABLE
;
359 w_byte(TYPE_UNICODE
, p
);
360 w_pstring(PyString_AS_STRING(utf8
), PyString_GET_SIZE(utf8
), p
);
364 else if (PyTuple_CheckExact(v
)) {
365 w_byte(TYPE_TUPLE
, p
);
368 for (i
= 0; i
< n
; i
++) {
369 w_object(PyTuple_GET_ITEM(v
, i
), p
);
372 else if (PyList_CheckExact(v
)) {
373 w_byte(TYPE_LIST
, p
);
374 n
= PyList_GET_SIZE(v
);
376 for (i
= 0; i
< n
; i
++) {
377 w_object(PyList_GET_ITEM(v
, i
), p
);
380 else if (PyDict_CheckExact(v
)) {
382 PyObject
*key
, *value
;
383 w_byte(TYPE_DICT
, p
);
384 /* This one is NULL object terminated! */
386 while (PyDict_Next(v
, &pos
, &key
, &value
)) {
390 w_object((PyObject
*)NULL
, p
);
392 else if (PyAnySet_CheckExact(v
)) {
393 PyObject
*value
, *it
;
395 if (PyObject_TypeCheck(v
, &PySet_Type
))
398 w_byte(TYPE_FROZENSET
, p
);
399 n
= PyObject_Size(v
);
402 p
->error
= WFERR_UNMARSHALLABLE
;
406 it
= PyObject_GetIter(v
);
409 p
->error
= WFERR_UNMARSHALLABLE
;
412 while ((value
= PyIter_Next(it
)) != NULL
) {
417 if (PyErr_Occurred()) {
419 p
->error
= WFERR_UNMARSHALLABLE
;
423 else if (PyCode_Check(v
)) {
424 PyCodeObject
*co
= (PyCodeObject
*)v
;
425 w_byte(TYPE_CODE
, p
);
426 w_long(co
->co_argcount
, p
);
427 w_long(co
->co_nlocals
, p
);
428 w_long(co
->co_stacksize
, p
);
429 w_long(co
->co_flags
, p
);
430 w_object(co
->co_code
, p
);
431 w_object(co
->co_consts
, p
);
432 w_object(co
->co_names
, p
);
433 w_object(co
->co_varnames
, p
);
434 w_object(co
->co_freevars
, p
);
435 w_object(co
->co_cellvars
, p
);
436 w_object(co
->co_filename
, p
);
437 w_object(co
->co_name
, p
);
438 w_long(co
->co_firstlineno
, p
);
439 w_object(co
->co_lnotab
, p
);
441 else if (PyObject_CheckReadBuffer(v
)) {
442 /* Write unknown buffer-style objects as a string */
444 PyBufferProcs
*pb
= v
->ob_type
->tp_as_buffer
;
445 w_byte(TYPE_STRING
, p
);
446 n
= (*pb
->bf_getreadbuffer
)(v
, 0, (void **)&s
);
450 w_byte(TYPE_UNKNOWN
, p
);
451 p
->error
= WFERR_UNMARSHALLABLE
;
457 /* version currently has no effect for writing longs. */
459 PyMarshal_WriteLongToFile(long x
, FILE *fp
, int version
)
466 wf
.version
= version
;
471 PyMarshal_WriteObjectToFile(PyObject
*x
, FILE *fp
, int version
)
477 wf
.strings
= (version
> 0) ? PyDict_New() : NULL
;
478 wf
.version
= version
;
480 Py_XDECREF(wf
.strings
);
483 typedef WFILE RFILE
; /* Same struct with different invariants */
485 #define rs_byte(p) (((p)->ptr < (p)->end) ? (unsigned char)*(p)->ptr++ : EOF)
487 #define r_byte(p) ((p)->fp ? getc((p)->fp) : rs_byte(p))
490 r_string(char *s
, Py_ssize_t n
, RFILE
*p
)
493 /* The result fits into int because it must be <=n. */
494 return fread(s
, 1, n
, p
->fp
);
495 if (p
->end
- p
->ptr
< n
)
497 memcpy(s
, p
->ptr
, n
);
508 /* Sign-extension, in case short greater than 16 bits */
517 register FILE *fp
= p
->fp
;
520 x
|= (long)getc(fp
) << 8;
521 x
|= (long)getc(fp
) << 16;
522 x
|= (long)getc(fp
) << 24;
526 x
|= (long)rs_byte(p
) << 8;
527 x
|= (long)rs_byte(p
) << 16;
528 x
|= (long)rs_byte(p
) << 24;
531 /* Sign extension for 64-bit machines */
532 x
|= -(x
& 0x80000000L
);
537 /* r_long64 deals with the TYPE_INT64 code. On a machine with
538 sizeof(long) > 4, it returns a Python int object, else a Python long
539 object. Note that w_long64 writes out TYPE_INT if 32 bits is enough,
540 so there's no inefficiency here in returning a PyLong on 32-bit boxes
541 for everything written via TYPE_INT64 (i.e., if an int is written via
542 TYPE_INT64, it *needs* more than 32 bits).
547 long lo4
= r_long(p
);
548 long hi4
= r_long(p
);
550 long x
= (hi4
<< 32) | (lo4
& 0xFFFFFFFFL
);
551 return PyInt_FromLong(x
);
553 unsigned char buf
[8];
555 int is_little_endian
= (int)*(char*)&one
;
556 if (is_little_endian
) {
557 memcpy(buf
, &lo4
, 4);
558 memcpy(buf
+4, &hi4
, 4);
561 memcpy(buf
, &hi4
, 4);
562 memcpy(buf
+4, &lo4
, 4);
564 return _PyLong_FromByteArray(buf
, 8, is_little_endian
, 1);
573 int j
, md
, shorts_in_top_digit
;
578 return (PyObject
*)_PyLong_New(0);
579 if (n
< -SIZE32_MAX
|| n
> SIZE32_MAX
) {
580 PyErr_SetString(PyExc_ValueError
,
581 "bad marshal data (long size out of range)");
585 size
= 1 + (ABS(n
) - 1) / PyLong_MARSHAL_RATIO
;
586 shorts_in_top_digit
= 1 + (ABS(n
) - 1) % PyLong_MARSHAL_RATIO
;
587 ob
= _PyLong_New(size
);
590 Py_SIZE(ob
) = n
> 0 ? size
: -size
;
592 for (i
= 0; i
< size
-1; i
++) {
594 for (j
=0; j
< PyLong_MARSHAL_RATIO
; j
++) {
596 if (md
< 0 || md
> PyLong_MARSHAL_BASE
)
598 d
+= (digit
)md
<< j
*PyLong_MARSHAL_SHIFT
;
603 for (j
=0; j
< shorts_in_top_digit
; j
++) {
605 if (md
< 0 || md
> PyLong_MARSHAL_BASE
)
607 /* topmost marshal digit should be nonzero */
608 if (md
== 0 && j
== shorts_in_top_digit
- 1) {
610 PyErr_SetString(PyExc_ValueError
,
611 "bad marshal data (unnormalized long data)");
614 d
+= (digit
)md
<< j
*PyLong_MARSHAL_SHIFT
;
616 /* top digit should be nonzero, else the resulting PyLong won't be
618 ob
->ob_digit
[size
-1] = d
;
619 return (PyObject
*)ob
;
622 PyErr_SetString(PyExc_ValueError
,
623 "bad marshal data (digit out of range in long)");
631 /* NULL is a valid return value, it does not necessarily means that
632 an exception is set. */
635 int type
= r_byte(p
);
640 if (p
->depth
> MAX_MARSHAL_STACK_DEPTH
) {
642 PyErr_SetString(PyExc_ValueError
, "recursion limit exceeded");
649 PyErr_SetString(PyExc_EOFError
,
650 "EOF read where object expected");
664 Py_INCREF(PyExc_StopIteration
);
665 retval
= PyExc_StopIteration
;
669 Py_INCREF(Py_Ellipsis
);
670 retval
= Py_Ellipsis
;
684 retval
= PyInt_FromLong(r_long(p
));
688 retval
= r_long64(p
);
692 retval
= r_PyLong(p
);
700 if (n
== EOF
|| r_string(buf
, n
, p
) != n
) {
701 PyErr_SetString(PyExc_EOFError
,
702 "EOF read where object expected");
707 dx
= PyOS_string_to_double(buf
, NULL
, NULL
);
708 if (dx
== -1.0 && PyErr_Occurred()) {
712 retval
= PyFloat_FromDouble(dx
);
716 case TYPE_BINARY_FLOAT
:
718 unsigned char buf
[8];
720 if (r_string((char*)buf
, 8, p
) != 8) {
721 PyErr_SetString(PyExc_EOFError
,
722 "EOF read where object expected");
726 x
= _PyFloat_Unpack8(buf
, 1);
727 if (x
== -1.0 && PyErr_Occurred()) {
731 retval
= PyFloat_FromDouble(x
);
735 #ifndef WITHOUT_COMPLEX
741 if (n
== EOF
|| r_string(buf
, n
, p
) != n
) {
742 PyErr_SetString(PyExc_EOFError
,
743 "EOF read where object expected");
748 c
.real
= PyOS_string_to_double(buf
, NULL
, NULL
);
749 if (c
.real
== -1.0 && PyErr_Occurred()) {
754 if (n
== EOF
|| r_string(buf
, n
, p
) != n
) {
755 PyErr_SetString(PyExc_EOFError
,
756 "EOF read where object expected");
761 c
.imag
= PyOS_string_to_double(buf
, NULL
, NULL
);
762 if (c
.imag
== -1.0 && PyErr_Occurred()) {
766 retval
= PyComplex_FromCComplex(c
);
770 case TYPE_BINARY_COMPLEX
:
772 unsigned char buf
[8];
774 if (r_string((char*)buf
, 8, p
) != 8) {
775 PyErr_SetString(PyExc_EOFError
,
776 "EOF read where object expected");
780 c
.real
= _PyFloat_Unpack8(buf
, 1);
781 if (c
.real
== -1.0 && PyErr_Occurred()) {
785 if (r_string((char*)buf
, 8, p
) != 8) {
786 PyErr_SetString(PyExc_EOFError
,
787 "EOF read where object expected");
791 c
.imag
= _PyFloat_Unpack8(buf
, 1);
792 if (c
.imag
== -1.0 && PyErr_Occurred()) {
796 retval
= PyComplex_FromCComplex(c
);
804 if (n
< 0 || n
> SIZE32_MAX
) {
805 PyErr_SetString(PyExc_ValueError
, "bad marshal data (string size out of range)");
809 v
= PyString_FromStringAndSize((char *)NULL
, n
);
814 if (r_string(PyString_AS_STRING(v
), n
, p
) != n
) {
816 PyErr_SetString(PyExc_EOFError
,
817 "EOF read where object expected");
821 if (type
== TYPE_INTERNED
) {
822 PyString_InternInPlace(&v
);
823 if (PyList_Append(p
->strings
, v
) < 0) {
833 if (n
< 0 || n
>= PyList_GET_SIZE(p
->strings
)) {
834 PyErr_SetString(PyExc_ValueError
, "bad marshal data (string ref out of range)");
838 v
= PyList_GET_ITEM(p
->strings
, n
);
843 #ifdef Py_USING_UNICODE
849 if (n
< 0 || n
> SIZE32_MAX
) {
850 PyErr_SetString(PyExc_ValueError
, "bad marshal data (unicode size out of range)");
854 buffer
= PyMem_NEW(char, n
);
855 if (buffer
== NULL
) {
856 retval
= PyErr_NoMemory();
859 if (r_string(buffer
, n
, p
) != n
) {
861 PyErr_SetString(PyExc_EOFError
,
862 "EOF read where object expected");
866 v
= PyUnicode_DecodeUTF8(buffer
, n
, NULL
);
875 if (n
< 0 || n
> SIZE32_MAX
) {
876 PyErr_SetString(PyExc_ValueError
, "bad marshal data (tuple size out of range)");
885 for (i
= 0; i
< n
; i
++) {
888 if (!PyErr_Occurred())
889 PyErr_SetString(PyExc_TypeError
,
890 "NULL object in marshal data for tuple");
895 PyTuple_SET_ITEM(v
, i
, v2
);
902 if (n
< 0 || n
> SIZE32_MAX
) {
903 PyErr_SetString(PyExc_ValueError
, "bad marshal data (list size out of range)");
912 for (i
= 0; i
< n
; i
++) {
915 if (!PyErr_Occurred())
916 PyErr_SetString(PyExc_TypeError
,
917 "NULL object in marshal data for list");
922 PyList_SET_ITEM(v
, i
, v2
);
940 PyDict_SetItem(v
, key
, val
);
944 if (PyErr_Occurred()) {
954 if (n
< 0 || n
> SIZE32_MAX
) {
955 PyErr_SetString(PyExc_ValueError
, "bad marshal data (set size out of range)");
959 v
= (type
== TYPE_SET
) ? PySet_New(NULL
) : PyFrozenSet_New(NULL
);
964 for (i
= 0; i
< n
; i
++) {
967 if (!PyErr_Occurred())
968 PyErr_SetString(PyExc_TypeError
,
969 "NULL object in marshal data for set");
974 if (PySet_Add(v
, v2
) == -1) {
986 if (PyEval_GetRestricted()) {
987 PyErr_SetString(PyExc_RuntimeError
,
988 "cannot unmarshal code objects in "
989 "restricted execution mode");
998 PyObject
*code
= NULL
;
999 PyObject
*consts
= NULL
;
1000 PyObject
*names
= NULL
;
1001 PyObject
*varnames
= NULL
;
1002 PyObject
*freevars
= NULL
;
1003 PyObject
*cellvars
= NULL
;
1004 PyObject
*filename
= NULL
;
1005 PyObject
*name
= NULL
;
1007 PyObject
*lnotab
= NULL
;
1011 /* XXX ignore long->int overflows for now */
1012 argcount
= (int)r_long(p
);
1013 nlocals
= (int)r_long(p
);
1014 stacksize
= (int)r_long(p
);
1015 flags
= (int)r_long(p
);
1019 consts
= r_object(p
);
1022 names
= r_object(p
);
1025 varnames
= r_object(p
);
1026 if (varnames
== NULL
)
1028 freevars
= r_object(p
);
1029 if (freevars
== NULL
)
1031 cellvars
= r_object(p
);
1032 if (cellvars
== NULL
)
1034 filename
= r_object(p
);
1035 if (filename
== NULL
)
1040 firstlineno
= (int)r_long(p
);
1041 lnotab
= r_object(p
);
1045 v
= (PyObject
*) PyCode_New(
1046 argcount
, nlocals
, stacksize
, flags
,
1047 code
, consts
, names
, varnames
,
1048 freevars
, cellvars
, filename
, name
,
1049 firstlineno
, lnotab
);
1055 Py_XDECREF(varnames
);
1056 Py_XDECREF(freevars
);
1057 Py_XDECREF(cellvars
);
1058 Py_XDECREF(filename
);
1067 /* Bogus data got written, which isn't ideal.
1068 This will let you keep working and recover. */
1069 PyErr_SetString(PyExc_ValueError
, "bad marshal data (unknown type code)");
1079 read_object(RFILE
*p
)
1082 if (PyErr_Occurred()) {
1083 fprintf(stderr
, "XXX readobject called with exception set\n");
1087 if (v
== NULL
&& !PyErr_Occurred())
1088 PyErr_SetString(PyExc_TypeError
, "NULL object in marshal data for object");
1093 PyMarshal_ReadShortFromFile(FILE *fp
)
1099 rf
.end
= rf
.ptr
= NULL
;
1100 return r_short(&rf
);
1104 PyMarshal_ReadLongFromFile(FILE *fp
)
1109 rf
.ptr
= rf
.end
= NULL
;
1114 /* Return size of file in bytes; < 0 if unknown. */
1116 getfilesize(FILE *fp
)
1119 if (fstat(fileno(fp
), &st
) != 0)
1126 /* If we can get the size of the file up-front, and it's reasonably small,
1127 * read it in one gulp and delegate to ...FromString() instead. Much quicker
1128 * than reading a byte at a time from file; speeds .pyc imports.
1129 * CAUTION: since this may read the entire remainder of the file, don't
1130 * call it unless you know you're done with the file.
1133 PyMarshal_ReadLastObjectFromFile(FILE *fp
)
1135 /* REASONABLE_FILE_LIMIT is by defn something big enough for Tkinter.pyc. */
1136 #define REASONABLE_FILE_LIMIT (1L << 18)
1139 filesize
= getfilesize(fp
);
1140 if (filesize
> 0 && filesize
<= REASONABLE_FILE_LIMIT
) {
1141 char* pBuf
= (char *)PyMem_MALLOC(filesize
);
1143 size_t n
= fread(pBuf
, 1, (size_t)filesize
, fp
);
1144 PyObject
* v
= PyMarshal_ReadObjectFromString(pBuf
, n
);
1151 /* We don't have fstat, or we do but the file is larger than
1152 * REASONABLE_FILE_LIMIT or malloc failed -- read a byte at a time.
1154 return PyMarshal_ReadObjectFromFile(fp
);
1156 #undef REASONABLE_FILE_LIMIT
1160 PyMarshal_ReadObjectFromFile(FILE *fp
)
1165 rf
.strings
= PyList_New(0);
1167 rf
.ptr
= rf
.end
= NULL
;
1168 result
= r_object(&rf
);
1169 Py_DECREF(rf
.strings
);
1174 PyMarshal_ReadObjectFromString(char *str
, Py_ssize_t len
)
1181 rf
.strings
= PyList_New(0);
1183 result
= r_object(&rf
);
1184 Py_DECREF(rf
.strings
);
1189 set_error(int error
)
1192 case WFERR_NOMEMORY
:
1195 case WFERR_UNMARSHALLABLE
:
1196 PyErr_SetString(PyExc_ValueError
, "unmarshallable object");
1198 case WFERR_NESTEDTOODEEP
:
1200 PyErr_SetString(PyExc_ValueError
,
1201 "object too deeply nested to marshal");
1207 PyMarshal_WriteObjectToString(PyObject
*x
, int version
)
1211 wf
.str
= PyString_FromStringAndSize((char *)NULL
, 50);
1214 wf
.ptr
= PyString_AS_STRING((PyStringObject
*)wf
.str
);
1215 wf
.end
= wf
.ptr
+ PyString_Size(wf
.str
);
1216 wf
.error
= WFERR_OK
;
1218 wf
.version
= version
;
1219 wf
.strings
= (version
> 0) ? PyDict_New() : NULL
;
1221 Py_XDECREF(wf
.strings
);
1222 if (wf
.str
!= NULL
) {
1223 char *base
= PyString_AS_STRING((PyStringObject
*)wf
.str
);
1224 if (wf
.ptr
- base
> PY_SSIZE_T_MAX
) {
1226 PyErr_SetString(PyExc_OverflowError
,
1227 "too much marshall data for a string");
1230 if (_PyString_Resize(&wf
.str
, (Py_ssize_t
)(wf
.ptr
- base
)))
1233 if (wf
.error
!= WFERR_OK
) {
1235 set_error(wf
.error
);
1241 /* And an interface for Python programs... */
1244 marshal_dump(PyObject
*self
, PyObject
*args
)
1249 int version
= Py_MARSHAL_VERSION
;
1250 if (!PyArg_ParseTuple(args
, "OO|i:dump", &x
, &f
, &version
))
1252 if (!PyFile_Check(f
)) {
1253 PyErr_SetString(PyExc_TypeError
,
1254 "marshal.dump() 2nd arg must be file");
1257 wf
.fp
= PyFile_AsFile(f
);
1259 wf
.ptr
= wf
.end
= NULL
;
1260 wf
.error
= WFERR_OK
;
1262 wf
.strings
= (version
> 0) ? PyDict_New() : 0;
1263 wf
.version
= version
;
1265 Py_XDECREF(wf
.strings
);
1266 if (wf
.error
!= WFERR_OK
) {
1267 set_error(wf
.error
);
1274 PyDoc_STRVAR(dump_doc
,
1275 "dump(value, file[, version])\n\
1277 Write the value on the open file. The value must be a supported type.\n\
1278 The file must be an open file object such as sys.stdout or returned by\n\
1279 open() or os.popen(). It must be opened in binary mode ('wb' or 'w+b').\n\
1281 If the value has (or contains an object that has) an unsupported type, a\n\
1282 ValueError exception is raised — but garbage data will also be written\n\
1283 to the file. The object will not be properly read back by load()\n\
1285 New in version 2.4: The version argument indicates the data format that\n\
1289 marshal_load(PyObject
*self
, PyObject
*f
)
1293 if (!PyFile_Check(f
)) {
1294 PyErr_SetString(PyExc_TypeError
,
1295 "marshal.load() arg must be file");
1298 rf
.fp
= PyFile_AsFile(f
);
1299 rf
.strings
= PyList_New(0);
1301 result
= read_object(&rf
);
1302 Py_DECREF(rf
.strings
);
1306 PyDoc_STRVAR(load_doc
,
1309 Read one value from the open file and return it. If no valid value is\n\
1310 read (e.g. because the data has a different Python version’s\n\
1311 incompatible marshal format), raise EOFError, ValueError or TypeError.\n\
1312 The file must be an open file object opened in binary mode ('rb' or\n\
1315 Note: If an object containing an unsupported type was marshalled with\n\
1316 dump(), load() will substitute None for the unmarshallable type.");
1320 marshal_dumps(PyObject
*self
, PyObject
*args
)
1323 int version
= Py_MARSHAL_VERSION
;
1324 if (!PyArg_ParseTuple(args
, "O|i:dumps", &x
, &version
))
1326 return PyMarshal_WriteObjectToString(x
, version
);
1329 PyDoc_STRVAR(dumps_doc
,
1330 "dumps(value[, version])\n\
1332 Return the string that would be written to a file by dump(value, file).\n\
1333 The value must be a supported type. Raise a ValueError exception if\n\
1334 value has (or contains an object that has) an unsupported type.\n\
1336 New in version 2.4: The version argument indicates the data format that\n\
1337 dumps should use.");
1341 marshal_loads(PyObject
*self
, PyObject
*args
)
1347 if (!PyArg_ParseTuple(args
, "s#:loads", &s
, &n
))
1352 rf
.strings
= PyList_New(0);
1354 result
= read_object(&rf
);
1355 Py_DECREF(rf
.strings
);
1359 PyDoc_STRVAR(loads_doc
,
1362 Convert the string to a value. If no valid value is found, raise\n\
1363 EOFError, ValueError or TypeError. Extra characters in the string are\n\
1366 static PyMethodDef marshal_methods
[] = {
1367 {"dump", marshal_dump
, METH_VARARGS
, dump_doc
},
1368 {"load", marshal_load
, METH_O
, load_doc
},
1369 {"dumps", marshal_dumps
, METH_VARARGS
, dumps_doc
},
1370 {"loads", marshal_loads
, METH_VARARGS
, loads_doc
},
1371 {NULL
, NULL
} /* sentinel */
1374 PyDoc_STRVAR(marshal_doc
,
1375 "This module contains functions that can read and write Python values in\n\
1376 a binary format. The format is specific to Python, but independent of\n\
1377 machine architecture issues.\n\
1379 Not all Python object types are supported; in general, only objects\n\
1380 whose value is independent from a particular invocation of Python can be\n\
1381 written and read by this module. The following types are supported:\n\
1382 None, integers, long integers, floating point numbers, strings, Unicode\n\
1383 objects, tuples, lists, sets, dictionaries, and code objects, where it\n\
1384 should be understood that tuples, lists and dictionaries are only\n\
1385 supported as long as the values contained therein are themselves\n\
1386 supported; and recursive lists and dictionaries should not be written\n\
1387 (they will cause infinite loops).\n\
1391 version -- indicates the format that the module uses. Version 0 is the\n\
1392 historical format, version 1 (added in Python 2.4) shares interned\n\
1393 strings and version 2 (added in Python 2.5) uses a binary format for\n\
1394 floating point numbers. (New in version 2.4)\n\
1398 dump() -- write value to a file\n\
1399 load() -- read value from a file\n\
1400 dumps() -- write value to a string\n\
1401 loads() -- read value from a string");
1405 PyMarshal_Init(void)
1407 PyObject
*mod
= Py_InitModule3("marshal", marshal_methods
,
1411 PyModule_AddIntConstant(mod
, "version", Py_MARSHAL_VERSION
);