2 /* Write Python objects to files and read them back.
3 This is intended for writing and reading compiled Python code only;
4 a true persistent storage facility would be much harder, since
5 it would have to take circular links and sharing into account. */
7 #define PY_SSIZE_T_CLEAN
10 #include "longintrepr.h"
15 #define ABS(x) ((x) < 0 ? -(x) : (x))
18 /* High water mark to determine when the marshalled object is dangerously deep
19 * and risks coring the interpreter. When the object stack gets this deep,
20 * raise an exception instead of continuing.
22 #define MAX_MARSHAL_STACK_DEPTH 2000
26 #define TYPE_FALSE 'F'
28 #define TYPE_STOPITER 'S'
29 #define TYPE_ELLIPSIS '.'
31 #define TYPE_INT64 'I'
32 #define TYPE_FLOAT 'f'
33 #define TYPE_BINARY_FLOAT 'g'
34 #define TYPE_COMPLEX 'x'
35 #define TYPE_BINARY_COMPLEX 'y'
37 #define TYPE_STRING 's'
38 #define TYPE_INTERNED 't'
39 #define TYPE_STRINGREF 'R'
40 #define TYPE_TUPLE '('
44 #define TYPE_UNICODE 'u'
45 #define TYPE_UNKNOWN '?'
47 #define TYPE_FROZENSET '>'
50 #define WFERR_UNMARSHALLABLE 1
51 #define WFERR_NESTEDTOODEEP 2
52 #define WFERR_NOMEMORY 3
56 int error
; /* see WFERR_* values */
58 /* If fp == NULL, the following are valid: */
62 PyObject
*strings
; /* dict on marshal, list on unmarshal */
66 #define w_byte(c, p) if (((p)->fp)) putc((c), (p)->fp); \
67 else if ((p)->ptr != (p)->end) *(p)->ptr++ = (c); \
71 w_more(int c
, WFILE
*p
)
73 Py_ssize_t size
, newsize
;
75 return; /* An error already occurred */
76 size
= PyString_Size(p
->str
);
77 newsize
= size
+ size
+ 1024;
78 if (newsize
> 32*1024*1024) {
79 newsize
= size
+ (size
>> 3); /* 12.5% overallocation */
81 if (_PyString_Resize(&p
->str
, newsize
) != 0) {
82 p
->ptr
= p
->end
= NULL
;
85 p
->ptr
= PyString_AS_STRING((PyStringObject
*)p
->str
) + size
;
87 PyString_AS_STRING((PyStringObject
*)p
->str
) + newsize
;
88 *p
->ptr
++ = Py_SAFE_DOWNCAST(c
, int, char);
93 w_string(char *s
, int n
, WFILE
*p
)
96 fwrite(s
, 1, n
, p
->fp
);
107 w_short(int x
, WFILE
*p
)
109 w_byte((char)( x
& 0xff), p
);
110 w_byte((char)((x
>> 8) & 0xff), p
);
114 w_long(long x
, WFILE
*p
)
116 w_byte((char)( x
& 0xff), p
);
117 w_byte((char)((x
>> 8) & 0xff), p
);
118 w_byte((char)((x
>>16) & 0xff), p
);
119 w_byte((char)((x
>>24) & 0xff), p
);
124 w_long64(long x
, WFILE
*p
)
131 /* We assume that Python longs are stored internally in base some power of
132 2**15; for the sake of portability we'll always read and write them in base
135 #define PyLong_MARSHAL_SHIFT 15
136 #define PyLong_MARSHAL_BASE ((short)1 << PyLong_MARSHAL_SHIFT)
137 #define PyLong_MARSHAL_MASK (PyLong_MARSHAL_BASE - 1)
138 #if PyLong_SHIFT % PyLong_MARSHAL_SHIFT != 0
139 #error "PyLong_SHIFT must be a multiple of PyLong_MARSHAL_SHIFT"
141 #define PyLong_MARSHAL_RATIO (PyLong_SHIFT / PyLong_MARSHAL_SHIFT)
144 w_PyLong(const PyLongObject
*ob
, WFILE
*p
)
146 Py_ssize_t i
, j
, n
, l
;
149 w_byte(TYPE_LONG
, p
);
150 if (Py_SIZE(ob
) == 0) {
155 /* set l to number of base PyLong_MARSHAL_BASE digits */
156 n
= ABS(Py_SIZE(ob
));
157 l
= (n
-1) * PyLong_MARSHAL_RATIO
;
158 d
= ob
->ob_digit
[n
-1];
159 assert(d
!= 0); /* a PyLong is always normalized */
161 d
>>= PyLong_MARSHAL_SHIFT
;
164 w_long((long)(Py_SIZE(ob
) > 0 ? l
: -l
), p
);
166 for (i
=0; i
< n
-1; i
++) {
168 for (j
=0; j
< PyLong_MARSHAL_RATIO
; j
++) {
169 w_short(d
& PyLong_MARSHAL_MASK
, p
);
170 d
>>= PyLong_MARSHAL_SHIFT
;
174 d
= ob
->ob_digit
[n
-1];
176 w_short(d
& PyLong_MARSHAL_MASK
, p
);
177 d
>>= PyLong_MARSHAL_SHIFT
;
182 w_object(PyObject
*v
, WFILE
*p
)
188 if (p
->depth
> MAX_MARSHAL_STACK_DEPTH
) {
189 p
->error
= WFERR_NESTEDTOODEEP
;
191 else if (v
== NULL
) {
192 w_byte(TYPE_NULL
, p
);
194 else if (v
== Py_None
) {
195 w_byte(TYPE_NONE
, p
);
197 else if (v
== PyExc_StopIteration
) {
198 w_byte(TYPE_STOPITER
, p
);
200 else if (v
== Py_Ellipsis
) {
201 w_byte(TYPE_ELLIPSIS
, p
);
203 else if (v
== Py_False
) {
204 w_byte(TYPE_FALSE
, p
);
206 else if (v
== Py_True
) {
207 w_byte(TYPE_TRUE
, p
);
209 else if (PyInt_CheckExact(v
)) {
210 long x
= PyInt_AS_LONG((PyIntObject
*)v
);
212 long y
= Py_ARITHMETIC_RIGHT_SHIFT(long, x
, 31);
214 w_byte(TYPE_INT64
, p
);
224 else if (PyLong_CheckExact(v
)) {
225 PyLongObject
*ob
= (PyLongObject
*)v
;
228 else if (PyFloat_CheckExact(v
)) {
229 if (p
->version
> 1) {
230 unsigned char buf
[8];
231 if (_PyFloat_Pack8(PyFloat_AsDouble(v
),
233 p
->error
= WFERR_UNMARSHALLABLE
;
236 w_byte(TYPE_BINARY_FLOAT
, p
);
237 w_string((char*)buf
, 8, p
);
240 char *buf
= PyOS_double_to_string(PyFloat_AS_DOUBLE(v
),
243 p
->error
= WFERR_NOMEMORY
;
247 w_byte(TYPE_FLOAT
, p
);
249 w_string(buf
, (int)n
, p
);
253 #ifndef WITHOUT_COMPLEX
254 else if (PyComplex_CheckExact(v
)) {
255 if (p
->version
> 1) {
256 unsigned char buf
[8];
257 if (_PyFloat_Pack8(PyComplex_RealAsDouble(v
),
259 p
->error
= WFERR_UNMARSHALLABLE
;
262 w_byte(TYPE_BINARY_COMPLEX
, p
);
263 w_string((char*)buf
, 8, p
);
264 if (_PyFloat_Pack8(PyComplex_ImagAsDouble(v
),
266 p
->error
= WFERR_UNMARSHALLABLE
;
269 w_string((char*)buf
, 8, p
);
273 w_byte(TYPE_COMPLEX
, p
);
274 buf
= PyOS_double_to_string(PyComplex_RealAsDouble(v
),
277 p
->error
= WFERR_NOMEMORY
;
282 w_string(buf
, (int)n
, p
);
284 buf
= PyOS_double_to_string(PyComplex_ImagAsDouble(v
),
287 p
->error
= WFERR_NOMEMORY
;
292 w_string(buf
, (int)n
, p
);
297 else if (PyString_CheckExact(v
)) {
298 if (p
->strings
&& PyString_CHECK_INTERNED(v
)) {
299 PyObject
*o
= PyDict_GetItem(p
->strings
, v
);
301 long w
= PyInt_AsLong(o
);
302 w_byte(TYPE_STRINGREF
, p
);
308 o
= PyInt_FromSsize_t(PyDict_Size(p
->strings
));
310 PyDict_SetItem(p
->strings
, v
, o
) >= 0;
314 p
->error
= WFERR_UNMARSHALLABLE
;
317 w_byte(TYPE_INTERNED
, p
);
321 w_byte(TYPE_STRING
, p
);
323 n
= PyString_GET_SIZE(v
);
325 /* huge strings are not supported */
327 p
->error
= WFERR_UNMARSHALLABLE
;
331 w_string(PyString_AS_STRING(v
), (int)n
, p
);
333 #ifdef Py_USING_UNICODE
334 else if (PyUnicode_CheckExact(v
)) {
336 utf8
= PyUnicode_AsUTF8String(v
);
339 p
->error
= WFERR_UNMARSHALLABLE
;
342 w_byte(TYPE_UNICODE
, p
);
343 n
= PyString_GET_SIZE(utf8
);
346 p
->error
= WFERR_UNMARSHALLABLE
;
350 w_string(PyString_AS_STRING(utf8
), (int)n
, p
);
354 else if (PyTuple_CheckExact(v
)) {
355 w_byte(TYPE_TUPLE
, p
);
358 for (i
= 0; i
< n
; i
++) {
359 w_object(PyTuple_GET_ITEM(v
, i
), p
);
362 else if (PyList_CheckExact(v
)) {
363 w_byte(TYPE_LIST
, p
);
364 n
= PyList_GET_SIZE(v
);
366 for (i
= 0; i
< n
; i
++) {
367 w_object(PyList_GET_ITEM(v
, i
), p
);
370 else if (PyDict_CheckExact(v
)) {
372 PyObject
*key
, *value
;
373 w_byte(TYPE_DICT
, p
);
374 /* This one is NULL object terminated! */
376 while (PyDict_Next(v
, &pos
, &key
, &value
)) {
380 w_object((PyObject
*)NULL
, p
);
382 else if (PyAnySet_CheckExact(v
)) {
383 PyObject
*value
, *it
;
385 if (PyObject_TypeCheck(v
, &PySet_Type
))
388 w_byte(TYPE_FROZENSET
, p
);
389 n
= PyObject_Size(v
);
392 p
->error
= WFERR_UNMARSHALLABLE
;
396 it
= PyObject_GetIter(v
);
399 p
->error
= WFERR_UNMARSHALLABLE
;
402 while ((value
= PyIter_Next(it
)) != NULL
) {
407 if (PyErr_Occurred()) {
409 p
->error
= WFERR_UNMARSHALLABLE
;
413 else if (PyCode_Check(v
)) {
414 PyCodeObject
*co
= (PyCodeObject
*)v
;
415 w_byte(TYPE_CODE
, p
);
416 w_long(co
->co_argcount
, p
);
417 w_long(co
->co_nlocals
, p
);
418 w_long(co
->co_stacksize
, p
);
419 w_long(co
->co_flags
, p
);
420 w_object(co
->co_code
, p
);
421 w_object(co
->co_consts
, p
);
422 w_object(co
->co_names
, p
);
423 w_object(co
->co_varnames
, p
);
424 w_object(co
->co_freevars
, p
);
425 w_object(co
->co_cellvars
, p
);
426 w_object(co
->co_filename
, p
);
427 w_object(co
->co_name
, p
);
428 w_long(co
->co_firstlineno
, p
);
429 w_object(co
->co_lnotab
, p
);
431 else if (PyObject_CheckReadBuffer(v
)) {
432 /* Write unknown buffer-style objects as a string */
434 PyBufferProcs
*pb
= v
->ob_type
->tp_as_buffer
;
435 w_byte(TYPE_STRING
, p
);
436 n
= (*pb
->bf_getreadbuffer
)(v
, 0, (void **)&s
);
439 p
->error
= WFERR_UNMARSHALLABLE
;
443 w_string(s
, (int)n
, p
);
446 w_byte(TYPE_UNKNOWN
, p
);
447 p
->error
= WFERR_UNMARSHALLABLE
;
453 /* version currently has no effect for writing longs. */
455 PyMarshal_WriteLongToFile(long x
, FILE *fp
, int version
)
462 wf
.version
= version
;
467 PyMarshal_WriteObjectToFile(PyObject
*x
, FILE *fp
, int version
)
473 wf
.strings
= (version
> 0) ? PyDict_New() : NULL
;
474 wf
.version
= version
;
476 Py_XDECREF(wf
.strings
);
479 typedef WFILE RFILE
; /* Same struct with different invariants */
481 #define rs_byte(p) (((p)->ptr < (p)->end) ? (unsigned char)*(p)->ptr++ : EOF)
483 #define r_byte(p) ((p)->fp ? getc((p)->fp) : rs_byte(p))
486 r_string(char *s
, int n
, RFILE
*p
)
489 /* The result fits into int because it must be <=n. */
490 return (int)fread(s
, 1, n
, p
->fp
);
491 if (p
->end
- p
->ptr
< n
)
492 n
= (int)(p
->end
- p
->ptr
);
493 memcpy(s
, p
->ptr
, n
);
504 /* Sign-extension, in case short greater than 16 bits */
513 register FILE *fp
= p
->fp
;
516 x
|= (long)getc(fp
) << 8;
517 x
|= (long)getc(fp
) << 16;
518 x
|= (long)getc(fp
) << 24;
522 x
|= (long)rs_byte(p
) << 8;
523 x
|= (long)rs_byte(p
) << 16;
524 x
|= (long)rs_byte(p
) << 24;
527 /* Sign extension for 64-bit machines */
528 x
|= -(x
& 0x80000000L
);
533 /* r_long64 deals with the TYPE_INT64 code. On a machine with
534 sizeof(long) > 4, it returns a Python int object, else a Python long
535 object. Note that w_long64 writes out TYPE_INT if 32 bits is enough,
536 so there's no inefficiency here in returning a PyLong on 32-bit boxes
537 for everything written via TYPE_INT64 (i.e., if an int is written via
538 TYPE_INT64, it *needs* more than 32 bits).
543 long lo4
= r_long(p
);
544 long hi4
= r_long(p
);
546 long x
= (hi4
<< 32) | (lo4
& 0xFFFFFFFFL
);
547 return PyInt_FromLong(x
);
549 unsigned char buf
[8];
551 int is_little_endian
= (int)*(char*)&one
;
552 if (is_little_endian
) {
553 memcpy(buf
, &lo4
, 4);
554 memcpy(buf
+4, &hi4
, 4);
557 memcpy(buf
, &hi4
, 4);
558 memcpy(buf
+4, &lo4
, 4);
560 return _PyLong_FromByteArray(buf
, 8, is_little_endian
, 1);
568 int size
, i
, j
, md
, shorts_in_top_digit
;
574 return (PyObject
*)_PyLong_New(0);
575 if (n
< -INT_MAX
|| n
> INT_MAX
) {
576 PyErr_SetString(PyExc_ValueError
,
577 "bad marshal data (long size out of range)");
581 size
= 1 + (ABS(n
) - 1) / PyLong_MARSHAL_RATIO
;
582 shorts_in_top_digit
= 1 + (ABS(n
) - 1) % PyLong_MARSHAL_RATIO
;
583 ob
= _PyLong_New(size
);
586 Py_SIZE(ob
) = n
> 0 ? size
: -size
;
588 for (i
= 0; i
< size
-1; i
++) {
590 for (j
=0; j
< PyLong_MARSHAL_RATIO
; j
++) {
592 if (md
< 0 || md
> PyLong_MARSHAL_BASE
)
594 d
+= (digit
)md
<< j
*PyLong_MARSHAL_SHIFT
;
599 for (j
=0; j
< shorts_in_top_digit
; j
++) {
601 if (md
< 0 || md
> PyLong_MARSHAL_BASE
)
603 /* topmost marshal digit should be nonzero */
604 if (md
== 0 && j
== shorts_in_top_digit
- 1) {
606 PyErr_SetString(PyExc_ValueError
,
607 "bad marshal data (unnormalized long data)");
610 d
+= (digit
)md
<< j
*PyLong_MARSHAL_SHIFT
;
612 /* top digit should be nonzero, else the resulting PyLong won't be
614 ob
->ob_digit
[size
-1] = d
;
615 return (PyObject
*)ob
;
618 PyErr_SetString(PyExc_ValueError
,
619 "bad marshal data (digit out of range in long)");
627 /* NULL is a valid return value, it does not necessarily means that
628 an exception is set. */
631 int type
= r_byte(p
);
636 if (p
->depth
> MAX_MARSHAL_STACK_DEPTH
) {
638 PyErr_SetString(PyExc_ValueError
, "recursion limit exceeded");
645 PyErr_SetString(PyExc_EOFError
,
646 "EOF read where object expected");
660 Py_INCREF(PyExc_StopIteration
);
661 retval
= PyExc_StopIteration
;
665 Py_INCREF(Py_Ellipsis
);
666 retval
= Py_Ellipsis
;
680 retval
= PyInt_FromLong(r_long(p
));
684 retval
= r_long64(p
);
688 retval
= r_PyLong(p
);
696 if (n
== EOF
|| r_string(buf
, (int)n
, p
) != n
) {
697 PyErr_SetString(PyExc_EOFError
,
698 "EOF read where object expected");
703 dx
= PyOS_string_to_double(buf
, NULL
, NULL
);
704 if (dx
== -1.0 && PyErr_Occurred()) {
708 retval
= PyFloat_FromDouble(dx
);
712 case TYPE_BINARY_FLOAT
:
714 unsigned char buf
[8];
716 if (r_string((char*)buf
, 8, p
) != 8) {
717 PyErr_SetString(PyExc_EOFError
,
718 "EOF read where object expected");
722 x
= _PyFloat_Unpack8(buf
, 1);
723 if (x
== -1.0 && PyErr_Occurred()) {
727 retval
= PyFloat_FromDouble(x
);
731 #ifndef WITHOUT_COMPLEX
737 if (n
== EOF
|| r_string(buf
, (int)n
, p
) != n
) {
738 PyErr_SetString(PyExc_EOFError
,
739 "EOF read where object expected");
744 c
.real
= PyOS_string_to_double(buf
, NULL
, NULL
);
745 if (c
.real
== -1.0 && PyErr_Occurred()) {
750 if (n
== EOF
|| r_string(buf
, (int)n
, p
) != n
) {
751 PyErr_SetString(PyExc_EOFError
,
752 "EOF read where object expected");
757 c
.imag
= PyOS_string_to_double(buf
, NULL
, NULL
);
758 if (c
.imag
== -1.0 && PyErr_Occurred()) {
762 retval
= PyComplex_FromCComplex(c
);
766 case TYPE_BINARY_COMPLEX
:
768 unsigned char buf
[8];
770 if (r_string((char*)buf
, 8, p
) != 8) {
771 PyErr_SetString(PyExc_EOFError
,
772 "EOF read where object expected");
776 c
.real
= _PyFloat_Unpack8(buf
, 1);
777 if (c
.real
== -1.0 && PyErr_Occurred()) {
781 if (r_string((char*)buf
, 8, p
) != 8) {
782 PyErr_SetString(PyExc_EOFError
,
783 "EOF read where object expected");
787 c
.imag
= _PyFloat_Unpack8(buf
, 1);
788 if (c
.imag
== -1.0 && PyErr_Occurred()) {
792 retval
= PyComplex_FromCComplex(c
);
800 if (n
< 0 || n
> INT_MAX
) {
801 PyErr_SetString(PyExc_ValueError
, "bad marshal data (string size out of range)");
805 v
= PyString_FromStringAndSize((char *)NULL
, n
);
810 if (r_string(PyString_AS_STRING(v
), (int)n
, p
) != n
) {
812 PyErr_SetString(PyExc_EOFError
,
813 "EOF read where object expected");
817 if (type
== TYPE_INTERNED
) {
818 PyString_InternInPlace(&v
);
819 if (PyList_Append(p
->strings
, v
) < 0) {
829 if (n
< 0 || n
>= PyList_GET_SIZE(p
->strings
)) {
830 PyErr_SetString(PyExc_ValueError
, "bad marshal data (string ref out of range)");
834 v
= PyList_GET_ITEM(p
->strings
, n
);
839 #ifdef Py_USING_UNICODE
845 if (n
< 0 || n
> INT_MAX
) {
846 PyErr_SetString(PyExc_ValueError
, "bad marshal data (unicode size out of range)");
850 buffer
= PyMem_NEW(char, n
);
851 if (buffer
== NULL
) {
852 retval
= PyErr_NoMemory();
855 if (r_string(buffer
, (int)n
, p
) != n
) {
857 PyErr_SetString(PyExc_EOFError
,
858 "EOF read where object expected");
862 v
= PyUnicode_DecodeUTF8(buffer
, n
, NULL
);
871 if (n
< 0 || n
> INT_MAX
) {
872 PyErr_SetString(PyExc_ValueError
, "bad marshal data (tuple size out of range)");
876 v
= PyTuple_New((int)n
);
881 for (i
= 0; i
< n
; i
++) {
884 if (!PyErr_Occurred())
885 PyErr_SetString(PyExc_TypeError
,
886 "NULL object in marshal data for tuple");
891 PyTuple_SET_ITEM(v
, (int)i
, v2
);
898 if (n
< 0 || n
> INT_MAX
) {
899 PyErr_SetString(PyExc_ValueError
, "bad marshal data (list size out of range)");
903 v
= PyList_New((int)n
);
908 for (i
= 0; i
< n
; i
++) {
911 if (!PyErr_Occurred())
912 PyErr_SetString(PyExc_TypeError
,
913 "NULL object in marshal data for list");
918 PyList_SET_ITEM(v
, (int)i
, v2
);
936 PyDict_SetItem(v
, key
, val
);
940 if (PyErr_Occurred()) {
950 if (n
< 0 || n
> INT_MAX
) {
951 PyErr_SetString(PyExc_ValueError
, "bad marshal data (set size out of range)");
955 v
= (type
== TYPE_SET
) ? PySet_New(NULL
) : PyFrozenSet_New(NULL
);
960 for (i
= 0; i
< n
; i
++) {
963 if (!PyErr_Occurred())
964 PyErr_SetString(PyExc_TypeError
,
965 "NULL object in marshal data for set");
970 if (PySet_Add(v
, v2
) == -1) {
982 if (PyEval_GetRestricted()) {
983 PyErr_SetString(PyExc_RuntimeError
,
984 "cannot unmarshal code objects in "
985 "restricted execution mode");
994 PyObject
*code
= NULL
;
995 PyObject
*consts
= NULL
;
996 PyObject
*names
= NULL
;
997 PyObject
*varnames
= NULL
;
998 PyObject
*freevars
= NULL
;
999 PyObject
*cellvars
= NULL
;
1000 PyObject
*filename
= NULL
;
1001 PyObject
*name
= NULL
;
1003 PyObject
*lnotab
= NULL
;
1007 /* XXX ignore long->int overflows for now */
1008 argcount
= (int)r_long(p
);
1009 nlocals
= (int)r_long(p
);
1010 stacksize
= (int)r_long(p
);
1011 flags
= (int)r_long(p
);
1015 consts
= r_object(p
);
1018 names
= r_object(p
);
1021 varnames
= r_object(p
);
1022 if (varnames
== NULL
)
1024 freevars
= r_object(p
);
1025 if (freevars
== NULL
)
1027 cellvars
= r_object(p
);
1028 if (cellvars
== NULL
)
1030 filename
= r_object(p
);
1031 if (filename
== NULL
)
1036 firstlineno
= (int)r_long(p
);
1037 lnotab
= r_object(p
);
1041 v
= (PyObject
*) PyCode_New(
1042 argcount
, nlocals
, stacksize
, flags
,
1043 code
, consts
, names
, varnames
,
1044 freevars
, cellvars
, filename
, name
,
1045 firstlineno
, lnotab
);
1051 Py_XDECREF(varnames
);
1052 Py_XDECREF(freevars
);
1053 Py_XDECREF(cellvars
);
1054 Py_XDECREF(filename
);
1063 /* Bogus data got written, which isn't ideal.
1064 This will let you keep working and recover. */
1065 PyErr_SetString(PyExc_ValueError
, "bad marshal data (unknown type code)");
1075 read_object(RFILE
*p
)
1078 if (PyErr_Occurred()) {
1079 fprintf(stderr
, "XXX readobject called with exception set\n");
1083 if (v
== NULL
&& !PyErr_Occurred())
1084 PyErr_SetString(PyExc_TypeError
, "NULL object in marshal data for object");
1089 PyMarshal_ReadShortFromFile(FILE *fp
)
1095 rf
.end
= rf
.ptr
= NULL
;
1096 return r_short(&rf
);
1100 PyMarshal_ReadLongFromFile(FILE *fp
)
1105 rf
.ptr
= rf
.end
= NULL
;
1110 /* Return size of file in bytes; < 0 if unknown. */
1112 getfilesize(FILE *fp
)
1115 if (fstat(fileno(fp
), &st
) != 0)
1122 /* If we can get the size of the file up-front, and it's reasonably small,
1123 * read it in one gulp and delegate to ...FromString() instead. Much quicker
1124 * than reading a byte at a time from file; speeds .pyc imports.
1125 * CAUTION: since this may read the entire remainder of the file, don't
1126 * call it unless you know you're done with the file.
1129 PyMarshal_ReadLastObjectFromFile(FILE *fp
)
1131 /* REASONABLE_FILE_LIMIT is by defn something big enough for Tkinter.pyc. */
1132 #define REASONABLE_FILE_LIMIT (1L << 18)
1135 filesize
= getfilesize(fp
);
1136 if (filesize
> 0 && filesize
<= REASONABLE_FILE_LIMIT
) {
1137 char* pBuf
= (char *)PyMem_MALLOC(filesize
);
1141 /* filesize must fit into an int, because it
1142 is smaller than REASONABLE_FILE_LIMIT */
1143 n
= fread(pBuf
, 1, (int)filesize
, fp
);
1144 v
= PyMarshal_ReadObjectFromString(pBuf
, n
);
1151 /* We don't have fstat, or we do but the file is larger than
1152 * REASONABLE_FILE_LIMIT or malloc failed -- read a byte at a time.
1154 return PyMarshal_ReadObjectFromFile(fp
);
1156 #undef REASONABLE_FILE_LIMIT
1160 PyMarshal_ReadObjectFromFile(FILE *fp
)
1165 rf
.strings
= PyList_New(0);
1167 rf
.ptr
= rf
.end
= NULL
;
1168 result
= r_object(&rf
);
1169 Py_DECREF(rf
.strings
);
1174 PyMarshal_ReadObjectFromString(char *str
, Py_ssize_t len
)
1181 rf
.strings
= PyList_New(0);
1183 result
= r_object(&rf
);
1184 Py_DECREF(rf
.strings
);
1189 set_error(int error
)
1192 case WFERR_NOMEMORY
:
1195 case WFERR_UNMARSHALLABLE
:
1196 PyErr_SetString(PyExc_ValueError
, "unmarshallable object");
1198 case WFERR_NESTEDTOODEEP
:
1200 PyErr_SetString(PyExc_ValueError
,
1201 "object too deeply nested to marshal");
1207 PyMarshal_WriteObjectToString(PyObject
*x
, int version
)
1211 wf
.str
= PyString_FromStringAndSize((char *)NULL
, 50);
1214 wf
.ptr
= PyString_AS_STRING((PyStringObject
*)wf
.str
);
1215 wf
.end
= wf
.ptr
+ PyString_Size(wf
.str
);
1216 wf
.error
= WFERR_OK
;
1218 wf
.version
= version
;
1219 wf
.strings
= (version
> 0) ? PyDict_New() : NULL
;
1221 Py_XDECREF(wf
.strings
);
1222 if (wf
.str
!= NULL
) {
1223 char *base
= PyString_AS_STRING((PyStringObject
*)wf
.str
);
1224 if (wf
.ptr
- base
> PY_SSIZE_T_MAX
) {
1226 PyErr_SetString(PyExc_OverflowError
,
1227 "too much marshall data for a string");
1230 if (_PyString_Resize(&wf
.str
, (Py_ssize_t
)(wf
.ptr
- base
)))
1233 if (wf
.error
!= WFERR_OK
) {
1235 set_error(wf
.error
);
1241 /* And an interface for Python programs... */
1244 marshal_dump(PyObject
*self
, PyObject
*args
)
1249 int version
= Py_MARSHAL_VERSION
;
1250 if (!PyArg_ParseTuple(args
, "OO|i:dump", &x
, &f
, &version
))
1252 if (!PyFile_Check(f
)) {
1253 PyErr_SetString(PyExc_TypeError
,
1254 "marshal.dump() 2nd arg must be file");
1257 wf
.fp
= PyFile_AsFile(f
);
1259 wf
.ptr
= wf
.end
= NULL
;
1260 wf
.error
= WFERR_OK
;
1262 wf
.strings
= (version
> 0) ? PyDict_New() : 0;
1263 wf
.version
= version
;
1265 Py_XDECREF(wf
.strings
);
1266 if (wf
.error
!= WFERR_OK
) {
1267 set_error(wf
.error
);
1274 PyDoc_STRVAR(dump_doc
,
1275 "dump(value, file[, version])\n\
1277 Write the value on the open file. The value must be a supported type.\n\
1278 The file must be an open file object such as sys.stdout or returned by\n\
1279 open() or os.popen(). It must be opened in binary mode ('wb' or 'w+b').\n\
1281 If the value has (or contains an object that has) an unsupported type, a\n\
1282 ValueError exception is raised — but garbage data will also be written\n\
1283 to the file. The object will not be properly read back by load()\n\
1285 New in version 2.4: The version argument indicates the data format that\n\
1289 marshal_load(PyObject
*self
, PyObject
*f
)
1293 if (!PyFile_Check(f
)) {
1294 PyErr_SetString(PyExc_TypeError
,
1295 "marshal.load() arg must be file");
1298 rf
.fp
= PyFile_AsFile(f
);
1299 rf
.strings
= PyList_New(0);
1301 result
= read_object(&rf
);
1302 Py_DECREF(rf
.strings
);
1306 PyDoc_STRVAR(load_doc
,
1309 Read one value from the open file and return it. If no valid value is\n\
1310 read (e.g. because the data has a different Python version’s\n\
1311 incompatible marshal format), raise EOFError, ValueError or TypeError.\n\
1312 The file must be an open file object opened in binary mode ('rb' or\n\
1315 Note: If an object containing an unsupported type was marshalled with\n\
1316 dump(), load() will substitute None for the unmarshallable type.");
1320 marshal_dumps(PyObject
*self
, PyObject
*args
)
1323 int version
= Py_MARSHAL_VERSION
;
1324 if (!PyArg_ParseTuple(args
, "O|i:dumps", &x
, &version
))
1326 return PyMarshal_WriteObjectToString(x
, version
);
1329 PyDoc_STRVAR(dumps_doc
,
1330 "dumps(value[, version])\n\
1332 Return the string that would be written to a file by dump(value, file).\n\
1333 The value must be a supported type. Raise a ValueError exception if\n\
1334 value has (or contains an object that has) an unsupported type.\n\
1336 New in version 2.4: The version argument indicates the data format that\n\
1337 dumps should use.");
1341 marshal_loads(PyObject
*self
, PyObject
*args
)
1347 if (!PyArg_ParseTuple(args
, "s#:loads", &s
, &n
))
1352 rf
.strings
= PyList_New(0);
1354 result
= read_object(&rf
);
1355 Py_DECREF(rf
.strings
);
1359 PyDoc_STRVAR(loads_doc
,
1362 Convert the string to a value. If no valid value is found, raise\n\
1363 EOFError, ValueError or TypeError. Extra characters in the string are\n\
1366 static PyMethodDef marshal_methods
[] = {
1367 {"dump", marshal_dump
, METH_VARARGS
, dump_doc
},
1368 {"load", marshal_load
, METH_O
, load_doc
},
1369 {"dumps", marshal_dumps
, METH_VARARGS
, dumps_doc
},
1370 {"loads", marshal_loads
, METH_VARARGS
, loads_doc
},
1371 {NULL
, NULL
} /* sentinel */
1374 PyDoc_STRVAR(marshal_doc
,
1375 "This module contains functions that can read and write Python values in\n\
1376 a binary format. The format is specific to Python, but independent of\n\
1377 machine architecture issues.\n\
1379 Not all Python object types are supported; in general, only objects\n\
1380 whose value is independent from a particular invocation of Python can be\n\
1381 written and read by this module. The following types are supported:\n\
1382 None, integers, long integers, floating point numbers, strings, Unicode\n\
1383 objects, tuples, lists, sets, dictionaries, and code objects, where it\n\
1384 should be understood that tuples, lists and dictionaries are only\n\
1385 supported as long as the values contained therein are themselves\n\
1386 supported; and recursive lists and dictionaries should not be written\n\
1387 (they will cause infinite loops).\n\
1391 version -- indicates the format that the module uses. Version 0 is the\n\
1392 historical format, version 1 (added in Python 2.4) shares interned\n\
1393 strings and version 2 (added in Python 2.5) uses a binary format for\n\
1394 floating point numbers. (New in version 2.4)\n\
1398 dump() -- write value to a file\n\
1399 load() -- read value from a file\n\
1400 dumps() -- write value to a string\n\
1401 loads() -- read value from a string");
1405 PyMarshal_Init(void)
1407 PyObject
*mod
= Py_InitModule3("marshal", marshal_methods
,
1411 PyModule_AddIntConstant(mod
, "version", Py_MARSHAL_VERSION
);