2 Write Python objects to files and read them back.
3 This is intended for writing and reading compiled Python code only;
4 a true persistent storage facility would be much harder, since
5 it would have to take circular links and sharing into account.
7 Copyright (c) 2011, Intel Corporation. All rights reserved.<BR>
8 This program and the accompanying materials are licensed and made available under
9 the terms and conditions of the BSD License that accompanies this distribution.
10 The full text of the license may be found at
11 http://opensource.org/licenses/bsd-license.
13 THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS,
14 WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED.
17 #define PY_SSIZE_T_CLEAN
20 #include "longintrepr.h"
25 #define ABS(x) ((x) < 0 ? -(x) : (x))
28 /* High water mark to determine when the marshalled object is dangerously deep
29 * and risks coring the interpreter. When the object stack gets this deep,
30 * raise an exception instead of continuing.
32 #define MAX_MARSHAL_STACK_DEPTH 2000
36 #define TYPE_FALSE 'F'
38 #define TYPE_STOPITER 'S'
39 #define TYPE_ELLIPSIS '.'
41 #define TYPE_INT64 'I'
42 #define TYPE_FLOAT 'f'
43 #define TYPE_BINARY_FLOAT 'g'
44 #define TYPE_COMPLEX 'x'
45 #define TYPE_BINARY_COMPLEX 'y'
47 #define TYPE_STRING 's'
48 #define TYPE_INTERNED 't'
49 #define TYPE_STRINGREF 'R'
50 #define TYPE_TUPLE '('
54 #define TYPE_UNICODE 'u'
55 #define TYPE_UNKNOWN '?'
57 #define TYPE_FROZENSET '>'
60 #define WFERR_UNMARSHALLABLE 1
61 #define WFERR_NESTEDTOODEEP 2
62 #define WFERR_NOMEMORY 3
66 int error
; /* see WFERR_* values */
68 /* If fp == NULL, the following are valid: */
72 PyObject
*strings
; /* dict on marshal, list on unmarshal */
76 #define w_byte(c, p) if (((p)->fp)) putc((c), (p)->fp); \
77 else if ((p)->ptr != (p)->end) *(p)->ptr++ = (c); \
81 w_more(int c
, WFILE
*p
)
83 Py_ssize_t size
, newsize
;
85 return; /* An error already occurred */
86 size
= PyString_Size(p
->str
);
87 newsize
= size
+ size
+ 1024;
88 if (newsize
> 32*1024*1024) {
89 newsize
= size
+ (size
>> 3); /* 12.5% overallocation */
91 if (_PyString_Resize(&p
->str
, newsize
) != 0) {
92 p
->ptr
= p
->end
= NULL
;
95 p
->ptr
= PyString_AS_STRING((PyStringObject
*)p
->str
) + size
;
97 PyString_AS_STRING((PyStringObject
*)p
->str
) + newsize
;
98 *p
->ptr
++ = Py_SAFE_DOWNCAST(c
, int, char);
103 w_string(char *s
, int n
, WFILE
*p
)
106 fwrite(s
, 1, n
, p
->fp
);
117 w_short(int x
, WFILE
*p
)
119 w_byte((char)( x
& 0xff), p
);
120 w_byte((char)((x
>> 8) & 0xff), p
);
124 w_long(long x
, WFILE
*p
)
126 w_byte((char)( x
& 0xff), p
);
127 w_byte((char)((x
>> 8) & 0xff), p
);
128 w_byte((char)((x
>>16) & 0xff), p
);
129 w_byte((char)((x
>>24) & 0xff), p
);
134 w_long64(long x
, WFILE
*p
)
141 /* We assume that Python longs are stored internally in base some power of
142 2**15; for the sake of portability we'll always read and write them in base
145 #define PyLong_MARSHAL_SHIFT 15
146 #define PyLong_MARSHAL_BASE ((short)1 << PyLong_MARSHAL_SHIFT)
147 #define PyLong_MARSHAL_MASK (PyLong_MARSHAL_BASE - 1)
148 #if PyLong_SHIFT % PyLong_MARSHAL_SHIFT != 0
149 #error "PyLong_SHIFT must be a multiple of PyLong_MARSHAL_SHIFT"
151 #define PyLong_MARSHAL_RATIO (PyLong_SHIFT / PyLong_MARSHAL_SHIFT)
154 w_PyLong(const PyLongObject
*ob
, WFILE
*p
)
156 Py_ssize_t i
, j
, n
, l
;
159 w_byte(TYPE_LONG
, p
);
160 if (Py_SIZE(ob
) == 0) {
165 /* set l to number of base PyLong_MARSHAL_BASE digits */
166 n
= ABS(Py_SIZE(ob
));
167 l
= (n
-1) * PyLong_MARSHAL_RATIO
;
168 d
= ob
->ob_digit
[n
-1];
169 assert(d
!= 0); /* a PyLong is always normalized */
171 d
>>= PyLong_MARSHAL_SHIFT
;
174 w_long((long)(Py_SIZE(ob
) > 0 ? l
: -l
), p
);
176 for (i
=0; i
< n
-1; i
++) {
178 for (j
=0; j
< PyLong_MARSHAL_RATIO
; j
++) {
179 w_short(d
& PyLong_MARSHAL_MASK
, p
);
180 d
>>= PyLong_MARSHAL_SHIFT
;
184 d
= ob
->ob_digit
[n
-1];
186 w_short(d
& PyLong_MARSHAL_MASK
, p
);
187 d
>>= PyLong_MARSHAL_SHIFT
;
192 w_object(PyObject
*v
, WFILE
*p
)
198 if (p
->depth
> MAX_MARSHAL_STACK_DEPTH
) {
199 p
->error
= WFERR_NESTEDTOODEEP
;
201 else if (v
== NULL
) {
202 w_byte(TYPE_NULL
, p
);
204 else if (v
== Py_None
) {
205 w_byte(TYPE_NONE
, p
);
207 else if (v
== PyExc_StopIteration
) {
208 w_byte(TYPE_STOPITER
, p
);
210 else if (v
== Py_Ellipsis
) {
211 w_byte(TYPE_ELLIPSIS
, p
);
213 else if (v
== Py_False
) {
214 w_byte(TYPE_FALSE
, p
);
216 else if (v
== Py_True
) {
217 w_byte(TYPE_TRUE
, p
);
219 else if (PyInt_CheckExact(v
)) {
220 long x
= PyInt_AS_LONG((PyIntObject
*)v
);
222 long y
= Py_ARITHMETIC_RIGHT_SHIFT(long, x
, 31);
224 w_byte(TYPE_INT64
, p
);
234 else if (PyLong_CheckExact(v
)) {
235 PyLongObject
*ob
= (PyLongObject
*)v
;
238 else if (PyFloat_CheckExact(v
)) {
239 if (p
->version
> 1) {
240 unsigned char buf
[8];
241 if (_PyFloat_Pack8(PyFloat_AsDouble(v
),
243 p
->error
= WFERR_UNMARSHALLABLE
;
246 w_byte(TYPE_BINARY_FLOAT
, p
);
247 w_string((char*)buf
, 8, p
);
250 char *buf
= PyOS_double_to_string(PyFloat_AS_DOUBLE(v
),
253 p
->error
= WFERR_NOMEMORY
;
257 w_byte(TYPE_FLOAT
, p
);
259 w_string(buf
, (int)n
, p
);
263 #ifndef WITHOUT_COMPLEX
264 else if (PyComplex_CheckExact(v
)) {
265 if (p
->version
> 1) {
266 unsigned char buf
[8];
267 if (_PyFloat_Pack8(PyComplex_RealAsDouble(v
),
269 p
->error
= WFERR_UNMARSHALLABLE
;
272 w_byte(TYPE_BINARY_COMPLEX
, p
);
273 w_string((char*)buf
, 8, p
);
274 if (_PyFloat_Pack8(PyComplex_ImagAsDouble(v
),
276 p
->error
= WFERR_UNMARSHALLABLE
;
279 w_string((char*)buf
, 8, p
);
283 w_byte(TYPE_COMPLEX
, p
);
284 buf
= PyOS_double_to_string(PyComplex_RealAsDouble(v
),
287 p
->error
= WFERR_NOMEMORY
;
292 w_string(buf
, (int)n
, p
);
294 buf
= PyOS_double_to_string(PyComplex_ImagAsDouble(v
),
297 p
->error
= WFERR_NOMEMORY
;
302 w_string(buf
, (int)n
, p
);
307 else if (PyString_CheckExact(v
)) {
308 if (p
->strings
&& PyString_CHECK_INTERNED(v
)) {
309 PyObject
*o
= PyDict_GetItem(p
->strings
, v
);
311 long w
= PyInt_AsLong(o
);
312 w_byte(TYPE_STRINGREF
, p
);
318 o
= PyInt_FromSsize_t(PyDict_Size(p
->strings
));
320 PyDict_SetItem(p
->strings
, v
, o
) >= 0;
324 p
->error
= WFERR_UNMARSHALLABLE
;
327 w_byte(TYPE_INTERNED
, p
);
331 w_byte(TYPE_STRING
, p
);
333 n
= PyString_GET_SIZE(v
);
335 /* huge strings are not supported */
337 p
->error
= WFERR_UNMARSHALLABLE
;
341 w_string(PyString_AS_STRING(v
), (int)n
, p
);
343 #ifdef Py_USING_UNICODE
344 else if (PyUnicode_CheckExact(v
)) {
346 utf8
= PyUnicode_AsUTF8String(v
);
349 p
->error
= WFERR_UNMARSHALLABLE
;
352 w_byte(TYPE_UNICODE
, p
);
353 n
= PyString_GET_SIZE(utf8
);
356 p
->error
= WFERR_UNMARSHALLABLE
;
360 w_string(PyString_AS_STRING(utf8
), (int)n
, p
);
364 else if (PyTuple_CheckExact(v
)) {
365 w_byte(TYPE_TUPLE
, p
);
368 for (i
= 0; i
< n
; i
++) {
369 w_object(PyTuple_GET_ITEM(v
, i
), p
);
372 else if (PyList_CheckExact(v
)) {
373 w_byte(TYPE_LIST
, p
);
374 n
= PyList_GET_SIZE(v
);
376 for (i
= 0; i
< n
; i
++) {
377 w_object(PyList_GET_ITEM(v
, i
), p
);
380 else if (PyDict_CheckExact(v
)) {
382 PyObject
*key
, *value
;
383 w_byte(TYPE_DICT
, p
);
384 /* This one is NULL object terminated! */
386 while (PyDict_Next(v
, &pos
, &key
, &value
)) {
390 w_object((PyObject
*)NULL
, p
);
392 else if (PyAnySet_CheckExact(v
)) {
393 PyObject
*value
, *it
;
395 if (PyObject_TypeCheck(v
, &PySet_Type
))
398 w_byte(TYPE_FROZENSET
, p
);
399 n
= PyObject_Size(v
);
402 p
->error
= WFERR_UNMARSHALLABLE
;
406 it
= PyObject_GetIter(v
);
409 p
->error
= WFERR_UNMARSHALLABLE
;
412 while ((value
= PyIter_Next(it
)) != NULL
) {
417 if (PyErr_Occurred()) {
419 p
->error
= WFERR_UNMARSHALLABLE
;
423 else if (PyCode_Check(v
)) {
424 PyCodeObject
*co
= (PyCodeObject
*)v
;
425 w_byte(TYPE_CODE
, p
);
426 w_long(co
->co_argcount
, p
);
427 w_long(co
->co_nlocals
, p
);
428 w_long(co
->co_stacksize
, p
);
429 w_long(co
->co_flags
, p
);
430 w_object(co
->co_code
, p
);
431 w_object(co
->co_consts
, p
);
432 w_object(co
->co_names
, p
);
433 w_object(co
->co_varnames
, p
);
434 w_object(co
->co_freevars
, p
);
435 w_object(co
->co_cellvars
, p
);
436 w_object(co
->co_filename
, p
);
437 w_object(co
->co_name
, p
);
438 w_long(co
->co_firstlineno
, p
);
439 w_object(co
->co_lnotab
, p
);
441 else if (PyObject_CheckReadBuffer(v
)) {
442 /* Write unknown buffer-style objects as a string */
444 PyBufferProcs
*pb
= v
->ob_type
->tp_as_buffer
;
445 w_byte(TYPE_STRING
, p
);
446 n
= (*pb
->bf_getreadbuffer
)(v
, 0, (void **)&s
);
449 p
->error
= WFERR_UNMARSHALLABLE
;
453 w_string(s
, (int)n
, p
);
456 w_byte(TYPE_UNKNOWN
, p
);
457 p
->error
= WFERR_UNMARSHALLABLE
;
463 /* version currently has no effect for writing longs. */
465 PyMarshal_WriteLongToFile(long x
, FILE *fp
, int version
)
472 wf
.version
= version
;
477 PyMarshal_WriteObjectToFile(PyObject
*x
, FILE *fp
, int version
)
483 wf
.strings
= (version
> 0) ? PyDict_New() : NULL
;
484 wf
.version
= version
;
486 Py_XDECREF(wf
.strings
);
489 typedef WFILE RFILE
; /* Same struct with different invariants */
491 #define rs_byte(p) (((p)->ptr < (p)->end) ? (unsigned char)*(p)->ptr++ : EOF)
493 #define r_byte(p) ((p)->fp ? getc((p)->fp) : rs_byte(p))
496 r_string(char *s
, int n
, RFILE
*p
)
499 /* The result fits into int because it must be <=n. */
500 return (int)fread(s
, 1, n
, p
->fp
);
501 if (p
->end
- p
->ptr
< n
)
502 n
= (int)(p
->end
- p
->ptr
);
503 memcpy(s
, p
->ptr
, n
);
514 /* Sign-extension, in case short greater than 16 bits */
523 register FILE *fp
= p
->fp
;
526 x
|= (long)getc(fp
) << 8;
527 x
|= (long)getc(fp
) << 16;
528 x
|= (long)getc(fp
) << 24;
532 x
|= (long)rs_byte(p
) << 8;
533 x
|= (long)rs_byte(p
) << 16;
534 x
|= (long)rs_byte(p
) << 24;
537 /* Sign extension for 64-bit machines */
538 x
|= -(x
& 0x80000000L
);
543 /* r_long64 deals with the TYPE_INT64 code. On a machine with
544 sizeof(long) > 4, it returns a Python int object, else a Python long
545 object. Note that w_long64 writes out TYPE_INT if 32 bits is enough,
546 so there's no inefficiency here in returning a PyLong on 32-bit boxes
547 for everything written via TYPE_INT64 (i.e., if an int is written via
548 TYPE_INT64, it *needs* more than 32 bits).
553 long lo4
= r_long(p
);
554 long hi4
= r_long(p
);
556 long x
= (hi4
<< 32) | (lo4
& 0xFFFFFFFFL
);
557 return PyInt_FromLong(x
);
559 unsigned char buf
[8];
561 int is_little_endian
= (int)*(char*)&one
;
562 if (is_little_endian
) {
563 memcpy(buf
, &lo4
, 4);
564 memcpy(buf
+4, &hi4
, 4);
567 memcpy(buf
, &hi4
, 4);
568 memcpy(buf
+4, &lo4
, 4);
570 return _PyLong_FromByteArray(buf
, 8, is_little_endian
, 1);
578 int size
, i
, j
, md
, shorts_in_top_digit
;
584 return (PyObject
*)_PyLong_New(0);
585 if (n
< -INT_MAX
|| n
> INT_MAX
) {
586 PyErr_SetString(PyExc_ValueError
,
587 "bad marshal data (long size out of range)");
591 size
= 1 + (ABS(n
) - 1) / PyLong_MARSHAL_RATIO
;
592 shorts_in_top_digit
= 1 + (ABS(n
) - 1) % PyLong_MARSHAL_RATIO
;
593 ob
= _PyLong_New(size
);
596 Py_SIZE(ob
) = n
> 0 ? size
: -size
;
598 for (i
= 0; i
< size
-1; i
++) {
600 for (j
=0; j
< PyLong_MARSHAL_RATIO
; j
++) {
602 if (md
< 0 || md
> PyLong_MARSHAL_BASE
)
604 d
+= (digit
)md
<< j
*PyLong_MARSHAL_SHIFT
;
609 for (j
=0; j
< shorts_in_top_digit
; j
++) {
611 if (md
< 0 || md
> PyLong_MARSHAL_BASE
)
613 /* topmost marshal digit should be nonzero */
614 if (md
== 0 && j
== shorts_in_top_digit
- 1) {
616 PyErr_SetString(PyExc_ValueError
,
617 "bad marshal data (unnormalized long data)");
620 d
+= (digit
)md
<< j
*PyLong_MARSHAL_SHIFT
;
622 /* top digit should be nonzero, else the resulting PyLong won't be
624 ob
->ob_digit
[size
-1] = d
;
625 return (PyObject
*)ob
;
628 PyErr_SetString(PyExc_ValueError
,
629 "bad marshal data (digit out of range in long)");
637 /* NULL is a valid return value, it does not necessarily means that
638 an exception is set. */
641 int type
= r_byte(p
);
646 if (p
->depth
> MAX_MARSHAL_STACK_DEPTH
) {
648 PyErr_SetString(PyExc_ValueError
, "recursion limit exceeded");
655 PyErr_SetString(PyExc_EOFError
,
656 "EOF read where object expected");
670 Py_INCREF(PyExc_StopIteration
);
671 retval
= PyExc_StopIteration
;
675 Py_INCREF(Py_Ellipsis
);
676 retval
= Py_Ellipsis
;
690 retval
= PyInt_FromLong(r_long(p
));
694 retval
= r_long64(p
);
698 retval
= r_PyLong(p
);
706 if (n
== EOF
|| r_string(buf
, (int)n
, p
) != n
) {
707 PyErr_SetString(PyExc_EOFError
,
708 "EOF read where object expected");
713 dx
= PyOS_string_to_double(buf
, NULL
, NULL
);
714 if (dx
== -1.0 && PyErr_Occurred()) {
718 retval
= PyFloat_FromDouble(dx
);
722 case TYPE_BINARY_FLOAT
:
724 unsigned char buf
[8];
726 if (r_string((char*)buf
, 8, p
) != 8) {
727 PyErr_SetString(PyExc_EOFError
,
728 "EOF read where object expected");
732 x
= _PyFloat_Unpack8(buf
, 1);
733 if (x
== -1.0 && PyErr_Occurred()) {
737 retval
= PyFloat_FromDouble(x
);
741 #ifndef WITHOUT_COMPLEX
747 if (n
== EOF
|| r_string(buf
, (int)n
, p
) != n
) {
748 PyErr_SetString(PyExc_EOFError
,
749 "EOF read where object expected");
754 c
.real
= PyOS_string_to_double(buf
, NULL
, NULL
);
755 if (c
.real
== -1.0 && PyErr_Occurred()) {
760 if (n
== EOF
|| r_string(buf
, (int)n
, p
) != n
) {
761 PyErr_SetString(PyExc_EOFError
,
762 "EOF read where object expected");
767 c
.imag
= PyOS_string_to_double(buf
, NULL
, NULL
);
768 if (c
.imag
== -1.0 && PyErr_Occurred()) {
772 retval
= PyComplex_FromCComplex(c
);
776 case TYPE_BINARY_COMPLEX
:
778 unsigned char buf
[8];
780 if (r_string((char*)buf
, 8, p
) != 8) {
781 PyErr_SetString(PyExc_EOFError
,
782 "EOF read where object expected");
786 c
.real
= _PyFloat_Unpack8(buf
, 1);
787 if (c
.real
== -1.0 && PyErr_Occurred()) {
791 if (r_string((char*)buf
, 8, p
) != 8) {
792 PyErr_SetString(PyExc_EOFError
,
793 "EOF read where object expected");
797 c
.imag
= _PyFloat_Unpack8(buf
, 1);
798 if (c
.imag
== -1.0 && PyErr_Occurred()) {
802 retval
= PyComplex_FromCComplex(c
);
810 if (n
< 0 || n
> INT_MAX
) {
811 PyErr_SetString(PyExc_ValueError
, "bad marshal data (string size out of range)");
815 v
= PyString_FromStringAndSize((char *)NULL
, n
);
820 if (r_string(PyString_AS_STRING(v
), (int)n
, p
) != n
) {
822 PyErr_SetString(PyExc_EOFError
,
823 "EOF read where object expected");
827 if (type
== TYPE_INTERNED
) {
828 PyString_InternInPlace(&v
);
829 if (PyList_Append(p
->strings
, v
) < 0) {
839 if (n
< 0 || n
>= PyList_GET_SIZE(p
->strings
)) {
840 PyErr_SetString(PyExc_ValueError
, "bad marshal data (string ref out of range)");
844 v
= PyList_GET_ITEM(p
->strings
, n
);
849 #ifdef Py_USING_UNICODE
855 if (n
< 0 || n
> INT_MAX
) {
856 PyErr_SetString(PyExc_ValueError
, "bad marshal data (unicode size out of range)");
860 buffer
= PyMem_NEW(char, n
);
861 if (buffer
== NULL
) {
862 retval
= PyErr_NoMemory();
865 if (r_string(buffer
, (int)n
, p
) != n
) {
867 PyErr_SetString(PyExc_EOFError
,
868 "EOF read where object expected");
872 v
= PyUnicode_DecodeUTF8(buffer
, n
, NULL
);
881 if (n
< 0 || n
> INT_MAX
) {
882 PyErr_SetString(PyExc_ValueError
, "bad marshal data (tuple size out of range)");
886 v
= PyTuple_New((int)n
);
891 for (i
= 0; i
< n
; i
++) {
894 if (!PyErr_Occurred())
895 PyErr_SetString(PyExc_TypeError
,
896 "NULL object in marshal data for tuple");
901 PyTuple_SET_ITEM(v
, (int)i
, v2
);
908 if (n
< 0 || n
> INT_MAX
) {
909 PyErr_SetString(PyExc_ValueError
, "bad marshal data (list size out of range)");
913 v
= PyList_New((int)n
);
918 for (i
= 0; i
< n
; i
++) {
921 if (!PyErr_Occurred())
922 PyErr_SetString(PyExc_TypeError
,
923 "NULL object in marshal data for list");
928 PyList_SET_ITEM(v
, (int)i
, v2
);
946 PyDict_SetItem(v
, key
, val
);
950 if (PyErr_Occurred()) {
960 if (n
< 0 || n
> INT_MAX
) {
961 PyErr_SetString(PyExc_ValueError
, "bad marshal data (set size out of range)");
965 v
= (type
== TYPE_SET
) ? PySet_New(NULL
) : PyFrozenSet_New(NULL
);
970 for (i
= 0; i
< n
; i
++) {
973 if (!PyErr_Occurred())
974 PyErr_SetString(PyExc_TypeError
,
975 "NULL object in marshal data for set");
980 if (PySet_Add(v
, v2
) == -1) {
992 if (PyEval_GetRestricted()) {
993 PyErr_SetString(PyExc_RuntimeError
,
994 "cannot unmarshal code objects in "
995 "restricted execution mode");
1004 PyObject
*code
= NULL
;
1005 PyObject
*consts
= NULL
;
1006 PyObject
*names
= NULL
;
1007 PyObject
*varnames
= NULL
;
1008 PyObject
*freevars
= NULL
;
1009 PyObject
*cellvars
= NULL
;
1010 PyObject
*filename
= NULL
;
1011 PyObject
*name
= NULL
;
1013 PyObject
*lnotab
= NULL
;
1017 /* XXX ignore long->int overflows for now */
1018 argcount
= (int)r_long(p
);
1019 nlocals
= (int)r_long(p
);
1020 stacksize
= (int)r_long(p
);
1021 flags
= (int)r_long(p
);
1025 consts
= r_object(p
);
1028 names
= r_object(p
);
1031 varnames
= r_object(p
);
1032 if (varnames
== NULL
)
1034 freevars
= r_object(p
);
1035 if (freevars
== NULL
)
1037 cellvars
= r_object(p
);
1038 if (cellvars
== NULL
)
1040 filename
= r_object(p
);
1041 if (filename
== NULL
)
1046 firstlineno
= (int)r_long(p
);
1047 lnotab
= r_object(p
);
1051 v
= (PyObject
*) PyCode_New(
1052 argcount
, nlocals
, stacksize
, flags
,
1053 code
, consts
, names
, varnames
,
1054 freevars
, cellvars
, filename
, name
,
1055 firstlineno
, lnotab
);
1061 Py_XDECREF(varnames
);
1062 Py_XDECREF(freevars
);
1063 Py_XDECREF(cellvars
);
1064 Py_XDECREF(filename
);
1073 /* Bogus data got written, which isn't ideal.
1074 This will let you keep working and recover. */
1075 PyErr_SetString(PyExc_ValueError
, "bad marshal data (unknown type code)");
1085 read_object(RFILE
*p
)
1088 if (PyErr_Occurred()) {
1089 fprintf(stderr
, "XXX readobject called with exception set\n");
1093 if (v
== NULL
&& !PyErr_Occurred())
1094 PyErr_SetString(PyExc_TypeError
, "NULL object in marshal data for object");
1099 PyMarshal_ReadShortFromFile(FILE *fp
)
1105 rf
.end
= rf
.ptr
= NULL
;
1106 return r_short(&rf
);
1110 PyMarshal_ReadLongFromFile(FILE *fp
)
1115 rf
.ptr
= rf
.end
= NULL
;
1120 /* Return size of file in bytes; < 0 if unknown. */
1122 getfilesize(FILE *fp
)
1125 if (fstat(fileno(fp
), &st
) != 0)
1132 /* If we can get the size of the file up-front, and it's reasonably small,
1133 * read it in one gulp and delegate to ...FromString() instead. Much quicker
1134 * than reading a byte at a time from file; speeds .pyc imports.
1135 * CAUTION: since this may read the entire remainder of the file, don't
1136 * call it unless you know you're done with the file.
1139 PyMarshal_ReadLastObjectFromFile(FILE *fp
)
1141 /* REASONABLE_FILE_LIMIT is by defn something big enough for Tkinter.pyc. */
1142 #define REASONABLE_FILE_LIMIT (1L << 18)
1145 filesize
= getfilesize(fp
);
1146 if (filesize
> 0 && filesize
<= REASONABLE_FILE_LIMIT
) {
1147 char* pBuf
= (char *)PyMem_MALLOC(filesize
);
1151 /* filesize must fit into an int, because it
1152 is smaller than REASONABLE_FILE_LIMIT */
1153 n
= fread(pBuf
, 1, (int)filesize
, fp
);
1154 v
= PyMarshal_ReadObjectFromString(pBuf
, n
);
1161 /* We don't have fstat, or we do but the file is larger than
1162 * REASONABLE_FILE_LIMIT or malloc failed -- read a byte at a time.
1164 return PyMarshal_ReadObjectFromFile(fp
);
1166 #undef REASONABLE_FILE_LIMIT
1170 PyMarshal_ReadObjectFromFile(FILE *fp
)
1175 rf
.strings
= PyList_New(0);
1177 rf
.ptr
= rf
.end
= NULL
;
1178 result
= r_object(&rf
);
1179 Py_DECREF(rf
.strings
);
1184 PyMarshal_ReadObjectFromString(char *str
, Py_ssize_t len
)
1191 rf
.strings
= PyList_New(0);
1193 result
= r_object(&rf
);
1194 Py_DECREF(rf
.strings
);
1199 set_error(int error
)
1202 case WFERR_NOMEMORY
:
1205 case WFERR_UNMARSHALLABLE
:
1206 PyErr_SetString(PyExc_ValueError
, "unmarshallable object");
1208 case WFERR_NESTEDTOODEEP
:
1210 PyErr_SetString(PyExc_ValueError
,
1211 "object too deeply nested to marshal");
1217 PyMarshal_WriteObjectToString(PyObject
*x
, int version
)
1221 wf
.str
= PyString_FromStringAndSize((char *)NULL
, 50);
1224 wf
.ptr
= PyString_AS_STRING((PyStringObject
*)wf
.str
);
1225 wf
.end
= wf
.ptr
+ PyString_Size(wf
.str
);
1226 wf
.error
= WFERR_OK
;
1228 wf
.version
= version
;
1229 wf
.strings
= (version
> 0) ? PyDict_New() : NULL
;
1231 Py_XDECREF(wf
.strings
);
1232 if (wf
.str
!= NULL
) {
1233 char *base
= PyString_AS_STRING((PyStringObject
*)wf
.str
);
1234 if (wf
.ptr
- base
> PY_SSIZE_T_MAX
) {
1236 PyErr_SetString(PyExc_OverflowError
,
1237 "too much marshall data for a string");
1240 if (_PyString_Resize(&wf
.str
, (Py_ssize_t
)(wf
.ptr
- base
)))
1243 if (wf
.error
!= WFERR_OK
) {
1245 set_error(wf
.error
);
1251 /* And an interface for Python programs... */
1254 marshal_dump(PyObject
*self
, PyObject
*args
)
1259 int version
= Py_MARSHAL_VERSION
;
1260 if (!PyArg_ParseTuple(args
, "OO|i:dump", &x
, &f
, &version
))
1262 if (!PyFile_Check(f
)) {
1263 PyErr_SetString(PyExc_TypeError
,
1264 "marshal.dump() 2nd arg must be file");
1267 wf
.fp
= PyFile_AsFile(f
);
1269 wf
.ptr
= wf
.end
= NULL
;
1270 wf
.error
= WFERR_OK
;
1272 wf
.strings
= (version
> 0) ? PyDict_New() : 0;
1273 wf
.version
= version
;
1275 Py_XDECREF(wf
.strings
);
1276 if (wf
.error
!= WFERR_OK
) {
1277 set_error(wf
.error
);
1284 PyDoc_STRVAR(dump_doc
,
1285 "dump(value, file[, version])\n\
1287 Write the value on the open file. The value must be a supported type.\n\
1288 The file must be an open file object such as sys.stdout or returned by\n\
1289 open() or os.popen(). It must be opened in binary mode ('wb' or 'w+b').\n\
1291 If the value has (or contains an object that has) an unsupported type, a\n\
1292 ValueError exception is raised - but garbage data will also be written\n\
1293 to the file. The object will not be properly read back by load()\n\
1295 New in version 2.4: The version argument indicates the data format that\n\
1299 marshal_load(PyObject
*self
, PyObject
*f
)
1303 if (!PyFile_Check(f
)) {
1304 PyErr_SetString(PyExc_TypeError
,
1305 "marshal.load() arg must be file");
1308 rf
.fp
= PyFile_AsFile(f
);
1309 rf
.strings
= PyList_New(0);
1311 result
= read_object(&rf
);
1312 Py_DECREF(rf
.strings
);
1316 PyDoc_STRVAR(load_doc
,
1319 Read one value from the open file and return it. If no valid value is\n\
1320 read (e.g. because the data has a different Python version's\n\
1321 incompatible marshal format), raise EOFError, ValueError or TypeError.\n\
1322 The file must be an open file object opened in binary mode ('rb' or\n\
1325 Note: If an object containing an unsupported type was marshalled with\n\
1326 dump(), load() will substitute None for the unmarshallable type.");
1330 marshal_dumps(PyObject
*self
, PyObject
*args
)
1333 int version
= Py_MARSHAL_VERSION
;
1334 if (!PyArg_ParseTuple(args
, "O|i:dumps", &x
, &version
))
1336 return PyMarshal_WriteObjectToString(x
, version
);
1339 PyDoc_STRVAR(dumps_doc
,
1340 "dumps(value[, version])\n\
1342 Return the string that would be written to a file by dump(value, file).\n\
1343 The value must be a supported type. Raise a ValueError exception if\n\
1344 value has (or contains an object that has) an unsupported type.\n\
1346 New in version 2.4: The version argument indicates the data format that\n\
1347 dumps should use.");
1351 marshal_loads(PyObject
*self
, PyObject
*args
)
1357 if (!PyArg_ParseTuple(args
, "s#:loads", &s
, &n
))
1362 rf
.strings
= PyList_New(0);
1364 result
= read_object(&rf
);
1365 Py_DECREF(rf
.strings
);
1369 PyDoc_STRVAR(loads_doc
,
1372 Convert the string to a value. If no valid value is found, raise\n\
1373 EOFError, ValueError or TypeError. Extra characters in the string are\n\
1376 static PyMethodDef marshal_methods
[] = {
1377 {"dump", marshal_dump
, METH_VARARGS
, dump_doc
},
1378 {"load", marshal_load
, METH_O
, load_doc
},
1379 {"dumps", marshal_dumps
, METH_VARARGS
, dumps_doc
},
1380 {"loads", marshal_loads
, METH_VARARGS
, loads_doc
},
1381 {NULL
, NULL
} /* sentinel */
1384 PyDoc_STRVAR(marshal_doc
,
1385 "This module contains functions that can read and write Python values in\n\
1386 a binary format. The format is specific to Python, but independent of\n\
1387 machine architecture issues.\n\
1389 Not all Python object types are supported; in general, only objects\n\
1390 whose value is independent from a particular invocation of Python can be\n\
1391 written and read by this module. The following types are supported:\n\
1392 None, integers, long integers, floating point numbers, strings, Unicode\n\
1393 objects, tuples, lists, sets, dictionaries, and code objects, where it\n\
1394 should be understood that tuples, lists and dictionaries are only\n\
1395 supported as long as the values contained therein are themselves\n\
1396 supported; and recursive lists and dictionaries should not be written\n\
1397 (they will cause infinite loops).\n\
1401 version -- indicates the format that the module uses. Version 0 is the\n\
1402 historical format, version 1 (added in Python 2.4) shares interned\n\
1403 strings and version 2 (added in Python 2.5) uses a binary format for\n\
1404 floating point numbers. (New in version 2.4)\n\
1408 dump() -- write value to a file\n\
1409 load() -- read value from a file\n\
1410 dumps() -- write value to a string\n\
1411 loads() -- read value from a string");
1415 PyMarshal_Init(void)
1417 PyObject
*mod
= Py_InitModule3("marshal", marshal_methods
,
1421 PyModule_AddIntConstant(mod
, "version", Py_MARSHAL_VERSION
);