2 Write Python objects to files and read them back.
3 This is intended for writing and reading compiled Python code only;
4 a true persistent storage facility would be much harder, since
5 it would have to take circular links and sharing into account.
7 Copyright (c) 2015, Daryl McDaniel. All rights reserved.<BR>
8 This program and the accompanying materials are licensed and made available under
9 the terms and conditions of the BSD License that accompanies this distribution.
10 The full text of the license may be found at
11 http://opensource.org/licenses/bsd-license.
13 THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS,
14 WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED.
17 #define PY_SSIZE_T_CLEAN
20 #include "longintrepr.h"
25 #define ABS(x) ((x) < 0 ? -(x) : (x))
27 /* High water mark to determine when the marshalled object is dangerously deep
28 * and risks coring the interpreter. When the object stack gets this deep,
29 * raise an exception instead of continuing.
31 #define MAX_MARSHAL_STACK_DEPTH 2000
35 #define TYPE_FALSE 'F'
37 #define TYPE_STOPITER 'S'
38 #define TYPE_ELLIPSIS '.'
40 #define TYPE_INT64 'I'
41 #define TYPE_FLOAT 'f'
42 #define TYPE_BINARY_FLOAT 'g'
43 #define TYPE_COMPLEX 'x'
44 #define TYPE_BINARY_COMPLEX 'y'
46 #define TYPE_STRING 's'
47 #define TYPE_INTERNED 't'
48 #define TYPE_STRINGREF 'R'
49 #define TYPE_TUPLE '('
53 #define TYPE_UNICODE 'u'
54 #define TYPE_UNKNOWN '?'
56 #define TYPE_FROZENSET '>'
59 #define WFERR_UNMARSHALLABLE 1
60 #define WFERR_NESTEDTOODEEP 2
61 #define WFERR_NOMEMORY 3
65 int error
; /* see WFERR_* values */
67 /* If fp == NULL, the following are valid: */
71 PyObject
*strings
; /* dict on marshal, list on unmarshal */
75 #define w_byte(c, p) if (((p)->fp)) putc((c), (p)->fp); \
76 else if ((p)->ptr != (p)->end) *(p)->ptr++ = (c); \
80 w_more(int c
, WFILE
*p
)
82 Py_ssize_t size
, newsize
;
84 return; /* An error already occurred */
85 size
= PyString_Size(p
->str
);
86 newsize
= size
+ size
+ 1024;
87 if (newsize
> 32*1024*1024) {
88 newsize
= size
+ (size
>> 3); /* 12.5% overallocation */
90 if (_PyString_Resize(&p
->str
, newsize
) != 0) {
91 p
->ptr
= p
->end
= NULL
;
94 p
->ptr
= PyString_AS_STRING((PyStringObject
*)p
->str
) + size
;
96 PyString_AS_STRING((PyStringObject
*)p
->str
) + newsize
;
97 *p
->ptr
++ = Py_SAFE_DOWNCAST(c
, int, char);
102 w_string(const char *s
, Py_ssize_t n
, WFILE
*p
)
105 fwrite(s
, 1, n
, p
->fp
);
116 w_short(int x
, WFILE
*p
)
118 w_byte((char)( x
& 0xff), p
);
119 w_byte((char)((x
>> 8) & 0xff), p
);
123 w_long(long x
, WFILE
*p
)
125 w_byte((char)( x
& 0xff), p
);
126 w_byte((char)((x
>> 8) & 0xff), p
);
127 w_byte((char)((x
>>16) & 0xff), p
);
128 w_byte((char)((x
>>24) & 0xff), p
);
133 w_long64(long x
, WFILE
*p
)
140 #define SIZE32_MAX 0x7FFFFFFF
142 #if SIZEOF_SIZE_T > 4
143 # define W_SIZE(n, p) do { \
144 if ((n) > SIZE32_MAX) { \
146 (p)->error = WFERR_UNMARSHALLABLE; \
149 w_long((long)(n), p); \
152 # define W_SIZE w_long
156 w_pstring(const char *s
, Py_ssize_t n
, WFILE
*p
)
162 /* We assume that Python longs are stored internally in base some power of
163 2**15; for the sake of portability we'll always read and write them in base
166 #define PyLong_MARSHAL_SHIFT 15
167 #define PyLong_MARSHAL_BASE ((short)1 << PyLong_MARSHAL_SHIFT)
168 #define PyLong_MARSHAL_MASK (PyLong_MARSHAL_BASE - 1)
169 #if PyLong_SHIFT % PyLong_MARSHAL_SHIFT != 0
170 #error "PyLong_SHIFT must be a multiple of PyLong_MARSHAL_SHIFT"
172 #define PyLong_MARSHAL_RATIO (PyLong_SHIFT / PyLong_MARSHAL_SHIFT)
175 w_PyLong(const PyLongObject
*ob
, WFILE
*p
)
177 Py_ssize_t i
, j
, n
, l
;
180 w_byte(TYPE_LONG
, p
);
181 if (Py_SIZE(ob
) == 0) {
186 /* set l to number of base PyLong_MARSHAL_BASE digits */
187 n
= ABS(Py_SIZE(ob
));
188 l
= (n
-1) * PyLong_MARSHAL_RATIO
;
189 d
= ob
->ob_digit
[n
-1];
190 assert(d
!= 0); /* a PyLong is always normalized */
192 d
>>= PyLong_MARSHAL_SHIFT
;
195 if (l
> SIZE32_MAX
) {
197 p
->error
= WFERR_UNMARSHALLABLE
;
200 w_long((long)(Py_SIZE(ob
) > 0 ? l
: -l
), p
);
202 for (i
=0; i
< n
-1; i
++) {
204 for (j
=0; j
< PyLong_MARSHAL_RATIO
; j
++) {
205 w_short(d
& PyLong_MARSHAL_MASK
, p
);
206 d
>>= PyLong_MARSHAL_SHIFT
;
210 d
= ob
->ob_digit
[n
-1];
212 w_short(d
& PyLong_MARSHAL_MASK
, p
);
213 d
>>= PyLong_MARSHAL_SHIFT
;
218 w_object(PyObject
*v
, WFILE
*p
)
224 if (p
->depth
> MAX_MARSHAL_STACK_DEPTH
) {
225 p
->error
= WFERR_NESTEDTOODEEP
;
227 else if (v
== NULL
) {
228 w_byte(TYPE_NULL
, p
);
230 else if (v
== Py_None
) {
231 w_byte(TYPE_NONE
, p
);
233 else if (v
== PyExc_StopIteration
) {
234 w_byte(TYPE_STOPITER
, p
);
236 else if (v
== Py_Ellipsis
) {
237 w_byte(TYPE_ELLIPSIS
, p
);
239 else if (v
== Py_False
) {
240 w_byte(TYPE_FALSE
, p
);
242 else if (v
== Py_True
) {
243 w_byte(TYPE_TRUE
, p
);
245 else if (PyInt_CheckExact(v
)) {
246 long x
= PyInt_AS_LONG((PyIntObject
*)v
);
248 long y
= Py_ARITHMETIC_RIGHT_SHIFT(long, x
, 31);
250 w_byte(TYPE_INT64
, p
);
260 else if (PyLong_CheckExact(v
)) {
261 PyLongObject
*ob
= (PyLongObject
*)v
;
264 else if (PyFloat_CheckExact(v
)) {
265 if (p
->version
> 1) {
266 unsigned char buf
[8];
267 if (_PyFloat_Pack8(PyFloat_AsDouble(v
),
269 p
->error
= WFERR_UNMARSHALLABLE
;
272 w_byte(TYPE_BINARY_FLOAT
, p
);
273 w_string((char*)buf
, 8, p
);
276 char *buf
= PyOS_double_to_string(PyFloat_AS_DOUBLE(v
),
279 p
->error
= WFERR_NOMEMORY
;
283 w_byte(TYPE_FLOAT
, p
);
289 #ifndef WITHOUT_COMPLEX
290 else if (PyComplex_CheckExact(v
)) {
291 if (p
->version
> 1) {
292 unsigned char buf
[8];
293 if (_PyFloat_Pack8(PyComplex_RealAsDouble(v
),
295 p
->error
= WFERR_UNMARSHALLABLE
;
298 w_byte(TYPE_BINARY_COMPLEX
, p
);
299 w_string((char*)buf
, 8, p
);
300 if (_PyFloat_Pack8(PyComplex_ImagAsDouble(v
),
302 p
->error
= WFERR_UNMARSHALLABLE
;
305 w_string((char*)buf
, 8, p
);
309 w_byte(TYPE_COMPLEX
, p
);
310 buf
= PyOS_double_to_string(PyComplex_RealAsDouble(v
),
313 p
->error
= WFERR_NOMEMORY
;
320 buf
= PyOS_double_to_string(PyComplex_ImagAsDouble(v
),
323 p
->error
= WFERR_NOMEMORY
;
333 else if (PyString_CheckExact(v
)) {
334 if (p
->strings
&& PyString_CHECK_INTERNED(v
)) {
335 PyObject
*o
= PyDict_GetItem(p
->strings
, v
);
337 long w
= PyInt_AsLong(o
);
338 w_byte(TYPE_STRINGREF
, p
);
344 o
= PyInt_FromSsize_t(PyDict_Size(p
->strings
));
346 PyDict_SetItem(p
->strings
, v
, o
) >= 0;
350 p
->error
= WFERR_UNMARSHALLABLE
;
353 w_byte(TYPE_INTERNED
, p
);
357 w_byte(TYPE_STRING
, p
);
359 w_pstring(PyBytes_AS_STRING(v
), PyString_GET_SIZE(v
), p
);
361 #ifdef Py_USING_UNICODE
362 else if (PyUnicode_CheckExact(v
)) {
364 utf8
= PyUnicode_AsUTF8String(v
);
367 p
->error
= WFERR_UNMARSHALLABLE
;
370 w_byte(TYPE_UNICODE
, p
);
371 w_pstring(PyString_AS_STRING(utf8
), PyString_GET_SIZE(utf8
), p
);
375 else if (PyTuple_CheckExact(v
)) {
376 w_byte(TYPE_TUPLE
, p
);
379 for (i
= 0; i
< n
; i
++) {
380 w_object(PyTuple_GET_ITEM(v
, i
), p
);
383 else if (PyList_CheckExact(v
)) {
384 w_byte(TYPE_LIST
, p
);
385 n
= PyList_GET_SIZE(v
);
387 for (i
= 0; i
< n
; i
++) {
388 w_object(PyList_GET_ITEM(v
, i
), p
);
391 else if (PyDict_CheckExact(v
)) {
393 PyObject
*key
, *value
;
394 w_byte(TYPE_DICT
, p
);
395 /* This one is NULL object terminated! */
397 while (PyDict_Next(v
, &pos
, &key
, &value
)) {
401 w_object((PyObject
*)NULL
, p
);
403 else if (PyAnySet_CheckExact(v
)) {
404 PyObject
*value
, *it
;
406 if (PyObject_TypeCheck(v
, &PySet_Type
))
409 w_byte(TYPE_FROZENSET
, p
);
410 n
= PyObject_Size(v
);
413 p
->error
= WFERR_UNMARSHALLABLE
;
417 it
= PyObject_GetIter(v
);
420 p
->error
= WFERR_UNMARSHALLABLE
;
423 while ((value
= PyIter_Next(it
)) != NULL
) {
428 if (PyErr_Occurred()) {
430 p
->error
= WFERR_UNMARSHALLABLE
;
434 else if (PyCode_Check(v
)) {
435 PyCodeObject
*co
= (PyCodeObject
*)v
;
436 w_byte(TYPE_CODE
, p
);
437 w_long(co
->co_argcount
, p
);
438 w_long(co
->co_nlocals
, p
);
439 w_long(co
->co_stacksize
, p
);
440 w_long(co
->co_flags
, p
);
441 w_object(co
->co_code
, p
);
442 w_object(co
->co_consts
, p
);
443 w_object(co
->co_names
, p
);
444 w_object(co
->co_varnames
, p
);
445 w_object(co
->co_freevars
, p
);
446 w_object(co
->co_cellvars
, p
);
447 w_object(co
->co_filename
, p
);
448 w_object(co
->co_name
, p
);
449 w_long(co
->co_firstlineno
, p
);
450 w_object(co
->co_lnotab
, p
);
452 else if (PyObject_CheckReadBuffer(v
)) {
453 /* Write unknown buffer-style objects as a string */
455 PyBufferProcs
*pb
= v
->ob_type
->tp_as_buffer
;
456 w_byte(TYPE_STRING
, p
);
457 n
= (*pb
->bf_getreadbuffer
)(v
, 0, (void **)&s
);
461 w_byte(TYPE_UNKNOWN
, p
);
462 p
->error
= WFERR_UNMARSHALLABLE
;
468 /* version currently has no effect for writing longs. */
470 PyMarshal_WriteLongToFile(long x
, FILE *fp
, int version
)
477 wf
.version
= version
;
482 PyMarshal_WriteObjectToFile(PyObject
*x
, FILE *fp
, int version
)
488 wf
.strings
= (version
> 0) ? PyDict_New() : NULL
;
489 wf
.version
= version
;
491 Py_XDECREF(wf
.strings
);
494 typedef WFILE RFILE
; /* Same struct with different invariants */
496 #define rs_byte(p) (((p)->ptr < (p)->end) ? (unsigned char)*(p)->ptr++ : EOF)
498 #define r_byte(p) ((p)->fp ? getc((p)->fp) : rs_byte(p))
501 r_string(char *s
, Py_ssize_t n
, RFILE
*p
)
504 /* The result fits into int because it must be <=n. */
505 return fread(s
, 1, n
, p
->fp
);
506 if (p
->end
- p
->ptr
< n
)
508 memcpy(s
, p
->ptr
, n
);
519 /* Sign-extension, in case short greater than 16 bits */
528 register FILE *fp
= p
->fp
;
531 x
|= (long)getc(fp
) << 8;
532 x
|= (long)getc(fp
) << 16;
533 x
|= (long)getc(fp
) << 24;
537 x
|= (long)rs_byte(p
) << 8;
538 x
|= (long)rs_byte(p
) << 16;
539 x
|= (long)rs_byte(p
) << 24;
542 /* Sign extension for 64-bit machines */
543 x
|= -(x
& 0x80000000L
);
548 /* r_long64 deals with the TYPE_INT64 code. On a machine with
549 sizeof(long) > 4, it returns a Python int object, else a Python long
550 object. Note that w_long64 writes out TYPE_INT if 32 bits is enough,
551 so there's no inefficiency here in returning a PyLong on 32-bit boxes
552 for everything written via TYPE_INT64 (i.e., if an int is written via
553 TYPE_INT64, it *needs* more than 32 bits).
558 long lo4
= r_long(p
);
559 long hi4
= r_long(p
);
561 long x
= (hi4
<< 32) | (lo4
& 0xFFFFFFFFL
);
562 return PyInt_FromLong(x
);
564 unsigned char buf
[8];
566 int is_little_endian
= (int)*(char*)&one
;
567 if (is_little_endian
) {
568 memcpy(buf
, &lo4
, 4);
569 memcpy(buf
+4, &hi4
, 4);
572 memcpy(buf
, &hi4
, 4);
573 memcpy(buf
+4, &lo4
, 4);
575 return _PyLong_FromByteArray(buf
, 8, is_little_endian
, 1);
584 int j
, md
, shorts_in_top_digit
;
589 return (PyObject
*)_PyLong_New(0);
590 if (n
< -SIZE32_MAX
|| n
> SIZE32_MAX
) {
591 PyErr_SetString(PyExc_ValueError
,
592 "bad marshal data (long size out of range)");
596 size
= 1 + (ABS(n
) - 1) / PyLong_MARSHAL_RATIO
;
597 shorts_in_top_digit
= 1 + (ABS(n
) - 1) % PyLong_MARSHAL_RATIO
;
598 ob
= _PyLong_New(size
);
601 Py_SIZE(ob
) = n
> 0 ? size
: -size
;
603 for (i
= 0; i
< size
-1; i
++) {
605 for (j
=0; j
< PyLong_MARSHAL_RATIO
; j
++) {
607 if (md
< 0 || md
> PyLong_MARSHAL_BASE
)
609 d
+= (digit
)md
<< j
*PyLong_MARSHAL_SHIFT
;
614 for (j
=0; j
< shorts_in_top_digit
; j
++) {
616 if (md
< 0 || md
> PyLong_MARSHAL_BASE
)
618 /* topmost marshal digit should be nonzero */
619 if (md
== 0 && j
== shorts_in_top_digit
- 1) {
621 PyErr_SetString(PyExc_ValueError
,
622 "bad marshal data (unnormalized long data)");
625 d
+= (digit
)md
<< j
*PyLong_MARSHAL_SHIFT
;
627 /* top digit should be nonzero, else the resulting PyLong won't be
629 ob
->ob_digit
[size
-1] = d
;
630 return (PyObject
*)ob
;
633 PyErr_SetString(PyExc_ValueError
,
634 "bad marshal data (digit out of range in long)");
642 /* NULL is a valid return value, it does not necessarily means that
643 an exception is set. */
646 int type
= r_byte(p
);
651 if (p
->depth
> MAX_MARSHAL_STACK_DEPTH
) {
653 PyErr_SetString(PyExc_ValueError
, "recursion limit exceeded");
660 PyErr_SetString(PyExc_EOFError
,
661 "EOF read where object expected");
675 Py_INCREF(PyExc_StopIteration
);
676 retval
= PyExc_StopIteration
;
680 Py_INCREF(Py_Ellipsis
);
681 retval
= Py_Ellipsis
;
695 retval
= PyInt_FromLong(r_long(p
));
699 retval
= r_long64(p
);
703 retval
= r_PyLong(p
);
711 if (n
== EOF
|| r_string(buf
, n
, p
) != n
) {
712 PyErr_SetString(PyExc_EOFError
,
713 "EOF read where object expected");
718 dx
= PyOS_string_to_double(buf
, NULL
, NULL
);
719 if (dx
== -1.0 && PyErr_Occurred()) {
723 retval
= PyFloat_FromDouble(dx
);
727 case TYPE_BINARY_FLOAT
:
729 unsigned char buf
[8];
731 if (r_string((char*)buf
, 8, p
) != 8) {
732 PyErr_SetString(PyExc_EOFError
,
733 "EOF read where object expected");
737 x
= _PyFloat_Unpack8(buf
, 1);
738 if (x
== -1.0 && PyErr_Occurred()) {
742 retval
= PyFloat_FromDouble(x
);
746 #ifndef WITHOUT_COMPLEX
752 if (n
== EOF
|| r_string(buf
, n
, p
) != n
) {
753 PyErr_SetString(PyExc_EOFError
,
754 "EOF read where object expected");
759 c
.real
= PyOS_string_to_double(buf
, NULL
, NULL
);
760 if (c
.real
== -1.0 && PyErr_Occurred()) {
765 if (n
== EOF
|| r_string(buf
, n
, p
) != n
) {
766 PyErr_SetString(PyExc_EOFError
,
767 "EOF read where object expected");
772 c
.imag
= PyOS_string_to_double(buf
, NULL
, NULL
);
773 if (c
.imag
== -1.0 && PyErr_Occurred()) {
777 retval
= PyComplex_FromCComplex(c
);
781 case TYPE_BINARY_COMPLEX
:
783 unsigned char buf
[8];
785 if (r_string((char*)buf
, 8, p
) != 8) {
786 PyErr_SetString(PyExc_EOFError
,
787 "EOF read where object expected");
791 c
.real
= _PyFloat_Unpack8(buf
, 1);
792 if (c
.real
== -1.0 && PyErr_Occurred()) {
796 if (r_string((char*)buf
, 8, p
) != 8) {
797 PyErr_SetString(PyExc_EOFError
,
798 "EOF read where object expected");
802 c
.imag
= _PyFloat_Unpack8(buf
, 1);
803 if (c
.imag
== -1.0 && PyErr_Occurred()) {
807 retval
= PyComplex_FromCComplex(c
);
815 if (n
< 0 || n
> SIZE32_MAX
) {
816 PyErr_SetString(PyExc_ValueError
, "bad marshal data (string size out of range)");
820 v
= PyString_FromStringAndSize((char *)NULL
, n
);
825 if (r_string(PyString_AS_STRING(v
), n
, p
) != n
) {
827 PyErr_SetString(PyExc_EOFError
,
828 "EOF read where object expected");
832 if (type
== TYPE_INTERNED
) {
833 PyString_InternInPlace(&v
);
834 if (PyList_Append(p
->strings
, v
) < 0) {
844 if (n
< 0 || n
>= PyList_GET_SIZE(p
->strings
)) {
845 PyErr_SetString(PyExc_ValueError
, "bad marshal data (string ref out of range)");
849 v
= PyList_GET_ITEM(p
->strings
, n
);
854 #ifdef Py_USING_UNICODE
860 if (n
< 0 || n
> SIZE32_MAX
) {
861 PyErr_SetString(PyExc_ValueError
, "bad marshal data (unicode size out of range)");
865 buffer
= PyMem_NEW(char, n
);
866 if (buffer
== NULL
) {
867 retval
= PyErr_NoMemory();
870 if (r_string(buffer
, n
, p
) != n
) {
872 PyErr_SetString(PyExc_EOFError
,
873 "EOF read where object expected");
877 v
= PyUnicode_DecodeUTF8(buffer
, n
, NULL
);
886 if (n
< 0 || n
> SIZE32_MAX
) {
887 PyErr_SetString(PyExc_ValueError
, "bad marshal data (tuple size out of range)");
896 for (i
= 0; i
< n
; i
++) {
899 if (!PyErr_Occurred())
900 PyErr_SetString(PyExc_TypeError
,
901 "NULL object in marshal data for tuple");
906 PyTuple_SET_ITEM(v
, i
, v2
);
913 if (n
< 0 || n
> SIZE32_MAX
) {
914 PyErr_SetString(PyExc_ValueError
, "bad marshal data (list size out of range)");
923 for (i
= 0; i
< n
; i
++) {
926 if (!PyErr_Occurred())
927 PyErr_SetString(PyExc_TypeError
,
928 "NULL object in marshal data for list");
933 PyList_SET_ITEM(v
, i
, v2
);
951 PyDict_SetItem(v
, key
, val
);
955 if (PyErr_Occurred()) {
965 if (n
< 0 || n
> SIZE32_MAX
) {
966 PyErr_SetString(PyExc_ValueError
, "bad marshal data (set size out of range)");
970 v
= (type
== TYPE_SET
) ? PySet_New(NULL
) : PyFrozenSet_New(NULL
);
975 for (i
= 0; i
< n
; i
++) {
978 if (!PyErr_Occurred())
979 PyErr_SetString(PyExc_TypeError
,
980 "NULL object in marshal data for set");
985 if (PySet_Add(v
, v2
) == -1) {
997 if (PyEval_GetRestricted()) {
998 PyErr_SetString(PyExc_RuntimeError
,
999 "cannot unmarshal code objects in "
1000 "restricted execution mode");
1009 PyObject
*code
= NULL
;
1010 PyObject
*consts
= NULL
;
1011 PyObject
*names
= NULL
;
1012 PyObject
*varnames
= NULL
;
1013 PyObject
*freevars
= NULL
;
1014 PyObject
*cellvars
= NULL
;
1015 PyObject
*filename
= NULL
;
1016 PyObject
*name
= NULL
;
1018 PyObject
*lnotab
= NULL
;
1022 /* XXX ignore long->int overflows for now */
1023 argcount
= (int)r_long(p
);
1024 nlocals
= (int)r_long(p
);
1025 stacksize
= (int)r_long(p
);
1026 flags
= (int)r_long(p
);
1030 consts
= r_object(p
);
1033 names
= r_object(p
);
1036 varnames
= r_object(p
);
1037 if (varnames
== NULL
)
1039 freevars
= r_object(p
);
1040 if (freevars
== NULL
)
1042 cellvars
= r_object(p
);
1043 if (cellvars
== NULL
)
1045 filename
= r_object(p
);
1046 if (filename
== NULL
)
1051 firstlineno
= (int)r_long(p
);
1052 lnotab
= r_object(p
);
1056 v
= (PyObject
*) PyCode_New(
1057 argcount
, nlocals
, stacksize
, flags
,
1058 code
, consts
, names
, varnames
,
1059 freevars
, cellvars
, filename
, name
,
1060 firstlineno
, lnotab
);
1066 Py_XDECREF(varnames
);
1067 Py_XDECREF(freevars
);
1068 Py_XDECREF(cellvars
);
1069 Py_XDECREF(filename
);
1078 /* Bogus data got written, which isn't ideal.
1079 This will let you keep working and recover. */
1080 PyErr_SetString(PyExc_ValueError
, "bad marshal data (unknown type code)");
1090 read_object(RFILE
*p
)
1093 if (PyErr_Occurred()) {
1094 fprintf(stderr
, "XXX readobject called with exception set\n");
1098 if (v
== NULL
&& !PyErr_Occurred())
1099 PyErr_SetString(PyExc_TypeError
, "NULL object in marshal data for object");
1104 PyMarshal_ReadShortFromFile(FILE *fp
)
1110 rf
.end
= rf
.ptr
= NULL
;
1111 return r_short(&rf
);
1115 PyMarshal_ReadLongFromFile(FILE *fp
)
1120 rf
.ptr
= rf
.end
= NULL
;
1125 /* Return size of file in bytes; < 0 if unknown. */
1127 getfilesize(FILE *fp
)
1130 if (fstat(fileno(fp
), &st
) != 0)
1137 /* If we can get the size of the file up-front, and it's reasonably small,
1138 * read it in one gulp and delegate to ...FromString() instead. Much quicker
1139 * than reading a byte at a time from file; speeds .pyc imports.
1140 * CAUTION: since this may read the entire remainder of the file, don't
1141 * call it unless you know you're done with the file.
1144 PyMarshal_ReadLastObjectFromFile(FILE *fp
)
1146 /* REASONABLE_FILE_LIMIT is by defn something big enough for Tkinter.pyc. */
1147 #define REASONABLE_FILE_LIMIT (1L << 18)
1150 filesize
= getfilesize(fp
);
1151 if (filesize
> 0 && filesize
<= REASONABLE_FILE_LIMIT
) {
1152 char* pBuf
= (char *)PyMem_MALLOC(filesize
);
1154 size_t n
= fread(pBuf
, 1, (size_t)filesize
, fp
);
1155 PyObject
* v
= PyMarshal_ReadObjectFromString(pBuf
, n
);
1162 /* We don't have fstat, or we do but the file is larger than
1163 * REASONABLE_FILE_LIMIT or malloc failed -- read a byte at a time.
1165 return PyMarshal_ReadObjectFromFile(fp
);
1167 #undef REASONABLE_FILE_LIMIT
1171 PyMarshal_ReadObjectFromFile(FILE *fp
)
1176 rf
.strings
= PyList_New(0);
1178 rf
.ptr
= rf
.end
= NULL
;
1179 result
= r_object(&rf
);
1180 Py_DECREF(rf
.strings
);
1185 PyMarshal_ReadObjectFromString(char *str
, Py_ssize_t len
)
1192 rf
.strings
= PyList_New(0);
1194 result
= r_object(&rf
);
1195 Py_DECREF(rf
.strings
);
1200 set_error(int error
)
1203 case WFERR_NOMEMORY
:
1206 case WFERR_UNMARSHALLABLE
:
1207 PyErr_SetString(PyExc_ValueError
, "unmarshallable object");
1209 case WFERR_NESTEDTOODEEP
:
1211 PyErr_SetString(PyExc_ValueError
,
1212 "object too deeply nested to marshal");
1218 PyMarshal_WriteObjectToString(PyObject
*x
, int version
)
1222 wf
.str
= PyString_FromStringAndSize((char *)NULL
, 50);
1225 wf
.ptr
= PyString_AS_STRING((PyStringObject
*)wf
.str
);
1226 wf
.end
= wf
.ptr
+ PyString_Size(wf
.str
);
1227 wf
.error
= WFERR_OK
;
1229 wf
.version
= version
;
1230 wf
.strings
= (version
> 0) ? PyDict_New() : NULL
;
1232 Py_XDECREF(wf
.strings
);
1233 if (wf
.str
!= NULL
) {
1234 char *base
= PyString_AS_STRING((PyStringObject
*)wf
.str
);
1235 if (wf
.ptr
- base
> PY_SSIZE_T_MAX
) {
1237 PyErr_SetString(PyExc_OverflowError
,
1238 "too much marshall data for a string");
1241 if (_PyString_Resize(&wf
.str
, (Py_ssize_t
)(wf
.ptr
- base
)))
1244 if (wf
.error
!= WFERR_OK
) {
1246 set_error(wf
.error
);
1252 /* And an interface for Python programs... */
1255 marshal_dump(PyObject
*self
, PyObject
*args
)
1260 int version
= Py_MARSHAL_VERSION
;
1261 if (!PyArg_ParseTuple(args
, "OO|i:dump", &x
, &f
, &version
))
1263 if (!PyFile_Check(f
)) {
1264 PyErr_SetString(PyExc_TypeError
,
1265 "marshal.dump() 2nd arg must be file");
1268 wf
.fp
= PyFile_AsFile(f
);
1270 wf
.ptr
= wf
.end
= NULL
;
1271 wf
.error
= WFERR_OK
;
1273 wf
.strings
= (version
> 0) ? PyDict_New() : 0;
1274 wf
.version
= version
;
1276 Py_XDECREF(wf
.strings
);
1277 if (wf
.error
!= WFERR_OK
) {
1278 set_error(wf
.error
);
1285 PyDoc_STRVAR(dump_doc
,
1286 "dump(value, file[, version])\n\
1288 Write the value on the open file. The value must be a supported type.\n\
1289 The file must be an open file object such as sys.stdout or returned by\n\
1290 open() or os.popen(). It must be opened in binary mode ('wb' or 'w+b').\n\
1292 If the value has (or contains an object that has) an unsupported type, a\n\
1293 ValueError exception is raised — but garbage data will also be written\n\
1294 to the file. The object will not be properly read back by load()\n\
1296 New in version 2.4: The version argument indicates the data format that\n\
1300 marshal_load(PyObject
*self
, PyObject
*f
)
1304 if (!PyFile_Check(f
)) {
1305 PyErr_SetString(PyExc_TypeError
,
1306 "marshal.load() arg must be file");
1309 rf
.fp
= PyFile_AsFile(f
);
1310 rf
.strings
= PyList_New(0);
1312 result
= read_object(&rf
);
1313 Py_DECREF(rf
.strings
);
1317 PyDoc_STRVAR(load_doc
,
1320 Read one value from the open file and return it. If no valid value is\n\
1321 read (e.g. because the data has a different Python version’s\n\
1322 incompatible marshal format), raise EOFError, ValueError or TypeError.\n\
1323 The file must be an open file object opened in binary mode ('rb' or\n\
1326 Note: If an object containing an unsupported type was marshalled with\n\
1327 dump(), load() will substitute None for the unmarshallable type.");
1331 marshal_dumps(PyObject
*self
, PyObject
*args
)
1334 int version
= Py_MARSHAL_VERSION
;
1335 if (!PyArg_ParseTuple(args
, "O|i:dumps", &x
, &version
))
1337 return PyMarshal_WriteObjectToString(x
, version
);
1340 PyDoc_STRVAR(dumps_doc
,
1341 "dumps(value[, version])\n\
1343 Return the string that would be written to a file by dump(value, file).\n\
1344 The value must be a supported type. Raise a ValueError exception if\n\
1345 value has (or contains an object that has) an unsupported type.\n\
1347 New in version 2.4: The version argument indicates the data format that\n\
1348 dumps should use.");
1352 marshal_loads(PyObject
*self
, PyObject
*args
)
1358 if (!PyArg_ParseTuple(args
, "s#:loads", &s
, &n
))
1363 rf
.strings
= PyList_New(0);
1365 result
= read_object(&rf
);
1366 Py_DECREF(rf
.strings
);
1370 PyDoc_STRVAR(loads_doc
,
1373 Convert the string to a value. If no valid value is found, raise\n\
1374 EOFError, ValueError or TypeError. Extra characters in the string are\n\
1377 static PyMethodDef marshal_methods
[] = {
1378 {"dump", marshal_dump
, METH_VARARGS
, dump_doc
},
1379 {"load", marshal_load
, METH_O
, load_doc
},
1380 {"dumps", marshal_dumps
, METH_VARARGS
, dumps_doc
},
1381 {"loads", marshal_loads
, METH_VARARGS
, loads_doc
},
1382 {NULL
, NULL
} /* sentinel */
1385 PyDoc_STRVAR(marshal_doc
,
1386 "This module contains functions that can read and write Python values in\n\
1387 a binary format. The format is specific to Python, but independent of\n\
1388 machine architecture issues.\n\
1390 Not all Python object types are supported; in general, only objects\n\
1391 whose value is independent from a particular invocation of Python can be\n\
1392 written and read by this module. The following types are supported:\n\
1393 None, integers, long integers, floating point numbers, strings, Unicode\n\
1394 objects, tuples, lists, sets, dictionaries, and code objects, where it\n\
1395 should be understood that tuples, lists and dictionaries are only\n\
1396 supported as long as the values contained therein are themselves\n\
1397 supported; and recursive lists and dictionaries should not be written\n\
1398 (they will cause infinite loops).\n\
1402 version -- indicates the format that the module uses. Version 0 is the\n\
1403 historical format, version 1 (added in Python 2.4) shares interned\n\
1404 strings and version 2 (added in Python 2.5) uses a binary format for\n\
1405 floating point numbers. (New in version 2.4)\n\
1409 dump() -- write value to a file\n\
1410 load() -- read value from a file\n\
1411 dumps() -- write value to a string\n\
1412 loads() -- read value from a string");
1416 PyMarshal_Init(void)
1418 PyObject
*mod
= Py_InitModule3("marshal", marshal_methods
,
1422 PyModule_AddIntConstant(mod
, "version", Py_MARSHAL_VERSION
);