1 // Licensed to the Apache Software Foundation (ASF) under one
2 // or more contributor license agreements. See the NOTICE file
3 // distributed with this work for additional information
4 // regarding copyright ownership. The ASF licenses this file
5 // to you under the Apache License, Version 2.0 (the
6 // "License"); you may not use this file except in compliance
7 // with the License. You may obtain a copy of the License at
9 // http://www.apache.org/licenses/LICENSE-2.0
11 // Unless required by applicable law or agreed to in writing,
12 // software distributed under the License is distributed on an
13 // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14 // KIND, either express or implied. See the License for the
15 // specific language governing permissions and limitations
24 #include "arrow/python/serialize.h"
25 #include "arrow/python/visibility.h"
26 #include "arrow/status.h"
35 class RandomAccessFile
;
41 struct ARROW_PYTHON_EXPORT SparseTensorCounts
{
48 int num_total_tensors() const { return coo
+ csr
+ csc
+ csf
; }
49 int num_total_buffers() const {
50 return coo
* 3 + csr
* 4 + csc
* 4 + 2 * ndim_csf
+ csf
;
54 /// \brief Read serialized Python sequence from file interface using Arrow IPC
55 /// \param[in] src a RandomAccessFile
56 /// \param[out] out the reconstructed data
59 Status
ReadSerializedObject(io::RandomAccessFile
* src
, SerializedPyObject
* out
);
61 /// \brief Reconstruct SerializedPyObject from representation produced by
62 /// SerializedPyObject::GetComponents.
64 /// \param[in] num_tensors number of tensors in the object
65 /// \param[in] num_sparse_tensors number of sparse tensors in the object
66 /// \param[in] num_ndarrays number of numpy Ndarrays in the object
67 /// \param[in] num_buffers number of buffers in the object
68 /// \param[in] data a list containing pyarrow.Buffer instances. It must be 1 +
69 /// num_tensors * 2 + num_coo_tensors * 3 + num_csr_tensors * 4 + num_csc_tensors * 4 +
70 /// num_csf_tensors * (2 * ndim_csf + 3) + num_buffers in length
71 /// \param[out] out the reconstructed object
74 Status
GetSerializedFromComponents(int num_tensors
,
75 const SparseTensorCounts
& num_sparse_tensors
,
76 int num_ndarrays
, int num_buffers
, PyObject
* data
,
77 SerializedPyObject
* out
);
79 /// \brief Reconstruct Python object from Arrow-serialized representation
80 /// \param[in] context Serialization context which contains custom serialization
81 /// and deserialization callbacks. Can be any Python object with a
82 /// _serialize_callback method for serialization and a _deserialize_callback
83 /// method for deserialization. If context is None, no custom serialization
84 /// will be attempted.
85 /// \param[in] object Object to deserialize
86 /// \param[in] base a Python object holding the underlying data that any NumPy
87 /// arrays will reference, to avoid premature deallocation
88 /// \param[out] out The returned object
90 /// This acquires the GIL
92 Status
DeserializeObject(PyObject
* context
, const SerializedPyObject
& object
,
93 PyObject
* base
, PyObject
** out
);
95 /// \brief Reconstruct Ndarray from Arrow-serialized representation
96 /// \param[in] object Object to deserialize
97 /// \param[out] out The deserialized tensor
100 Status
DeserializeNdarray(const SerializedPyObject
& object
, std::shared_ptr
<Tensor
>* out
);
103 Status
NdarrayFromBuffer(std::shared_ptr
<Buffer
> src
, std::shared_ptr
<Tensor
>* out
);