1 # Licensed to the Apache Software Foundation (ASF) under one
2 # or more contributor license agreements. See the NOTICE file
3 # distributed with this work for additional information
4 # regarding copyright ownership. The ASF licenses this file
5 # to you under the Apache License, Version 2.0 (the
6 # "License"); you may not use this file except in compliance
7 # with the License. You may obtain a copy of the License at
9 # http://www.apache.org/licenses/LICENSE-2.0
11 # Unless required by applicable law or agreed to in writing,
12 # software distributed under the License is distributed on an
13 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14 # KIND, either express or implied. See the License for the
15 # specific language governing permissions and limitations
21 cdef class Scalar(_Weakrefable):
23 The base class for scalars.
27 raise TypeError("Do not call {}'s constructor directly, use "
28 "pa.scalar() instead.".format(self.__class__.__name__))
30 cdef void init(self, const shared_ptr[CScalar]& wrapped):
31 self.wrapped = wrapped
34 cdef wrap(const shared_ptr[CScalar]& wrapped):
37 Type type_id = wrapped.get().type.get().id()
39 if type_id == _Type_NA:
43 typ = _scalar_classes[type_id]
45 raise NotImplementedError(
46 "Wrapping scalar of type " +
47 frombytes(wrapped.get().type.get().ToString()))
48 self = typ.__new__(typ)
53 cdef inline shared_ptr[CScalar] unwrap(self) nogil:
59 Data type of the Scalar object.
61 return pyarrow_wrap_data_type(self.wrapped.get().type)
66 Holds a valid (non-null) value.
68 return self.wrapped.get().is_valid
70 def cast(self, object target_type):
72 Attempt a safe cast to target data type.
75 DataType type = ensure_type(target_type)
76 shared_ptr[CScalar] result
79 result = GetResultValue(self.wrapped.get().CastTo(type.sp_type))
81 return Scalar.wrap(result)
84 return '<pyarrow.{}: {!r}>'.format(
85 self.__class__.__name__, self.as_py()
89 return str(self.as_py())
91 def equals(self, Scalar other not None):
92 return self.wrapped.get().Equals(other.unwrap().get()[0])
94 def __eq__(self, other):
96 return self.equals(other)
101 cdef CScalarHash hasher
102 return hasher(self.wrapped)
104 def __reduce__(self):
105 return scalar, (self.as_py(), self.type)
108 raise NotImplementedError()
114 cdef class NullScalar(Scalar):
116 Concrete class for null scalars.
122 raise RuntimeError('Cannot create multiple NullScalar instances')
123 self.init(shared_ptr[CScalar](new CNullScalar()))
130 Return this value as a Python None.
135 _NULL = NA = NullScalar()
138 cdef class BooleanScalar(Scalar):
140 Concrete class for boolean scalars.
145 Return this value as a Python bool.
147 cdef CBooleanScalar* sp = <CBooleanScalar*> self.wrapped.get()
148 return sp.value if sp.is_valid else None
151 cdef class UInt8Scalar(Scalar):
153 Concrete class for uint8 scalars.
158 Return this value as a Python int.
160 cdef CUInt8Scalar* sp = <CUInt8Scalar*> self.wrapped.get()
161 return sp.value if sp.is_valid else None
164 cdef class Int8Scalar(Scalar):
166 Concrete class for int8 scalars.
171 Return this value as a Python int.
173 cdef CInt8Scalar* sp = <CInt8Scalar*> self.wrapped.get()
174 return sp.value if sp.is_valid else None
177 cdef class UInt16Scalar(Scalar):
179 Concrete class for uint16 scalars.
184 Return this value as a Python int.
186 cdef CUInt16Scalar* sp = <CUInt16Scalar*> self.wrapped.get()
187 return sp.value if sp.is_valid else None
190 cdef class Int16Scalar(Scalar):
192 Concrete class for int16 scalars.
197 Return this value as a Python int.
199 cdef CInt16Scalar* sp = <CInt16Scalar*> self.wrapped.get()
200 return sp.value if sp.is_valid else None
203 cdef class UInt32Scalar(Scalar):
205 Concrete class for uint32 scalars.
210 Return this value as a Python int.
212 cdef CUInt32Scalar* sp = <CUInt32Scalar*> self.wrapped.get()
213 return sp.value if sp.is_valid else None
216 cdef class Int32Scalar(Scalar):
218 Concrete class for int32 scalars.
223 Return this value as a Python int.
225 cdef CInt32Scalar* sp = <CInt32Scalar*> self.wrapped.get()
226 return sp.value if sp.is_valid else None
229 cdef class UInt64Scalar(Scalar):
231 Concrete class for uint64 scalars.
236 Return this value as a Python int.
238 cdef CUInt64Scalar* sp = <CUInt64Scalar*> self.wrapped.get()
239 return sp.value if sp.is_valid else None
242 cdef class Int64Scalar(Scalar):
244 Concrete class for int64 scalars.
249 Return this value as a Python int.
251 cdef CInt64Scalar* sp = <CInt64Scalar*> self.wrapped.get()
252 return sp.value if sp.is_valid else None
255 cdef class HalfFloatScalar(Scalar):
257 Concrete class for float scalars.
262 Return this value as a Python float.
264 cdef CHalfFloatScalar* sp = <CHalfFloatScalar*> self.wrapped.get()
265 return PyHalf_FromHalf(sp.value) if sp.is_valid else None
268 cdef class FloatScalar(Scalar):
270 Concrete class for float scalars.
275 Return this value as a Python float.
277 cdef CFloatScalar* sp = <CFloatScalar*> self.wrapped.get()
278 return sp.value if sp.is_valid else None
281 cdef class DoubleScalar(Scalar):
283 Concrete class for double scalars.
288 Return this value as a Python float.
290 cdef CDoubleScalar* sp = <CDoubleScalar*> self.wrapped.get()
291 return sp.value if sp.is_valid else None
294 cdef class Decimal128Scalar(Scalar):
296 Concrete class for decimal128 scalars.
301 Return this value as a Python Decimal.
304 CDecimal128Scalar* sp = <CDecimal128Scalar*> self.wrapped.get()
305 CDecimal128Type* dtype = <CDecimal128Type*> sp.type.get()
307 return _pydecimal.Decimal(
308 frombytes(sp.value.ToString(dtype.scale()))
314 cdef class Decimal256Scalar(Scalar):
316 Concrete class for decimal256 scalars.
321 Return this value as a Python Decimal.
324 CDecimal256Scalar* sp = <CDecimal256Scalar*> self.wrapped.get()
325 CDecimal256Type* dtype = <CDecimal256Type*> sp.type.get()
327 return _pydecimal.Decimal(
328 frombytes(sp.value.ToString(dtype.scale()))
334 cdef class Date32Scalar(Scalar):
336 Concrete class for date32 scalars.
341 Return this value as a Python datetime.datetime instance.
343 cdef CDate32Scalar* sp = <CDate32Scalar*> self.wrapped.get()
346 # shift to seconds since epoch
348 datetime.date(1970, 1, 1) + datetime.timedelta(days=sp.value)
354 cdef class Date64Scalar(Scalar):
356 Concrete class for date64 scalars.
361 Return this value as a Python datetime.datetime instance.
363 cdef CDate64Scalar* sp = <CDate64Scalar*> self.wrapped.get()
367 datetime.date(1970, 1, 1) +
368 datetime.timedelta(days=sp.value / 86400000)
374 def _datetime_from_int(int64_t value, TimeUnit unit, tzinfo=None):
375 if unit == TimeUnit_SECOND:
376 delta = datetime.timedelta(seconds=value)
377 elif unit == TimeUnit_MILLI:
378 delta = datetime.timedelta(milliseconds=value)
379 elif unit == TimeUnit_MICRO:
380 delta = datetime.timedelta(microseconds=value)
382 # TimeUnit_NANO: prefer pandas timestamps if available
383 if _pandas_api.have_pandas:
384 return _pandas_api.pd.Timestamp(value, tz=tzinfo, unit='ns')
385 # otherwise safely truncate to microsecond resolution datetime
386 if value % 1000 != 0:
388 "Nanosecond resolution temporal type {} is not safely "
389 "convertible to microseconds to convert to datetime.datetime. "
390 "Install pandas to return as Timestamp with nanosecond "
391 "support or access the .value attribute.".format(value)
393 delta = datetime.timedelta(microseconds=value // 1000)
395 dt = datetime.datetime(1970, 1, 1) + delta
396 # adjust timezone if set to the datatype
397 if tzinfo is not None:
398 dt = tzinfo.fromutc(dt)
403 cdef class Time32Scalar(Scalar):
405 Concrete class for time32 scalars.
410 Return this value as a Python datetime.timedelta instance.
413 CTime32Scalar* sp = <CTime32Scalar*> self.wrapped.get()
414 CTime32Type* dtype = <CTime32Type*> sp.type.get()
417 return _datetime_from_int(sp.value, unit=dtype.unit()).time()
422 cdef class Time64Scalar(Scalar):
424 Concrete class for time64 scalars.
429 Return this value as a Python datetime.timedelta instance.
432 CTime64Scalar* sp = <CTime64Scalar*> self.wrapped.get()
433 CTime64Type* dtype = <CTime64Type*> sp.type.get()
436 return _datetime_from_int(sp.value, unit=dtype.unit()).time()
441 cdef class TimestampScalar(Scalar):
443 Concrete class for timestamp scalars.
448 cdef CTimestampScalar* sp = <CTimestampScalar*> self.wrapped.get()
449 return sp.value if sp.is_valid else None
453 Return this value as a Pandas Timestamp instance (if units are
454 nanoseconds and pandas is available), otherwise as a Python
455 datetime.datetime instance.
458 CTimestampScalar* sp = <CTimestampScalar*> self.wrapped.get()
459 CTimestampType* dtype = <CTimestampType*> sp.type.get()
464 if not dtype.timezone().empty():
465 tzinfo = string_to_tzinfo(frombytes(dtype.timezone()))
469 return _datetime_from_int(sp.value, unit=dtype.unit(), tzinfo=tzinfo)
472 cdef class DurationScalar(Scalar):
474 Concrete class for duration scalars.
479 cdef CDurationScalar* sp = <CDurationScalar*> self.wrapped.get()
480 return sp.value if sp.is_valid else None
484 Return this value as a Pandas Timedelta instance (if units are
485 nanoseconds and pandas is available), otherwise as a Python
486 datetime.timedelta instance.
489 CDurationScalar* sp = <CDurationScalar*> self.wrapped.get()
490 CDurationType* dtype = <CDurationType*> sp.type.get()
491 TimeUnit unit = dtype.unit()
496 if unit == TimeUnit_SECOND:
497 return datetime.timedelta(seconds=sp.value)
498 elif unit == TimeUnit_MILLI:
499 return datetime.timedelta(milliseconds=sp.value)
500 elif unit == TimeUnit_MICRO:
501 return datetime.timedelta(microseconds=sp.value)
503 # TimeUnit_NANO: prefer pandas timestamps if available
504 if _pandas_api.have_pandas:
505 return _pandas_api.pd.Timedelta(sp.value, unit='ns')
506 # otherwise safely truncate to microsecond resolution timedelta
507 if sp.value % 1000 != 0:
509 "Nanosecond duration {} is not safely convertible to "
510 "microseconds to convert to datetime.timedelta. Install "
511 "pandas to return as Timedelta with nanosecond support or "
512 "access the .value attribute.".format(sp.value)
514 return datetime.timedelta(microseconds=sp.value // 1000)
517 cdef class MonthDayNanoIntervalScalar(Scalar):
519 Concrete class for month, day, nanosecond interval scalars.
531 Return this value as a pyarrow.MonthDayNano.
535 CMonthDayNanoIntervalScalar* scalar
536 scalar = <CMonthDayNanoIntervalScalar*>self.wrapped.get()
537 val = GetResultValue(MonthDayNanoIntervalScalarToPyObject(
539 return PyObject_to_object(val)
542 cdef class BinaryScalar(Scalar):
544 Concrete class for binary-like scalars.
549 Return a view over this value as a Buffer object.
551 cdef CBaseBinaryScalar* sp = <CBaseBinaryScalar*> self.wrapped.get()
552 return pyarrow_wrap_buffer(sp.value) if sp.is_valid else None
556 Return this value as a Python bytes.
558 buffer = self.as_buffer()
559 return None if buffer is None else buffer.to_pybytes()
562 cdef class LargeBinaryScalar(BinaryScalar):
566 cdef class FixedSizeBinaryScalar(BinaryScalar):
570 cdef class StringScalar(BinaryScalar):
572 Concrete class for string-like (utf8) scalars.
577 Return this value as a Python string.
579 buffer = self.as_buffer()
580 return None if buffer is None else str(buffer, 'utf8')
583 cdef class LargeStringScalar(StringScalar):
587 cdef class ListScalar(Scalar):
589 Concrete class for list-like scalars.
594 cdef CBaseListScalar* sp = <CBaseListScalar*> self.wrapped.get()
596 return pyarrow_wrap_array(sp.value)
602 Return the number of values.
604 return len(self.values)
606 def __getitem__(self, i):
608 Return the value at the given index.
610 return self.values[_normalize_index(i, len(self))]
614 Iterate over this element's values.
616 return iter(self.values)
620 Return this value as a Python list.
623 return None if arr is None else arr.to_pylist()
626 cdef class FixedSizeListScalar(ListScalar):
630 cdef class LargeListScalar(ListScalar):
634 cdef class StructScalar(Scalar, collections.abc.Mapping):
636 Concrete class for struct scalars.
640 cdef CStructScalar* sp = <CStructScalar*> self.wrapped.get()
641 return sp.value.size()
645 CStructScalar* sp = <CStructScalar*> self.wrapped.get()
646 CStructType* dtype = <CStructType*> sp.type.get()
647 vector[shared_ptr[CField]] fields = dtype.fields()
649 for i in range(dtype.num_fields()):
650 yield frombytes(fields[i].get().name())
653 return ((key, self[i]) for i, key in enumerate(self))
655 def __contains__(self, key):
656 return key in list(self)
658 def __getitem__(self, key):
660 Return the child value for the given field.
664 index : Union[int, str]
665 Index / position or name of the field.
673 CStructScalar* sp = <CStructScalar*> self.wrapped.get()
675 if isinstance(key, (bytes, str)):
676 ref = CFieldRef(<c_string> tobytes(key))
677 elif isinstance(key, int):
678 ref = CFieldRef(<int> key)
680 raise TypeError('Expected integer or string index')
683 return Scalar.wrap(GetResultValue(sp.field(ref)))
684 except ArrowInvalid as exc:
685 if isinstance(key, int):
686 raise IndexError(key) from exc
688 raise KeyError(key) from exc
692 Return this value as a Python dict.
696 return {k: self[k].as_py() for k in self.keys()}
699 "Converting to Python dictionary is not supported when "
700 "duplicate field names are present")
704 def _as_py_tuple(self):
705 # a version that returns a tuple instead of dict to support repr/str
706 # with the presence of duplicate field names
708 return [(key, self[i].as_py()) for i, key in enumerate(self)]
713 return '<pyarrow.{}: {!r}>'.format(
714 self.__class__.__name__, self._as_py_tuple()
718 return str(self._as_py_tuple())
721 cdef class MapScalar(ListScalar):
723 Concrete class for map scalars.
726 def __getitem__(self, i):
728 Return the value at the given index.
733 dct = arr[_normalize_index(i, len(arr))]
734 return (dct['key'], dct['value'])
738 Iterate over this element's values.
743 for k, v in zip(arr.field('key'), arr.field('value')):
744 yield (k.as_py(), v.as_py())
748 Return this value as a Python list.
750 cdef CStructScalar* sp = <CStructScalar*> self.wrapped.get()
751 return list(self) if sp.is_valid else None
754 cdef class DictionaryScalar(Scalar):
756 Concrete class for dictionary-encoded scalars.
760 def _reconstruct(cls, type, is_valid, index, dictionary):
762 CDictionaryScalarIndexAndDictionary value
763 shared_ptr[CDictionaryScalar] wrapped
768 type_ = ensure_type(type, allow_none=False)
769 if not isinstance(type_, DictionaryType):
770 raise TypeError('Must pass a DictionaryType instance')
772 if isinstance(index, Scalar):
773 if not index.type.equals(type.index_type):
774 raise TypeError("The Scalar value passed as index must have "
775 "identical type to the dictionary type's "
779 index_ = scalar(index, type=type_.index_type)
781 if isinstance(dictionary, Array):
782 if not dictionary.type.equals(type.value_type):
783 raise TypeError("The Array passed as dictionary must have "
784 "identical type to the dictionary type's "
786 dictionary_ = dictionary
788 dictionary_ = array(dictionary, type=type_.value_type)
790 value.index = pyarrow_unwrap_scalar(index_)
791 value.dictionary = pyarrow_unwrap_array(dictionary_)
793 wrapped = make_shared[CDictionaryScalar](
794 value, pyarrow_unwrap_data_type(type_), <c_bool>(is_valid)
796 return Scalar.wrap(<shared_ptr[CScalar]> wrapped)
798 def __reduce__(self):
799 return DictionaryScalar._reconstruct, (
800 self.type, self.is_valid, self.index, self.dictionary
806 Return this value's underlying index as a scalar.
808 cdef CDictionaryScalar* sp = <CDictionaryScalar*> self.wrapped.get()
809 return Scalar.wrap(sp.value.index)
814 Return the encoded value as a scalar.
816 cdef CDictionaryScalar* sp = <CDictionaryScalar*> self.wrapped.get()
817 return Scalar.wrap(GetResultValue(sp.GetEncodedValue()))
820 def dictionary(self):
821 cdef CDictionaryScalar* sp = <CDictionaryScalar*> self.wrapped.get()
822 return pyarrow_wrap_array(sp.value.dictionary)
826 Return this encoded value as a Python object.
828 return self.value.as_py() if self.is_valid else None
831 def index_value(self):
832 warnings.warn("`index_value` property is deprecated as of 1.0.0"
833 "please use the `index` property instead",
838 def dictionary_value(self):
839 warnings.warn("`dictionary_value` property is deprecated as of 1.0.0, "
840 "please use the `value` property instead", FutureWarning)
844 cdef class UnionScalar(Scalar):
846 Concrete class for Union scalars.
852 Return underlying value as a scalar.
854 cdef CUnionScalar* sp = <CUnionScalar*> self.wrapped.get()
855 return Scalar.wrap(sp.value) if sp.is_valid else None
859 Return underlying value as a Python object.
862 return None if value is None else value.as_py()
867 Return the union type code for this scalar.
869 cdef CUnionScalar* sp = <CUnionScalar*> self.wrapped.get()
873 cdef class ExtensionScalar(Scalar):
875 Concrete class for Extension scalars.
881 Return storage value as a scalar.
883 cdef CExtensionScalar* sp = <CExtensionScalar*> self.wrapped.get()
884 return Scalar.wrap(sp.value) if sp.is_valid else None
888 Return this scalar as a Python object.
890 # XXX should there be a hook to wrap the result in a custom class?
892 return None if value is None else value.as_py()
895 def from_storage(BaseExtensionType typ, value):
897 Construct ExtensionScalar from type and storage value.
902 The extension type for the result scalar.
904 The storage value for the result scalar.
908 ext_scalar : ExtensionScalar
911 shared_ptr[CExtensionScalar] sp_scalar
912 CExtensionScalar* ext_scalar
916 elif isinstance(value, Scalar):
917 if value.type != typ.storage_type:
918 raise TypeError("Incompatible storage type {0} "
919 "for extension type {1}"
920 .format(value.type, typ))
923 storage = scalar(value, typ.storage_type)
925 sp_scalar = make_shared[CExtensionScalar](typ.sp_type)
926 ext_scalar = sp_scalar.get()
927 ext_scalar.is_valid = storage is not None and storage.is_valid
928 if ext_scalar.is_valid:
929 ext_scalar.value = pyarrow_unwrap_scalar(storage)
930 check_status(ext_scalar.Validate())
931 return pyarrow_wrap_scalar(<shared_ptr[CScalar]> sp_scalar)
934 cdef dict _scalar_classes = {
935 _Type_BOOL: BooleanScalar,
936 _Type_UINT8: UInt8Scalar,
937 _Type_UINT16: UInt16Scalar,
938 _Type_UINT32: UInt32Scalar,
939 _Type_UINT64: UInt64Scalar,
940 _Type_INT8: Int8Scalar,
941 _Type_INT16: Int16Scalar,
942 _Type_INT32: Int32Scalar,
943 _Type_INT64: Int64Scalar,
944 _Type_HALF_FLOAT: HalfFloatScalar,
945 _Type_FLOAT: FloatScalar,
946 _Type_DOUBLE: DoubleScalar,
947 _Type_DECIMAL128: Decimal128Scalar,
948 _Type_DECIMAL256: Decimal256Scalar,
949 _Type_DATE32: Date32Scalar,
950 _Type_DATE64: Date64Scalar,
951 _Type_TIME32: Time32Scalar,
952 _Type_TIME64: Time64Scalar,
953 _Type_TIMESTAMP: TimestampScalar,
954 _Type_DURATION: DurationScalar,
955 _Type_BINARY: BinaryScalar,
956 _Type_LARGE_BINARY: LargeBinaryScalar,
957 _Type_FIXED_SIZE_BINARY: FixedSizeBinaryScalar,
958 _Type_STRING: StringScalar,
959 _Type_LARGE_STRING: LargeStringScalar,
960 _Type_LIST: ListScalar,
961 _Type_LARGE_LIST: LargeListScalar,
962 _Type_FIXED_SIZE_LIST: FixedSizeListScalar,
963 _Type_STRUCT: StructScalar,
964 _Type_MAP: MapScalar,
965 _Type_DICTIONARY: DictionaryScalar,
966 _Type_SPARSE_UNION: UnionScalar,
967 _Type_DENSE_UNION: UnionScalar,
968 _Type_INTERVAL_MONTH_DAY_NANO: MonthDayNanoIntervalScalar,
969 _Type_EXTENSION: ExtensionScalar,
973 def scalar(value, type=None, *, from_pandas=None, MemoryPool memory_pool=None):
975 Create a pyarrow.Scalar instance from a Python object.
980 Python object coercible to arrow's type system.
981 type : pyarrow.DataType
982 Explicit type to attempt to coerce to, otherwise will be inferred from
984 from_pandas : bool, default None
985 Use pandas's semantics for inferring nulls from values in
986 ndarray-like data. Defaults to False if not passed explicitly by user,
987 or True if a pandas object is passed in.
988 memory_pool : pyarrow.MemoryPool, optional
989 If not passed, will allocate memory from the currently-set default
994 scalar : pyarrow.Scalar
998 >>> import pyarrow as pa
1001 <pyarrow.Int64Scalar: 42>
1003 >>> pa.scalar("string")
1004 <pyarrow.StringScalar: 'string'>
1006 >>> pa.scalar([1, 2])
1007 <pyarrow.ListScalar: [1, 2]>
1009 >>> pa.scalar([1, 2], type=pa.list_(pa.int16()))
1010 <pyarrow.ListScalar: [1, 2]>
1014 PyConversionOptions options
1015 shared_ptr[CScalar] scalar
1016 shared_ptr[CArray] array
1017 shared_ptr[CChunkedArray] chunked
1018 bint is_pandas_object = False
1021 type = ensure_type(type, allow_none=True)
1022 pool = maybe_unbox_memory_pool(memory_pool)
1024 if _is_array_like(value):
1025 value = get_values(value, &is_pandas_object)
1029 if type is not None:
1030 ty = ensure_type(type)
1031 options.type = ty.sp_type
1033 if from_pandas is None:
1034 options.from_pandas = is_pandas_object
1036 options.from_pandas = from_pandas
1040 chunked = GetResultValue(ConvertPySequence(value, None, options, pool))
1042 # get the first chunk
1043 assert chunked.get().num_chunks() == 1
1044 array = chunked.get().chunk(0)
1046 # retrieve the scalar from the first position
1047 scalar = GetResultValue(array.get().GetScalar(0))
1048 return Scalar.wrap(scalar)