]> git.proxmox.com Git - ceph.git/blame - ceph/src/arrow/go/arrow/scalar/scalar_test.go
projects / ceph.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
import quincy 17.2.0
[ceph.git] / ceph / src / arrow / go / arrow / scalar / scalar_test.go
CommitLineData
1d09f67e
TL		 1// Licensed to the Apache Software Foundation (ASF) under one
2// or more contributor license agreements. See the NOTICE file
3// distributed with this work for additional information
4// regarding copyright ownership. The ASF licenses this file
5// to you under the Apache License, Version 2.0 (the
6// "License"); you may not use this file except in compliance
7// with the License. You may obtain a copy of the License at
8//
9// http://www.apache.org/licenses/LICENSE-2.0
10//
11// Unless required by applicable law or agreed to in writing, software
12// distributed under the License is distributed on an "AS IS" BASIS,
13// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14// See the License for the specific language governing permissions and
15// limitations under the License.
16
17package scalar_test
18
19import (
20 "bytes"
21 "hash/maphash"
22 "math/bits"
23 "testing"
24 "time"
25
26 "github.com/apache/arrow/go/v6/arrow"
27 "github.com/apache/arrow/go/v6/arrow/array"
28 "github.com/apache/arrow/go/v6/arrow/decimal128"
29 "github.com/apache/arrow/go/v6/arrow/memory"
30 "github.com/apache/arrow/go/v6/arrow/scalar"
31 "github.com/stretchr/testify/assert"
32 "github.com/stretchr/testify/suite"
33)
34
35func assertScalarsEqual(t *testing.T, expected, actual scalar.Scalar) {
36 assert.Truef(t, scalar.Equals(expected, actual), "Expected:\n%s\nActual:\n%s", expected, actual)
37 seed := maphash.MakeSeed()
38 assert.Equal(t, scalar.Hash(seed, expected), scalar.Hash(seed, actual))
39}
40
41func assertMakeScalarParam(t *testing.T, expected scalar.Scalar, dt arrow.DataType, val interface{}) {
42 out, err := scalar.MakeScalarParam(val, dt)
43 assert.NoError(t, err)
44 assert.NoError(t, out.Validate())
45 assert.NoError(t, out.ValidateFull())
46 assertScalarsEqual(t, expected, out)
47}
48
49func assertMakeScalar(t *testing.T, expected scalar.Scalar, val interface{}) {
50 out := scalar.MakeScalar(val)
51 assert.NoError(t, out.Validate())
52 assert.NoError(t, out.ValidateFull())
53 assertScalarsEqual(t, expected, out)
54}
55
56func assertParseScalar(t *testing.T, dt arrow.DataType, str string, expected scalar.Scalar) {
57 out, err := scalar.ParseScalar(dt, str)
58 assert.NoError(t, err)
59 assert.NoError(t, out.Validate())
60 assert.NoError(t, out.ValidateFull())
61 assertScalarsEqual(t, expected, out)
62}
63
64func TestMakeScalarInt(t *testing.T) {
65 three := scalar.MakeScalar(int(3))
66 assert.NoError(t, three.ValidateFull())
67
68 var expected scalar.Scalar
69 if bits.UintSize == 32 {
70 expected = scalar.NewInt32Scalar(3)
71 } else {
72 expected = scalar.NewInt64Scalar(3)
73 }
74
75 assert.Equal(t, expected, three)
76 assertMakeScalar(t, expected, int(3))
77 assertParseScalar(t, expected.DataType(), "3", expected)
78}
79
80func checkMakeNullScalar(t *testing.T, dt arrow.DataType) scalar.Scalar {
81 s := scalar.MakeNullScalar(dt)
82 assert.NoError(t, s.Validate())
83 assert.NoError(t, s.ValidateFull())
84 assert.True(t, arrow.TypeEqual(s.DataType(), dt))
85 assert.False(t, s.IsValid())
86 return s
87}
88
89func TestMakeScalarUint(t *testing.T) {
90 three := scalar.MakeScalar(uint(3))
91 assert.NoError(t, three.ValidateFull())
92
93 var expected scalar.Scalar
94 if bits.UintSize == 32 {
95 expected = scalar.NewUint32Scalar(3)
96 } else {
97 expected = scalar.NewUint64Scalar(3)
98 }
99
100 assert.Equal(t, expected, three)
101 assertMakeScalar(t, expected, uint(3))
102 assertParseScalar(t, expected.DataType(), "3", expected)
103}
104
105func TestBasicDecimal128(t *testing.T) {
106 ty := &arrow.Decimal128Type{Precision: 3, Scale: 2}
107 pi := scalar.NewDecimal128Scalar(decimal128.New(0, 314), ty)
108 pi2 := scalar.NewDecimal128Scalar(decimal128.FromI64(628), ty)
109 null := checkMakeNullScalar(t, ty)
110
111 assert.NoError(t, pi.ValidateFull())
112 assert.True(t, pi.IsValid())
113 assert.Equal(t, decimal128.FromI64(314), pi.Value)
114
115 assert.NoError(t, null.ValidateFull())
116 assert.False(t, null.IsValid())
117
118 assert.False(t, scalar.Equals(pi, pi2))
119}
120
121func TestBinaryScalarBasics(t *testing.T) {
122 data := "test data"
123 buf := memory.NewBufferBytes([]byte(data))
124
125 value := scalar.NewBinaryScalar(buf, arrow.BinaryTypes.Binary)
126 assert.NoError(t, value.ValidateFull())
127 assert.True(t, bytes.Equal(value.Value.Bytes(), buf.Bytes()))
128 assert.True(t, value.IsValid())
129 assert.True(t, arrow.TypeEqual(value.DataType(), arrow.BinaryTypes.Binary))
130
131 nullValue := checkMakeNullScalar(t, arrow.BinaryTypes.Binary)
132 assert.False(t, nullValue.IsValid())
133 assert.Nil(t, nullValue.(*scalar.Binary).Value)
134 assert.NoError(t, nullValue.ValidateFull())
135
136 value2 := scalar.NewStringScalarFromBuffer(buf)
137 assert.NoError(t, value2.ValidateFull())
138 assert.True(t, bytes.Equal(value2.Value.Bytes(), buf.Bytes()))
139 assert.True(t, value2.IsValid())
140 assert.True(t, arrow.TypeEqual(arrow.BinaryTypes.String, value2.DataType()))
141
142 assert.NotEqual(t, value2, value)
143 assert.False(t, scalar.Equals(value2, value))
144
145 value3 := scalar.NewStringScalar(data)
146 assert.True(t, scalar.Equals(value2, value3))
147}
148
149func TestBinaryScalarValidateErrors(t *testing.T) {
150 sc := scalar.NewBinaryScalar(memory.NewBufferBytes([]byte("xxx")), arrow.BinaryTypes.Binary)
151 sc.Valid = false
152 assert.Error(t, sc.Validate())
153 assert.Error(t, sc.ValidateFull())
154
155 nullScalar := scalar.MakeNullScalar(arrow.BinaryTypes.Binary)
156 nullScalar.(*scalar.Binary).Valid = true
157 assert.Error(t, sc.Validate())
158 assert.Error(t, sc.ValidateFull())
159}
160
161func TestStringMakeScalar(t *testing.T) {
162 assertMakeScalar(t, scalar.NewStringScalar("three"), "three")
163 assertParseScalar(t, arrow.BinaryTypes.String, "three", scalar.NewStringScalar("three"))
164}
165
166func TestStringScalarValidateErrors(t *testing.T) {
167 sc := scalar.NewStringScalar("xxx")
168 sc.Valid = false
169 assert.Error(t, sc.Validate())
170 assert.Error(t, sc.ValidateFull())
171
172 nullScalar := scalar.MakeNullScalar(arrow.BinaryTypes.String)
173 nullScalar.(*scalar.String).Valid = true
174 assert.Error(t, sc.Validate())
175 assert.Error(t, sc.ValidateFull())
176
177 // invalid utf8
178 sc = scalar.NewStringScalarFromBuffer(memory.NewBufferBytes([]byte{0xff}))
179 assert.NoError(t, sc.Validate())
180 assert.Error(t, sc.ValidateFull())
181}
182
183func TestFixedSizeBinaryScalarBasics(t *testing.T) {
184 data := "test data"
185 buf := memory.NewBufferBytes([]byte(data))
186
187 exType := &arrow.FixedSizeBinaryType{ByteWidth: 9}
188
189 value := scalar.NewFixedSizeBinaryScalar(buf, exType)
190 assert.NoError(t, value.ValidateFull())
191 assert.True(t, bytes.Equal(value.Value.Bytes(), buf.Bytes()))
192 assert.True(t, value.Valid)
193 assert.True(t, arrow.TypeEqual(value.DataType(), exType))
194
195 nullValue := scalar.MakeNullScalar(exType)
196 assert.NoError(t, nullValue.ValidateFull())
197 assert.False(t, nullValue.IsValid())
198 assert.Nil(t, nullValue.(*scalar.FixedSizeBinary).Value)
199}
200
201func TestFixedSizeBinaryMakeScalar(t *testing.T) {
202 data := "test data"
203 buf := memory.NewBufferBytes([]byte(data))
204 exType := &arrow.FixedSizeBinaryType{ByteWidth: 9}
205
206 assertMakeScalarParam(t, scalar.NewFixedSizeBinaryScalar(buf, exType), exType, buf)
207 assertParseScalar(t, exType, data, scalar.NewFixedSizeBinaryScalar(buf, exType))
208
209 _, err := scalar.MakeScalarParam(buf.Bytes()[:3], exType)
210 assert.Error(t, err)
211 _, err = scalar.ParseScalar(exType, data[:3])
212 assert.Error(t, err)
213}
214
215func TestFixedSizeBinaryScalarValidateErrors(t *testing.T) {
216 data := "test data"
217 buf := memory.NewBufferBytes([]byte(data))
218 exType := &arrow.FixedSizeBinaryType{ByteWidth: 9}
219
220 value := scalar.NewFixedSizeBinaryScalar(buf, exType)
221 assert.NoError(t, value.ValidateFull())
222
223 value.Value.Reset(buf.Bytes()[:1])
224 assert.Error(t, value.ValidateFull())
225}
226
227func TestDateScalarBasics(t *testing.T) {
228 i32Val := arrow.Date32(1)
229 date32Val := scalar.NewDate32Scalar(i32Val)
230 date32Null := scalar.MakeNullScalar(arrow.FixedWidthTypes.Date32)
231 assert.NoError(t, date32Null.ValidateFull())
232 assert.NoError(t, date32Val.ValidateFull())
233
234 assert.True(t, arrow.TypeEqual(arrow.FixedWidthTypes.Date32, date32Val.DataType()))
235 assert.True(t, date32Val.IsValid())
236 assert.False(t, date32Null.IsValid())
237
238 i64Val := arrow.Date64(2)
239 date64Val := scalar.NewDate64Scalar(i64Val)
240 date64Null := scalar.MakeNullScalar(arrow.FixedWidthTypes.Date64)
241 assert.NoError(t, date64Null.ValidateFull())
242 assert.NoError(t, date64Val.ValidateFull())
243
244 assert.True(t, arrow.TypeEqual(arrow.FixedWidthTypes.Date64, date64Val.DataType()))
245 assert.True(t, date64Val.IsValid())
246 assert.False(t, date64Null.IsValid())
247}
248
249func TestDateScalarMakeScalar(t *testing.T) {
250 assertMakeScalar(t, scalar.NewDate32Scalar(arrow.Date32(1)), arrow.Date32(1))
251 assertParseScalar(t, arrow.FixedWidthTypes.Date32, "1454-10-22", scalar.NewDate32Scalar(arrow.Date32(-188171)))
252 assert.Equal(t, "1454-10-22", scalar.NewDate32Scalar(arrow.Date32(-188171)).String())
253
254 assertMakeScalar(t, scalar.NewDate64Scalar(arrow.Date64(1)), arrow.Date64(1))
255 assertParseScalar(t, arrow.FixedWidthTypes.Date64, "1454-10-22", scalar.NewDate64Scalar(arrow.Date64(-188171*(time.Hour*24).Milliseconds())))
256 assert.Equal(t, "1454-10-22", scalar.NewDate64Scalar(arrow.Date64(-188171*(time.Hour*24).Milliseconds())).String())
257
258 d32 := scalar.NewDate32Scalar(arrow.Date32(-188171))
259 d64 := scalar.NewDate64Scalar(arrow.Date64(-188171 * (time.Hour * 24).Milliseconds()))
260
261 d32Casted, err := d32.CastTo(arrow.FixedWidthTypes.Date64)
262 assert.NoError(t, err)
263 assert.True(t, scalar.Equals(d64, d32Casted))
264
265 d64Casted, err := d64.CastTo(arrow.FixedWidthTypes.Date32)
266 assert.NoError(t, err)
267 assert.True(t, scalar.Equals(d64Casted, d32))
268}
269
270func TestTimeScalarsBasics(t *testing.T) {
271 typ1 := arrow.FixedWidthTypes.Time32ms
272 typ2 := arrow.FixedWidthTypes.Time32s
273 typ3 := arrow.FixedWidthTypes.Time64us
274 typ4 := arrow.FixedWidthTypes.Time64ns
275
276 t32val := arrow.Time32(1)
277 time32Val := scalar.NewTime32Scalar(t32val, typ1)
278 time32Null := scalar.MakeNullScalar(typ2)
279 assert.NoError(t, time32Val.ValidateFull())
280 assert.NoError(t, time32Null.ValidateFull())
281
282 assert.Equal(t, t32val, time32Val.Value)
283 assert.True(t, arrow.TypeEqual(time32Val.Type, typ1))
284 assert.True(t, time32Val.IsValid())
285 assert.False(t, time32Null.IsValid())
286 assert.True(t, arrow.TypeEqual(time32Null.DataType(), typ2))
287
288 t64val := arrow.Time64(1)
289 time64Val := scalar.NewTime64Scalar(t64val, typ3)
290 time64Null := scalar.MakeNullScalar(typ4)
291 assert.NoError(t, time64Val.ValidateFull())
292 assert.NoError(t, time64Null.ValidateFull())
293
294 assert.Equal(t, t64val, time64Val.Value)
295 assert.True(t, arrow.TypeEqual(time64Val.Type, typ3))
296 assert.True(t, time64Val.IsValid())
297 assert.False(t, time64Null.IsValid())
298 assert.True(t, arrow.TypeEqual(time64Null.DataType(), typ4))
299}
300
301func TestTimeScalarsMakeScalar(t *testing.T) {
302 typ1 := arrow.FixedWidthTypes.Time32s
303 typ2 := arrow.FixedWidthTypes.Time32ms
304 typ3 := arrow.FixedWidthTypes.Time64us
305 typ4 := arrow.FixedWidthTypes.Time64ns
306
307 assertMakeScalarParam(t, scalar.NewTime32Scalar(arrow.Time32(1), typ1), typ1, arrow.Time32(1))
308 assertMakeScalarParam(t, scalar.NewTime32Scalar(arrow.Time32(1), typ2), typ2, arrow.Time32(1))
309 assertMakeScalarParam(t, scalar.NewTime64Scalar(arrow.Time64(1), typ3), typ3, arrow.Time64(1))
310 assertMakeScalarParam(t, scalar.NewTime64Scalar(arrow.Time64(1), typ4), typ4, arrow.Time64(1))
311
312 tententen := 60*(60*(10)+10) + 10
313 assertParseScalar(t, typ1, "10:10:10", scalar.NewTime32Scalar(arrow.Time32(tententen), typ1))
314 assert.Equal(t, "10:10:10", scalar.NewTime32Scalar(arrow.Time32(tententen), typ1).String())
315
316 tententen = 1000*tententen + 123
317 assertParseScalar(t, typ2, "10:10:10.123", scalar.NewTime32Scalar(arrow.Time32(tententen), typ2))
318 assert.Equal(t, "10:10:10.123", scalar.NewTime32Scalar(arrow.Time32(tententen), typ2).String())
319
320 tententen = 1000*tententen + 456
321 assertParseScalar(t, typ3, "10:10:10.123456", scalar.NewTime64Scalar(arrow.Time64(tententen), typ3))
322 assert.Equal(t, "10:10:10.123456", scalar.NewTime64Scalar(arrow.Time64(tententen), typ3).String())
323
324 tententen = 1000*tententen + 789
325 assertParseScalar(t, typ4, "10:10:10.123456789", scalar.NewTime64Scalar(arrow.Time64(tententen), typ4))
326 assert.Equal(t, "10:10:10.123456789", scalar.NewTime64Scalar(arrow.Time64(tententen), typ4).String())
327}
328
329func TestTimestampScalarBasics(t *testing.T) {
330 typ1 := arrow.FixedWidthTypes.Timestamp_ms
331 typ2 := arrow.FixedWidthTypes.Timestamp_s
332
333 val1 := arrow.Timestamp(1)
334 val2 := arrow.Timestamp(2)
335 tsVal1 := scalar.NewTimestampScalar(val1, typ1)
336 tsVal2 := scalar.NewTimestampScalar(val2, typ2)
337 tsNull := scalar.MakeNullScalar(typ1)
338 assert.NoError(t, tsVal1.ValidateFull())
339 assert.NoError(t, tsVal2.ValidateFull())
340 assert.NoError(t, tsNull.ValidateFull())
341
342 assert.Equal(t, val1, tsVal1.Value)
343
344 assert.True(t, arrow.TypeEqual(tsVal1.Type, typ1))
345 assert.True(t, arrow.TypeEqual(tsVal2.DataType(), typ2))
346 assert.True(t, tsVal1.Valid)
347 assert.True(t, tsVal2.IsValid())
348 assert.False(t, tsNull.IsValid())
349 assert.True(t, arrow.TypeEqual(tsNull.DataType(), typ1))
350
351 assert.NotEqual(t, tsVal1, tsVal2)
352 assert.False(t, scalar.Equals(tsVal1, tsVal2))
353 assert.NotEqual(t, tsVal1, tsNull)
354 assert.False(t, scalar.Equals(tsVal1, tsNull))
355 assert.NotEqual(t, tsVal2, tsNull)
356 assert.False(t, scalar.Equals(tsVal2, tsNull))
357}
358
359func TestTimestampScalarsMakeScalar(t *testing.T) {
360 typ1 := arrow.FixedWidthTypes.Timestamp_ms
361 typ2 := arrow.FixedWidthTypes.Timestamp_s
362 typ3 := arrow.FixedWidthTypes.Timestamp_us
363 typ4 := arrow.FixedWidthTypes.Timestamp_ns
364
365 epochPlus1s := "1970-01-01 00:00:01"
366
367 assertMakeScalarParam(t, scalar.NewTimestampScalar(arrow.Timestamp(1), typ1), typ1, arrow.Timestamp(1))
368 assertParseScalar(t, typ1, epochPlus1s, scalar.NewTimestampScalar(1000, typ1))
369
370 assertMakeScalarParam(t, scalar.NewTimestampScalar(arrow.Timestamp(1), typ2), typ2, arrow.Timestamp(1))
371 assertParseScalar(t, typ2, epochPlus1s, scalar.NewTimestampScalar(arrow.Timestamp(1), typ2))
372
373 assertMakeScalarParam(t, scalar.NewTimestampScalar(arrow.Timestamp(1), typ3), typ3, arrow.Timestamp(1))
374 assertParseScalar(t, typ3, epochPlus1s, scalar.NewTimestampScalar(arrow.Timestamp(1000*1000), typ3))
375
376 assertMakeScalarParam(t, scalar.NewTimestampScalar(arrow.Timestamp(1), typ4), typ4, arrow.Timestamp(1))
377 assertParseScalar(t, typ4, epochPlus1s, scalar.NewTimestampScalar(arrow.Timestamp(1000*1000*1000), typ4))
378}
379
380func TestTimestampScalarsCasting(t *testing.T) {
381 convert := func(in, out arrow.TimeUnit, val arrow.Timestamp) arrow.Timestamp {
382 s, err := scalar.NewTimestampScalar(val, &arrow.TimestampType{Unit: in}).CastTo(&arrow.TimestampType{Unit: out})
383 assert.NoError(t, err)
384 return s.(*scalar.Timestamp).Value
385 }
386
387 assert.EqualValues(t, convert(arrow.Second, arrow.Millisecond, arrow.Timestamp(1)), 1000)
388 assert.EqualValues(t, convert(arrow.Second, arrow.Nanosecond, arrow.Timestamp(1)), 1000000000)
389
390 assert.EqualValues(t, convert(arrow.Nanosecond, arrow.Microsecond, arrow.Timestamp(1234)), 1)
391 assert.EqualValues(t, convert(arrow.Microsecond, arrow.Millisecond, arrow.Timestamp(4567)), 4)
392
393 str, err := scalar.NewTimestampScalar(arrow.Timestamp(1024), arrow.FixedWidthTypes.Timestamp_ms).CastTo(arrow.BinaryTypes.String)
394 assert.NoError(t, err)
395 assert.Truef(t, scalar.Equals(scalar.NewStringScalar("1970-01-01 00:00:01.024"), str), "expected: '1970-01-01 00:00:01.024', got: %s", str)
396
397 i64, err := scalar.NewTimestampScalar(arrow.Timestamp(1024), arrow.FixedWidthTypes.Timestamp_ms).CastTo(arrow.PrimitiveTypes.Int64)
398 assert.NoError(t, err)
399 assert.Truef(t, scalar.Equals(scalar.NewInt64Scalar(1024), i64), "expected 1024, got %s", i64)
400
401 const millisInDay = 86400000
402 d64, err := scalar.NewTimestampScalar(arrow.Timestamp(1024*millisInDay+3), arrow.FixedWidthTypes.Timestamp_ms).CastTo(arrow.FixedWidthTypes.Date64)
403 assert.NoError(t, err)
404
405 d32, err := scalar.NewTimestampScalar(arrow.Timestamp(1024*millisInDay+3), arrow.FixedWidthTypes.Timestamp_ms).CastTo(arrow.FixedWidthTypes.Date32)
406 assert.NoError(t, err)
407
408 assert.True(t, scalar.Equals(scalar.NewDate32Scalar(arrow.Date32(1024)), d32))
409 assert.Truef(t, scalar.Equals(scalar.NewDate64Scalar(arrow.Date64(1024*millisInDay)), d64), "got %s", d64)
410 tms, err := scalar.NewDate64Scalar(arrow.Date64(1024 * millisInDay)).CastTo(arrow.FixedWidthTypes.Timestamp_ms)
411 assert.NoError(t, err)
412 assert.True(t, scalar.Equals(tms, scalar.NewTimestampScalar(arrow.Timestamp(1024*millisInDay), arrow.FixedWidthTypes.Timestamp_ms)))
413
414 tms, err = scalar.NewDate32Scalar(arrow.Date32(1024)).CastTo(arrow.FixedWidthTypes.Timestamp_ms)
415 assert.NoError(t, err)
416 assert.True(t, scalar.Equals(tms, scalar.NewTimestampScalar(arrow.Timestamp(1024*millisInDay), arrow.FixedWidthTypes.Timestamp_ms)))
417}
418
419func TestDurationScalarBasics(t *testing.T) {
420 typ1 := arrow.FixedWidthTypes.Duration_ms
421 typ2 := arrow.FixedWidthTypes.Duration_s
422
423 val1 := arrow.Duration(1)
424 val2 := arrow.Duration(2)
425 tsVal1 := scalar.NewDurationScalar(val1, typ1)
426 tsVal2 := scalar.NewDurationScalar(val2, typ2)
427 tsNull := scalar.MakeNullScalar(typ1)
428 assert.NoError(t, tsVal1.ValidateFull())
429 assert.NoError(t, tsVal2.ValidateFull())
430 assert.NoError(t, tsNull.ValidateFull())
431
432 assert.Equal(t, val1, tsVal1.Value)
433
434 assert.True(t, arrow.TypeEqual(tsVal1.Type, typ1))
435 assert.True(t, arrow.TypeEqual(tsVal2.DataType(), typ2))
436 assert.True(t, tsVal1.Valid)
437 assert.False(t, tsNull.IsValid())
438 assert.True(t, arrow.TypeEqual(typ1, tsNull.DataType()))
439
440 assert.False(t, scalar.Equals(tsVal1, tsVal2))
441 assert.False(t, scalar.Equals(tsVal1, tsNull))
442 assert.False(t, scalar.Equals(tsNull, tsVal2))
443}
444
445func TestMonthIntervalScalarBasics(t *testing.T) {
446 typ1 := arrow.FixedWidthTypes.MonthInterval
447 typ2 := arrow.FixedWidthTypes.MonthInterval
448
449 val1 := arrow.MonthInterval(1)
450 val2 := arrow.MonthInterval(2)
451 tsVal1 := scalar.NewMonthIntervalScalar(val1)
452 tsVal2 := scalar.NewMonthIntervalScalar(val2)
453 tsNull := scalar.MakeNullScalar(typ1)
454 assert.NoError(t, tsVal1.ValidateFull())
455 assert.NoError(t, tsVal2.ValidateFull())
456 assert.NoError(t, tsNull.ValidateFull())
457
458 assert.Equal(t, val1, tsVal1.Value)
459
460 assert.True(t, arrow.TypeEqual(tsVal1.Type, typ1))
461 assert.True(t, arrow.TypeEqual(tsVal2.DataType(), typ2))
462 assert.True(t, tsVal1.Valid)
463 assert.False(t, tsNull.IsValid())
464 assert.True(t, arrow.TypeEqual(typ1, tsNull.DataType()))
465
466 assert.False(t, scalar.Equals(tsVal1, tsVal2))
467 assert.False(t, scalar.Equals(tsVal1, tsNull))
468 assert.False(t, scalar.Equals(tsNull, tsVal2))
469}
470
471func TestDayTimeIntervalScalarBasics(t *testing.T) {
472 typ := arrow.FixedWidthTypes.DayTimeInterval
473
474 val1 := arrow.DayTimeInterval{Days: 1, Milliseconds: 1}
475 val2 := arrow.DayTimeInterval{Days: 2, Milliseconds: 2}
476 tsVal1 := scalar.NewDayTimeIntervalScalar(val1)
477 tsVal2 := scalar.NewDayTimeIntervalScalar(val2)
478 tsNull := scalar.MakeNullScalar(typ)
479 assert.NoError(t, tsVal1.ValidateFull())
480 assert.NoError(t, tsVal2.ValidateFull())
481 assert.NoError(t, tsNull.ValidateFull())
482
483 assert.Equal(t, val1, tsVal1.Value)
484
485 assert.True(t, arrow.TypeEqual(tsVal1.Type, typ))
486 assert.True(t, arrow.TypeEqual(tsVal2.DataType(), typ))
487 assert.True(t, tsVal1.Valid)
488 assert.False(t, tsNull.IsValid())
489 assert.True(t, arrow.TypeEqual(typ, tsNull.DataType()))
490
491 assert.False(t, scalar.Equals(tsVal1, tsVal2))
492 assert.False(t, scalar.Equals(tsVal1, tsNull))
493 assert.False(t, scalar.Equals(tsNull, tsVal2))
494}
495
496func TestMonthDayNanoIntervalScalarBasics(t *testing.T) {
497 typ := arrow.FixedWidthTypes.MonthDayNanoInterval
498
499 val1 := arrow.MonthDayNanoInterval{Months: 1, Days: 2, Nanoseconds: 3000}
500 val2 := arrow.MonthDayNanoInterval{Months: 2, Days: 3, Nanoseconds: 4000}
501 tsVal1 := scalar.NewMonthDayNanoIntervalScalar(val1)
502 tsVal2 := scalar.NewMonthDayNanoIntervalScalar(val2)
503 tsNull := scalar.MakeNullScalar(typ)
504 assert.NoError(t, tsVal1.ValidateFull())
505 assert.NoError(t, tsVal2.ValidateFull())
506 assert.NoError(t, tsNull.ValidateFull())
507
508 assert.Equal(t, val1, tsVal1.Value)
509
510 assert.True(t, arrow.TypeEqual(tsVal1.Type, typ))
511 assert.True(t, arrow.TypeEqual(tsVal2.DataType(), typ))
512 assert.True(t, tsVal1.Valid)
513 assert.False(t, tsNull.IsValid())
514 assert.True(t, arrow.TypeEqual(typ, tsNull.DataType()))
515
516 assert.False(t, scalar.Equals(tsVal1, tsVal2))
517 assert.False(t, scalar.Equals(tsVal1, tsNull))
518 assert.False(t, scalar.Equals(tsNull, tsVal2))
519}
520
521func TestNumericScalarCasts(t *testing.T) {
522 tests := []arrow.DataType{
523 arrow.PrimitiveTypes.Int8,
524 arrow.PrimitiveTypes.Int16,
525 arrow.PrimitiveTypes.Int32,
526 arrow.PrimitiveTypes.Int64,
527 arrow.PrimitiveTypes.Uint8,
528 arrow.PrimitiveTypes.Uint16,
529 arrow.PrimitiveTypes.Uint32,
530 arrow.PrimitiveTypes.Uint64,
531 arrow.PrimitiveTypes.Float32,
532 arrow.PrimitiveTypes.Float64,
533 arrow.FixedWidthTypes.Float16,
534 }
535
536 temporalTypes := []arrow.DataType{
537 arrow.FixedWidthTypes.Date32,
538 arrow.FixedWidthTypes.Date64,
539 arrow.FixedWidthTypes.Date64,
540 arrow.FixedWidthTypes.Time32ms,
541 arrow.FixedWidthTypes.Time64us,
542 arrow.FixedWidthTypes.Timestamp_ms,
543 arrow.FixedWidthTypes.MonthInterval,
544 }
545
546 falseScalar := scalar.NewBooleanScalar(false)
547 trueScalar := scalar.NewBooleanScalar(true)
548 nullBool := scalar.MakeNullScalar(arrow.FixedWidthTypes.Boolean)
549
550 for _, tt := range tests {
551 t.Run(tt.ID().String()+"from bool", func(t *testing.T) {
552 zero, _ := scalar.ParseScalar(tt, "0")
553 zeroFromBool, err := falseScalar.CastTo(tt)
554 assert.NoError(t, err)
555 assert.True(t, scalar.Equals(zero, zeroFromBool))
556
557 one, _ := scalar.ParseScalar(tt, "1")
558 oneFromBool, err := trueScalar.CastTo(tt)
559 assert.NoError(t, err)
560 assert.True(t, scalar.Equals(one, oneFromBool))
561 })
562 t.Run(tt.ID().String(), func(t *testing.T) {
563 for _, repr := range []string{"0", "1", "3"} {
564 nullTest := scalar.MakeNullScalar(tt)
565 assert.Equal(t, "null", nullTest.String())
566
567 castedNull, err := nullBool.CastTo(tt)
568 assert.NoError(t, err)
569 assert.True(t, scalar.Equals(castedNull, nullTest))
570
571 s, err := scalar.ParseScalar(tt, repr)
572 assert.NoError(t, err)
573
574 for _, other := range []arrow.DataType{arrow.PrimitiveTypes.Float32, arrow.PrimitiveTypes.Int8, arrow.PrimitiveTypes.Int64, arrow.PrimitiveTypes.Uint32} {
575 otherNull, err := nullTest.CastTo(other)
576 assert.NoError(t, err)
577
578 expectedNull := scalar.MakeNullScalar(other)
579 assert.True(t, scalar.Equals(otherNull, expectedNull))
580
581 otherScalar, err := scalar.ParseScalar(other, repr)
582 assert.NoError(t, err)
583
584 castToOther, err := s.CastTo(other)
585 assert.NoError(t, err)
586 assert.True(t, scalar.Equals(castToOther, otherScalar))
587
588 castFromOther, err := otherScalar.CastTo(tt)
589 assert.NoError(t, err)
590 assert.True(t, scalar.Equals(castFromOther, s))
591 }
592
593 castToBool, err := s.CastTo(arrow.FixedWidthTypes.Boolean)
594 assert.NoError(t, err)
595 assert.True(t, castToBool.IsValid())
596 assert.Equal(t, repr != "0", castToBool.(*scalar.Boolean).Value)
597
598 castFromStr, err := scalar.NewStringScalar(repr).CastTo(tt)
599 assert.NoError(t, err)
600
601 assert.True(t, scalar.Equals(castFromStr, s))
602 assert.Equal(t, repr, s.String())
603 if tt == arrow.FixedWidthTypes.Float16 {
604 continue
605 }
606
607 for _, tmtyp := range temporalTypes {
608 castToTemporal, err := s.CastTo(tmtyp)
609 assert.NoError(t, err)
610 assert.NoError(t, castToTemporal.ValidateFull())
611 assert.True(t, arrow.TypeEqual(tmtyp, castToTemporal.DataType()))
612 }
613
614 if tt == arrow.PrimitiveTypes.Float32 || tt == arrow.PrimitiveTypes.Float64 {
615 continue
616 }
617
618 castToStr, err := s.CastTo(arrow.BinaryTypes.String)
619 assert.NoError(t, err)
620 assert.Equal(t, repr, string(castToStr.(*scalar.String).Value.Bytes()))
621 }
622 })
623 }
624}
625
626type ListScalarSuite struct {
627 suite.Suite
628
629 typ arrow.DataType
630 val array.Interface
631}
632
633func (l *ListScalarSuite) SetupTest() {
634 bld := array.NewInt16Builder(memory.DefaultAllocator)
635 defer bld.Release()
636 bld.AppendValues([]int16{1, 2, 0}, []bool{true, true, false})
637
638 l.val = bld.NewInt16Array()
639}
640
641func (l *ListScalarSuite) TearDownTest() {
642 l.val.Release()
643}
644
645func (l *ListScalarSuite) TestBasics() {
646 s, err := scalar.MakeScalarParam(l.val, l.typ)
647 l.NoError(err)
648
649 l.NoError(s.ValidateFull())
650 l.True(s.IsValid())
651 l.True(arrow.TypeEqual(l.typ, s.DataType()))
652
653 nullScalar := checkMakeNullScalar(l.T(), l.typ)
654 l.NoError(nullScalar.ValidateFull())
655 l.False(nullScalar.IsValid())
656 l.True(arrow.TypeEqual(nullScalar.DataType(), l.typ))
657
658 l.Equal("[1 2 (null)]", s.String())
659}
660
661func (l *ListScalarSuite) TestValidateErrors() {
662 // inconsistent isvalid / value
663 s, _ := scalar.MakeScalarParam(l.val, l.typ)
664 switch s := s.(type) {
665 case *scalar.List:
666 s.Valid = false
667 case *scalar.FixedSizeList:
668 s.Valid = false
669 }
670 l.Error(s.Validate())
671
672 s, _ = scalar.MakeScalarParam(l.val, l.typ)
673 switch s := s.(type) {
674 case *scalar.List:
675 s.Value = nil
676 case *scalar.FixedSizeList:
677 s.Value = nil
678 }
679 l.Error(s.Validate())
680
681 // inconsistent child type
682 bld := array.NewInt32Builder(memory.DefaultAllocator)
683 defer bld.Release()
684 bld.AppendValues([]int32{1, 2, 0}, []bool{true, true, false})
685 arr := bld.NewArray()
686 defer arr.Release()
687
688 s, _ = scalar.MakeScalarParam(l.val, l.typ)
689 switch s := s.(type) {
690 case *scalar.List:
691 s.Value = arr
692 case *scalar.FixedSizeList:
693 s.Value = arr
694 }
695 l.Error(s.Validate())
696}
697
698func TestListScalars(t *testing.T) {
699 ls := new(ListScalarSuite)
700 ls.typ = arrow.ListOf(arrow.PrimitiveTypes.Int16)
701 suite.Run(t, ls)
702 ls.typ = arrow.FixedSizeListOf(3, arrow.PrimitiveTypes.Int16)
703 suite.Run(t, ls)
704}
705
706func TestFixedSizeListScalarWrongNumber(t *testing.T) {
707 typ := arrow.FixedSizeListOf(3, arrow.PrimitiveTypes.Int16)
708 bld := array.NewInt16Builder(memory.DefaultAllocator)
709 defer bld.Release()
710 bld.AppendValues([]int16{1, 2, 5}, nil)
711 arr := bld.NewArray()
712 defer arr.Release()
713
714 sc := scalar.NewFixedSizeListScalarWithType(arr, typ)
715 assert.NoError(t, sc.ValidateFull())
716
717 sc.Type = arrow.FixedSizeListOf(4, arrow.PrimitiveTypes.Int16)
718 assert.Error(t, sc.ValidateFull())
719}
720
721func TestMapScalarBasics(t *testing.T) {
722 bld := array.NewStructBuilder(memory.DefaultAllocator, arrow.StructOf(
723 arrow.Field{Name: "key", Type: arrow.BinaryTypes.String, Nullable: false},
724 arrow.Field{Name: "value", Type: arrow.PrimitiveTypes.Int8, Nullable: true}))
725 defer bld.Release()
726 bld.FieldBuilder(0).(*array.StringBuilder).AppendValues([]string{"a", "b"}, nil)
727 bld.FieldBuilder(1).(*array.Int8Builder).AppendValues([]int8{1, 2}, nil)
728 value := bld.NewArray()
729 defer value.Release()
730
731 s := scalar.NewMapScalar(value)
732 assert.NoError(t, s.ValidateFull())
733
734 expectedScalarType := arrow.MapOf(arrow.BinaryTypes.String, arrow.PrimitiveTypes.Int8)
735 assert.True(t, arrow.TypeEqual(s.DataType(), expectedScalarType))
736 assert.True(t, array.ArrayEqual(value, s.GetList()))
737
738 checkMakeNullScalar(t, expectedScalarType)
739}
740
741func TestStructScalar(t *testing.T) {
742 abc := scalar.NewStructScalar([]scalar.Scalar{
743 scalar.MakeScalar(true),
744 scalar.MakeNullScalar(arrow.PrimitiveTypes.Int32),
745 scalar.MakeScalar("hello"),
746 scalar.MakeNullScalar(arrow.PrimitiveTypes.Int64),
747 }, arrow.StructOf(
748 arrow.Field{Name: "a", Type: arrow.FixedWidthTypes.Boolean, Nullable: true},
749 arrow.Field{Name: "b", Type: arrow.PrimitiveTypes.Int32, Nullable: true},
750 arrow.Field{Name: "c", Type: arrow.BinaryTypes.String, Nullable: true},
751 arrow.Field{Name: "d", Type: arrow.PrimitiveTypes.Int64, Nullable: true}))
752
753 assert.NoError(t, abc.Validate())
754 assert.NoError(t, abc.ValidateFull())
755
756 a, err := abc.Field("a")
757 assert.NoError(t, err)
758 assert.True(t, scalar.Equals(a, abc.Value[0]))
759
760 _, err = abc.Field("f")
761 assert.Error(t, err)
762
763 d, err := abc.Field("d")
764 assert.NoError(t, err)
765 assert.True(t, scalar.Equals(scalar.MakeNullScalar(arrow.PrimitiveTypes.Int64), d))
766 assert.False(t, scalar.Equals(scalar.MakeScalar(int64(12)), d))
767
768 abc2, err := scalar.NewStructScalarWithNames(abc.Value, []string{"a", "b", "c", "d"})
769 assert.NoError(t, err)
770 assert.True(t, scalar.Equals(abc, abc2))
771
772 assert.Equal(t, "{a:bool = true, b:int32 = null, c:utf8 = hello, d:int64 = null}", abc.String())
773}
774
775func TestNullStructScalar(t *testing.T) {
776 ty := arrow.StructOf(
777 arrow.Field{Name: "a", Type: arrow.FixedWidthTypes.Boolean, Nullable: true},
778 arrow.Field{Name: "b", Type: arrow.PrimitiveTypes.Int32, Nullable: true},
779 arrow.Field{Name: "c", Type: arrow.BinaryTypes.String, Nullable: true},
780 arrow.Field{Name: "d", Type: arrow.PrimitiveTypes.Int64, Nullable: true})
781 nullScalar := scalar.MakeNullScalar(ty)
782 assert.NoError(t, nullScalar.ValidateFull())
783 assert.False(t, nullScalar.IsValid())
784
785 sc := checkMakeNullScalar(t, ty)
786 assert.True(t, scalar.Equals(nullScalar, sc))
787}
788
789func TestStructScalarValidateErrors(t *testing.T) {
790 ty := arrow.StructOf(arrow.Field{Name: "a", Type: arrow.BinaryTypes.String})
791
792 // inconsistent isvalid value
793 sc := scalar.NewStructScalar([]scalar.Scalar{scalar.MakeScalar("hello")}, ty)
794 sc.Valid = false
795 assert.Error(t, sc.ValidateFull())
796
797 sc = scalar.NewStructScalar(nil, ty)
798 sc.Valid = true
799 assert.Error(t, sc.ValidateFull())
800
801 // inconsistent number of fields
802 sc = scalar.NewStructScalar([]scalar.Scalar{}, ty)
803 assert.Error(t, sc.ValidateFull())
804
805 sc = scalar.NewStructScalar([]scalar.Scalar{scalar.MakeScalar("foo"), scalar.MakeScalar("bar")}, ty)
806 assert.Error(t, sc.ValidateFull())
807
808 // inconsistent child value type
809 sc = scalar.NewStructScalar([]scalar.Scalar{scalar.MakeScalar(42)}, ty)
810 assert.Error(t, sc.ValidateFull())
811
812 // child value has invalid utf8 data
813 sc = scalar.NewStructScalar([]scalar.Scalar{scalar.MakeScalar("\xff")}, ty)
814 assert.NoError(t, sc.Validate())
815 assert.Error(t, sc.ValidateFull())
816}
817
818func getScalars(mem memory.Allocator) []scalar.Scalar {
819 hello := memory.NewBufferBytes([]byte("hello"))
820 daytime := arrow.DayTimeInterval{Days: 1, Milliseconds: 100}
821 monthdaynano := arrow.MonthDayNanoInterval{Months: 5, Days: 4, Nanoseconds: 100}
822
823 int8Bldr := array.NewInt8Builder(mem)
824 defer int8Bldr.Release()
825
826 int8Bldr.AppendValues([]int8{1, 2, 3, 4}, nil)
827 int8Arr := int8Bldr.NewInt8Array()
828 defer int8Arr.Release()
829
830 mapBldr := array.NewMapBuilder(mem, arrow.PrimitiveTypes.Int8, arrow.BinaryTypes.String, false)
831 defer mapBldr.Release()
832
833 kb := mapBldr.KeyBuilder().(*array.Int8Builder)
834 ib := mapBldr.ItemBuilder().(*array.StringBuilder)
835
836 mapBldr.Append(true)
837 kb.AppendValues([]int8{1, 2, 3}, nil)
838 ib.AppendValues([]string{"foo", "bar", "baz"}, nil)
839
840 mapArr := mapBldr.NewMapArray()
841 defer mapArr.Release()
842
843 return []scalar.Scalar{
844 scalar.NewBooleanScalar(false),
845 scalar.NewInt8Scalar(3),
846 scalar.NewUint16Scalar(3),
847 scalar.NewInt32Scalar(3),
848 scalar.NewUint64Scalar(3),
849 scalar.NewFloat64Scalar(3.0),
850 scalar.NewDate32Scalar(10),
851 scalar.NewDate64Scalar(11),
852 scalar.NewTime32Scalar(1000, arrow.FixedWidthTypes.Time32s),
853 scalar.NewTime64Scalar(1111, arrow.FixedWidthTypes.Time64us),
854 scalar.NewTimestampScalar(111, arrow.FixedWidthTypes.Timestamp_ms),
855 scalar.NewMonthIntervalScalar(1),
856 scalar.NewDayTimeIntervalScalar(daytime),
857 scalar.NewMonthDayNanoIntervalScalar(monthdaynano),
858 scalar.NewDurationScalar(60, arrow.FixedWidthTypes.Duration_s),
859 scalar.NewBinaryScalar(hello, arrow.BinaryTypes.Binary),
860 scalar.NewFixedSizeBinaryScalar(hello, &arrow.FixedSizeBinaryType{ByteWidth: hello.Len()}),
861 scalar.NewDecimal128Scalar(decimal128.FromI64(10), &arrow.Decimal128Type{Precision: 16, Scale: 4}),
862 scalar.NewStringScalarFromBuffer(hello),
863 scalar.NewListScalar(int8Arr),
864 scalar.NewMapScalar(mapArr.List.ListValues()),
865 scalar.NewFixedSizeListScalar(int8Arr),
866 scalar.NewStructScalar([]scalar.Scalar{scalar.NewInt32Scalar(2), scalar.NewInt32Scalar(6)},
867 arrow.StructOf([]arrow.Field{{Name: "min", Type: arrow.PrimitiveTypes.Int32}, {Name: "max", Type: arrow.PrimitiveTypes.Int32}}...)),
868 }
869}
870
871func TestMakeArrayFromScalar(t *testing.T) {
872 mem := memory.NewCheckedAllocator(memory.NewGoAllocator())
873 defer mem.AssertSize(t, 0)
874
875 nullArray, err := scalar.MakeArrayFromScalar(scalar.ScalarNull, 5, mem)
876 assert.NoError(t, err)
877 defer nullArray.Release()
878
879 assert.Equal(t, 5, nullArray.Len())
880 assert.Equal(t, 5, nullArray.NullN())
881
882 scalars := getScalars(mem)
883
884 for _, length := range []int{16} {
885 for _, s := range scalars {
886 t.Run(s.DataType().Name(), func(t *testing.T) {
887 if ls, ok := s.(scalar.Releasable); ok {
888 defer ls.Release()
889 }
890
891 arr, err := scalar.MakeArrayFromScalar(s, length, mem)
892 assert.NoError(t, err)
893 defer arr.Release()
894
895 assert.Equal(t, length, arr.Len())
896 assert.Zero(t, arr.NullN())
897
898 for _, i := range []int{0, length / 2, length - 1} {
899 scalarCompare, err := scalar.GetScalar(arr, i)
900 assert.NoError(t, err)
901 assert.True(t, scalar.Equals(s, scalarCompare))
902 if ls, ok := scalarCompare.(scalar.Releasable); ok {
903 ls.Release()
904 }
905 }
906 })
907 }
908 }
909}
Ceph