1 // Licensed to the Apache Software Foundation (ASF) under one
2 // or more contributor license agreements. See the NOTICE file
3 // distributed with this work for additional information
4 // regarding copyright ownership. The ASF licenses this file
5 // to you under the Apache License, Version 2.0 (the
6 // "License"); you may not use this file except in compliance
7 // with the License. You may obtain a copy of the License at
9 // http://www.apache.org/licenses/LICENSE-2.0
11 // Unless required by applicable law or agreed to in writing, software
12 // distributed under the License is distributed on an "AS IS" BASIS,
13 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 // See the License for the specific language governing permissions and
15 // limitations under the License.
20 // use test tag so that we only run these tests when the "test" tag is present
21 // so that the .c and other framework infrastructure is only compiled in during
22 // testing, and the .c files and symbols are not present in release builds.
33 "github.com/apache/arrow/go/v6/arrow"
34 "github.com/apache/arrow/go/v6/arrow/array"
35 "github.com/apache/arrow/go/v6/arrow/decimal128"
36 "github.com/apache/arrow/go/v6/arrow/memory"
37 "github.com/stretchr/testify/assert"
40 func TestSchemaExport(t *testing.T) {
41 sc := exportInt32TypeSchema()
42 f, err := importSchema(&sc)
43 assert.NoError(t, err)
45 keys, _ := getMetadataKeys()
46 vals, _ := getMetadataValues()
48 assert.Equal(t, arrow.PrimitiveTypes.Int32, f.Type)
49 assert.Equal(t, keys, f.Metadata.Keys())
50 assert.Equal(t, vals, f.Metadata.Values())
52 // schema was released when importing
53 assert.True(t, schemaIsReleased(&sc))
56 func TestSimpleArrayExport(t *testing.T) {
57 assert.False(t, test1IsReleased())
59 testarr := exportInt32Array()
60 arr, err := ImportCArrayWithType(testarr, arrow.PrimitiveTypes.Int32)
61 assert.NoError(t, err)
63 assert.False(t, test1IsReleased())
64 assert.True(t, isReleased(testarr))
68 assert.Eventually(t, test1IsReleased, 1*time.Second, 10*time.Millisecond)
71 func TestSimpleArrayAndSchema(t *testing.T) {
72 sc := exportInt32TypeSchema()
73 testarr := exportInt32Array()
75 // grab address of the buffer we stuck into the ArrowArray object
76 buflist := (*[2]unsafe.Pointer)(unsafe.Pointer(testarr.buffers))
77 origvals := (*[10]int32)(unsafe.Pointer(buflist[1]))
79 fld, arr, err := ImportCArray(testarr, &sc)
80 assert.NoError(t, err)
81 assert.Equal(t, arrow.PrimitiveTypes.Int32, fld.Type)
82 assert.EqualValues(t, 10, arr.Len())
84 // verify that the address is the same of the first integer for the
85 // slice that is being used by the array.Interface and the original buffer
86 vals := arr.(*array.Int32).Int32Values()
87 assert.Same(t, &vals[0], &origvals[0])
89 // and that the values are correct
90 for i, v := range vals {
91 assert.Equal(t, int32(i+1), v)
95 func TestPrimitiveSchemas(t *testing.T) {
100 {arrow.PrimitiveTypes.Int8, "c"},
101 {arrow.PrimitiveTypes.Int16, "s"},
102 {arrow.PrimitiveTypes.Int32, "i"},
103 {arrow.PrimitiveTypes.Int64, "l"},
104 {arrow.PrimitiveTypes.Uint8, "C"},
105 {arrow.PrimitiveTypes.Uint16, "S"},
106 {arrow.PrimitiveTypes.Uint32, "I"},
107 {arrow.PrimitiveTypes.Uint64, "L"},
108 {arrow.FixedWidthTypes.Boolean, "b"},
110 {arrow.FixedWidthTypes.Float16, "e"},
111 {arrow.PrimitiveTypes.Float32, "f"},
112 {arrow.PrimitiveTypes.Float64, "g"},
113 {&arrow.FixedSizeBinaryType{ByteWidth: 3}, "w:3"},
114 {arrow.BinaryTypes.Binary, "z"},
115 {arrow.BinaryTypes.String, "u"},
116 {&arrow.Decimal128Type{Precision: 16, Scale: 4}, "d:16,4"},
117 {&arrow.Decimal128Type{Precision: 15, Scale: 0}, "d:15,0"},
118 {&arrow.Decimal128Type{Precision: 15, Scale: -4}, "d:15,-4"},
121 for _, tt := range tests {
122 t.Run(tt.typ.Name(), func(t *testing.T) {
123 sc := testPrimitive(tt.fmt)
125 f, err := ImportCArrowField(&sc)
126 assert.NoError(t, err)
128 assert.True(t, arrow.TypeEqual(tt.typ, f.Type))
130 assert.True(t, schemaIsReleased(&sc))
135 func TestImportTemporalSchema(t *testing.T) {
140 {arrow.FixedWidthTypes.Date32, "tdD"},
141 {arrow.FixedWidthTypes.Date64, "tdm"},
142 {arrow.FixedWidthTypes.Time32s, "tts"},
143 {arrow.FixedWidthTypes.Time32ms, "ttm"},
144 {arrow.FixedWidthTypes.Time64us, "ttu"},
145 {arrow.FixedWidthTypes.Time64ns, "ttn"},
146 {arrow.FixedWidthTypes.Duration_s, "tDs"},
147 {arrow.FixedWidthTypes.Duration_ms, "tDm"},
148 {arrow.FixedWidthTypes.Duration_us, "tDu"},
149 {arrow.FixedWidthTypes.Duration_ns, "tDn"},
150 {arrow.FixedWidthTypes.MonthInterval, "tiM"},
151 {arrow.FixedWidthTypes.DayTimeInterval, "tiD"},
152 {arrow.FixedWidthTypes.MonthDayNanoInterval, "tin"},
153 {arrow.FixedWidthTypes.Timestamp_s, "tss:"},
154 {&arrow.TimestampType{Unit: arrow.Second, TimeZone: "Europe/Paris"}, "tss:Europe/Paris"},
155 {arrow.FixedWidthTypes.Timestamp_ms, "tsm:"},
156 {&arrow.TimestampType{Unit: arrow.Millisecond, TimeZone: "Europe/Paris"}, "tsm:Europe/Paris"},
157 {arrow.FixedWidthTypes.Timestamp_us, "tsu:"},
158 {&arrow.TimestampType{Unit: arrow.Microsecond, TimeZone: "Europe/Paris"}, "tsu:Europe/Paris"},
159 {arrow.FixedWidthTypes.Timestamp_ns, "tsn:"},
160 {&arrow.TimestampType{Unit: arrow.Nanosecond, TimeZone: "Europe/Paris"}, "tsn:Europe/Paris"},
163 for _, tt := range tests {
164 t.Run(tt.typ.Name(), func(t *testing.T) {
165 sc := testPrimitive(tt.fmt)
167 f, err := ImportCArrowField(&sc)
168 assert.NoError(t, err)
170 assert.True(t, arrow.TypeEqual(tt.typ, f.Type))
172 assert.True(t, schemaIsReleased(&sc))
177 func TestListSchemas(t *testing.T) {
184 {arrow.ListOf(arrow.PrimitiveTypes.Int8), []string{"+l", "c"}, []string{"", "item"}, []bool{true}},
185 {arrow.FixedSizeListOfNonNullable(2, arrow.PrimitiveTypes.Int64), []string{"+w:2", "l"}, []string{"", "item"}, []bool{false}},
186 {arrow.ListOfNonNullable(arrow.ListOf(arrow.PrimitiveTypes.Int32)), []string{"+l", "+l", "i"}, []string{"", "item", "item"}, []bool{false, true}},
189 for _, tt := range tests {
190 t.Run(tt.typ.Name(), func(t *testing.T) {
191 sc := testNested(tt.fmts, tt.names, tt.isnull)
192 defer freeMallocedSchemas(sc)
194 top := (*[1]*CArrowSchema)(unsafe.Pointer(sc))[0]
195 f, err := ImportCArrowField(top)
196 assert.NoError(t, err)
198 assert.True(t, arrow.TypeEqual(tt.typ, f.Type))
200 assert.True(t, schemaIsReleased(top))
205 func TestStructSchemas(t *testing.T) {
213 arrow.Field{Name: "a", Type: arrow.PrimitiveTypes.Int8, Nullable: true},
214 arrow.Field{Name: "b", Type: arrow.BinaryTypes.String, Nullable: true, Metadata: metadata2},
215 ), []string{"+s", "c", "u"}, []string{"", "a", "b"}, []int64{flagIsNullable, flagIsNullable, flagIsNullable}},
218 for _, tt := range tests {
219 t.Run(tt.typ.Name(), func(t *testing.T) {
220 sc := testStruct(tt.fmts, tt.names, tt.flags)
221 defer freeMallocedSchemas(sc)
223 top := (*[1]*CArrowSchema)(unsafe.Pointer(sc))[0]
224 f, err := ImportCArrowField(top)
225 assert.NoError(t, err)
227 assert.True(t, arrow.TypeEqual(tt.typ, f.Type))
229 assert.True(t, schemaIsReleased(top))
234 func TestMapSchemas(t *testing.T) {
242 {arrow.MapOf(arrow.PrimitiveTypes.Int8, arrow.BinaryTypes.String), false, []string{"+m", "+s", "c", "u"}, []string{"", "entries", "key", "value"}, []int64{flagIsNullable, 0, 0, flagIsNullable}},
243 {arrow.MapOf(arrow.PrimitiveTypes.Int8, arrow.BinaryTypes.String), true, []string{"+m", "+s", "c", "u"}, []string{"", "entries", "key", "value"}, []int64{flagIsNullable | flagMapKeysSorted, 0, 0, flagIsNullable}},
246 for _, tt := range tests {
247 t.Run(tt.typ.Name(), func(t *testing.T) {
248 sc := testMap(tt.fmts, tt.names, tt.flags)
249 defer freeMallocedSchemas(sc)
251 top := (*[1]*CArrowSchema)(unsafe.Pointer(sc))[0]
252 f, err := ImportCArrowField(top)
253 assert.NoError(t, err)
255 tt.typ.KeysSorted = tt.keysSorted
256 assert.True(t, arrow.TypeEqual(tt.typ, f.Type))
258 assert.True(t, schemaIsReleased(top))
263 func TestSchema(t *testing.T) {
264 // schema is exported as an equivalent struct type (+ top-level metadata)
265 sc := arrow.NewSchema([]arrow.Field{
266 {Name: "nulls", Type: arrow.Null, Nullable: false},
267 {Name: "values", Type: arrow.PrimitiveTypes.Int64, Nullable: true, Metadata: metadata1},
270 cst := testSchema([]string{"+s", "n", "l"}, []string{"", "nulls", "values"}, []int64{0, 0, flagIsNullable})
271 defer freeMallocedSchemas(cst)
273 top := (*[1]*CArrowSchema)(unsafe.Pointer(cst))[0]
274 out, err := ImportCArrowSchema(top)
275 assert.NoError(t, err)
277 assert.True(t, sc.Equal(out))
278 assert.True(t, sc.Metadata().Equal(out.Metadata()))
280 assert.True(t, schemaIsReleased(top))
283 func createTestInt8Arr() array.Interface {
284 bld := array.NewInt8Builder(memory.DefaultAllocator)
287 bld.AppendValues([]int8{1, 2, 0, -3}, []bool{true, true, false, true})
288 return bld.NewInt8Array()
291 func createTestInt16Arr() array.Interface {
292 bld := array.NewInt16Builder(memory.DefaultAllocator)
295 bld.AppendValues([]int16{1, 2, -3}, []bool{true, true, true})
296 return bld.NewInt16Array()
299 func createTestInt32Arr() array.Interface {
300 bld := array.NewInt32Builder(memory.DefaultAllocator)
303 bld.AppendValues([]int32{1, 2, 0, -3}, []bool{true, true, false, true})
304 return bld.NewInt32Array()
307 func createTestInt64Arr() array.Interface {
308 bld := array.NewInt64Builder(memory.DefaultAllocator)
311 bld.AppendValues([]int64{1, 2, -3}, []bool{true, true, true})
312 return bld.NewInt64Array()
315 func createTestUint8Arr() array.Interface {
316 bld := array.NewUint8Builder(memory.DefaultAllocator)
319 bld.AppendValues([]uint8{1, 2, 0, 3}, []bool{true, true, false, true})
320 return bld.NewUint8Array()
323 func createTestUint16Arr() array.Interface {
324 bld := array.NewUint16Builder(memory.DefaultAllocator)
327 bld.AppendValues([]uint16{1, 2, 3}, []bool{true, true, true})
328 return bld.NewUint16Array()
331 func createTestUint32Arr() array.Interface {
332 bld := array.NewUint32Builder(memory.DefaultAllocator)
335 bld.AppendValues([]uint32{1, 2, 0, 3}, []bool{true, true, false, true})
336 return bld.NewUint32Array()
339 func createTestUint64Arr() array.Interface {
340 bld := array.NewUint64Builder(memory.DefaultAllocator)
343 bld.AppendValues([]uint64{1, 2, 3}, []bool{true, true, true})
344 return bld.NewUint64Array()
347 func createTestBoolArr() array.Interface {
348 bld := array.NewBooleanBuilder(memory.DefaultAllocator)
351 bld.AppendValues([]bool{true, false, false}, []bool{true, true, false})
352 return bld.NewBooleanArray()
355 func createTestNullArr() array.Interface {
356 return array.NewNull(2)
359 func createTestFloat32Arr() array.Interface {
360 bld := array.NewFloat32Builder(memory.DefaultAllocator)
363 bld.AppendValues([]float32{1.5, 0}, []bool{true, false})
364 return bld.NewFloat32Array()
367 func createTestFloat64Arr() array.Interface {
368 bld := array.NewFloat64Builder(memory.DefaultAllocator)
371 bld.AppendValues([]float64{1.5, 0}, []bool{true, false})
372 return bld.NewFloat64Array()
375 func createTestFSBArr() array.Interface {
376 bld := array.NewFixedSizeBinaryBuilder(memory.DefaultAllocator, &arrow.FixedSizeBinaryType{ByteWidth: 3})
379 bld.AppendValues([][]byte{[]byte("foo"), []byte("bar"), nil}, []bool{true, true, false})
380 return bld.NewFixedSizeBinaryArray()
383 func createTestBinaryArr() array.Interface {
384 bld := array.NewBinaryBuilder(memory.DefaultAllocator, arrow.BinaryTypes.Binary)
387 bld.AppendValues([][]byte{[]byte("foo"), []byte("bar"), nil}, []bool{true, true, false})
388 return bld.NewBinaryArray()
391 func createTestStrArr() array.Interface {
392 bld := array.NewStringBuilder(memory.DefaultAllocator)
395 bld.AppendValues([]string{"foo", "bar", ""}, []bool{true, true, false})
396 return bld.NewStringArray()
399 func createTestDecimalArr() array.Interface {
400 bld := array.NewDecimal128Builder(memory.DefaultAllocator, &arrow.Decimal128Type{Precision: 16, Scale: 4})
403 bld.AppendValues([]decimal128.Num{decimal128.FromU64(12345670), decimal128.FromU64(0)}, []bool{true, false})
404 return bld.NewDecimal128Array()
407 func TestPrimitiveArrs(t *testing.T) {
410 fn func() array.Interface
412 {"int8", createTestInt8Arr},
413 {"uint8", createTestUint8Arr},
414 {"int16", createTestInt16Arr},
415 {"uint16", createTestUint16Arr},
416 {"int32", createTestInt32Arr},
417 {"uint32", createTestUint32Arr},
418 {"int64", createTestInt64Arr},
419 {"uint64", createTestUint64Arr},
420 {"bool", createTestBoolArr},
421 {"null", createTestNullArr},
422 {"float32", createTestFloat32Arr},
423 {"float64", createTestFloat64Arr},
424 {"fixed size binary", createTestFSBArr},
425 {"binary", createTestBinaryArr},
426 {"utf8", createTestStrArr},
427 {"decimal128", createTestDecimalArr},
430 for _, tt := range tests {
431 t.Run(tt.name, func(t *testing.T) {
435 carr := createCArr(arr)
436 defer freeTestArr(carr)
438 imported, err := ImportCArrayWithType(carr, arr.DataType())
439 assert.NoError(t, err)
440 assert.True(t, array.ArrayEqual(arr, imported))
441 assert.True(t, isReleased(carr))
448 func TestPrimitiveSliced(t *testing.T) {
449 arr := createTestInt16Arr()
452 sl := array.NewSlice(arr, 1, 2)
455 carr := createCArr(sl)
456 defer freeTestArr(carr)
458 imported, err := ImportCArrayWithType(carr, arr.DataType())
459 assert.NoError(t, err)
460 assert.True(t, array.ArrayEqual(sl, imported))
461 assert.True(t, array.ArraySliceEqual(arr, 1, 2, imported, 0, int64(imported.Len())))
462 assert.True(t, isReleased(carr))
467 func createTestListArr() array.Interface {
468 bld := array.NewListBuilder(memory.DefaultAllocator, arrow.PrimitiveTypes.Int8)
471 vb := bld.ValueBuilder().(*array.Int8Builder)
474 vb.AppendValues([]int8{1, 2}, []bool{true, true})
477 vb.AppendValues([]int8{3, 0}, []bool{true, false})
481 return bld.NewArray()
484 func createTestFixedSizeList() array.Interface {
485 bld := array.NewFixedSizeListBuilder(memory.DefaultAllocator, 2, arrow.PrimitiveTypes.Int64)
488 vb := bld.ValueBuilder().(*array.Int64Builder)
491 vb.AppendValues([]int64{1, 2}, []bool{true, true})
494 vb.AppendValues([]int64{3, 0}, []bool{true, false})
497 return bld.NewArray()
500 func createTestStructArr() array.Interface {
501 bld := array.NewStructBuilder(memory.DefaultAllocator, arrow.StructOf(
502 arrow.Field{Name: "a", Type: arrow.PrimitiveTypes.Int8, Nullable: true},
503 arrow.Field{Name: "b", Type: arrow.BinaryTypes.String, Nullable: true},
507 f1bld := bld.FieldBuilder(0).(*array.Int8Builder)
508 f2bld := bld.FieldBuilder(1).(*array.StringBuilder)
518 return bld.NewArray()
521 func createTestMapArr() array.Interface {
522 bld := array.NewMapBuilder(memory.DefaultAllocator, arrow.PrimitiveTypes.Int8, arrow.BinaryTypes.String, false)
525 kb := bld.KeyBuilder().(*array.Int8Builder)
526 vb := bld.ItemBuilder().(*array.StringBuilder)
538 return bld.NewArray()
541 func TestNestedArrays(t *testing.T) {
544 fn func() array.Interface
546 {"list", createTestListArr},
547 {"fixed size list", createTestFixedSizeList},
548 {"struct", createTestStructArr},
549 {"map", createTestMapArr},
552 for _, tt := range tests {
553 t.Run(tt.name, func(t *testing.T) {
557 carr := createCArr(arr)
558 defer freeTestArr(carr)
560 imported, err := ImportCArrayWithType(carr, arr.DataType())
561 assert.NoError(t, err)
562 assert.True(t, array.ArrayEqual(arr, imported))
563 assert.True(t, isReleased(carr))
570 func TestRecordBatch(t *testing.T) {
571 arr := createTestStructArr()
574 carr := createCArr(arr)
575 defer freeTestArr(carr)
577 sc := testStruct([]string{"+s", "c", "u"}, []string{"", "a", "b"}, []int64{0, flagIsNullable, flagIsNullable})
578 defer freeMallocedSchemas(sc)
580 top := (*[1]*CArrowSchema)(unsafe.Pointer(sc))[0]
581 rb, err := ImportCRecordBatch(carr, top)
582 assert.NoError(t, err)
585 assert.EqualValues(t, 2, rb.NumCols())
586 rbschema := rb.Schema()
587 assert.Equal(t, "a", rbschema.Field(0).Name)
588 assert.Equal(t, "b", rbschema.Field(1).Name)
590 rec := array.NewRecord(rbschema, []array.Interface{arr.(*array.Struct).Field(0), arr.(*array.Struct).Field(1)}, -1)
593 assert.True(t, array.RecordEqual(rb, rec))
596 func TestRecordReaderStream(t *testing.T) {
597 stream := arrayStreamTest()
598 defer releaseStream(stream)
600 rdr := ImportCArrayStream(stream, nil)
603 rec, err := rdr.Read()
608 assert.NoError(t, err)
612 assert.EqualValues(t, 2, rec.NumCols())
613 assert.Equal(t, "a", rec.ColumnName(0))
614 assert.Equal(t, "b", rec.ColumnName(1))
616 for j := 0; j < int(rec.NumRows()); j++ {
617 assert.Equal(t, int32((j+1)*i), rec.Column(0).(*array.Int32).Value(j))
619 assert.Equal(t, "foo", rec.Column(1).(*array.String).Value(0))
620 assert.Equal(t, "bar", rec.Column(1).(*array.String).Value(1))
621 assert.Equal(t, "baz", rec.Column(1).(*array.String).Value(2))