1 // Licensed to the Apache Software Foundation (ASF) under one
2 // or more contributor license agreements. See the NOTICE file
3 // distributed with this work for additional information
4 // regarding copyright ownership. The ASF licenses this file
5 // to you under the Apache License, Version 2.0 (the
6 // "License"); you may not use this file except in compliance
7 // with the License. You may obtain a copy of the License at
9 // http://www.apache.org/licenses/LICENSE-2.0
11 // Unless required by applicable law or agreed to in writing, software
12 // distributed under the License is distributed on an "AS IS" BASIS,
13 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 // See the License for the specific language governing permissions and
15 // limitations under the License.
24 "github.com/apache/arrow/go/v6/arrow"
25 "github.com/apache/arrow/go/v6/arrow/array"
26 "github.com/apache/arrow/go/v6/arrow/memory"
29 func TestRecord(t *testing.T) {
30 mem := memory.NewCheckedAllocator(memory.NewGoAllocator())
31 defer mem.AssertSize(t, 0)
33 schema := arrow.NewSchema(
35 arrow.Field{Name: "f1-i32", Type: arrow.PrimitiveTypes.Int32},
36 arrow.Field{Name: "f2-f64", Type: arrow.PrimitiveTypes.Float64},
40 col1 := func() array.Interface {
41 ib := array.NewInt32Builder(mem)
44 ib.AppendValues([]int32{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}, nil)
45 return ib.NewInt32Array()
49 col2 := func() array.Interface {
50 b := array.NewFloat64Builder(mem)
53 b.AppendValues([]float64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}, nil)
54 return b.NewFloat64Array()
58 cols := []array.Interface{col1, col2}
59 rec := array.NewRecord(schema, cols, -1)
65 if got, want := rec.Schema(), schema; !got.Equal(want) {
66 t.Fatalf("invalid schema: got=%#v, want=%#v", got, want)
69 if got, want := rec.NumRows(), int64(10); got != want {
70 t.Fatalf("invalid number of rows: got=%d, want=%d", got, want)
72 if got, want := rec.NumCols(), int64(2); got != want {
73 t.Fatalf("invalid number of columns: got=%d, want=%d", got, want)
75 if got, want := rec.Columns()[0], cols[0]; got != want {
76 t.Fatalf("invalid column: got=%q, want=%q", got, want)
78 if got, want := rec.Column(0), cols[0]; got != want {
79 t.Fatalf("invalid column: got=%q, want=%q", got, want)
81 if got, want := rec.ColumnName(0), schema.Field(0).Name; got != want {
82 t.Fatalf("invalid column name: got=%q, want=%q", got, want)
85 for _, tc := range []struct {
89 {i: 0, j: 10, err: nil},
90 {i: 1, j: 10, err: nil},
91 {i: 1, j: 9, err: nil},
92 {i: 0, j: 0, err: nil},
93 {i: 1, j: 1, err: nil},
94 {i: 10, j: 10, err: nil},
95 {i: 1, j: 0, err: fmt.Errorf("arrow/array: index out of range")},
96 {i: 1, j: 11, err: fmt.Errorf("arrow/array: index out of range")},
98 t.Run(fmt.Sprintf("slice-%02d-%02d", tc.i, tc.j), func(t *testing.T) {
103 t.Fatalf("expected an error %q", tc.err)
105 switch err := e.(type) {
107 if err != tc.err.Error() {
108 t.Fatalf("invalid panic message. got=%q, want=%q", err, tc.err)
111 if err.Error() != tc.err.Error() {
112 t.Fatalf("invalid panic message. got=%q, want=%q", err, tc.err)
115 t.Fatalf("invalid type for panic message: %T (err=%v)", err, err)
119 sub := rec.NewSlice(tc.i, tc.j)
122 if got, want := sub.NumRows(), tc.j-tc.i; got != want {
123 t.Fatalf("invalid rec-slice number of rows: got=%d, want=%d", got, want)
128 for _, tc := range []struct {
130 cols []array.Interface
138 err: fmt.Errorf("arrow/array: number of columns/fields mismatch"),
144 err: fmt.Errorf("arrow/array: number of columns/fields mismatch"),
147 schema: arrow.NewSchema(
149 arrow.Field{Name: "f1-i32", Type: arrow.PrimitiveTypes.Int32},
155 err: fmt.Errorf("arrow/array: number of columns/fields mismatch"),
158 schema: arrow.NewSchema(
160 arrow.Field{Name: "f1-i32", Type: arrow.PrimitiveTypes.Int32},
161 arrow.Field{Name: "f2-f64", Type: arrow.PrimitiveTypes.Int32},
167 err: fmt.Errorf(`arrow/array: column "f2-f64" type mismatch: got=float64, want=int32`),
173 err: fmt.Errorf(`arrow/array: mismatch number of rows in column "f1-i32": got=10, want=11`),
194 t.Run("", func(t *testing.T) {
199 t.Fatalf("expected an error %q", tc.err)
201 switch err := e.(type) {
203 if err != tc.err.Error() {
204 t.Fatalf("invalid panic message. got=%q, want=%q", err, tc.err)
207 if err.Error() != tc.err.Error() {
208 t.Fatalf("invalid panic message. got=%q, want=%q", err, tc.err)
211 t.Fatalf("invalid type for panic message: %T (err=%v)", err, err)
215 rec := array.NewRecord(tc.schema, tc.cols, tc.rows)
217 if got, want := rec.NumRows(), tc.rows; got != want {
218 t.Fatalf("invalid number of rows: got=%d, want=%d", got, want)
224 func TestRecordReader(t *testing.T) {
225 mem := memory.NewCheckedAllocator(memory.NewGoAllocator())
226 defer mem.AssertSize(t, 0)
228 schema := arrow.NewSchema(
230 arrow.Field{Name: "f1-i32", Type: arrow.PrimitiveTypes.Int32},
231 arrow.Field{Name: "f2-f64", Type: arrow.PrimitiveTypes.Float64},
235 rec1 := func() array.Record {
236 col1 := func() array.Interface {
237 ib := array.NewInt32Builder(mem)
240 ib.AppendValues([]int32{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}, nil)
241 return ib.NewInt32Array()
245 col2 := func() array.Interface {
246 b := array.NewFloat64Builder(mem)
249 b.AppendValues([]float64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}, nil)
250 return b.NewFloat64Array()
254 cols := []array.Interface{col1, col2}
255 return array.NewRecord(schema, cols, -1)
259 rec2 := func() array.Record {
260 col1 := func() array.Interface {
261 ib := array.NewInt32Builder(mem)
264 ib.AppendValues([]int32{11, 12, 13, 14, 15, 16, 17, 18, 19, 20}, nil)
265 return ib.NewInt32Array()
269 col2 := func() array.Interface {
270 b := array.NewFloat64Builder(mem)
273 b.AppendValues([]float64{11, 12, 13, 14, 15, 16, 17, 18, 19, 20}, nil)
274 return b.NewFloat64Array()
278 cols := []array.Interface{col1, col2}
279 return array.NewRecord(schema, cols, -1)
283 recs := []array.Record{rec1, rec2}
284 itr, err := array.NewRecordReader(schema, recs)
293 if got, want := itr.Schema(), schema; !got.Equal(want) {
294 t.Fatalf("invalid schema. got=%#v, want=%#v", got, want)
300 if got, want := itr.Record(), recs[n-1]; !reflect.DeepEqual(got, want) {
301 t.Fatalf("itr[%d], invalid record. got=%#v, want=%#v", n-1, got, want)
306 t.Fatalf("invalid number of iterations. got=%d, want=%d", n, len(recs))
309 for _, tc := range []struct {
315 name: "mismatch-name",
316 schema: arrow.NewSchema(
318 arrow.Field{Name: "f1-i32", Type: arrow.PrimitiveTypes.Int32},
319 arrow.Field{Name: "f2-XXX", Type: arrow.PrimitiveTypes.Float64},
323 err: fmt.Errorf("arrow/array: mismatch schema"),
326 name: "mismatch-type",
327 schema: arrow.NewSchema(
329 arrow.Field{Name: "f1-i32", Type: arrow.PrimitiveTypes.Int32},
330 arrow.Field{Name: "f2-f64", Type: arrow.PrimitiveTypes.Int64},
334 err: fmt.Errorf("arrow/array: mismatch schema"),
337 t.Run(tc.name, func(t *testing.T) {
338 itr, err := array.NewRecordReader(tc.schema, recs)
343 t.Fatalf("expected an error: %v", tc.err)
345 if !reflect.DeepEqual(tc.err, err) {
346 t.Fatalf("invalid error: got=%v, want=%v", err, tc.err)
352 func TestRecordBuilder(t *testing.T) {
353 mem := memory.NewCheckedAllocator(memory.NewGoAllocator())
354 defer mem.AssertSize(t, 0)
356 schema := arrow.NewSchema(
358 arrow.Field{Name: "f1-i32", Type: arrow.PrimitiveTypes.Int32},
359 arrow.Field{Name: "f2-f64", Type: arrow.PrimitiveTypes.Float64},
364 b := array.NewRecordBuilder(mem, schema)
370 b.Field(0).(*array.Int32Builder).AppendValues([]int32{1, 2, 3, 4, 5, 6}, nil)
371 b.Field(0).(*array.Int32Builder).AppendValues([]int32{7, 8, 9, 10}, nil)
372 b.Field(1).(*array.Float64Builder).AppendValues([]float64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}, nil)
377 if got, want := rec.Schema(), schema; !got.Equal(want) {
378 t.Fatalf("invalid schema: got=%#v, want=%#v", got, want)
381 if got, want := rec.NumRows(), int64(10); got != want {
382 t.Fatalf("invalid number of rows: got=%d, want=%d", got, want)
384 if got, want := rec.NumCols(), int64(2); got != want {
385 t.Fatalf("invalid number of columns: got=%d, want=%d", got, want)
387 if got, want := rec.ColumnName(0), schema.Field(0).Name; got != want {
388 t.Fatalf("invalid column name: got=%q, want=%q", got, want)
392 type testMessage struct {
394 Bars []*testMessageBar
397 func (m *testMessage) Reset() { *m = testMessage{} }
399 func (m *testMessage) GetFoo() *testMessageFoo {
406 func (m *testMessage) GetBars() []*testMessageBar {
413 type testMessageFoo struct {
418 func (m *testMessageFoo) Reset() { *m = testMessageFoo{} }
420 func (m *testMessageFoo) GetA() int32 {
427 func (m *testMessageFoo) GetB() []uint32 {
434 type testMessageBar struct {
439 func (m *testMessageBar) Reset() { *m = testMessageBar{} }
441 func (m *testMessageBar) GetC() int64 {
448 func (m *testMessageBar) GetD() []uint64 {
455 var testMessageSchema = arrow.NewSchema(
457 arrow.Field{Name: "foo", Type: arrow.StructOf(
458 arrow.Field{Name: "a", Type: arrow.PrimitiveTypes.Int32},
459 arrow.Field{Name: "b", Type: arrow.ListOf(
460 arrow.PrimitiveTypes.Uint32,
463 arrow.Field{Name: "bars", Type: arrow.ListOf(
465 arrow.Field{Name: "c", Type: arrow.PrimitiveTypes.Int64},
466 arrow.Field{Name: "d", Type: arrow.ListOf(
467 arrow.PrimitiveTypes.Uint64,
475 func (m *testMessage) Fill(rec array.Record, row int) error {
479 if 0 < rec.NumCols() {
480 src0 := rec.Column(0).Data()
481 typedSrc0 := array.NewStructData(src0)
482 defer typedSrc0.Release()
483 if typedSrc0.IsValid(row) {
484 m0 := &testMessageFoo{}
488 if 0 < typedSrc0.NumField() {
489 src0_0 := typedSrc0.Field(0).Data()
490 typedSrc0_0 := array.NewInt32Data(src0_0)
491 defer typedSrc0_0.Release()
492 m0.A = typedSrc0_0.Value(row)
496 if 1 < typedSrc0.NumField() {
497 src0_1 := typedSrc0.Field(1).Data()
498 listSrc0_1 := array.NewListData(src0_1)
499 defer listSrc0_1.Release()
500 if listSrc0_1.IsValid(row) {
501 typedSrc0_1 := array.NewUint32Data(listSrc0_1.ListValues().Data())
502 typedSrc0_1.Release()
503 start0_1 := int(listSrc0_1.Offsets()[row])
504 end0_1 := int(listSrc0_1.Offsets()[row+1])
505 for row := start0_1; row < end0_1; row++ {
506 m0.B = append(m0.B, typedSrc0_1.Value(row))
516 if 1 < rec.NumCols() {
517 src1 := rec.Column(1).Data()
518 listSrc1 := array.NewListData(src1)
519 defer listSrc1.Release()
520 if listSrc1.IsValid(row) {
521 typedSrc1 := array.NewStructData(listSrc1.ListValues().Data())
522 defer typedSrc1.Release()
523 start1 := int(listSrc1.Offsets()[row])
524 end1 := int(listSrc1.Offsets()[row+1])
525 for row := start1; row < end1; row++ {
526 if typedSrc1.IsValid(row) {
527 m1 := &testMessageBar{}
531 if 0 < typedSrc1.NumField() {
532 src1_0 := typedSrc1.Field(0).Data()
533 typedSrc1_0 := array.NewInt64Data(src1_0)
534 defer typedSrc1_0.Release()
535 m1.C = typedSrc1_0.Value(row)
539 if 1 < typedSrc1.NumField() {
540 src1_1 := typedSrc1.Field(1).Data()
541 listSrc1_1 := array.NewListData(src1_1)
542 defer listSrc1_1.Release()
543 if listSrc1_1.IsValid(row) {
544 typedSrc1_1 := array.NewUint64Data(listSrc1_1.ListValues().Data())
545 defer typedSrc1_1.Release()
546 start1_1 := int(listSrc1_1.Offsets()[row])
547 end1_1 := int(listSrc1_1.Offsets()[row+1])
548 for row := start1_1; row < end1_1; row++ {
549 m1.D = append(m1.D, typedSrc1_1.Value(row))
554 m.Bars = append(m.Bars, m1)
556 m.Bars = append(m.Bars, nil)
564 func newTestMessageArrowRecordBuilder(mem memory.Allocator) *testMessageArrowRecordBuilder {
565 return &testMessageArrowRecordBuilder{
566 rb: array.NewRecordBuilder(mem, testMessageSchema),
570 type testMessageArrowRecordBuilder struct {
571 rb *array.RecordBuilder
574 func (b *testMessageArrowRecordBuilder) Build() array.Record {
575 return b.rb.NewRecord()
578 func (b *testMessageArrowRecordBuilder) Release() {
582 func (b *testMessageArrowRecordBuilder) Append(m *testMessage) {
586 builder0 := b.rb.Field(0)
588 valueBuilder0 := builder0.(*array.StructBuilder)
590 valueBuilder0.AppendNull()
592 valueBuilder0.Append(true)
597 builder0_0 := valueBuilder0.FieldBuilder(0)
598 valueBuilder0_0 := builder0_0.(*array.Int32Builder)
599 valueBuilder0_0.Append(v0_0)
605 builder0_1 := valueBuilder0.FieldBuilder(1)
606 listBuilder0_1 := builder0_1.(*array.ListBuilder)
608 listBuilder0_1.AppendNull()
610 listBuilder0_1.Append(true)
611 valueBuilder0_1 := listBuilder0_1.ValueBuilder().(*array.Uint32Builder)
612 for _, item := range v0_1 {
613 valueBuilder0_1.Append(item)
622 builder1 := b.rb.Field(1)
624 listBuilder1 := builder1.(*array.ListBuilder)
626 listBuilder1.AppendNull()
628 listBuilder1.Append(true)
629 valueBuilder1 := listBuilder1.ValueBuilder().(*array.StructBuilder)
630 for _, item := range v1 {
632 valueBuilder1.AppendNull()
634 valueBuilder1.Append(true)
639 builder1_0 := valueBuilder1.FieldBuilder(0)
640 valueBuilder1_0 := builder1_0.(*array.Int64Builder)
641 valueBuilder1_0.Append(v1_0)
647 builder1_1 := valueBuilder1.FieldBuilder(1)
648 listBuilder1_1 := builder1_1.(*array.ListBuilder)
650 listBuilder1_1.AppendNull()
652 listBuilder1_1.Append(true)
653 valueBuilder1_1 := listBuilder1_1.ValueBuilder().(*array.Uint64Builder)
654 for _, item := range v1_1 {
655 valueBuilder1_1.Append(item)
665 func TestRecordBuilderMessages(t *testing.T) {
666 mem := memory.NewCheckedAllocator(memory.NewGoAllocator())
667 defer mem.AssertSize(t, 0)
669 b := newTestMessageArrowRecordBuilder(mem)
672 var msgs []*testMessage
673 for i := 0; i < 1000; i++ {
675 Foo: &testMessageFoo{
677 B: []uint32{2, 3, 4, 5, 6, 7, 8, 9},
679 Bars: []*testMessageBar{
682 D: []uint64{12, 13, 14},
686 D: []uint64{16, 17, 18, 19},
695 msgs = append(msgs, msg)
703 for i := 0; i < 1000; i++ {
705 if !reflect.DeepEqual(&got, msgs[i]) {
706 t.Fatalf("row[%d], invalid record. got=%#v, want=%#v", i, &got, msgs[i])