]>
Commit | Line | Data |
---|---|---|
1d09f67e TL |
1 | /* |
2 | * Licensed to the Apache Software Foundation (ASF) under one or more | |
3 | * contributor license agreements. See the NOTICE file distributed with | |
4 | * this work for additional information regarding copyright ownership. | |
5 | * The ASF licenses this file to You under the Apache License, Version 2.0 | |
6 | * (the "License"); you may not use this file except in compliance with | |
7 | * the License. You may obtain a copy of the License at | |
8 | * | |
9 | * http://www.apache.org/licenses/LICENSE-2.0 | |
10 | * | |
11 | * Unless required by applicable law or agreed to in writing, software | |
12 | * distributed under the License is distributed on an "AS IS" BASIS, | |
13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |
14 | * See the License for the specific language governing permissions and | |
15 | * limitations under the License. | |
16 | */ | |
17 | ||
18 | package org.apache.arrow.vector; | |
19 | ||
20 | import static org.junit.Assert.assertEquals; | |
21 | import static org.junit.Assert.assertTrue; | |
22 | import static org.junit.jupiter.api.Assertions.assertFalse; | |
23 | ||
24 | import org.apache.arrow.memory.ArrowBuf; | |
25 | import org.apache.arrow.memory.BufferAllocator; | |
26 | import org.apache.arrow.memory.RootAllocator; | |
27 | import org.junit.Test; | |
28 | ||
29 | import io.netty.util.internal.PlatformDependent; | |
30 | ||
31 | public class TestBitVectorHelper { | |
32 | @Test | |
33 | public void testGetNullCount() throws Exception { | |
34 | try (BufferAllocator root = new RootAllocator()) { | |
35 | // test case 1, 1 null value for 0b110 | |
36 | ArrowBuf validityBuffer = root.buffer(3); | |
37 | // we set validity buffer to be 0b10110, but only have 3 items with 1st item is null | |
38 | validityBuffer.setByte(0, 0b10110); | |
39 | ||
40 | // we will only consider 0b110 here, since we only 3 items and only one is null | |
41 | int count = BitVectorHelper.getNullCount(validityBuffer, 3); | |
42 | assertEquals(count, 1); | |
43 | validityBuffer.close(); | |
44 | ||
45 | // test case 2, no null value for 0xFF | |
46 | validityBuffer = root.buffer(8); | |
47 | validityBuffer.setByte(0, 0xFF); | |
48 | ||
49 | count = BitVectorHelper.getNullCount(validityBuffer, 8); | |
50 | assertEquals(count, 0); | |
51 | validityBuffer.close(); | |
52 | ||
53 | // test case 3, 1 null value for 0x7F | |
54 | validityBuffer = root.buffer(8); | |
55 | validityBuffer.setByte(0, 0x7F); | |
56 | ||
57 | count = BitVectorHelper.getNullCount(validityBuffer, 8); | |
58 | assertEquals(count, 1); | |
59 | validityBuffer.close(); | |
60 | ||
61 | // test case 4, validity buffer has multiple bytes, 11 items | |
62 | validityBuffer = root.buffer(11); | |
63 | validityBuffer.setByte(0, 0b10101010); | |
64 | validityBuffer.setByte(1, 0b01010101); | |
65 | ||
66 | count = BitVectorHelper.getNullCount(validityBuffer, 11); | |
67 | assertEquals(count, 5); | |
68 | validityBuffer.close(); | |
69 | } | |
70 | } | |
71 | ||
72 | @Test | |
73 | public void testAllBitsNull() { | |
74 | final int bufferLength = 32 * 1024; | |
75 | try (RootAllocator allocator = new RootAllocator(bufferLength); | |
76 | ArrowBuf validityBuffer = allocator.buffer(bufferLength)) { | |
77 | ||
78 | validityBuffer.setZero(0, bufferLength); | |
79 | int bitLength = 1024; | |
80 | assertTrue(BitVectorHelper.checkAllBitsEqualTo(validityBuffer, bitLength, false)); | |
81 | ||
82 | bitLength = 1027; | |
83 | assertTrue(BitVectorHelper.checkAllBitsEqualTo(validityBuffer, bitLength, false)); | |
84 | ||
85 | validityBuffer.setZero(0, bufferLength); | |
86 | bitLength = 1025; | |
87 | BitVectorHelper.setBit(validityBuffer, 12); | |
88 | assertFalse(BitVectorHelper.checkAllBitsEqualTo(validityBuffer, bitLength, false)); | |
89 | ||
90 | validityBuffer.setZero(0, bufferLength); | |
91 | bitLength = 1025; | |
92 | BitVectorHelper.setBit(validityBuffer, 1024); | |
93 | assertFalse(BitVectorHelper.checkAllBitsEqualTo(validityBuffer, bitLength, false)); | |
94 | ||
95 | validityBuffer.setZero(0, bufferLength); | |
96 | bitLength = 1026; | |
97 | BitVectorHelper.setBit(validityBuffer, 1024); | |
98 | assertFalse(BitVectorHelper.checkAllBitsEqualTo(validityBuffer, bitLength, false)); | |
99 | ||
100 | validityBuffer.setZero(0, bufferLength); | |
101 | bitLength = 1027; | |
102 | BitVectorHelper.setBit(validityBuffer, 1025); | |
103 | assertFalse(BitVectorHelper.checkAllBitsEqualTo(validityBuffer, bitLength, false)); | |
104 | ||
105 | validityBuffer.setZero(0, bufferLength); | |
106 | bitLength = 1031; | |
107 | BitVectorHelper.setBit(validityBuffer, 1029); | |
108 | BitVectorHelper.setBit(validityBuffer, 1030); | |
109 | assertFalse(BitVectorHelper.checkAllBitsEqualTo(validityBuffer, bitLength, false)); | |
110 | } | |
111 | } | |
112 | ||
113 | @Test | |
114 | public void testAllBitsSet() { | |
115 | final int bufferLength = 32 * 1024; | |
116 | try (RootAllocator allocator = new RootAllocator(bufferLength); | |
117 | ArrowBuf validityBuffer = allocator.buffer(bufferLength)) { | |
118 | ||
119 | PlatformDependent.setMemory(validityBuffer.memoryAddress(), bufferLength, (byte) -1); | |
120 | int bitLength = 1024; | |
121 | assertTrue(BitVectorHelper.checkAllBitsEqualTo(validityBuffer, bitLength, true)); | |
122 | ||
123 | bitLength = 1028; | |
124 | assertTrue(BitVectorHelper.checkAllBitsEqualTo(validityBuffer, bitLength, true)); | |
125 | ||
126 | PlatformDependent.setMemory(validityBuffer.memoryAddress(), bufferLength, (byte) -1); | |
127 | bitLength = 1025; | |
128 | BitVectorHelper.unsetBit(validityBuffer, 12); | |
129 | assertFalse(BitVectorHelper.checkAllBitsEqualTo(validityBuffer, bitLength, true)); | |
130 | ||
131 | PlatformDependent.setMemory(validityBuffer.memoryAddress(), bufferLength, (byte) -1); | |
132 | bitLength = 1025; | |
133 | BitVectorHelper.unsetBit(validityBuffer, 1024); | |
134 | assertFalse(BitVectorHelper.checkAllBitsEqualTo(validityBuffer, bitLength, true)); | |
135 | ||
136 | PlatformDependent.setMemory(validityBuffer.memoryAddress(), bufferLength, (byte) -1); | |
137 | bitLength = 1026; | |
138 | BitVectorHelper.unsetBit(validityBuffer, 1024); | |
139 | assertFalse(BitVectorHelper.checkAllBitsEqualTo(validityBuffer, bitLength, true)); | |
140 | ||
141 | PlatformDependent.setMemory(validityBuffer.memoryAddress(), bufferLength, (byte) -1); | |
142 | bitLength = 1027; | |
143 | BitVectorHelper.unsetBit(validityBuffer, 1025); | |
144 | assertFalse(BitVectorHelper.checkAllBitsEqualTo(validityBuffer, bitLength, true)); | |
145 | ||
146 | PlatformDependent.setMemory(validityBuffer.memoryAddress(), bufferLength, (byte) -1); | |
147 | bitLength = 1031; | |
148 | BitVectorHelper.unsetBit(validityBuffer, 1029); | |
149 | BitVectorHelper.unsetBit(validityBuffer, 1030); | |
150 | assertFalse(BitVectorHelper.checkAllBitsEqualTo(validityBuffer, bitLength, true)); | |
151 | } | |
152 | } | |
153 | ||
154 | @Test | |
155 | public void testConcatBits() { | |
156 | try (RootAllocator allocator = new RootAllocator(1024 * 1024)) { | |
157 | try (ArrowBuf buf1 = allocator.buffer(1024); | |
158 | ArrowBuf buf2 = allocator.buffer(1024); | |
159 | ArrowBuf output = allocator.buffer(1024)) { | |
160 | ||
161 | buf1.setZero(0, buf1.capacity()); | |
162 | buf2.setZero(0, buf2.capacity()); | |
163 | ||
164 | final int maxCount = 100; | |
165 | for (int i = 0; i < maxCount; i++) { | |
166 | if (i % 3 == 0) { | |
167 | BitVectorHelper.setBit(buf1, i); | |
168 | BitVectorHelper.setBit(buf2, i); | |
169 | } | |
170 | } | |
171 | ||
172 | // test the case where the number of bits for both sets are multiples of 8. | |
173 | concatAndVerify(buf1, 40, buf2, 48, output); | |
174 | ||
175 | // only the number of bits in the first set is a multiple of 8 | |
176 | concatAndVerify(buf1, 32, buf2, 47, output); | |
177 | ||
178 | // only the number of bits in the second set is a multiple of 8 | |
179 | concatAndVerify(buf1, 31, buf2, 48, output); | |
180 | ||
181 | // neither set has a size that is a multiple of 8 | |
182 | concatAndVerify(buf1, 27, buf2, 52, output); | |
183 | ||
184 | // the remaining bits in the second set is spread in two bytes | |
185 | concatAndVerify(buf1, 31, buf2, 55, output); | |
186 | } | |
187 | } | |
188 | } | |
189 | ||
190 | @Test | |
191 | public void testConcatBitsInPlace() { | |
192 | try (RootAllocator allocator = new RootAllocator(1024 * 1024)) { | |
193 | try (ArrowBuf buf1 = allocator.buffer(1024); | |
194 | ArrowBuf buf2 = allocator.buffer(1024)) { | |
195 | ||
196 | buf1.setZero(0, buf1.capacity()); | |
197 | buf2.setZero(0, buf2.capacity()); | |
198 | ||
199 | final int maxCount = 100; | |
200 | for (int i = 0; i < maxCount; i++) { | |
201 | if (i % 3 == 0) { | |
202 | BitVectorHelper.setBit(buf1, i); | |
203 | BitVectorHelper.setBit(buf2, i); | |
204 | } | |
205 | } | |
206 | ||
207 | // test the case where the number of bits for both sets are multiples of 8. | |
208 | concatAndVerify(buf1, 40, buf2, 48, buf1); | |
209 | ||
210 | // only the number of bits in the first set is a multiple of 8 | |
211 | concatAndVerify(buf1, 32, buf2, 47, buf1); | |
212 | ||
213 | // only the number of bits in the second set is a multiple of 8 | |
214 | concatAndVerify(buf1, 31, buf2, 48, buf1); | |
215 | ||
216 | // neither set has a size that is a multiple of 8 | |
217 | concatAndVerify(buf1, 27, buf2, 52, buf1); | |
218 | ||
219 | // the remaining bits in the second set is spread in two bytes | |
220 | concatAndVerify(buf1, 31, buf2, 55, buf1); | |
221 | } | |
222 | } | |
223 | } | |
224 | ||
225 | private void concatAndVerify(ArrowBuf buf1, int count1, ArrowBuf buf2, int count2, ArrowBuf output) { | |
226 | BitVectorHelper.concatBits(buf1, count1, buf2, count2, output); | |
227 | int outputIdx = 0; | |
228 | for (int i = 0; i < count1; i++, outputIdx++) { | |
229 | assertEquals(BitVectorHelper.get(output, outputIdx), BitVectorHelper.get(buf1, i)); | |
230 | } | |
231 | for (int i = 0; i < count2; i++, outputIdx++) { | |
232 | assertEquals(BitVectorHelper.get(output, outputIdx), BitVectorHelper.get(buf2, i)); | |
233 | } | |
234 | } | |
235 | } |