]>
Commit | Line | Data |
---|---|---|
1d09f67e TL |
1 | /* |
2 | * Licensed to the Apache Software Foundation (ASF) under one or more | |
3 | * contributor license agreements. See the NOTICE file distributed with | |
4 | * this work for additional information regarding copyright ownership. | |
5 | * The ASF licenses this file to You under the Apache License, Version 2.0 | |
6 | * (the "License"); you may not use this file except in compliance with | |
7 | * the License. You may obtain a copy of the License at | |
8 | * | |
9 | * http://www.apache.org/licenses/LICENSE-2.0 | |
10 | * | |
11 | * Unless required by applicable law or agreed to in writing, software | |
12 | * distributed under the License is distributed on an "AS IS" BASIS, | |
13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |
14 | * See the License for the specific language governing permissions and | |
15 | * limitations under the License. | |
16 | */ | |
17 | ||
18 | package org.apache.arrow.vector; | |
19 | ||
20 | import java.util.Collections; | |
21 | import java.util.Iterator; | |
22 | ||
23 | import org.apache.arrow.memory.ArrowBuf; | |
24 | import org.apache.arrow.memory.BufferAllocator; | |
25 | import org.apache.arrow.memory.ReferenceManager; | |
26 | import org.apache.arrow.util.Preconditions; | |
27 | import org.apache.arrow.vector.util.DataSizeRoundingUtil; | |
28 | import org.apache.arrow.vector.util.TransferPair; | |
29 | import org.apache.arrow.vector.util.ValueVectorUtility; | |
30 | import org.slf4j.Logger; | |
31 | import org.slf4j.LoggerFactory; | |
32 | ||
33 | /** | |
34 | * Base class for other Arrow Vector Types. Provides basic functionality around | |
35 | * memory management. | |
36 | */ | |
37 | public abstract class BaseValueVector implements ValueVector { | |
38 | private static final Logger logger = LoggerFactory.getLogger(BaseValueVector.class); | |
39 | ||
40 | public static final String MAX_ALLOCATION_SIZE_PROPERTY = "arrow.vector.max_allocation_bytes"; | |
41 | public static final long MAX_ALLOCATION_SIZE = Long.getLong(MAX_ALLOCATION_SIZE_PROPERTY, Long.MAX_VALUE); | |
42 | /* | |
43 | * For all fixed width vectors, the value and validity buffers are sliced from a single buffer. | |
44 | * Similarly, for variable width vectors, the offsets and validity buffers are sliced from a | |
45 | * single buffer. To ensure the single buffer is power-of-2 size, the initial value allocation | |
46 | * should be less than power-of-2. For IntVectors, this comes to 3970*4 (15880) for the data | |
47 | * buffer and 504 bytes for the validity buffer, totalling to 16384 (2^16). | |
48 | */ | |
49 | public static final int INITIAL_VALUE_ALLOCATION = 3970; | |
50 | ||
51 | protected final BufferAllocator allocator; | |
52 | ||
53 | protected BaseValueVector(BufferAllocator allocator) { | |
54 | this.allocator = Preconditions.checkNotNull(allocator, "allocator cannot be null"); | |
55 | } | |
56 | ||
57 | @Override | |
58 | public abstract String getName(); | |
59 | ||
60 | /** | |
61 | * Representation of vector suitable for debugging. | |
62 | */ | |
63 | @Override | |
64 | public String toString() { | |
65 | return ValueVectorUtility.getToString(this, 0, getValueCount()); | |
66 | } | |
67 | ||
68 | @Override | |
69 | public void clear() { | |
70 | } | |
71 | ||
72 | @Override | |
73 | public void close() { | |
74 | clear(); | |
75 | } | |
76 | ||
77 | @Override | |
78 | public TransferPair getTransferPair(BufferAllocator allocator) { | |
79 | return getTransferPair(getName(), allocator); | |
80 | } | |
81 | ||
82 | @Override | |
83 | public Iterator<ValueVector> iterator() { | |
84 | return Collections.emptyIterator(); | |
85 | } | |
86 | ||
87 | /** | |
88 | * Checks to ensure that every buffer <code>vv</code> uses | |
89 | * has a positive reference count, throws if this precondition | |
90 | * isn't met. Returns true otherwise. | |
91 | */ | |
92 | public static boolean checkBufRefs(final ValueVector vv) { | |
93 | for (final ArrowBuf buffer : vv.getBuffers(false)) { | |
94 | if (buffer.refCnt() <= 0) { | |
95 | throw new IllegalStateException("zero refcount"); | |
96 | } | |
97 | } | |
98 | ||
99 | return true; | |
100 | } | |
101 | ||
102 | @Override | |
103 | public BufferAllocator getAllocator() { | |
104 | return allocator; | |
105 | } | |
106 | ||
107 | void compareTypes(BaseValueVector target, String caller) { | |
108 | if (this.getMinorType() != target.getMinorType()) { | |
109 | throw new UnsupportedOperationException(caller + " should have vectors of exact same type"); | |
110 | } | |
111 | } | |
112 | ||
113 | protected ArrowBuf releaseBuffer(ArrowBuf buffer) { | |
114 | buffer.getReferenceManager().release(); | |
115 | buffer = allocator.getEmpty(); | |
116 | return buffer; | |
117 | } | |
118 | ||
119 | /* number of bytes for the validity buffer for the given valueCount */ | |
120 | protected static int getValidityBufferSizeFromCount(final int valueCount) { | |
121 | return DataSizeRoundingUtil.divideBy8Ceil(valueCount); | |
122 | } | |
123 | ||
124 | /* round up bytes for the validity buffer for the given valueCount */ | |
125 | private static long roundUp8ForValidityBuffer(long valueCount) { | |
126 | return ((valueCount + 63) >> 6) << 3; | |
127 | } | |
128 | ||
129 | long computeCombinedBufferSize(int valueCount, int typeWidth) { | |
130 | Preconditions.checkArgument(valueCount >= 0, "valueCount must be >= 0"); | |
131 | Preconditions.checkArgument(typeWidth >= 0, "typeWidth must be >= 0"); | |
132 | ||
133 | // compute size of validity buffer. | |
134 | long bufferSize = roundUp8ForValidityBuffer(valueCount); | |
135 | ||
136 | // add the size of the value buffer. | |
137 | if (typeWidth == 0) { | |
138 | // for boolean type, value-buffer and validity-buffer are of same size. | |
139 | bufferSize *= 2; | |
140 | } else { | |
141 | bufferSize += DataSizeRoundingUtil.roundUpTo8Multiple((long) valueCount * typeWidth); | |
142 | } | |
143 | return allocator.getRoundingPolicy().getRoundedSize(bufferSize); | |
144 | } | |
145 | ||
146 | /** | |
147 | * Container for primitive vectors (1 for the validity bit-mask and one to hold the values). | |
148 | */ | |
149 | class DataAndValidityBuffers { | |
150 | private ArrowBuf dataBuf; | |
151 | private ArrowBuf validityBuf; | |
152 | ||
153 | DataAndValidityBuffers(ArrowBuf dataBuf, ArrowBuf validityBuf) { | |
154 | this.dataBuf = dataBuf; | |
155 | this.validityBuf = validityBuf; | |
156 | } | |
157 | ||
158 | ArrowBuf getDataBuf() { | |
159 | return dataBuf; | |
160 | } | |
161 | ||
162 | ArrowBuf getValidityBuf() { | |
163 | return validityBuf; | |
164 | } | |
165 | } | |
166 | ||
167 | DataAndValidityBuffers allocFixedDataAndValidityBufs(int valueCount, int typeWidth) { | |
168 | long bufferSize = computeCombinedBufferSize(valueCount, typeWidth); | |
169 | assert bufferSize <= MAX_ALLOCATION_SIZE; | |
170 | ||
171 | long validityBufferSize; | |
172 | long dataBufferSize; | |
173 | if (typeWidth == 0) { | |
174 | validityBufferSize = dataBufferSize = bufferSize / 2; | |
175 | } else { | |
176 | // Due to the rounding policy, the bufferSize could be greater than the | |
177 | // requested size. Utilize the allocated buffer fully.; | |
178 | long actualCount = (long) ((bufferSize * 8.0) / (8 * typeWidth + 1)); | |
179 | do { | |
180 | validityBufferSize = roundUp8ForValidityBuffer(actualCount); | |
181 | dataBufferSize = DataSizeRoundingUtil.roundUpTo8Multiple(actualCount * typeWidth); | |
182 | if (validityBufferSize + dataBufferSize <= bufferSize) { | |
183 | break; | |
184 | } | |
185 | --actualCount; | |
186 | } | |
187 | while (true); | |
188 | } | |
189 | ||
190 | ||
191 | /* allocate combined buffer */ | |
192 | ArrowBuf combinedBuffer = allocator.buffer(bufferSize); | |
193 | ||
194 | /* slice into requested lengths */ | |
195 | ArrowBuf dataBuf = null; | |
196 | ArrowBuf validityBuf = null; | |
197 | long bufferOffset = 0; | |
198 | for (int numBuffers = 0; numBuffers < 2; ++numBuffers) { | |
199 | long len = (numBuffers == 0 ? dataBufferSize : validityBufferSize); | |
200 | ArrowBuf buf = combinedBuffer.slice(bufferOffset, len); | |
201 | buf.getReferenceManager().retain(); | |
202 | buf.readerIndex(0); | |
203 | buf.writerIndex(0); | |
204 | ||
205 | bufferOffset += len; | |
206 | if (numBuffers == 0) { | |
207 | dataBuf = buf; | |
208 | } else { | |
209 | validityBuf = buf; | |
210 | } | |
211 | } | |
212 | combinedBuffer.getReferenceManager().release(); | |
213 | return new DataAndValidityBuffers(dataBuf, validityBuf); | |
214 | } | |
215 | ||
216 | public static ArrowBuf transferBuffer(final ArrowBuf srcBuffer, final BufferAllocator targetAllocator) { | |
217 | final ReferenceManager referenceManager = srcBuffer.getReferenceManager(); | |
218 | return referenceManager.transferOwnership(srcBuffer, targetAllocator).getTransferredBuffer(); | |
219 | } | |
220 | ||
221 | @Override | |
222 | public void copyFrom(int fromIndex, int thisIndex, ValueVector from) { | |
223 | throw new UnsupportedOperationException(); | |
224 | } | |
225 | ||
226 | @Override | |
227 | public void copyFromSafe(int fromIndex, int thisIndex, ValueVector from) { | |
228 | throw new UnsupportedOperationException(); | |
229 | } | |
230 | } | |
231 |