[ceph.git] / ceph / src / arrow / java / vector / src / main / java / org / apache / arrow / vector / BaseFixedWidthVector.java

/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.arrow.vector;

import static org.apache.arrow.memory.util.LargeMemoryUtil.capAtMaxInt;

import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;

import org.apache.arrow.memory.ArrowBuf;
import org.apache.arrow.memory.BufferAllocator;
import org.apache.arrow.memory.util.ArrowBufPointer;
import org.apache.arrow.memory.util.ByteFunctionHelpers;
import org.apache.arrow.memory.util.hash.ArrowBufHasher;
import org.apache.arrow.util.Preconditions;
import org.apache.arrow.vector.compare.VectorVisitor;
import org.apache.arrow.vector.ipc.message.ArrowFieldNode;
import org.apache.arrow.vector.types.pojo.Field;
import org.apache.arrow.vector.util.CallBack;
import org.apache.arrow.vector.util.OversizedAllocationException;
import org.apache.arrow.vector.util.TransferPair;

import io.netty.util.internal.PlatformDependent;

/**
 * BaseFixedWidthVector provides an abstract interface for
 * implementing vectors of fixed width values. The vectors are nullable
 * implying that zero or more elements in the vector could be NULL.
 */
public abstract class BaseFixedWidthVector extends BaseValueVector
        implements FixedWidthVector, FieldVector, VectorDefinitionSetter {
  private final int typeWidth;

  protected int lastValueCapacity;
  protected int actualValueCapacity;

  protected final Field field;
  private int allocationMonitor;
  protected ArrowBuf validityBuffer;
  protected ArrowBuf valueBuffer;
  protected int valueCount;

  /**
   * Constructs a new instance.
   *
   * @param field field materialized by this vector
   * @param allocator The allocator to use for allocating memory for the vector.
   * @param typeWidth The width in bytes of the type.
   */
  public BaseFixedWidthVector(Field field, final BufferAllocator allocator, final int typeWidth) {
    super(allocator);
    this.typeWidth = typeWidth;
    this.field = field;
    valueCount = 0;
    allocationMonitor = 0;
    validityBuffer = allocator.getEmpty();
    valueBuffer = allocator.getEmpty();
    lastValueCapacity = INITIAL_VALUE_ALLOCATION;
    refreshValueCapacity();
  }


  public int getTypeWidth() {
    return typeWidth;
  }

  @Override
  public String getName() {
    return field.getName();
  }

  /* TODO:
   * see if getNullCount() can be made faster -- O(1)
   */

  /* TODO:
   * Once the entire hierarchy has been refactored, move common functions
   * like getNullCount(), splitAndTransferValidityBuffer to top level
   * base class BaseValueVector.
   *
   * Along with this, some class members (validityBuffer) can also be
   * abstracted out to top level base class.
   *
   * Right now BaseValueVector is the top level base class for other
   * vector types in ValueVector hierarchy (non-nullable) and those
   * vectors have not yet been refactored/removed so moving things to
   * the top class as of now is not a good idea.
   */

  /**
   * Get the memory address of buffer that manages the validity
   * (NULL or NON-NULL nature) of elements in the vector.
   * @return starting address of the buffer
   */
  @Override
  public long getValidityBufferAddress() {
    return (validityBuffer.memoryAddress());
  }

  /**
   * Get the memory address of buffer that stores the data for elements
   * in the vector.
   * @return starting address of the buffer
   */
  @Override
  public long getDataBufferAddress() {
    return (valueBuffer.memoryAddress());
  }

  /**
   * Get the memory address of buffer that stores the offsets for elements
   * in the vector. This operation is not supported for fixed-width vectors.
   * @return starting address of the buffer
   * @throws UnsupportedOperationException for fixed width vectors
   */
  @Override
  public long getOffsetBufferAddress() {
    throw new UnsupportedOperationException("not supported for fixed-width vectors");
  }

  /**
   * Get buffer that manages the validity (NULL or NON-NULL nature) of
   * elements in the vector. Consider it as a buffer for internal bit vector
   * data structure.
   * @return buffer
   */
  @Override
  public ArrowBuf getValidityBuffer() {
    return validityBuffer;
  }

  /**
   * Get the buffer that stores the data for elements in the vector.
   * @return buffer
   */
  @Override
  public ArrowBuf getDataBuffer() {
    return valueBuffer;
  }

  /**
   * buffer that stores the offsets for elements
   * in the vector. This operation is not supported for fixed-width vectors.
   * @return buffer
   * @throws UnsupportedOperationException for fixed width vectors
   */
  @Override
  public ArrowBuf getOffsetBuffer() {
    throw new UnsupportedOperationException("not supported for fixed-width vectors");
  }

  /**
   * Sets the desired value capacity for the vector. This function doesn't
   * allocate any memory for the vector.
   * @param valueCount desired number of elements in the vector
   */
  @Override
  public void setInitialCapacity(int valueCount) {
    computeAndCheckBufferSize(valueCount);
    lastValueCapacity = valueCount;
  }

  /**
   * Get the current value capacity for the vector.
   *
   * @return number of elements that vector can hold.
   */
  @Override
  public int getValueCapacity() {
    return actualValueCapacity;
  }

  /**
   * Call this if you change the capacity of valueBuffer or validityBuffer.
   */
  protected void refreshValueCapacity() {
    actualValueCapacity = Math.min(getValueBufferValueCapacity(), getValidityBufferValueCapacity());
  }

  protected int getValueBufferValueCapacity() {
    return capAtMaxInt(valueBuffer.capacity() / typeWidth);
  }

  protected int getValidityBufferValueCapacity() {
    return capAtMaxInt(validityBuffer.capacity() * 8);
  }

  /**
   * zero out the vector and the data in associated buffers.
   */
  @Override
  public void zeroVector() {
    initValidityBuffer();
    initValueBuffer();
  }

  /* zero out the validity buffer */
  private void initValidityBuffer() {
    validityBuffer.setZero(0, validityBuffer.capacity());
  }

  /* zero out the data buffer */
  private void initValueBuffer() {
    valueBuffer.setZero(0, valueBuffer.capacity());
  }

  /**
   * Reset the vector to initial state. Same as {@link #zeroVector()}.
   * Note that this method doesn't release any memory.
   */
  @Override
  public void reset() {
    valueCount = 0;
    zeroVector();
  }

  /**
   * Close the vector and release the associated buffers.
   */
  @Override
  public void close() {
    clear();
  }

  /**
   * Same as {@link #close()}.
   */
  @Override
  public void clear() {
    valueCount = 0;
    validityBuffer = releaseBuffer(validityBuffer);
    valueBuffer = releaseBuffer(valueBuffer);
    refreshValueCapacity();
  }

  /* used to step down the memory allocation */
  protected void incrementAllocationMonitor() {
    if (allocationMonitor < 0) {
      allocationMonitor = 0;
    }
    allocationMonitor++;
  }

  /* used to step up the memory allocation */
  protected void decrementAllocationMonitor() {
    if (allocationMonitor > 0) {
      allocationMonitor = 0;
    }
    allocationMonitor--;
  }

  /**
   * Same as {@link #allocateNewSafe()}.
   */
  @Override
  public void allocateNew() {
    allocateNew(lastValueCapacity);
  }

  /**
   * Allocate memory for the vector. We internally use a default value count
   * of 4096 to allocate memory for at least these many elements in the
   * vector. See {@link #allocateNew(int)} for allocating memory for specific
   * number of elements in the vector.
   *
   * @return false if memory allocation fails, true otherwise.
   */
  @Override
  public boolean allocateNewSafe() {
    try {
      allocateNew(lastValueCapacity);
      return true;
    } catch (Exception e) {
      return false;
    }
  }

  /**
   * Allocate memory for the vector to support storing at least the provided number of
   * elements in the vector. This method must be called prior to using the ValueVector.
   *
   * @param valueCount the desired number of elements in the vector
   * @throws org.apache.arrow.memory.OutOfMemoryException on error
   */
  public void allocateNew(int valueCount) {
    computeAndCheckBufferSize(valueCount);

    /* we are doing a new allocation -- release the current buffers */
    clear();

    try {
      allocateBytes(valueCount);
    } catch (Exception e) {
      clear();
      throw e;
    }
  }

  /*
   * Compute the buffer size required for 'valueCount', and check if it's within bounds.
   */
  private long computeAndCheckBufferSize(int valueCount) {
    final long size = computeCombinedBufferSize(valueCount, typeWidth);
    if (size > MAX_ALLOCATION_SIZE) {
      throw new OversizedAllocationException("Memory required for vector capacity " +
          valueCount +
          " is (" + size + "), which is more than max allowed (" + MAX_ALLOCATION_SIZE + ")");
    }
    return size;
  }

  /**
   * Actual memory allocation is done by this function. All the calculations
   * and knowledge about what size to allocate is upto the callers of this
   * method.
   * Callers appropriately handle errors if memory allocation fails here.
   * Callers should also take care of determining that desired size is
   * within the bounds of max allocation allowed and any other error
   * conditions.
   */
  private void allocateBytes(int valueCount) {
    DataAndValidityBuffers buffers = allocFixedDataAndValidityBufs(valueCount, typeWidth);
    valueBuffer = buffers.getDataBuf();
    validityBuffer = buffers.getValidityBuf();
    zeroVector();

    refreshValueCapacity();
    lastValueCapacity = getValueCapacity();
  }

  /**
   * During splitAndTransfer, if we splitting from a random position within a byte,
   * we can't just slice the source buffer so we have to explicitly allocate the
   * validityBuffer of the target vector. This is unlike the databuffer which we can
   * always slice for the target vector.
   */
  private void allocateValidityBuffer(final int validityBufferSize) {
    validityBuffer = allocator.buffer(validityBufferSize);
    validityBuffer.readerIndex(0);
    refreshValueCapacity();
  }

  /**
   * Get the potential buffer size for a particular number of records.
   * @param count desired number of elements in the vector
   * @return estimated size of underlying buffers if the vector holds
   *         a given number of elements
   */
  @Override
  public int getBufferSizeFor(final int count) {
    if (count == 0) {
      return 0;
    }
    return (count * typeWidth) + getValidityBufferSizeFromCount(count);
  }

  /**
   * Get the size (number of bytes) of underlying buffers used by this
   * vector.
   * @return size of underlying buffers.
   */
  @Override
  public int getBufferSize() {
    if (valueCount == 0) {
      return 0;
    }
    return (valueCount * typeWidth) + getValidityBufferSizeFromCount(valueCount);
  }

  /**
   * Get information about how this field is materialized.
   * @return the field corresponding to this vector
   */
  @Override
  public Field getField() {
    return field;
  }

  /**
   * Return the underlying buffers associated with this vector. Note that this doesn't
   * impact the reference counts for this buffer so it only should be used for in-context
   * access. Also note that this buffer changes regularly thus
   * external classes shouldn't hold a reference to it (unless they change it).
   *
   * @param clear Whether to clear vector before returning; the buffers will still be refcounted
   *              but the returned array will be the only reference to them
   * @return The underlying {@link ArrowBuf buffers} that is used by this
   *         vector instance.
   */
  @Override
  public ArrowBuf[] getBuffers(boolean clear) {
    final ArrowBuf[] buffers;
    setReaderAndWriterIndex();
    if (getBufferSize() == 0) {
      buffers = new ArrowBuf[0];
    } else {
      buffers = new ArrowBuf[2];
      buffers[0] = validityBuffer;
      buffers[1] = valueBuffer;
    }
    if (clear) {
      for (final ArrowBuf buffer : buffers) {
        buffer.getReferenceManager().retain(1);
      }
      clear();
    }
    return buffers;
  }

  /**
   * Resize the vector to increase the capacity. The internal behavior is to
   * double the current value capacity.
   */
  @Override
  public void reAlloc() {
    int targetValueCount = getValueCapacity() * 2;
    if (targetValueCount == 0) {
      if (lastValueCapacity > 0) {
        targetValueCount = lastValueCapacity;
      } else {
        targetValueCount = INITIAL_VALUE_ALLOCATION * 2;
      }
    }
    computeAndCheckBufferSize(targetValueCount);

    DataAndValidityBuffers buffers = allocFixedDataAndValidityBufs(targetValueCount, typeWidth);
    final ArrowBuf newValueBuffer = buffers.getDataBuf();
    newValueBuffer.setBytes(0, valueBuffer, 0, valueBuffer.capacity());
    newValueBuffer.setZero(valueBuffer.capacity(), newValueBuffer.capacity() - valueBuffer.capacity());
    valueBuffer.getReferenceManager().release();
    valueBuffer = newValueBuffer;

    final ArrowBuf newValidityBuffer = buffers.getValidityBuf();
    newValidityBuffer.setBytes(0, validityBuffer, 0, validityBuffer.capacity());
    newValidityBuffer.setZero(validityBuffer.capacity(), newValidityBuffer.capacity() - validityBuffer.capacity());
    validityBuffer.getReferenceManager().release();
    validityBuffer = newValidityBuffer;

    refreshValueCapacity();
    lastValueCapacity = getValueCapacity();
  }

  /**
   * Get the inner vectors.
   *
   * @deprecated This API will be removed as the current implementations no longer support inner vectors.
   *
   * @return the inner vectors for this field as defined by the TypeLayout
   */
  @Deprecated
  @Override
  public List<BufferBacked> getFieldInnerVectors() {
    throw new UnsupportedOperationException("There are no inner vectors. Use getFieldBuffers");
  }

  /**
   * Initialize the children in schema for this Field. This operation is a
   * NO-OP for scalar types since they don't have any children.
   * @param children the schema
   * @throws IllegalArgumentException if children is a non-empty list for scalar types.
   */
  @Override
  public void initializeChildrenFromFields(List<Field> children) {
    if (!children.isEmpty()) {
      throw new IllegalArgumentException("primitive type vector can not have children");
    }
  }

  /**
   * Get the inner child vectors.
   * @return list of child vectors for complex types, empty list for scalar vector types
   */
  @Override
  public List<FieldVector> getChildrenFromFields() {
    return Collections.emptyList();
  }

  /**
   * Load the buffers of this vector with provided source buffers.
   * The caller manages the source buffers and populates them before invoking
   * this method.
   * @param fieldNode  the fieldNode indicating the value count
   * @param ownBuffers the buffers for this Field (own buffers only, children not included)
   */
  @Override
  public void loadFieldBuffers(ArrowFieldNode fieldNode, List<ArrowBuf> ownBuffers) {
    if (ownBuffers.size() != 2) {
      throw new IllegalArgumentException("Illegal buffer count, expected " + 2 + ", got: " + ownBuffers.size());
    }

    ArrowBuf bitBuffer = ownBuffers.get(0);
    ArrowBuf dataBuffer = ownBuffers.get(1);

    validityBuffer.getReferenceManager().release();
    validityBuffer = BitVectorHelper.loadValidityBuffer(fieldNode, bitBuffer, allocator);
    valueBuffer.getReferenceManager().release();
    valueBuffer = dataBuffer.getReferenceManager().retain(dataBuffer, allocator);
    refreshValueCapacity();

    valueCount = fieldNode.getLength();
  }

  /**
   * Get the buffers belonging to this vector.
   *
   * @return the inner buffers.
   */
  public List<ArrowBuf> getFieldBuffers() {
    List<ArrowBuf> result = new ArrayList<>(2);
    setReaderAndWriterIndex();
    result.add(validityBuffer);
    result.add(valueBuffer);

    return result;
  }

  /**
   * Set the reader and writer indexes for the inner buffers.
   */
  private void setReaderAndWriterIndex() {
    validityBuffer.readerIndex(0);
    valueBuffer.readerIndex(0);
    if (valueCount == 0) {
      validityBuffer.writerIndex(0);
      valueBuffer.writerIndex(0);
    } else {
      validityBuffer.writerIndex(getValidityBufferSizeFromCount(valueCount));
      if (typeWidth == 0) {
        /* specialized handling for BitVector */
        valueBuffer.writerIndex(getValidityBufferSizeFromCount(valueCount));
      } else {
        valueBuffer.writerIndex((long) valueCount * typeWidth);
      }
    }
  }

  /**
   * Construct a transfer pair of this vector and another vector of same type.
   * @param ref name of the target vector
   * @param allocator allocator for the target vector
   * @param callBack not used
   * @return TransferPair
   */
  @Override
  public TransferPair getTransferPair(String ref, BufferAllocator allocator, CallBack callBack) {
    return getTransferPair(ref, allocator);
  }

  /**
   * Construct a transfer pair of this vector and another vector of same type.
   * @param allocator allocator for the target vector
   * @return TransferPair
   */
  @Override
  public TransferPair getTransferPair(BufferAllocator allocator) {
    return getTransferPair(getName(), allocator);
  }

  /**
   * Construct a transfer pair of this vector and another vector of same type.
   * @param ref name of the target vector
   * @param allocator allocator for the target vector
   * @return TransferPair
   */
  public abstract TransferPair getTransferPair(String ref, BufferAllocator allocator);

  /**
   * Transfer this vector'data to another vector. The memory associated
   * with this vector is transferred to the allocator of target vector
   * for accounting and management purposes.
   * @param target destination vector for transfer
   */
  public void transferTo(BaseFixedWidthVector target) {
    compareTypes(target, "transferTo");
    target.clear();
    target.validityBuffer = transferBuffer(validityBuffer, target.allocator);
    target.valueBuffer = transferBuffer(valueBuffer, target.allocator);
    target.valueCount = valueCount;
    target.refreshValueCapacity();
    clear();
  }

  /**
   * Slice this vector at desired index and length and transfer the
   * corresponding data to the target vector.
   * @param startIndex start position of the split in source vector.
   * @param length length of the split.
   * @param target destination vector
   */
  public void splitAndTransferTo(int startIndex, int length,
                                 BaseFixedWidthVector target) {
    Preconditions.checkArgument(startIndex >= 0 && length >= 0 && startIndex + length <= valueCount,
        "Invalid parameters startIndex: %s, length: %s for valueCount: %s", startIndex, length, valueCount);
    compareTypes(target, "splitAndTransferTo");
    target.clear();
    splitAndTransferValidityBuffer(startIndex, length, target);
    splitAndTransferValueBuffer(startIndex, length, target);
    target.setValueCount(length);
  }

  /**
   * Data buffer can always be split and transferred using slicing.
   */
  private void splitAndTransferValueBuffer(int startIndex, int length,
                                           BaseFixedWidthVector target) {
    final int startPoint = startIndex * typeWidth;
    final int sliceLength = length * typeWidth;
    final ArrowBuf slicedBuffer = valueBuffer.slice(startPoint, sliceLength);
    target.valueBuffer = transferBuffer(slicedBuffer, target.allocator);
    target.refreshValueCapacity();
  }

  /**
   * Validity buffer has multiple cases of split and transfer depending on
   * the starting position of the source index.
   */
  private void splitAndTransferValidityBuffer(int startIndex, int length,
                                              BaseFixedWidthVector target) {
    int firstByteSource = BitVectorHelper.byteIndex(startIndex);
    int lastByteSource = BitVectorHelper.byteIndex(valueCount - 1);
    int byteSizeTarget = getValidityBufferSizeFromCount(length);
    int offset = startIndex % 8;

    if (length > 0) {
      if (offset == 0) {
        /* slice */
        if (target.validityBuffer != null) {
          target.validityBuffer.getReferenceManager().release();
        }
        target.validityBuffer = validityBuffer.slice(firstByteSource, byteSizeTarget);
        target.validityBuffer.getReferenceManager().retain(1);
        target.refreshValueCapacity();
      } else {
        /* Copy data
         * When the first bit starts from the middle of a byte (offset != 0),
         * copy data from src BitVector.
         * Each byte in the target is composed by a part in i-th byte,
         * another part in (i+1)-th byte.
         */
        target.allocateValidityBuffer(byteSizeTarget);

        for (int i = 0; i < byteSizeTarget - 1; i++) {
          byte b1 = BitVectorHelper.getBitsFromCurrentByte(this.validityBuffer,
                  firstByteSource + i, offset);
          byte b2 = BitVectorHelper.getBitsFromNextByte(this.validityBuffer,
                  firstByteSource + i + 1, offset);

          target.validityBuffer.setByte(i, (b1 + b2));
        }

        /* Copying the last piece is done in the following manner:
         * if the source vector has 1 or more bytes remaining, we copy
         * the last piece as a byte formed by shifting data
         * from the current byte and the next byte.
         *
         * if the source vector has no more bytes remaining
         * (we are at the last byte), we copy the last piece as a byte
         * by shifting data from the current byte.
         */
        if ((firstByteSource + byteSizeTarget - 1) < lastByteSource) {
          byte b1 = BitVectorHelper.getBitsFromCurrentByte(this.validityBuffer,
                  firstByteSource + byteSizeTarget - 1, offset);
          byte b2 = BitVectorHelper.getBitsFromNextByte(this.validityBuffer,
                  firstByteSource + byteSizeTarget, offset);

          target.validityBuffer.setByte(byteSizeTarget - 1, b1 + b2);
        } else {
          byte b1 = BitVectorHelper.getBitsFromCurrentByte(this.validityBuffer,
                  firstByteSource + byteSizeTarget - 1, offset);
          target.validityBuffer.setByte(byteSizeTarget - 1, b1);
        }
      }
    }
  }


  /*----------------------------------------------------------------*
   |                                                                |
   |          common getters and setters                            |
   |                                                                |
   *----------------------------------------------------------------*/


  /**
   * Get the number of elements that are null in the vector.
   *
   * @return the number of null elements.
   */
  @Override
  public int getNullCount() {
    return BitVectorHelper.getNullCount(validityBuffer, valueCount);
  }

  /**
   * Get the value count of vector. This will always be zero unless
   * {@link #setValueCount(int)} has been called prior to calling this.
   *
   * @return valueCount for the vector
   */
  @Override
  public int getValueCount() {
    return valueCount;
  }

  /**
   * Set value count for the vector.
   *
   * @param valueCount  value count to set
   */
  @Override
  public void setValueCount(int valueCount) {
    this.valueCount = valueCount;
    final int currentValueCapacity = getValueCapacity();
    while (valueCount > getValueCapacity()) {
      reAlloc();
    }
    /*
     * We are trying to understand the pattern of memory allocation.
     * If initially, the user did vector.allocateNew(), we would have
     * allocated memory of default size (4096 * type width).
     * Later on user invokes setValueCount(count).
     *
     * If the existing value capacity is twice as large as the
     * valueCount, we know that we over-provisioned memory in the
     * first place when default memory allocation was done because user
     * really needs a much less value count in the vector.
     *
     * We record this by bumping up the allocationMonitor. If this pattern
     * happens for certain number of times and allocationMonitor
     * reaches the threshold (internal hardcoded) value, subsequent
     * call to allocateNew() will take care of stepping down the
     * default memory allocation size.
     *
     * Another case would be under-provisioning the initial memory and
     * thus going through a lot of realloc(). Here the goal is to
     * see if we can minimize the number of reallocations. Again the
     * state is recorded in allocationMonitor by decrementing it
     * (negative value). If a threshold is hit, realloc will try to
     * allocate more memory in order to possibly avoid a future realloc.
     * This case is also applicable to setSafe() methods which can trigger
     * a realloc() and thus we record the state there as well.
     */
    if (valueCount > 0) {
      if (currentValueCapacity >= (valueCount * 2)) {
        incrementAllocationMonitor();
      } else if (currentValueCapacity <= (valueCount / 2)) {
        decrementAllocationMonitor();
      }
    }
    setReaderAndWriterIndex();
  }

  /**
   * Check if the given index is within the current value capacity
   * of the vector.
   *
   * @param index  position to check
   * @return true if index is within the current value capacity
   */
  public boolean isSafe(int index) {
    return index < getValueCapacity();
  }

  /**
   * Check if element at given index is null.
   *
   * @param index  position of element
   * @return true if element at given index is null, false otherwise
   */
  @Override
  public boolean isNull(int index) {
    return (isSet(index) == 0);
  }

  /**
   * Same as {@link #isNull(int)}.
   *
   * @param index  position of element
   * @return 1 if element at given index is not null, 0 otherwise
   */
  public int isSet(int index) {
    final int byteIndex = index >> 3;
    final byte b = validityBuffer.getByte(byteIndex);
    final int bitIndex = index & 7;
    return (b >> bitIndex) & 0x01;
  }

  /**
   * Mark the particular position in the vector as non-null.
   *
   * @param index position of the element.
   */
  @Override
  public void setIndexDefined(int index) {
    handleSafe(index);
    BitVectorHelper.setBit(validityBuffer, index);
  }

  public void set(int index, byte[] value, int start, int length) {
    throw new UnsupportedOperationException();
  }

  public void setSafe(int index, byte[] value, int start, int length) {
    throw new UnsupportedOperationException();
  }

  public void set(int index, ByteBuffer value, int start, int length) {
    throw new UnsupportedOperationException();
  }

  public void setSafe(int index, ByteBuffer value, int start, int length) {
    throw new UnsupportedOperationException();
  }


  /*----------------------------------------------------------------*
   |                                                                |
   |                helper methods for setters                      |
   |                                                                |
   *----------------------------------------------------------------*/


  protected void handleSafe(int index) {
    while (index >= getValueCapacity()) {
      decrementAllocationMonitor();
      reAlloc();
    }
  }

  /**
   * Copy a cell value from a particular index in source vector to a particular
   * position in this vector. The source vector should be of the same type as this one.
   *
   * @param fromIndex position to copy from in source vector
   * @param thisIndex position to copy to in this vector
   * @param from      source vector
   */
  @Override
  public void copyFrom(int fromIndex, int thisIndex, ValueVector from) {
    Preconditions.checkArgument(this.getMinorType() == from.getMinorType());
    if (from.isNull(fromIndex)) {
      BitVectorHelper.unsetBit(this.getValidityBuffer(), thisIndex);
    } else {
      BitVectorHelper.setBit(this.getValidityBuffer(), thisIndex);
      PlatformDependent.copyMemory(from.getDataBuffer().memoryAddress() + (long) fromIndex * typeWidth,
              this.getDataBuffer().memoryAddress() + (long) thisIndex * typeWidth, typeWidth);
    }
  }

  /**
   * Same as {@link #copyFrom(int, int, ValueVector)} except that
   * it handles the case when the capacity of the vector needs to be expanded
   * before copy.
   *
   * @param fromIndex position to copy from in source vector
   * @param thisIndex position to copy to in this vector
   * @param from      source vector
   */
  @Override
  public void copyFromSafe(int fromIndex, int thisIndex, ValueVector from) {
    Preconditions.checkArgument(this.getMinorType() == from.getMinorType());
    handleSafe(thisIndex);
    copyFrom(fromIndex, thisIndex, from);
  }

  /**
   * Set the element at the given index to null.
   *
   * @param index position of element
   */
  public void setNull(int index) {
    handleSafe(index);
    // not really needed to set the bit to 0 as long as
    // the buffer always starts from 0.
    BitVectorHelper.unsetBit(validityBuffer, index);
  }

  @Override
  public ArrowBufPointer getDataPointer(int index) {
    return getDataPointer(index, new ArrowBufPointer());
  }

  @Override
  public ArrowBufPointer getDataPointer(int index, ArrowBufPointer reuse) {
    if (isNull(index)) {
      reuse.set(null, 0, 0);
    } else {
      reuse.set(valueBuffer, (long) index * typeWidth, typeWidth);
    }
    return reuse;
  }

  @Override
  public int hashCode(int index) {
    return hashCode(index, null);
  }

  @Override
  public int hashCode(int index, ArrowBufHasher hasher) {
    if (isNull(index)) {
      return ArrowBufPointer.NULL_HASH_CODE;
    }
    long start = (long) typeWidth * index;
    long end = (long) typeWidth * (index + 1);
    return ByteFunctionHelpers.hash(hasher, this.getDataBuffer(), start, end);
  }

  @Override
  public <OUT, IN> OUT accept(VectorVisitor<OUT, IN> visitor, IN value) {
    return visitor.visit(this, value);
  }

}