ceph/src/rocksdb/utilities/column_aware_encoding_test.cc

   1 //  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
   2 //  This source code is licensed under both the GPLv2 (found in the
   3 //  COPYING file in the root directory) and Apache 2.0 License
   4 //  (found in the LICENSE.Apache file in the root directory).
   5 //
   6 #ifndef ROCKSDB_LITE
   7
   8 #include <vector>
   9 #include "util/testharness.h"
  10 #include "util/testutil.h"
  11 #include "utilities/col_buf_decoder.h"
  12 #include "utilities/col_buf_encoder.h"
  13
  14 namespace rocksdb {
  15
  16 class ColumnAwareEncodingTest : public testing::Test {
  17  public:
  18   ColumnAwareEncodingTest() {}
  19
  20   ~ColumnAwareEncodingTest() {}
  21 };
  22
  23 class ColumnAwareEncodingTestWithSize
  24     : public ColumnAwareEncodingTest,
  25       public testing::WithParamInterface<size_t> {
  26  public:
  27   ColumnAwareEncodingTestWithSize() {}
  28
  29   ~ColumnAwareEncodingTestWithSize() {}
  30
  31   static std::vector<size_t> GetValues() { return {4, 8}; }
  32 };
  33
  34 INSTANTIATE_TEST_CASE_P(
  35     ColumnAwareEncodingTestWithSize, ColumnAwareEncodingTestWithSize,
  36     ::testing::ValuesIn(ColumnAwareEncodingTestWithSize::GetValues()));
  37
  38 TEST_P(ColumnAwareEncodingTestWithSize, NoCompressionEncodeDecode) {
  39   size_t col_size = GetParam();
  40   std::unique_ptr<ColBufEncoder> col_buf_encoder(
  41       new FixedLengthColBufEncoder(col_size, kColNoCompression, false, true));
  42   std::string str_buf;
  43   uint64_t base_val = 0x0102030405060708;
  44   uint64_t val = 0;
  45   memcpy(&val, &base_val, col_size);
  46   const int row_count = 4;
  47   for (int i = 0; i < row_count; ++i) {
  48     str_buf.append(reinterpret_cast<char*>(&val), col_size);
  49   }
  50   const char* str_buf_ptr = str_buf.c_str();
  51   for (int i = 0; i < row_count; ++i) {
  52     col_buf_encoder->Append(str_buf_ptr);
  53   }
  54   col_buf_encoder->Finish();
  55   const std::string& encoded_data = col_buf_encoder->GetData();
  56   // Check correctness of encoded string length
  57   ASSERT_EQ(row_count * col_size, encoded_data.size());
  58
  59   const char* encoded_data_ptr = encoded_data.c_str();
  60   uint64_t expected_encoded_val;
  61   if (col_size == 8) {
  62     expected_encoded_val = port::kLittleEndian ? 0x0807060504030201 : 0x0102030405060708;
  63   } else if (col_size == 4) {
  64     expected_encoded_val = port::kLittleEndian ? 0x08070605 : 0x0102030400000000;
  65   }
  66   uint64_t encoded_val = 0;
  67   for (int i = 0; i < row_count; ++i) {
  68     memcpy(&encoded_val, encoded_data_ptr, col_size);
  69     // Check correctness of encoded value
  70     ASSERT_EQ(expected_encoded_val, encoded_val);
  71     encoded_data_ptr += col_size;
  72   }
  73
  74   std::unique_ptr<ColBufDecoder> col_buf_decoder(
  75       new FixedLengthColBufDecoder(col_size, kColNoCompression, false, true));
  76   encoded_data_ptr = encoded_data.c_str();
  77   encoded_data_ptr += col_buf_decoder->Init(encoded_data_ptr);
  78   char* decoded_data = new char[100];
  79   char* decoded_data_base = decoded_data;
  80   for (int i = 0; i < row_count; ++i) {
  81     encoded_data_ptr +=
  82         col_buf_decoder->Decode(encoded_data_ptr, &decoded_data);
  83   }
  84
  85   // Check correctness of decoded string length
  86   ASSERT_EQ(row_count * col_size, decoded_data - decoded_data_base);
  87   decoded_data = decoded_data_base;
  88   for (int i = 0; i < row_count; ++i) {
  89     uint64_t decoded_val;
  90     decoded_val = 0;
  91     memcpy(&decoded_val, decoded_data, col_size);
  92     // Check correctness of decoded value
  93     ASSERT_EQ(val, decoded_val);
  94     decoded_data += col_size;
  95   }
  96   delete[] decoded_data_base;
  97 }
  98
  99 TEST_P(ColumnAwareEncodingTestWithSize, RleEncodeDecode) {
 100   size_t col_size = GetParam();
 101   std::unique_ptr<ColBufEncoder> col_buf_encoder(
 102       new FixedLengthColBufEncoder(col_size, kColRle, false, true));
 103   std::string str_buf;
 104   uint64_t base_val = 0x0102030405060708;
 105   uint64_t val = 0;
 106   memcpy(&val, &base_val, col_size);
 107   const int row_count = 4;
 108   for (int i = 0; i < row_count; ++i) {
 109     str_buf.append(reinterpret_cast<char*>(&val), col_size);
 110   }
 111   const char* str_buf_ptr = str_buf.c_str();
 112   for (int i = 0; i < row_count; ++i) {
 113     str_buf_ptr += col_buf_encoder->Append(str_buf_ptr);
 114   }
 115   col_buf_encoder->Finish();
 116   const std::string& encoded_data = col_buf_encoder->GetData();
 117   // Check correctness of encoded string length
 118   ASSERT_EQ(col_size + 1, encoded_data.size());
 119
 120   const char* encoded_data_ptr = encoded_data.c_str();
 121   uint64_t encoded_val = 0;
 122   memcpy(&encoded_val, encoded_data_ptr, col_size);
 123   uint64_t expected_encoded_val;
 124   if (col_size == 8) {
 125     expected_encoded_val = port::kLittleEndian ? 0x0807060504030201 : 0x0102030405060708;
 126   } else if (col_size == 4) {
 127     expected_encoded_val = port::kLittleEndian ? 0x08070605 : 0x0102030400000000;
 128   }
 129   // Check correctness of encoded value
 130   ASSERT_EQ(expected_encoded_val, encoded_val);
 131
 132   std::unique_ptr<ColBufDecoder> col_buf_decoder(
 133       new FixedLengthColBufDecoder(col_size, kColRle, false, true));
 134   char* decoded_data = new char[100];
 135   char* decoded_data_base = decoded_data;
 136   encoded_data_ptr += col_buf_decoder->Init(encoded_data_ptr);
 137   for (int i = 0; i < row_count; ++i) {
 138     encoded_data_ptr +=
 139         col_buf_decoder->Decode(encoded_data_ptr, &decoded_data);
 140   }
 141   // Check correctness of decoded string length
 142   ASSERT_EQ(decoded_data - decoded_data_base, row_count * col_size);
 143   decoded_data = decoded_data_base;
 144   for (int i = 0; i < row_count; ++i) {
 145     uint64_t decoded_val;
 146     decoded_val = 0;
 147     memcpy(&decoded_val, decoded_data, col_size);
 148     // Check correctness of decoded value
 149     ASSERT_EQ(val, decoded_val);
 150     decoded_data += col_size;
 151   }
 152   delete[] decoded_data_base;
 153 }
 154
 155 TEST_P(ColumnAwareEncodingTestWithSize, DeltaEncodeDecode) {
 156   size_t col_size = GetParam();
 157   int row_count = 4;
 158   std::unique_ptr<ColBufEncoder> col_buf_encoder(
 159       new FixedLengthColBufEncoder(col_size, kColDeltaVarint, false, true));
 160   std::string str_buf;
 161   uint64_t base_val1 = port::kLittleEndian ? 0x0102030405060708 : 0x0807060504030201;
 162   uint64_t base_val2 = port::kLittleEndian ? 0x0202030405060708 : 0x0807060504030202;
 163   uint64_t val1 = 0, val2 = 0;
 164   memcpy(&val1, &base_val1, col_size);
 165   memcpy(&val2, &base_val2, col_size);
 166   const char* str_buf_ptr;
 167   for (int i = 0; i < row_count / 2; ++i) {
 168     str_buf = std::string(reinterpret_cast<char*>(&val1), col_size);
 169     str_buf_ptr = str_buf.c_str();
 170     col_buf_encoder->Append(str_buf_ptr);
 171
 172     str_buf = std::string(reinterpret_cast<char*>(&val2), col_size);
 173     str_buf_ptr = str_buf.c_str();
 174     col_buf_encoder->Append(str_buf_ptr);
 175   }
 176   col_buf_encoder->Finish();
 177   const std::string& encoded_data = col_buf_encoder->GetData();
 178   // Check encoded string length
 179   int varint_len = 0;
 180   if (col_size == 8) {
 181     varint_len = 9;
 182   } else if (col_size == 4) {
 183     varint_len = port::kLittleEndian ? 5 : 9;
 184   }
 185   // Check encoded string length: first value is original one (val - 0), the
 186   // coming three are encoded as 1, -1, 1, so they should take 1 byte in varint.
 187   ASSERT_EQ(varint_len + 3 * 1, encoded_data.size());
 188
 189   std::unique_ptr<ColBufDecoder> col_buf_decoder(
 190       new FixedLengthColBufDecoder(col_size, kColDeltaVarint, false, true));
 191   char* decoded_data = new char[100];
 192   char* decoded_data_base = decoded_data;
 193   const char* encoded_data_ptr = encoded_data.c_str();
 194   encoded_data_ptr += col_buf_decoder->Init(encoded_data_ptr);
 195   for (int i = 0; i < row_count; ++i) {
 196     encoded_data_ptr +=
 197         col_buf_decoder->Decode(encoded_data_ptr, &decoded_data);
 198   }
 199
 200   // Check correctness of decoded string length
 201   ASSERT_EQ(row_count * col_size, decoded_data - decoded_data_base);
 202   decoded_data = decoded_data_base;
 203
 204   // Check correctness of decoded data
 205   for (int i = 0; i < row_count / 2; ++i) {
 206     uint64_t decoded_val = 0;
 207     memcpy(&decoded_val, decoded_data, col_size);
 208     ASSERT_EQ(val1, decoded_val);
 209     decoded_data += col_size;
 210     memcpy(&decoded_val, decoded_data, col_size);
 211     ASSERT_EQ(val2, decoded_val);
 212     decoded_data += col_size;
 213   }
 214   delete[] decoded_data_base;
 215 }
 216
 217 TEST_F(ColumnAwareEncodingTest, ChunkBufEncodeDecode) {
 218   std::unique_ptr<ColBufEncoder> col_buf_encoder(
 219       new VariableChunkColBufEncoder(kColDict));
 220   std::string buf("12345678\377\1\0\0\0\0\0\0\0\376", 18);
 221   col_buf_encoder->Append(buf.c_str());
 222   col_buf_encoder->Finish();
 223   const std::string& encoded_data = col_buf_encoder->GetData();
 224   const char* str_ptr = encoded_data.c_str();
 225
 226   std::unique_ptr<ColBufDecoder> col_buf_decoder(
 227       new VariableChunkColBufDecoder(kColDict));
 228   str_ptr += col_buf_decoder->Init(str_ptr);
 229   char* decoded_data = new char[100];
 230   char* decoded_data_base = decoded_data;
 231   col_buf_decoder->Decode(str_ptr, &decoded_data);
 232   for (size_t i = 0; i < buf.size(); ++i) {
 233     ASSERT_EQ(buf[i], decoded_data_base[i]);
 234   }
 235   delete[] decoded_data_base;
 236 }
 237
 238 }  // namespace rocksdb
 239
 240 int main(int argc, char** argv) {
 241   ::testing::InitGoogleTest(&argc, argv);
 242   return RUN_ALL_TESTS();
 243 }
 244
 245 #else
 246
 247 #include <cstdio>
 248
 249 int main() {
 250   fprintf(stderr,
 251           "SKIPPED as column aware encoding experiment is not enabled in "
 252           "ROCKSDB_LITE\n");
 253 }
 254 #endif  // ROCKSDB_LITE