ceph/src/rocksdb/db/cuckoo_table_db_test.cc

   1 //  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
   2 //  This source code is licensed under both the GPLv2 (found in the
   3 //  COPYING file in the root directory) and Apache 2.0 License
   4 //  (found in the LICENSE.Apache file in the root directory).
   5
   6 #ifndef ROCKSDB_LITE
   7
   8 #include "db/db_impl/db_impl.h"
   9 #include "rocksdb/db.h"
  10 #include "rocksdb/env.h"
  11 #include "table/cuckoo/cuckoo_table_factory.h"
  12 #include "table/cuckoo/cuckoo_table_reader.h"
  13 #include "table/meta_blocks.h"
  14 #include "test_util/testharness.h"
  15 #include "test_util/testutil.h"
  16 #include "util/cast_util.h"
  17 #include "util/string_util.h"
  18
  19 namespace ROCKSDB_NAMESPACE {
  20
  21 class CuckooTableDBTest : public testing::Test {
  22  private:
  23   std::string dbname_;
  24   Env* env_;
  25   DB* db_;
  26
  27  public:
  28   CuckooTableDBTest() : env_(Env::Default()) {
  29     dbname_ = test::PerThreadDBPath("cuckoo_table_db_test");
  30     EXPECT_OK(DestroyDB(dbname_, Options()));
  31     db_ = nullptr;
  32     Reopen();
  33   }
  34
  35   ~CuckooTableDBTest() override {
  36     delete db_;
  37     EXPECT_OK(DestroyDB(dbname_, Options()));
  38   }
  39
  40   Options CurrentOptions() {
  41     Options options;
  42     options.table_factory.reset(NewCuckooTableFactory());
  43     options.memtable_factory.reset(NewHashLinkListRepFactory(4, 0, 3, true));
  44     options.allow_mmap_reads = true;
  45     options.create_if_missing = true;
  46     options.allow_concurrent_memtable_write = false;
  47     return options;
  48   }
  49
  50   DBImpl* dbfull() { return static_cast_with_check<DBImpl>(db_); }
  51
  52   // The following util methods are copied from plain_table_db_test.
  53   void Reopen(Options* options = nullptr) {
  54     delete db_;
  55     db_ = nullptr;
  56     Options opts;
  57     if (options != nullptr) {
  58       opts = *options;
  59     } else {
  60       opts = CurrentOptions();
  61       opts.create_if_missing = true;
  62     }
  63     ASSERT_OK(DB::Open(opts, dbname_, &db_));
  64   }
  65
  66   void DestroyAndReopen(Options* options) {
  67     assert(options);
  68     ASSERT_OK(db_->Close());
  69     delete db_;
  70     db_ = nullptr;
  71     ASSERT_OK(DestroyDB(dbname_, *options));
  72     Reopen(options);
  73   }
  74
  75   Status Put(const Slice& k, const Slice& v) {
  76     return db_->Put(WriteOptions(), k, v);
  77   }
  78
  79   Status Delete(const std::string& k) {
  80     return db_->Delete(WriteOptions(), k);
  81   }
  82
  83   std::string Get(const std::string& k) {
  84     ReadOptions options;
  85     std::string result;
  86     Status s = db_->Get(options, k, &result);
  87     if (s.IsNotFound()) {
  88       result = "NOT_FOUND";
  89     } else if (!s.ok()) {
  90       result = s.ToString();
  91     }
  92     return result;
  93   }
  94
  95   int NumTableFilesAtLevel(int level) {
  96     std::string property;
  97     EXPECT_TRUE(db_->GetProperty(
  98         "rocksdb.num-files-at-level" + NumberToString(level), &property));
  99     return atoi(property.c_str());
 100   }
 101
 102   // Return spread of files per level
 103   std::string FilesPerLevel() {
 104     std::string result;
 105     size_t last_non_zero_offset = 0;
 106     for (int level = 0; level < db_->NumberLevels(); level++) {
 107       int f = NumTableFilesAtLevel(level);
 108       char buf[100];
 109       snprintf(buf, sizeof(buf), "%s%d", (level ? "," : ""), f);
 110       result += buf;
 111       if (f > 0) {
 112         last_non_zero_offset = result.size();
 113       }
 114     }
 115     result.resize(last_non_zero_offset);
 116     return result;
 117   }
 118 };
 119
 120 TEST_F(CuckooTableDBTest, Flush) {
 121   // Try with empty DB first.
 122   ASSERT_TRUE(dbfull() != nullptr);
 123   ASSERT_EQ("NOT_FOUND", Get("key2"));
 124
 125   // Add some values to db.
 126   Options options = CurrentOptions();
 127   Reopen(&options);
 128
 129   ASSERT_OK(Put("key1", "v1"));
 130   ASSERT_OK(Put("key2", "v2"));
 131   ASSERT_OK(Put("key3", "v3"));
 132   dbfull()->TEST_FlushMemTable();
 133
 134   TablePropertiesCollection ptc;
 135   reinterpret_cast<DB*>(dbfull())->GetPropertiesOfAllTables(&ptc);
 136   ASSERT_EQ(1U, ptc.size());
 137   ASSERT_EQ(3U, ptc.begin()->second->num_entries);
 138   ASSERT_EQ("1", FilesPerLevel());
 139
 140   ASSERT_EQ("v1", Get("key1"));
 141   ASSERT_EQ("v2", Get("key2"));
 142   ASSERT_EQ("v3", Get("key3"));
 143   ASSERT_EQ("NOT_FOUND", Get("key4"));
 144
 145   // Now add more keys and flush.
 146   ASSERT_OK(Put("key4", "v4"));
 147   ASSERT_OK(Put("key5", "v5"));
 148   ASSERT_OK(Put("key6", "v6"));
 149   dbfull()->TEST_FlushMemTable();
 150
 151   reinterpret_cast<DB*>(dbfull())->GetPropertiesOfAllTables(&ptc);
 152   ASSERT_EQ(2U, ptc.size());
 153   auto row = ptc.begin();
 154   ASSERT_EQ(3U, row->second->num_entries);
 155   ASSERT_EQ(3U, (++row)->second->num_entries);
 156   ASSERT_EQ("2", FilesPerLevel());
 157   ASSERT_EQ("v1", Get("key1"));
 158   ASSERT_EQ("v2", Get("key2"));
 159   ASSERT_EQ("v3", Get("key3"));
 160   ASSERT_EQ("v4", Get("key4"));
 161   ASSERT_EQ("v5", Get("key5"));
 162   ASSERT_EQ("v6", Get("key6"));
 163
 164   ASSERT_OK(Delete("key6"));
 165   ASSERT_OK(Delete("key5"));
 166   ASSERT_OK(Delete("key4"));
 167   dbfull()->TEST_FlushMemTable();
 168   reinterpret_cast<DB*>(dbfull())->GetPropertiesOfAllTables(&ptc);
 169   ASSERT_EQ(3U, ptc.size());
 170   row = ptc.begin();
 171   ASSERT_EQ(3U, row->second->num_entries);
 172   ASSERT_EQ(3U, (++row)->second->num_entries);
 173   ASSERT_EQ(3U, (++row)->second->num_entries);
 174   ASSERT_EQ("3", FilesPerLevel());
 175   ASSERT_EQ("v1", Get("key1"));
 176   ASSERT_EQ("v2", Get("key2"));
 177   ASSERT_EQ("v3", Get("key3"));
 178   ASSERT_EQ("NOT_FOUND", Get("key4"));
 179   ASSERT_EQ("NOT_FOUND", Get("key5"));
 180   ASSERT_EQ("NOT_FOUND", Get("key6"));
 181 }
 182
 183 TEST_F(CuckooTableDBTest, FlushWithDuplicateKeys) {
 184   Options options = CurrentOptions();
 185   Reopen(&options);
 186   ASSERT_OK(Put("key1", "v1"));
 187   ASSERT_OK(Put("key2", "v2"));
 188   ASSERT_OK(Put("key1", "v3"));  // Duplicate
 189   dbfull()->TEST_FlushMemTable();
 190
 191   TablePropertiesCollection ptc;
 192   reinterpret_cast<DB*>(dbfull())->GetPropertiesOfAllTables(&ptc);
 193   ASSERT_EQ(1U, ptc.size());
 194   ASSERT_EQ(2U, ptc.begin()->second->num_entries);
 195   ASSERT_EQ("1", FilesPerLevel());
 196   ASSERT_EQ("v3", Get("key1"));
 197   ASSERT_EQ("v2", Get("key2"));
 198 }
 199
 200 namespace {
 201 static std::string Key(int i) {
 202   char buf[100];
 203   snprintf(buf, sizeof(buf), "key_______%06d", i);
 204   return std::string(buf);
 205 }
 206 static std::string Uint64Key(uint64_t i) {
 207   std::string str;
 208   str.resize(8);
 209   memcpy(&str[0], static_cast<void*>(&i), 8);
 210   return str;
 211 }
 212 }  // namespace.
 213
 214 TEST_F(CuckooTableDBTest, Uint64Comparator) {
 215   Options options = CurrentOptions();
 216   options.comparator = test::Uint64Comparator();
 217   DestroyAndReopen(&options);
 218
 219   ASSERT_OK(Put(Uint64Key(1), "v1"));
 220   ASSERT_OK(Put(Uint64Key(2), "v2"));
 221   ASSERT_OK(Put(Uint64Key(3), "v3"));
 222   dbfull()->TEST_FlushMemTable();
 223
 224   ASSERT_EQ("v1", Get(Uint64Key(1)));
 225   ASSERT_EQ("v2", Get(Uint64Key(2)));
 226   ASSERT_EQ("v3", Get(Uint64Key(3)));
 227   ASSERT_EQ("NOT_FOUND", Get(Uint64Key(4)));
 228
 229   // Add more keys.
 230   ASSERT_OK(Delete(Uint64Key(2)));  // Delete.
 231   dbfull()->TEST_FlushMemTable();
 232   ASSERT_OK(Put(Uint64Key(3), "v0"));  // Update.
 233   ASSERT_OK(Put(Uint64Key(4), "v4"));
 234   dbfull()->TEST_FlushMemTable();
 235   ASSERT_EQ("v1", Get(Uint64Key(1)));
 236   ASSERT_EQ("NOT_FOUND", Get(Uint64Key(2)));
 237   ASSERT_EQ("v0", Get(Uint64Key(3)));
 238   ASSERT_EQ("v4", Get(Uint64Key(4)));
 239 }
 240
 241 TEST_F(CuckooTableDBTest, CompactionIntoMultipleFiles) {
 242   // Create a big L0 file and check it compacts into multiple files in L1.
 243   Options options = CurrentOptions();
 244   options.write_buffer_size = 270 << 10;
 245   // Two SST files should be created, each containing 14 keys.
 246   // Number of buckets will be 16. Total size ~156 KB.
 247   options.target_file_size_base = 160 << 10;
 248   Reopen(&options);
 249
 250   // Write 28 values, each 10016 B ~ 10KB
 251   for (int idx = 0; idx < 28; ++idx) {
 252     ASSERT_OK(Put(Key(idx), std::string(10000, 'a' + char(idx))));
 253   }
 254   dbfull()->TEST_WaitForFlushMemTable();
 255   ASSERT_EQ("1", FilesPerLevel());
 256
 257   dbfull()->TEST_CompactRange(0, nullptr, nullptr, nullptr,
 258                               true /* disallow trivial move */);
 259   ASSERT_EQ("0,2", FilesPerLevel());
 260   for (int idx = 0; idx < 28; ++idx) {
 261     ASSERT_EQ(std::string(10000, 'a' + char(idx)), Get(Key(idx)));
 262   }
 263 }
 264
 265 TEST_F(CuckooTableDBTest, SameKeyInsertedInTwoDifferentFilesAndCompacted) {
 266   // Insert same key twice so that they go to different SST files. Then wait for
 267   // compaction and check if the latest value is stored and old value removed.
 268   Options options = CurrentOptions();
 269   options.write_buffer_size = 100 << 10;  // 100KB
 270   options.level0_file_num_compaction_trigger = 2;
 271   Reopen(&options);
 272
 273   // Write 11 values, each 10016 B
 274   for (int idx = 0; idx < 11; ++idx) {
 275     ASSERT_OK(Put(Key(idx), std::string(10000, 'a')));
 276   }
 277   dbfull()->TEST_WaitForFlushMemTable();
 278   ASSERT_EQ("1", FilesPerLevel());
 279
 280   // Generate one more file in level-0, and should trigger level-0 compaction
 281   for (int idx = 0; idx < 11; ++idx) {
 282     ASSERT_OK(Put(Key(idx), std::string(10000, 'a' + char(idx))));
 283   }
 284   dbfull()->TEST_WaitForFlushMemTable();
 285   dbfull()->TEST_CompactRange(0, nullptr, nullptr);
 286
 287   ASSERT_EQ("0,1", FilesPerLevel());
 288   for (int idx = 0; idx < 11; ++idx) {
 289     ASSERT_EQ(std::string(10000, 'a' + char(idx)), Get(Key(idx)));
 290   }
 291 }
 292
 293 TEST_F(CuckooTableDBTest, AdaptiveTable) {
 294   Options options = CurrentOptions();
 295
 296   // Ensure options compatible with PlainTable
 297   options.prefix_extractor.reset(NewCappedPrefixTransform(8));
 298
 299   // Write some keys using cuckoo table.
 300   options.table_factory.reset(NewCuckooTableFactory());
 301   Reopen(&options);
 302
 303   ASSERT_OK(Put("key1", "v1"));
 304   ASSERT_OK(Put("key2", "v2"));
 305   ASSERT_OK(Put("key3", "v3"));
 306   dbfull()->TEST_FlushMemTable();
 307
 308   // Write some keys using plain table.
 309   std::shared_ptr<TableFactory> block_based_factory(
 310       NewBlockBasedTableFactory());
 311   std::shared_ptr<TableFactory> plain_table_factory(
 312       NewPlainTableFactory());
 313   std::shared_ptr<TableFactory> cuckoo_table_factory(
 314       NewCuckooTableFactory());
 315   options.create_if_missing = false;
 316   options.table_factory.reset(NewAdaptiveTableFactory(
 317     plain_table_factory, block_based_factory, plain_table_factory,
 318     cuckoo_table_factory));
 319   Reopen(&options);
 320   ASSERT_OK(Put("key4", "v4"));
 321   ASSERT_OK(Put("key1", "v5"));
 322   dbfull()->TEST_FlushMemTable();
 323
 324   // Write some keys using block based table.
 325   options.table_factory.reset(NewAdaptiveTableFactory(
 326     block_based_factory, block_based_factory, plain_table_factory,
 327     cuckoo_table_factory));
 328   Reopen(&options);
 329   ASSERT_OK(Put("key5", "v6"));
 330   ASSERT_OK(Put("key2", "v7"));
 331   dbfull()->TEST_FlushMemTable();
 332
 333   ASSERT_EQ("v5", Get("key1"));
 334   ASSERT_EQ("v7", Get("key2"));
 335   ASSERT_EQ("v3", Get("key3"));
 336   ASSERT_EQ("v4", Get("key4"));
 337   ASSERT_EQ("v6", Get("key5"));
 338 }
 339 }  // namespace ROCKSDB_NAMESPACE
 340
 341 int main(int argc, char** argv) {
 342   if (ROCKSDB_NAMESPACE::port::kLittleEndian) {
 343     ::testing::InitGoogleTest(&argc, argv);
 344     return RUN_ALL_TESTS();
 345   } else {
 346     fprintf(stderr, "SKIPPED as Cuckoo table doesn't support Big Endian\n");
 347     return 0;
 348   }
 349 }
 350
 351 #else
 352 #include <stdio.h>
 353
 354 int main(int /*argc*/, char** /*argv*/) {
 355   fprintf(stderr, "SKIPPED as Cuckoo table is not supported in ROCKSDB_LITE\n");
 356   return 0;
 357 }
 358
 359 #endif  // ROCKSDB_LITE