ceph/src/rocksdb/utilities/options/options_util_test.cc

   1 //  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
   2 //  This source code is licensed under both the GPLv2 (found in the
   3 //  COPYING file in the root directory) and Apache 2.0 License
   4 //  (found in the LICENSE.Apache file in the root directory).
   5
   6 #ifndef ROCKSDB_LITE
   7 #ifndef __STDC_FORMAT_MACROS
   8 #define __STDC_FORMAT_MACROS
   9 #endif
  10
  11 #include <inttypes.h>
  12
  13 #include <cctype>
  14 #include <unordered_map>
  15
  16 #include "options/options_parser.h"
  17 #include "rocksdb/db.h"
  18 #include "rocksdb/table.h"
  19 #include "rocksdb/utilities/options_util.h"
  20 #include "util/random.h"
  21 #include "util/testharness.h"
  22 #include "util/testutil.h"
  23
  24 #ifndef GFLAGS
  25 bool FLAGS_enable_print = false;
  26 #else
  27 #include "util/gflags_compat.h"
  28 using GFLAGS_NAMESPACE::ParseCommandLineFlags;
  29 DEFINE_bool(enable_print, false, "Print options generated to console.");
  30 #endif  // GFLAGS
  31
  32 namespace rocksdb {
  33 class OptionsUtilTest : public testing::Test {
  34  public:
  35   OptionsUtilTest() : rnd_(0xFB) {
  36     env_.reset(new test::StringEnv(Env::Default()));
  37     dbname_ = test::PerThreadDBPath("options_util_test");
  38   }
  39
  40  protected:
  41   std::unique_ptr<test::StringEnv> env_;
  42   std::string dbname_;
  43   Random rnd_;
  44 };
  45
  46 bool IsBlockBasedTableFactory(TableFactory* tf) {
  47   return tf->Name() == BlockBasedTableFactory().Name();
  48 }
  49
  50 TEST_F(OptionsUtilTest, SaveAndLoad) {
  51   const size_t kCFCount = 5;
  52
  53   DBOptions db_opt;
  54   std::vector<std::string> cf_names;
  55   std::vector<ColumnFamilyOptions> cf_opts;
  56   test::RandomInitDBOptions(&db_opt, &rnd_);
  57   for (size_t i = 0; i < kCFCount; ++i) {
  58     cf_names.push_back(i == 0 ? kDefaultColumnFamilyName
  59                               : test::RandomName(&rnd_, 10));
  60     cf_opts.emplace_back();
  61     test::RandomInitCFOptions(&cf_opts.back(), &rnd_);
  62   }
  63
  64   const std::string kFileName = "OPTIONS-123456";
  65   PersistRocksDBOptions(db_opt, cf_names, cf_opts, kFileName, env_.get());
  66
  67   DBOptions loaded_db_opt;
  68   std::vector<ColumnFamilyDescriptor> loaded_cf_descs;
  69   ASSERT_OK(LoadOptionsFromFile(kFileName, env_.get(), &loaded_db_opt,
  70                                 &loaded_cf_descs));
  71
  72   ASSERT_OK(RocksDBOptionsParser::VerifyDBOptions(db_opt, loaded_db_opt));
  73   test::RandomInitDBOptions(&db_opt, &rnd_);
  74   ASSERT_NOK(RocksDBOptionsParser::VerifyDBOptions(db_opt, loaded_db_opt));
  75
  76   for (size_t i = 0; i < kCFCount; ++i) {
  77     ASSERT_EQ(cf_names[i], loaded_cf_descs[i].name);
  78     ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(
  79         cf_opts[i], loaded_cf_descs[i].options));
  80     if (IsBlockBasedTableFactory(cf_opts[i].table_factory.get())) {
  81       ASSERT_OK(RocksDBOptionsParser::VerifyTableFactory(
  82           cf_opts[i].table_factory.get(),
  83           loaded_cf_descs[i].options.table_factory.get()));
  84     }
  85     test::RandomInitCFOptions(&cf_opts[i], &rnd_);
  86     ASSERT_NOK(RocksDBOptionsParser::VerifyCFOptions(
  87         cf_opts[i], loaded_cf_descs[i].options));
  88   }
  89
  90   for (size_t i = 0; i < kCFCount; ++i) {
  91     if (cf_opts[i].compaction_filter) {
  92       delete cf_opts[i].compaction_filter;
  93     }
  94   }
  95 }
  96
  97 TEST_F(OptionsUtilTest, SaveAndLoadWithCacheCheck) {
  98   // creating db
  99   DBOptions db_opt;
 100   db_opt.create_if_missing = true;
 101   // initialize BlockBasedTableOptions
 102   std::shared_ptr<Cache> cache = NewLRUCache(1 * 1024);
 103   BlockBasedTableOptions bbt_opts;
 104   bbt_opts.block_size = 32 * 1024;
 105   // saving cf options
 106   std::vector<ColumnFamilyOptions> cf_opts;
 107   ColumnFamilyOptions default_column_family_opt = ColumnFamilyOptions();
 108   default_column_family_opt.table_factory.reset(
 109       NewBlockBasedTableFactory(bbt_opts));
 110   cf_opts.push_back(default_column_family_opt);
 111
 112   ColumnFamilyOptions cf_opt_sample = ColumnFamilyOptions();
 113   cf_opt_sample.table_factory.reset(NewBlockBasedTableFactory(bbt_opts));
 114   cf_opts.push_back(cf_opt_sample);
 115
 116   ColumnFamilyOptions cf_opt_plain_table_opt = ColumnFamilyOptions();
 117   cf_opt_plain_table_opt.table_factory.reset(NewPlainTableFactory());
 118   cf_opts.push_back(cf_opt_plain_table_opt);
 119
 120   std::vector<std::string> cf_names;
 121   cf_names.push_back(kDefaultColumnFamilyName);
 122   cf_names.push_back("cf_sample");
 123   cf_names.push_back("cf_plain_table_sample");
 124   // Saving DB in file
 125   const std::string kFileName = "OPTIONS-LOAD_CACHE_123456";
 126   PersistRocksDBOptions(db_opt, cf_names, cf_opts, kFileName, env_.get());
 127   DBOptions loaded_db_opt;
 128   std::vector<ColumnFamilyDescriptor> loaded_cf_descs;
 129   ASSERT_OK(LoadOptionsFromFile(kFileName, env_.get(), &loaded_db_opt,
 130                                 &loaded_cf_descs, false, &cache));
 131   for (size_t i = 0; i < loaded_cf_descs.size(); i++) {
 132     if (IsBlockBasedTableFactory(cf_opts[i].table_factory.get())) {
 133       auto* loaded_bbt_opt = reinterpret_cast<BlockBasedTableOptions*>(
 134           loaded_cf_descs[i].options.table_factory->GetOptions());
 135       // Expect the same cache will be loaded
 136       if (loaded_bbt_opt != nullptr) {
 137         ASSERT_EQ(loaded_bbt_opt->block_cache.get(), cache.get());
 138       }
 139     }
 140   }
 141 }
 142
 143 namespace {
 144 class DummyTableFactory : public TableFactory {
 145  public:
 146   DummyTableFactory() {}
 147   ~DummyTableFactory() override {}
 148
 149   const char* Name() const override { return "DummyTableFactory"; }
 150
 151   Status NewTableReader(
 152       const TableReaderOptions& /*table_reader_options*/,
 153       std::unique_ptr<RandomAccessFileReader>&& /*file*/,
 154       uint64_t /*file_size*/, std::unique_ptr<TableReader>* /*table_reader*/,
 155       bool /*prefetch_index_and_filter_in_cache*/) const override {
 156     return Status::NotSupported();
 157   }
 158
 159   TableBuilder* NewTableBuilder(
 160       const TableBuilderOptions& /*table_builder_options*/,
 161       uint32_t /*column_family_id*/,
 162       WritableFileWriter* /*file*/) const override {
 163     return nullptr;
 164   }
 165
 166   Status SanitizeOptions(
 167       const DBOptions& /*db_opts*/,
 168       const ColumnFamilyOptions& /*cf_opts*/) const override {
 169     return Status::NotSupported();
 170   }
 171
 172   std::string GetPrintableTableOptions() const override { return ""; }
 173
 174   Status GetOptionString(std::string* /*opt_string*/,
 175                          const std::string& /*delimiter*/) const override {
 176     return Status::OK();
 177   }
 178 };
 179
 180 class DummyMergeOperator : public MergeOperator {
 181  public:
 182   DummyMergeOperator() {}
 183   ~DummyMergeOperator() override {}
 184
 185   bool FullMergeV2(const MergeOperationInput& /*merge_in*/,
 186                    MergeOperationOutput* /*merge_out*/) const override {
 187     return false;
 188   }
 189
 190   bool PartialMergeMulti(const Slice& /*key*/,
 191                          const std::deque<Slice>& /*operand_list*/,
 192                          std::string* /*new_value*/,
 193                          Logger* /*logger*/) const override {
 194     return false;
 195   }
 196
 197   const char* Name() const override { return "DummyMergeOperator"; }
 198 };
 199
 200 class DummySliceTransform : public SliceTransform {
 201  public:
 202   DummySliceTransform() {}
 203   ~DummySliceTransform() override {}
 204
 205   // Return the name of this transformation.
 206   const char* Name() const override { return "DummySliceTransform"; }
 207
 208   // transform a src in domain to a dst in the range
 209   Slice Transform(const Slice& src) const override { return src; }
 210
 211   // determine whether this is a valid src upon the function applies
 212   bool InDomain(const Slice& /*src*/) const override { return false; }
 213
 214   // determine whether dst=Transform(src) for some src
 215   bool InRange(const Slice& /*dst*/) const override { return false; }
 216 };
 217
 218 }  // namespace
 219
 220 TEST_F(OptionsUtilTest, SanityCheck) {
 221   DBOptions db_opt;
 222   std::vector<ColumnFamilyDescriptor> cf_descs;
 223   const size_t kCFCount = 5;
 224   for (size_t i = 0; i < kCFCount; ++i) {
 225     cf_descs.emplace_back();
 226     cf_descs.back().name =
 227         (i == 0) ? kDefaultColumnFamilyName : test::RandomName(&rnd_, 10);
 228
 229     cf_descs.back().options.table_factory.reset(NewBlockBasedTableFactory());
 230     // Assign non-null values to prefix_extractors except the first cf.
 231     cf_descs.back().options.prefix_extractor.reset(
 232         i != 0 ? test::RandomSliceTransform(&rnd_) : nullptr);
 233     cf_descs.back().options.merge_operator.reset(
 234         test::RandomMergeOperator(&rnd_));
 235   }
 236
 237   db_opt.create_missing_column_families = true;
 238   db_opt.create_if_missing = true;
 239
 240   DestroyDB(dbname_, Options(db_opt, cf_descs[0].options));
 241   DB* db;
 242   std::vector<ColumnFamilyHandle*> handles;
 243   // open and persist the options
 244   ASSERT_OK(DB::Open(db_opt, dbname_, cf_descs, &handles, &db));
 245
 246   // close the db
 247   for (auto* handle : handles) {
 248     delete handle;
 249   }
 250   delete db;
 251
 252   // perform sanity check
 253   ASSERT_OK(
 254       CheckOptionsCompatibility(dbname_, Env::Default(), db_opt, cf_descs));
 255
 256   ASSERT_GE(kCFCount, 5);
 257   // merge operator
 258   {
 259     std::shared_ptr<MergeOperator> merge_op =
 260         cf_descs[0].options.merge_operator;
 261
 262     ASSERT_NE(merge_op.get(), nullptr);
 263     cf_descs[0].options.merge_operator.reset();
 264     ASSERT_NOK(
 265         CheckOptionsCompatibility(dbname_, Env::Default(), db_opt, cf_descs));
 266
 267     cf_descs[0].options.merge_operator.reset(new DummyMergeOperator());
 268     ASSERT_NOK(
 269         CheckOptionsCompatibility(dbname_, Env::Default(), db_opt, cf_descs));
 270
 271     cf_descs[0].options.merge_operator = merge_op;
 272     ASSERT_OK(
 273         CheckOptionsCompatibility(dbname_, Env::Default(), db_opt, cf_descs));
 274   }
 275
 276   // prefix extractor
 277   {
 278     std::shared_ptr<const SliceTransform> prefix_extractor =
 279         cf_descs[1].options.prefix_extractor;
 280
 281     // It's okay to set prefix_extractor to nullptr.
 282     ASSERT_NE(prefix_extractor, nullptr);
 283     cf_descs[1].options.prefix_extractor.reset();
 284     ASSERT_OK(
 285         CheckOptionsCompatibility(dbname_, Env::Default(), db_opt, cf_descs));
 286
 287     cf_descs[1].options.prefix_extractor.reset(new DummySliceTransform());
 288     ASSERT_OK(
 289         CheckOptionsCompatibility(dbname_, Env::Default(), db_opt, cf_descs));
 290
 291     cf_descs[1].options.prefix_extractor = prefix_extractor;
 292     ASSERT_OK(
 293         CheckOptionsCompatibility(dbname_, Env::Default(), db_opt, cf_descs));
 294   }
 295
 296   // prefix extractor nullptr case
 297   {
 298     std::shared_ptr<const SliceTransform> prefix_extractor =
 299         cf_descs[0].options.prefix_extractor;
 300
 301     // It's okay to set prefix_extractor to nullptr.
 302     ASSERT_EQ(prefix_extractor, nullptr);
 303     cf_descs[0].options.prefix_extractor.reset();
 304     ASSERT_OK(
 305         CheckOptionsCompatibility(dbname_, Env::Default(), db_opt, cf_descs));
 306
 307     // It's okay to change prefix_extractor from nullptr to non-nullptr
 308     cf_descs[0].options.prefix_extractor.reset(new DummySliceTransform());
 309     ASSERT_OK(
 310         CheckOptionsCompatibility(dbname_, Env::Default(), db_opt, cf_descs));
 311
 312     cf_descs[0].options.prefix_extractor = prefix_extractor;
 313     ASSERT_OK(
 314         CheckOptionsCompatibility(dbname_, Env::Default(), db_opt, cf_descs));
 315   }
 316
 317   // comparator
 318   {
 319     test::SimpleSuffixReverseComparator comparator;
 320
 321     auto* prev_comparator = cf_descs[2].options.comparator;
 322     cf_descs[2].options.comparator = &comparator;
 323     ASSERT_NOK(
 324         CheckOptionsCompatibility(dbname_, Env::Default(), db_opt, cf_descs));
 325
 326     cf_descs[2].options.comparator = prev_comparator;
 327     ASSERT_OK(
 328         CheckOptionsCompatibility(dbname_, Env::Default(), db_opt, cf_descs));
 329   }
 330
 331   // table factory
 332   {
 333     std::shared_ptr<TableFactory> table_factory =
 334         cf_descs[3].options.table_factory;
 335
 336     ASSERT_NE(table_factory, nullptr);
 337     cf_descs[3].options.table_factory.reset(new DummyTableFactory());
 338     ASSERT_NOK(
 339         CheckOptionsCompatibility(dbname_, Env::Default(), db_opt, cf_descs));
 340
 341     cf_descs[3].options.table_factory = table_factory;
 342     ASSERT_OK(
 343         CheckOptionsCompatibility(dbname_, Env::Default(), db_opt, cf_descs));
 344   }
 345 }
 346
 347 }  // namespace rocksdb
 348
 349 int main(int argc, char** argv) {
 350   ::testing::InitGoogleTest(&argc, argv);
 351 #ifdef GFLAGS
 352   ParseCommandLineFlags(&argc, &argv, true);
 353 #endif  // GFLAGS
 354   return RUN_ALL_TESTS();
 355 }
 356
 357 #else
 358 #include <cstdio>
 359
 360 int main(int /*argc*/, char** /*argv*/) {
 361   printf("Skipped in RocksDBLite as utilities are not supported.\n");
 362   return 0;
 363 }
 364 #endif  // !ROCKSDB_LITE