ceph/src/rocksdb/tools/db_bench_tool_test.cc

   1 //  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
   2 //  This source code is licensed under both the GPLv2 (found in the
   3 //  COPYING file in the root directory) and Apache 2.0 License
   4 //  (found in the LICENSE.Apache file in the root directory).
   5 //
   6 // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
   7 // Use of this source code is governed by a BSD-style license that can be
   8 // found in the LICENSE file. See the AUTHORS file for names of contributors.
   9
  10 #include "rocksdb/db_bench_tool.h"
  11 #include "options/options_parser.h"
  12 #include "rocksdb/utilities/options_util.h"
  13 #include "test_util/testharness.h"
  14 #include "test_util/testutil.h"
  15 #include "util/random.h"
  16
  17 #ifdef GFLAGS
  18 #include "util/gflags_compat.h"
  19
  20 namespace ROCKSDB_NAMESPACE {
  21 namespace {
  22 static const int kMaxArgCount = 100;
  23 static const size_t kArgBufferSize = 100000;
  24 }  // namespace
  25
  26 class DBBenchTest : public testing::Test {
  27  public:
  28   DBBenchTest() : rnd_(0xFB) {
  29     test_path_ = test::PerThreadDBPath("db_bench_test");
  30     Env::Default()->CreateDir(test_path_);
  31     db_path_ = test_path_ + "/db";
  32     wal_path_ = test_path_ + "/wal";
  33   }
  34
  35   ~DBBenchTest() {
  36     //  DestroyDB(db_path_, Options());
  37   }
  38
  39   void ResetArgs() {
  40     argc_ = 0;
  41     cursor_ = 0;
  42     memset(arg_buffer_, 0, kArgBufferSize);
  43   }
  44
  45   void AppendArgs(const std::vector<std::string>& args) {
  46     for (const auto& arg : args) {
  47       ASSERT_LE(cursor_ + arg.size() + 1, kArgBufferSize);
  48       ASSERT_LE(argc_ + 1, kMaxArgCount);
  49       snprintf(arg_buffer_ + cursor_, arg.size() + 1, "%s", arg.c_str());
  50
  51       argv_[argc_++] = arg_buffer_ + cursor_;
  52       cursor_ += arg.size() + 1;
  53     }
  54   }
  55
  56   void RunDbBench(const std::string& options_file_name) {
  57     AppendArgs({"./db_bench", "--benchmarks=fillseq", "--use_existing_db=0",
  58                 "--num=1000",
  59                 std::string(std::string("--db=") + db_path_).c_str(),
  60                 std::string(std::string("--wal_dir=") + wal_path_).c_str(),
  61                 std::string(std::string("--options_file=") + options_file_name)
  62                     .c_str()});
  63     ASSERT_EQ(0, db_bench_tool(argc(), argv()));
  64   }
  65
  66   void VerifyOptions(const Options& opt) {
  67     DBOptions loaded_db_opts;
  68     std::vector<ColumnFamilyDescriptor> cf_descs;
  69     ASSERT_OK(LoadLatestOptions(db_path_, FileSystem::Default(),
  70                                 &loaded_db_opts, &cf_descs));
  71
  72     ASSERT_OK(
  73         RocksDBOptionsParser::VerifyDBOptions(DBOptions(opt), loaded_db_opts));
  74     ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(ColumnFamilyOptions(opt),
  75                                                     cf_descs[0].options));
  76
  77     // check with the default rocksdb options and expect failure
  78     ASSERT_NOK(
  79         RocksDBOptionsParser::VerifyDBOptions(DBOptions(), loaded_db_opts));
  80     ASSERT_NOK(RocksDBOptionsParser::VerifyCFOptions(ColumnFamilyOptions(),
  81                                                      cf_descs[0].options));
  82   }
  83
  84   char** argv() { return argv_; }
  85
  86   int argc() { return argc_; }
  87
  88   std::string db_path_;
  89   std::string test_path_;
  90   std::string wal_path_;
  91
  92   char arg_buffer_[kArgBufferSize];
  93   char* argv_[kMaxArgCount];
  94   int argc_ = 0;
  95   int cursor_ = 0;
  96   Random rnd_;
  97 };
  98
  99 namespace {}  // namespace
 100
 101 TEST_F(DBBenchTest, OptionsFile) {
 102   const std::string kOptionsFileName = test_path_ + "/OPTIONS_test";
 103
 104   Options opt;
 105   opt.create_if_missing = true;
 106   opt.max_open_files = 256;
 107   opt.max_background_compactions = 10;
 108   opt.arena_block_size = 8388608;
 109   ASSERT_OK(PersistRocksDBOptions(DBOptions(opt), {"default"},
 110                                   {ColumnFamilyOptions(opt)}, kOptionsFileName,
 111                                   Env::Default()));
 112
 113   // override the following options as db_bench will not take these
 114   // options from the options file
 115   opt.wal_dir = wal_path_;
 116
 117   RunDbBench(kOptionsFileName);
 118
 119   VerifyOptions(opt);
 120 }
 121
 122 TEST_F(DBBenchTest, OptionsFileUniversal) {
 123   const std::string kOptionsFileName = test_path_ + "/OPTIONS_test";
 124
 125   Options opt;
 126   opt.compaction_style = kCompactionStyleUniversal;
 127   opt.num_levels = 1;
 128   opt.create_if_missing = true;
 129   opt.max_open_files = 256;
 130   opt.max_background_compactions = 10;
 131   opt.arena_block_size = 8388608;
 132   ASSERT_OK(PersistRocksDBOptions(DBOptions(opt), {"default"},
 133                                   {ColumnFamilyOptions(opt)}, kOptionsFileName,
 134                                   Env::Default()));
 135
 136   // override the following options as db_bench will not take these
 137   // options from the options file
 138   opt.wal_dir = wal_path_;
 139
 140   RunDbBench(kOptionsFileName);
 141
 142   VerifyOptions(opt);
 143 }
 144
 145 TEST_F(DBBenchTest, OptionsFileMultiLevelUniversal) {
 146   const std::string kOptionsFileName = test_path_ + "/OPTIONS_test";
 147
 148   Options opt;
 149   opt.compaction_style = kCompactionStyleUniversal;
 150   opt.num_levels = 12;
 151   opt.create_if_missing = true;
 152   opt.max_open_files = 256;
 153   opt.max_background_compactions = 10;
 154   opt.arena_block_size = 8388608;
 155   ASSERT_OK(PersistRocksDBOptions(DBOptions(opt), {"default"},
 156                                   {ColumnFamilyOptions(opt)}, kOptionsFileName,
 157                                   Env::Default()));
 158
 159   // override the following options as db_bench will not take these
 160   // options from the options file
 161   opt.wal_dir = wal_path_;
 162
 163   RunDbBench(kOptionsFileName);
 164
 165   VerifyOptions(opt);
 166 }
 167
 168 const std::string options_file_content = R"OPTIONS_FILE(
 169 [Version]
 170   rocksdb_version=4.3.1
 171   options_file_version=1.1
 172
 173 [DBOptions]
 174   wal_bytes_per_sync=1048576
 175   delete_obsolete_files_period_micros=0
 176   WAL_ttl_seconds=0
 177   WAL_size_limit_MB=0
 178   db_write_buffer_size=0
 179   max_subcompactions=1
 180   table_cache_numshardbits=4
 181   max_open_files=-1
 182   max_file_opening_threads=10
 183   max_background_compactions=5
 184   use_fsync=false
 185   use_adaptive_mutex=false
 186   max_total_wal_size=18446744073709551615
 187   compaction_readahead_size=0
 188   new_table_reader_for_compaction_inputs=false
 189   keep_log_file_num=10
 190   skip_stats_update_on_db_open=false
 191   max_manifest_file_size=18446744073709551615
 192   db_log_dir=
 193   skip_log_error_on_recovery=false
 194   writable_file_max_buffer_size=1048576
 195   paranoid_checks=true
 196   is_fd_close_on_exec=true
 197   bytes_per_sync=1048576
 198   enable_thread_tracking=true
 199   recycle_log_file_num=0
 200   create_missing_column_families=false
 201   log_file_time_to_roll=0
 202   max_background_flushes=1
 203   create_if_missing=true
 204   error_if_exists=false
 205   delayed_write_rate=1048576
 206   manifest_preallocation_size=4194304
 207   allow_mmap_reads=false
 208   allow_mmap_writes=false
 209   use_direct_reads=false
 210   use_direct_io_for_flush_and_compaction=false
 211   stats_dump_period_sec=600
 212   allow_fallocate=true
 213   max_log_file_size=83886080
 214   random_access_max_buffer_size=1048576
 215   advise_random_on_open=true
 216
 217
 218 [CFOptions "default"]
 219   compaction_filter_factory=nullptr
 220   table_factory=BlockBasedTable
 221   prefix_extractor=nullptr
 222   comparator=leveldb.BytewiseComparator
 223   compression_per_level=
 224   max_bytes_for_level_base=104857600
 225   bloom_locality=0
 226   target_file_size_base=10485760
 227   memtable_huge_page_size=0
 228   max_successive_merges=1000
 229   max_sequential_skip_in_iterations=8
 230   arena_block_size=52428800
 231   target_file_size_multiplier=1
 232   source_compaction_factor=1
 233   min_write_buffer_number_to_merge=1
 234   max_write_buffer_number=2
 235   write_buffer_size=419430400
 236   max_grandparent_overlap_factor=10
 237   max_bytes_for_level_multiplier=10
 238   memtable_factory=SkipListFactory
 239   compression=kSnappyCompression
 240   min_partial_merge_operands=2
 241   level0_stop_writes_trigger=100
 242   num_levels=1
 243   level0_slowdown_writes_trigger=50
 244   level0_file_num_compaction_trigger=10
 245   expanded_compaction_factor=25
 246   soft_rate_limit=0.000000
 247   max_write_buffer_number_to_maintain=0
 248   max_write_buffer_size_to_maintain=0
 249   verify_checksums_in_compaction=true
 250   merge_operator=nullptr
 251   memtable_prefix_bloom_bits=0
 252   memtable_whole_key_filtering=true
 253   paranoid_file_checks=false
 254   inplace_update_num_locks=10000
 255   optimize_filters_for_hits=false
 256   level_compaction_dynamic_level_bytes=false
 257   inplace_update_support=false
 258   compaction_style=kCompactionStyleUniversal
 259   memtable_prefix_bloom_probes=6
 260   purge_redundant_kvs_while_flush=true
 261   filter_deletes=false
 262   hard_pending_compaction_bytes_limit=0
 263   disable_auto_compactions=false
 264   compaction_measure_io_stats=false
 265
 266 [TableOptions/BlockBasedTable "default"]
 267   format_version=0
 268   skip_table_builder_flush=false
 269   cache_index_and_filter_blocks=false
 270   flush_block_policy_factory=FlushBlockBySizePolicyFactory
 271   hash_index_allow_collision=true
 272   index_type=kBinarySearch
 273   whole_key_filtering=true
 274   checksum=kCRC32c
 275   no_block_cache=false
 276   block_size=32768
 277   block_size_deviation=10
 278   block_restart_interval=16
 279   filter_policy=rocksdb.BuiltinBloomFilter
 280 )OPTIONS_FILE";
 281
 282 TEST_F(DBBenchTest, OptionsFileFromFile) {
 283   const std::string kOptionsFileName = test_path_ + "/OPTIONS_flash";
 284   std::unique_ptr<WritableFile> writable;
 285   ASSERT_OK(Env::Default()->NewWritableFile(kOptionsFileName, &writable,
 286                                             EnvOptions()));
 287   ASSERT_OK(writable->Append(options_file_content));
 288   ASSERT_OK(writable->Close());
 289
 290   DBOptions db_opt;
 291   std::vector<ColumnFamilyDescriptor> cf_descs;
 292   ASSERT_OK(LoadOptionsFromFile(kOptionsFileName, Env::Default(), &db_opt,
 293                                 &cf_descs));
 294   Options opt(db_opt, cf_descs[0].options);
 295
 296   opt.create_if_missing = true;
 297
 298   // override the following options as db_bench will not take these
 299   // options from the options file
 300   opt.wal_dir = wal_path_;
 301
 302   RunDbBench(kOptionsFileName);
 303
 304   VerifyOptions(opt);
 305 }
 306
 307 }  // namespace ROCKSDB_NAMESPACE
 308
 309 int main(int argc, char** argv) {
 310   ::testing::InitGoogleTest(&argc, argv);
 311   google::ParseCommandLineFlags(&argc, &argv, true);
 312   return RUN_ALL_TESTS();
 313 }
 314
 315 #else
 316
 317 int main(int argc, char** argv) {
 318   printf("Skip db_bench_tool_test as the required library GFLAG is missing.");
 319 }
 320 #endif  // #ifdef GFLAGS