ceph/src/rocksdb/tools/db_bench_tool_test.cc

   1 //  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
   2 //  This source code is licensed under both the GPLv2 (found in the
   3 //  COPYING file in the root directory) and Apache 2.0 License
   4 //  (found in the LICENSE.Apache file in the root directory).
   5 //
   6 // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
   7 // Use of this source code is governed by a BSD-style license that can be
   8 // found in the LICENSE file. See the AUTHORS file for names of contributors.
   9
  10 #include "rocksdb/db_bench_tool.h"
  11
  12 #include "db/db_impl/db_impl.h"
  13 #include "options/options_parser.h"
  14 #include "rocksdb/utilities/options_util.h"
  15 #include "test_util/testharness.h"
  16 #include "test_util/testutil.h"
  17 #include "util/random.h"
  18
  19 #ifdef GFLAGS
  20 #include "util/gflags_compat.h"
  21
  22 namespace ROCKSDB_NAMESPACE {
  23 namespace {
  24 static const int kMaxArgCount = 100;
  25 static const size_t kArgBufferSize = 100000;
  26 }  // namespace
  27
  28 class DBBenchTest : public testing::Test {
  29  public:
  30   DBBenchTest() : rnd_(0xFB) {
  31     test_path_ = test::PerThreadDBPath("db_bench_test");
  32     Env::Default()->CreateDir(test_path_);
  33     db_path_ = test_path_ + "/db";
  34     wal_path_ = test_path_ + "/wal";
  35   }
  36
  37   ~DBBenchTest() {
  38     //  DestroyDB(db_path_, Options());
  39   }
  40
  41   void ResetArgs() {
  42     argc_ = 0;
  43     cursor_ = 0;
  44     memset(arg_buffer_, 0, kArgBufferSize);
  45   }
  46
  47   void AppendArgs(const std::vector<std::string>& args) {
  48     for (const auto& arg : args) {
  49       ASSERT_LE(cursor_ + arg.size() + 1, kArgBufferSize);
  50       ASSERT_LE(argc_ + 1, kMaxArgCount);
  51       snprintf(arg_buffer_ + cursor_, arg.size() + 1, "%s", arg.c_str());
  52
  53       argv_[argc_++] = arg_buffer_ + cursor_;
  54       cursor_ += arg.size() + 1;
  55     }
  56   }
  57
  58   // Gets the default options for this test/db_bench.
  59   // Note that db_bench may change some of the default option values and that
  60   // the database might as well.  The options changed by db_bench are
  61   // specified here; the ones by the DB are set via SanitizeOptions
  62   Options GetDefaultOptions(CompactionStyle style = kCompactionStyleLevel,
  63                             int levels = 7) const {
  64     Options opt;
  65
  66     opt.create_if_missing = true;
  67     opt.max_open_files = 256;
  68     opt.max_background_compactions = 10;
  69     opt.dump_malloc_stats = true;  // db_bench uses a different default
  70     opt.compaction_style = style;
  71     opt.num_levels = levels;
  72     opt.compression = kNoCompression;
  73     opt.arena_block_size = 8388608;
  74
  75     return SanitizeOptions(db_path_, opt);
  76   }
  77
  78   void RunDbBench(const std::string& options_file_name) {
  79     AppendArgs({"./db_bench", "--benchmarks=fillseq", "--use_existing_db=0",
  80                 "--num=1000", "--compression_type=none",
  81                 std::string(std::string("--db=") + db_path_).c_str(),
  82                 std::string(std::string("--wal_dir=") + wal_path_).c_str(),
  83                 std::string(std::string("--options_file=") + options_file_name)
  84                     .c_str()});
  85     ASSERT_EQ(0, db_bench_tool(argc(), argv()));
  86   }
  87
  88   void VerifyOptions(const Options& opt) {
  89     DBOptions loaded_db_opts;
  90     std::vector<ColumnFamilyDescriptor> cf_descs;
  91     ASSERT_OK(LoadLatestOptions(db_path_, Env::Default(), &loaded_db_opts,
  92                                 &cf_descs));
  93
  94     ConfigOptions exact;
  95     exact.input_strings_escaped = false;
  96     exact.sanity_level = ConfigOptions::kSanityLevelExactMatch;
  97     ASSERT_OK(RocksDBOptionsParser::VerifyDBOptions(exact, DBOptions(opt),
  98                                                     loaded_db_opts));
  99     ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(
 100         exact, ColumnFamilyOptions(opt), cf_descs[0].options));
 101
 102     // check with the default rocksdb options and expect failure
 103     ASSERT_NOK(RocksDBOptionsParser::VerifyDBOptions(exact, DBOptions(),
 104                                                      loaded_db_opts));
 105     ASSERT_NOK(RocksDBOptionsParser::VerifyCFOptions(
 106         exact, ColumnFamilyOptions(), cf_descs[0].options));
 107   }
 108
 109   char** argv() { return argv_; }
 110
 111   int argc() { return argc_; }
 112
 113   std::string db_path_;
 114   std::string test_path_;
 115   std::string wal_path_;
 116
 117   char arg_buffer_[kArgBufferSize];
 118   char* argv_[kMaxArgCount];
 119   int argc_ = 0;
 120   int cursor_ = 0;
 121   Random rnd_;
 122 };
 123
 124 namespace {}  // namespace
 125
 126 TEST_F(DBBenchTest, OptionsFile) {
 127   const std::string kOptionsFileName = test_path_ + "/OPTIONS_test";
 128   Options opt = GetDefaultOptions();
 129   ASSERT_OK(PersistRocksDBOptions(DBOptions(opt), {"default"},
 130                                   {ColumnFamilyOptions(opt)}, kOptionsFileName,
 131                                   opt.env->GetFileSystem().get()));
 132
 133   // override the following options as db_bench will not take these
 134   // options from the options file
 135   opt.wal_dir = wal_path_;
 136
 137   RunDbBench(kOptionsFileName);
 138   opt.delayed_write_rate = 16 * 1024 * 1024;  // Set by SanitizeOptions
 139
 140   VerifyOptions(opt);
 141 }
 142
 143 TEST_F(DBBenchTest, OptionsFileUniversal) {
 144   const std::string kOptionsFileName = test_path_ + "/OPTIONS_test";
 145
 146   Options opt = GetDefaultOptions(kCompactionStyleUniversal, 1);
 147
 148   ASSERT_OK(PersistRocksDBOptions(DBOptions(opt), {"default"},
 149                                   {ColumnFamilyOptions(opt)}, kOptionsFileName,
 150                                   opt.env->GetFileSystem().get()));
 151
 152   // override the following options as db_bench will not take these
 153   // options from the options file
 154   opt.wal_dir = wal_path_;
 155   RunDbBench(kOptionsFileName);
 156
 157   VerifyOptions(opt);
 158 }
 159
 160 TEST_F(DBBenchTest, OptionsFileMultiLevelUniversal) {
 161   const std::string kOptionsFileName = test_path_ + "/OPTIONS_test";
 162
 163   Options opt = GetDefaultOptions(kCompactionStyleUniversal, 12);
 164
 165   ASSERT_OK(PersistRocksDBOptions(DBOptions(opt), {"default"},
 166                                   {ColumnFamilyOptions(opt)}, kOptionsFileName,
 167                                   opt.env->GetFileSystem().get()));
 168
 169   // override the following options as db_bench will not take these
 170   // options from the options file
 171   opt.wal_dir = wal_path_;
 172
 173   RunDbBench(kOptionsFileName);
 174   VerifyOptions(opt);
 175 }
 176
 177 const std::string options_file_content = R"OPTIONS_FILE(
 178 [Version]
 179   rocksdb_version=4.3.1
 180   options_file_version=1.1
 181
 182 [DBOptions]
 183   wal_bytes_per_sync=1048576
 184   delete_obsolete_files_period_micros=0
 185   WAL_ttl_seconds=0
 186   WAL_size_limit_MB=0
 187   db_write_buffer_size=0
 188   max_subcompactions=1
 189   table_cache_numshardbits=4
 190   max_open_files=-1
 191   max_file_opening_threads=10
 192   max_background_compactions=5
 193   use_fsync=false
 194   use_adaptive_mutex=false
 195   max_total_wal_size=18446744073709551615
 196   compaction_readahead_size=0
 197   keep_log_file_num=10
 198   skip_stats_update_on_db_open=false
 199   max_manifest_file_size=18446744073709551615
 200   db_log_dir=
 201   writable_file_max_buffer_size=1048576
 202   paranoid_checks=true
 203   is_fd_close_on_exec=true
 204   bytes_per_sync=1048576
 205   enable_thread_tracking=true
 206   recycle_log_file_num=0
 207   create_missing_column_families=false
 208   log_file_time_to_roll=0
 209   max_background_flushes=1
 210   create_if_missing=true
 211   error_if_exists=false
 212   delayed_write_rate=1048576
 213   manifest_preallocation_size=4194304
 214   allow_mmap_reads=false
 215   allow_mmap_writes=false
 216   use_direct_reads=false
 217   use_direct_io_for_flush_and_compaction=false
 218   stats_dump_period_sec=600
 219   allow_fallocate=true
 220   max_log_file_size=83886080
 221   random_access_max_buffer_size=1048576
 222   advise_random_on_open=true
 223   dump_malloc_stats=true
 224
 225 [CFOptions "default"]
 226   compaction_filter_factory=nullptr
 227   table_factory=BlockBasedTable
 228   prefix_extractor=nullptr
 229   comparator=leveldb.BytewiseComparator
 230   compression_per_level=
 231   max_bytes_for_level_base=104857600
 232   bloom_locality=0
 233   target_file_size_base=10485760
 234   memtable_huge_page_size=0
 235   max_successive_merges=1000
 236   max_sequential_skip_in_iterations=8
 237   arena_block_size=52428800
 238   target_file_size_multiplier=1
 239   source_compaction_factor=1
 240   min_write_buffer_number_to_merge=1
 241   max_write_buffer_number=2
 242   write_buffer_size=419430400
 243   max_grandparent_overlap_factor=10
 244   max_bytes_for_level_multiplier=10
 245   memtable_factory=SkipListFactory
 246   compression=kNoCompression
 247   min_partial_merge_operands=2
 248   level0_stop_writes_trigger=100
 249   num_levels=1
 250   level0_slowdown_writes_trigger=50
 251   level0_file_num_compaction_trigger=10
 252   expanded_compaction_factor=25
 253   max_write_buffer_number_to_maintain=0
 254   max_write_buffer_size_to_maintain=0
 255   verify_checksums_in_compaction=true
 256   merge_operator=nullptr
 257   memtable_prefix_bloom_bits=0
 258   memtable_whole_key_filtering=true
 259   paranoid_file_checks=false
 260   inplace_update_num_locks=10000
 261   optimize_filters_for_hits=false
 262   level_compaction_dynamic_level_bytes=false
 263   inplace_update_support=false
 264   compaction_style=kCompactionStyleUniversal
 265   memtable_prefix_bloom_probes=6
 266   filter_deletes=false
 267   hard_pending_compaction_bytes_limit=0
 268   disable_auto_compactions=false
 269   compaction_measure_io_stats=false
 270   enable_blob_files=true
 271   min_blob_size=16
 272   blob_file_size=10485760
 273   blob_compression_type=kNoCompression
 274   enable_blob_garbage_collection=true
 275   blob_garbage_collection_age_cutoff=0.5
 276   blob_garbage_collection_force_threshold=0.75
 277   blob_compaction_readahead_size=262144
 278   blob_file_starting_level=0
 279   prepopulate_blob_cache=kDisable;
 280
 281 [TableOptions/BlockBasedTable "default"]
 282   format_version=0
 283   skip_table_builder_flush=false
 284   cache_index_and_filter_blocks=false
 285   flush_block_policy_factory=FlushBlockBySizePolicyFactory
 286   index_type=kBinarySearch
 287   whole_key_filtering=true
 288   checksum=kCRC32c
 289   no_block_cache=false
 290   block_size=32768
 291   block_size_deviation=10
 292   block_restart_interval=16
 293   filter_policy=rocksdb.BuiltinBloomFilter
 294 )OPTIONS_FILE";
 295
 296 TEST_F(DBBenchTest, OptionsFileFromFile) {
 297   const std::string kOptionsFileName = test_path_ + "/OPTIONS_flash";
 298   std::unique_ptr<WritableFile> writable;
 299   ASSERT_OK(Env::Default()->NewWritableFile(kOptionsFileName, &writable,
 300                                             EnvOptions()));
 301   ASSERT_OK(writable->Append(options_file_content));
 302   ASSERT_OK(writable->Close());
 303
 304   DBOptions db_opt;
 305   std::vector<ColumnFamilyDescriptor> cf_descs;
 306   ASSERT_OK(LoadOptionsFromFile(kOptionsFileName, Env::Default(), &db_opt,
 307                                 &cf_descs));
 308   Options opt(db_opt, cf_descs[0].options);
 309   opt.create_if_missing = true;
 310
 311   // override the following options as db_bench will not take these
 312   // options from the options file
 313   opt.wal_dir = wal_path_;
 314
 315   RunDbBench(kOptionsFileName);
 316
 317   VerifyOptions(SanitizeOptions(db_path_, opt));
 318 }
 319
 320 }  // namespace ROCKSDB_NAMESPACE
 321
 322 int main(int argc, char** argv) {
 323   ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
 324   ::testing::InitGoogleTest(&argc, argv);
 325   GFLAGS_NAMESPACE::ParseCommandLineFlags(&argc, &argv, true);
 326   return RUN_ALL_TESTS();
 327 }
 328
 329 #else
 330
 331 int main(int argc, char** argv) {
 332   printf("Skip db_bench_tool_test as the required library GFLAG is missing.");
 333 }
 334 #endif  // #ifdef GFLAGS