ceph/src/rocksdb/options/options_settable_test.cc

   1 //  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
   2 //  This source code is licensed under both the GPLv2 (found in the
   3 //  COPYING file in the root directory) and Apache 2.0 License
   4 //  (found in the LICENSE.Apache file in the root directory).
   5 //
   6 // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
   7 // Use of this source code is governed by a BSD-style license that can be
   8 // found in the LICENSE file. See the AUTHORS file for names of contributors.
   9
  10 #ifndef __STDC_FORMAT_MACROS
  11 #define __STDC_FORMAT_MACROS
  12 #endif
  13
  14 #include <cstring>
  15
  16 #include "options/options_helper.h"
  17 #include "rocksdb/convenience.h"
  18 #include "util/testharness.h"
  19
  20 #ifndef GFLAGS
  21 bool FLAGS_enable_print = false;
  22 #else
  23 #include "util/gflags_compat.h"
  24 using GFLAGS_NAMESPACE::ParseCommandLineFlags;
  25 DEFINE_bool(enable_print, false, "Print options generated to console.");
  26 #endif  // GFLAGS
  27
  28 namespace rocksdb {
  29
  30 // Verify options are settable from options strings.
  31 // We take the approach that depends on compiler behavior that copy constructor
  32 // won't touch implicit padding bytes, so that the test is fragile.
  33 // As a result, we only run the tests to verify new fields in options are
  34 // settable through string on limited platforms as it depends on behavior of
  35 // compilers.
  36 #ifndef ROCKSDB_LITE
  37 #if defined OS_LINUX || defined OS_WIN
  38 #ifndef __clang__
  39
  40 class OptionsSettableTest : public testing::Test {
  41  public:
  42   OptionsSettableTest() {}
  43 };
  44
  45 const char kSpecialChar = 'z';
  46 typedef std::vector<std::pair<size_t, size_t>> OffsetGap;
  47
  48 void FillWithSpecialChar(char* start_ptr, size_t total_size,
  49                          const OffsetGap& blacklist) {
  50   size_t offset = 0;
  51   for (auto& pair : blacklist) {
  52     std::memset(start_ptr + offset, kSpecialChar, pair.first - offset);
  53     offset = pair.first + pair.second;
  54   }
  55   std::memset(start_ptr + offset, kSpecialChar, total_size - offset);
  56 }
  57
  58 int NumUnsetBytes(char* start_ptr, size_t total_size,
  59                   const OffsetGap& blacklist) {
  60   int total_unset_bytes_base = 0;
  61   size_t offset = 0;
  62   for (auto& pair : blacklist) {
  63     for (char* ptr = start_ptr + offset; ptr < start_ptr + pair.first; ptr++) {
  64       if (*ptr == kSpecialChar) {
  65         total_unset_bytes_base++;
  66       }
  67     }
  68     offset = pair.first + pair.second;
  69   }
  70   for (char* ptr = start_ptr + offset; ptr < start_ptr + total_size; ptr++) {
  71     if (*ptr == kSpecialChar) {
  72       total_unset_bytes_base++;
  73     }
  74   }
  75   return total_unset_bytes_base;
  76 }
  77
  78 // If the test fails, likely a new option is added to BlockBasedTableOptions
  79 // but it cannot be set through GetBlockBasedTableOptionsFromString(), or the
  80 // test is not updated accordingly.
  81 // After adding an option, we need to make sure it is settable by
  82 // GetBlockBasedTableOptionsFromString() and add the option to the input string
  83 // passed to the GetBlockBasedTableOptionsFromString() in this test.
  84 // If it is a complicated type, you also need to add the field to
  85 // kBbtoBlacklist, and maybe add customized verification for it.
  86 TEST_F(OptionsSettableTest, BlockBasedTableOptionsAllFieldsSettable) {
  87   // Items in the form of <offset, size>. Need to be in ascending order
  88   // and not overlapping. Need to updated if new pointer-option is added.
  89   const OffsetGap kBbtoBlacklist = {
  90       {offsetof(struct BlockBasedTableOptions, flush_block_policy_factory),
  91        sizeof(std::shared_ptr<FlushBlockPolicyFactory>)},
  92       {offsetof(struct BlockBasedTableOptions, block_cache),
  93        sizeof(std::shared_ptr<Cache>)},
  94       {offsetof(struct BlockBasedTableOptions, persistent_cache),
  95        sizeof(std::shared_ptr<PersistentCache>)},
  96       {offsetof(struct BlockBasedTableOptions, block_cache_compressed),
  97        sizeof(std::shared_ptr<Cache>)},
  98       {offsetof(struct BlockBasedTableOptions, filter_policy),
  99        sizeof(std::shared_ptr<const FilterPolicy>)},
 100   };
 101
 102   // In this test, we catch a new option of BlockBasedTableOptions that is not
 103   // settable through GetBlockBasedTableOptionsFromString().
 104   // We count padding bytes of the option struct, and assert it to be the same
 105   // as unset bytes of an option struct initialized by
 106   // GetBlockBasedTableOptionsFromString().
 107
 108   char* bbto_ptr = new char[sizeof(BlockBasedTableOptions)];
 109
 110   // Count padding bytes by setting all bytes in the memory to a special char,
 111   // copy a well constructed struct to this memory and see how many special
 112   // bytes left.
 113   BlockBasedTableOptions* bbto = new (bbto_ptr) BlockBasedTableOptions();
 114   FillWithSpecialChar(bbto_ptr, sizeof(BlockBasedTableOptions), kBbtoBlacklist);
 115   // It based on the behavior of compiler that padding bytes are not changed
 116   // when copying the struct. It's prone to failure when compiler behavior
 117   // changes. We verify there is unset bytes to detect the case.
 118   *bbto = BlockBasedTableOptions();
 119   int unset_bytes_base =
 120       NumUnsetBytes(bbto_ptr, sizeof(BlockBasedTableOptions), kBbtoBlacklist);
 121   ASSERT_GT(unset_bytes_base, 0);
 122   bbto->~BlockBasedTableOptions();
 123
 124   // Construct the base option passed into
 125   // GetBlockBasedTableOptionsFromString().
 126   bbto = new (bbto_ptr) BlockBasedTableOptions();
 127   FillWithSpecialChar(bbto_ptr, sizeof(BlockBasedTableOptions), kBbtoBlacklist);
 128   // This option is not setable:
 129   bbto->use_delta_encoding = true;
 130
 131   char* new_bbto_ptr = new char[sizeof(BlockBasedTableOptions)];
 132   BlockBasedTableOptions* new_bbto =
 133       new (new_bbto_ptr) BlockBasedTableOptions();
 134   FillWithSpecialChar(new_bbto_ptr, sizeof(BlockBasedTableOptions),
 135                       kBbtoBlacklist);
 136
 137   // Need to update the option string if a new option is added.
 138   ASSERT_OK(GetBlockBasedTableOptionsFromString(
 139       *bbto,
 140       "cache_index_and_filter_blocks=1;"
 141       "cache_index_and_filter_blocks_with_high_priority=true;"
 142       "pin_l0_filter_and_index_blocks_in_cache=1;"
 143       "pin_top_level_index_and_filter=1;"
 144       "index_type=kHashSearch;"
 145       "data_block_index_type=kDataBlockBinaryAndHash;"
 146       "data_block_hash_table_util_ratio=0.75;"
 147       "checksum=kxxHash;hash_index_allow_collision=1;no_block_cache=1;"
 148       "block_cache=1M;block_cache_compressed=1k;block_size=1024;"
 149       "block_size_deviation=8;block_restart_interval=4; "
 150       "metadata_block_size=1024;"
 151       "partition_filters=false;"
 152       "index_block_restart_interval=4;"
 153       "filter_policy=bloomfilter:4:true;whole_key_filtering=1;"
 154       "format_version=1;"
 155       "hash_index_allow_collision=false;"
 156       "verify_compression=true;read_amp_bytes_per_bit=0;"
 157       "enable_index_compression=false;"
 158       "block_align=true",
 159       new_bbto));
 160
 161   ASSERT_EQ(unset_bytes_base,
 162             NumUnsetBytes(new_bbto_ptr, sizeof(BlockBasedTableOptions),
 163                           kBbtoBlacklist));
 164
 165   ASSERT_TRUE(new_bbto->block_cache.get() != nullptr);
 166   ASSERT_TRUE(new_bbto->block_cache_compressed.get() != nullptr);
 167   ASSERT_TRUE(new_bbto->filter_policy.get() != nullptr);
 168
 169   bbto->~BlockBasedTableOptions();
 170   new_bbto->~BlockBasedTableOptions();
 171
 172   delete[] bbto_ptr;
 173   delete[] new_bbto_ptr;
 174 }
 175
 176 // If the test fails, likely a new option is added to DBOptions
 177 // but it cannot be set through GetDBOptionsFromString(), or the test is not
 178 // updated accordingly.
 179 // After adding an option, we need to make sure it is settable by
 180 // GetDBOptionsFromString() and add the option to the input string passed to
 181 // DBOptionsFromString()in this test.
 182 // If it is a complicated type, you also need to add the field to
 183 // kDBOptionsBlacklist, and maybe add customized verification for it.
 184 TEST_F(OptionsSettableTest, DBOptionsAllFieldsSettable) {
 185   const OffsetGap kDBOptionsBlacklist = {
 186       {offsetof(struct DBOptions, env), sizeof(Env*)},
 187       {offsetof(struct DBOptions, rate_limiter),
 188        sizeof(std::shared_ptr<RateLimiter>)},
 189       {offsetof(struct DBOptions, sst_file_manager),
 190        sizeof(std::shared_ptr<SstFileManager>)},
 191       {offsetof(struct DBOptions, info_log), sizeof(std::shared_ptr<Logger>)},
 192       {offsetof(struct DBOptions, statistics),
 193        sizeof(std::shared_ptr<Statistics>)},
 194       {offsetof(struct DBOptions, db_paths), sizeof(std::vector<DbPath>)},
 195       {offsetof(struct DBOptions, db_log_dir), sizeof(std::string)},
 196       {offsetof(struct DBOptions, wal_dir), sizeof(std::string)},
 197       {offsetof(struct DBOptions, write_buffer_manager),
 198        sizeof(std::shared_ptr<WriteBufferManager>)},
 199       {offsetof(struct DBOptions, listeners),
 200        sizeof(std::vector<std::shared_ptr<EventListener>>)},
 201       {offsetof(struct DBOptions, row_cache), sizeof(std::shared_ptr<Cache>)},
 202       {offsetof(struct DBOptions, wal_filter), sizeof(const WalFilter*)},
 203   };
 204
 205   char* options_ptr = new char[sizeof(DBOptions)];
 206
 207   // Count padding bytes by setting all bytes in the memory to a special char,
 208   // copy a well constructed struct to this memory and see how many special
 209   // bytes left.
 210   DBOptions* options = new (options_ptr) DBOptions();
 211   FillWithSpecialChar(options_ptr, sizeof(DBOptions), kDBOptionsBlacklist);
 212   // It based on the behavior of compiler that padding bytes are not changed
 213   // when copying the struct. It's prone to failure when compiler behavior
 214   // changes. We verify there is unset bytes to detect the case.
 215   *options = DBOptions();
 216   int unset_bytes_base =
 217       NumUnsetBytes(options_ptr, sizeof(DBOptions), kDBOptionsBlacklist);
 218   ASSERT_GT(unset_bytes_base, 0);
 219   options->~DBOptions();
 220
 221   options = new (options_ptr) DBOptions();
 222   FillWithSpecialChar(options_ptr, sizeof(DBOptions), kDBOptionsBlacklist);
 223
 224   char* new_options_ptr = new char[sizeof(DBOptions)];
 225   DBOptions* new_options = new (new_options_ptr) DBOptions();
 226   FillWithSpecialChar(new_options_ptr, sizeof(DBOptions), kDBOptionsBlacklist);
 227
 228   // Need to update the option string if a new option is added.
 229   ASSERT_OK(
 230       GetDBOptionsFromString(*options,
 231                              "wal_bytes_per_sync=4295048118;"
 232                              "delete_obsolete_files_period_micros=4294967758;"
 233                              "WAL_ttl_seconds=4295008036;"
 234                              "WAL_size_limit_MB=4295036161;"
 235                              "wal_dir=path/to/wal_dir;"
 236                              "db_write_buffer_size=2587;"
 237                              "max_subcompactions=64330;"
 238                              "table_cache_numshardbits=28;"
 239                              "max_open_files=72;"
 240                              "max_file_opening_threads=35;"
 241                              "max_background_jobs=8;"
 242                              "base_background_compactions=3;"
 243                              "max_background_compactions=33;"
 244                              "use_fsync=true;"
 245                              "use_adaptive_mutex=false;"
 246                              "max_total_wal_size=4295005604;"
 247                              "compaction_readahead_size=0;"
 248                              "new_table_reader_for_compaction_inputs=false;"
 249                              "keep_log_file_num=4890;"
 250                              "skip_stats_update_on_db_open=false;"
 251                              "max_manifest_file_size=4295009941;"
 252                              "db_log_dir=path/to/db_log_dir;"
 253                              "skip_log_error_on_recovery=true;"
 254                              "writable_file_max_buffer_size=1048576;"
 255                              "paranoid_checks=true;"
 256                              "is_fd_close_on_exec=false;"
 257                              "bytes_per_sync=4295013613;"
 258                              "enable_thread_tracking=false;"
 259                              "recycle_log_file_num=0;"
 260                              "create_missing_column_families=true;"
 261                              "log_file_time_to_roll=3097;"
 262                              "max_background_flushes=35;"
 263                              "create_if_missing=false;"
 264                              "error_if_exists=true;"
 265                              "delayed_write_rate=4294976214;"
 266                              "manifest_preallocation_size=1222;"
 267                              "allow_mmap_writes=false;"
 268                              "stats_dump_period_sec=70127;"
 269                              "stats_persist_period_sec=54321;"
 270                              "stats_history_buffer_size=14159;"
 271                              "allow_fallocate=true;"
 272                              "allow_mmap_reads=false;"
 273                              "use_direct_reads=false;"
 274                              "use_direct_io_for_flush_and_compaction=false;"
 275                              "max_log_file_size=4607;"
 276                              "random_access_max_buffer_size=1048576;"
 277                              "advise_random_on_open=true;"
 278                              "fail_if_options_file_error=false;"
 279                              "enable_pipelined_write=false;"
 280                              "allow_concurrent_memtable_write=true;"
 281                              "wal_recovery_mode=kPointInTimeRecovery;"
 282                              "enable_write_thread_adaptive_yield=true;"
 283                              "write_thread_slow_yield_usec=5;"
 284                              "write_thread_max_yield_usec=1000;"
 285                              "access_hint_on_compaction_start=NONE;"
 286                              "info_log_level=DEBUG_LEVEL;"
 287                              "dump_malloc_stats=false;"
 288                              "allow_2pc=false;"
 289                              "avoid_flush_during_recovery=false;"
 290                              "avoid_flush_during_shutdown=false;"
 291                              "allow_ingest_behind=false;"
 292                              "preserve_deletes=false;"
 293                              "concurrent_prepare=false;"
 294                              "two_write_queues=false;"
 295                              "manual_wal_flush=false;"
 296                              "seq_per_batch=false;"
 297                              "atomic_flush=false;"
 298                              "avoid_unnecessary_blocking_io=false",
 299                              new_options));
 300
 301   ASSERT_EQ(unset_bytes_base, NumUnsetBytes(new_options_ptr, sizeof(DBOptions),
 302                                             kDBOptionsBlacklist));
 303
 304   options->~DBOptions();
 305   new_options->~DBOptions();
 306
 307   delete[] options_ptr;
 308   delete[] new_options_ptr;
 309 }
 310
 311 template <typename T1, typename T2>
 312 inline int offset_of(T1 T2::*member) {
 313   static T2 obj;
 314   return int(size_t(&(obj.*member)) - size_t(&obj));
 315 }
 316
 317 // If the test fails, likely a new option is added to ColumnFamilyOptions
 318 // but it cannot be set through GetColumnFamilyOptionsFromString(), or the
 319 // test is not updated accordingly.
 320 // After adding an option, we need to make sure it is settable by
 321 // GetColumnFamilyOptionsFromString() and add the option to the input
 322 // string passed to GetColumnFamilyOptionsFromString()in this test.
 323 // If it is a complicated type, you also need to add the field to
 324 // kColumnFamilyOptionsBlacklist, and maybe add customized verification
 325 // for it.
 326 TEST_F(OptionsSettableTest, ColumnFamilyOptionsAllFieldsSettable) {
 327   // options in the blacklist need to appear in the same order as in
 328   // ColumnFamilyOptions.
 329   const OffsetGap kColumnFamilyOptionsBlacklist = {
 330       {offset_of(&ColumnFamilyOptions::inplace_callback),
 331        sizeof(UpdateStatus(*)(char*, uint32_t*, Slice, std::string*))},
 332       {offset_of(
 333            &ColumnFamilyOptions::memtable_insert_with_hint_prefix_extractor),
 334        sizeof(std::shared_ptr<const SliceTransform>)},
 335       {offset_of(&ColumnFamilyOptions::compression_per_level),
 336        sizeof(std::vector<CompressionType>)},
 337       {offset_of(
 338            &ColumnFamilyOptions::max_bytes_for_level_multiplier_additional),
 339        sizeof(std::vector<int>)},
 340       {offset_of(&ColumnFamilyOptions::memtable_factory),
 341        sizeof(std::shared_ptr<MemTableRepFactory>)},
 342       {offset_of(&ColumnFamilyOptions::table_properties_collector_factories),
 343        sizeof(ColumnFamilyOptions::TablePropertiesCollectorFactories)},
 344       {offset_of(&ColumnFamilyOptions::comparator), sizeof(Comparator*)},
 345       {offset_of(&ColumnFamilyOptions::merge_operator),
 346        sizeof(std::shared_ptr<MergeOperator>)},
 347       {offset_of(&ColumnFamilyOptions::compaction_filter),
 348        sizeof(const CompactionFilter*)},
 349       {offset_of(&ColumnFamilyOptions::compaction_filter_factory),
 350        sizeof(std::shared_ptr<CompactionFilterFactory>)},
 351       {offset_of(&ColumnFamilyOptions::prefix_extractor),
 352        sizeof(std::shared_ptr<const SliceTransform>)},
 353       {offset_of(&ColumnFamilyOptions::table_factory),
 354        sizeof(std::shared_ptr<TableFactory>)},
 355       {offset_of(&ColumnFamilyOptions::cf_paths),
 356        sizeof(std::vector<DbPath>)},
 357       {offset_of(&ColumnFamilyOptions::compaction_thread_limiter),
 358        sizeof(std::shared_ptr<ConcurrentTaskLimiter>)},
 359   };
 360
 361   char* options_ptr = new char[sizeof(ColumnFamilyOptions)];
 362
 363   // Count padding bytes by setting all bytes in the memory to a special char,
 364   // copy a well constructed struct to this memory and see how many special
 365   // bytes left.
 366   ColumnFamilyOptions* options = new (options_ptr) ColumnFamilyOptions();
 367   FillWithSpecialChar(options_ptr, sizeof(ColumnFamilyOptions),
 368                       kColumnFamilyOptionsBlacklist);
 369   // It based on the behavior of compiler that padding bytes are not changed
 370   // when copying the struct. It's prone to failure when compiler behavior
 371   // changes. We verify there is unset bytes to detect the case.
 372   *options = ColumnFamilyOptions();
 373
 374   // Deprecatd option which is not initialized. Need to set it to avoid
 375   // Valgrind error
 376   options->max_mem_compaction_level = 0;
 377
 378   int unset_bytes_base = NumUnsetBytes(options_ptr, sizeof(ColumnFamilyOptions),
 379                                        kColumnFamilyOptionsBlacklist);
 380   ASSERT_GT(unset_bytes_base, 0);
 381   options->~ColumnFamilyOptions();
 382
 383   options = new (options_ptr) ColumnFamilyOptions();
 384   FillWithSpecialChar(options_ptr, sizeof(ColumnFamilyOptions),
 385                       kColumnFamilyOptionsBlacklist);
 386
 387   // Following options are not settable through
 388   // GetColumnFamilyOptionsFromString():
 389   options->rate_limit_delay_max_milliseconds = 33;
 390   options->compaction_options_universal = CompactionOptionsUniversal();
 391   options->compression_opts = CompressionOptions();
 392   options->bottommost_compression_opts = CompressionOptions();
 393   options->hard_rate_limit = 0;
 394   options->soft_rate_limit = 0;
 395   options->purge_redundant_kvs_while_flush = false;
 396   options->max_mem_compaction_level = 0;
 397   options->compaction_filter = nullptr;
 398
 399   char* new_options_ptr = new char[sizeof(ColumnFamilyOptions)];
 400   ColumnFamilyOptions* new_options =
 401       new (new_options_ptr) ColumnFamilyOptions();
 402   FillWithSpecialChar(new_options_ptr, sizeof(ColumnFamilyOptions),
 403                       kColumnFamilyOptionsBlacklist);
 404
 405   // Need to update the option string if a new option is added.
 406   ASSERT_OK(GetColumnFamilyOptionsFromString(
 407       *options,
 408       "compaction_filter_factory=mpudlojcujCompactionFilterFactory;"
 409       "table_factory=PlainTable;"
 410       "prefix_extractor=rocksdb.CappedPrefix.13;"
 411       "comparator=leveldb.BytewiseComparator;"
 412       "compression_per_level=kBZip2Compression:kBZip2Compression:"
 413       "kBZip2Compression:kNoCompression:kZlibCompression:kBZip2Compression:"
 414       "kSnappyCompression;"
 415       "max_bytes_for_level_base=986;"
 416       "bloom_locality=8016;"
 417       "target_file_size_base=4294976376;"
 418       "memtable_huge_page_size=2557;"
 419       "max_successive_merges=5497;"
 420       "max_sequential_skip_in_iterations=4294971408;"
 421       "arena_block_size=1893;"
 422       "target_file_size_multiplier=35;"
 423       "min_write_buffer_number_to_merge=9;"
 424       "max_write_buffer_number=84;"
 425       "write_buffer_size=1653;"
 426       "max_compaction_bytes=64;"
 427       "max_bytes_for_level_multiplier=60;"
 428       "memtable_factory=SkipListFactory;"
 429       "compression=kNoCompression;"
 430       "bottommost_compression=kDisableCompressionOption;"
 431       "level0_stop_writes_trigger=33;"
 432       "num_levels=99;"
 433       "level0_slowdown_writes_trigger=22;"
 434       "level0_file_num_compaction_trigger=14;"
 435       "compaction_filter=urxcqstuwnCompactionFilter;"
 436       "soft_rate_limit=530.615385;"
 437       "soft_pending_compaction_bytes_limit=0;"
 438       "max_write_buffer_number_to_maintain=84;"
 439       "merge_operator=aabcxehazrMergeOperator;"
 440       "memtable_prefix_bloom_size_ratio=0.4642;"
 441       "memtable_whole_key_filtering=true;"
 442       "memtable_insert_with_hint_prefix_extractor=rocksdb.CappedPrefix.13;"
 443       "paranoid_file_checks=true;"
 444       "force_consistency_checks=true;"
 445       "inplace_update_num_locks=7429;"
 446       "optimize_filters_for_hits=false;"
 447       "level_compaction_dynamic_level_bytes=false;"
 448       "inplace_update_support=false;"
 449       "compaction_style=kCompactionStyleFIFO;"
 450       "compaction_pri=kMinOverlappingRatio;"
 451       "hard_pending_compaction_bytes_limit=0;"
 452       "disable_auto_compactions=false;"
 453       "report_bg_io_stats=true;"
 454       "ttl=60;"
 455       "sample_for_compression=0;"
 456       "compaction_options_fifo={max_table_files_size=3;allow_"
 457       "compaction=false;};",
 458       new_options));
 459
 460   ASSERT_EQ(unset_bytes_base,
 461             NumUnsetBytes(new_options_ptr, sizeof(ColumnFamilyOptions),
 462                           kColumnFamilyOptionsBlacklist));
 463
 464   options->~ColumnFamilyOptions();
 465   new_options->~ColumnFamilyOptions();
 466
 467   delete[] options_ptr;
 468   delete[] new_options_ptr;
 469 }
 470 #endif  // !__clang__
 471 #endif  // OS_LINUX || OS_WIN
 472 #endif  // !ROCKSDB_LITE
 473
 474 }  // namespace rocksdb
 475
 476 int main(int argc, char** argv) {
 477   ::testing::InitGoogleTest(&argc, argv);
 478 #ifdef GFLAGS
 479   ParseCommandLineFlags(&argc, &argv, true);
 480 #endif  // GFLAGS
 481   return RUN_ALL_TESTS();
 482 }