ceph/src/rocksdb/options/options_settable_test.cc

   1 //  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
   2 //  This source code is licensed under the BSD-style license found in the
   3 //  LICENSE file in the root directory of this source tree. An additional grant
   4 //  of patent rights can be found in the PATENTS file in the same directory.
   5 //
   6 // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
   7 // Use of this source code is governed by a BSD-style license that can be
   8 // found in the LICENSE file. See the AUTHORS file for names of contributors.
   9
  10 #ifndef __STDC_FORMAT_MACROS
  11 #define __STDC_FORMAT_MACROS
  12 #endif
  13
  14 #include <inttypes.h>
  15 #include <cctype>
  16 #include <cstring>
  17 #include <unordered_map>
  18
  19 #include "options/options_helper.h"
  20 #include "options/options_parser.h"
  21 #include "options/options_sanity_check.h"
  22 #include "rocksdb/cache.h"
  23 #include "rocksdb/convenience.h"
  24 #include "rocksdb/memtablerep.h"
  25 #include "rocksdb/utilities/leveldb_options.h"
  26 #include "util/random.h"
  27 #include "util/stderr_logger.h"
  28 #include "util/testharness.h"
  29 #include "util/testutil.h"
  30
  31 #ifndef GFLAGS
  32 bool FLAGS_enable_print = false;
  33 #else
  34 #include <gflags/gflags.h>
  35 using GFLAGS::ParseCommandLineFlags;
  36 DEFINE_bool(enable_print, false, "Print options generated to console.");
  37 #endif  // GFLAGS
  38
  39 namespace rocksdb {
  40
  41 // Verify options are settable from options strings.
  42 // We take the approach that depends on compiler behavior that copy constructor
  43 // won't touch implicit padding bytes, so that the test is fragile.
  44 // As a result, we only run the tests to verify new fields in options are
  45 // settable through string on limited platforms as it depends on behavior of
  46 // compilers.
  47 #ifndef ROCKSDB_LITE
  48 #ifdef OS_LINUX
  49 #ifndef __clang__
  50
  51 class OptionsSettableTest : public testing::Test {
  52  public:
  53   OptionsSettableTest() {}
  54 };
  55
  56 const char kSpecialChar = 'z';
  57 typedef std::vector<std::pair<int, size_t>> OffsetGap;
  58
  59 void FillWithSpecialChar(char* start_ptr, size_t total_size,
  60                          const OffsetGap& blacklist) {
  61   size_t offset = 0;
  62   for (auto& pair : blacklist) {
  63     std::memset(start_ptr + offset, kSpecialChar, pair.first - offset);
  64     offset = pair.first + pair.second;
  65   }
  66   std::memset(start_ptr + offset, kSpecialChar, total_size - offset);
  67 }
  68
  69 int NumUnsetBytes(char* start_ptr, size_t total_size,
  70                   const OffsetGap& blacklist) {
  71   int total_unset_bytes_base = 0;
  72   size_t offset = 0;
  73   for (auto& pair : blacklist) {
  74     for (char* ptr = start_ptr + offset; ptr < start_ptr + pair.first; ptr++) {
  75       if (*ptr == kSpecialChar) {
  76         total_unset_bytes_base++;
  77       }
  78     }
  79     offset = pair.first + pair.second;
  80   }
  81   for (char* ptr = start_ptr + offset; ptr < start_ptr + total_size; ptr++) {
  82     if (*ptr == kSpecialChar) {
  83       total_unset_bytes_base++;
  84     }
  85   }
  86   return total_unset_bytes_base;
  87 }
  88
  89 // If the test fails, likely a new option is added to BlockBasedTableOptions
  90 // but it cannot be set through GetBlockBasedTableOptionsFromString(), or the
  91 // test is not updated accordingly.
  92 // After adding an option, we need to make sure it is settable by
  93 // GetBlockBasedTableOptionsFromString() and add the option to the input string
  94 // passed to the GetBlockBasedTableOptionsFromString() in this test.
  95 // If it is a complicated type, you also need to add the field to
  96 // kBbtoBlacklist, and maybe add customized verification for it.
  97 TEST_F(OptionsSettableTest, BlockBasedTableOptionsAllFieldsSettable) {
  98   // Items in the form of <offset, size>. Need to be in ascending order
  99   // and not overlapping. Need to updated if new pointer-option is added.
 100   const OffsetGap kBbtoBlacklist = {
 101       {offsetof(struct BlockBasedTableOptions, flush_block_policy_factory),
 102        sizeof(std::shared_ptr<FlushBlockPolicyFactory>)},
 103       {offsetof(struct BlockBasedTableOptions, block_cache),
 104        sizeof(std::shared_ptr<Cache>)},
 105       {offsetof(struct BlockBasedTableOptions, persistent_cache),
 106        sizeof(std::shared_ptr<PersistentCache>)},
 107       {offsetof(struct BlockBasedTableOptions, block_cache_compressed),
 108        sizeof(std::shared_ptr<Cache>)},
 109       {offsetof(struct BlockBasedTableOptions, filter_policy),
 110        sizeof(std::shared_ptr<const FilterPolicy>)},
 111   };
 112
 113   // In this test, we catch a new option of BlockBasedTableOptions that is not
 114   // settable through GetBlockBasedTableOptionsFromString().
 115   // We count padding bytes of the option struct, and assert it to be the same
 116   // as unset bytes of an option struct initialized by
 117   // GetBlockBasedTableOptionsFromString().
 118
 119   char* bbto_ptr = new char[sizeof(BlockBasedTableOptions)];
 120
 121   // Count padding bytes by setting all bytes in the memory to a special char,
 122   // copy a well constructed struct to this memory and see how many special
 123   // bytes left.
 124   BlockBasedTableOptions* bbto = new (bbto_ptr) BlockBasedTableOptions();
 125   FillWithSpecialChar(bbto_ptr, sizeof(BlockBasedTableOptions), kBbtoBlacklist);
 126   // It based on the behavior of compiler that padding bytes are not changed
 127   // when copying the struct. It's prone to failure when compiler behavior
 128   // changes. We verify there is unset bytes to detect the case.
 129   *bbto = BlockBasedTableOptions();
 130   int unset_bytes_base =
 131       NumUnsetBytes(bbto_ptr, sizeof(BlockBasedTableOptions), kBbtoBlacklist);
 132   ASSERT_GT(unset_bytes_base, 0);
 133   bbto->~BlockBasedTableOptions();
 134
 135   // Construct the base option passed into
 136   // GetBlockBasedTableOptionsFromString().
 137   bbto = new (bbto_ptr) BlockBasedTableOptions();
 138   FillWithSpecialChar(bbto_ptr, sizeof(BlockBasedTableOptions), kBbtoBlacklist);
 139   // This option is not setable:
 140   bbto->use_delta_encoding = true;
 141
 142   char* new_bbto_ptr = new char[sizeof(BlockBasedTableOptions)];
 143   BlockBasedTableOptions* new_bbto =
 144       new (new_bbto_ptr) BlockBasedTableOptions();
 145   FillWithSpecialChar(new_bbto_ptr, sizeof(BlockBasedTableOptions),
 146                       kBbtoBlacklist);
 147
 148   // Need to update the option string if a new option is added.
 149   ASSERT_OK(GetBlockBasedTableOptionsFromString(
 150       *bbto,
 151       "cache_index_and_filter_blocks=1;"
 152       "cache_index_and_filter_blocks_with_high_priority=true;"
 153       "pin_l0_filter_and_index_blocks_in_cache=1;"
 154       "index_type=kHashSearch;"
 155       "checksum=kxxHash;hash_index_allow_collision=1;no_block_cache=1;"
 156       "block_cache=1M;block_cache_compressed=1k;block_size=1024;"
 157       "block_size_deviation=8;block_restart_interval=4; "
 158       "metadata_block_size=1024;"
 159       "partition_filters=false;"
 160       "index_block_restart_interval=4;"
 161       "filter_policy=bloomfilter:4:true;whole_key_filtering=1;"
 162       "format_version=1;"
 163       "hash_index_allow_collision=false;"
 164       "verify_compression=true;read_amp_bytes_per_bit=0",
 165       new_bbto));
 166
 167   ASSERT_EQ(unset_bytes_base,
 168             NumUnsetBytes(new_bbto_ptr, sizeof(BlockBasedTableOptions),
 169                           kBbtoBlacklist));
 170
 171   ASSERT_TRUE(new_bbto->block_cache.get() != nullptr);
 172   ASSERT_TRUE(new_bbto->block_cache_compressed.get() != nullptr);
 173   ASSERT_TRUE(new_bbto->filter_policy.get() != nullptr);
 174
 175   bbto->~BlockBasedTableOptions();
 176   new_bbto->~BlockBasedTableOptions();
 177
 178   delete[] bbto_ptr;
 179   delete[] new_bbto_ptr;
 180 }
 181
 182 // If the test fails, likely a new option is added to DBOptions
 183 // but it cannot be set through GetDBOptionsFromString(), or the test is not
 184 // updated accordingly.
 185 // After adding an option, we need to make sure it is settable by
 186 // GetDBOptionsFromString() and add the option to the input string passed to
 187 // DBOptionsFromString()in this test.
 188 // If it is a complicated type, you also need to add the field to
 189 // kDBOptionsBlacklist, and maybe add customized verification for it.
 190 TEST_F(OptionsSettableTest, DBOptionsAllFieldsSettable) {
 191   const OffsetGap kDBOptionsBlacklist = {
 192       {offsetof(struct DBOptions, env), sizeof(Env*)},
 193       {offsetof(struct DBOptions, rate_limiter),
 194        sizeof(std::shared_ptr<RateLimiter>)},
 195       {offsetof(struct DBOptions, sst_file_manager),
 196        sizeof(std::shared_ptr<SstFileManager>)},
 197       {offsetof(struct DBOptions, info_log), sizeof(std::shared_ptr<Logger>)},
 198       {offsetof(struct DBOptions, statistics),
 199        sizeof(std::shared_ptr<Statistics>)},
 200       {offsetof(struct DBOptions, db_paths), sizeof(std::vector<DbPath>)},
 201       {offsetof(struct DBOptions, db_log_dir), sizeof(std::string)},
 202       {offsetof(struct DBOptions, wal_dir), sizeof(std::string)},
 203       {offsetof(struct DBOptions, write_buffer_manager),
 204        sizeof(std::shared_ptr<WriteBufferManager>)},
 205       {offsetof(struct DBOptions, listeners),
 206        sizeof(std::vector<std::shared_ptr<EventListener>>)},
 207       {offsetof(struct DBOptions, row_cache), sizeof(std::shared_ptr<Cache>)},
 208       {offsetof(struct DBOptions, wal_filter), sizeof(const WalFilter*)},
 209   };
 210
 211   char* options_ptr = new char[sizeof(DBOptions)];
 212
 213   // Count padding bytes by setting all bytes in the memory to a special char,
 214   // copy a well constructed struct to this memory and see how many special
 215   // bytes left.
 216   DBOptions* options = new (options_ptr) DBOptions();
 217   FillWithSpecialChar(options_ptr, sizeof(DBOptions), kDBOptionsBlacklist);
 218   // It based on the behavior of compiler that padding bytes are not changed
 219   // when copying the struct. It's prone to failure when compiler behavior
 220   // changes. We verify there is unset bytes to detect the case.
 221   *options = DBOptions();
 222   int unset_bytes_base =
 223       NumUnsetBytes(options_ptr, sizeof(DBOptions), kDBOptionsBlacklist);
 224   ASSERT_GT(unset_bytes_base, 0);
 225   options->~DBOptions();
 226
 227   options = new (options_ptr) DBOptions();
 228   FillWithSpecialChar(options_ptr, sizeof(DBOptions), kDBOptionsBlacklist);
 229
 230   char* new_options_ptr = new char[sizeof(DBOptions)];
 231   DBOptions* new_options = new (new_options_ptr) DBOptions();
 232   FillWithSpecialChar(new_options_ptr, sizeof(DBOptions), kDBOptionsBlacklist);
 233
 234   // Need to update the option string if a new option is added.
 235   ASSERT_OK(
 236       GetDBOptionsFromString(*options,
 237                              "wal_bytes_per_sync=4295048118;"
 238                              "delete_obsolete_files_period_micros=4294967758;"
 239                              "WAL_ttl_seconds=4295008036;"
 240                              "WAL_size_limit_MB=4295036161;"
 241                              "wal_dir=path/to/wal_dir;"
 242                              "db_write_buffer_size=2587;"
 243                              "max_subcompactions=64330;"
 244                              "table_cache_numshardbits=28;"
 245                              "max_open_files=72;"
 246                              "max_file_opening_threads=35;"
 247                              "base_background_compactions=3;"
 248                              "max_background_compactions=33;"
 249                              "use_fsync=true;"
 250                              "use_adaptive_mutex=false;"
 251                              "max_total_wal_size=4295005604;"
 252                              "compaction_readahead_size=0;"
 253                              "new_table_reader_for_compaction_inputs=false;"
 254                              "keep_log_file_num=4890;"
 255                              "skip_stats_update_on_db_open=false;"
 256                              "max_manifest_file_size=4295009941;"
 257                              "db_log_dir=path/to/db_log_dir;"
 258                              "skip_log_error_on_recovery=true;"
 259                              "writable_file_max_buffer_size=1048576;"
 260                              "paranoid_checks=true;"
 261                              "is_fd_close_on_exec=false;"
 262                              "bytes_per_sync=4295013613;"
 263                              "enable_thread_tracking=false;"
 264                              "recycle_log_file_num=0;"
 265                              "create_missing_column_families=true;"
 266                              "log_file_time_to_roll=3097;"
 267                              "max_background_flushes=35;"
 268                              "create_if_missing=false;"
 269                              "error_if_exists=true;"
 270                              "delayed_write_rate=4294976214;"
 271                              "manifest_preallocation_size=1222;"
 272                              "allow_mmap_writes=false;"
 273                              "stats_dump_period_sec=70127;"
 274                              "allow_fallocate=true;"
 275                              "allow_mmap_reads=false;"
 276                              "use_direct_reads=false;"
 277                              "use_direct_io_for_flush_and_compaction=false;"
 278                              "max_log_file_size=4607;"
 279                              "random_access_max_buffer_size=1048576;"
 280                              "advise_random_on_open=true;"
 281                              "fail_if_options_file_error=false;"
 282                              "allow_concurrent_memtable_write=true;"
 283                              "wal_recovery_mode=kPointInTimeRecovery;"
 284                              "enable_write_thread_adaptive_yield=true;"
 285                              "write_thread_slow_yield_usec=5;"
 286                              "write_thread_max_yield_usec=1000;"
 287                              "access_hint_on_compaction_start=NONE;"
 288                              "info_log_level=DEBUG_LEVEL;"
 289                              "dump_malloc_stats=false;"
 290                              "allow_2pc=false;"
 291                              "avoid_flush_during_recovery=false;"
 292                              "avoid_flush_during_shutdown=false;",
 293                              new_options));
 294
 295   ASSERT_EQ(unset_bytes_base, NumUnsetBytes(new_options_ptr, sizeof(DBOptions),
 296                                             kDBOptionsBlacklist));
 297
 298   options->~DBOptions();
 299   new_options->~DBOptions();
 300
 301   delete[] options_ptr;
 302   delete[] new_options_ptr;
 303 }
 304
 305 // If the test fails, likely a new option is added to ColumnFamilyOptions
 306 // but it cannot be set through GetColumnFamilyOptionsFromString(), or the
 307 // test is not updated accordingly.
 308 // After adding an option, we need to make sure it is settable by
 309 // GetColumnFamilyOptionsFromString() and add the option to the input
 310 // string passed to GetColumnFamilyOptionsFromString()in this test.
 311 // If it is a complicated type, you also need to add the field to
 312 // kColumnFamilyOptionsBlacklist, and maybe add customized verification
 313 // for it.
 314 TEST_F(OptionsSettableTest, ColumnFamilyOptionsAllFieldsSettable) {
 315   // options in the blacklist need to appear in the same order as in
 316   // ColumnFamilyOptions.
 317   const OffsetGap kColumnFamilyOptionsBlacklist = {
 318       {offset_of(&ColumnFamilyOptions::inplace_callback),
 319        sizeof(UpdateStatus(*)(char*, uint32_t*, Slice, std::string*))},
 320       {offset_of(
 321            &ColumnFamilyOptions::memtable_insert_with_hint_prefix_extractor),
 322        sizeof(std::shared_ptr<const SliceTransform>)},
 323       {offset_of(&ColumnFamilyOptions::compression_per_level),
 324        sizeof(std::vector<CompressionType>)},
 325       {offset_of(
 326            &ColumnFamilyOptions::max_bytes_for_level_multiplier_additional),
 327        sizeof(std::vector<int>)},
 328       {offset_of(&ColumnFamilyOptions::memtable_factory),
 329        sizeof(std::shared_ptr<MemTableRepFactory>)},
 330       {offset_of(&ColumnFamilyOptions::table_properties_collector_factories),
 331        sizeof(ColumnFamilyOptions::TablePropertiesCollectorFactories)},
 332       {offset_of(&ColumnFamilyOptions::comparator), sizeof(Comparator*)},
 333       {offset_of(&ColumnFamilyOptions::merge_operator),
 334        sizeof(std::shared_ptr<MergeOperator>)},
 335       {offset_of(&ColumnFamilyOptions::compaction_filter),
 336        sizeof(const CompactionFilter*)},
 337       {offset_of(&ColumnFamilyOptions::compaction_filter_factory),
 338        sizeof(std::shared_ptr<CompactionFilterFactory>)},
 339       {offset_of(&ColumnFamilyOptions::prefix_extractor),
 340        sizeof(std::shared_ptr<const SliceTransform>)},
 341       {offset_of(&ColumnFamilyOptions::table_factory),
 342        sizeof(std::shared_ptr<TableFactory>)},
 343   };
 344
 345   char* options_ptr = new char[sizeof(ColumnFamilyOptions)];
 346
 347   // Count padding bytes by setting all bytes in the memory to a special char,
 348   // copy a well constructed struct to this memory and see how many special
 349   // bytes left.
 350   ColumnFamilyOptions* options = new (options_ptr) ColumnFamilyOptions();
 351   FillWithSpecialChar(options_ptr, sizeof(ColumnFamilyOptions),
 352                       kColumnFamilyOptionsBlacklist);
 353   // It based on the behavior of compiler that padding bytes are not changed
 354   // when copying the struct. It's prone to failure when compiler behavior
 355   // changes. We verify there is unset bytes to detect the case.
 356   *options = ColumnFamilyOptions();
 357
 358   // Deprecatd option which is not initialized. Need to set it to avoid
 359   // Valgrind error
 360   options->max_mem_compaction_level = 0;
 361
 362   int unset_bytes_base = NumUnsetBytes(options_ptr, sizeof(ColumnFamilyOptions),
 363                                        kColumnFamilyOptionsBlacklist);
 364   ASSERT_GT(unset_bytes_base, 0);
 365   options->~ColumnFamilyOptions();
 366
 367   options = new (options_ptr) ColumnFamilyOptions();
 368   FillWithSpecialChar(options_ptr, sizeof(ColumnFamilyOptions),
 369                       kColumnFamilyOptionsBlacklist);
 370
 371   // Following options are not settable through
 372   // GetColumnFamilyOptionsFromString():
 373   options->rate_limit_delay_max_milliseconds = 33;
 374   options->compaction_options_universal = CompactionOptionsUniversal();
 375   options->compression_opts = CompressionOptions();
 376   options->hard_rate_limit = 0;
 377   options->soft_rate_limit = 0;
 378   options->compaction_options_fifo = CompactionOptionsFIFO();
 379   options->max_mem_compaction_level = 0;
 380
 381   char* new_options_ptr = new char[sizeof(ColumnFamilyOptions)];
 382   ColumnFamilyOptions* new_options =
 383       new (new_options_ptr) ColumnFamilyOptions();
 384   FillWithSpecialChar(new_options_ptr, sizeof(ColumnFamilyOptions),
 385                       kColumnFamilyOptionsBlacklist);
 386
 387   // Need to update the option string if a new option is added.
 388   ASSERT_OK(GetColumnFamilyOptionsFromString(
 389       *options,
 390       "compaction_filter_factory=mpudlojcujCompactionFilterFactory;"
 391       "table_factory=PlainTable;"
 392       "prefix_extractor=rocksdb.CappedPrefix.13;"
 393       "comparator=leveldb.BytewiseComparator;"
 394       "compression_per_level=kBZip2Compression:kBZip2Compression:"
 395       "kBZip2Compression:kNoCompression:kZlibCompression:kBZip2Compression:"
 396       "kSnappyCompression;"
 397       "max_bytes_for_level_base=986;"
 398       "bloom_locality=8016;"
 399       "target_file_size_base=4294976376;"
 400       "memtable_huge_page_size=2557;"
 401       "max_successive_merges=5497;"
 402       "max_sequential_skip_in_iterations=4294971408;"
 403       "arena_block_size=1893;"
 404       "target_file_size_multiplier=35;"
 405       "min_write_buffer_number_to_merge=9;"
 406       "max_write_buffer_number=84;"
 407       "write_buffer_size=1653;"
 408       "max_compaction_bytes=64;"
 409       "max_bytes_for_level_multiplier=60;"
 410       "memtable_factory=SkipListFactory;"
 411       "compression=kNoCompression;"
 412       "bottommost_compression=kDisableCompressionOption;"
 413       "level0_stop_writes_trigger=33;"
 414       "num_levels=99;"
 415       "level0_slowdown_writes_trigger=22;"
 416       "level0_file_num_compaction_trigger=14;"
 417       "compaction_filter=urxcqstuwnCompactionFilter;"
 418       "soft_rate_limit=530.615385;"
 419       "soft_pending_compaction_bytes_limit=0;"
 420       "max_write_buffer_number_to_maintain=84;"
 421       "merge_operator=aabcxehazrMergeOperator;"
 422       "memtable_prefix_bloom_size_ratio=0.4642;"
 423       "memtable_insert_with_hint_prefix_extractor=rocksdb.CappedPrefix.13;"
 424       "paranoid_file_checks=true;"
 425       "force_consistency_checks=true;"
 426       "inplace_update_num_locks=7429;"
 427       "optimize_filters_for_hits=false;"
 428       "level_compaction_dynamic_level_bytes=false;"
 429       "inplace_update_support=false;"
 430       "compaction_style=kCompactionStyleFIFO;"
 431       "compaction_pri=kMinOverlappingRatio;"
 432       "purge_redundant_kvs_while_flush=true;"
 433       "hard_pending_compaction_bytes_limit=0;"
 434       "disable_auto_compactions=false;"
 435       "report_bg_io_stats=true;",
 436       new_options));
 437
 438   ASSERT_EQ(unset_bytes_base,
 439             NumUnsetBytes(new_options_ptr, sizeof(ColumnFamilyOptions),
 440                           kColumnFamilyOptionsBlacklist));
 441
 442   options->~ColumnFamilyOptions();
 443   new_options->~ColumnFamilyOptions();
 444
 445   delete[] options_ptr;
 446   delete[] new_options_ptr;
 447 }
 448 #endif  // !__clang__
 449 #endif  // OS_LINUX
 450 #endif  // !ROCKSDB_LITE
 451
 452 }  // namespace rocksdb
 453
 454 int main(int argc, char** argv) {
 455   ::testing::InitGoogleTest(&argc, argv);
 456 #ifdef GFLAGS
 457   ParseCommandLineFlags(&argc, &argv, true);
 458 #endif  // GFLAGS
 459   return RUN_ALL_TESTS();
 460 }