1 // Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
2 // This source code is licensed under both the GPLv2 (found in the
3 // COPYING file in the root directory) and Apache 2.0 License
4 // (found in the LICENSE.Apache file in the root directory).
6 // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
7 // Use of this source code is governed by a BSD-style license that can be
8 // found in the LICENSE file. See the AUTHORS file for names of contributors.
10 #ifndef __STDC_FORMAT_MACROS
11 #define __STDC_FORMAT_MACROS
16 #include "options/options_helper.h"
17 #include "rocksdb/convenience.h"
18 #include "util/testharness.h"
21 bool FLAGS_enable_print
= false;
23 #include "util/gflags_compat.h"
24 using GFLAGS_NAMESPACE::ParseCommandLineFlags
;
25 DEFINE_bool(enable_print
, false, "Print options generated to console.");
30 // Verify options are settable from options strings.
31 // We take the approach that depends on compiler behavior that copy constructor
32 // won't touch implicit padding bytes, so that the test is fragile.
33 // As a result, we only run the tests to verify new fields in options are
34 // settable through string on limited platforms as it depends on behavior of
37 #if defined OS_LINUX || defined OS_WIN
40 class OptionsSettableTest
: public testing::Test
{
42 OptionsSettableTest() {}
45 const char kSpecialChar
= 'z';
46 typedef std::vector
<std::pair
<size_t, size_t>> OffsetGap
;
48 void FillWithSpecialChar(char* start_ptr
, size_t total_size
,
49 const OffsetGap
& blacklist
) {
51 for (auto& pair
: blacklist
) {
52 std::memset(start_ptr
+ offset
, kSpecialChar
, pair
.first
- offset
);
53 offset
= pair
.first
+ pair
.second
;
55 std::memset(start_ptr
+ offset
, kSpecialChar
, total_size
- offset
);
58 int NumUnsetBytes(char* start_ptr
, size_t total_size
,
59 const OffsetGap
& blacklist
) {
60 int total_unset_bytes_base
= 0;
62 for (auto& pair
: blacklist
) {
63 for (char* ptr
= start_ptr
+ offset
; ptr
< start_ptr
+ pair
.first
; ptr
++) {
64 if (*ptr
== kSpecialChar
) {
65 total_unset_bytes_base
++;
68 offset
= pair
.first
+ pair
.second
;
70 for (char* ptr
= start_ptr
+ offset
; ptr
< start_ptr
+ total_size
; ptr
++) {
71 if (*ptr
== kSpecialChar
) {
72 total_unset_bytes_base
++;
75 return total_unset_bytes_base
;
78 // If the test fails, likely a new option is added to BlockBasedTableOptions
79 // but it cannot be set through GetBlockBasedTableOptionsFromString(), or the
80 // test is not updated accordingly.
81 // After adding an option, we need to make sure it is settable by
82 // GetBlockBasedTableOptionsFromString() and add the option to the input string
83 // passed to the GetBlockBasedTableOptionsFromString() in this test.
84 // If it is a complicated type, you also need to add the field to
85 // kBbtoBlacklist, and maybe add customized verification for it.
86 TEST_F(OptionsSettableTest
, BlockBasedTableOptionsAllFieldsSettable
) {
87 // Items in the form of <offset, size>. Need to be in ascending order
88 // and not overlapping. Need to updated if new pointer-option is added.
89 const OffsetGap kBbtoBlacklist
= {
90 {offsetof(struct BlockBasedTableOptions
, flush_block_policy_factory
),
91 sizeof(std::shared_ptr
<FlushBlockPolicyFactory
>)},
92 {offsetof(struct BlockBasedTableOptions
, block_cache
),
93 sizeof(std::shared_ptr
<Cache
>)},
94 {offsetof(struct BlockBasedTableOptions
, persistent_cache
),
95 sizeof(std::shared_ptr
<PersistentCache
>)},
96 {offsetof(struct BlockBasedTableOptions
, block_cache_compressed
),
97 sizeof(std::shared_ptr
<Cache
>)},
98 {offsetof(struct BlockBasedTableOptions
, filter_policy
),
99 sizeof(std::shared_ptr
<const FilterPolicy
>)},
102 // In this test, we catch a new option of BlockBasedTableOptions that is not
103 // settable through GetBlockBasedTableOptionsFromString().
104 // We count padding bytes of the option struct, and assert it to be the same
105 // as unset bytes of an option struct initialized by
106 // GetBlockBasedTableOptionsFromString().
108 char* bbto_ptr
= new char[sizeof(BlockBasedTableOptions
)];
110 // Count padding bytes by setting all bytes in the memory to a special char,
111 // copy a well constructed struct to this memory and see how many special
113 BlockBasedTableOptions
* bbto
= new (bbto_ptr
) BlockBasedTableOptions();
114 FillWithSpecialChar(bbto_ptr
, sizeof(BlockBasedTableOptions
), kBbtoBlacklist
);
115 // It based on the behavior of compiler that padding bytes are not changed
116 // when copying the struct. It's prone to failure when compiler behavior
117 // changes. We verify there is unset bytes to detect the case.
118 *bbto
= BlockBasedTableOptions();
119 int unset_bytes_base
=
120 NumUnsetBytes(bbto_ptr
, sizeof(BlockBasedTableOptions
), kBbtoBlacklist
);
121 ASSERT_GT(unset_bytes_base
, 0);
122 bbto
->~BlockBasedTableOptions();
124 // Construct the base option passed into
125 // GetBlockBasedTableOptionsFromString().
126 bbto
= new (bbto_ptr
) BlockBasedTableOptions();
127 FillWithSpecialChar(bbto_ptr
, sizeof(BlockBasedTableOptions
), kBbtoBlacklist
);
128 // This option is not setable:
129 bbto
->use_delta_encoding
= true;
131 char* new_bbto_ptr
= new char[sizeof(BlockBasedTableOptions
)];
132 BlockBasedTableOptions
* new_bbto
=
133 new (new_bbto_ptr
) BlockBasedTableOptions();
134 FillWithSpecialChar(new_bbto_ptr
, sizeof(BlockBasedTableOptions
),
137 // Need to update the option string if a new option is added.
138 ASSERT_OK(GetBlockBasedTableOptionsFromString(
140 "cache_index_and_filter_blocks=1;"
141 "cache_index_and_filter_blocks_with_high_priority=true;"
142 "pin_l0_filter_and_index_blocks_in_cache=1;"
143 "pin_top_level_index_and_filter=1;"
144 "index_type=kHashSearch;"
145 "data_block_index_type=kDataBlockBinaryAndHash;"
146 "data_block_hash_table_util_ratio=0.75;"
147 "checksum=kxxHash;hash_index_allow_collision=1;no_block_cache=1;"
148 "block_cache=1M;block_cache_compressed=1k;block_size=1024;"
149 "block_size_deviation=8;block_restart_interval=4; "
150 "metadata_block_size=1024;"
151 "partition_filters=false;"
152 "index_block_restart_interval=4;"
153 "filter_policy=bloomfilter:4:true;whole_key_filtering=1;"
155 "hash_index_allow_collision=false;"
156 "verify_compression=true;read_amp_bytes_per_bit=0;"
157 "enable_index_compression=false;"
161 ASSERT_EQ(unset_bytes_base
,
162 NumUnsetBytes(new_bbto_ptr
, sizeof(BlockBasedTableOptions
),
165 ASSERT_TRUE(new_bbto
->block_cache
.get() != nullptr);
166 ASSERT_TRUE(new_bbto
->block_cache_compressed
.get() != nullptr);
167 ASSERT_TRUE(new_bbto
->filter_policy
.get() != nullptr);
169 bbto
->~BlockBasedTableOptions();
170 new_bbto
->~BlockBasedTableOptions();
173 delete[] new_bbto_ptr
;
176 // If the test fails, likely a new option is added to DBOptions
177 // but it cannot be set through GetDBOptionsFromString(), or the test is not
178 // updated accordingly.
179 // After adding an option, we need to make sure it is settable by
180 // GetDBOptionsFromString() and add the option to the input string passed to
181 // DBOptionsFromString()in this test.
182 // If it is a complicated type, you also need to add the field to
183 // kDBOptionsBlacklist, and maybe add customized verification for it.
184 TEST_F(OptionsSettableTest
, DBOptionsAllFieldsSettable
) {
185 const OffsetGap kDBOptionsBlacklist
= {
186 {offsetof(struct DBOptions
, env
), sizeof(Env
*)},
187 {offsetof(struct DBOptions
, rate_limiter
),
188 sizeof(std::shared_ptr
<RateLimiter
>)},
189 {offsetof(struct DBOptions
, sst_file_manager
),
190 sizeof(std::shared_ptr
<SstFileManager
>)},
191 {offsetof(struct DBOptions
, info_log
), sizeof(std::shared_ptr
<Logger
>)},
192 {offsetof(struct DBOptions
, statistics
),
193 sizeof(std::shared_ptr
<Statistics
>)},
194 {offsetof(struct DBOptions
, db_paths
), sizeof(std::vector
<DbPath
>)},
195 {offsetof(struct DBOptions
, db_log_dir
), sizeof(std::string
)},
196 {offsetof(struct DBOptions
, wal_dir
), sizeof(std::string
)},
197 {offsetof(struct DBOptions
, write_buffer_manager
),
198 sizeof(std::shared_ptr
<WriteBufferManager
>)},
199 {offsetof(struct DBOptions
, listeners
),
200 sizeof(std::vector
<std::shared_ptr
<EventListener
>>)},
201 {offsetof(struct DBOptions
, row_cache
), sizeof(std::shared_ptr
<Cache
>)},
202 {offsetof(struct DBOptions
, wal_filter
), sizeof(const WalFilter
*)},
205 char* options_ptr
= new char[sizeof(DBOptions
)];
207 // Count padding bytes by setting all bytes in the memory to a special char,
208 // copy a well constructed struct to this memory and see how many special
210 DBOptions
* options
= new (options_ptr
) DBOptions();
211 FillWithSpecialChar(options_ptr
, sizeof(DBOptions
), kDBOptionsBlacklist
);
212 // It based on the behavior of compiler that padding bytes are not changed
213 // when copying the struct. It's prone to failure when compiler behavior
214 // changes. We verify there is unset bytes to detect the case.
215 *options
= DBOptions();
216 int unset_bytes_base
=
217 NumUnsetBytes(options_ptr
, sizeof(DBOptions
), kDBOptionsBlacklist
);
218 ASSERT_GT(unset_bytes_base
, 0);
219 options
->~DBOptions();
221 options
= new (options_ptr
) DBOptions();
222 FillWithSpecialChar(options_ptr
, sizeof(DBOptions
), kDBOptionsBlacklist
);
224 char* new_options_ptr
= new char[sizeof(DBOptions
)];
225 DBOptions
* new_options
= new (new_options_ptr
) DBOptions();
226 FillWithSpecialChar(new_options_ptr
, sizeof(DBOptions
), kDBOptionsBlacklist
);
228 // Need to update the option string if a new option is added.
230 GetDBOptionsFromString(*options
,
231 "wal_bytes_per_sync=4295048118;"
232 "delete_obsolete_files_period_micros=4294967758;"
233 "WAL_ttl_seconds=4295008036;"
234 "WAL_size_limit_MB=4295036161;"
235 "wal_dir=path/to/wal_dir;"
236 "db_write_buffer_size=2587;"
237 "max_subcompactions=64330;"
238 "table_cache_numshardbits=28;"
240 "max_file_opening_threads=35;"
241 "max_background_jobs=8;"
242 "base_background_compactions=3;"
243 "max_background_compactions=33;"
245 "use_adaptive_mutex=false;"
246 "max_total_wal_size=4295005604;"
247 "compaction_readahead_size=0;"
248 "new_table_reader_for_compaction_inputs=false;"
249 "keep_log_file_num=4890;"
250 "skip_stats_update_on_db_open=false;"
251 "max_manifest_file_size=4295009941;"
252 "db_log_dir=path/to/db_log_dir;"
253 "skip_log_error_on_recovery=true;"
254 "writable_file_max_buffer_size=1048576;"
255 "paranoid_checks=true;"
256 "is_fd_close_on_exec=false;"
257 "bytes_per_sync=4295013613;"
258 "enable_thread_tracking=false;"
259 "recycle_log_file_num=0;"
260 "create_missing_column_families=true;"
261 "log_file_time_to_roll=3097;"
262 "max_background_flushes=35;"
263 "create_if_missing=false;"
264 "error_if_exists=true;"
265 "delayed_write_rate=4294976214;"
266 "manifest_preallocation_size=1222;"
267 "allow_mmap_writes=false;"
268 "stats_dump_period_sec=70127;"
269 "stats_persist_period_sec=54321;"
270 "stats_history_buffer_size=14159;"
271 "allow_fallocate=true;"
272 "allow_mmap_reads=false;"
273 "use_direct_reads=false;"
274 "use_direct_io_for_flush_and_compaction=false;"
275 "max_log_file_size=4607;"
276 "random_access_max_buffer_size=1048576;"
277 "advise_random_on_open=true;"
278 "fail_if_options_file_error=false;"
279 "enable_pipelined_write=false;"
280 "allow_concurrent_memtable_write=true;"
281 "wal_recovery_mode=kPointInTimeRecovery;"
282 "enable_write_thread_adaptive_yield=true;"
283 "write_thread_slow_yield_usec=5;"
284 "write_thread_max_yield_usec=1000;"
285 "access_hint_on_compaction_start=NONE;"
286 "info_log_level=DEBUG_LEVEL;"
287 "dump_malloc_stats=false;"
289 "avoid_flush_during_recovery=false;"
290 "avoid_flush_during_shutdown=false;"
291 "allow_ingest_behind=false;"
292 "preserve_deletes=false;"
293 "concurrent_prepare=false;"
294 "two_write_queues=false;"
295 "manual_wal_flush=false;"
296 "seq_per_batch=false;"
297 "atomic_flush=false;"
298 "avoid_unnecessary_blocking_io=false",
301 ASSERT_EQ(unset_bytes_base
, NumUnsetBytes(new_options_ptr
, sizeof(DBOptions
),
302 kDBOptionsBlacklist
));
304 options
->~DBOptions();
305 new_options
->~DBOptions();
307 delete[] options_ptr
;
308 delete[] new_options_ptr
;
311 template <typename T1
, typename T2
>
312 inline int offset_of(T1
T2::*member
) {
314 return int(size_t(&(obj
.*member
)) - size_t(&obj
));
317 // If the test fails, likely a new option is added to ColumnFamilyOptions
318 // but it cannot be set through GetColumnFamilyOptionsFromString(), or the
319 // test is not updated accordingly.
320 // After adding an option, we need to make sure it is settable by
321 // GetColumnFamilyOptionsFromString() and add the option to the input
322 // string passed to GetColumnFamilyOptionsFromString()in this test.
323 // If it is a complicated type, you also need to add the field to
324 // kColumnFamilyOptionsBlacklist, and maybe add customized verification
326 TEST_F(OptionsSettableTest
, ColumnFamilyOptionsAllFieldsSettable
) {
327 // options in the blacklist need to appear in the same order as in
328 // ColumnFamilyOptions.
329 const OffsetGap kColumnFamilyOptionsBlacklist
= {
330 {offset_of(&ColumnFamilyOptions::inplace_callback
),
331 sizeof(UpdateStatus(*)(char*, uint32_t*, Slice
, std::string
*))},
333 &ColumnFamilyOptions::memtable_insert_with_hint_prefix_extractor
),
334 sizeof(std::shared_ptr
<const SliceTransform
>)},
335 {offset_of(&ColumnFamilyOptions::compression_per_level
),
336 sizeof(std::vector
<CompressionType
>)},
338 &ColumnFamilyOptions::max_bytes_for_level_multiplier_additional
),
339 sizeof(std::vector
<int>)},
340 {offset_of(&ColumnFamilyOptions::memtable_factory
),
341 sizeof(std::shared_ptr
<MemTableRepFactory
>)},
342 {offset_of(&ColumnFamilyOptions::table_properties_collector_factories
),
343 sizeof(ColumnFamilyOptions::TablePropertiesCollectorFactories
)},
344 {offset_of(&ColumnFamilyOptions::comparator
), sizeof(Comparator
*)},
345 {offset_of(&ColumnFamilyOptions::merge_operator
),
346 sizeof(std::shared_ptr
<MergeOperator
>)},
347 {offset_of(&ColumnFamilyOptions::compaction_filter
),
348 sizeof(const CompactionFilter
*)},
349 {offset_of(&ColumnFamilyOptions::compaction_filter_factory
),
350 sizeof(std::shared_ptr
<CompactionFilterFactory
>)},
351 {offset_of(&ColumnFamilyOptions::prefix_extractor
),
352 sizeof(std::shared_ptr
<const SliceTransform
>)},
353 {offset_of(&ColumnFamilyOptions::table_factory
),
354 sizeof(std::shared_ptr
<TableFactory
>)},
355 {offset_of(&ColumnFamilyOptions::cf_paths
),
356 sizeof(std::vector
<DbPath
>)},
357 {offset_of(&ColumnFamilyOptions::compaction_thread_limiter
),
358 sizeof(std::shared_ptr
<ConcurrentTaskLimiter
>)},
361 char* options_ptr
= new char[sizeof(ColumnFamilyOptions
)];
363 // Count padding bytes by setting all bytes in the memory to a special char,
364 // copy a well constructed struct to this memory and see how many special
366 ColumnFamilyOptions
* options
= new (options_ptr
) ColumnFamilyOptions();
367 FillWithSpecialChar(options_ptr
, sizeof(ColumnFamilyOptions
),
368 kColumnFamilyOptionsBlacklist
);
369 // It based on the behavior of compiler that padding bytes are not changed
370 // when copying the struct. It's prone to failure when compiler behavior
371 // changes. We verify there is unset bytes to detect the case.
372 *options
= ColumnFamilyOptions();
374 // Deprecatd option which is not initialized. Need to set it to avoid
376 options
->max_mem_compaction_level
= 0;
378 int unset_bytes_base
= NumUnsetBytes(options_ptr
, sizeof(ColumnFamilyOptions
),
379 kColumnFamilyOptionsBlacklist
);
380 ASSERT_GT(unset_bytes_base
, 0);
381 options
->~ColumnFamilyOptions();
383 options
= new (options_ptr
) ColumnFamilyOptions();
384 FillWithSpecialChar(options_ptr
, sizeof(ColumnFamilyOptions
),
385 kColumnFamilyOptionsBlacklist
);
387 // Following options are not settable through
388 // GetColumnFamilyOptionsFromString():
389 options
->rate_limit_delay_max_milliseconds
= 33;
390 options
->compaction_options_universal
= CompactionOptionsUniversal();
391 options
->compression_opts
= CompressionOptions();
392 options
->bottommost_compression_opts
= CompressionOptions();
393 options
->hard_rate_limit
= 0;
394 options
->soft_rate_limit
= 0;
395 options
->purge_redundant_kvs_while_flush
= false;
396 options
->max_mem_compaction_level
= 0;
397 options
->compaction_filter
= nullptr;
399 char* new_options_ptr
= new char[sizeof(ColumnFamilyOptions
)];
400 ColumnFamilyOptions
* new_options
=
401 new (new_options_ptr
) ColumnFamilyOptions();
402 FillWithSpecialChar(new_options_ptr
, sizeof(ColumnFamilyOptions
),
403 kColumnFamilyOptionsBlacklist
);
405 // Need to update the option string if a new option is added.
406 ASSERT_OK(GetColumnFamilyOptionsFromString(
408 "compaction_filter_factory=mpudlojcujCompactionFilterFactory;"
409 "table_factory=PlainTable;"
410 "prefix_extractor=rocksdb.CappedPrefix.13;"
411 "comparator=leveldb.BytewiseComparator;"
412 "compression_per_level=kBZip2Compression:kBZip2Compression:"
413 "kBZip2Compression:kNoCompression:kZlibCompression:kBZip2Compression:"
414 "kSnappyCompression;"
415 "max_bytes_for_level_base=986;"
416 "bloom_locality=8016;"
417 "target_file_size_base=4294976376;"
418 "memtable_huge_page_size=2557;"
419 "max_successive_merges=5497;"
420 "max_sequential_skip_in_iterations=4294971408;"
421 "arena_block_size=1893;"
422 "target_file_size_multiplier=35;"
423 "min_write_buffer_number_to_merge=9;"
424 "max_write_buffer_number=84;"
425 "write_buffer_size=1653;"
426 "max_compaction_bytes=64;"
427 "max_bytes_for_level_multiplier=60;"
428 "memtable_factory=SkipListFactory;"
429 "compression=kNoCompression;"
430 "bottommost_compression=kDisableCompressionOption;"
431 "level0_stop_writes_trigger=33;"
433 "level0_slowdown_writes_trigger=22;"
434 "level0_file_num_compaction_trigger=14;"
435 "compaction_filter=urxcqstuwnCompactionFilter;"
436 "soft_rate_limit=530.615385;"
437 "soft_pending_compaction_bytes_limit=0;"
438 "max_write_buffer_number_to_maintain=84;"
439 "merge_operator=aabcxehazrMergeOperator;"
440 "memtable_prefix_bloom_size_ratio=0.4642;"
441 "memtable_whole_key_filtering=true;"
442 "memtable_insert_with_hint_prefix_extractor=rocksdb.CappedPrefix.13;"
443 "paranoid_file_checks=true;"
444 "force_consistency_checks=true;"
445 "inplace_update_num_locks=7429;"
446 "optimize_filters_for_hits=false;"
447 "level_compaction_dynamic_level_bytes=false;"
448 "inplace_update_support=false;"
449 "compaction_style=kCompactionStyleFIFO;"
450 "compaction_pri=kMinOverlappingRatio;"
451 "hard_pending_compaction_bytes_limit=0;"
452 "disable_auto_compactions=false;"
453 "report_bg_io_stats=true;"
455 "sample_for_compression=0;"
456 "compaction_options_fifo={max_table_files_size=3;allow_"
457 "compaction=false;};",
460 ASSERT_EQ(unset_bytes_base
,
461 NumUnsetBytes(new_options_ptr
, sizeof(ColumnFamilyOptions
),
462 kColumnFamilyOptionsBlacklist
));
464 options
->~ColumnFamilyOptions();
465 new_options
->~ColumnFamilyOptions();
467 delete[] options_ptr
;
468 delete[] new_options_ptr
;
471 #endif // OS_LINUX || OS_WIN
472 #endif // !ROCKSDB_LITE
474 } // namespace rocksdb
476 int main(int argc
, char** argv
) {
477 ::testing::InitGoogleTest(&argc
, argv
);
479 ParseCommandLineFlags(&argc
, &argv
, true);
481 return RUN_ALL_TESTS();