]>
Commit | Line | Data |
---|---|---|
7c673cae | 1 | // Copyright (c) 2011-present, Facebook, Inc. All rights reserved. |
11fdf7f2 TL |
2 | // This source code is licensed under both the GPLv2 (found in the |
3 | // COPYING file in the root directory) and Apache 2.0 License | |
4 | // (found in the LICENSE.Apache file in the root directory). | |
7c673cae FG |
5 | // |
6 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. | |
7 | // Use of this source code is governed by a BSD-style license that can be | |
8 | // found in the LICENSE file. See the AUTHORS file for names of contributors. | |
9 | ||
7c673cae | 10 | #include <cstring> |
7c673cae | 11 | |
20effc67 TL |
12 | #include "options/cf_options.h" |
13 | #include "options/db_options.h" | |
7c673cae | 14 | #include "options/options_helper.h" |
7c673cae | 15 | #include "rocksdb/convenience.h" |
f67539c2 | 16 | #include "test_util/testharness.h" |
7c673cae FG |
17 | |
18 | #ifndef GFLAGS | |
19 | bool FLAGS_enable_print = false; | |
20 | #else | |
11fdf7f2 TL |
21 | #include "util/gflags_compat.h" |
22 | using GFLAGS_NAMESPACE::ParseCommandLineFlags; | |
7c673cae FG |
23 | DEFINE_bool(enable_print, false, "Print options generated to console."); |
24 | #endif // GFLAGS | |
25 | ||
f67539c2 | 26 | namespace ROCKSDB_NAMESPACE { |
7c673cae FG |
27 | |
28 | // Verify options are settable from options strings. | |
29 | // We take the approach that depends on compiler behavior that copy constructor | |
30 | // won't touch implicit padding bytes, so that the test is fragile. | |
31 | // As a result, we only run the tests to verify new fields in options are | |
32 | // settable through string on limited platforms as it depends on behavior of | |
33 | // compilers. | |
34 | #ifndef ROCKSDB_LITE | |
11fdf7f2 | 35 | #if defined OS_LINUX || defined OS_WIN |
7c673cae FG |
36 | #ifndef __clang__ |
37 | ||
38 | class OptionsSettableTest : public testing::Test { | |
39 | public: | |
40 | OptionsSettableTest() {} | |
41 | }; | |
42 | ||
43 | const char kSpecialChar = 'z'; | |
11fdf7f2 | 44 | typedef std::vector<std::pair<size_t, size_t>> OffsetGap; |
7c673cae FG |
45 | |
46 | void FillWithSpecialChar(char* start_ptr, size_t total_size, | |
20effc67 TL |
47 | const OffsetGap& excluded, |
48 | char special_char = kSpecialChar) { | |
7c673cae | 49 | size_t offset = 0; |
20effc67 TL |
50 | for (auto& pair : excluded) { |
51 | std::memset(start_ptr + offset, special_char, pair.first - offset); | |
7c673cae FG |
52 | offset = pair.first + pair.second; |
53 | } | |
20effc67 | 54 | std::memset(start_ptr + offset, special_char, total_size - offset); |
7c673cae FG |
55 | } |
56 | ||
57 | int NumUnsetBytes(char* start_ptr, size_t total_size, | |
20effc67 | 58 | const OffsetGap& excluded) { |
7c673cae FG |
59 | int total_unset_bytes_base = 0; |
60 | size_t offset = 0; | |
20effc67 | 61 | for (auto& pair : excluded) { |
7c673cae FG |
62 | for (char* ptr = start_ptr + offset; ptr < start_ptr + pair.first; ptr++) { |
63 | if (*ptr == kSpecialChar) { | |
64 | total_unset_bytes_base++; | |
65 | } | |
66 | } | |
67 | offset = pair.first + pair.second; | |
68 | } | |
69 | for (char* ptr = start_ptr + offset; ptr < start_ptr + total_size; ptr++) { | |
70 | if (*ptr == kSpecialChar) { | |
71 | total_unset_bytes_base++; | |
72 | } | |
73 | } | |
74 | return total_unset_bytes_base; | |
75 | } | |
76 | ||
20effc67 TL |
77 | // Return true iff two structs are the same except excluded fields. |
78 | bool CompareBytes(char* start_ptr1, char* start_ptr2, size_t total_size, | |
79 | const OffsetGap& excluded) { | |
80 | size_t offset = 0; | |
81 | for (auto& pair : excluded) { | |
82 | for (; offset < pair.first; offset++) { | |
83 | if (*(start_ptr1 + offset) != *(start_ptr2 + offset)) { | |
84 | return false; | |
85 | } | |
86 | } | |
87 | offset = pair.first + pair.second; | |
88 | } | |
89 | for (; offset < total_size; offset++) { | |
90 | if (*(start_ptr1 + offset) != *(start_ptr2 + offset)) { | |
91 | return false; | |
92 | } | |
93 | } | |
94 | return true; | |
95 | } | |
96 | ||
7c673cae FG |
97 | // If the test fails, likely a new option is added to BlockBasedTableOptions |
98 | // but it cannot be set through GetBlockBasedTableOptionsFromString(), or the | |
99 | // test is not updated accordingly. | |
100 | // After adding an option, we need to make sure it is settable by | |
101 | // GetBlockBasedTableOptionsFromString() and add the option to the input string | |
102 | // passed to the GetBlockBasedTableOptionsFromString() in this test. | |
103 | // If it is a complicated type, you also need to add the field to | |
20effc67 | 104 | // kBbtoExcluded, and maybe add customized verification for it. |
7c673cae FG |
105 | TEST_F(OptionsSettableTest, BlockBasedTableOptionsAllFieldsSettable) { |
106 | // Items in the form of <offset, size>. Need to be in ascending order | |
107 | // and not overlapping. Need to updated if new pointer-option is added. | |
20effc67 | 108 | const OffsetGap kBbtoExcluded = { |
7c673cae FG |
109 | {offsetof(struct BlockBasedTableOptions, flush_block_policy_factory), |
110 | sizeof(std::shared_ptr<FlushBlockPolicyFactory>)}, | |
111 | {offsetof(struct BlockBasedTableOptions, block_cache), | |
112 | sizeof(std::shared_ptr<Cache>)}, | |
113 | {offsetof(struct BlockBasedTableOptions, persistent_cache), | |
114 | sizeof(std::shared_ptr<PersistentCache>)}, | |
115 | {offsetof(struct BlockBasedTableOptions, block_cache_compressed), | |
116 | sizeof(std::shared_ptr<Cache>)}, | |
117 | {offsetof(struct BlockBasedTableOptions, filter_policy), | |
118 | sizeof(std::shared_ptr<const FilterPolicy>)}, | |
119 | }; | |
120 | ||
121 | // In this test, we catch a new option of BlockBasedTableOptions that is not | |
122 | // settable through GetBlockBasedTableOptionsFromString(). | |
123 | // We count padding bytes of the option struct, and assert it to be the same | |
124 | // as unset bytes of an option struct initialized by | |
125 | // GetBlockBasedTableOptionsFromString(). | |
126 | ||
127 | char* bbto_ptr = new char[sizeof(BlockBasedTableOptions)]; | |
128 | ||
129 | // Count padding bytes by setting all bytes in the memory to a special char, | |
130 | // copy a well constructed struct to this memory and see how many special | |
131 | // bytes left. | |
132 | BlockBasedTableOptions* bbto = new (bbto_ptr) BlockBasedTableOptions(); | |
20effc67 | 133 | FillWithSpecialChar(bbto_ptr, sizeof(BlockBasedTableOptions), kBbtoExcluded); |
7c673cae FG |
134 | // It based on the behavior of compiler that padding bytes are not changed |
135 | // when copying the struct. It's prone to failure when compiler behavior | |
136 | // changes. We verify there is unset bytes to detect the case. | |
137 | *bbto = BlockBasedTableOptions(); | |
138 | int unset_bytes_base = | |
20effc67 | 139 | NumUnsetBytes(bbto_ptr, sizeof(BlockBasedTableOptions), kBbtoExcluded); |
7c673cae FG |
140 | ASSERT_GT(unset_bytes_base, 0); |
141 | bbto->~BlockBasedTableOptions(); | |
142 | ||
143 | // Construct the base option passed into | |
144 | // GetBlockBasedTableOptionsFromString(). | |
145 | bbto = new (bbto_ptr) BlockBasedTableOptions(); | |
20effc67 | 146 | FillWithSpecialChar(bbto_ptr, sizeof(BlockBasedTableOptions), kBbtoExcluded); |
7c673cae FG |
147 | // This option is not setable: |
148 | bbto->use_delta_encoding = true; | |
149 | ||
150 | char* new_bbto_ptr = new char[sizeof(BlockBasedTableOptions)]; | |
151 | BlockBasedTableOptions* new_bbto = | |
152 | new (new_bbto_ptr) BlockBasedTableOptions(); | |
153 | FillWithSpecialChar(new_bbto_ptr, sizeof(BlockBasedTableOptions), | |
20effc67 | 154 | kBbtoExcluded); |
7c673cae FG |
155 | |
156 | // Need to update the option string if a new option is added. | |
157 | ASSERT_OK(GetBlockBasedTableOptionsFromString( | |
158 | *bbto, | |
159 | "cache_index_and_filter_blocks=1;" | |
160 | "cache_index_and_filter_blocks_with_high_priority=true;" | |
20effc67 TL |
161 | "metadata_cache_options={top_level_index_pinning=kFallback;" |
162 | "partition_pinning=kAll;" | |
163 | "unpartitioned_pinning=kFlushedAndSimilar;};" | |
7c673cae | 164 | "pin_l0_filter_and_index_blocks_in_cache=1;" |
11fdf7f2 | 165 | "pin_top_level_index_and_filter=1;" |
7c673cae | 166 | "index_type=kHashSearch;" |
11fdf7f2 | 167 | "data_block_index_type=kDataBlockBinaryAndHash;" |
f67539c2 | 168 | "index_shortening=kNoShortening;" |
11fdf7f2 | 169 | "data_block_hash_table_util_ratio=0.75;" |
7c673cae FG |
170 | "checksum=kxxHash;hash_index_allow_collision=1;no_block_cache=1;" |
171 | "block_cache=1M;block_cache_compressed=1k;block_size=1024;" | |
172 | "block_size_deviation=8;block_restart_interval=4; " | |
173 | "metadata_block_size=1024;" | |
174 | "partition_filters=false;" | |
20effc67 | 175 | "optimize_filters_for_memory=true;" |
7c673cae FG |
176 | "index_block_restart_interval=4;" |
177 | "filter_policy=bloomfilter:4:true;whole_key_filtering=1;" | |
178 | "format_version=1;" | |
179 | "hash_index_allow_collision=false;" | |
11fdf7f2 TL |
180 | "verify_compression=true;read_amp_bytes_per_bit=0;" |
181 | "enable_index_compression=false;" | |
182 | "block_align=true", | |
7c673cae FG |
183 | new_bbto)); |
184 | ||
185 | ASSERT_EQ(unset_bytes_base, | |
186 | NumUnsetBytes(new_bbto_ptr, sizeof(BlockBasedTableOptions), | |
20effc67 | 187 | kBbtoExcluded)); |
7c673cae FG |
188 | |
189 | ASSERT_TRUE(new_bbto->block_cache.get() != nullptr); | |
190 | ASSERT_TRUE(new_bbto->block_cache_compressed.get() != nullptr); | |
191 | ASSERT_TRUE(new_bbto->filter_policy.get() != nullptr); | |
192 | ||
193 | bbto->~BlockBasedTableOptions(); | |
194 | new_bbto->~BlockBasedTableOptions(); | |
195 | ||
196 | delete[] bbto_ptr; | |
197 | delete[] new_bbto_ptr; | |
198 | } | |
199 | ||
200 | // If the test fails, likely a new option is added to DBOptions | |
201 | // but it cannot be set through GetDBOptionsFromString(), or the test is not | |
202 | // updated accordingly. | |
203 | // After adding an option, we need to make sure it is settable by | |
204 | // GetDBOptionsFromString() and add the option to the input string passed to | |
205 | // DBOptionsFromString()in this test. | |
206 | // If it is a complicated type, you also need to add the field to | |
20effc67 | 207 | // kDBOptionsExcluded, and maybe add customized verification for it. |
7c673cae | 208 | TEST_F(OptionsSettableTest, DBOptionsAllFieldsSettable) { |
20effc67 | 209 | const OffsetGap kDBOptionsExcluded = { |
7c673cae FG |
210 | {offsetof(struct DBOptions, env), sizeof(Env*)}, |
211 | {offsetof(struct DBOptions, rate_limiter), | |
212 | sizeof(std::shared_ptr<RateLimiter>)}, | |
213 | {offsetof(struct DBOptions, sst_file_manager), | |
214 | sizeof(std::shared_ptr<SstFileManager>)}, | |
215 | {offsetof(struct DBOptions, info_log), sizeof(std::shared_ptr<Logger>)}, | |
216 | {offsetof(struct DBOptions, statistics), | |
217 | sizeof(std::shared_ptr<Statistics>)}, | |
218 | {offsetof(struct DBOptions, db_paths), sizeof(std::vector<DbPath>)}, | |
219 | {offsetof(struct DBOptions, db_log_dir), sizeof(std::string)}, | |
220 | {offsetof(struct DBOptions, wal_dir), sizeof(std::string)}, | |
221 | {offsetof(struct DBOptions, write_buffer_manager), | |
222 | sizeof(std::shared_ptr<WriteBufferManager>)}, | |
223 | {offsetof(struct DBOptions, listeners), | |
224 | sizeof(std::vector<std::shared_ptr<EventListener>>)}, | |
225 | {offsetof(struct DBOptions, row_cache), sizeof(std::shared_ptr<Cache>)}, | |
226 | {offsetof(struct DBOptions, wal_filter), sizeof(const WalFilter*)}, | |
20effc67 TL |
227 | {offsetof(struct DBOptions, file_checksum_gen_factory), |
228 | sizeof(std::shared_ptr<FileChecksumGenFactory>)}, | |
229 | {offsetof(struct DBOptions, db_host_id), sizeof(std::string)}, | |
7c673cae FG |
230 | }; |
231 | ||
232 | char* options_ptr = new char[sizeof(DBOptions)]; | |
233 | ||
234 | // Count padding bytes by setting all bytes in the memory to a special char, | |
235 | // copy a well constructed struct to this memory and see how many special | |
236 | // bytes left. | |
237 | DBOptions* options = new (options_ptr) DBOptions(); | |
20effc67 | 238 | FillWithSpecialChar(options_ptr, sizeof(DBOptions), kDBOptionsExcluded); |
7c673cae FG |
239 | // It based on the behavior of compiler that padding bytes are not changed |
240 | // when copying the struct. It's prone to failure when compiler behavior | |
241 | // changes. We verify there is unset bytes to detect the case. | |
242 | *options = DBOptions(); | |
243 | int unset_bytes_base = | |
20effc67 | 244 | NumUnsetBytes(options_ptr, sizeof(DBOptions), kDBOptionsExcluded); |
7c673cae FG |
245 | ASSERT_GT(unset_bytes_base, 0); |
246 | options->~DBOptions(); | |
247 | ||
248 | options = new (options_ptr) DBOptions(); | |
20effc67 | 249 | FillWithSpecialChar(options_ptr, sizeof(DBOptions), kDBOptionsExcluded); |
7c673cae FG |
250 | |
251 | char* new_options_ptr = new char[sizeof(DBOptions)]; | |
252 | DBOptions* new_options = new (new_options_ptr) DBOptions(); | |
20effc67 | 253 | FillWithSpecialChar(new_options_ptr, sizeof(DBOptions), kDBOptionsExcluded); |
7c673cae FG |
254 | |
255 | // Need to update the option string if a new option is added. | |
256 | ASSERT_OK( | |
257 | GetDBOptionsFromString(*options, | |
258 | "wal_bytes_per_sync=4295048118;" | |
259 | "delete_obsolete_files_period_micros=4294967758;" | |
260 | "WAL_ttl_seconds=4295008036;" | |
261 | "WAL_size_limit_MB=4295036161;" | |
f67539c2 | 262 | "max_write_batch_group_size_bytes=1048576;" |
7c673cae FG |
263 | "wal_dir=path/to/wal_dir;" |
264 | "db_write_buffer_size=2587;" | |
265 | "max_subcompactions=64330;" | |
266 | "table_cache_numshardbits=28;" | |
267 | "max_open_files=72;" | |
268 | "max_file_opening_threads=35;" | |
11fdf7f2 | 269 | "max_background_jobs=8;" |
7c673cae FG |
270 | "base_background_compactions=3;" |
271 | "max_background_compactions=33;" | |
272 | "use_fsync=true;" | |
273 | "use_adaptive_mutex=false;" | |
274 | "max_total_wal_size=4295005604;" | |
275 | "compaction_readahead_size=0;" | |
276 | "new_table_reader_for_compaction_inputs=false;" | |
277 | "keep_log_file_num=4890;" | |
278 | "skip_stats_update_on_db_open=false;" | |
f67539c2 | 279 | "skip_checking_sst_file_sizes_on_db_open=false;" |
7c673cae FG |
280 | "max_manifest_file_size=4295009941;" |
281 | "db_log_dir=path/to/db_log_dir;" | |
282 | "skip_log_error_on_recovery=true;" | |
283 | "writable_file_max_buffer_size=1048576;" | |
284 | "paranoid_checks=true;" | |
20effc67 | 285 | "track_and_verify_wals_in_manifest=true;" |
7c673cae FG |
286 | "is_fd_close_on_exec=false;" |
287 | "bytes_per_sync=4295013613;" | |
f67539c2 | 288 | "strict_bytes_per_sync=true;" |
7c673cae FG |
289 | "enable_thread_tracking=false;" |
290 | "recycle_log_file_num=0;" | |
291 | "create_missing_column_families=true;" | |
292 | "log_file_time_to_roll=3097;" | |
293 | "max_background_flushes=35;" | |
294 | "create_if_missing=false;" | |
295 | "error_if_exists=true;" | |
296 | "delayed_write_rate=4294976214;" | |
297 | "manifest_preallocation_size=1222;" | |
298 | "allow_mmap_writes=false;" | |
299 | "stats_dump_period_sec=70127;" | |
494da23a | 300 | "stats_persist_period_sec=54321;" |
f67539c2 | 301 | "persist_stats_to_disk=true;" |
494da23a | 302 | "stats_history_buffer_size=14159;" |
7c673cae FG |
303 | "allow_fallocate=true;" |
304 | "allow_mmap_reads=false;" | |
305 | "use_direct_reads=false;" | |
306 | "use_direct_io_for_flush_and_compaction=false;" | |
307 | "max_log_file_size=4607;" | |
308 | "random_access_max_buffer_size=1048576;" | |
309 | "advise_random_on_open=true;" | |
310 | "fail_if_options_file_error=false;" | |
11fdf7f2 | 311 | "enable_pipelined_write=false;" |
f67539c2 | 312 | "unordered_write=false;" |
7c673cae FG |
313 | "allow_concurrent_memtable_write=true;" |
314 | "wal_recovery_mode=kPointInTimeRecovery;" | |
315 | "enable_write_thread_adaptive_yield=true;" | |
316 | "write_thread_slow_yield_usec=5;" | |
317 | "write_thread_max_yield_usec=1000;" | |
318 | "access_hint_on_compaction_start=NONE;" | |
319 | "info_log_level=DEBUG_LEVEL;" | |
320 | "dump_malloc_stats=false;" | |
321 | "allow_2pc=false;" | |
322 | "avoid_flush_during_recovery=false;" | |
11fdf7f2 TL |
323 | "avoid_flush_during_shutdown=false;" |
324 | "allow_ingest_behind=false;" | |
325 | "preserve_deletes=false;" | |
326 | "concurrent_prepare=false;" | |
327 | "two_write_queues=false;" | |
328 | "manual_wal_flush=false;" | |
494da23a TL |
329 | "seq_per_batch=false;" |
330 | "atomic_flush=false;" | |
f67539c2 TL |
331 | "avoid_unnecessary_blocking_io=false;" |
332 | "log_readahead_size=0;" | |
20effc67 TL |
333 | "write_dbid_to_manifest=false;" |
334 | "best_efforts_recovery=false;" | |
335 | "max_bgerror_resume_count=2;" | |
336 | "bgerror_resume_retry_interval=1000000" | |
337 | "db_host_id=hostname;" | |
338 | "allow_data_in_errors=false", | |
7c673cae FG |
339 | new_options)); |
340 | ||
341 | ASSERT_EQ(unset_bytes_base, NumUnsetBytes(new_options_ptr, sizeof(DBOptions), | |
20effc67 | 342 | kDBOptionsExcluded)); |
7c673cae FG |
343 | |
344 | options->~DBOptions(); | |
345 | new_options->~DBOptions(); | |
346 | ||
347 | delete[] options_ptr; | |
348 | delete[] new_options_ptr; | |
349 | } | |
350 | ||
11fdf7f2 TL |
351 | template <typename T1, typename T2> |
352 | inline int offset_of(T1 T2::*member) { | |
353 | static T2 obj; | |
354 | return int(size_t(&(obj.*member)) - size_t(&obj)); | |
355 | } | |
356 | ||
7c673cae FG |
357 | // If the test fails, likely a new option is added to ColumnFamilyOptions |
358 | // but it cannot be set through GetColumnFamilyOptionsFromString(), or the | |
359 | // test is not updated accordingly. | |
360 | // After adding an option, we need to make sure it is settable by | |
361 | // GetColumnFamilyOptionsFromString() and add the option to the input | |
362 | // string passed to GetColumnFamilyOptionsFromString()in this test. | |
363 | // If it is a complicated type, you also need to add the field to | |
20effc67 | 364 | // kColumnFamilyOptionsExcluded, and maybe add customized verification |
7c673cae FG |
365 | // for it. |
366 | TEST_F(OptionsSettableTest, ColumnFamilyOptionsAllFieldsSettable) { | |
20effc67 | 367 | // options in the excluded set need to appear in the same order as in |
7c673cae | 368 | // ColumnFamilyOptions. |
20effc67 | 369 | const OffsetGap kColumnFamilyOptionsExcluded = { |
7c673cae FG |
370 | {offset_of(&ColumnFamilyOptions::inplace_callback), |
371 | sizeof(UpdateStatus(*)(char*, uint32_t*, Slice, std::string*))}, | |
372 | {offset_of( | |
373 | &ColumnFamilyOptions::memtable_insert_with_hint_prefix_extractor), | |
374 | sizeof(std::shared_ptr<const SliceTransform>)}, | |
375 | {offset_of(&ColumnFamilyOptions::compression_per_level), | |
376 | sizeof(std::vector<CompressionType>)}, | |
377 | {offset_of( | |
378 | &ColumnFamilyOptions::max_bytes_for_level_multiplier_additional), | |
379 | sizeof(std::vector<int>)}, | |
380 | {offset_of(&ColumnFamilyOptions::memtable_factory), | |
381 | sizeof(std::shared_ptr<MemTableRepFactory>)}, | |
382 | {offset_of(&ColumnFamilyOptions::table_properties_collector_factories), | |
383 | sizeof(ColumnFamilyOptions::TablePropertiesCollectorFactories)}, | |
384 | {offset_of(&ColumnFamilyOptions::comparator), sizeof(Comparator*)}, | |
385 | {offset_of(&ColumnFamilyOptions::merge_operator), | |
386 | sizeof(std::shared_ptr<MergeOperator>)}, | |
387 | {offset_of(&ColumnFamilyOptions::compaction_filter), | |
388 | sizeof(const CompactionFilter*)}, | |
389 | {offset_of(&ColumnFamilyOptions::compaction_filter_factory), | |
390 | sizeof(std::shared_ptr<CompactionFilterFactory>)}, | |
391 | {offset_of(&ColumnFamilyOptions::prefix_extractor), | |
392 | sizeof(std::shared_ptr<const SliceTransform>)}, | |
f67539c2 | 393 | {offset_of(&ColumnFamilyOptions::snap_refresh_nanos), sizeof(uint64_t)}, |
7c673cae FG |
394 | {offset_of(&ColumnFamilyOptions::table_factory), |
395 | sizeof(std::shared_ptr<TableFactory>)}, | |
f67539c2 | 396 | {offset_of(&ColumnFamilyOptions::cf_paths), sizeof(std::vector<DbPath>)}, |
494da23a TL |
397 | {offset_of(&ColumnFamilyOptions::compaction_thread_limiter), |
398 | sizeof(std::shared_ptr<ConcurrentTaskLimiter>)}, | |
20effc67 TL |
399 | {offset_of(&ColumnFamilyOptions::sst_partitioner_factory), |
400 | sizeof(std::shared_ptr<SstPartitionerFactory>)}, | |
7c673cae FG |
401 | }; |
402 | ||
403 | char* options_ptr = new char[sizeof(ColumnFamilyOptions)]; | |
404 | ||
405 | // Count padding bytes by setting all bytes in the memory to a special char, | |
406 | // copy a well constructed struct to this memory and see how many special | |
407 | // bytes left. | |
408 | ColumnFamilyOptions* options = new (options_ptr) ColumnFamilyOptions(); | |
409 | FillWithSpecialChar(options_ptr, sizeof(ColumnFamilyOptions), | |
20effc67 TL |
410 | kColumnFamilyOptionsExcluded); |
411 | ||
7c673cae FG |
412 | // It based on the behavior of compiler that padding bytes are not changed |
413 | // when copying the struct. It's prone to failure when compiler behavior | |
414 | // changes. We verify there is unset bytes to detect the case. | |
415 | *options = ColumnFamilyOptions(); | |
416 | ||
417 | // Deprecatd option which is not initialized. Need to set it to avoid | |
418 | // Valgrind error | |
419 | options->max_mem_compaction_level = 0; | |
420 | ||
421 | int unset_bytes_base = NumUnsetBytes(options_ptr, sizeof(ColumnFamilyOptions), | |
20effc67 | 422 | kColumnFamilyOptionsExcluded); |
7c673cae FG |
423 | ASSERT_GT(unset_bytes_base, 0); |
424 | options->~ColumnFamilyOptions(); | |
425 | ||
426 | options = new (options_ptr) ColumnFamilyOptions(); | |
427 | FillWithSpecialChar(options_ptr, sizeof(ColumnFamilyOptions), | |
20effc67 | 428 | kColumnFamilyOptionsExcluded); |
7c673cae FG |
429 | |
430 | // Following options are not settable through | |
431 | // GetColumnFamilyOptionsFromString(): | |
432 | options->rate_limit_delay_max_milliseconds = 33; | |
433 | options->compaction_options_universal = CompactionOptionsUniversal(); | |
7c673cae FG |
434 | options->hard_rate_limit = 0; |
435 | options->soft_rate_limit = 0; | |
20effc67 | 436 | options->num_levels = 42; // Initialize options for MutableCF |
11fdf7f2 | 437 | options->purge_redundant_kvs_while_flush = false; |
7c673cae | 438 | options->max_mem_compaction_level = 0; |
494da23a | 439 | options->compaction_filter = nullptr; |
20effc67 | 440 | options->sst_partitioner_factory = nullptr; |
7c673cae FG |
441 | |
442 | char* new_options_ptr = new char[sizeof(ColumnFamilyOptions)]; | |
443 | ColumnFamilyOptions* new_options = | |
444 | new (new_options_ptr) ColumnFamilyOptions(); | |
445 | FillWithSpecialChar(new_options_ptr, sizeof(ColumnFamilyOptions), | |
20effc67 | 446 | kColumnFamilyOptionsExcluded); |
7c673cae FG |
447 | |
448 | // Need to update the option string if a new option is added. | |
449 | ASSERT_OK(GetColumnFamilyOptionsFromString( | |
450 | *options, | |
451 | "compaction_filter_factory=mpudlojcujCompactionFilterFactory;" | |
452 | "table_factory=PlainTable;" | |
453 | "prefix_extractor=rocksdb.CappedPrefix.13;" | |
454 | "comparator=leveldb.BytewiseComparator;" | |
455 | "compression_per_level=kBZip2Compression:kBZip2Compression:" | |
456 | "kBZip2Compression:kNoCompression:kZlibCompression:kBZip2Compression:" | |
457 | "kSnappyCompression;" | |
458 | "max_bytes_for_level_base=986;" | |
459 | "bloom_locality=8016;" | |
460 | "target_file_size_base=4294976376;" | |
461 | "memtable_huge_page_size=2557;" | |
462 | "max_successive_merges=5497;" | |
463 | "max_sequential_skip_in_iterations=4294971408;" | |
464 | "arena_block_size=1893;" | |
465 | "target_file_size_multiplier=35;" | |
466 | "min_write_buffer_number_to_merge=9;" | |
467 | "max_write_buffer_number=84;" | |
468 | "write_buffer_size=1653;" | |
469 | "max_compaction_bytes=64;" | |
470 | "max_bytes_for_level_multiplier=60;" | |
471 | "memtable_factory=SkipListFactory;" | |
472 | "compression=kNoCompression;" | |
20effc67 TL |
473 | "compression_opts=5:6:7:8:9:true;" |
474 | "bottommost_compression_opts=4:5:6:7:8:true;" | |
7c673cae FG |
475 | "bottommost_compression=kDisableCompressionOption;" |
476 | "level0_stop_writes_trigger=33;" | |
477 | "num_levels=99;" | |
478 | "level0_slowdown_writes_trigger=22;" | |
479 | "level0_file_num_compaction_trigger=14;" | |
480 | "compaction_filter=urxcqstuwnCompactionFilter;" | |
481 | "soft_rate_limit=530.615385;" | |
482 | "soft_pending_compaction_bytes_limit=0;" | |
483 | "max_write_buffer_number_to_maintain=84;" | |
f67539c2 | 484 | "max_write_buffer_size_to_maintain=2147483648;" |
7c673cae FG |
485 | "merge_operator=aabcxehazrMergeOperator;" |
486 | "memtable_prefix_bloom_size_ratio=0.4642;" | |
494da23a | 487 | "memtable_whole_key_filtering=true;" |
7c673cae | 488 | "memtable_insert_with_hint_prefix_extractor=rocksdb.CappedPrefix.13;" |
20effc67 | 489 | "check_flush_compaction_key_order=false;" |
7c673cae FG |
490 | "paranoid_file_checks=true;" |
491 | "force_consistency_checks=true;" | |
492 | "inplace_update_num_locks=7429;" | |
493 | "optimize_filters_for_hits=false;" | |
494 | "level_compaction_dynamic_level_bytes=false;" | |
495 | "inplace_update_support=false;" | |
496 | "compaction_style=kCompactionStyleFIFO;" | |
497 | "compaction_pri=kMinOverlappingRatio;" | |
7c673cae FG |
498 | "hard_pending_compaction_bytes_limit=0;" |
499 | "disable_auto_compactions=false;" | |
11fdf7f2 TL |
500 | "report_bg_io_stats=true;" |
501 | "ttl=60;" | |
f67539c2 | 502 | "periodic_compaction_seconds=3600;" |
494da23a | 503 | "sample_for_compression=0;" |
20effc67 TL |
504 | "enable_blob_files=true;" |
505 | "min_blob_size=256;" | |
506 | "blob_file_size=1000000;" | |
507 | "blob_compression_type=kBZip2Compression;" | |
508 | "enable_blob_garbage_collection=true;" | |
509 | "blob_garbage_collection_age_cutoff=0.5;" | |
494da23a | 510 | "compaction_options_fifo={max_table_files_size=3;allow_" |
11fdf7f2 | 511 | "compaction=false;};", |
7c673cae FG |
512 | new_options)); |
513 | ||
514 | ASSERT_EQ(unset_bytes_base, | |
515 | NumUnsetBytes(new_options_ptr, sizeof(ColumnFamilyOptions), | |
20effc67 TL |
516 | kColumnFamilyOptionsExcluded)); |
517 | ||
518 | ColumnFamilyOptions rnd_filled_options = *new_options; | |
7c673cae FG |
519 | |
520 | options->~ColumnFamilyOptions(); | |
521 | new_options->~ColumnFamilyOptions(); | |
522 | ||
523 | delete[] options_ptr; | |
524 | delete[] new_options_ptr; | |
20effc67 TL |
525 | |
526 | // Test copying to mutabable and immutable options and copy back the mutable | |
527 | // part. | |
528 | const OffsetGap kMutableCFOptionsExcluded = { | |
529 | {offset_of(&MutableCFOptions::prefix_extractor), | |
530 | sizeof(std::shared_ptr<const SliceTransform>)}, | |
531 | {offset_of(&MutableCFOptions::max_bytes_for_level_multiplier_additional), | |
532 | sizeof(std::vector<int>)}, | |
533 | {offset_of(&MutableCFOptions::max_file_size), | |
534 | sizeof(std::vector<uint64_t>)}, | |
535 | }; | |
536 | ||
537 | // For all memory used for options, pre-fill every char. Otherwise, the | |
538 | // padding bytes might be different so that byte-wise comparison doesn't | |
539 | // general equal results even if objects are equal. | |
540 | const char kMySpecialChar = 'x'; | |
541 | char* mcfo1_ptr = new char[sizeof(MutableCFOptions)]; | |
542 | FillWithSpecialChar(mcfo1_ptr, sizeof(MutableCFOptions), | |
543 | kMutableCFOptionsExcluded, kMySpecialChar); | |
544 | char* mcfo2_ptr = new char[sizeof(MutableCFOptions)]; | |
545 | FillWithSpecialChar(mcfo2_ptr, sizeof(MutableCFOptions), | |
546 | kMutableCFOptionsExcluded, kMySpecialChar); | |
547 | ||
548 | // A clean column family options is constructed after filling the same special | |
549 | // char as the initial one. So that the padding bytes are the same. | |
550 | char* cfo_clean_ptr = new char[sizeof(ColumnFamilyOptions)]; | |
551 | FillWithSpecialChar(cfo_clean_ptr, sizeof(ColumnFamilyOptions), | |
552 | kColumnFamilyOptionsExcluded); | |
553 | rnd_filled_options.num_levels = 66; | |
554 | ColumnFamilyOptions* cfo_clean = new (cfo_clean_ptr) ColumnFamilyOptions(); | |
555 | ||
556 | MutableCFOptions* mcfo1 = | |
557 | new (mcfo1_ptr) MutableCFOptions(rnd_filled_options); | |
558 | ColumnFamilyOptions cfo_back = BuildColumnFamilyOptions(*cfo_clean, *mcfo1); | |
559 | MutableCFOptions* mcfo2 = new (mcfo2_ptr) MutableCFOptions(cfo_back); | |
560 | ||
561 | ASSERT_TRUE(CompareBytes(mcfo1_ptr, mcfo2_ptr, sizeof(MutableCFOptions), | |
562 | kMutableCFOptionsExcluded)); | |
563 | ||
564 | cfo_clean->~ColumnFamilyOptions(); | |
565 | mcfo1->~MutableCFOptions(); | |
566 | mcfo2->~MutableCFOptions(); | |
567 | delete[] mcfo1_ptr; | |
568 | delete[] mcfo2_ptr; | |
569 | delete[] cfo_clean_ptr; | |
7c673cae FG |
570 | } |
571 | #endif // !__clang__ | |
11fdf7f2 | 572 | #endif // OS_LINUX || OS_WIN |
7c673cae FG |
573 | #endif // !ROCKSDB_LITE |
574 | ||
f67539c2 | 575 | } // namespace ROCKSDB_NAMESPACE |
7c673cae FG |
576 | |
577 | int main(int argc, char** argv) { | |
578 | ::testing::InitGoogleTest(&argc, argv); | |
579 | #ifdef GFLAGS | |
580 | ParseCommandLineFlags(&argc, &argv, true); | |
581 | #endif // GFLAGS | |
582 | return RUN_ALL_TESTS(); | |
583 | } |