]>
Commit | Line | Data |
---|---|---|
7c673cae | 1 | // Copyright (c) 2011-present, Facebook, Inc. All rights reserved. |
11fdf7f2 TL |
2 | // This source code is licensed under both the GPLv2 (found in the |
3 | // COPYING file in the root directory) and Apache 2.0 License | |
4 | // (found in the LICENSE.Apache file in the root directory). | |
7c673cae FG |
5 | // |
6 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. | |
7 | // Use of this source code is governed by a BSD-style license that can be | |
8 | // found in the LICENSE file. See the AUTHORS file for names of contributors. | |
9 | ||
10 | #ifndef __STDC_FORMAT_MACROS | |
11 | #define __STDC_FORMAT_MACROS | |
12 | #endif | |
13 | ||
14 | #include <cctype> | |
15 | #include <cstring> | |
16 | #include <unordered_map> | |
17 | #include <inttypes.h> | |
18 | ||
11fdf7f2 TL |
19 | #include "cache/lru_cache.h" |
20 | #include "cache/sharded_cache.h" | |
7c673cae FG |
21 | #include "options/options_helper.h" |
22 | #include "options/options_parser.h" | |
23 | #include "options/options_sanity_check.h" | |
24 | #include "rocksdb/cache.h" | |
25 | #include "rocksdb/convenience.h" | |
26 | #include "rocksdb/memtablerep.h" | |
27 | #include "rocksdb/utilities/leveldb_options.h" | |
28 | #include "util/random.h" | |
29 | #include "util/stderr_logger.h" | |
30 | #include "util/string_util.h" | |
31 | #include "util/testharness.h" | |
32 | #include "util/testutil.h" | |
33 | ||
34 | #ifndef GFLAGS | |
35 | bool FLAGS_enable_print = false; | |
36 | #else | |
11fdf7f2 TL |
37 | #include "util/gflags_compat.h" |
38 | using GFLAGS_NAMESPACE::ParseCommandLineFlags; | |
7c673cae FG |
39 | DEFINE_bool(enable_print, false, "Print options generated to console."); |
40 | #endif // GFLAGS | |
41 | ||
42 | namespace rocksdb { | |
43 | ||
44 | class OptionsTest : public testing::Test {}; | |
45 | ||
46 | #ifndef ROCKSDB_LITE // GetOptionsFromMap is not supported in ROCKSDB_LITE | |
47 | TEST_F(OptionsTest, GetOptionsFromMapTest) { | |
48 | std::unordered_map<std::string, std::string> cf_options_map = { | |
49 | {"write_buffer_size", "1"}, | |
50 | {"max_write_buffer_number", "2"}, | |
51 | {"min_write_buffer_number_to_merge", "3"}, | |
52 | {"max_write_buffer_number_to_maintain", "99"}, | |
53 | {"compression", "kSnappyCompression"}, | |
54 | {"compression_per_level", | |
55 | "kNoCompression:" | |
56 | "kSnappyCompression:" | |
57 | "kZlibCompression:" | |
58 | "kBZip2Compression:" | |
59 | "kLZ4Compression:" | |
60 | "kLZ4HCCompression:" | |
61 | "kXpressCompression:" | |
62 | "kZSTD:" | |
63 | "kZSTDNotFinalCompression"}, | |
64 | {"bottommost_compression", "kLZ4Compression"}, | |
11fdf7f2 TL |
65 | {"bottommost_compression_opts", "5:6:7:8:9:true"}, |
66 | {"compression_opts", "4:5:6:7:8:true"}, | |
7c673cae FG |
67 | {"num_levels", "8"}, |
68 | {"level0_file_num_compaction_trigger", "8"}, | |
69 | {"level0_slowdown_writes_trigger", "9"}, | |
70 | {"level0_stop_writes_trigger", "10"}, | |
71 | {"target_file_size_base", "12"}, | |
72 | {"target_file_size_multiplier", "13"}, | |
73 | {"max_bytes_for_level_base", "14"}, | |
74 | {"level_compaction_dynamic_level_bytes", "true"}, | |
75 | {"max_bytes_for_level_multiplier", "15.0"}, | |
76 | {"max_bytes_for_level_multiplier_additional", "16:17:18"}, | |
77 | {"max_compaction_bytes", "21"}, | |
78 | {"soft_rate_limit", "1.1"}, | |
79 | {"hard_rate_limit", "2.1"}, | |
80 | {"hard_pending_compaction_bytes_limit", "211"}, | |
81 | {"arena_block_size", "22"}, | |
82 | {"disable_auto_compactions", "true"}, | |
83 | {"compaction_style", "kCompactionStyleLevel"}, | |
84 | {"compaction_pri", "kOldestSmallestSeqFirst"}, | |
85 | {"verify_checksums_in_compaction", "false"}, | |
86 | {"compaction_options_fifo", "23"}, | |
87 | {"max_sequential_skip_in_iterations", "24"}, | |
88 | {"inplace_update_support", "true"}, | |
89 | {"report_bg_io_stats", "true"}, | |
90 | {"compaction_measure_io_stats", "false"}, | |
91 | {"inplace_update_num_locks", "25"}, | |
92 | {"memtable_prefix_bloom_size_ratio", "0.26"}, | |
93 | {"memtable_huge_page_size", "28"}, | |
94 | {"bloom_locality", "29"}, | |
95 | {"max_successive_merges", "30"}, | |
96 | {"min_partial_merge_operands", "31"}, | |
97 | {"prefix_extractor", "fixed:31"}, | |
98 | {"optimize_filters_for_hits", "true"}, | |
99 | }; | |
100 | ||
101 | std::unordered_map<std::string, std::string> db_options_map = { | |
102 | {"create_if_missing", "false"}, | |
103 | {"create_missing_column_families", "true"}, | |
104 | {"error_if_exists", "false"}, | |
105 | {"paranoid_checks", "true"}, | |
106 | {"max_open_files", "32"}, | |
107 | {"max_total_wal_size", "33"}, | |
108 | {"use_fsync", "true"}, | |
109 | {"db_log_dir", "/db_log_dir"}, | |
110 | {"wal_dir", "/wal_dir"}, | |
111 | {"delete_obsolete_files_period_micros", "34"}, | |
112 | {"max_background_compactions", "35"}, | |
113 | {"max_background_flushes", "36"}, | |
114 | {"max_log_file_size", "37"}, | |
115 | {"log_file_time_to_roll", "38"}, | |
116 | {"keep_log_file_num", "39"}, | |
117 | {"recycle_log_file_num", "5"}, | |
118 | {"max_manifest_file_size", "40"}, | |
119 | {"table_cache_numshardbits", "41"}, | |
120 | {"WAL_ttl_seconds", "43"}, | |
121 | {"WAL_size_limit_MB", "44"}, | |
122 | {"manifest_preallocation_size", "45"}, | |
123 | {"allow_mmap_reads", "true"}, | |
124 | {"allow_mmap_writes", "false"}, | |
125 | {"use_direct_reads", "false"}, | |
126 | {"use_direct_io_for_flush_and_compaction", "false"}, | |
127 | {"is_fd_close_on_exec", "true"}, | |
128 | {"skip_log_error_on_recovery", "false"}, | |
129 | {"stats_dump_period_sec", "46"}, | |
130 | {"advise_random_on_open", "true"}, | |
131 | {"use_adaptive_mutex", "false"}, | |
132 | {"new_table_reader_for_compaction_inputs", "true"}, | |
133 | {"compaction_readahead_size", "100"}, | |
134 | {"random_access_max_buffer_size", "3145728"}, | |
135 | {"writable_file_max_buffer_size", "314159"}, | |
136 | {"bytes_per_sync", "47"}, | |
137 | {"wal_bytes_per_sync", "48"}, | |
138 | }; | |
139 | ||
140 | ColumnFamilyOptions base_cf_opt; | |
141 | ColumnFamilyOptions new_cf_opt; | |
142 | ASSERT_OK(GetColumnFamilyOptionsFromMap( | |
143 | base_cf_opt, cf_options_map, &new_cf_opt)); | |
144 | ASSERT_EQ(new_cf_opt.write_buffer_size, 1U); | |
145 | ASSERT_EQ(new_cf_opt.max_write_buffer_number, 2); | |
146 | ASSERT_EQ(new_cf_opt.min_write_buffer_number_to_merge, 3); | |
147 | ASSERT_EQ(new_cf_opt.max_write_buffer_number_to_maintain, 99); | |
148 | ASSERT_EQ(new_cf_opt.compression, kSnappyCompression); | |
149 | ASSERT_EQ(new_cf_opt.compression_per_level.size(), 9U); | |
150 | ASSERT_EQ(new_cf_opt.compression_per_level[0], kNoCompression); | |
151 | ASSERT_EQ(new_cf_opt.compression_per_level[1], kSnappyCompression); | |
152 | ASSERT_EQ(new_cf_opt.compression_per_level[2], kZlibCompression); | |
153 | ASSERT_EQ(new_cf_opt.compression_per_level[3], kBZip2Compression); | |
154 | ASSERT_EQ(new_cf_opt.compression_per_level[4], kLZ4Compression); | |
155 | ASSERT_EQ(new_cf_opt.compression_per_level[5], kLZ4HCCompression); | |
156 | ASSERT_EQ(new_cf_opt.compression_per_level[6], kXpressCompression); | |
157 | ASSERT_EQ(new_cf_opt.compression_per_level[7], kZSTD); | |
158 | ASSERT_EQ(new_cf_opt.compression_per_level[8], kZSTDNotFinalCompression); | |
159 | ASSERT_EQ(new_cf_opt.compression_opts.window_bits, 4); | |
160 | ASSERT_EQ(new_cf_opt.compression_opts.level, 5); | |
161 | ASSERT_EQ(new_cf_opt.compression_opts.strategy, 6); | |
162 | ASSERT_EQ(new_cf_opt.compression_opts.max_dict_bytes, 7); | |
11fdf7f2 TL |
163 | ASSERT_EQ(new_cf_opt.compression_opts.zstd_max_train_bytes, 8); |
164 | ASSERT_EQ(new_cf_opt.compression_opts.enabled, true); | |
7c673cae | 165 | ASSERT_EQ(new_cf_opt.bottommost_compression, kLZ4Compression); |
11fdf7f2 TL |
166 | ASSERT_EQ(new_cf_opt.bottommost_compression_opts.window_bits, 5); |
167 | ASSERT_EQ(new_cf_opt.bottommost_compression_opts.level, 6); | |
168 | ASSERT_EQ(new_cf_opt.bottommost_compression_opts.strategy, 7); | |
169 | ASSERT_EQ(new_cf_opt.bottommost_compression_opts.max_dict_bytes, 8); | |
170 | ASSERT_EQ(new_cf_opt.bottommost_compression_opts.zstd_max_train_bytes, 9); | |
171 | ASSERT_EQ(new_cf_opt.bottommost_compression_opts.enabled, true); | |
7c673cae FG |
172 | ASSERT_EQ(new_cf_opt.num_levels, 8); |
173 | ASSERT_EQ(new_cf_opt.level0_file_num_compaction_trigger, 8); | |
174 | ASSERT_EQ(new_cf_opt.level0_slowdown_writes_trigger, 9); | |
175 | ASSERT_EQ(new_cf_opt.level0_stop_writes_trigger, 10); | |
176 | ASSERT_EQ(new_cf_opt.target_file_size_base, static_cast<uint64_t>(12)); | |
177 | ASSERT_EQ(new_cf_opt.target_file_size_multiplier, 13); | |
178 | ASSERT_EQ(new_cf_opt.max_bytes_for_level_base, 14U); | |
179 | ASSERT_EQ(new_cf_opt.level_compaction_dynamic_level_bytes, true); | |
180 | ASSERT_EQ(new_cf_opt.max_bytes_for_level_multiplier, 15.0); | |
181 | ASSERT_EQ(new_cf_opt.max_bytes_for_level_multiplier_additional.size(), 3U); | |
182 | ASSERT_EQ(new_cf_opt.max_bytes_for_level_multiplier_additional[0], 16); | |
183 | ASSERT_EQ(new_cf_opt.max_bytes_for_level_multiplier_additional[1], 17); | |
184 | ASSERT_EQ(new_cf_opt.max_bytes_for_level_multiplier_additional[2], 18); | |
185 | ASSERT_EQ(new_cf_opt.max_compaction_bytes, 21); | |
186 | ASSERT_EQ(new_cf_opt.hard_pending_compaction_bytes_limit, 211); | |
187 | ASSERT_EQ(new_cf_opt.arena_block_size, 22U); | |
188 | ASSERT_EQ(new_cf_opt.disable_auto_compactions, true); | |
189 | ASSERT_EQ(new_cf_opt.compaction_style, kCompactionStyleLevel); | |
190 | ASSERT_EQ(new_cf_opt.compaction_pri, kOldestSmallestSeqFirst); | |
191 | ASSERT_EQ(new_cf_opt.compaction_options_fifo.max_table_files_size, | |
192 | static_cast<uint64_t>(23)); | |
193 | ASSERT_EQ(new_cf_opt.max_sequential_skip_in_iterations, | |
194 | static_cast<uint64_t>(24)); | |
195 | ASSERT_EQ(new_cf_opt.inplace_update_support, true); | |
196 | ASSERT_EQ(new_cf_opt.inplace_update_num_locks, 25U); | |
197 | ASSERT_EQ(new_cf_opt.memtable_prefix_bloom_size_ratio, 0.26); | |
198 | ASSERT_EQ(new_cf_opt.memtable_huge_page_size, 28U); | |
199 | ASSERT_EQ(new_cf_opt.bloom_locality, 29U); | |
200 | ASSERT_EQ(new_cf_opt.max_successive_merges, 30U); | |
201 | ASSERT_TRUE(new_cf_opt.prefix_extractor != nullptr); | |
202 | ASSERT_EQ(new_cf_opt.optimize_filters_for_hits, true); | |
203 | ASSERT_EQ(std::string(new_cf_opt.prefix_extractor->Name()), | |
204 | "rocksdb.FixedPrefix.31"); | |
205 | ||
206 | cf_options_map["write_buffer_size"] = "hello"; | |
207 | ASSERT_NOK(GetColumnFamilyOptionsFromMap( | |
208 | base_cf_opt, cf_options_map, &new_cf_opt)); | |
209 | ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(base_cf_opt, new_cf_opt)); | |
210 | ||
211 | cf_options_map["write_buffer_size"] = "1"; | |
212 | ASSERT_OK(GetColumnFamilyOptionsFromMap( | |
213 | base_cf_opt, cf_options_map, &new_cf_opt)); | |
7c673cae | 214 | |
11fdf7f2 | 215 | cf_options_map["unknown_option"] = "1"; |
7c673cae FG |
216 | ASSERT_NOK(GetColumnFamilyOptionsFromMap( |
217 | base_cf_opt, cf_options_map, &new_cf_opt)); | |
218 | ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(base_cf_opt, new_cf_opt)); | |
219 | ||
11fdf7f2 TL |
220 | ASSERT_OK(GetColumnFamilyOptionsFromMap(base_cf_opt, cf_options_map, |
221 | &new_cf_opt, | |
222 | false, /* input_strings_escaped */ | |
223 | true /* ignore_unknown_options */)); | |
224 | ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions( | |
225 | base_cf_opt, new_cf_opt, nullptr, /* new_opt_map */ | |
226 | kSanityLevelLooselyCompatible /* from CheckOptionsCompatibility*/)); | |
227 | ASSERT_NOK(RocksDBOptionsParser::VerifyCFOptions( | |
228 | base_cf_opt, new_cf_opt, nullptr, /* new_opt_map */ | |
229 | kSanityLevelExactMatch /* default for VerifyCFOptions */)); | |
230 | ||
7c673cae FG |
231 | DBOptions base_db_opt; |
232 | DBOptions new_db_opt; | |
233 | ASSERT_OK(GetDBOptionsFromMap(base_db_opt, db_options_map, &new_db_opt)); | |
234 | ASSERT_EQ(new_db_opt.create_if_missing, false); | |
235 | ASSERT_EQ(new_db_opt.create_missing_column_families, true); | |
236 | ASSERT_EQ(new_db_opt.error_if_exists, false); | |
237 | ASSERT_EQ(new_db_opt.paranoid_checks, true); | |
238 | ASSERT_EQ(new_db_opt.max_open_files, 32); | |
239 | ASSERT_EQ(new_db_opt.max_total_wal_size, static_cast<uint64_t>(33)); | |
240 | ASSERT_EQ(new_db_opt.use_fsync, true); | |
241 | ASSERT_EQ(new_db_opt.db_log_dir, "/db_log_dir"); | |
242 | ASSERT_EQ(new_db_opt.wal_dir, "/wal_dir"); | |
243 | ASSERT_EQ(new_db_opt.delete_obsolete_files_period_micros, | |
244 | static_cast<uint64_t>(34)); | |
245 | ASSERT_EQ(new_db_opt.max_background_compactions, 35); | |
246 | ASSERT_EQ(new_db_opt.max_background_flushes, 36); | |
247 | ASSERT_EQ(new_db_opt.max_log_file_size, 37U); | |
248 | ASSERT_EQ(new_db_opt.log_file_time_to_roll, 38U); | |
249 | ASSERT_EQ(new_db_opt.keep_log_file_num, 39U); | |
250 | ASSERT_EQ(new_db_opt.recycle_log_file_num, 5U); | |
251 | ASSERT_EQ(new_db_opt.max_manifest_file_size, static_cast<uint64_t>(40)); | |
252 | ASSERT_EQ(new_db_opt.table_cache_numshardbits, 41); | |
253 | ASSERT_EQ(new_db_opt.WAL_ttl_seconds, static_cast<uint64_t>(43)); | |
254 | ASSERT_EQ(new_db_opt.WAL_size_limit_MB, static_cast<uint64_t>(44)); | |
255 | ASSERT_EQ(new_db_opt.manifest_preallocation_size, 45U); | |
256 | ASSERT_EQ(new_db_opt.allow_mmap_reads, true); | |
257 | ASSERT_EQ(new_db_opt.allow_mmap_writes, false); | |
258 | ASSERT_EQ(new_db_opt.use_direct_reads, false); | |
259 | ASSERT_EQ(new_db_opt.use_direct_io_for_flush_and_compaction, false); | |
260 | ASSERT_EQ(new_db_opt.is_fd_close_on_exec, true); | |
261 | ASSERT_EQ(new_db_opt.skip_log_error_on_recovery, false); | |
262 | ASSERT_EQ(new_db_opt.stats_dump_period_sec, 46U); | |
263 | ASSERT_EQ(new_db_opt.advise_random_on_open, true); | |
264 | ASSERT_EQ(new_db_opt.use_adaptive_mutex, false); | |
265 | ASSERT_EQ(new_db_opt.new_table_reader_for_compaction_inputs, true); | |
266 | ASSERT_EQ(new_db_opt.compaction_readahead_size, 100); | |
267 | ASSERT_EQ(new_db_opt.random_access_max_buffer_size, 3145728); | |
268 | ASSERT_EQ(new_db_opt.writable_file_max_buffer_size, 314159); | |
269 | ASSERT_EQ(new_db_opt.bytes_per_sync, static_cast<uint64_t>(47)); | |
270 | ASSERT_EQ(new_db_opt.wal_bytes_per_sync, static_cast<uint64_t>(48)); | |
11fdf7f2 TL |
271 | |
272 | db_options_map["max_open_files"] = "hello"; | |
273 | ASSERT_NOK(GetDBOptionsFromMap(base_db_opt, db_options_map, &new_db_opt)); | |
274 | ASSERT_OK(RocksDBOptionsParser::VerifyDBOptions(base_db_opt, new_db_opt)); | |
275 | ASSERT_OK(RocksDBOptionsParser::VerifyDBOptions( | |
276 | base_db_opt, new_db_opt, nullptr, /* new_opt_map */ | |
277 | kSanityLevelLooselyCompatible /* from CheckOptionsCompatibility */)); | |
278 | ||
279 | // unknow options should fail parsing without ignore_unknown_options = true | |
280 | db_options_map["unknown_db_option"] = "1"; | |
281 | ASSERT_NOK(GetDBOptionsFromMap(base_db_opt, db_options_map, &new_db_opt)); | |
282 | ASSERT_OK(RocksDBOptionsParser::VerifyDBOptions(base_db_opt, new_db_opt)); | |
283 | ||
284 | ASSERT_OK(GetDBOptionsFromMap(base_db_opt, db_options_map, &new_db_opt, | |
285 | false, /* input_strings_escaped */ | |
286 | true /* ignore_unknown_options */)); | |
287 | ASSERT_OK(RocksDBOptionsParser::VerifyDBOptions( | |
288 | base_db_opt, new_db_opt, nullptr, /* new_opt_map */ | |
289 | kSanityLevelLooselyCompatible /* from CheckOptionsCompatibility */)); | |
290 | ASSERT_NOK(RocksDBOptionsParser::VerifyDBOptions( | |
291 | base_db_opt, new_db_opt, nullptr, /* new_opt_mat */ | |
292 | kSanityLevelExactMatch /* default for VerifyDBOptions */)); | |
7c673cae FG |
293 | } |
294 | #endif // !ROCKSDB_LITE | |
295 | ||
296 | #ifndef ROCKSDB_LITE // GetColumnFamilyOptionsFromString is not supported in | |
297 | // ROCKSDB_LITE | |
298 | TEST_F(OptionsTest, GetColumnFamilyOptionsFromStringTest) { | |
299 | ColumnFamilyOptions base_cf_opt; | |
300 | ColumnFamilyOptions new_cf_opt; | |
301 | base_cf_opt.table_factory.reset(); | |
302 | ASSERT_OK(GetColumnFamilyOptionsFromString(base_cf_opt, "", &new_cf_opt)); | |
303 | ASSERT_OK(GetColumnFamilyOptionsFromString(base_cf_opt, | |
304 | "write_buffer_size=5", &new_cf_opt)); | |
305 | ASSERT_EQ(new_cf_opt.write_buffer_size, 5U); | |
306 | ASSERT_TRUE(new_cf_opt.table_factory == nullptr); | |
307 | ASSERT_OK(GetColumnFamilyOptionsFromString(base_cf_opt, | |
308 | "write_buffer_size=6;", &new_cf_opt)); | |
309 | ASSERT_EQ(new_cf_opt.write_buffer_size, 6U); | |
310 | ASSERT_OK(GetColumnFamilyOptionsFromString(base_cf_opt, | |
311 | " write_buffer_size = 7 ", &new_cf_opt)); | |
312 | ASSERT_EQ(new_cf_opt.write_buffer_size, 7U); | |
313 | ASSERT_OK(GetColumnFamilyOptionsFromString(base_cf_opt, | |
314 | " write_buffer_size = 8 ; ", &new_cf_opt)); | |
315 | ASSERT_EQ(new_cf_opt.write_buffer_size, 8U); | |
316 | ASSERT_OK(GetColumnFamilyOptionsFromString(base_cf_opt, | |
317 | "write_buffer_size=9;max_write_buffer_number=10", &new_cf_opt)); | |
318 | ASSERT_EQ(new_cf_opt.write_buffer_size, 9U); | |
319 | ASSERT_EQ(new_cf_opt.max_write_buffer_number, 10); | |
320 | ASSERT_OK(GetColumnFamilyOptionsFromString(base_cf_opt, | |
321 | "write_buffer_size=11; max_write_buffer_number = 12 ;", | |
322 | &new_cf_opt)); | |
323 | ASSERT_EQ(new_cf_opt.write_buffer_size, 11U); | |
324 | ASSERT_EQ(new_cf_opt.max_write_buffer_number, 12); | |
325 | // Wrong name "max_write_buffer_number_" | |
326 | ASSERT_NOK(GetColumnFamilyOptionsFromString(base_cf_opt, | |
327 | "write_buffer_size=13;max_write_buffer_number_=14;", | |
328 | &new_cf_opt)); | |
329 | ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(base_cf_opt, new_cf_opt)); | |
330 | ||
331 | // Wrong key/value pair | |
332 | ASSERT_NOK(GetColumnFamilyOptionsFromString(base_cf_opt, | |
333 | "write_buffer_size=13;max_write_buffer_number;", &new_cf_opt)); | |
334 | ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(base_cf_opt, new_cf_opt)); | |
335 | ||
336 | // Error Paring value | |
337 | ASSERT_NOK(GetColumnFamilyOptionsFromString(base_cf_opt, | |
338 | "write_buffer_size=13;max_write_buffer_number=;", &new_cf_opt)); | |
339 | ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(base_cf_opt, new_cf_opt)); | |
340 | ||
341 | // Missing option name | |
342 | ASSERT_NOK(GetColumnFamilyOptionsFromString(base_cf_opt, | |
343 | "write_buffer_size=13; =100;", &new_cf_opt)); | |
344 | ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(base_cf_opt, new_cf_opt)); | |
345 | ||
346 | const int64_t kilo = 1024UL; | |
347 | const int64_t mega = 1024 * kilo; | |
348 | const int64_t giga = 1024 * mega; | |
349 | const int64_t tera = 1024 * giga; | |
350 | ||
351 | // Units (k) | |
352 | ASSERT_OK(GetColumnFamilyOptionsFromString( | |
353 | base_cf_opt, "max_write_buffer_number=15K", &new_cf_opt)); | |
354 | ASSERT_EQ(new_cf_opt.max_write_buffer_number, 15 * kilo); | |
355 | // Units (m) | |
356 | ASSERT_OK(GetColumnFamilyOptionsFromString(base_cf_opt, | |
357 | "max_write_buffer_number=16m;inplace_update_num_locks=17M", | |
358 | &new_cf_opt)); | |
359 | ASSERT_EQ(new_cf_opt.max_write_buffer_number, 16 * mega); | |
360 | ASSERT_EQ(new_cf_opt.inplace_update_num_locks, 17 * mega); | |
361 | // Units (g) | |
362 | ASSERT_OK(GetColumnFamilyOptionsFromString( | |
363 | base_cf_opt, | |
364 | "write_buffer_size=18g;prefix_extractor=capped:8;" | |
365 | "arena_block_size=19G", | |
366 | &new_cf_opt)); | |
367 | ||
368 | ASSERT_EQ(new_cf_opt.write_buffer_size, 18 * giga); | |
369 | ASSERT_EQ(new_cf_opt.arena_block_size, 19 * giga); | |
370 | ASSERT_TRUE(new_cf_opt.prefix_extractor.get() != nullptr); | |
371 | std::string prefix_name(new_cf_opt.prefix_extractor->Name()); | |
372 | ASSERT_EQ(prefix_name, "rocksdb.CappedPrefix.8"); | |
373 | ||
374 | // Units (t) | |
375 | ASSERT_OK(GetColumnFamilyOptionsFromString(base_cf_opt, | |
376 | "write_buffer_size=20t;arena_block_size=21T", &new_cf_opt)); | |
377 | ASSERT_EQ(new_cf_opt.write_buffer_size, 20 * tera); | |
378 | ASSERT_EQ(new_cf_opt.arena_block_size, 21 * tera); | |
379 | ||
380 | // Nested block based table options | |
11fdf7f2 | 381 | // Empty |
7c673cae FG |
382 | ASSERT_OK(GetColumnFamilyOptionsFromString(base_cf_opt, |
383 | "write_buffer_size=10;max_write_buffer_number=16;" | |
384 | "block_based_table_factory={};arena_block_size=1024", | |
385 | &new_cf_opt)); | |
386 | ASSERT_TRUE(new_cf_opt.table_factory != nullptr); | |
387 | // Non-empty | |
388 | ASSERT_OK(GetColumnFamilyOptionsFromString(base_cf_opt, | |
389 | "write_buffer_size=10;max_write_buffer_number=16;" | |
390 | "block_based_table_factory={block_cache=1M;block_size=4;};" | |
391 | "arena_block_size=1024", | |
392 | &new_cf_opt)); | |
393 | ASSERT_TRUE(new_cf_opt.table_factory != nullptr); | |
394 | // Last one | |
395 | ASSERT_OK(GetColumnFamilyOptionsFromString(base_cf_opt, | |
396 | "write_buffer_size=10;max_write_buffer_number=16;" | |
397 | "block_based_table_factory={block_cache=1M;block_size=4;}", | |
398 | &new_cf_opt)); | |
399 | ASSERT_TRUE(new_cf_opt.table_factory != nullptr); | |
400 | // Mismatch curly braces | |
401 | ASSERT_NOK(GetColumnFamilyOptionsFromString(base_cf_opt, | |
402 | "write_buffer_size=10;max_write_buffer_number=16;" | |
403 | "block_based_table_factory={{{block_size=4;};" | |
404 | "arena_block_size=1024", | |
405 | &new_cf_opt)); | |
406 | ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(base_cf_opt, new_cf_opt)); | |
407 | ||
408 | // Unexpected chars after closing curly brace | |
409 | ASSERT_NOK(GetColumnFamilyOptionsFromString(base_cf_opt, | |
410 | "write_buffer_size=10;max_write_buffer_number=16;" | |
411 | "block_based_table_factory={block_size=4;}};" | |
412 | "arena_block_size=1024", | |
413 | &new_cf_opt)); | |
414 | ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(base_cf_opt, new_cf_opt)); | |
415 | ||
416 | ASSERT_NOK(GetColumnFamilyOptionsFromString(base_cf_opt, | |
417 | "write_buffer_size=10;max_write_buffer_number=16;" | |
418 | "block_based_table_factory={block_size=4;}xdfa;" | |
419 | "arena_block_size=1024", | |
420 | &new_cf_opt)); | |
421 | ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(base_cf_opt, new_cf_opt)); | |
422 | ||
423 | ASSERT_NOK(GetColumnFamilyOptionsFromString(base_cf_opt, | |
424 | "write_buffer_size=10;max_write_buffer_number=16;" | |
425 | "block_based_table_factory={block_size=4;}xdfa", | |
426 | &new_cf_opt)); | |
427 | ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(base_cf_opt, new_cf_opt)); | |
428 | ||
429 | // Invalid block based table option | |
430 | ASSERT_NOK(GetColumnFamilyOptionsFromString(base_cf_opt, | |
431 | "write_buffer_size=10;max_write_buffer_number=16;" | |
432 | "block_based_table_factory={xx_block_size=4;}", | |
433 | &new_cf_opt)); | |
434 | ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(base_cf_opt, new_cf_opt)); | |
435 | ||
436 | ASSERT_OK(GetColumnFamilyOptionsFromString(base_cf_opt, | |
437 | "optimize_filters_for_hits=true", | |
438 | &new_cf_opt)); | |
439 | ASSERT_OK(GetColumnFamilyOptionsFromString(base_cf_opt, | |
440 | "optimize_filters_for_hits=false", | |
441 | &new_cf_opt)); | |
442 | ||
443 | ASSERT_NOK(GetColumnFamilyOptionsFromString(base_cf_opt, | |
444 | "optimize_filters_for_hits=junk", | |
445 | &new_cf_opt)); | |
446 | ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(base_cf_opt, new_cf_opt)); | |
447 | ||
448 | // Nested plain table options | |
11fdf7f2 | 449 | // Empty |
7c673cae FG |
450 | ASSERT_OK(GetColumnFamilyOptionsFromString(base_cf_opt, |
451 | "write_buffer_size=10;max_write_buffer_number=16;" | |
452 | "plain_table_factory={};arena_block_size=1024", | |
453 | &new_cf_opt)); | |
454 | ASSERT_TRUE(new_cf_opt.table_factory != nullptr); | |
455 | ASSERT_EQ(std::string(new_cf_opt.table_factory->Name()), "PlainTable"); | |
456 | // Non-empty | |
457 | ASSERT_OK(GetColumnFamilyOptionsFromString(base_cf_opt, | |
458 | "write_buffer_size=10;max_write_buffer_number=16;" | |
459 | "plain_table_factory={user_key_len=66;bloom_bits_per_key=20;};" | |
460 | "arena_block_size=1024", | |
461 | &new_cf_opt)); | |
462 | ASSERT_TRUE(new_cf_opt.table_factory != nullptr); | |
463 | ASSERT_EQ(std::string(new_cf_opt.table_factory->Name()), "PlainTable"); | |
464 | ||
465 | // memtable factory | |
466 | ASSERT_OK(GetColumnFamilyOptionsFromString(base_cf_opt, | |
467 | "write_buffer_size=10;max_write_buffer_number=16;" | |
468 | "memtable=skip_list:10;arena_block_size=1024", | |
469 | &new_cf_opt)); | |
470 | ASSERT_TRUE(new_cf_opt.memtable_factory != nullptr); | |
471 | ASSERT_EQ(std::string(new_cf_opt.memtable_factory->Name()), "SkipListFactory"); | |
472 | } | |
473 | #endif // !ROCKSDB_LITE | |
474 | ||
475 | #ifndef ROCKSDB_LITE // GetBlockBasedTableOptionsFromString is not supported | |
476 | TEST_F(OptionsTest, GetBlockBasedTableOptionsFromString) { | |
477 | BlockBasedTableOptions table_opt; | |
478 | BlockBasedTableOptions new_opt; | |
479 | // make sure default values are overwritten by something else | |
480 | ASSERT_OK(GetBlockBasedTableOptionsFromString(table_opt, | |
481 | "cache_index_and_filter_blocks=1;index_type=kHashSearch;" | |
482 | "checksum=kxxHash;hash_index_allow_collision=1;no_block_cache=1;" | |
483 | "block_cache=1M;block_cache_compressed=1k;block_size=1024;" | |
484 | "block_size_deviation=8;block_restart_interval=4;" | |
485 | "filter_policy=bloomfilter:4:true;whole_key_filtering=1;", | |
486 | &new_opt)); | |
487 | ASSERT_TRUE(new_opt.cache_index_and_filter_blocks); | |
488 | ASSERT_EQ(new_opt.index_type, BlockBasedTableOptions::kHashSearch); | |
489 | ASSERT_EQ(new_opt.checksum, ChecksumType::kxxHash); | |
490 | ASSERT_TRUE(new_opt.hash_index_allow_collision); | |
491 | ASSERT_TRUE(new_opt.no_block_cache); | |
492 | ASSERT_TRUE(new_opt.block_cache != nullptr); | |
493 | ASSERT_EQ(new_opt.block_cache->GetCapacity(), 1024UL*1024UL); | |
494 | ASSERT_TRUE(new_opt.block_cache_compressed != nullptr); | |
495 | ASSERT_EQ(new_opt.block_cache_compressed->GetCapacity(), 1024UL); | |
496 | ASSERT_EQ(new_opt.block_size, 1024UL); | |
497 | ASSERT_EQ(new_opt.block_size_deviation, 8); | |
498 | ASSERT_EQ(new_opt.block_restart_interval, 4); | |
499 | ASSERT_TRUE(new_opt.filter_policy != nullptr); | |
500 | ||
501 | // unknown option | |
502 | ASSERT_NOK(GetBlockBasedTableOptionsFromString(table_opt, | |
503 | "cache_index_and_filter_blocks=1;index_type=kBinarySearch;" | |
504 | "bad_option=1", | |
505 | &new_opt)); | |
506 | ASSERT_EQ(table_opt.cache_index_and_filter_blocks, | |
507 | new_opt.cache_index_and_filter_blocks); | |
508 | ASSERT_EQ(table_opt.index_type, new_opt.index_type); | |
509 | ||
510 | // unrecognized index type | |
511 | ASSERT_NOK(GetBlockBasedTableOptionsFromString(table_opt, | |
512 | "cache_index_and_filter_blocks=1;index_type=kBinarySearchXX", | |
513 | &new_opt)); | |
514 | ASSERT_EQ(table_opt.cache_index_and_filter_blocks, | |
515 | new_opt.cache_index_and_filter_blocks); | |
516 | ASSERT_EQ(table_opt.index_type, new_opt.index_type); | |
517 | ||
518 | // unrecognized checksum type | |
519 | ASSERT_NOK(GetBlockBasedTableOptionsFromString(table_opt, | |
520 | "cache_index_and_filter_blocks=1;checksum=kxxHashXX", | |
521 | &new_opt)); | |
522 | ASSERT_EQ(table_opt.cache_index_and_filter_blocks, | |
523 | new_opt.cache_index_and_filter_blocks); | |
524 | ASSERT_EQ(table_opt.index_type, new_opt.index_type); | |
525 | ||
526 | // unrecognized filter policy name | |
527 | ASSERT_NOK(GetBlockBasedTableOptionsFromString(table_opt, | |
528 | "cache_index_and_filter_blocks=1;" | |
529 | "filter_policy=bloomfilterxx:4:true", | |
530 | &new_opt)); | |
531 | ASSERT_EQ(table_opt.cache_index_and_filter_blocks, | |
532 | new_opt.cache_index_and_filter_blocks); | |
533 | ASSERT_EQ(table_opt.filter_policy, new_opt.filter_policy); | |
534 | ||
535 | // unrecognized filter policy config | |
536 | ASSERT_NOK(GetBlockBasedTableOptionsFromString(table_opt, | |
537 | "cache_index_and_filter_blocks=1;" | |
538 | "filter_policy=bloomfilter:4", | |
539 | &new_opt)); | |
540 | ASSERT_EQ(table_opt.cache_index_and_filter_blocks, | |
541 | new_opt.cache_index_and_filter_blocks); | |
542 | ASSERT_EQ(table_opt.filter_policy, new_opt.filter_policy); | |
11fdf7f2 TL |
543 | |
544 | // Check block cache options are overwritten when specified | |
545 | // in new format as a struct. | |
546 | ASSERT_OK(GetBlockBasedTableOptionsFromString(table_opt, | |
547 | "block_cache={capacity=1M;num_shard_bits=4;" | |
548 | "strict_capacity_limit=true;high_pri_pool_ratio=0.5;};" | |
549 | "block_cache_compressed={capacity=1M;num_shard_bits=4;" | |
550 | "strict_capacity_limit=true;high_pri_pool_ratio=0.5;}", | |
551 | &new_opt)); | |
552 | ASSERT_TRUE(new_opt.block_cache != nullptr); | |
553 | ASSERT_EQ(new_opt.block_cache->GetCapacity(), 1024UL*1024UL); | |
554 | ASSERT_EQ(std::dynamic_pointer_cast<ShardedCache>( | |
555 | new_opt.block_cache)->GetNumShardBits(), 4); | |
556 | ASSERT_EQ(new_opt.block_cache->HasStrictCapacityLimit(), true); | |
557 | ASSERT_EQ(std::dynamic_pointer_cast<LRUCache>( | |
558 | new_opt.block_cache)->GetHighPriPoolRatio(), 0.5); | |
559 | ASSERT_TRUE(new_opt.block_cache_compressed != nullptr); | |
560 | ASSERT_EQ(new_opt.block_cache_compressed->GetCapacity(), 1024UL*1024UL); | |
561 | ASSERT_EQ(std::dynamic_pointer_cast<ShardedCache>( | |
562 | new_opt.block_cache_compressed)->GetNumShardBits(), 4); | |
563 | ASSERT_EQ(new_opt.block_cache_compressed->HasStrictCapacityLimit(), true); | |
564 | ASSERT_EQ(std::dynamic_pointer_cast<LRUCache>( | |
565 | new_opt.block_cache_compressed)->GetHighPriPoolRatio(), | |
566 | 0.5); | |
567 | ||
568 | // Set only block cache capacity. Check other values are | |
569 | // reset to default values. | |
570 | ASSERT_OK(GetBlockBasedTableOptionsFromString(table_opt, | |
571 | "block_cache={capacity=2M};" | |
572 | "block_cache_compressed={capacity=2M}", | |
573 | &new_opt)); | |
574 | ASSERT_TRUE(new_opt.block_cache != nullptr); | |
575 | ASSERT_EQ(new_opt.block_cache->GetCapacity(), 2*1024UL*1024UL); | |
576 | // Default values | |
577 | ASSERT_EQ(std::dynamic_pointer_cast<ShardedCache>( | |
578 | new_opt.block_cache)->GetNumShardBits(), | |
579 | GetDefaultCacheShardBits(new_opt.block_cache->GetCapacity())); | |
580 | ASSERT_EQ(new_opt.block_cache->HasStrictCapacityLimit(), false); | |
581 | ASSERT_EQ(std::dynamic_pointer_cast<LRUCache>( | |
582 | new_opt.block_cache)->GetHighPriPoolRatio(), 0.0); | |
583 | ASSERT_TRUE(new_opt.block_cache_compressed != nullptr); | |
584 | ASSERT_EQ(new_opt.block_cache_compressed->GetCapacity(), 2*1024UL*1024UL); | |
585 | // Default values | |
586 | ASSERT_EQ(std::dynamic_pointer_cast<ShardedCache>( | |
587 | new_opt.block_cache_compressed)->GetNumShardBits(), | |
588 | GetDefaultCacheShardBits( | |
589 | new_opt.block_cache_compressed->GetCapacity())); | |
590 | ASSERT_EQ(new_opt.block_cache_compressed->HasStrictCapacityLimit(), false); | |
591 | ASSERT_EQ(std::dynamic_pointer_cast<LRUCache>( | |
592 | new_opt.block_cache_compressed)->GetHighPriPoolRatio(), | |
593 | 0.0); | |
594 | ||
595 | // Set couple of block cache options. | |
596 | ASSERT_OK(GetBlockBasedTableOptionsFromString(table_opt, | |
597 | "block_cache={num_shard_bits=5;high_pri_pool_ratio=0.5;};" | |
598 | "block_cache_compressed={num_shard_bits=5;" | |
599 | "high_pri_pool_ratio=0.5;}", | |
600 | &new_opt)); | |
601 | ASSERT_EQ(new_opt.block_cache->GetCapacity(), 0); | |
602 | ASSERT_EQ(std::dynamic_pointer_cast<ShardedCache>( | |
603 | new_opt.block_cache)->GetNumShardBits(), 5); | |
604 | ASSERT_EQ(new_opt.block_cache->HasStrictCapacityLimit(), false); | |
605 | ASSERT_EQ(std::dynamic_pointer_cast<LRUCache>( | |
606 | new_opt.block_cache)->GetHighPriPoolRatio(), 0.5); | |
607 | ASSERT_TRUE(new_opt.block_cache_compressed != nullptr); | |
608 | ASSERT_EQ(new_opt.block_cache_compressed->GetCapacity(), 0); | |
609 | ASSERT_EQ(std::dynamic_pointer_cast<ShardedCache>( | |
610 | new_opt.block_cache_compressed)->GetNumShardBits(), 5); | |
611 | ASSERT_EQ(new_opt.block_cache_compressed->HasStrictCapacityLimit(), false); | |
612 | ASSERT_EQ(std::dynamic_pointer_cast<LRUCache>( | |
613 | new_opt.block_cache_compressed)->GetHighPriPoolRatio(), | |
614 | 0.5); | |
615 | ||
616 | // Set couple of block cache options. | |
617 | ASSERT_OK(GetBlockBasedTableOptionsFromString(table_opt, | |
618 | "block_cache={capacity=1M;num_shard_bits=4;" | |
619 | "strict_capacity_limit=true;};" | |
620 | "block_cache_compressed={capacity=1M;num_shard_bits=4;" | |
621 | "strict_capacity_limit=true;}", | |
622 | &new_opt)); | |
623 | ASSERT_TRUE(new_opt.block_cache != nullptr); | |
624 | ASSERT_EQ(new_opt.block_cache->GetCapacity(), 1024UL*1024UL); | |
625 | ASSERT_EQ(std::dynamic_pointer_cast<ShardedCache>( | |
626 | new_opt.block_cache)->GetNumShardBits(), 4); | |
627 | ASSERT_EQ(new_opt.block_cache->HasStrictCapacityLimit(), true); | |
628 | ASSERT_EQ(std::dynamic_pointer_cast<LRUCache>( | |
629 | new_opt.block_cache)->GetHighPriPoolRatio(), 0.0); | |
630 | ASSERT_TRUE(new_opt.block_cache_compressed != nullptr); | |
631 | ASSERT_EQ(new_opt.block_cache_compressed->GetCapacity(), 1024UL*1024UL); | |
632 | ASSERT_EQ(std::dynamic_pointer_cast<ShardedCache>( | |
633 | new_opt.block_cache_compressed)->GetNumShardBits(), 4); | |
634 | ASSERT_EQ(new_opt.block_cache_compressed->HasStrictCapacityLimit(), true); | |
635 | ASSERT_EQ(std::dynamic_pointer_cast<LRUCache>( | |
636 | new_opt.block_cache_compressed)->GetHighPriPoolRatio(), | |
637 | 0.0); | |
7c673cae FG |
638 | } |
639 | #endif // !ROCKSDB_LITE | |
640 | ||
641 | ||
642 | #ifndef ROCKSDB_LITE // GetPlainTableOptionsFromString is not supported | |
643 | TEST_F(OptionsTest, GetPlainTableOptionsFromString) { | |
644 | PlainTableOptions table_opt; | |
645 | PlainTableOptions new_opt; | |
646 | // make sure default values are overwritten by something else | |
647 | ASSERT_OK(GetPlainTableOptionsFromString(table_opt, | |
648 | "user_key_len=66;bloom_bits_per_key=20;hash_table_ratio=0.5;" | |
649 | "index_sparseness=8;huge_page_tlb_size=4;encoding_type=kPrefix;" | |
650 | "full_scan_mode=true;store_index_in_file=true", | |
651 | &new_opt)); | |
652 | ASSERT_EQ(new_opt.user_key_len, 66); | |
653 | ASSERT_EQ(new_opt.bloom_bits_per_key, 20); | |
654 | ASSERT_EQ(new_opt.hash_table_ratio, 0.5); | |
655 | ASSERT_EQ(new_opt.index_sparseness, 8); | |
656 | ASSERT_EQ(new_opt.huge_page_tlb_size, 4); | |
657 | ASSERT_EQ(new_opt.encoding_type, EncodingType::kPrefix); | |
658 | ASSERT_TRUE(new_opt.full_scan_mode); | |
659 | ASSERT_TRUE(new_opt.store_index_in_file); | |
660 | ||
661 | // unknown option | |
662 | ASSERT_NOK(GetPlainTableOptionsFromString(table_opt, | |
663 | "user_key_len=66;bloom_bits_per_key=20;hash_table_ratio=0.5;" | |
664 | "bad_option=1", | |
665 | &new_opt)); | |
666 | ||
667 | // unrecognized EncodingType | |
668 | ASSERT_NOK(GetPlainTableOptionsFromString(table_opt, | |
669 | "user_key_len=66;bloom_bits_per_key=20;hash_table_ratio=0.5;" | |
670 | "encoding_type=kPrefixXX", | |
671 | &new_opt)); | |
672 | } | |
673 | #endif // !ROCKSDB_LITE | |
674 | ||
675 | #ifndef ROCKSDB_LITE // GetMemTableRepFactoryFromString is not supported | |
676 | TEST_F(OptionsTest, GetMemTableRepFactoryFromString) { | |
677 | std::unique_ptr<MemTableRepFactory> new_mem_factory = nullptr; | |
678 | ||
679 | ASSERT_OK(GetMemTableRepFactoryFromString("skip_list", &new_mem_factory)); | |
680 | ASSERT_OK(GetMemTableRepFactoryFromString("skip_list:16", &new_mem_factory)); | |
681 | ASSERT_EQ(std::string(new_mem_factory->Name()), "SkipListFactory"); | |
682 | ASSERT_NOK(GetMemTableRepFactoryFromString("skip_list:16:invalid_opt", | |
683 | &new_mem_factory)); | |
684 | ||
685 | ASSERT_OK(GetMemTableRepFactoryFromString("prefix_hash", &new_mem_factory)); | |
686 | ASSERT_OK(GetMemTableRepFactoryFromString("prefix_hash:1000", | |
687 | &new_mem_factory)); | |
688 | ASSERT_EQ(std::string(new_mem_factory->Name()), "HashSkipListRepFactory"); | |
689 | ASSERT_NOK(GetMemTableRepFactoryFromString("prefix_hash:1000:invalid_opt", | |
690 | &new_mem_factory)); | |
691 | ||
692 | ASSERT_OK(GetMemTableRepFactoryFromString("hash_linkedlist", | |
693 | &new_mem_factory)); | |
694 | ASSERT_OK(GetMemTableRepFactoryFromString("hash_linkedlist:1000", | |
695 | &new_mem_factory)); | |
696 | ASSERT_EQ(std::string(new_mem_factory->Name()), "HashLinkListRepFactory"); | |
697 | ASSERT_NOK(GetMemTableRepFactoryFromString("hash_linkedlist:1000:invalid_opt", | |
698 | &new_mem_factory)); | |
699 | ||
700 | ASSERT_OK(GetMemTableRepFactoryFromString("vector", &new_mem_factory)); | |
701 | ASSERT_OK(GetMemTableRepFactoryFromString("vector:1024", &new_mem_factory)); | |
702 | ASSERT_EQ(std::string(new_mem_factory->Name()), "VectorRepFactory"); | |
703 | ASSERT_NOK(GetMemTableRepFactoryFromString("vector:1024:invalid_opt", | |
704 | &new_mem_factory)); | |
705 | ||
706 | ASSERT_NOK(GetMemTableRepFactoryFromString("cuckoo", &new_mem_factory)); | |
707 | ASSERT_OK(GetMemTableRepFactoryFromString("cuckoo:1024", &new_mem_factory)); | |
708 | ASSERT_EQ(std::string(new_mem_factory->Name()), "HashCuckooRepFactory"); | |
709 | ||
710 | ASSERT_NOK(GetMemTableRepFactoryFromString("bad_factory", &new_mem_factory)); | |
711 | } | |
712 | #endif // !ROCKSDB_LITE | |
713 | ||
714 | #ifndef ROCKSDB_LITE // GetOptionsFromString is not supported in RocksDB Lite | |
715 | TEST_F(OptionsTest, GetOptionsFromStringTest) { | |
716 | Options base_options, new_options; | |
717 | base_options.write_buffer_size = 20; | |
718 | base_options.min_write_buffer_number_to_merge = 15; | |
719 | BlockBasedTableOptions block_based_table_options; | |
720 | block_based_table_options.cache_index_and_filter_blocks = true; | |
721 | base_options.table_factory.reset( | |
722 | NewBlockBasedTableFactory(block_based_table_options)); | |
723 | ASSERT_OK(GetOptionsFromString( | |
724 | base_options, | |
725 | "write_buffer_size=10;max_write_buffer_number=16;" | |
726 | "block_based_table_factory={block_cache=1M;block_size=4;};" | |
727 | "compression_opts=4:5:6;create_if_missing=true;max_open_files=1;" | |
11fdf7f2 TL |
728 | "bottommost_compression_opts=5:6:7;create_if_missing=true;max_open_files=" |
729 | "1;" | |
7c673cae FG |
730 | "rate_limiter_bytes_per_sec=1024", |
731 | &new_options)); | |
732 | ||
733 | ASSERT_EQ(new_options.compression_opts.window_bits, 4); | |
734 | ASSERT_EQ(new_options.compression_opts.level, 5); | |
735 | ASSERT_EQ(new_options.compression_opts.strategy, 6); | |
736 | ASSERT_EQ(new_options.compression_opts.max_dict_bytes, 0); | |
11fdf7f2 TL |
737 | ASSERT_EQ(new_options.compression_opts.zstd_max_train_bytes, 0); |
738 | ASSERT_EQ(new_options.compression_opts.enabled, false); | |
7c673cae | 739 | ASSERT_EQ(new_options.bottommost_compression, kDisableCompressionOption); |
11fdf7f2 TL |
740 | ASSERT_EQ(new_options.bottommost_compression_opts.window_bits, 5); |
741 | ASSERT_EQ(new_options.bottommost_compression_opts.level, 6); | |
742 | ASSERT_EQ(new_options.bottommost_compression_opts.strategy, 7); | |
743 | ASSERT_EQ(new_options.bottommost_compression_opts.max_dict_bytes, 0); | |
744 | ASSERT_EQ(new_options.bottommost_compression_opts.zstd_max_train_bytes, 0); | |
745 | ASSERT_EQ(new_options.bottommost_compression_opts.enabled, false); | |
7c673cae FG |
746 | ASSERT_EQ(new_options.write_buffer_size, 10U); |
747 | ASSERT_EQ(new_options.max_write_buffer_number, 16); | |
748 | BlockBasedTableOptions new_block_based_table_options = | |
749 | dynamic_cast<BlockBasedTableFactory*>(new_options.table_factory.get()) | |
750 | ->table_options(); | |
751 | ASSERT_EQ(new_block_based_table_options.block_cache->GetCapacity(), 1U << 20); | |
752 | ASSERT_EQ(new_block_based_table_options.block_size, 4U); | |
753 | // don't overwrite block based table options | |
754 | ASSERT_TRUE(new_block_based_table_options.cache_index_and_filter_blocks); | |
755 | ||
756 | ASSERT_EQ(new_options.create_if_missing, true); | |
757 | ASSERT_EQ(new_options.max_open_files, 1); | |
758 | ASSERT_TRUE(new_options.rate_limiter.get() != nullptr); | |
759 | } | |
760 | ||
761 | TEST_F(OptionsTest, DBOptionsSerialization) { | |
762 | Options base_options, new_options; | |
763 | Random rnd(301); | |
764 | ||
765 | // Phase 1: Make big change in base_options | |
766 | test::RandomInitDBOptions(&base_options, &rnd); | |
767 | ||
768 | // Phase 2: obtain a string from base_option | |
769 | std::string base_options_file_content; | |
770 | ASSERT_OK(GetStringFromDBOptions(&base_options_file_content, base_options)); | |
771 | ||
772 | // Phase 3: Set new_options from the derived string and expect | |
773 | // new_options == base_options | |
774 | ASSERT_OK(GetDBOptionsFromString(DBOptions(), base_options_file_content, | |
775 | &new_options)); | |
776 | ASSERT_OK(RocksDBOptionsParser::VerifyDBOptions(base_options, new_options)); | |
777 | } | |
778 | ||
11fdf7f2 TL |
779 | TEST_F(OptionsTest, OptionsComposeDecompose) { |
780 | // build an Options from DBOptions + CFOptions, then decompose it to verify | |
781 | // we get same constituent options. | |
782 | DBOptions base_db_opts; | |
783 | ColumnFamilyOptions base_cf_opts; | |
784 | ||
785 | Random rnd(301); | |
786 | test::RandomInitDBOptions(&base_db_opts, &rnd); | |
787 | test::RandomInitCFOptions(&base_cf_opts, &rnd); | |
788 | ||
789 | Options base_opts(base_db_opts, base_cf_opts); | |
790 | DBOptions new_db_opts(base_opts); | |
791 | ColumnFamilyOptions new_cf_opts(base_opts); | |
792 | ||
793 | ASSERT_OK(RocksDBOptionsParser::VerifyDBOptions(base_db_opts, new_db_opts)); | |
794 | ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(base_cf_opts, new_cf_opts)); | |
795 | delete new_cf_opts.compaction_filter; | |
796 | } | |
797 | ||
7c673cae FG |
798 | TEST_F(OptionsTest, ColumnFamilyOptionsSerialization) { |
799 | ColumnFamilyOptions base_opt, new_opt; | |
800 | Random rnd(302); | |
801 | // Phase 1: randomly assign base_opt | |
802 | // custom type options | |
803 | test::RandomInitCFOptions(&base_opt, &rnd); | |
804 | ||
805 | // Phase 2: obtain a string from base_opt | |
806 | std::string base_options_file_content; | |
807 | ASSERT_OK( | |
808 | GetStringFromColumnFamilyOptions(&base_options_file_content, base_opt)); | |
809 | ||
810 | // Phase 3: Set new_opt from the derived string and expect | |
811 | // new_opt == base_opt | |
812 | ASSERT_OK(GetColumnFamilyOptionsFromString( | |
813 | ColumnFamilyOptions(), base_options_file_content, &new_opt)); | |
814 | ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(base_opt, new_opt)); | |
815 | if (base_opt.compaction_filter) { | |
816 | delete base_opt.compaction_filter; | |
817 | } | |
818 | } | |
819 | ||
820 | #endif // !ROCKSDB_LITE | |
821 | ||
822 | Status StringToMap( | |
823 | const std::string& opts_str, | |
824 | std::unordered_map<std::string, std::string>* opts_map); | |
825 | ||
826 | #ifndef ROCKSDB_LITE // StringToMap is not supported in ROCKSDB_LITE | |
827 | TEST_F(OptionsTest, StringToMapTest) { | |
828 | std::unordered_map<std::string, std::string> opts_map; | |
829 | // Regular options | |
830 | ASSERT_OK(StringToMap("k1=v1;k2=v2;k3=v3", &opts_map)); | |
831 | ASSERT_EQ(opts_map["k1"], "v1"); | |
832 | ASSERT_EQ(opts_map["k2"], "v2"); | |
833 | ASSERT_EQ(opts_map["k3"], "v3"); | |
834 | // Value with '=' | |
835 | opts_map.clear(); | |
836 | ASSERT_OK(StringToMap("k1==v1;k2=v2=;", &opts_map)); | |
837 | ASSERT_EQ(opts_map["k1"], "=v1"); | |
838 | ASSERT_EQ(opts_map["k2"], "v2="); | |
839 | // Overwrriten option | |
840 | opts_map.clear(); | |
841 | ASSERT_OK(StringToMap("k1=v1;k1=v2;k3=v3", &opts_map)); | |
842 | ASSERT_EQ(opts_map["k1"], "v2"); | |
843 | ASSERT_EQ(opts_map["k3"], "v3"); | |
844 | // Empty value | |
845 | opts_map.clear(); | |
846 | ASSERT_OK(StringToMap("k1=v1;k2=;k3=v3;k4=", &opts_map)); | |
847 | ASSERT_EQ(opts_map["k1"], "v1"); | |
848 | ASSERT_TRUE(opts_map.find("k2") != opts_map.end()); | |
849 | ASSERT_EQ(opts_map["k2"], ""); | |
850 | ASSERT_EQ(opts_map["k3"], "v3"); | |
851 | ASSERT_TRUE(opts_map.find("k4") != opts_map.end()); | |
852 | ASSERT_EQ(opts_map["k4"], ""); | |
853 | opts_map.clear(); | |
854 | ASSERT_OK(StringToMap("k1=v1;k2=;k3=v3;k4= ", &opts_map)); | |
855 | ASSERT_EQ(opts_map["k1"], "v1"); | |
856 | ASSERT_TRUE(opts_map.find("k2") != opts_map.end()); | |
857 | ASSERT_EQ(opts_map["k2"], ""); | |
858 | ASSERT_EQ(opts_map["k3"], "v3"); | |
859 | ASSERT_TRUE(opts_map.find("k4") != opts_map.end()); | |
860 | ASSERT_EQ(opts_map["k4"], ""); | |
861 | opts_map.clear(); | |
862 | ASSERT_OK(StringToMap("k1=v1;k2=;k3=", &opts_map)); | |
863 | ASSERT_EQ(opts_map["k1"], "v1"); | |
864 | ASSERT_TRUE(opts_map.find("k2") != opts_map.end()); | |
865 | ASSERT_EQ(opts_map["k2"], ""); | |
866 | ASSERT_TRUE(opts_map.find("k3") != opts_map.end()); | |
867 | ASSERT_EQ(opts_map["k3"], ""); | |
868 | opts_map.clear(); | |
869 | ASSERT_OK(StringToMap("k1=v1;k2=;k3=;", &opts_map)); | |
870 | ASSERT_EQ(opts_map["k1"], "v1"); | |
871 | ASSERT_TRUE(opts_map.find("k2") != opts_map.end()); | |
872 | ASSERT_EQ(opts_map["k2"], ""); | |
873 | ASSERT_TRUE(opts_map.find("k3") != opts_map.end()); | |
874 | ASSERT_EQ(opts_map["k3"], ""); | |
875 | // Regular nested options | |
876 | opts_map.clear(); | |
877 | ASSERT_OK(StringToMap("k1=v1;k2={nk1=nv1;nk2=nv2};k3=v3", &opts_map)); | |
878 | ASSERT_EQ(opts_map["k1"], "v1"); | |
879 | ASSERT_EQ(opts_map["k2"], "nk1=nv1;nk2=nv2"); | |
880 | ASSERT_EQ(opts_map["k3"], "v3"); | |
881 | // Multi-level nested options | |
882 | opts_map.clear(); | |
883 | ASSERT_OK(StringToMap("k1=v1;k2={nk1=nv1;nk2={nnk1=nnk2}};" | |
884 | "k3={nk1={nnk1={nnnk1=nnnv1;nnnk2;nnnv2}}};k4=v4", | |
885 | &opts_map)); | |
886 | ASSERT_EQ(opts_map["k1"], "v1"); | |
887 | ASSERT_EQ(opts_map["k2"], "nk1=nv1;nk2={nnk1=nnk2}"); | |
888 | ASSERT_EQ(opts_map["k3"], "nk1={nnk1={nnnk1=nnnv1;nnnk2;nnnv2}}"); | |
889 | ASSERT_EQ(opts_map["k4"], "v4"); | |
890 | // Garbage inside curly braces | |
891 | opts_map.clear(); | |
892 | ASSERT_OK(StringToMap("k1=v1;k2={dfad=};k3={=};k4=v4", | |
893 | &opts_map)); | |
894 | ASSERT_EQ(opts_map["k1"], "v1"); | |
895 | ASSERT_EQ(opts_map["k2"], "dfad="); | |
896 | ASSERT_EQ(opts_map["k3"], "="); | |
897 | ASSERT_EQ(opts_map["k4"], "v4"); | |
898 | // Empty nested options | |
899 | opts_map.clear(); | |
900 | ASSERT_OK(StringToMap("k1=v1;k2={};", &opts_map)); | |
901 | ASSERT_EQ(opts_map["k1"], "v1"); | |
902 | ASSERT_EQ(opts_map["k2"], ""); | |
903 | opts_map.clear(); | |
904 | ASSERT_OK(StringToMap("k1=v1;k2={{{{}}}{}{}};", &opts_map)); | |
905 | ASSERT_EQ(opts_map["k1"], "v1"); | |
906 | ASSERT_EQ(opts_map["k2"], "{{{}}}{}{}"); | |
907 | // With random spaces | |
908 | opts_map.clear(); | |
909 | ASSERT_OK(StringToMap(" k1 = v1 ; k2= {nk1=nv1; nk2={nnk1=nnk2}} ; " | |
910 | "k3={ { } }; k4= v4 ", | |
911 | &opts_map)); | |
912 | ASSERT_EQ(opts_map["k1"], "v1"); | |
913 | ASSERT_EQ(opts_map["k2"], "nk1=nv1; nk2={nnk1=nnk2}"); | |
914 | ASSERT_EQ(opts_map["k3"], "{ }"); | |
915 | ASSERT_EQ(opts_map["k4"], "v4"); | |
916 | ||
917 | // Empty key | |
918 | ASSERT_NOK(StringToMap("k1=v1;k2=v2;=", &opts_map)); | |
919 | ASSERT_NOK(StringToMap("=v1;k2=v2", &opts_map)); | |
920 | ASSERT_NOK(StringToMap("k1=v1;k2v2;", &opts_map)); | |
921 | ASSERT_NOK(StringToMap("k1=v1;k2=v2;fadfa", &opts_map)); | |
922 | ASSERT_NOK(StringToMap("k1=v1;k2=v2;;", &opts_map)); | |
923 | // Mismatch curly braces | |
924 | ASSERT_NOK(StringToMap("k1=v1;k2={;k3=v3", &opts_map)); | |
925 | ASSERT_NOK(StringToMap("k1=v1;k2={{};k3=v3", &opts_map)); | |
926 | ASSERT_NOK(StringToMap("k1=v1;k2={}};k3=v3", &opts_map)); | |
927 | ASSERT_NOK(StringToMap("k1=v1;k2={{}{}}};k3=v3", &opts_map)); | |
928 | // However this is valid! | |
929 | opts_map.clear(); | |
930 | ASSERT_OK(StringToMap("k1=v1;k2=};k3=v3", &opts_map)); | |
931 | ASSERT_EQ(opts_map["k1"], "v1"); | |
932 | ASSERT_EQ(opts_map["k2"], "}"); | |
933 | ASSERT_EQ(opts_map["k3"], "v3"); | |
934 | ||
935 | // Invalid chars after closing curly brace | |
936 | ASSERT_NOK(StringToMap("k1=v1;k2={{}}{};k3=v3", &opts_map)); | |
937 | ASSERT_NOK(StringToMap("k1=v1;k2={{}}cfda;k3=v3", &opts_map)); | |
938 | ASSERT_NOK(StringToMap("k1=v1;k2={{}} cfda;k3=v3", &opts_map)); | |
939 | ASSERT_NOK(StringToMap("k1=v1;k2={{}} cfda", &opts_map)); | |
940 | ASSERT_NOK(StringToMap("k1=v1;k2={{}}{}", &opts_map)); | |
941 | ASSERT_NOK(StringToMap("k1=v1;k2={{dfdl}adfa}{}", &opts_map)); | |
942 | } | |
943 | #endif // ROCKSDB_LITE | |
944 | ||
945 | #ifndef ROCKSDB_LITE // StringToMap is not supported in ROCKSDB_LITE | |
946 | TEST_F(OptionsTest, StringToMapRandomTest) { | |
947 | std::unordered_map<std::string, std::string> opts_map; | |
948 | // Make sure segfault is not hit by semi-random strings | |
949 | ||
950 | std::vector<std::string> bases = { | |
951 | "a={aa={};tt={xxx={}}};c=defff", | |
952 | "a={aa={};tt={xxx={}}};c=defff;d={{}yxx{}3{xx}}", | |
953 | "abc={{}{}{}{{{}}}{{}{}{}{}{}{}{}"}; | |
954 | ||
955 | for (std::string base : bases) { | |
956 | for (int rand_seed = 301; rand_seed < 401; rand_seed++) { | |
957 | Random rnd(rand_seed); | |
958 | for (int attempt = 0; attempt < 10; attempt++) { | |
959 | std::string str = base; | |
960 | // Replace random position to space | |
961 | size_t pos = static_cast<size_t>( | |
962 | rnd.Uniform(static_cast<int>(base.size()))); | |
963 | str[pos] = ' '; | |
964 | Status s = StringToMap(str, &opts_map); | |
965 | ASSERT_TRUE(s.ok() || s.IsInvalidArgument()); | |
966 | opts_map.clear(); | |
967 | } | |
968 | } | |
969 | } | |
970 | ||
971 | // Random Construct a string | |
972 | std::vector<char> chars = {'{', '}', ' ', '=', ';', 'c'}; | |
973 | for (int rand_seed = 301; rand_seed < 1301; rand_seed++) { | |
974 | Random rnd(rand_seed); | |
975 | int len = rnd.Uniform(30); | |
976 | std::string str = ""; | |
977 | for (int attempt = 0; attempt < len; attempt++) { | |
978 | // Add a random character | |
979 | size_t pos = static_cast<size_t>( | |
980 | rnd.Uniform(static_cast<int>(chars.size()))); | |
981 | str.append(1, chars[pos]); | |
982 | } | |
983 | Status s = StringToMap(str, &opts_map); | |
984 | ASSERT_TRUE(s.ok() || s.IsInvalidArgument()); | |
985 | s = StringToMap("name=" + str, &opts_map); | |
986 | ASSERT_TRUE(s.ok() || s.IsInvalidArgument()); | |
987 | opts_map.clear(); | |
988 | } | |
989 | } | |
990 | ||
991 | TEST_F(OptionsTest, GetStringFromCompressionType) { | |
992 | std::string res; | |
993 | ||
994 | ASSERT_OK(GetStringFromCompressionType(&res, kNoCompression)); | |
995 | ASSERT_EQ(res, "kNoCompression"); | |
996 | ||
997 | ASSERT_OK(GetStringFromCompressionType(&res, kSnappyCompression)); | |
998 | ASSERT_EQ(res, "kSnappyCompression"); | |
999 | ||
1000 | ASSERT_OK(GetStringFromCompressionType(&res, kDisableCompressionOption)); | |
1001 | ASSERT_EQ(res, "kDisableCompressionOption"); | |
1002 | ||
1003 | ASSERT_OK(GetStringFromCompressionType(&res, kLZ4Compression)); | |
1004 | ASSERT_EQ(res, "kLZ4Compression"); | |
1005 | ||
1006 | ASSERT_OK(GetStringFromCompressionType(&res, kZlibCompression)); | |
1007 | ASSERT_EQ(res, "kZlibCompression"); | |
1008 | ||
1009 | ASSERT_NOK( | |
1010 | GetStringFromCompressionType(&res, static_cast<CompressionType>(-10))); | |
1011 | } | |
1012 | #endif // !ROCKSDB_LITE | |
1013 | ||
1014 | TEST_F(OptionsTest, ConvertOptionsTest) { | |
1015 | LevelDBOptions leveldb_opt; | |
1016 | Options converted_opt = ConvertOptions(leveldb_opt); | |
1017 | ||
1018 | ASSERT_EQ(converted_opt.create_if_missing, leveldb_opt.create_if_missing); | |
1019 | ASSERT_EQ(converted_opt.error_if_exists, leveldb_opt.error_if_exists); | |
1020 | ASSERT_EQ(converted_opt.paranoid_checks, leveldb_opt.paranoid_checks); | |
1021 | ASSERT_EQ(converted_opt.env, leveldb_opt.env); | |
1022 | ASSERT_EQ(converted_opt.info_log.get(), leveldb_opt.info_log); | |
1023 | ASSERT_EQ(converted_opt.write_buffer_size, leveldb_opt.write_buffer_size); | |
1024 | ASSERT_EQ(converted_opt.max_open_files, leveldb_opt.max_open_files); | |
1025 | ASSERT_EQ(converted_opt.compression, leveldb_opt.compression); | |
1026 | ||
11fdf7f2 TL |
1027 | std::shared_ptr<TableFactory> tb_guard = converted_opt.table_factory; |
1028 | BlockBasedTableFactory* table_factory = | |
1029 | dynamic_cast<BlockBasedTableFactory*>(converted_opt.table_factory.get()); | |
7c673cae | 1030 | |
11fdf7f2 | 1031 | ASSERT_TRUE(table_factory != nullptr); |
7c673cae FG |
1032 | |
1033 | const BlockBasedTableOptions table_opt = table_factory->table_options(); | |
1034 | ||
1035 | ASSERT_EQ(table_opt.block_cache->GetCapacity(), 8UL << 20); | |
1036 | ASSERT_EQ(table_opt.block_size, leveldb_opt.block_size); | |
1037 | ASSERT_EQ(table_opt.block_restart_interval, | |
1038 | leveldb_opt.block_restart_interval); | |
1039 | ASSERT_EQ(table_opt.filter_policy.get(), leveldb_opt.filter_policy); | |
1040 | } | |
1041 | ||
1042 | #ifndef ROCKSDB_LITE | |
1043 | class OptionsParserTest : public testing::Test { | |
1044 | public: | |
1045 | OptionsParserTest() { env_.reset(new test::StringEnv(Env::Default())); } | |
1046 | ||
1047 | protected: | |
1048 | std::unique_ptr<test::StringEnv> env_; | |
1049 | }; | |
1050 | ||
1051 | TEST_F(OptionsParserTest, Comment) { | |
1052 | DBOptions db_opt; | |
1053 | db_opt.max_open_files = 12345; | |
1054 | db_opt.max_background_flushes = 301; | |
1055 | db_opt.max_total_wal_size = 1024; | |
1056 | ColumnFamilyOptions cf_opt; | |
1057 | ||
1058 | std::string options_file_content = | |
1059 | "# This is a testing option string.\n" | |
1060 | "# Currently we only support \"#\" styled comment.\n" | |
1061 | "\n" | |
1062 | "[Version]\n" | |
1063 | " rocksdb_version=3.14.0\n" | |
1064 | " options_file_version=1\n" | |
1065 | "[ DBOptions ]\n" | |
1066 | " # note that we don't support space around \"=\"\n" | |
1067 | " max_open_files=12345;\n" | |
1068 | " max_background_flushes=301 # comment after a statement is fine\n" | |
1069 | " # max_background_flushes=1000 # this line would be ignored\n" | |
1070 | " # max_background_compactions=2000 # so does this one\n" | |
1071 | " max_total_wal_size=1024 # keep_log_file_num=1000\n" | |
1072 | "[CFOptions \"default\"] # column family must be specified\n" | |
1073 | " # in the correct order\n" | |
1074 | " # if a section is blank, we will use the default\n"; | |
1075 | ||
1076 | const std::string kTestFileName = "test-rocksdb-options.ini"; | |
1077 | env_->WriteToNewFile(kTestFileName, options_file_content); | |
1078 | RocksDBOptionsParser parser; | |
1079 | ASSERT_OK(parser.Parse(kTestFileName, env_.get())); | |
1080 | ||
1081 | ASSERT_OK(RocksDBOptionsParser::VerifyDBOptions(*parser.db_opt(), db_opt)); | |
1082 | ASSERT_EQ(parser.NumColumnFamilies(), 1U); | |
1083 | ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions( | |
1084 | *parser.GetCFOptions("default"), cf_opt)); | |
1085 | } | |
1086 | ||
1087 | TEST_F(OptionsParserTest, ExtraSpace) { | |
1088 | std::string options_file_content = | |
1089 | "# This is a testing option string.\n" | |
1090 | "# Currently we only support \"#\" styled comment.\n" | |
1091 | "\n" | |
1092 | "[ Version ]\n" | |
1093 | " rocksdb_version = 3.14.0 \n" | |
1094 | " options_file_version=1 # some comment\n" | |
1095 | "[DBOptions ] # some comment\n" | |
1096 | "max_open_files=12345 \n" | |
1097 | " max_background_flushes = 301 \n" | |
1098 | " max_total_wal_size = 1024 # keep_log_file_num=1000\n" | |
1099 | " [CFOptions \"default\" ]\n" | |
1100 | " # if a section is blank, we will use the default\n"; | |
1101 | ||
1102 | const std::string kTestFileName = "test-rocksdb-options.ini"; | |
1103 | env_->WriteToNewFile(kTestFileName, options_file_content); | |
1104 | RocksDBOptionsParser parser; | |
1105 | ASSERT_OK(parser.Parse(kTestFileName, env_.get())); | |
1106 | } | |
1107 | ||
1108 | TEST_F(OptionsParserTest, MissingDBOptions) { | |
1109 | std::string options_file_content = | |
1110 | "# This is a testing option string.\n" | |
1111 | "# Currently we only support \"#\" styled comment.\n" | |
1112 | "\n" | |
1113 | "[Version]\n" | |
1114 | " rocksdb_version=3.14.0\n" | |
1115 | " options_file_version=1\n" | |
1116 | "[CFOptions \"default\"]\n" | |
1117 | " # if a section is blank, we will use the default\n"; | |
1118 | ||
1119 | const std::string kTestFileName = "test-rocksdb-options.ini"; | |
1120 | env_->WriteToNewFile(kTestFileName, options_file_content); | |
1121 | RocksDBOptionsParser parser; | |
1122 | ASSERT_NOK(parser.Parse(kTestFileName, env_.get())); | |
1123 | } | |
1124 | ||
1125 | TEST_F(OptionsParserTest, DoubleDBOptions) { | |
1126 | DBOptions db_opt; | |
1127 | db_opt.max_open_files = 12345; | |
1128 | db_opt.max_background_flushes = 301; | |
1129 | db_opt.max_total_wal_size = 1024; | |
1130 | ColumnFamilyOptions cf_opt; | |
1131 | ||
1132 | std::string options_file_content = | |
1133 | "# This is a testing option string.\n" | |
1134 | "# Currently we only support \"#\" styled comment.\n" | |
1135 | "\n" | |
1136 | "[Version]\n" | |
1137 | " rocksdb_version=3.14.0\n" | |
1138 | " options_file_version=1\n" | |
1139 | "[DBOptions]\n" | |
1140 | " max_open_files=12345\n" | |
1141 | " max_background_flushes=301\n" | |
1142 | " max_total_wal_size=1024 # keep_log_file_num=1000\n" | |
1143 | "[DBOptions]\n" | |
1144 | "[CFOptions \"default\"]\n" | |
1145 | " # if a section is blank, we will use the default\n"; | |
1146 | ||
1147 | const std::string kTestFileName = "test-rocksdb-options.ini"; | |
1148 | env_->WriteToNewFile(kTestFileName, options_file_content); | |
1149 | RocksDBOptionsParser parser; | |
1150 | ASSERT_NOK(parser.Parse(kTestFileName, env_.get())); | |
1151 | } | |
1152 | ||
1153 | TEST_F(OptionsParserTest, NoDefaultCFOptions) { | |
1154 | DBOptions db_opt; | |
1155 | db_opt.max_open_files = 12345; | |
1156 | db_opt.max_background_flushes = 301; | |
1157 | db_opt.max_total_wal_size = 1024; | |
1158 | ColumnFamilyOptions cf_opt; | |
1159 | ||
1160 | std::string options_file_content = | |
1161 | "# This is a testing option string.\n" | |
1162 | "# Currently we only support \"#\" styled comment.\n" | |
1163 | "\n" | |
1164 | "[Version]\n" | |
1165 | " rocksdb_version=3.14.0\n" | |
1166 | " options_file_version=1\n" | |
1167 | "[DBOptions]\n" | |
1168 | " max_open_files=12345\n" | |
1169 | " max_background_flushes=301\n" | |
1170 | " max_total_wal_size=1024 # keep_log_file_num=1000\n" | |
1171 | "[CFOptions \"something_else\"]\n" | |
1172 | " # if a section is blank, we will use the default\n"; | |
1173 | ||
1174 | const std::string kTestFileName = "test-rocksdb-options.ini"; | |
1175 | env_->WriteToNewFile(kTestFileName, options_file_content); | |
1176 | RocksDBOptionsParser parser; | |
1177 | ASSERT_NOK(parser.Parse(kTestFileName, env_.get())); | |
1178 | } | |
1179 | ||
1180 | TEST_F(OptionsParserTest, DefaultCFOptionsMustBeTheFirst) { | |
1181 | DBOptions db_opt; | |
1182 | db_opt.max_open_files = 12345; | |
1183 | db_opt.max_background_flushes = 301; | |
1184 | db_opt.max_total_wal_size = 1024; | |
1185 | ColumnFamilyOptions cf_opt; | |
1186 | ||
1187 | std::string options_file_content = | |
1188 | "# This is a testing option string.\n" | |
1189 | "# Currently we only support \"#\" styled comment.\n" | |
1190 | "\n" | |
1191 | "[Version]\n" | |
1192 | " rocksdb_version=3.14.0\n" | |
1193 | " options_file_version=1\n" | |
1194 | "[DBOptions]\n" | |
1195 | " max_open_files=12345\n" | |
1196 | " max_background_flushes=301\n" | |
1197 | " max_total_wal_size=1024 # keep_log_file_num=1000\n" | |
1198 | "[CFOptions \"something_else\"]\n" | |
1199 | " # if a section is blank, we will use the default\n" | |
1200 | "[CFOptions \"default\"]\n" | |
1201 | " # if a section is blank, we will use the default\n"; | |
1202 | ||
1203 | const std::string kTestFileName = "test-rocksdb-options.ini"; | |
1204 | env_->WriteToNewFile(kTestFileName, options_file_content); | |
1205 | RocksDBOptionsParser parser; | |
1206 | ASSERT_NOK(parser.Parse(kTestFileName, env_.get())); | |
1207 | } | |
1208 | ||
1209 | TEST_F(OptionsParserTest, DuplicateCFOptions) { | |
1210 | DBOptions db_opt; | |
1211 | db_opt.max_open_files = 12345; | |
1212 | db_opt.max_background_flushes = 301; | |
1213 | db_opt.max_total_wal_size = 1024; | |
1214 | ColumnFamilyOptions cf_opt; | |
1215 | ||
1216 | std::string options_file_content = | |
1217 | "# This is a testing option string.\n" | |
1218 | "# Currently we only support \"#\" styled comment.\n" | |
1219 | "\n" | |
1220 | "[Version]\n" | |
1221 | " rocksdb_version=3.14.0\n" | |
1222 | " options_file_version=1\n" | |
1223 | "[DBOptions]\n" | |
1224 | " max_open_files=12345\n" | |
1225 | " max_background_flushes=301\n" | |
1226 | " max_total_wal_size=1024 # keep_log_file_num=1000\n" | |
1227 | "[CFOptions \"default\"]\n" | |
1228 | "[CFOptions \"something_else\"]\n" | |
1229 | "[CFOptions \"something_else\"]\n"; | |
1230 | ||
1231 | const std::string kTestFileName = "test-rocksdb-options.ini"; | |
1232 | env_->WriteToNewFile(kTestFileName, options_file_content); | |
1233 | RocksDBOptionsParser parser; | |
1234 | ASSERT_NOK(parser.Parse(kTestFileName, env_.get())); | |
1235 | } | |
1236 | ||
11fdf7f2 TL |
1237 | TEST_F(OptionsParserTest, IgnoreUnknownOptions) { |
1238 | for (int case_id = 0; case_id < 5; case_id++) { | |
1239 | DBOptions db_opt; | |
1240 | db_opt.max_open_files = 12345; | |
1241 | db_opt.max_background_flushes = 301; | |
1242 | db_opt.max_total_wal_size = 1024; | |
1243 | ColumnFamilyOptions cf_opt; | |
1244 | ||
1245 | std::string version_string; | |
1246 | bool should_ignore = true; | |
1247 | if (case_id == 0) { | |
1248 | // same version | |
1249 | should_ignore = false; | |
1250 | version_string = | |
1251 | ToString(ROCKSDB_MAJOR) + "." + ToString(ROCKSDB_MINOR) + ".0"; | |
1252 | } else if (case_id == 1) { | |
1253 | // higher minor version | |
1254 | should_ignore = true; | |
1255 | version_string = | |
1256 | ToString(ROCKSDB_MAJOR) + "." + ToString(ROCKSDB_MINOR + 1) + ".0"; | |
1257 | } else if (case_id == 2) { | |
1258 | // higher major version. | |
1259 | should_ignore = true; | |
1260 | version_string = ToString(ROCKSDB_MAJOR + 1) + ".0.0"; | |
1261 | } else if (case_id == 3) { | |
1262 | // lower minor version | |
1263 | #if ROCKSDB_MINOR == 0 | |
1264 | continue; | |
1265 | #else | |
1266 | version_string = | |
1267 | ToString(ROCKSDB_MAJOR) + "." + ToString(ROCKSDB_MINOR - 1) + ".0"; | |
1268 | should_ignore = false; | |
1269 | #endif | |
1270 | } else { | |
1271 | // lower major version | |
1272 | should_ignore = false; | |
1273 | version_string = | |
1274 | ToString(ROCKSDB_MAJOR - 1) + "." + ToString(ROCKSDB_MINOR) + ".0"; | |
1275 | } | |
1276 | ||
1277 | std::string options_file_content = | |
1278 | "# This is a testing option string.\n" | |
1279 | "# Currently we only support \"#\" styled comment.\n" | |
1280 | "\n" | |
1281 | "[Version]\n" | |
1282 | " rocksdb_version=" + | |
1283 | version_string + | |
1284 | "\n" | |
1285 | " options_file_version=1\n" | |
1286 | "[DBOptions]\n" | |
1287 | " max_open_files=12345\n" | |
1288 | " max_background_flushes=301\n" | |
1289 | " max_total_wal_size=1024 # keep_log_file_num=1000\n" | |
1290 | " unknown_db_option1=321\n" | |
1291 | " unknown_db_option2=false\n" | |
1292 | "[CFOptions \"default\"]\n" | |
1293 | " unknown_cf_option1=hello\n" | |
1294 | "[CFOptions \"something_else\"]\n" | |
1295 | " unknown_cf_option2=world\n" | |
1296 | " # if a section is blank, we will use the default\n"; | |
1297 | ||
1298 | const std::string kTestFileName = "test-rocksdb-options.ini"; | |
1299 | env_->DeleteFile(kTestFileName); | |
1300 | env_->WriteToNewFile(kTestFileName, options_file_content); | |
1301 | RocksDBOptionsParser parser; | |
1302 | ASSERT_NOK(parser.Parse(kTestFileName, env_.get())); | |
1303 | if (should_ignore) { | |
1304 | ASSERT_OK(parser.Parse(kTestFileName, env_.get(), | |
1305 | true /* ignore_unknown_options */)); | |
1306 | } else { | |
1307 | ASSERT_NOK(parser.Parse(kTestFileName, env_.get(), | |
1308 | true /* ignore_unknown_options */)); | |
1309 | } | |
1310 | } | |
1311 | } | |
1312 | ||
7c673cae FG |
1313 | TEST_F(OptionsParserTest, ParseVersion) { |
1314 | DBOptions db_opt; | |
1315 | db_opt.max_open_files = 12345; | |
1316 | db_opt.max_background_flushes = 301; | |
1317 | db_opt.max_total_wal_size = 1024; | |
1318 | ColumnFamilyOptions cf_opt; | |
1319 | ||
1320 | std::string file_template = | |
1321 | "# This is a testing option string.\n" | |
1322 | "# Currently we only support \"#\" styled comment.\n" | |
1323 | "\n" | |
1324 | "[Version]\n" | |
1325 | " rocksdb_version=3.13.1\n" | |
1326 | " options_file_version=%s\n" | |
1327 | "[DBOptions]\n" | |
1328 | "[CFOptions \"default\"]\n"; | |
1329 | const int kLength = 1000; | |
1330 | char buffer[kLength]; | |
1331 | RocksDBOptionsParser parser; | |
1332 | ||
1333 | const std::vector<std::string> invalid_versions = { | |
1334 | "a.b.c", "3.2.2b", "3.-12", "3. 1", // only digits and dots are allowed | |
1335 | "1.2.3.4", | |
1336 | "1.2.3" // can only contains at most one dot. | |
1337 | "0", // options_file_version must be at least one | |
1338 | "3..2", | |
1339 | ".", ".1.2", // must have at least one digit before each dot | |
1340 | "1.2.", "1.", "2.34."}; // must have at least one digit after each dot | |
1341 | for (auto iv : invalid_versions) { | |
1342 | snprintf(buffer, kLength - 1, file_template.c_str(), iv.c_str()); | |
1343 | ||
1344 | parser.Reset(); | |
1345 | env_->WriteToNewFile(iv, buffer); | |
1346 | ASSERT_NOK(parser.Parse(iv, env_.get())); | |
1347 | } | |
1348 | ||
1349 | const std::vector<std::string> valid_versions = { | |
1350 | "1.232", "100", "3.12", "1", "12.3 ", " 1.25 "}; | |
1351 | for (auto vv : valid_versions) { | |
1352 | snprintf(buffer, kLength - 1, file_template.c_str(), vv.c_str()); | |
1353 | parser.Reset(); | |
1354 | env_->WriteToNewFile(vv, buffer); | |
1355 | ASSERT_OK(parser.Parse(vv, env_.get())); | |
1356 | } | |
1357 | } | |
1358 | ||
1359 | void VerifyCFPointerTypedOptions( | |
1360 | ColumnFamilyOptions* base_cf_opt, const ColumnFamilyOptions* new_cf_opt, | |
1361 | const std::unordered_map<std::string, std::string>* new_cf_opt_map) { | |
1362 | std::string name_buffer; | |
1363 | ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(*base_cf_opt, *new_cf_opt, | |
1364 | new_cf_opt_map)); | |
1365 | ||
1366 | // change the name of merge operator back-and-forth | |
1367 | { | |
1368 | auto* merge_operator = dynamic_cast<test::ChanglingMergeOperator*>( | |
1369 | base_cf_opt->merge_operator.get()); | |
1370 | if (merge_operator != nullptr) { | |
1371 | name_buffer = merge_operator->Name(); | |
1372 | // change the name and expect non-ok status | |
1373 | merge_operator->SetName("some-other-name"); | |
1374 | ASSERT_NOK(RocksDBOptionsParser::VerifyCFOptions( | |
1375 | *base_cf_opt, *new_cf_opt, new_cf_opt_map)); | |
1376 | // change the name back and expect ok status | |
1377 | merge_operator->SetName(name_buffer); | |
1378 | ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(*base_cf_opt, *new_cf_opt, | |
1379 | new_cf_opt_map)); | |
1380 | } | |
1381 | } | |
1382 | ||
1383 | // change the name of the compaction filter factory back-and-forth | |
1384 | { | |
1385 | auto* compaction_filter_factory = | |
1386 | dynamic_cast<test::ChanglingCompactionFilterFactory*>( | |
1387 | base_cf_opt->compaction_filter_factory.get()); | |
1388 | if (compaction_filter_factory != nullptr) { | |
1389 | name_buffer = compaction_filter_factory->Name(); | |
1390 | // change the name and expect non-ok status | |
1391 | compaction_filter_factory->SetName("some-other-name"); | |
1392 | ASSERT_NOK(RocksDBOptionsParser::VerifyCFOptions( | |
1393 | *base_cf_opt, *new_cf_opt, new_cf_opt_map)); | |
1394 | // change the name back and expect ok status | |
1395 | compaction_filter_factory->SetName(name_buffer); | |
1396 | ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(*base_cf_opt, *new_cf_opt, | |
1397 | new_cf_opt_map)); | |
1398 | } | |
1399 | } | |
1400 | ||
1401 | // test by setting compaction_filter to nullptr | |
1402 | { | |
1403 | auto* tmp_compaction_filter = base_cf_opt->compaction_filter; | |
1404 | if (tmp_compaction_filter != nullptr) { | |
1405 | base_cf_opt->compaction_filter = nullptr; | |
1406 | // set compaction_filter to nullptr and expect non-ok status | |
1407 | ASSERT_NOK(RocksDBOptionsParser::VerifyCFOptions( | |
1408 | *base_cf_opt, *new_cf_opt, new_cf_opt_map)); | |
1409 | // set the value back and expect ok status | |
1410 | base_cf_opt->compaction_filter = tmp_compaction_filter; | |
1411 | ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(*base_cf_opt, *new_cf_opt, | |
1412 | new_cf_opt_map)); | |
1413 | } | |
1414 | } | |
1415 | ||
1416 | // test by setting table_factory to nullptr | |
1417 | { | |
1418 | auto tmp_table_factory = base_cf_opt->table_factory; | |
1419 | if (tmp_table_factory != nullptr) { | |
1420 | base_cf_opt->table_factory.reset(); | |
1421 | // set table_factory to nullptr and expect non-ok status | |
1422 | ASSERT_NOK(RocksDBOptionsParser::VerifyCFOptions( | |
1423 | *base_cf_opt, *new_cf_opt, new_cf_opt_map)); | |
1424 | // set the value back and expect ok status | |
1425 | base_cf_opt->table_factory = tmp_table_factory; | |
1426 | ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(*base_cf_opt, *new_cf_opt, | |
1427 | new_cf_opt_map)); | |
1428 | } | |
1429 | } | |
1430 | ||
1431 | // test by setting memtable_factory to nullptr | |
1432 | { | |
1433 | auto tmp_memtable_factory = base_cf_opt->memtable_factory; | |
1434 | if (tmp_memtable_factory != nullptr) { | |
1435 | base_cf_opt->memtable_factory.reset(); | |
1436 | // set memtable_factory to nullptr and expect non-ok status | |
1437 | ASSERT_NOK(RocksDBOptionsParser::VerifyCFOptions( | |
1438 | *base_cf_opt, *new_cf_opt, new_cf_opt_map)); | |
1439 | // set the value back and expect ok status | |
1440 | base_cf_opt->memtable_factory = tmp_memtable_factory; | |
1441 | ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(*base_cf_opt, *new_cf_opt, | |
1442 | new_cf_opt_map)); | |
1443 | } | |
1444 | } | |
1445 | } | |
1446 | ||
1447 | TEST_F(OptionsParserTest, DumpAndParse) { | |
1448 | DBOptions base_db_opt; | |
1449 | std::vector<ColumnFamilyOptions> base_cf_opts; | |
1450 | std::vector<std::string> cf_names = {"default", "cf1", "cf2", "cf3", | |
1451 | "c:f:4:4:4" | |
1452 | "p\\i\\k\\a\\chu\\\\\\", | |
1453 | "###rocksdb#1-testcf#2###"}; | |
1454 | const int num_cf = static_cast<int>(cf_names.size()); | |
1455 | Random rnd(302); | |
1456 | test::RandomInitDBOptions(&base_db_opt, &rnd); | |
1457 | base_db_opt.db_log_dir += "/#odd #but #could #happen #path #/\\\\#OMG"; | |
11fdf7f2 TL |
1458 | |
1459 | BlockBasedTableOptions special_bbto; | |
1460 | special_bbto.cache_index_and_filter_blocks = true; | |
1461 | special_bbto.block_size = 999999; | |
1462 | ||
7c673cae FG |
1463 | for (int c = 0; c < num_cf; ++c) { |
1464 | ColumnFamilyOptions cf_opt; | |
1465 | Random cf_rnd(0xFB + c); | |
1466 | test::RandomInitCFOptions(&cf_opt, &cf_rnd); | |
1467 | if (c < 4) { | |
1468 | cf_opt.prefix_extractor.reset(test::RandomSliceTransform(&rnd, c)); | |
1469 | } | |
1470 | if (c < 3) { | |
1471 | cf_opt.table_factory.reset(test::RandomTableFactory(&rnd, c)); | |
11fdf7f2 TL |
1472 | } else if (c == 4) { |
1473 | cf_opt.table_factory.reset(NewBlockBasedTableFactory(special_bbto)); | |
7c673cae FG |
1474 | } |
1475 | base_cf_opts.emplace_back(cf_opt); | |
1476 | } | |
1477 | ||
1478 | const std::string kOptionsFileName = "test-persisted-options.ini"; | |
1479 | ASSERT_OK(PersistRocksDBOptions(base_db_opt, cf_names, base_cf_opts, | |
1480 | kOptionsFileName, env_.get())); | |
1481 | ||
1482 | RocksDBOptionsParser parser; | |
1483 | ASSERT_OK(parser.Parse(kOptionsFileName, env_.get())); | |
1484 | ||
11fdf7f2 TL |
1485 | // Make sure block-based table factory options was deserialized correctly |
1486 | std::shared_ptr<TableFactory> ttf = (*parser.cf_opts())[4].table_factory; | |
1487 | ASSERT_EQ(BlockBasedTableFactory::kName, std::string(ttf->Name())); | |
1488 | const BlockBasedTableOptions& parsed_bbto = | |
1489 | static_cast<BlockBasedTableFactory*>(ttf.get())->table_options(); | |
1490 | ASSERT_EQ(special_bbto.block_size, parsed_bbto.block_size); | |
1491 | ASSERT_EQ(special_bbto.cache_index_and_filter_blocks, | |
1492 | parsed_bbto.cache_index_and_filter_blocks); | |
1493 | ||
7c673cae FG |
1494 | ASSERT_OK(RocksDBOptionsParser::VerifyRocksDBOptionsFromFile( |
1495 | base_db_opt, cf_names, base_cf_opts, kOptionsFileName, env_.get())); | |
1496 | ||
1497 | ASSERT_OK( | |
1498 | RocksDBOptionsParser::VerifyDBOptions(*parser.db_opt(), base_db_opt)); | |
1499 | for (int c = 0; c < num_cf; ++c) { | |
1500 | const auto* cf_opt = parser.GetCFOptions(cf_names[c]); | |
1501 | ASSERT_NE(cf_opt, nullptr); | |
1502 | ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions( | |
1503 | base_cf_opts[c], *cf_opt, &(parser.cf_opt_maps()->at(c)))); | |
1504 | } | |
1505 | ||
1506 | // Further verify pointer-typed options | |
1507 | for (int c = 0; c < num_cf; ++c) { | |
1508 | const auto* cf_opt = parser.GetCFOptions(cf_names[c]); | |
1509 | ASSERT_NE(cf_opt, nullptr); | |
1510 | VerifyCFPointerTypedOptions(&base_cf_opts[c], cf_opt, | |
1511 | &(parser.cf_opt_maps()->at(c))); | |
1512 | } | |
1513 | ||
1514 | ASSERT_EQ(parser.GetCFOptions("does not exist"), nullptr); | |
1515 | ||
1516 | base_db_opt.max_open_files++; | |
1517 | ASSERT_NOK(RocksDBOptionsParser::VerifyRocksDBOptionsFromFile( | |
1518 | base_db_opt, cf_names, base_cf_opts, kOptionsFileName, env_.get())); | |
1519 | ||
1520 | for (int c = 0; c < num_cf; ++c) { | |
1521 | if (base_cf_opts[c].compaction_filter) { | |
1522 | delete base_cf_opts[c].compaction_filter; | |
1523 | } | |
1524 | } | |
1525 | } | |
1526 | ||
1527 | TEST_F(OptionsParserTest, DifferentDefault) { | |
1528 | const std::string kOptionsFileName = "test-persisted-options.ini"; | |
1529 | ||
1530 | ColumnFamilyOptions cf_level_opts; | |
1531 | cf_level_opts.OptimizeLevelStyleCompaction(); | |
1532 | ||
1533 | ColumnFamilyOptions cf_univ_opts; | |
1534 | cf_univ_opts.OptimizeUniversalStyleCompaction(); | |
1535 | ||
1536 | ASSERT_OK(PersistRocksDBOptions(DBOptions(), {"default", "universal"}, | |
1537 | {cf_level_opts, cf_univ_opts}, | |
1538 | kOptionsFileName, env_.get())); | |
1539 | ||
1540 | RocksDBOptionsParser parser; | |
1541 | ASSERT_OK(parser.Parse(kOptionsFileName, env_.get())); | |
1542 | ||
1543 | { | |
1544 | Options old_default_opts; | |
1545 | old_default_opts.OldDefaults(); | |
1546 | ASSERT_EQ(10 * 1048576, old_default_opts.max_bytes_for_level_base); | |
1547 | ASSERT_EQ(5000, old_default_opts.max_open_files); | |
7c673cae FG |
1548 | ASSERT_EQ(2 * 1024U * 1024U, old_default_opts.delayed_write_rate); |
1549 | ASSERT_EQ(WALRecoveryMode::kTolerateCorruptedTailRecords, | |
1550 | old_default_opts.wal_recovery_mode); | |
1551 | } | |
1552 | { | |
1553 | Options old_default_opts; | |
1554 | old_default_opts.OldDefaults(4, 6); | |
1555 | ASSERT_EQ(10 * 1048576, old_default_opts.max_bytes_for_level_base); | |
1556 | ASSERT_EQ(5000, old_default_opts.max_open_files); | |
1557 | } | |
1558 | { | |
1559 | Options old_default_opts; | |
1560 | old_default_opts.OldDefaults(4, 7); | |
1561 | ASSERT_NE(10 * 1048576, old_default_opts.max_bytes_for_level_base); | |
1562 | ASSERT_NE(4, old_default_opts.table_cache_numshardbits); | |
1563 | ASSERT_EQ(5000, old_default_opts.max_open_files); | |
1564 | ASSERT_EQ(2 * 1024U * 1024U, old_default_opts.delayed_write_rate); | |
1565 | } | |
1566 | { | |
1567 | ColumnFamilyOptions old_default_cf_opts; | |
1568 | old_default_cf_opts.OldDefaults(); | |
1569 | ASSERT_EQ(2 * 1048576, old_default_cf_opts.target_file_size_base); | |
1570 | ASSERT_EQ(4 << 20, old_default_cf_opts.write_buffer_size); | |
1571 | ASSERT_EQ(2 * 1048576, old_default_cf_opts.target_file_size_base); | |
1572 | ASSERT_EQ(0, old_default_cf_opts.soft_pending_compaction_bytes_limit); | |
1573 | ASSERT_EQ(0, old_default_cf_opts.hard_pending_compaction_bytes_limit); | |
1574 | ASSERT_EQ(CompactionPri::kByCompensatedSize, | |
1575 | old_default_cf_opts.compaction_pri); | |
1576 | } | |
1577 | { | |
1578 | ColumnFamilyOptions old_default_cf_opts; | |
1579 | old_default_cf_opts.OldDefaults(4, 6); | |
1580 | ASSERT_EQ(2 * 1048576, old_default_cf_opts.target_file_size_base); | |
1581 | ASSERT_EQ(CompactionPri::kByCompensatedSize, | |
1582 | old_default_cf_opts.compaction_pri); | |
1583 | } | |
1584 | { | |
1585 | ColumnFamilyOptions old_default_cf_opts; | |
1586 | old_default_cf_opts.OldDefaults(4, 7); | |
1587 | ASSERT_NE(2 * 1048576, old_default_cf_opts.target_file_size_base); | |
1588 | ASSERT_EQ(CompactionPri::kByCompensatedSize, | |
1589 | old_default_cf_opts.compaction_pri); | |
1590 | } | |
1591 | { | |
1592 | Options old_default_opts; | |
1593 | old_default_opts.OldDefaults(5, 1); | |
1594 | ASSERT_EQ(2 * 1024U * 1024U, old_default_opts.delayed_write_rate); | |
1595 | } | |
1596 | { | |
1597 | Options old_default_opts; | |
1598 | old_default_opts.OldDefaults(5, 2); | |
1599 | ASSERT_EQ(16 * 1024U * 1024U, old_default_opts.delayed_write_rate); | |
1600 | } | |
1601 | ||
1602 | Options small_opts; | |
1603 | small_opts.OptimizeForSmallDb(); | |
1604 | ASSERT_EQ(2 << 20, small_opts.write_buffer_size); | |
1605 | ASSERT_EQ(5000, small_opts.max_open_files); | |
1606 | } | |
1607 | ||
1608 | class OptionsSanityCheckTest : public OptionsParserTest { | |
1609 | public: | |
1610 | OptionsSanityCheckTest() {} | |
1611 | ||
1612 | protected: | |
1613 | Status SanityCheckCFOptions(const ColumnFamilyOptions& cf_opts, | |
1614 | OptionsSanityCheckLevel level) { | |
1615 | return RocksDBOptionsParser::VerifyRocksDBOptionsFromFile( | |
1616 | DBOptions(), {"default"}, {cf_opts}, kOptionsFileName, env_.get(), | |
1617 | level); | |
1618 | } | |
1619 | ||
1620 | Status PersistCFOptions(const ColumnFamilyOptions& cf_opts) { | |
1621 | Status s = env_->DeleteFile(kOptionsFileName); | |
1622 | if (!s.ok()) { | |
1623 | return s; | |
1624 | } | |
1625 | return PersistRocksDBOptions(DBOptions(), {"default"}, {cf_opts}, | |
1626 | kOptionsFileName, env_.get()); | |
1627 | } | |
1628 | ||
1629 | const std::string kOptionsFileName = "OPTIONS"; | |
1630 | }; | |
1631 | ||
1632 | TEST_F(OptionsSanityCheckTest, SanityCheck) { | |
1633 | ColumnFamilyOptions opts; | |
1634 | Random rnd(301); | |
1635 | ||
1636 | // default ColumnFamilyOptions | |
1637 | { | |
1638 | ASSERT_OK(PersistCFOptions(opts)); | |
1639 | ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelExactMatch)); | |
1640 | } | |
1641 | ||
1642 | // prefix_extractor | |
1643 | { | |
1644 | // Okay to change prefix_extractor form nullptr to non-nullptr | |
1645 | ASSERT_EQ(opts.prefix_extractor.get(), nullptr); | |
1646 | opts.prefix_extractor.reset(NewCappedPrefixTransform(10)); | |
1647 | ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelLooselyCompatible)); | |
1648 | ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelNone)); | |
1649 | ||
1650 | // persist the change | |
1651 | ASSERT_OK(PersistCFOptions(opts)); | |
1652 | ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelExactMatch)); | |
1653 | ||
1654 | // use same prefix extractor but with different parameter | |
1655 | opts.prefix_extractor.reset(NewCappedPrefixTransform(15)); | |
11fdf7f2 TL |
1656 | // expect pass only in kSanityLevelLooselyCompatible |
1657 | ASSERT_NOK(SanityCheckCFOptions(opts, kSanityLevelExactMatch)); | |
1658 | ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelLooselyCompatible)); | |
7c673cae FG |
1659 | ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelNone)); |
1660 | ||
1661 | // repeat the test with FixedPrefixTransform | |
1662 | opts.prefix_extractor.reset(NewFixedPrefixTransform(10)); | |
11fdf7f2 TL |
1663 | ASSERT_NOK(SanityCheckCFOptions(opts, kSanityLevelExactMatch)); |
1664 | ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelLooselyCompatible)); | |
7c673cae FG |
1665 | ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelNone)); |
1666 | ||
1667 | // persist the change of prefix_extractor | |
1668 | ASSERT_OK(PersistCFOptions(opts)); | |
1669 | ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelExactMatch)); | |
1670 | ||
1671 | // use same prefix extractor but with different parameter | |
1672 | opts.prefix_extractor.reset(NewFixedPrefixTransform(15)); | |
11fdf7f2 TL |
1673 | // expect pass only in kSanityLevelLooselyCompatible |
1674 | ASSERT_NOK(SanityCheckCFOptions(opts, kSanityLevelExactMatch)); | |
1675 | ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelLooselyCompatible)); | |
7c673cae FG |
1676 | ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelNone)); |
1677 | ||
1678 | // Change prefix extractor from non-nullptr to nullptr | |
1679 | opts.prefix_extractor.reset(); | |
1680 | // expect pass as it's safe to change prefix_extractor | |
1681 | // from non-null to null | |
1682 | ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelLooselyCompatible)); | |
1683 | ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelNone)); | |
1684 | } | |
1685 | // persist the change | |
1686 | ASSERT_OK(PersistCFOptions(opts)); | |
1687 | ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelExactMatch)); | |
1688 | ||
1689 | // table_factory | |
1690 | { | |
1691 | for (int tb = 0; tb <= 2; ++tb) { | |
1692 | // change the table factory | |
1693 | opts.table_factory.reset(test::RandomTableFactory(&rnd, tb)); | |
1694 | ASSERT_NOK(SanityCheckCFOptions(opts, kSanityLevelLooselyCompatible)); | |
1695 | ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelNone)); | |
1696 | ||
1697 | // persist the change | |
1698 | ASSERT_OK(PersistCFOptions(opts)); | |
1699 | ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelExactMatch)); | |
1700 | } | |
1701 | } | |
1702 | ||
1703 | // merge_operator | |
1704 | { | |
11fdf7f2 TL |
1705 | // Test when going from nullptr -> merge operator |
1706 | opts.merge_operator.reset(test::RandomMergeOperator(&rnd)); | |
1707 | ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelLooselyCompatible)); | |
1708 | ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelNone)); | |
1709 | ||
1710 | // persist the change | |
1711 | ASSERT_OK(PersistCFOptions(opts)); | |
1712 | ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelExactMatch)); | |
1713 | ||
7c673cae FG |
1714 | for (int test = 0; test < 5; ++test) { |
1715 | // change the merge operator | |
1716 | opts.merge_operator.reset(test::RandomMergeOperator(&rnd)); | |
1717 | ASSERT_NOK(SanityCheckCFOptions(opts, kSanityLevelLooselyCompatible)); | |
1718 | ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelNone)); | |
1719 | ||
1720 | // persist the change | |
1721 | ASSERT_OK(PersistCFOptions(opts)); | |
1722 | ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelExactMatch)); | |
1723 | } | |
11fdf7f2 TL |
1724 | |
1725 | // Test when going from merge operator -> nullptr | |
1726 | opts.merge_operator = nullptr; | |
1727 | ASSERT_NOK(SanityCheckCFOptions(opts, kSanityLevelLooselyCompatible)); | |
1728 | ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelNone)); | |
1729 | ||
1730 | // persist the change | |
1731 | ASSERT_OK(PersistCFOptions(opts)); | |
1732 | ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelExactMatch)); | |
7c673cae FG |
1733 | } |
1734 | ||
1735 | // compaction_filter | |
1736 | { | |
1737 | for (int test = 0; test < 5; ++test) { | |
1738 | // change the compaction filter | |
1739 | opts.compaction_filter = test::RandomCompactionFilter(&rnd); | |
1740 | ASSERT_NOK(SanityCheckCFOptions(opts, kSanityLevelExactMatch)); | |
1741 | ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelLooselyCompatible)); | |
1742 | ||
1743 | // persist the change | |
1744 | ASSERT_OK(PersistCFOptions(opts)); | |
1745 | ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelExactMatch)); | |
1746 | delete opts.compaction_filter; | |
1747 | opts.compaction_filter = nullptr; | |
1748 | } | |
1749 | } | |
1750 | ||
1751 | // compaction_filter_factory | |
1752 | { | |
1753 | for (int test = 0; test < 5; ++test) { | |
1754 | // change the compaction filter factory | |
1755 | opts.compaction_filter_factory.reset( | |
1756 | test::RandomCompactionFilterFactory(&rnd)); | |
1757 | ASSERT_NOK(SanityCheckCFOptions(opts, kSanityLevelExactMatch)); | |
1758 | ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelLooselyCompatible)); | |
1759 | ||
1760 | // persist the change | |
1761 | ASSERT_OK(PersistCFOptions(opts)); | |
1762 | ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelExactMatch)); | |
1763 | } | |
1764 | } | |
1765 | } | |
1766 | ||
1767 | namespace { | |
1768 | bool IsEscapedString(const std::string& str) { | |
1769 | for (size_t i = 0; i < str.size(); ++i) { | |
1770 | if (str[i] == '\\') { | |
1771 | // since we already handle those two consecutive '\'s in | |
1772 | // the next if-then branch, any '\' appear at the end | |
1773 | // of an escaped string in such case is not valid. | |
1774 | if (i == str.size() - 1) { | |
1775 | return false; | |
1776 | } | |
1777 | if (str[i + 1] == '\\') { | |
1778 | // if there're two consecutive '\'s, skip the second one. | |
1779 | i++; | |
1780 | continue; | |
1781 | } | |
1782 | switch (str[i + 1]) { | |
1783 | case ':': | |
1784 | case '\\': | |
1785 | case '#': | |
1786 | continue; | |
1787 | default: | |
1788 | // if true, '\' together with str[i + 1] is not a valid escape. | |
1789 | if (UnescapeChar(str[i + 1]) == str[i + 1]) { | |
1790 | return false; | |
1791 | } | |
1792 | } | |
1793 | } else if (isSpecialChar(str[i]) && (i == 0 || str[i - 1] != '\\')) { | |
1794 | return false; | |
1795 | } | |
1796 | } | |
1797 | return true; | |
1798 | } | |
1799 | } // namespace | |
1800 | ||
1801 | TEST_F(OptionsParserTest, EscapeOptionString) { | |
1802 | ASSERT_EQ(UnescapeOptionString( | |
1803 | "This is a test string with \\# \\: and \\\\ escape chars."), | |
1804 | "This is a test string with # : and \\ escape chars."); | |
1805 | ||
1806 | ASSERT_EQ( | |
1807 | EscapeOptionString("This is a test string with # : and \\ escape chars."), | |
1808 | "This is a test string with \\# \\: and \\\\ escape chars."); | |
1809 | ||
1810 | std::string readible_chars = | |
1811 | "A String like this \"1234567890-=_)(*&^%$#@!ertyuiop[]{POIU" | |
1812 | "YTREWQasdfghjkl;':LKJHGFDSAzxcvbnm,.?>" | |
1813 | "<MNBVCXZ\\\" should be okay to \\#\\\\\\:\\#\\#\\#\\ " | |
1814 | "be serialized and deserialized"; | |
1815 | ||
1816 | std::string escaped_string = EscapeOptionString(readible_chars); | |
1817 | ASSERT_TRUE(IsEscapedString(escaped_string)); | |
1818 | // This two transformations should be canceled and should output | |
1819 | // the original input. | |
1820 | ASSERT_EQ(UnescapeOptionString(escaped_string), readible_chars); | |
1821 | ||
1822 | std::string all_chars; | |
1823 | for (unsigned char c = 0;; ++c) { | |
1824 | all_chars += c; | |
1825 | if (c == 255) { | |
1826 | break; | |
1827 | } | |
1828 | } | |
1829 | escaped_string = EscapeOptionString(all_chars); | |
1830 | ASSERT_TRUE(IsEscapedString(escaped_string)); | |
1831 | ASSERT_EQ(UnescapeOptionString(escaped_string), all_chars); | |
1832 | ||
1833 | ASSERT_EQ(RocksDBOptionsParser::TrimAndRemoveComment( | |
1834 | " A simple statement with a comment. # like this :)"), | |
1835 | "A simple statement with a comment."); | |
1836 | ||
1837 | ASSERT_EQ(RocksDBOptionsParser::TrimAndRemoveComment( | |
1838 | "Escape \\# and # comment together ."), | |
1839 | "Escape \\# and"); | |
1840 | } | |
1841 | #endif // !ROCKSDB_LITE | |
1842 | } // namespace rocksdb | |
1843 | ||
1844 | int main(int argc, char** argv) { | |
1845 | ::testing::InitGoogleTest(&argc, argv); | |
1846 | #ifdef GFLAGS | |
1847 | ParseCommandLineFlags(&argc, &argv, true); | |
1848 | #endif // GFLAGS | |
1849 | return RUN_ALL_TESTS(); | |
1850 | } |