]>
Commit | Line | Data |
---|---|---|
7c673cae | 1 | // Copyright (c) 2011-present, Facebook, Inc. All rights reserved. |
11fdf7f2 TL |
2 | // This source code is licensed under both the GPLv2 (found in the |
3 | // COPYING file in the root directory) and Apache 2.0 License | |
4 | // (found in the LICENSE.Apache file in the root directory). | |
7c673cae FG |
5 | // |
6 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. | |
7 | // Use of this source code is governed by a BSD-style license that can be | |
8 | // found in the LICENSE file. See the AUTHORS file for names of contributors. | |
9 | ||
10 | #include "rocksdb/db_bench_tool.h" | |
11 | #include "options/options_parser.h" | |
12 | #include "rocksdb/utilities/options_util.h" | |
13 | #include "util/random.h" | |
14 | #include "util/testharness.h" | |
15 | #include "util/testutil.h" | |
16 | ||
17 | #ifdef GFLAGS | |
11fdf7f2 | 18 | #include "util/gflags_compat.h" |
7c673cae FG |
19 | |
20 | namespace rocksdb { | |
21 | namespace { | |
22 | static const int kMaxArgCount = 100; | |
23 | static const size_t kArgBufferSize = 100000; | |
24 | } // namespace | |
25 | ||
26 | class DBBenchTest : public testing::Test { | |
27 | public: | |
28 | DBBenchTest() : rnd_(0xFB) { | |
11fdf7f2 | 29 | test_path_ = test::PerThreadDBPath("db_bench_test"); |
7c673cae FG |
30 | Env::Default()->CreateDir(test_path_); |
31 | db_path_ = test_path_ + "/db"; | |
32 | wal_path_ = test_path_ + "/wal"; | |
33 | } | |
34 | ||
35 | ~DBBenchTest() { | |
36 | // DestroyDB(db_path_, Options()); | |
37 | } | |
38 | ||
39 | void ResetArgs() { | |
40 | argc_ = 0; | |
41 | cursor_ = 0; | |
42 | memset(arg_buffer_, 0, kArgBufferSize); | |
43 | } | |
44 | ||
45 | void AppendArgs(const std::vector<std::string>& args) { | |
46 | for (const auto& arg : args) { | |
47 | ASSERT_LE(cursor_ + arg.size() + 1, kArgBufferSize); | |
48 | ASSERT_LE(argc_ + 1, kMaxArgCount); | |
49 | snprintf(arg_buffer_ + cursor_, arg.size() + 1, "%s", arg.c_str()); | |
50 | ||
51 | argv_[argc_++] = arg_buffer_ + cursor_; | |
52 | cursor_ += arg.size() + 1; | |
53 | } | |
54 | } | |
55 | ||
56 | void RunDbBench(const std::string& options_file_name) { | |
57 | AppendArgs({"./db_bench", "--benchmarks=fillseq", "--use_existing_db=0", | |
58 | "--num=1000", | |
59 | std::string(std::string("--db=") + db_path_).c_str(), | |
60 | std::string(std::string("--wal_dir=") + wal_path_).c_str(), | |
61 | std::string(std::string("--options_file=") + options_file_name) | |
62 | .c_str()}); | |
63 | ASSERT_EQ(0, db_bench_tool(argc(), argv())); | |
64 | } | |
65 | ||
66 | void VerifyOptions(const Options& opt) { | |
67 | DBOptions loaded_db_opts; | |
68 | std::vector<ColumnFamilyDescriptor> cf_descs; | |
69 | ASSERT_OK(LoadLatestOptions(db_path_, Env::Default(), &loaded_db_opts, | |
70 | &cf_descs)); | |
71 | ||
72 | ASSERT_OK( | |
73 | RocksDBOptionsParser::VerifyDBOptions(DBOptions(opt), loaded_db_opts)); | |
74 | ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(ColumnFamilyOptions(opt), | |
75 | cf_descs[0].options)); | |
76 | ||
77 | // check with the default rocksdb options and expect failure | |
78 | ASSERT_NOK( | |
79 | RocksDBOptionsParser::VerifyDBOptions(DBOptions(), loaded_db_opts)); | |
80 | ASSERT_NOK(RocksDBOptionsParser::VerifyCFOptions(ColumnFamilyOptions(), | |
81 | cf_descs[0].options)); | |
82 | } | |
83 | ||
84 | char** argv() { return argv_; } | |
85 | ||
86 | int argc() { return argc_; } | |
87 | ||
88 | std::string db_path_; | |
89 | std::string test_path_; | |
90 | std::string wal_path_; | |
91 | ||
92 | char arg_buffer_[kArgBufferSize]; | |
93 | char* argv_[kMaxArgCount]; | |
94 | int argc_ = 0; | |
95 | int cursor_ = 0; | |
96 | Random rnd_; | |
97 | }; | |
98 | ||
99 | namespace {} // namespace | |
100 | ||
101 | TEST_F(DBBenchTest, OptionsFile) { | |
102 | const std::string kOptionsFileName = test_path_ + "/OPTIONS_test"; | |
103 | ||
104 | Options opt; | |
105 | opt.create_if_missing = true; | |
106 | opt.max_open_files = 256; | |
7c673cae FG |
107 | opt.max_background_compactions = 10; |
108 | opt.arena_block_size = 8388608; | |
109 | ASSERT_OK(PersistRocksDBOptions(DBOptions(opt), {"default"}, | |
110 | {ColumnFamilyOptions(opt)}, kOptionsFileName, | |
111 | Env::Default())); | |
112 | ||
113 | // override the following options as db_bench will not take these | |
114 | // options from the options file | |
115 | opt.wal_dir = wal_path_; | |
116 | ||
117 | RunDbBench(kOptionsFileName); | |
118 | ||
119 | VerifyOptions(opt); | |
120 | } | |
121 | ||
122 | TEST_F(DBBenchTest, OptionsFileUniversal) { | |
123 | const std::string kOptionsFileName = test_path_ + "/OPTIONS_test"; | |
124 | ||
125 | Options opt; | |
126 | opt.compaction_style = kCompactionStyleUniversal; | |
127 | opt.num_levels = 1; | |
128 | opt.create_if_missing = true; | |
129 | opt.max_open_files = 256; | |
7c673cae FG |
130 | opt.max_background_compactions = 10; |
131 | opt.arena_block_size = 8388608; | |
132 | ASSERT_OK(PersistRocksDBOptions(DBOptions(opt), {"default"}, | |
133 | {ColumnFamilyOptions(opt)}, kOptionsFileName, | |
134 | Env::Default())); | |
135 | ||
136 | // override the following options as db_bench will not take these | |
137 | // options from the options file | |
138 | opt.wal_dir = wal_path_; | |
139 | ||
140 | RunDbBench(kOptionsFileName); | |
141 | ||
142 | VerifyOptions(opt); | |
143 | } | |
144 | ||
145 | TEST_F(DBBenchTest, OptionsFileMultiLevelUniversal) { | |
146 | const std::string kOptionsFileName = test_path_ + "/OPTIONS_test"; | |
147 | ||
148 | Options opt; | |
149 | opt.compaction_style = kCompactionStyleUniversal; | |
150 | opt.num_levels = 12; | |
151 | opt.create_if_missing = true; | |
152 | opt.max_open_files = 256; | |
7c673cae FG |
153 | opt.max_background_compactions = 10; |
154 | opt.arena_block_size = 8388608; | |
155 | ASSERT_OK(PersistRocksDBOptions(DBOptions(opt), {"default"}, | |
156 | {ColumnFamilyOptions(opt)}, kOptionsFileName, | |
157 | Env::Default())); | |
158 | ||
159 | // override the following options as db_bench will not take these | |
160 | // options from the options file | |
161 | opt.wal_dir = wal_path_; | |
162 | ||
163 | RunDbBench(kOptionsFileName); | |
164 | ||
165 | VerifyOptions(opt); | |
166 | } | |
167 | ||
168 | const std::string options_file_content = R"OPTIONS_FILE( | |
169 | [Version] | |
170 | rocksdb_version=4.3.1 | |
171 | options_file_version=1.1 | |
172 | ||
173 | [DBOptions] | |
174 | wal_bytes_per_sync=1048576 | |
175 | delete_obsolete_files_period_micros=0 | |
176 | WAL_ttl_seconds=0 | |
177 | WAL_size_limit_MB=0 | |
178 | db_write_buffer_size=0 | |
179 | max_subcompactions=1 | |
180 | table_cache_numshardbits=4 | |
181 | max_open_files=-1 | |
182 | max_file_opening_threads=10 | |
7c673cae FG |
183 | max_background_compactions=5 |
184 | use_fsync=false | |
185 | use_adaptive_mutex=false | |
186 | max_total_wal_size=18446744073709551615 | |
187 | compaction_readahead_size=0 | |
188 | new_table_reader_for_compaction_inputs=false | |
189 | keep_log_file_num=10 | |
190 | skip_stats_update_on_db_open=false | |
191 | max_manifest_file_size=18446744073709551615 | |
192 | db_log_dir= | |
193 | skip_log_error_on_recovery=false | |
194 | writable_file_max_buffer_size=1048576 | |
195 | paranoid_checks=true | |
196 | is_fd_close_on_exec=true | |
197 | bytes_per_sync=1048576 | |
198 | enable_thread_tracking=true | |
199 | recycle_log_file_num=0 | |
200 | create_missing_column_families=false | |
201 | log_file_time_to_roll=0 | |
202 | max_background_flushes=1 | |
203 | create_if_missing=true | |
204 | error_if_exists=false | |
205 | delayed_write_rate=1048576 | |
206 | manifest_preallocation_size=4194304 | |
207 | allow_mmap_reads=false | |
208 | allow_mmap_writes=false | |
209 | use_direct_reads=false | |
210 | use_direct_io_for_flush_and_compaction=false | |
211 | stats_dump_period_sec=600 | |
212 | allow_fallocate=true | |
213 | max_log_file_size=83886080 | |
214 | random_access_max_buffer_size=1048576 | |
215 | advise_random_on_open=true | |
216 | ||
217 | ||
218 | [CFOptions "default"] | |
219 | compaction_filter_factory=nullptr | |
220 | table_factory=BlockBasedTable | |
221 | prefix_extractor=nullptr | |
222 | comparator=leveldb.BytewiseComparator | |
223 | compression_per_level= | |
224 | max_bytes_for_level_base=104857600 | |
225 | bloom_locality=0 | |
226 | target_file_size_base=10485760 | |
227 | memtable_huge_page_size=0 | |
228 | max_successive_merges=1000 | |
229 | max_sequential_skip_in_iterations=8 | |
230 | arena_block_size=52428800 | |
231 | target_file_size_multiplier=1 | |
232 | source_compaction_factor=1 | |
233 | min_write_buffer_number_to_merge=1 | |
234 | max_write_buffer_number=2 | |
235 | write_buffer_size=419430400 | |
236 | max_grandparent_overlap_factor=10 | |
237 | max_bytes_for_level_multiplier=10 | |
238 | memtable_factory=SkipListFactory | |
239 | compression=kSnappyCompression | |
240 | min_partial_merge_operands=2 | |
241 | level0_stop_writes_trigger=100 | |
242 | num_levels=1 | |
243 | level0_slowdown_writes_trigger=50 | |
244 | level0_file_num_compaction_trigger=10 | |
245 | expanded_compaction_factor=25 | |
246 | soft_rate_limit=0.000000 | |
247 | max_write_buffer_number_to_maintain=0 | |
248 | verify_checksums_in_compaction=true | |
249 | merge_operator=nullptr | |
250 | memtable_prefix_bloom_bits=0 | |
251 | paranoid_file_checks=false | |
252 | inplace_update_num_locks=10000 | |
253 | optimize_filters_for_hits=false | |
254 | level_compaction_dynamic_level_bytes=false | |
255 | inplace_update_support=false | |
256 | compaction_style=kCompactionStyleUniversal | |
257 | memtable_prefix_bloom_probes=6 | |
258 | purge_redundant_kvs_while_flush=true | |
259 | filter_deletes=false | |
260 | hard_pending_compaction_bytes_limit=0 | |
261 | disable_auto_compactions=false | |
262 | compaction_measure_io_stats=false | |
263 | ||
264 | [TableOptions/BlockBasedTable "default"] | |
265 | format_version=0 | |
266 | skip_table_builder_flush=false | |
267 | cache_index_and_filter_blocks=false | |
268 | flush_block_policy_factory=FlushBlockBySizePolicyFactory | |
269 | hash_index_allow_collision=true | |
270 | index_type=kBinarySearch | |
271 | whole_key_filtering=true | |
272 | checksum=kCRC32c | |
273 | no_block_cache=false | |
274 | block_size=32768 | |
275 | block_size_deviation=10 | |
276 | block_restart_interval=16 | |
277 | filter_policy=rocksdb.BuiltinBloomFilter | |
278 | )OPTIONS_FILE"; | |
279 | ||
280 | TEST_F(DBBenchTest, OptionsFileFromFile) { | |
281 | const std::string kOptionsFileName = test_path_ + "/OPTIONS_flash"; | |
282 | unique_ptr<WritableFile> writable; | |
283 | ASSERT_OK(Env::Default()->NewWritableFile(kOptionsFileName, &writable, | |
284 | EnvOptions())); | |
285 | ASSERT_OK(writable->Append(options_file_content)); | |
286 | ASSERT_OK(writable->Close()); | |
287 | ||
288 | DBOptions db_opt; | |
289 | std::vector<ColumnFamilyDescriptor> cf_descs; | |
290 | ASSERT_OK(LoadOptionsFromFile(kOptionsFileName, Env::Default(), &db_opt, | |
291 | &cf_descs)); | |
292 | Options opt(db_opt, cf_descs[0].options); | |
293 | ||
294 | opt.create_if_missing = true; | |
295 | ||
296 | // override the following options as db_bench will not take these | |
297 | // options from the options file | |
298 | opt.wal_dir = wal_path_; | |
299 | ||
300 | RunDbBench(kOptionsFileName); | |
301 | ||
302 | VerifyOptions(opt); | |
303 | } | |
304 | ||
305 | } // namespace rocksdb | |
306 | ||
307 | int main(int argc, char** argv) { | |
308 | ::testing::InitGoogleTest(&argc, argv); | |
309 | google::ParseCommandLineFlags(&argc, &argv, true); | |
310 | return RUN_ALL_TESTS(); | |
311 | } | |
312 | ||
313 | #else | |
314 | ||
315 | int main(int argc, char** argv) { | |
316 | printf("Skip db_bench_tool_test as the required library GFLAG is missing."); | |
317 | } | |
318 | #endif // #ifdef GFLAGS |