1 // Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
2 // This source code is licensed under both the GPLv2 (found in the
3 // COPYING file in the root directory) and Apache 2.0 License
4 // (found in the LICENSE.Apache file in the root directory).
6 // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
7 // Use of this source code is governed by a BSD-style license that can be
8 // found in the LICENSE file. See the AUTHORS file for names of contributors.
10 #include "rocksdb/db_bench_tool.h"
12 #include "db/db_impl/db_impl.h"
13 #include "options/options_parser.h"
14 #include "rocksdb/utilities/options_util.h"
15 #include "test_util/testharness.h"
16 #include "test_util/testutil.h"
17 #include "util/random.h"
20 #include "util/gflags_compat.h"
22 namespace ROCKSDB_NAMESPACE
{
24 static const int kMaxArgCount
= 100;
25 static const size_t kArgBufferSize
= 100000;
28 class DBBenchTest
: public testing::Test
{
30 DBBenchTest() : rnd_(0xFB) {
31 test_path_
= test::PerThreadDBPath("db_bench_test");
32 Env::Default()->CreateDir(test_path_
);
33 db_path_
= test_path_
+ "/db";
34 wal_path_
= test_path_
+ "/wal";
35 fs_
.reset(new LegacyFileSystemWrapper(Env::Default()));
39 // DestroyDB(db_path_, Options());
45 memset(arg_buffer_
, 0, kArgBufferSize
);
48 void AppendArgs(const std::vector
<std::string
>& args
) {
49 for (const auto& arg
: args
) {
50 ASSERT_LE(cursor_
+ arg
.size() + 1, kArgBufferSize
);
51 ASSERT_LE(argc_
+ 1, kMaxArgCount
);
52 snprintf(arg_buffer_
+ cursor_
, arg
.size() + 1, "%s", arg
.c_str());
54 argv_
[argc_
++] = arg_buffer_
+ cursor_
;
55 cursor_
+= arg
.size() + 1;
59 // Gets the default options for this test/db_bench.
60 // Note that db_bench may change some of the default option values and that
61 // the database might as well. The options changed by db_bench are
62 // specified here; the ones by the DB are set via SanitizeOptions
63 Options
GetDefaultOptions(CompactionStyle style
= kCompactionStyleLevel
,
64 int levels
= 7) const {
67 opt
.create_if_missing
= true;
68 opt
.max_open_files
= 256;
69 opt
.max_background_compactions
= 10;
70 opt
.dump_malloc_stats
= true; // db_bench uses a different default
71 opt
.compaction_style
= style
;
72 opt
.num_levels
= levels
;
73 opt
.compression
= kNoCompression
;
74 opt
.arena_block_size
= 8388608;
76 return SanitizeOptions(db_path_
, opt
);
79 void RunDbBench(const std::string
& options_file_name
) {
80 AppendArgs({"./db_bench", "--benchmarks=fillseq", "--use_existing_db=0",
81 "--num=1000", "--compression_type=none",
82 std::string(std::string("--db=") + db_path_
).c_str(),
83 std::string(std::string("--wal_dir=") + wal_path_
).c_str(),
84 std::string(std::string("--options_file=") + options_file_name
)
86 ASSERT_EQ(0, db_bench_tool(argc(), argv()));
89 void VerifyOptions(const Options
& opt
) {
90 DBOptions loaded_db_opts
;
91 std::vector
<ColumnFamilyDescriptor
> cf_descs
;
92 ASSERT_OK(LoadLatestOptions(db_path_
, Env::Default(), &loaded_db_opts
,
96 exact
.input_strings_escaped
= false;
97 exact
.sanity_level
= ConfigOptions::kSanityLevelExactMatch
;
98 ASSERT_OK(RocksDBOptionsParser::VerifyDBOptions(exact
, DBOptions(opt
),
100 ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(
101 exact
, ColumnFamilyOptions(opt
), cf_descs
[0].options
));
103 // check with the default rocksdb options and expect failure
104 ASSERT_NOK(RocksDBOptionsParser::VerifyDBOptions(exact
, DBOptions(),
106 ASSERT_NOK(RocksDBOptionsParser::VerifyCFOptions(
107 exact
, ColumnFamilyOptions(), cf_descs
[0].options
));
110 char** argv() { return argv_
; }
112 int argc() { return argc_
; }
114 std::string db_path_
;
115 std::string test_path_
;
116 std::string wal_path_
;
117 std::unique_ptr
<LegacyFileSystemWrapper
> fs_
;
119 char arg_buffer_
[kArgBufferSize
];
120 char* argv_
[kMaxArgCount
];
126 namespace {} // namespace
128 TEST_F(DBBenchTest
, OptionsFile
) {
129 const std::string kOptionsFileName
= test_path_
+ "/OPTIONS_test";
130 Options opt
= GetDefaultOptions();
131 ASSERT_OK(PersistRocksDBOptions(DBOptions(opt
), {"default"},
132 {ColumnFamilyOptions()}, kOptionsFileName
,
135 // override the following options as db_bench will not take these
136 // options from the options file
137 opt
.wal_dir
= wal_path_
;
139 RunDbBench(kOptionsFileName
);
140 opt
.delayed_write_rate
= 16 * 1024 * 1024; // Set by SanitizeOptions
145 TEST_F(DBBenchTest
, OptionsFileUniversal
) {
146 const std::string kOptionsFileName
= test_path_
+ "/OPTIONS_test";
148 Options opt
= GetDefaultOptions(kCompactionStyleUniversal
, 1);
150 ASSERT_OK(PersistRocksDBOptions(DBOptions(opt
), {"default"},
151 {ColumnFamilyOptions(opt
)}, kOptionsFileName
,
154 // override the following options as db_bench will not take these
155 // options from the options file
156 opt
.wal_dir
= wal_path_
;
157 RunDbBench(kOptionsFileName
);
162 TEST_F(DBBenchTest
, OptionsFileMultiLevelUniversal
) {
163 const std::string kOptionsFileName
= test_path_
+ "/OPTIONS_test";
165 Options opt
= GetDefaultOptions(kCompactionStyleUniversal
, 12);
167 ASSERT_OK(PersistRocksDBOptions(DBOptions(opt
), {"default"},
168 {ColumnFamilyOptions(opt
)}, kOptionsFileName
,
171 // override the following options as db_bench will not take these
172 // options from the options file
173 opt
.wal_dir
= wal_path_
;
175 RunDbBench(kOptionsFileName
);
179 const std::string options_file_content
= R
"OPTIONS_FILE(
181 rocksdb_version=4.3.1
182 options_file_version=1.1
185 wal_bytes_per_sync=1048576
186 delete_obsolete_files_period_micros=0
189 db_write_buffer_size=0
191 table_cache_numshardbits=4
193 max_file_opening_threads=10
194 max_background_compactions=5
196 use_adaptive_mutex=false
197 max_total_wal_size=18446744073709551615
198 compaction_readahead_size=0
199 new_table_reader_for_compaction_inputs=false
201 skip_stats_update_on_db_open=false
202 max_manifest_file_size=18446744073709551615
204 skip_log_error_on_recovery=false
205 writable_file_max_buffer_size=1048576
207 is_fd_close_on_exec=true
208 bytes_per_sync=1048576
209 enable_thread_tracking=true
210 recycle_log_file_num=0
211 create_missing_column_families=false
212 log_file_time_to_roll=0
213 max_background_flushes=1
214 create_if_missing=true
215 error_if_exists=false
216 delayed_write_rate=1048576
217 manifest_preallocation_size=4194304
218 allow_mmap_reads=false
219 allow_mmap_writes=false
220 use_direct_reads=false
221 use_direct_io_for_flush_and_compaction=false
222 stats_dump_period_sec=600
224 max_log_file_size=83886080
225 random_access_max_buffer_size=1048576
226 advise_random_on_open=true
227 dump_malloc_stats=true
229 [CFOptions "default"]
230 compaction_filter_factory=nullptr
231 table_factory=BlockBasedTable
232 prefix_extractor=nullptr
233 comparator=leveldb.BytewiseComparator
234 compression_per_level=
235 max_bytes_for_level_base=104857600
237 target_file_size_base=10485760
238 memtable_huge_page_size=0
239 max_successive_merges=1000
240 max_sequential_skip_in_iterations=8
241 arena_block_size=52428800
242 target_file_size_multiplier=1
243 source_compaction_factor=1
244 min_write_buffer_number_to_merge=1
245 max_write_buffer_number=2
246 write_buffer_size=419430400
247 max_grandparent_overlap_factor=10
248 max_bytes_for_level_multiplier=10
249 memtable_factory=SkipListFactory
250 compression=kNoCompression
251 min_partial_merge_operands=2
252 level0_stop_writes_trigger=100
254 level0_slowdown_writes_trigger=50
255 level0_file_num_compaction_trigger=10
256 expanded_compaction_factor=25
257 soft_rate_limit=0.000000
258 max_write_buffer_number_to_maintain=0
259 max_write_buffer_size_to_maintain=0
260 verify_checksums_in_compaction=true
261 merge_operator=nullptr
262 memtable_prefix_bloom_bits=0
263 memtable_whole_key_filtering=true
264 paranoid_file_checks=false
265 inplace_update_num_locks=10000
266 optimize_filters_for_hits=false
267 level_compaction_dynamic_level_bytes=false
268 inplace_update_support=false
269 compaction_style=kCompactionStyleUniversal
270 memtable_prefix_bloom_probes=6
271 purge_redundant_kvs_while_flush=true
273 hard_pending_compaction_bytes_limit=0
274 disable_auto_compactions=false
275 compaction_measure_io_stats=false
277 [TableOptions/BlockBasedTable "default"]
279 skip_table_builder_flush=false
280 cache_index_and_filter_blocks=false
281 flush_block_policy_factory=FlushBlockBySizePolicyFactory
282 hash_index_allow_collision=true
283 index_type=kBinarySearch
284 whole_key_filtering=true
288 block_size_deviation=10
289 block_restart_interval=16
290 filter_policy=rocksdb.BuiltinBloomFilter
293 TEST_F(DBBenchTest
, OptionsFileFromFile
) {
294 const std::string kOptionsFileName
= test_path_
+ "/OPTIONS_flash";
295 std::unique_ptr
<WritableFile
> writable
;
296 ASSERT_OK(Env::Default()->NewWritableFile(kOptionsFileName
, &writable
,
298 ASSERT_OK(writable
->Append(options_file_content
));
299 ASSERT_OK(writable
->Close());
302 std::vector
<ColumnFamilyDescriptor
> cf_descs
;
303 ASSERT_OK(LoadOptionsFromFile(kOptionsFileName
, Env::Default(), &db_opt
,
305 Options
opt(db_opt
, cf_descs
[0].options
);
306 opt
.create_if_missing
= true;
308 // override the following options as db_bench will not take these
309 // options from the options file
310 opt
.wal_dir
= wal_path_
;
312 RunDbBench(kOptionsFileName
);
314 VerifyOptions(SanitizeOptions(db_path_
, opt
));
317 } // namespace ROCKSDB_NAMESPACE
319 int main(int argc
, char** argv
) {
320 ::testing::InitGoogleTest(&argc
, argv
);
321 google::ParseCommandLineFlags(&argc
, &argv
, true);
322 return RUN_ALL_TESTS();
327 int main(int argc
, char** argv
) {
328 printf("Skip db_bench_tool_test as the required library GFLAG is missing.");
330 #endif // #ifdef GFLAGS