]> git.proxmox.com Git - ceph.git/blame - ceph/src/rocksdb/db_stress_tool/db_stress_tool.cc
import quincy beta 17.1.0
[ceph.git] / ceph / src / rocksdb / db_stress_tool / db_stress_tool.cc
CommitLineData
f67539c2
TL
1// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
2// This source code is licensed under both the GPLv2 (found in the
3// COPYING file in the root directory) and Apache 2.0 License
4// (found in the LICENSE.Apache file in the root directory).
5//
6// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
7// Use of this source code is governed by a BSD-style license that can be
8// found in the LICENSE file. See the AUTHORS file for names of contributors.
9//
10// The test uses an array to compare against values written to the database.
11// Keys written to the array are in 1:1 correspondence to the actual values in
12// the database according to the formula in the function GenerateValue.
13
14// Space is reserved in the array from 0 to FLAGS_max_key and values are
15// randomly written/deleted/read from those positions. During verification we
16// compare all the positions in the array. To shorten/elongate the running
17// time, you could change the settings: FLAGS_max_key, FLAGS_ops_per_thread,
18// (sometimes also FLAGS_threads).
19//
20// NOTE that if FLAGS_test_batches_snapshots is set, the test will have
21// different behavior. See comment of the flag for details.
22
23#ifdef GFLAGS
24#include "db_stress_tool/db_stress_common.h"
25#include "db_stress_tool/db_stress_driver.h"
20effc67
TL
26#ifndef NDEBUG
27#include "utilities/fault_injection_fs.h"
28#endif
f67539c2
TL
29
30namespace ROCKSDB_NAMESPACE {
31namespace {
32static std::shared_ptr<ROCKSDB_NAMESPACE::Env> env_guard;
33static std::shared_ptr<ROCKSDB_NAMESPACE::DbStressEnvWrapper> env_wrapper_guard;
20effc67 34static std::shared_ptr<CompositeEnvWrapper> fault_env_guard;
f67539c2
TL
35} // namespace
36
20effc67
TL
37static Env* GetCompositeEnv(std::shared_ptr<FileSystem> fs) {
38 static std::shared_ptr<Env> composite_env = NewCompositeEnv(fs);
39 return composite_env.get();
40}
41
f67539c2
TL
42KeyGenContext key_gen_ctx;
43
44int db_stress_tool(int argc, char** argv) {
45 SetUsageMessage(std::string("\nUSAGE:\n") + std::string(argv[0]) +
46 " [OPTIONS]...");
47 ParseCommandLineFlags(&argc, &argv, true);
48
49 SanitizeDoubleParam(&FLAGS_bloom_bits);
50 SanitizeDoubleParam(&FLAGS_memtable_prefix_bloom_size_ratio);
51 SanitizeDoubleParam(&FLAGS_max_bytes_for_level_multiplier);
52
20effc67
TL
53#ifndef NDEBUG
54 if (FLAGS_mock_direct_io) {
55 SetupSyncPointsToMockDirectIO();
56 }
57#endif
f67539c2
TL
58 if (FLAGS_statistics) {
59 dbstats = ROCKSDB_NAMESPACE::CreateDBStatistics();
60 if (FLAGS_test_secondary) {
61 dbstats_secondaries = ROCKSDB_NAMESPACE::CreateDBStatistics();
62 }
63 }
64 compression_type_e = StringToCompressionType(FLAGS_compression_type.c_str());
65 bottommost_compression_type_e =
66 StringToCompressionType(FLAGS_bottommost_compression_type.c_str());
67 checksum_type_e = StringToChecksumType(FLAGS_checksum_type.c_str());
68
69 Env* raw_env;
70
20effc67
TL
71 int env_opts =
72 !FLAGS_hdfs.empty() + !FLAGS_env_uri.empty() + !FLAGS_fs_uri.empty();
73 if (env_opts > 1) {
74 fprintf(stderr,
75 "Error: --hdfs, --env_uri and --fs_uri are mutually exclusive\n");
76 exit(1);
77 }
78
f67539c2 79 if (!FLAGS_hdfs.empty()) {
f67539c2
TL
80 raw_env = new ROCKSDB_NAMESPACE::HdfsEnv(FLAGS_hdfs);
81 } else if (!FLAGS_env_uri.empty()) {
82 Status s = Env::LoadEnv(FLAGS_env_uri, &raw_env, &env_guard);
83 if (raw_env == nullptr) {
84 fprintf(stderr, "No Env registered for URI: %s\n", FLAGS_env_uri.c_str());
85 exit(1);
86 }
20effc67
TL
87 } else if (!FLAGS_fs_uri.empty()) {
88 std::shared_ptr<FileSystem> fs;
89 Status s = FileSystem::Load(FLAGS_fs_uri, &fs);
90 if (!s.ok()) {
91 fprintf(stderr, "Error: %s\n", s.ToString().c_str());
92 exit(1);
93 }
94 raw_env = GetCompositeEnv(fs);
f67539c2
TL
95 } else {
96 raw_env = Env::Default();
97 }
20effc67
TL
98
99#ifndef NDEBUG
100 if (FLAGS_read_fault_one_in || FLAGS_sync_fault_injection) {
101 FaultInjectionTestFS* fs =
102 new FaultInjectionTestFS(raw_env->GetFileSystem());
103 fault_fs_guard.reset(fs);
104 fault_fs_guard->SetFilesystemDirectWritable(true);
105 fault_env_guard =
106 std::make_shared<CompositeEnvWrapper>(raw_env, fault_fs_guard);
107 raw_env = fault_env_guard.get();
108 }
109#endif
110
f67539c2
TL
111 env_wrapper_guard = std::make_shared<DbStressEnvWrapper>(raw_env);
112 db_stress_env = env_wrapper_guard.get();
113
114 FLAGS_rep_factory = StringToRepFactory(FLAGS_memtablerep.c_str());
115
116 // The number of background threads should be at least as much the
117 // max number of concurrent compactions.
118 db_stress_env->SetBackgroundThreads(FLAGS_max_background_compactions,
119 ROCKSDB_NAMESPACE::Env::Priority::LOW);
120 db_stress_env->SetBackgroundThreads(FLAGS_num_bottom_pri_threads,
121 ROCKSDB_NAMESPACE::Env::Priority::BOTTOM);
122 if (FLAGS_prefixpercent > 0 && FLAGS_prefix_size < 0) {
123 fprintf(stderr,
124 "Error: prefixpercent is non-zero while prefix_size is "
125 "not positive!\n");
126 exit(1);
127 }
128 if (FLAGS_test_batches_snapshots && FLAGS_prefix_size <= 0) {
129 fprintf(stderr,
130 "Error: please specify prefix_size for "
131 "test_batches_snapshots test!\n");
132 exit(1);
133 }
134 if (FLAGS_memtable_prefix_bloom_size_ratio > 0.0 && FLAGS_prefix_size < 0) {
135 fprintf(stderr,
136 "Error: please specify positive prefix_size in order to use "
137 "memtable_prefix_bloom_size_ratio\n");
138 exit(1);
139 }
140 if ((FLAGS_readpercent + FLAGS_prefixpercent + FLAGS_writepercent +
141 FLAGS_delpercent + FLAGS_delrangepercent + FLAGS_iterpercent) != 100) {
142 fprintf(stderr,
143 "Error: Read+Prefix+Write+Delete+DeleteRange+Iterate percents != "
144 "100!\n");
145 exit(1);
146 }
147 if (FLAGS_disable_wal == 1 && FLAGS_reopen > 0) {
148 fprintf(stderr, "Error: Db cannot reopen safely with disable_wal set!\n");
149 exit(1);
150 }
151 if ((unsigned)FLAGS_reopen >= FLAGS_ops_per_thread) {
152 fprintf(stderr,
153 "Error: #DB-reopens should be < ops_per_thread\n"
154 "Provided reopens = %d and ops_per_thread = %lu\n",
155 FLAGS_reopen, (unsigned long)FLAGS_ops_per_thread);
156 exit(1);
157 }
158 if (FLAGS_test_batches_snapshots && FLAGS_delrangepercent > 0) {
159 fprintf(stderr,
160 "Error: nonzero delrangepercent unsupported in "
161 "test_batches_snapshots mode\n");
162 exit(1);
163 }
164 if (FLAGS_active_width > FLAGS_max_key) {
165 fprintf(stderr, "Error: active_width can be at most max_key\n");
166 exit(1);
167 } else if (FLAGS_active_width == 0) {
168 FLAGS_active_width = FLAGS_max_key;
169 }
170 if (FLAGS_value_size_mult * kRandomValueMaxFactor > kValueMaxLen) {
171 fprintf(stderr, "Error: value_size_mult can be at most %d\n",
172 kValueMaxLen / kRandomValueMaxFactor);
173 exit(1);
174 }
175 if (FLAGS_use_merge && FLAGS_nooverwritepercent == 100) {
176 fprintf(
177 stderr,
178 "Error: nooverwritepercent must not be 100 when using merge operands");
179 exit(1);
180 }
181 if (FLAGS_ingest_external_file_one_in > 0 && FLAGS_nooverwritepercent > 0) {
182 fprintf(stderr,
183 "Error: nooverwritepercent must be 0 when using file ingestion\n");
184 exit(1);
185 }
186 if (FLAGS_clear_column_family_one_in > 0 && FLAGS_backup_one_in > 0) {
187 fprintf(stderr,
188 "Error: clear_column_family_one_in must be 0 when using backup\n");
189 exit(1);
190 }
191 if (FLAGS_test_cf_consistency && FLAGS_disable_wal) {
192 FLAGS_atomic_flush = true;
193 }
194
195 if (FLAGS_read_only) {
196 if (FLAGS_writepercent != 0 || FLAGS_delpercent != 0 ||
197 FLAGS_delrangepercent != 0) {
198 fprintf(stderr, "Error: updates are not supported in read only mode\n");
199 exit(1);
200 } else if (FLAGS_checkpoint_one_in > 0 &&
201 FLAGS_clear_column_family_one_in > 0) {
202 fprintf(stdout,
203 "Warn: checkpoint won't be validated since column families may "
204 "be dropped.\n");
205 }
206 }
207
208 // Choose a location for the test database if none given with --db=<path>
209 if (FLAGS_db.empty()) {
210 std::string default_db_path;
211 db_stress_env->GetTestDirectory(&default_db_path);
212 default_db_path += "/dbstress";
213 FLAGS_db = default_db_path;
214 }
215
216 if ((FLAGS_test_secondary || FLAGS_continuous_verification_interval > 0) &&
217 FLAGS_secondaries_base.empty()) {
218 std::string default_secondaries_path;
219 db_stress_env->GetTestDirectory(&default_secondaries_path);
220 default_secondaries_path += "/dbstress_secondaries";
221 ROCKSDB_NAMESPACE::Status s =
222 db_stress_env->CreateDirIfMissing(default_secondaries_path);
223 if (!s.ok()) {
224 fprintf(stderr, "Failed to create directory %s: %s\n",
225 default_secondaries_path.c_str(), s.ToString().c_str());
226 exit(1);
227 }
228 FLAGS_secondaries_base = default_secondaries_path;
229 }
230
231 if (!FLAGS_test_secondary && FLAGS_secondary_catch_up_one_in > 0) {
232 fprintf(
233 stderr,
234 "Must set -test_secondary=true if secondary_catch_up_one_in > 0.\n");
235 exit(1);
236 }
20effc67
TL
237 if (FLAGS_best_efforts_recovery && !FLAGS_skip_verifydb &&
238 !FLAGS_disable_wal) {
239 fprintf(stderr,
240 "With best-efforts recovery, either skip_verifydb or disable_wal "
241 "should be set to true.\n");
242 exit(1);
243 }
244 if (FLAGS_skip_verifydb) {
245 if (FLAGS_verify_db_one_in > 0) {
246 fprintf(stderr,
247 "Must set -verify_db_one_in=0 if skip_verifydb is true.\n");
248 exit(1);
249 }
250 if (FLAGS_continuous_verification_interval > 0) {
251 fprintf(stderr,
252 "Must set -continuous_verification_interval=0 if skip_verifydb "
253 "is true.\n");
254 exit(1);
255 }
256 }
257 if (FLAGS_enable_compaction_filter &&
258 (FLAGS_acquire_snapshot_one_in > 0 || FLAGS_compact_range_one_in > 0 ||
259 FLAGS_iterpercent > 0 || FLAGS_test_batches_snapshots ||
260 FLAGS_test_cf_consistency)) {
261 fprintf(
262 stderr,
263 "Error: acquire_snapshot_one_in, compact_range_one_in, iterpercent, "
264 "test_batches_snapshots must all be 0 when using compaction filter\n");
265 exit(1);
266 }
f67539c2
TL
267
268 rocksdb_kill_odds = FLAGS_kill_random_test;
20effc67 269 rocksdb_kill_exclude_prefixes = SplitString(FLAGS_kill_exclude_prefixes);
f67539c2
TL
270
271 unsigned int levels = FLAGS_max_key_len;
272 std::vector<std::string> weights;
273 uint64_t scale_factor = FLAGS_key_window_scale_factor;
274 key_gen_ctx.window = scale_factor * 100;
275 if (!FLAGS_key_len_percent_dist.empty()) {
276 weights = SplitString(FLAGS_key_len_percent_dist);
277 if (weights.size() != levels) {
278 fprintf(stderr,
279 "Number of weights in key_len_dist should be equal to"
280 " max_key_len");
281 exit(1);
282 }
283
284 uint64_t total_weight = 0;
285 for (std::string& weight : weights) {
286 uint64_t val = std::stoull(weight);
287 key_gen_ctx.weights.emplace_back(val * scale_factor);
288 total_weight += val;
289 }
290 if (total_weight != 100) {
291 fprintf(stderr, "Sum of all weights in key_len_dist should be 100");
292 exit(1);
293 }
294 } else {
295 uint64_t keys_per_level = key_gen_ctx.window / levels;
20effc67 296 for (unsigned int level = 0; level + 1 < levels; ++level) {
f67539c2
TL
297 key_gen_ctx.weights.emplace_back(keys_per_level);
298 }
299 key_gen_ctx.weights.emplace_back(key_gen_ctx.window -
300 keys_per_level * (levels - 1));
301 }
302
303 std::unique_ptr<ROCKSDB_NAMESPACE::StressTest> stress;
304 if (FLAGS_test_cf_consistency) {
305 stress.reset(CreateCfConsistencyStressTest());
306 } else if (FLAGS_test_batches_snapshots) {
307 stress.reset(CreateBatchedOpsStressTest());
308 } else {
309 stress.reset(CreateNonBatchedOpsStressTest());
310 }
311 // Initialize the Zipfian pre-calculated array
312 InitializeHotKeyGenerator(FLAGS_hot_key_alpha);
313 if (RunStressTest(stress.get())) {
314 return 0;
315 } else {
316 return 1;
317 }
318}
319
320} // namespace ROCKSDB_NAMESPACE
321#endif // GFLAGS