]>
Commit | Line | Data |
---|---|---|
f67539c2 TL |
1 | // Copyright (c) 2011-present, Facebook, Inc. All rights reserved. |
2 | // This source code is licensed under both the GPLv2 (found in the | |
3 | // COPYING file in the root directory) and Apache 2.0 License | |
4 | // (found in the LICENSE.Apache file in the root directory). | |
5 | // | |
6 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. | |
7 | // Use of this source code is governed by a BSD-style license that can be | |
8 | // found in the LICENSE file. See the AUTHORS file for names of contributors. | |
9 | // | |
10 | // The test uses an array to compare against values written to the database. | |
11 | // Keys written to the array are in 1:1 correspondence to the actual values in | |
12 | // the database according to the formula in the function GenerateValue. | |
13 | ||
14 | // Space is reserved in the array from 0 to FLAGS_max_key and values are | |
15 | // randomly written/deleted/read from those positions. During verification we | |
16 | // compare all the positions in the array. To shorten/elongate the running | |
17 | // time, you could change the settings: FLAGS_max_key, FLAGS_ops_per_thread, | |
18 | // (sometimes also FLAGS_threads). | |
19 | // | |
20 | // NOTE that if FLAGS_test_batches_snapshots is set, the test will have | |
21 | // different behavior. See comment of the flag for details. | |
22 | ||
23 | #ifdef GFLAGS | |
24 | #include "db_stress_tool/db_stress_common.h" | |
25 | #include "db_stress_tool/db_stress_driver.h" | |
20effc67 TL |
26 | #ifndef NDEBUG |
27 | #include "utilities/fault_injection_fs.h" | |
28 | #endif | |
f67539c2 TL |
29 | |
30 | namespace ROCKSDB_NAMESPACE { | |
31 | namespace { | |
32 | static std::shared_ptr<ROCKSDB_NAMESPACE::Env> env_guard; | |
33 | static std::shared_ptr<ROCKSDB_NAMESPACE::DbStressEnvWrapper> env_wrapper_guard; | |
20effc67 | 34 | static std::shared_ptr<CompositeEnvWrapper> fault_env_guard; |
f67539c2 TL |
35 | } // namespace |
36 | ||
20effc67 TL |
37 | static Env* GetCompositeEnv(std::shared_ptr<FileSystem> fs) { |
38 | static std::shared_ptr<Env> composite_env = NewCompositeEnv(fs); | |
39 | return composite_env.get(); | |
40 | } | |
41 | ||
f67539c2 TL |
42 | KeyGenContext key_gen_ctx; |
43 | ||
44 | int db_stress_tool(int argc, char** argv) { | |
45 | SetUsageMessage(std::string("\nUSAGE:\n") + std::string(argv[0]) + | |
46 | " [OPTIONS]..."); | |
47 | ParseCommandLineFlags(&argc, &argv, true); | |
48 | ||
49 | SanitizeDoubleParam(&FLAGS_bloom_bits); | |
50 | SanitizeDoubleParam(&FLAGS_memtable_prefix_bloom_size_ratio); | |
51 | SanitizeDoubleParam(&FLAGS_max_bytes_for_level_multiplier); | |
52 | ||
20effc67 TL |
53 | #ifndef NDEBUG |
54 | if (FLAGS_mock_direct_io) { | |
55 | SetupSyncPointsToMockDirectIO(); | |
56 | } | |
57 | #endif | |
f67539c2 TL |
58 | if (FLAGS_statistics) { |
59 | dbstats = ROCKSDB_NAMESPACE::CreateDBStatistics(); | |
60 | if (FLAGS_test_secondary) { | |
61 | dbstats_secondaries = ROCKSDB_NAMESPACE::CreateDBStatistics(); | |
62 | } | |
63 | } | |
64 | compression_type_e = StringToCompressionType(FLAGS_compression_type.c_str()); | |
65 | bottommost_compression_type_e = | |
66 | StringToCompressionType(FLAGS_bottommost_compression_type.c_str()); | |
67 | checksum_type_e = StringToChecksumType(FLAGS_checksum_type.c_str()); | |
68 | ||
69 | Env* raw_env; | |
70 | ||
20effc67 TL |
71 | int env_opts = |
72 | !FLAGS_hdfs.empty() + !FLAGS_env_uri.empty() + !FLAGS_fs_uri.empty(); | |
73 | if (env_opts > 1) { | |
74 | fprintf(stderr, | |
75 | "Error: --hdfs, --env_uri and --fs_uri are mutually exclusive\n"); | |
76 | exit(1); | |
77 | } | |
78 | ||
f67539c2 | 79 | if (!FLAGS_hdfs.empty()) { |
f67539c2 TL |
80 | raw_env = new ROCKSDB_NAMESPACE::HdfsEnv(FLAGS_hdfs); |
81 | } else if (!FLAGS_env_uri.empty()) { | |
82 | Status s = Env::LoadEnv(FLAGS_env_uri, &raw_env, &env_guard); | |
83 | if (raw_env == nullptr) { | |
84 | fprintf(stderr, "No Env registered for URI: %s\n", FLAGS_env_uri.c_str()); | |
85 | exit(1); | |
86 | } | |
20effc67 TL |
87 | } else if (!FLAGS_fs_uri.empty()) { |
88 | std::shared_ptr<FileSystem> fs; | |
89 | Status s = FileSystem::Load(FLAGS_fs_uri, &fs); | |
90 | if (!s.ok()) { | |
91 | fprintf(stderr, "Error: %s\n", s.ToString().c_str()); | |
92 | exit(1); | |
93 | } | |
94 | raw_env = GetCompositeEnv(fs); | |
f67539c2 TL |
95 | } else { |
96 | raw_env = Env::Default(); | |
97 | } | |
20effc67 TL |
98 | |
99 | #ifndef NDEBUG | |
100 | if (FLAGS_read_fault_one_in || FLAGS_sync_fault_injection) { | |
101 | FaultInjectionTestFS* fs = | |
102 | new FaultInjectionTestFS(raw_env->GetFileSystem()); | |
103 | fault_fs_guard.reset(fs); | |
104 | fault_fs_guard->SetFilesystemDirectWritable(true); | |
105 | fault_env_guard = | |
106 | std::make_shared<CompositeEnvWrapper>(raw_env, fault_fs_guard); | |
107 | raw_env = fault_env_guard.get(); | |
108 | } | |
109 | #endif | |
110 | ||
f67539c2 TL |
111 | env_wrapper_guard = std::make_shared<DbStressEnvWrapper>(raw_env); |
112 | db_stress_env = env_wrapper_guard.get(); | |
113 | ||
114 | FLAGS_rep_factory = StringToRepFactory(FLAGS_memtablerep.c_str()); | |
115 | ||
116 | // The number of background threads should be at least as much the | |
117 | // max number of concurrent compactions. | |
118 | db_stress_env->SetBackgroundThreads(FLAGS_max_background_compactions, | |
119 | ROCKSDB_NAMESPACE::Env::Priority::LOW); | |
120 | db_stress_env->SetBackgroundThreads(FLAGS_num_bottom_pri_threads, | |
121 | ROCKSDB_NAMESPACE::Env::Priority::BOTTOM); | |
122 | if (FLAGS_prefixpercent > 0 && FLAGS_prefix_size < 0) { | |
123 | fprintf(stderr, | |
124 | "Error: prefixpercent is non-zero while prefix_size is " | |
125 | "not positive!\n"); | |
126 | exit(1); | |
127 | } | |
128 | if (FLAGS_test_batches_snapshots && FLAGS_prefix_size <= 0) { | |
129 | fprintf(stderr, | |
130 | "Error: please specify prefix_size for " | |
131 | "test_batches_snapshots test!\n"); | |
132 | exit(1); | |
133 | } | |
134 | if (FLAGS_memtable_prefix_bloom_size_ratio > 0.0 && FLAGS_prefix_size < 0) { | |
135 | fprintf(stderr, | |
136 | "Error: please specify positive prefix_size in order to use " | |
137 | "memtable_prefix_bloom_size_ratio\n"); | |
138 | exit(1); | |
139 | } | |
140 | if ((FLAGS_readpercent + FLAGS_prefixpercent + FLAGS_writepercent + | |
141 | FLAGS_delpercent + FLAGS_delrangepercent + FLAGS_iterpercent) != 100) { | |
142 | fprintf(stderr, | |
143 | "Error: Read+Prefix+Write+Delete+DeleteRange+Iterate percents != " | |
144 | "100!\n"); | |
145 | exit(1); | |
146 | } | |
147 | if (FLAGS_disable_wal == 1 && FLAGS_reopen > 0) { | |
148 | fprintf(stderr, "Error: Db cannot reopen safely with disable_wal set!\n"); | |
149 | exit(1); | |
150 | } | |
151 | if ((unsigned)FLAGS_reopen >= FLAGS_ops_per_thread) { | |
152 | fprintf(stderr, | |
153 | "Error: #DB-reopens should be < ops_per_thread\n" | |
154 | "Provided reopens = %d and ops_per_thread = %lu\n", | |
155 | FLAGS_reopen, (unsigned long)FLAGS_ops_per_thread); | |
156 | exit(1); | |
157 | } | |
158 | if (FLAGS_test_batches_snapshots && FLAGS_delrangepercent > 0) { | |
159 | fprintf(stderr, | |
160 | "Error: nonzero delrangepercent unsupported in " | |
161 | "test_batches_snapshots mode\n"); | |
162 | exit(1); | |
163 | } | |
164 | if (FLAGS_active_width > FLAGS_max_key) { | |
165 | fprintf(stderr, "Error: active_width can be at most max_key\n"); | |
166 | exit(1); | |
167 | } else if (FLAGS_active_width == 0) { | |
168 | FLAGS_active_width = FLAGS_max_key; | |
169 | } | |
170 | if (FLAGS_value_size_mult * kRandomValueMaxFactor > kValueMaxLen) { | |
171 | fprintf(stderr, "Error: value_size_mult can be at most %d\n", | |
172 | kValueMaxLen / kRandomValueMaxFactor); | |
173 | exit(1); | |
174 | } | |
175 | if (FLAGS_use_merge && FLAGS_nooverwritepercent == 100) { | |
176 | fprintf( | |
177 | stderr, | |
178 | "Error: nooverwritepercent must not be 100 when using merge operands"); | |
179 | exit(1); | |
180 | } | |
181 | if (FLAGS_ingest_external_file_one_in > 0 && FLAGS_nooverwritepercent > 0) { | |
182 | fprintf(stderr, | |
183 | "Error: nooverwritepercent must be 0 when using file ingestion\n"); | |
184 | exit(1); | |
185 | } | |
186 | if (FLAGS_clear_column_family_one_in > 0 && FLAGS_backup_one_in > 0) { | |
187 | fprintf(stderr, | |
188 | "Error: clear_column_family_one_in must be 0 when using backup\n"); | |
189 | exit(1); | |
190 | } | |
191 | if (FLAGS_test_cf_consistency && FLAGS_disable_wal) { | |
192 | FLAGS_atomic_flush = true; | |
193 | } | |
194 | ||
195 | if (FLAGS_read_only) { | |
196 | if (FLAGS_writepercent != 0 || FLAGS_delpercent != 0 || | |
197 | FLAGS_delrangepercent != 0) { | |
198 | fprintf(stderr, "Error: updates are not supported in read only mode\n"); | |
199 | exit(1); | |
200 | } else if (FLAGS_checkpoint_one_in > 0 && | |
201 | FLAGS_clear_column_family_one_in > 0) { | |
202 | fprintf(stdout, | |
203 | "Warn: checkpoint won't be validated since column families may " | |
204 | "be dropped.\n"); | |
205 | } | |
206 | } | |
207 | ||
208 | // Choose a location for the test database if none given with --db=<path> | |
209 | if (FLAGS_db.empty()) { | |
210 | std::string default_db_path; | |
211 | db_stress_env->GetTestDirectory(&default_db_path); | |
212 | default_db_path += "/dbstress"; | |
213 | FLAGS_db = default_db_path; | |
214 | } | |
215 | ||
216 | if ((FLAGS_test_secondary || FLAGS_continuous_verification_interval > 0) && | |
217 | FLAGS_secondaries_base.empty()) { | |
218 | std::string default_secondaries_path; | |
219 | db_stress_env->GetTestDirectory(&default_secondaries_path); | |
220 | default_secondaries_path += "/dbstress_secondaries"; | |
221 | ROCKSDB_NAMESPACE::Status s = | |
222 | db_stress_env->CreateDirIfMissing(default_secondaries_path); | |
223 | if (!s.ok()) { | |
224 | fprintf(stderr, "Failed to create directory %s: %s\n", | |
225 | default_secondaries_path.c_str(), s.ToString().c_str()); | |
226 | exit(1); | |
227 | } | |
228 | FLAGS_secondaries_base = default_secondaries_path; | |
229 | } | |
230 | ||
231 | if (!FLAGS_test_secondary && FLAGS_secondary_catch_up_one_in > 0) { | |
232 | fprintf( | |
233 | stderr, | |
234 | "Must set -test_secondary=true if secondary_catch_up_one_in > 0.\n"); | |
235 | exit(1); | |
236 | } | |
20effc67 TL |
237 | if (FLAGS_best_efforts_recovery && !FLAGS_skip_verifydb && |
238 | !FLAGS_disable_wal) { | |
239 | fprintf(stderr, | |
240 | "With best-efforts recovery, either skip_verifydb or disable_wal " | |
241 | "should be set to true.\n"); | |
242 | exit(1); | |
243 | } | |
244 | if (FLAGS_skip_verifydb) { | |
245 | if (FLAGS_verify_db_one_in > 0) { | |
246 | fprintf(stderr, | |
247 | "Must set -verify_db_one_in=0 if skip_verifydb is true.\n"); | |
248 | exit(1); | |
249 | } | |
250 | if (FLAGS_continuous_verification_interval > 0) { | |
251 | fprintf(stderr, | |
252 | "Must set -continuous_verification_interval=0 if skip_verifydb " | |
253 | "is true.\n"); | |
254 | exit(1); | |
255 | } | |
256 | } | |
257 | if (FLAGS_enable_compaction_filter && | |
258 | (FLAGS_acquire_snapshot_one_in > 0 || FLAGS_compact_range_one_in > 0 || | |
259 | FLAGS_iterpercent > 0 || FLAGS_test_batches_snapshots || | |
260 | FLAGS_test_cf_consistency)) { | |
261 | fprintf( | |
262 | stderr, | |
263 | "Error: acquire_snapshot_one_in, compact_range_one_in, iterpercent, " | |
264 | "test_batches_snapshots must all be 0 when using compaction filter\n"); | |
265 | exit(1); | |
266 | } | |
f67539c2 TL |
267 | |
268 | rocksdb_kill_odds = FLAGS_kill_random_test; | |
20effc67 | 269 | rocksdb_kill_exclude_prefixes = SplitString(FLAGS_kill_exclude_prefixes); |
f67539c2 TL |
270 | |
271 | unsigned int levels = FLAGS_max_key_len; | |
272 | std::vector<std::string> weights; | |
273 | uint64_t scale_factor = FLAGS_key_window_scale_factor; | |
274 | key_gen_ctx.window = scale_factor * 100; | |
275 | if (!FLAGS_key_len_percent_dist.empty()) { | |
276 | weights = SplitString(FLAGS_key_len_percent_dist); | |
277 | if (weights.size() != levels) { | |
278 | fprintf(stderr, | |
279 | "Number of weights in key_len_dist should be equal to" | |
280 | " max_key_len"); | |
281 | exit(1); | |
282 | } | |
283 | ||
284 | uint64_t total_weight = 0; | |
285 | for (std::string& weight : weights) { | |
286 | uint64_t val = std::stoull(weight); | |
287 | key_gen_ctx.weights.emplace_back(val * scale_factor); | |
288 | total_weight += val; | |
289 | } | |
290 | if (total_weight != 100) { | |
291 | fprintf(stderr, "Sum of all weights in key_len_dist should be 100"); | |
292 | exit(1); | |
293 | } | |
294 | } else { | |
295 | uint64_t keys_per_level = key_gen_ctx.window / levels; | |
20effc67 | 296 | for (unsigned int level = 0; level + 1 < levels; ++level) { |
f67539c2 TL |
297 | key_gen_ctx.weights.emplace_back(keys_per_level); |
298 | } | |
299 | key_gen_ctx.weights.emplace_back(key_gen_ctx.window - | |
300 | keys_per_level * (levels - 1)); | |
301 | } | |
302 | ||
303 | std::unique_ptr<ROCKSDB_NAMESPACE::StressTest> stress; | |
304 | if (FLAGS_test_cf_consistency) { | |
305 | stress.reset(CreateCfConsistencyStressTest()); | |
306 | } else if (FLAGS_test_batches_snapshots) { | |
307 | stress.reset(CreateBatchedOpsStressTest()); | |
308 | } else { | |
309 | stress.reset(CreateNonBatchedOpsStressTest()); | |
310 | } | |
311 | // Initialize the Zipfian pre-calculated array | |
312 | InitializeHotKeyGenerator(FLAGS_hot_key_alpha); | |
313 | if (RunStressTest(stress.get())) { | |
314 | return 0; | |
315 | } else { | |
316 | return 1; | |
317 | } | |
318 | } | |
319 | ||
320 | } // namespace ROCKSDB_NAMESPACE | |
321 | #endif // GFLAGS |