]>
Commit | Line | Data |
---|---|---|
f67539c2 TL |
1 | // Copyright (c) 2011-present, Facebook, Inc. All rights reserved. |
2 | // This source code is licensed under both the GPLv2 (found in the | |
3 | // COPYING file in the root directory) and Apache 2.0 License | |
4 | // (found in the LICENSE.Apache file in the root directory). | |
5 | // | |
6 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. | |
7 | // Use of this source code is governed by a BSD-style license that can be | |
8 | // found in the LICENSE file. See the AUTHORS file for names of contributors. | |
9 | ||
10 | #ifdef GFLAGS | |
11 | #include "db_stress_tool/db_stress_common.h" | |
20effc67 | 12 | #include "file/file_util.h" |
f67539c2 TL |
13 | |
14 | namespace ROCKSDB_NAMESPACE { | |
15 | class CfConsistencyStressTest : public StressTest { | |
16 | public: | |
17 | CfConsistencyStressTest() : batch_id_(0) {} | |
18 | ||
19 | ~CfConsistencyStressTest() override {} | |
20 | ||
1e59de90 TL |
21 | bool IsStateTracked() const override { return false; } |
22 | ||
f67539c2 TL |
23 | Status TestPut(ThreadState* thread, WriteOptions& write_opts, |
24 | const ReadOptions& /* read_opts */, | |
25 | const std::vector<int>& rand_column_families, | |
1e59de90 TL |
26 | const std::vector<int64_t>& rand_keys, |
27 | char (&value)[100]) override { | |
28 | assert(!rand_column_families.empty()); | |
29 | assert(!rand_keys.empty()); | |
30 | ||
31 | const std::string k = Key(rand_keys[0]); | |
32 | ||
33 | const uint32_t value_base = batch_id_.fetch_add(1); | |
34 | const size_t sz = GenerateValue(value_base, value, sizeof(value)); | |
35 | const Slice v(value, sz); | |
36 | ||
f67539c2 | 37 | WriteBatch batch; |
1e59de90 TL |
38 | |
39 | const bool use_put_entity = !FLAGS_use_merge && | |
40 | FLAGS_use_put_entity_one_in > 0 && | |
41 | (value_base % FLAGS_use_put_entity_one_in) == 0; | |
42 | ||
f67539c2 | 43 | for (auto cf : rand_column_families) { |
1e59de90 TL |
44 | ColumnFamilyHandle* const cfh = column_families_[cf]; |
45 | assert(cfh); | |
46 | ||
f67539c2 | 47 | if (FLAGS_use_merge) { |
1e59de90 TL |
48 | batch.Merge(cfh, k, v); |
49 | } else if (use_put_entity) { | |
50 | batch.PutEntity(cfh, k, GenerateWideColumns(value_base, v)); | |
51 | } else { | |
52 | batch.Put(cfh, k, v); | |
f67539c2 TL |
53 | } |
54 | } | |
1e59de90 | 55 | |
f67539c2 | 56 | Status s = db_->Write(write_opts, &batch); |
1e59de90 | 57 | |
f67539c2 TL |
58 | if (!s.ok()) { |
59 | fprintf(stderr, "multi put or merge error: %s\n", s.ToString().c_str()); | |
60 | thread->stats.AddErrors(1); | |
61 | } else { | |
62 | auto num = static_cast<long>(rand_column_families.size()); | |
63 | thread->stats.AddBytesForWrites(num, (sz + 1) * num); | |
64 | } | |
65 | ||
66 | return s; | |
67 | } | |
68 | ||
69 | Status TestDelete(ThreadState* thread, WriteOptions& write_opts, | |
70 | const std::vector<int>& rand_column_families, | |
1e59de90 | 71 | const std::vector<int64_t>& rand_keys) override { |
f67539c2 TL |
72 | std::string key_str = Key(rand_keys[0]); |
73 | Slice key = key_str; | |
74 | WriteBatch batch; | |
75 | for (auto cf : rand_column_families) { | |
76 | ColumnFamilyHandle* cfh = column_families_[cf]; | |
77 | batch.Delete(cfh, key); | |
78 | } | |
79 | Status s = db_->Write(write_opts, &batch); | |
80 | if (!s.ok()) { | |
81 | fprintf(stderr, "multidel error: %s\n", s.ToString().c_str()); | |
82 | thread->stats.AddErrors(1); | |
83 | } else { | |
84 | thread->stats.AddDeletes(static_cast<long>(rand_column_families.size())); | |
85 | } | |
86 | return s; | |
87 | } | |
88 | ||
89 | Status TestDeleteRange(ThreadState* thread, WriteOptions& write_opts, | |
90 | const std::vector<int>& rand_column_families, | |
1e59de90 | 91 | const std::vector<int64_t>& rand_keys) override { |
f67539c2 TL |
92 | int64_t rand_key = rand_keys[0]; |
93 | auto shared = thread->shared; | |
94 | int64_t max_key = shared->GetMaxKey(); | |
95 | if (rand_key > max_key - FLAGS_range_deletion_width) { | |
96 | rand_key = | |
97 | thread->rand.Next() % (max_key - FLAGS_range_deletion_width + 1); | |
98 | } | |
99 | std::string key_str = Key(rand_key); | |
100 | Slice key = key_str; | |
101 | std::string end_key_str = Key(rand_key + FLAGS_range_deletion_width); | |
102 | Slice end_key = end_key_str; | |
103 | WriteBatch batch; | |
104 | for (auto cf : rand_column_families) { | |
105 | ColumnFamilyHandle* cfh = column_families_[rand_column_families[cf]]; | |
106 | batch.DeleteRange(cfh, key, end_key); | |
107 | } | |
108 | Status s = db_->Write(write_opts, &batch); | |
109 | if (!s.ok()) { | |
110 | fprintf(stderr, "multi del range error: %s\n", s.ToString().c_str()); | |
111 | thread->stats.AddErrors(1); | |
112 | } else { | |
113 | thread->stats.AddRangeDeletions( | |
114 | static_cast<long>(rand_column_families.size())); | |
115 | } | |
116 | return s; | |
117 | } | |
118 | ||
119 | void TestIngestExternalFile( | |
120 | ThreadState* /* thread */, | |
121 | const std::vector<int>& /* rand_column_families */, | |
1e59de90 | 122 | const std::vector<int64_t>& /* rand_keys */) override { |
f67539c2 TL |
123 | assert(false); |
124 | fprintf(stderr, | |
125 | "CfConsistencyStressTest does not support TestIngestExternalFile " | |
126 | "because it's not possible to verify the result\n"); | |
127 | std::terminate(); | |
128 | } | |
129 | ||
130 | Status TestGet(ThreadState* thread, const ReadOptions& readoptions, | |
131 | const std::vector<int>& rand_column_families, | |
132 | const std::vector<int64_t>& rand_keys) override { | |
133 | std::string key_str = Key(rand_keys[0]); | |
134 | Slice key = key_str; | |
135 | Status s; | |
136 | bool is_consistent = true; | |
137 | ||
138 | if (thread->rand.OneIn(2)) { | |
139 | // 1/2 chance, does a random read from random CF | |
140 | auto cfh = | |
141 | column_families_[rand_column_families[thread->rand.Next() % | |
142 | rand_column_families.size()]]; | |
143 | std::string from_db; | |
144 | s = db_->Get(readoptions, cfh, key, &from_db); | |
145 | } else { | |
146 | // 1/2 chance, comparing one key is the same across all CFs | |
147 | const Snapshot* snapshot = db_->GetSnapshot(); | |
148 | ReadOptions readoptionscopy = readoptions; | |
149 | readoptionscopy.snapshot = snapshot; | |
150 | ||
151 | std::string value0; | |
152 | s = db_->Get(readoptionscopy, column_families_[rand_column_families[0]], | |
153 | key, &value0); | |
154 | if (s.ok() || s.IsNotFound()) { | |
155 | bool found = s.ok(); | |
156 | for (size_t i = 1; i < rand_column_families.size(); i++) { | |
157 | std::string value1; | |
158 | s = db_->Get(readoptionscopy, | |
159 | column_families_[rand_column_families[i]], key, &value1); | |
160 | if (!s.ok() && !s.IsNotFound()) { | |
161 | break; | |
162 | } | |
163 | if (!found && s.ok()) { | |
164 | fprintf(stderr, "Get() return different results with key %s\n", | |
165 | Slice(key_str).ToString(true).c_str()); | |
166 | fprintf(stderr, "CF %s is not found\n", | |
167 | column_family_names_[0].c_str()); | |
168 | fprintf(stderr, "CF %s returns value %s\n", | |
169 | column_family_names_[i].c_str(), | |
170 | Slice(value1).ToString(true).c_str()); | |
171 | is_consistent = false; | |
172 | } else if (found && s.IsNotFound()) { | |
173 | fprintf(stderr, "Get() return different results with key %s\n", | |
174 | Slice(key_str).ToString(true).c_str()); | |
175 | fprintf(stderr, "CF %s returns value %s\n", | |
176 | column_family_names_[0].c_str(), | |
177 | Slice(value0).ToString(true).c_str()); | |
178 | fprintf(stderr, "CF %s is not found\n", | |
179 | column_family_names_[i].c_str()); | |
180 | is_consistent = false; | |
181 | } else if (s.ok() && value0 != value1) { | |
182 | fprintf(stderr, "Get() return different results with key %s\n", | |
183 | Slice(key_str).ToString(true).c_str()); | |
184 | fprintf(stderr, "CF %s returns value %s\n", | |
185 | column_family_names_[0].c_str(), | |
186 | Slice(value0).ToString(true).c_str()); | |
187 | fprintf(stderr, "CF %s returns value %s\n", | |
188 | column_family_names_[i].c_str(), | |
189 | Slice(value1).ToString(true).c_str()); | |
190 | is_consistent = false; | |
191 | } | |
192 | if (!is_consistent) { | |
193 | break; | |
194 | } | |
195 | } | |
196 | } | |
197 | ||
198 | db_->ReleaseSnapshot(snapshot); | |
199 | } | |
200 | if (!is_consistent) { | |
201 | fprintf(stderr, "TestGet error: is_consistent is false\n"); | |
202 | thread->stats.AddErrors(1); | |
203 | // Fail fast to preserve the DB state. | |
204 | thread->shared->SetVerificationFailure(); | |
205 | } else if (s.ok()) { | |
206 | thread->stats.AddGets(1, 1); | |
207 | } else if (s.IsNotFound()) { | |
208 | thread->stats.AddGets(1, 0); | |
209 | } else { | |
210 | fprintf(stderr, "TestGet error: %s\n", s.ToString().c_str()); | |
211 | thread->stats.AddErrors(1); | |
212 | } | |
213 | return s; | |
214 | } | |
215 | ||
216 | std::vector<Status> TestMultiGet( | |
217 | ThreadState* thread, const ReadOptions& read_opts, | |
218 | const std::vector<int>& rand_column_families, | |
219 | const std::vector<int64_t>& rand_keys) override { | |
220 | size_t num_keys = rand_keys.size(); | |
221 | std::vector<std::string> key_str; | |
222 | std::vector<Slice> keys; | |
223 | keys.reserve(num_keys); | |
224 | key_str.reserve(num_keys); | |
225 | std::vector<PinnableSlice> values(num_keys); | |
226 | std::vector<Status> statuses(num_keys); | |
227 | ColumnFamilyHandle* cfh = column_families_[rand_column_families[0]]; | |
1e59de90 TL |
228 | ReadOptions readoptionscopy = read_opts; |
229 | readoptionscopy.rate_limiter_priority = | |
230 | FLAGS_rate_limit_user_ops ? Env::IO_USER : Env::IO_TOTAL; | |
f67539c2 TL |
231 | |
232 | for (size_t i = 0; i < num_keys; ++i) { | |
233 | key_str.emplace_back(Key(rand_keys[i])); | |
234 | keys.emplace_back(key_str.back()); | |
235 | } | |
1e59de90 | 236 | db_->MultiGet(readoptionscopy, cfh, num_keys, keys.data(), values.data(), |
f67539c2 TL |
237 | statuses.data()); |
238 | for (auto s : statuses) { | |
239 | if (s.ok()) { | |
240 | // found case | |
241 | thread->stats.AddGets(1, 1); | |
242 | } else if (s.IsNotFound()) { | |
243 | // not found case | |
244 | thread->stats.AddGets(1, 0); | |
245 | } else { | |
246 | // errors case | |
247 | fprintf(stderr, "MultiGet error: %s\n", s.ToString().c_str()); | |
248 | thread->stats.AddErrors(1); | |
249 | } | |
250 | } | |
251 | return statuses; | |
252 | } | |
253 | ||
254 | Status TestPrefixScan(ThreadState* thread, const ReadOptions& readoptions, | |
255 | const std::vector<int>& rand_column_families, | |
256 | const std::vector<int64_t>& rand_keys) override { | |
1e59de90 TL |
257 | assert(!rand_column_families.empty()); |
258 | assert(!rand_keys.empty()); | |
259 | ||
260 | const std::string key = Key(rand_keys[0]); | |
261 | ||
262 | const size_t prefix_to_use = | |
f67539c2 TL |
263 | (FLAGS_prefix_size < 0) ? 7 : static_cast<size_t>(FLAGS_prefix_size); |
264 | ||
1e59de90 | 265 | const Slice prefix(key.data(), prefix_to_use); |
f67539c2 TL |
266 | |
267 | std::string upper_bound; | |
268 | Slice ub_slice; | |
1e59de90 | 269 | |
f67539c2 | 270 | ReadOptions ro_copy = readoptions; |
1e59de90 | 271 | |
f67539c2 TL |
272 | // Get the next prefix first and then see if we want to set upper bound. |
273 | // We'll use the next prefix in an assertion later on | |
274 | if (GetNextPrefix(prefix, &upper_bound) && thread->rand.OneIn(2)) { | |
275 | ub_slice = Slice(upper_bound); | |
276 | ro_copy.iterate_upper_bound = &ub_slice; | |
277 | } | |
1e59de90 TL |
278 | |
279 | ColumnFamilyHandle* const cfh = | |
280 | column_families_[rand_column_families[thread->rand.Uniform( | |
281 | static_cast<int>(rand_column_families.size()))]]; | |
282 | assert(cfh); | |
283 | ||
284 | std::unique_ptr<Iterator> iter(db_->NewIterator(ro_copy, cfh)); | |
285 | ||
286 | uint64_t count = 0; | |
287 | Status s; | |
288 | ||
f67539c2 TL |
289 | for (iter->Seek(prefix); iter->Valid() && iter->key().starts_with(prefix); |
290 | iter->Next()) { | |
291 | ++count; | |
1e59de90 TL |
292 | |
293 | const WideColumns expected_columns = GenerateExpectedWideColumns( | |
294 | GetValueBase(iter->value()), iter->value()); | |
295 | if (iter->columns() != expected_columns) { | |
296 | s = Status::Corruption( | |
297 | "Value and columns inconsistent", | |
298 | DebugString(iter->value(), iter->columns(), expected_columns)); | |
299 | break; | |
300 | } | |
f67539c2 | 301 | } |
1e59de90 | 302 | |
f67539c2 TL |
303 | assert(prefix_to_use == 0 || |
304 | count <= GetPrefixKeyCount(prefix.ToString(), upper_bound)); | |
1e59de90 | 305 | |
f67539c2 | 306 | if (s.ok()) { |
1e59de90 TL |
307 | s = iter->status(); |
308 | } | |
309 | ||
310 | if (!s.ok()) { | |
f67539c2 TL |
311 | fprintf(stderr, "TestPrefixScan error: %s\n", s.ToString().c_str()); |
312 | thread->stats.AddErrors(1); | |
1e59de90 TL |
313 | |
314 | return s; | |
f67539c2 | 315 | } |
1e59de90 TL |
316 | |
317 | thread->stats.AddPrefixes(1, count); | |
318 | ||
319 | return Status::OK(); | |
f67539c2 TL |
320 | } |
321 | ||
322 | ColumnFamilyHandle* GetControlCfh(ThreadState* thread, | |
323 | int /*column_family_id*/ | |
324 | ) override { | |
325 | // All column families should contain the same data. Randomly pick one. | |
326 | return column_families_[thread->rand.Next() % column_families_.size()]; | |
327 | } | |
328 | ||
f67539c2 | 329 | void VerifyDb(ThreadState* thread) const override { |
1e59de90 TL |
330 | // This `ReadOptions` is for validation purposes. Ignore |
331 | // `FLAGS_rate_limit_user_ops` to avoid slowing any validation. | |
f67539c2 | 332 | ReadOptions options(FLAGS_verify_checksum, true); |
1e59de90 | 333 | |
f67539c2 TL |
334 | // We must set total_order_seek to true because we are doing a SeekToFirst |
335 | // on a column family whose memtables may support (by default) prefix-based | |
336 | // iterator. In this case, NewIterator with options.total_order_seek being | |
337 | // false returns a prefix-based iterator. Calling SeekToFirst using this | |
338 | // iterator causes the iterator to become invalid. That means we cannot | |
339 | // iterate the memtable using this iterator any more, although the memtable | |
340 | // contains the most up-to-date key-values. | |
341 | options.total_order_seek = true; | |
1e59de90 TL |
342 | |
343 | ManagedSnapshot snapshot_guard(db_); | |
344 | options.snapshot = snapshot_guard.snapshot(); | |
345 | ||
346 | const size_t num = column_families_.size(); | |
347 | ||
348 | std::vector<std::unique_ptr<Iterator>> iters; | |
349 | iters.reserve(num); | |
350 | ||
351 | for (size_t i = 0; i < num; ++i) { | |
352 | iters.emplace_back(db_->NewIterator(options, column_families_[i])); | |
353 | iters.back()->SeekToFirst(); | |
f67539c2 | 354 | } |
1e59de90 | 355 | |
f67539c2 | 356 | std::vector<Status> statuses(num, Status::OK()); |
1e59de90 TL |
357 | |
358 | assert(thread); | |
359 | ||
360 | auto shared = thread->shared; | |
361 | assert(shared); | |
362 | ||
f67539c2 TL |
363 | do { |
364 | if (shared->HasVerificationFailedYet()) { | |
365 | break; | |
366 | } | |
1e59de90 | 367 | |
f67539c2 | 368 | size_t valid_cnt = 0; |
1e59de90 TL |
369 | |
370 | for (size_t i = 0; i < num; ++i) { | |
371 | const auto& iter = iters[i]; | |
372 | assert(iter); | |
373 | ||
f67539c2 | 374 | if (iter->Valid()) { |
1e59de90 TL |
375 | const WideColumns expected_columns = GenerateExpectedWideColumns( |
376 | GetValueBase(iter->value()), iter->value()); | |
377 | if (iter->columns() != expected_columns) { | |
378 | statuses[i] = Status::Corruption( | |
379 | "Value and columns inconsistent", | |
380 | DebugString(iter->value(), iter->columns(), expected_columns)); | |
381 | } else { | |
382 | ++valid_cnt; | |
383 | } | |
f67539c2 | 384 | } else { |
1e59de90 | 385 | statuses[i] = iter->status(); |
f67539c2 | 386 | } |
f67539c2 | 387 | } |
1e59de90 | 388 | |
f67539c2 | 389 | if (valid_cnt == 0) { |
1e59de90 | 390 | for (size_t i = 0; i < num; ++i) { |
f67539c2 TL |
391 | const auto& s = statuses[i]; |
392 | if (!s.ok()) { | |
f67539c2 TL |
393 | fprintf(stderr, "Iterator on cf %s has error: %s\n", |
394 | column_families_[i]->GetName().c_str(), | |
395 | s.ToString().c_str()); | |
396 | shared->SetVerificationFailure(); | |
397 | } | |
398 | } | |
1e59de90 | 399 | |
f67539c2 | 400 | break; |
1e59de90 TL |
401 | } |
402 | ||
403 | if (valid_cnt < num) { | |
f67539c2 | 404 | shared->SetVerificationFailure(); |
1e59de90 TL |
405 | |
406 | for (size_t i = 0; i < num; ++i) { | |
407 | assert(iters[i]); | |
408 | ||
f67539c2 TL |
409 | if (!iters[i]->Valid()) { |
410 | if (statuses[i].ok()) { | |
411 | fprintf(stderr, "Finished scanning cf %s\n", | |
412 | column_families_[i]->GetName().c_str()); | |
413 | } else { | |
414 | fprintf(stderr, "Iterator on cf %s has error: %s\n", | |
415 | column_families_[i]->GetName().c_str(), | |
416 | statuses[i].ToString().c_str()); | |
417 | } | |
418 | } else { | |
419 | fprintf(stderr, "cf %s has remaining data to scan\n", | |
420 | column_families_[i]->GetName().c_str()); | |
421 | } | |
422 | } | |
1e59de90 | 423 | |
f67539c2 TL |
424 | break; |
425 | } | |
1e59de90 | 426 | |
f67539c2 TL |
427 | if (shared->HasVerificationFailedYet()) { |
428 | break; | |
429 | } | |
1e59de90 | 430 | |
f67539c2 TL |
431 | // If the program reaches here, then all column families' iterators are |
432 | // still valid. | |
1e59de90 TL |
433 | assert(valid_cnt == num); |
434 | ||
f67539c2 TL |
435 | if (shared->PrintingVerificationResults()) { |
436 | continue; | |
437 | } | |
1e59de90 TL |
438 | |
439 | assert(iters[0]); | |
440 | ||
441 | const Slice key = iters[0]->key(); | |
442 | const Slice value = iters[0]->value(); | |
443 | ||
f67539c2 | 444 | int num_mismatched_cfs = 0; |
1e59de90 TL |
445 | |
446 | for (size_t i = 1; i < num; ++i) { | |
447 | assert(iters[i]); | |
448 | ||
449 | const int cmp = key.compare(iters[i]->key()); | |
450 | ||
451 | if (cmp != 0) { | |
452 | ++num_mismatched_cfs; | |
453 | ||
454 | if (1 == num_mismatched_cfs) { | |
455 | fprintf(stderr, "Verification failed\n"); | |
456 | fprintf(stderr, "Latest Sequence Number: %" PRIu64 "\n", | |
457 | db_->GetLatestSequenceNumber()); | |
f67539c2 | 458 | fprintf(stderr, "[%s] %s => %s\n", |
1e59de90 TL |
459 | column_families_[0]->GetName().c_str(), |
460 | key.ToString(true /* hex */).c_str(), | |
461 | value.ToString(true /* hex */).c_str()); | |
462 | } | |
463 | ||
464 | fprintf(stderr, "[%s] %s => %s\n", | |
465 | column_families_[i]->GetName().c_str(), | |
466 | iters[i]->key().ToString(true /* hex */).c_str(), | |
467 | iters[i]->value().ToString(true /* hex */).c_str()); | |
468 | ||
f67539c2 | 469 | #ifndef ROCKSDB_LITE |
1e59de90 TL |
470 | Slice begin_key; |
471 | Slice end_key; | |
472 | if (cmp < 0) { | |
473 | begin_key = key; | |
474 | end_key = iters[i]->key(); | |
475 | } else { | |
476 | begin_key = iters[i]->key(); | |
477 | end_key = key; | |
478 | } | |
479 | ||
480 | const auto print_key_versions = [&](ColumnFamilyHandle* cfh) { | |
481 | constexpr size_t kMaxNumIKeys = 8; | |
482 | ||
f67539c2 | 483 | std::vector<KeyVersion> versions; |
1e59de90 TL |
484 | const Status s = GetAllKeyVersions(db_, cfh, begin_key, end_key, |
485 | kMaxNumIKeys, &versions); | |
486 | if (!s.ok()) { | |
487 | fprintf(stderr, "%s\n", s.ToString().c_str()); | |
488 | return; | |
f67539c2 | 489 | } |
1e59de90 TL |
490 | |
491 | assert(cfh); | |
492 | ||
493 | fprintf(stderr, | |
494 | "Internal keys in CF '%s', [%s, %s] (max %" ROCKSDB_PRIszt | |
495 | ")\n", | |
496 | cfh->GetName().c_str(), | |
497 | begin_key.ToString(true /* hex */).c_str(), | |
498 | end_key.ToString(true /* hex */).c_str(), kMaxNumIKeys); | |
499 | ||
500 | for (const KeyVersion& kv : versions) { | |
501 | fprintf(stderr, " key %s seq %" PRIu64 " type %d\n", | |
502 | Slice(kv.user_key).ToString(true).c_str(), kv.sequence, | |
503 | kv.type); | |
504 | } | |
505 | }; | |
506 | ||
507 | if (1 == num_mismatched_cfs) { | |
508 | print_key_versions(column_families_[0]); | |
f67539c2 | 509 | } |
1e59de90 TL |
510 | |
511 | print_key_versions(column_families_[i]); | |
512 | #endif // ROCKSDB_LITE | |
513 | ||
514 | shared->SetVerificationFailure(); | |
f67539c2 TL |
515 | } |
516 | } | |
1e59de90 | 517 | |
f67539c2 | 518 | shared->FinishPrintingVerificationResults(); |
1e59de90 | 519 | |
f67539c2 | 520 | for (auto& iter : iters) { |
1e59de90 | 521 | assert(iter); |
f67539c2 TL |
522 | iter->Next(); |
523 | } | |
524 | } while (true); | |
525 | } | |
526 | ||
527 | #ifndef ROCKSDB_LITE | |
528 | void ContinuouslyVerifyDb(ThreadState* thread) const override { | |
529 | assert(thread); | |
530 | Status status; | |
531 | ||
532 | DB* db_ptr = cmp_db_ ? cmp_db_ : db_; | |
533 | const auto& cfhs = cmp_db_ ? cmp_cfhs_ : column_families_; | |
1e59de90 TL |
534 | |
535 | // Take a snapshot to preserve the state of primary db. | |
536 | ManagedSnapshot snapshot_guard(db_); | |
537 | ||
f67539c2 TL |
538 | SharedState* shared = thread->shared; |
539 | assert(shared); | |
1e59de90 TL |
540 | |
541 | if (cmp_db_) { | |
542 | status = cmp_db_->TryCatchUpWithPrimary(); | |
543 | if (!status.ok()) { | |
544 | fprintf(stderr, "TryCatchUpWithPrimary: %s\n", | |
545 | status.ToString().c_str()); | |
546 | shared->SetShouldStopTest(); | |
547 | assert(false); | |
548 | return; | |
549 | } | |
f67539c2 | 550 | } |
1e59de90 | 551 | |
f67539c2 TL |
552 | const auto checksum_column_family = [](Iterator* iter, |
553 | uint32_t* checksum) -> Status { | |
554 | assert(nullptr != checksum); | |
1e59de90 | 555 | |
f67539c2 TL |
556 | uint32_t ret = 0; |
557 | for (iter->SeekToFirst(); iter->Valid(); iter->Next()) { | |
558 | ret = crc32c::Extend(ret, iter->key().data(), iter->key().size()); | |
559 | ret = crc32c::Extend(ret, iter->value().data(), iter->value().size()); | |
1e59de90 TL |
560 | |
561 | for (const auto& column : iter->columns()) { | |
562 | ret = crc32c::Extend(ret, column.name().data(), column.name().size()); | |
563 | ret = | |
564 | crc32c::Extend(ret, column.value().data(), column.value().size()); | |
565 | } | |
f67539c2 | 566 | } |
1e59de90 | 567 | |
f67539c2 TL |
568 | *checksum = ret; |
569 | return iter->status(); | |
570 | }; | |
1e59de90 TL |
571 | // This `ReadOptions` is for validation purposes. Ignore |
572 | // `FLAGS_rate_limit_user_ops` to avoid slowing any validation. | |
573 | ReadOptions ropts(FLAGS_verify_checksum, true); | |
f67539c2 | 574 | ropts.total_order_seek = true; |
1e59de90 TL |
575 | if (nullptr == cmp_db_) { |
576 | ropts.snapshot = snapshot_guard.snapshot(); | |
577 | } | |
f67539c2 TL |
578 | uint32_t crc = 0; |
579 | { | |
580 | // Compute crc for all key-values of default column family. | |
581 | std::unique_ptr<Iterator> it(db_ptr->NewIterator(ropts)); | |
582 | status = checksum_column_family(it.get(), &crc); | |
1e59de90 TL |
583 | if (!status.ok()) { |
584 | fprintf(stderr, "Computing checksum of default cf: %s\n", | |
585 | status.ToString().c_str()); | |
586 | assert(false); | |
587 | } | |
f67539c2 | 588 | } |
1e59de90 TL |
589 | // Since we currently intentionally disallow reading from the secondary |
590 | // instance with snapshot, we cannot achieve cross-cf consistency if WAL is | |
591 | // enabled because there is no guarantee that secondary instance replays | |
592 | // the primary's WAL to a consistent point where all cfs have the same | |
593 | // data. | |
594 | if (status.ok() && FLAGS_disable_wal) { | |
595 | uint32_t tmp_crc = 0; | |
f67539c2 TL |
596 | for (ColumnFamilyHandle* cfh : cfhs) { |
597 | if (cfh == db_ptr->DefaultColumnFamily()) { | |
598 | continue; | |
599 | } | |
600 | std::unique_ptr<Iterator> it(db_ptr->NewIterator(ropts, cfh)); | |
601 | status = checksum_column_family(it.get(), &tmp_crc); | |
602 | if (!status.ok() || tmp_crc != crc) { | |
603 | break; | |
604 | } | |
605 | } | |
1e59de90 TL |
606 | if (!status.ok()) { |
607 | fprintf(stderr, "status: %s\n", status.ToString().c_str()); | |
608 | shared->SetShouldStopTest(); | |
609 | assert(false); | |
610 | } else if (tmp_crc != crc) { | |
611 | fprintf(stderr, "tmp_crc=%" PRIu32 " crc=%" PRIu32 "\n", tmp_crc, crc); | |
612 | shared->SetShouldStopTest(); | |
613 | assert(false); | |
614 | } | |
f67539c2 TL |
615 | } |
616 | } | |
1e59de90 TL |
617 | #else // ROCKSDB_LITE |
618 | void ContinuouslyVerifyDb(ThreadState* /*thread*/) const override {} | |
f67539c2 TL |
619 | #endif // !ROCKSDB_LITE |
620 | ||
621 | std::vector<int> GenerateColumnFamilies( | |
622 | const int /* num_column_families */, | |
623 | int /* rand_column_family */) const override { | |
624 | std::vector<int> ret; | |
625 | int num = static_cast<int>(column_families_.size()); | |
626 | int k = 0; | |
627 | std::generate_n(back_inserter(ret), num, [&k]() -> int { return k++; }); | |
628 | return ret; | |
629 | } | |
630 | ||
631 | private: | |
1e59de90 | 632 | std::atomic<uint32_t> batch_id_; |
f67539c2 TL |
633 | }; |
634 | ||
635 | StressTest* CreateCfConsistencyStressTest() { | |
636 | return new CfConsistencyStressTest(); | |
637 | } | |
638 | ||
639 | } // namespace ROCKSDB_NAMESPACE | |
640 | #endif // GFLAGS |