]>
Commit | Line | Data |
---|---|---|
7c673cae | 1 | // Copyright (c) 2011-present, Facebook, Inc. All rights reserved. |
11fdf7f2 TL |
2 | // This source code is licensed under both the GPLv2 (found in the |
3 | // COPYING file in the root directory) and Apache 2.0 License | |
4 | // (found in the LICENSE.Apache file in the root directory). | |
7c673cae FG |
5 | // |
6 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. | |
7 | // Use of this source code is governed by a BSD-style license that can be | |
8 | // found in the LICENSE file. See the AUTHORS file for names of contributors. | |
9 | ||
10 | #ifndef ROCKSDB_LITE | |
11 | ||
7c673cae | 12 | #include <fcntl.h> |
7c673cae FG |
13 | #include <sys/stat.h> |
14 | #include <sys/types.h> | |
20effc67 | 15 | |
f67539c2 | 16 | #include <cinttypes> |
20effc67 | 17 | |
f67539c2 TL |
18 | #include "db/db_impl/db_impl.h" |
19 | #include "db/db_test_util.h" | |
7c673cae FG |
20 | #include "db/log_format.h" |
21 | #include "db/version_set.h" | |
f67539c2 TL |
22 | #include "env/composite_env_wrapper.h" |
23 | #include "file/filename.h" | |
7c673cae | 24 | #include "rocksdb/cache.h" |
11fdf7f2 | 25 | #include "rocksdb/convenience.h" |
20effc67 | 26 | #include "rocksdb/db.h" |
7c673cae FG |
27 | #include "rocksdb/env.h" |
28 | #include "rocksdb/table.h" | |
29 | #include "rocksdb/write_batch.h" | |
f67539c2 | 30 | #include "table/block_based/block_based_table_builder.h" |
494da23a | 31 | #include "table/meta_blocks.h" |
20effc67 | 32 | #include "table/mock_table.h" |
f67539c2 TL |
33 | #include "test_util/testharness.h" |
34 | #include "test_util/testutil.h" | |
20effc67 TL |
35 | #include "util/cast_util.h" |
36 | #include "util/random.h" | |
7c673cae | 37 | #include "util/string_util.h" |
7c673cae | 38 | |
f67539c2 | 39 | namespace ROCKSDB_NAMESPACE { |
7c673cae | 40 | |
20effc67 | 41 | static constexpr int kValueSize = 1000; |
7c673cae FG |
42 | |
43 | class CorruptionTest : public testing::Test { | |
44 | public: | |
45 | test::ErrorEnv env_; | |
46 | std::string dbname_; | |
494da23a | 47 | std::shared_ptr<Cache> tiny_cache_; |
7c673cae FG |
48 | Options options_; |
49 | DB* db_; | |
50 | ||
51 | CorruptionTest() { | |
52 | // If LRU cache shard bit is smaller than 2 (or -1 which will automatically | |
53 | // set it to 0), test SequenceNumberRecovery will fail, likely because of a | |
54 | // bug in recovery code. Keep it 4 for now to make the test passes. | |
55 | tiny_cache_ = NewLRUCache(100, 4); | |
56 | options_.wal_recovery_mode = WALRecoveryMode::kTolerateCorruptedTailRecords; | |
57 | options_.env = &env_; | |
11fdf7f2 | 58 | dbname_ = test::PerThreadDBPath("corruption_test"); |
20effc67 TL |
59 | Status s = DestroyDB(dbname_, options_); |
60 | EXPECT_OK(s); | |
7c673cae FG |
61 | |
62 | db_ = nullptr; | |
63 | options_.create_if_missing = true; | |
64 | BlockBasedTableOptions table_options; | |
65 | table_options.block_size_deviation = 0; // make unit test pass for now | |
66 | options_.table_factory.reset(NewBlockBasedTableFactory(table_options)); | |
67 | Reopen(); | |
68 | options_.create_if_missing = false; | |
69 | } | |
70 | ||
494da23a | 71 | ~CorruptionTest() override { |
20effc67 TL |
72 | SyncPoint::GetInstance()->DisableProcessing(); |
73 | SyncPoint::GetInstance()->LoadDependency({}); | |
74 | SyncPoint::GetInstance()->ClearAllCallBacks(); | |
494da23a | 75 | delete db_; |
20effc67 TL |
76 | db_ = nullptr; |
77 | if (getenv("KEEP_DB")) { | |
78 | fprintf(stdout, "db is still at %s\n", dbname_.c_str()); | |
79 | } else { | |
80 | EXPECT_OK(DestroyDB(dbname_, Options())); | |
81 | } | |
7c673cae FG |
82 | } |
83 | ||
84 | void CloseDb() { | |
85 | delete db_; | |
86 | db_ = nullptr; | |
87 | } | |
88 | ||
89 | Status TryReopen(Options* options = nullptr) { | |
90 | delete db_; | |
91 | db_ = nullptr; | |
92 | Options opt = (options ? *options : options_); | |
f67539c2 TL |
93 | if (opt.env == Options().env) { |
94 | // If env is not overridden, replace it with ErrorEnv. | |
95 | // Otherwise, the test already uses a non-default Env. | |
96 | opt.env = &env_; | |
97 | } | |
7c673cae FG |
98 | opt.arena_block_size = 4096; |
99 | BlockBasedTableOptions table_options; | |
100 | table_options.block_cache = tiny_cache_; | |
101 | table_options.block_size_deviation = 0; | |
102 | opt.table_factory.reset(NewBlockBasedTableFactory(table_options)); | |
103 | return DB::Open(opt, dbname_, &db_); | |
104 | } | |
105 | ||
106 | void Reopen(Options* options = nullptr) { | |
107 | ASSERT_OK(TryReopen(options)); | |
108 | } | |
109 | ||
110 | void RepairDB() { | |
111 | delete db_; | |
112 | db_ = nullptr; | |
f67539c2 | 113 | ASSERT_OK(::ROCKSDB_NAMESPACE::RepairDB(dbname_, options_)); |
7c673cae FG |
114 | } |
115 | ||
20effc67 | 116 | void Build(int n, int start, int flush_every) { |
7c673cae FG |
117 | std::string key_space, value_space; |
118 | WriteBatch batch; | |
119 | for (int i = 0; i < n; i++) { | |
120 | if (flush_every != 0 && i != 0 && i % flush_every == 0) { | |
20effc67 TL |
121 | DBImpl* dbi = static_cast_with_check<DBImpl>(db_); |
122 | ASSERT_OK(dbi->TEST_FlushMemTable()); | |
7c673cae FG |
123 | } |
124 | //if ((i % 100) == 0) fprintf(stderr, "@ %d of %d\n", i, n); | |
20effc67 | 125 | Slice key = Key(i + start, &key_space); |
7c673cae | 126 | batch.Clear(); |
20effc67 | 127 | ASSERT_OK(batch.Put(key, Value(i + start, &value_space))); |
7c673cae FG |
128 | ASSERT_OK(db_->Write(WriteOptions(), &batch)); |
129 | } | |
130 | } | |
131 | ||
20effc67 TL |
132 | void Build(int n, int flush_every = 0) { Build(n, 0, flush_every); } |
133 | ||
7c673cae FG |
134 | void Check(int min_expected, int max_expected) { |
135 | uint64_t next_expected = 0; | |
136 | uint64_t missed = 0; | |
137 | int bad_keys = 0; | |
138 | int bad_values = 0; | |
139 | int correct = 0; | |
140 | std::string value_space; | |
141 | // Do not verify checksums. If we verify checksums then the | |
142 | // db itself will raise errors because data is corrupted. | |
143 | // Instead, we want the reads to be successful and this test | |
144 | // will detect whether the appropriate corruptions have | |
145 | // occurred. | |
146 | Iterator* iter = db_->NewIterator(ReadOptions(false, true)); | |
147 | for (iter->SeekToFirst(); iter->Valid(); iter->Next()) { | |
20effc67 | 148 | ASSERT_OK(iter->status()); |
7c673cae FG |
149 | uint64_t key; |
150 | Slice in(iter->key()); | |
151 | if (!ConsumeDecimalNumber(&in, &key) || | |
152 | !in.empty() || | |
153 | key < next_expected) { | |
154 | bad_keys++; | |
155 | continue; | |
156 | } | |
157 | missed += (key - next_expected); | |
158 | next_expected = key + 1; | |
159 | if (iter->value() != Value(static_cast<int>(key), &value_space)) { | |
160 | bad_values++; | |
161 | } else { | |
162 | correct++; | |
163 | } | |
164 | } | |
20effc67 | 165 | iter->status().PermitUncheckedError(); |
7c673cae FG |
166 | delete iter; |
167 | ||
168 | fprintf(stderr, | |
169 | "expected=%d..%d; got=%d; bad_keys=%d; bad_values=%d; missed=%llu\n", | |
170 | min_expected, max_expected, correct, bad_keys, bad_values, | |
171 | static_cast<unsigned long long>(missed)); | |
172 | ASSERT_LE(min_expected, correct); | |
173 | ASSERT_GE(max_expected, correct); | |
174 | } | |
175 | ||
7c673cae FG |
176 | void Corrupt(FileType filetype, int offset, int bytes_to_corrupt) { |
177 | // Pick file to corrupt | |
178 | std::vector<std::string> filenames; | |
179 | ASSERT_OK(env_.GetChildren(dbname_, &filenames)); | |
180 | uint64_t number; | |
181 | FileType type; | |
182 | std::string fname; | |
183 | int picked_number = -1; | |
184 | for (size_t i = 0; i < filenames.size(); i++) { | |
185 | if (ParseFileName(filenames[i], &number, &type) && | |
186 | type == filetype && | |
187 | static_cast<int>(number) > picked_number) { // Pick latest file | |
188 | fname = dbname_ + "/" + filenames[i]; | |
189 | picked_number = static_cast<int>(number); | |
190 | } | |
191 | } | |
192 | ASSERT_TRUE(!fname.empty()) << filetype; | |
193 | ||
20effc67 | 194 | ASSERT_OK(test::CorruptFile(&env_, fname, offset, bytes_to_corrupt)); |
7c673cae FG |
195 | } |
196 | ||
197 | // corrupts exactly one file at level `level`. if no file found at level, | |
198 | // asserts | |
199 | void CorruptTableFileAtLevel(int level, int offset, int bytes_to_corrupt) { | |
200 | std::vector<LiveFileMetaData> metadata; | |
201 | db_->GetLiveFilesMetaData(&metadata); | |
202 | for (const auto& m : metadata) { | |
203 | if (m.level == level) { | |
20effc67 TL |
204 | ASSERT_OK(test::CorruptFile(&env_, dbname_ + "/" + m.name, offset, |
205 | bytes_to_corrupt)); | |
7c673cae FG |
206 | return; |
207 | } | |
208 | } | |
11fdf7f2 | 209 | FAIL() << "no file found at level"; |
7c673cae FG |
210 | } |
211 | ||
212 | ||
213 | int Property(const std::string& name) { | |
214 | std::string property; | |
215 | int result; | |
216 | if (db_->GetProperty(name, &property) && | |
217 | sscanf(property.c_str(), "%d", &result) == 1) { | |
218 | return result; | |
219 | } else { | |
220 | return -1; | |
221 | } | |
222 | } | |
223 | ||
224 | // Return the ith key | |
225 | Slice Key(int i, std::string* storage) { | |
226 | char buf[100]; | |
227 | snprintf(buf, sizeof(buf), "%016d", i); | |
228 | storage->assign(buf, strlen(buf)); | |
229 | return Slice(*storage); | |
230 | } | |
231 | ||
232 | // Return the value to associate with the specified key | |
233 | Slice Value(int k, std::string* storage) { | |
234 | if (k == 0) { | |
235 | // Ugh. Random seed of 0 used to produce no entropy. This code | |
236 | // preserves the implementation that was in place when all of the | |
237 | // magic values in this file were picked. | |
238 | *storage = std::string(kValueSize, ' '); | |
7c673cae FG |
239 | } else { |
240 | Random r(k); | |
20effc67 | 241 | *storage = r.RandomString(kValueSize); |
7c673cae | 242 | } |
20effc67 | 243 | return Slice(*storage); |
7c673cae FG |
244 | } |
245 | }; | |
246 | ||
247 | TEST_F(CorruptionTest, Recovery) { | |
248 | Build(100); | |
249 | Check(100, 100); | |
250 | #ifdef OS_WIN | |
251 | // On Wndows OS Disk cache does not behave properly | |
252 | // We do not call FlushBuffers on every Flush. If we do not close | |
253 | // the log file prior to the corruption we end up with the first | |
254 | // block not corrupted but only the second. However, under the debugger | |
255 | // things work just fine but never pass when running normally | |
256 | // For that reason people may want to run with unbuffered I/O. That option | |
257 | // is not available for WAL though. | |
258 | CloseDb(); | |
259 | #endif | |
20effc67 TL |
260 | Corrupt(kWalFile, 19, 1); // WriteBatch tag for first record |
261 | Corrupt(kWalFile, log::kBlockSize + 1000, 1); // Somewhere in second block | |
7c673cae FG |
262 | ASSERT_TRUE(!TryReopen().ok()); |
263 | options_.paranoid_checks = false; | |
264 | Reopen(&options_); | |
265 | ||
266 | // The 64 records in the first two log blocks are completely lost. | |
267 | Check(36, 36); | |
268 | } | |
269 | ||
270 | TEST_F(CorruptionTest, RecoverWriteError) { | |
271 | env_.writable_file_error_ = true; | |
272 | Status s = TryReopen(); | |
273 | ASSERT_TRUE(!s.ok()); | |
274 | } | |
275 | ||
276 | TEST_F(CorruptionTest, NewFileErrorDuringWrite) { | |
277 | // Do enough writing to force minor compaction | |
278 | env_.writable_file_error_ = true; | |
279 | const int num = | |
280 | static_cast<int>(3 + (Options().write_buffer_size / kValueSize)); | |
281 | std::string value_storage; | |
282 | Status s; | |
283 | bool failed = false; | |
284 | for (int i = 0; i < num; i++) { | |
285 | WriteBatch batch; | |
20effc67 | 286 | ASSERT_OK(batch.Put("a", Value(100, &value_storage))); |
7c673cae FG |
287 | s = db_->Write(WriteOptions(), &batch); |
288 | if (!s.ok()) { | |
289 | failed = true; | |
290 | } | |
291 | ASSERT_TRUE(!failed || !s.ok()); | |
292 | } | |
293 | ASSERT_TRUE(!s.ok()); | |
294 | ASSERT_GE(env_.num_writable_file_errors_, 1); | |
295 | env_.writable_file_error_ = false; | |
296 | Reopen(); | |
297 | } | |
298 | ||
299 | TEST_F(CorruptionTest, TableFile) { | |
300 | Build(100); | |
20effc67 TL |
301 | DBImpl* dbi = static_cast_with_check<DBImpl>(db_); |
302 | ASSERT_OK(dbi->TEST_FlushMemTable()); | |
303 | ASSERT_OK(dbi->TEST_CompactRange(0, nullptr, nullptr)); | |
304 | ASSERT_OK(dbi->TEST_CompactRange(1, nullptr, nullptr)); | |
7c673cae FG |
305 | |
306 | Corrupt(kTableFile, 100, 1); | |
307 | Check(99, 99); | |
11fdf7f2 | 308 | ASSERT_NOK(dbi->VerifyChecksum()); |
7c673cae FG |
309 | } |
310 | ||
f67539c2 TL |
311 | TEST_F(CorruptionTest, VerifyChecksumReadahead) { |
312 | Options options; | |
313 | SpecialEnv senv(Env::Default()); | |
314 | options.env = &senv; | |
315 | // Disable block cache as we are going to check checksum for | |
316 | // the same file twice and measure number of reads. | |
317 | BlockBasedTableOptions table_options_no_bc; | |
318 | table_options_no_bc.no_block_cache = true; | |
319 | options.table_factory.reset(NewBlockBasedTableFactory(table_options_no_bc)); | |
320 | ||
321 | Reopen(&options); | |
322 | ||
323 | Build(10000); | |
20effc67 TL |
324 | DBImpl* dbi = static_cast_with_check<DBImpl>(db_); |
325 | ASSERT_OK(dbi->TEST_FlushMemTable()); | |
326 | ASSERT_OK(dbi->TEST_CompactRange(0, nullptr, nullptr)); | |
327 | ASSERT_OK(dbi->TEST_CompactRange(1, nullptr, nullptr)); | |
f67539c2 TL |
328 | |
329 | senv.count_random_reads_ = true; | |
330 | senv.random_read_counter_.Reset(); | |
331 | ASSERT_OK(dbi->VerifyChecksum()); | |
332 | ||
333 | // Make sure the counter is enabled. | |
334 | ASSERT_GT(senv.random_read_counter_.Read(), 0); | |
335 | ||
336 | // The SST file is about 10MB. Default readahead size is 256KB. | |
337 | // Give a conservative 20 reads for metadata blocks, The number | |
338 | // of random reads should be within 10 MB / 256KB + 20 = 60. | |
339 | ASSERT_LT(senv.random_read_counter_.Read(), 60); | |
340 | ||
341 | senv.random_read_bytes_counter_ = 0; | |
342 | ReadOptions ro; | |
343 | ro.readahead_size = size_t{32 * 1024}; | |
344 | ASSERT_OK(dbi->VerifyChecksum(ro)); | |
345 | // The SST file is about 10MB. We set readahead size to 32KB. | |
346 | // Give 0 to 20 reads for metadata blocks, and allow real read | |
347 | // to range from 24KB to 48KB. The lower bound would be: | |
348 | // 10MB / 48KB + 0 = 213 | |
349 | // The higher bound is | |
350 | // 10MB / 24KB + 20 = 447. | |
351 | ASSERT_GE(senv.random_read_counter_.Read(), 213); | |
352 | ASSERT_LE(senv.random_read_counter_.Read(), 447); | |
353 | ||
354 | // Test readahead shouldn't break mmap mode (where it should be | |
355 | // disabled). | |
356 | options.allow_mmap_reads = true; | |
357 | Reopen(&options); | |
358 | dbi = static_cast<DBImpl*>(db_); | |
359 | ASSERT_OK(dbi->VerifyChecksum(ro)); | |
360 | ||
361 | CloseDb(); | |
362 | } | |
363 | ||
7c673cae FG |
364 | TEST_F(CorruptionTest, TableFileIndexData) { |
365 | Options options; | |
366 | // very big, we'll trigger flushes manually | |
367 | options.write_buffer_size = 100 * 1024 * 1024; | |
368 | Reopen(&options); | |
369 | // build 2 tables, flush at 5000 | |
370 | Build(10000, 5000); | |
20effc67 TL |
371 | DBImpl* dbi = static_cast_with_check<DBImpl>(db_); |
372 | ASSERT_OK(dbi->TEST_FlushMemTable()); | |
7c673cae FG |
373 | |
374 | // corrupt an index block of an entire file | |
375 | Corrupt(kTableFile, -2000, 500); | |
f67539c2 TL |
376 | options.paranoid_checks = false; |
377 | Reopen(&options); | |
20effc67 | 378 | dbi = static_cast_with_check<DBImpl>(db_); |
11fdf7f2 | 379 | // one full file may be readable, since only one was corrupted |
7c673cae | 380 | // the other file should be fully non-readable, since index was corrupted |
11fdf7f2 TL |
381 | Check(0, 5000); |
382 | ASSERT_NOK(dbi->VerifyChecksum()); | |
f67539c2 TL |
383 | |
384 | // In paranoid mode, the db cannot be opened due to the corrupted file. | |
385 | ASSERT_TRUE(TryReopen().IsCorruption()); | |
7c673cae FG |
386 | } |
387 | ||
388 | TEST_F(CorruptionTest, MissingDescriptor) { | |
389 | Build(1000); | |
390 | RepairDB(); | |
391 | Reopen(); | |
392 | Check(1000, 1000); | |
393 | } | |
394 | ||
395 | TEST_F(CorruptionTest, SequenceNumberRecovery) { | |
396 | ASSERT_OK(db_->Put(WriteOptions(), "foo", "v1")); | |
397 | ASSERT_OK(db_->Put(WriteOptions(), "foo", "v2")); | |
398 | ASSERT_OK(db_->Put(WriteOptions(), "foo", "v3")); | |
399 | ASSERT_OK(db_->Put(WriteOptions(), "foo", "v4")); | |
400 | ASSERT_OK(db_->Put(WriteOptions(), "foo", "v5")); | |
401 | RepairDB(); | |
402 | Reopen(); | |
403 | std::string v; | |
404 | ASSERT_OK(db_->Get(ReadOptions(), "foo", &v)); | |
405 | ASSERT_EQ("v5", v); | |
406 | // Write something. If sequence number was not recovered properly, | |
407 | // it will be hidden by an earlier write. | |
408 | ASSERT_OK(db_->Put(WriteOptions(), "foo", "v6")); | |
409 | ASSERT_OK(db_->Get(ReadOptions(), "foo", &v)); | |
410 | ASSERT_EQ("v6", v); | |
411 | Reopen(); | |
412 | ASSERT_OK(db_->Get(ReadOptions(), "foo", &v)); | |
413 | ASSERT_EQ("v6", v); | |
414 | } | |
415 | ||
416 | TEST_F(CorruptionTest, CorruptedDescriptor) { | |
417 | ASSERT_OK(db_->Put(WriteOptions(), "foo", "hello")); | |
20effc67 TL |
418 | DBImpl* dbi = static_cast_with_check<DBImpl>(db_); |
419 | ASSERT_OK(dbi->TEST_FlushMemTable()); | |
420 | ASSERT_OK(dbi->TEST_CompactRange(0, nullptr, nullptr)); | |
7c673cae FG |
421 | |
422 | Corrupt(kDescriptorFile, 0, 1000); | |
423 | Status s = TryReopen(); | |
424 | ASSERT_TRUE(!s.ok()); | |
425 | ||
426 | RepairDB(); | |
427 | Reopen(); | |
428 | std::string v; | |
429 | ASSERT_OK(db_->Get(ReadOptions(), "foo", &v)); | |
430 | ASSERT_EQ("hello", v); | |
431 | } | |
432 | ||
433 | TEST_F(CorruptionTest, CompactionInputError) { | |
434 | Options options; | |
435 | Reopen(&options); | |
436 | Build(10); | |
20effc67 TL |
437 | DBImpl* dbi = static_cast_with_check<DBImpl>(db_); |
438 | ASSERT_OK(dbi->TEST_FlushMemTable()); | |
439 | ASSERT_OK(dbi->TEST_CompactRange(0, nullptr, nullptr)); | |
440 | ASSERT_OK(dbi->TEST_CompactRange(1, nullptr, nullptr)); | |
7c673cae FG |
441 | ASSERT_EQ(1, Property("rocksdb.num-files-at-level2")); |
442 | ||
443 | Corrupt(kTableFile, 100, 1); | |
444 | Check(9, 9); | |
11fdf7f2 | 445 | ASSERT_NOK(dbi->VerifyChecksum()); |
7c673cae FG |
446 | |
447 | // Force compactions by writing lots of values | |
448 | Build(10000); | |
449 | Check(10000, 10000); | |
11fdf7f2 | 450 | ASSERT_NOK(dbi->VerifyChecksum()); |
7c673cae FG |
451 | } |
452 | ||
453 | TEST_F(CorruptionTest, CompactionInputErrorParanoid) { | |
454 | Options options; | |
455 | options.paranoid_checks = true; | |
456 | options.write_buffer_size = 131072; | |
457 | options.max_write_buffer_number = 2; | |
458 | Reopen(&options); | |
20effc67 | 459 | DBImpl* dbi = static_cast_with_check<DBImpl>(db_); |
7c673cae FG |
460 | |
461 | // Fill levels >= 1 | |
462 | for (int level = 1; level < dbi->NumberLevels(); level++) { | |
20effc67 TL |
463 | ASSERT_OK(dbi->Put(WriteOptions(), "", "begin")); |
464 | ASSERT_OK(dbi->Put(WriteOptions(), "~", "end")); | |
465 | ASSERT_OK(dbi->TEST_FlushMemTable()); | |
7c673cae FG |
466 | for (int comp_level = 0; comp_level < dbi->NumberLevels() - level; |
467 | ++comp_level) { | |
20effc67 | 468 | ASSERT_OK(dbi->TEST_CompactRange(comp_level, nullptr, nullptr)); |
7c673cae FG |
469 | } |
470 | } | |
471 | ||
472 | Reopen(&options); | |
473 | ||
20effc67 | 474 | dbi = static_cast_with_check<DBImpl>(db_); |
7c673cae | 475 | Build(10); |
20effc67 TL |
476 | ASSERT_OK(dbi->TEST_FlushMemTable()); |
477 | ASSERT_OK(dbi->TEST_WaitForCompact()); | |
7c673cae FG |
478 | ASSERT_EQ(1, Property("rocksdb.num-files-at-level0")); |
479 | ||
480 | CorruptTableFileAtLevel(0, 100, 1); | |
481 | Check(9, 9); | |
11fdf7f2 | 482 | ASSERT_NOK(dbi->VerifyChecksum()); |
7c673cae FG |
483 | |
484 | // Write must eventually fail because of corrupted table | |
485 | Status s; | |
486 | std::string tmp1, tmp2; | |
487 | bool failed = false; | |
488 | for (int i = 0; i < 10000; i++) { | |
489 | s = db_->Put(WriteOptions(), Key(i, &tmp1), Value(i, &tmp2)); | |
490 | if (!s.ok()) { | |
491 | failed = true; | |
492 | } | |
493 | // if one write failed, every subsequent write must fail, too | |
494 | ASSERT_TRUE(!failed || !s.ok()) << "write did not fail in a corrupted db"; | |
495 | } | |
496 | ASSERT_TRUE(!s.ok()) << "write did not fail in corrupted paranoid db"; | |
497 | } | |
498 | ||
499 | TEST_F(CorruptionTest, UnrelatedKeys) { | |
500 | Build(10); | |
20effc67 TL |
501 | DBImpl* dbi = static_cast_with_check<DBImpl>(db_); |
502 | ASSERT_OK(dbi->TEST_FlushMemTable()); | |
7c673cae | 503 | Corrupt(kTableFile, 100, 1); |
11fdf7f2 | 504 | ASSERT_NOK(dbi->VerifyChecksum()); |
7c673cae FG |
505 | |
506 | std::string tmp1, tmp2; | |
507 | ASSERT_OK(db_->Put(WriteOptions(), Key(1000, &tmp1), Value(1000, &tmp2))); | |
508 | std::string v; | |
509 | ASSERT_OK(db_->Get(ReadOptions(), Key(1000, &tmp1), &v)); | |
510 | ASSERT_EQ(Value(1000, &tmp2).ToString(), v); | |
20effc67 | 511 | ASSERT_OK(dbi->TEST_FlushMemTable()); |
7c673cae FG |
512 | ASSERT_OK(db_->Get(ReadOptions(), Key(1000, &tmp1), &v)); |
513 | ASSERT_EQ(Value(1000, &tmp2).ToString(), v); | |
514 | } | |
515 | ||
494da23a TL |
516 | TEST_F(CorruptionTest, RangeDeletionCorrupted) { |
517 | ASSERT_OK( | |
518 | db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), "a", "b")); | |
519 | ASSERT_OK(db_->Flush(FlushOptions())); | |
520 | std::vector<LiveFileMetaData> metadata; | |
521 | db_->GetLiveFilesMetaData(&metadata); | |
522 | ASSERT_EQ(static_cast<size_t>(1), metadata.size()); | |
523 | std::string filename = dbname_ + metadata[0].name; | |
524 | ||
525 | std::unique_ptr<RandomAccessFile> file; | |
526 | ASSERT_OK(options_.env->NewRandomAccessFile(filename, &file, EnvOptions())); | |
527 | std::unique_ptr<RandomAccessFileReader> file_reader( | |
f67539c2 TL |
528 | new RandomAccessFileReader(NewLegacyRandomAccessFileWrapper(file), |
529 | filename)); | |
494da23a TL |
530 | |
531 | uint64_t file_size; | |
532 | ASSERT_OK(options_.env->GetFileSize(filename, &file_size)); | |
533 | ||
534 | BlockHandle range_del_handle; | |
535 | ASSERT_OK(FindMetaBlock( | |
536 | file_reader.get(), file_size, kBlockBasedTableMagicNumber, | |
537 | ImmutableCFOptions(options_), kRangeDelBlock, &range_del_handle)); | |
538 | ||
539 | ASSERT_OK(TryReopen()); | |
20effc67 TL |
540 | ASSERT_OK(test::CorruptFile(&env_, filename, |
541 | static_cast<int>(range_del_handle.offset()), 1)); | |
f67539c2 | 542 | ASSERT_TRUE(TryReopen().IsCorruption()); |
494da23a TL |
543 | } |
544 | ||
7c673cae FG |
545 | TEST_F(CorruptionTest, FileSystemStateCorrupted) { |
546 | for (int iter = 0; iter < 2; ++iter) { | |
547 | Options options; | |
548 | options.paranoid_checks = true; | |
549 | options.create_if_missing = true; | |
550 | Reopen(&options); | |
551 | Build(10); | |
552 | ASSERT_OK(db_->Flush(FlushOptions())); | |
20effc67 | 553 | DBImpl* dbi = static_cast_with_check<DBImpl>(db_); |
7c673cae FG |
554 | std::vector<LiveFileMetaData> metadata; |
555 | dbi->GetLiveFilesMetaData(&metadata); | |
20effc67 | 556 | ASSERT_GT(metadata.size(), 0); |
7c673cae FG |
557 | std::string filename = dbname_ + metadata[0].name; |
558 | ||
559 | delete db_; | |
560 | db_ = nullptr; | |
561 | ||
562 | if (iter == 0) { // corrupt file size | |
494da23a | 563 | std::unique_ptr<WritableFile> file; |
7c673cae | 564 | env_.NewWritableFile(filename, &file, EnvOptions()); |
20effc67 | 565 | ASSERT_OK(file->Append(Slice("corrupted sst"))); |
7c673cae | 566 | file.reset(); |
f67539c2 TL |
567 | Status x = TryReopen(&options); |
568 | ASSERT_TRUE(x.IsCorruption()); | |
7c673cae | 569 | } else { // delete the file |
20effc67 | 570 | ASSERT_OK(env_.DeleteFile(filename)); |
f67539c2 | 571 | Status x = TryReopen(&options); |
20effc67 TL |
572 | ASSERT_TRUE(x.IsCorruption()); |
573 | } | |
574 | ||
575 | ASSERT_OK(DestroyDB(dbname_, options_)); | |
576 | } | |
577 | } | |
578 | ||
579 | static const auto& corruption_modes = { | |
580 | mock::MockTableFactory::kCorruptNone, mock::MockTableFactory::kCorruptKey, | |
581 | mock::MockTableFactory::kCorruptValue, | |
582 | mock::MockTableFactory::kCorruptReorderKey}; | |
583 | ||
584 | TEST_F(CorruptionTest, ParanoidFileChecksOnFlush) { | |
585 | Options options; | |
586 | options.check_flush_compaction_key_order = false; | |
587 | options.paranoid_file_checks = true; | |
588 | options.create_if_missing = true; | |
589 | Status s; | |
590 | for (const auto& mode : corruption_modes) { | |
591 | delete db_; | |
592 | db_ = nullptr; | |
593 | s = DestroyDB(dbname_, options); | |
594 | ASSERT_OK(s); | |
595 | std::shared_ptr<mock::MockTableFactory> mock = | |
596 | std::make_shared<mock::MockTableFactory>(); | |
597 | options.table_factory = mock; | |
598 | mock->SetCorruptionMode(mode); | |
599 | ASSERT_OK(DB::Open(options, dbname_, &db_)); | |
600 | assert(db_ != nullptr); | |
601 | Build(10); | |
602 | s = db_->Flush(FlushOptions()); | |
603 | if (mode == mock::MockTableFactory::kCorruptNone) { | |
604 | ASSERT_OK(s); | |
605 | } else { | |
606 | ASSERT_NOK(s); | |
607 | } | |
608 | } | |
609 | } | |
610 | ||
611 | TEST_F(CorruptionTest, ParanoidFileChecksOnCompact) { | |
612 | Options options; | |
613 | options.paranoid_file_checks = true; | |
614 | options.create_if_missing = true; | |
615 | options.check_flush_compaction_key_order = false; | |
616 | Status s; | |
617 | for (const auto& mode : corruption_modes) { | |
618 | delete db_; | |
619 | db_ = nullptr; | |
620 | s = DestroyDB(dbname_, options); | |
621 | std::shared_ptr<mock::MockTableFactory> mock = | |
622 | std::make_shared<mock::MockTableFactory>(); | |
623 | options.table_factory = mock; | |
624 | ASSERT_OK(DB::Open(options, dbname_, &db_)); | |
625 | assert(db_ != nullptr); | |
626 | Build(100, 2); | |
627 | // ASSERT_OK(db_->Flush(FlushOptions())); | |
628 | DBImpl* dbi = static_cast_with_check<DBImpl>(db_); | |
629 | ASSERT_OK(dbi->TEST_FlushMemTable()); | |
630 | mock->SetCorruptionMode(mode); | |
631 | s = dbi->TEST_CompactRange(0, nullptr, nullptr, nullptr, true); | |
632 | if (mode == mock::MockTableFactory::kCorruptNone) { | |
633 | ASSERT_OK(s); | |
634 | } else { | |
635 | ASSERT_NOK(s); | |
636 | } | |
637 | } | |
638 | } | |
639 | ||
640 | TEST_F(CorruptionTest, ParanoidFileChecksWithDeleteRangeFirst) { | |
641 | Options options; | |
642 | options.check_flush_compaction_key_order = false; | |
643 | options.paranoid_file_checks = true; | |
644 | options.create_if_missing = true; | |
645 | for (bool do_flush : {true, false}) { | |
646 | delete db_; | |
647 | db_ = nullptr; | |
648 | ASSERT_OK(DestroyDB(dbname_, options)); | |
649 | ASSERT_OK(DB::Open(options, dbname_, &db_)); | |
650 | std::string start, end; | |
651 | assert(db_ != nullptr); | |
652 | ASSERT_OK(db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), | |
653 | Key(3, &start), Key(7, &end))); | |
654 | auto snap = db_->GetSnapshot(); | |
655 | ASSERT_NE(snap, nullptr); | |
656 | ASSERT_OK(db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), | |
657 | Key(8, &start), Key(9, &end))); | |
658 | ASSERT_OK(db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), | |
659 | Key(2, &start), Key(5, &end))); | |
660 | Build(10); | |
661 | if (do_flush) { | |
662 | ASSERT_OK(db_->Flush(FlushOptions())); | |
663 | } else { | |
664 | DBImpl* dbi = static_cast_with_check<DBImpl>(db_); | |
665 | ASSERT_OK(dbi->TEST_FlushMemTable()); | |
666 | ASSERT_OK(dbi->TEST_CompactRange(0, nullptr, nullptr, nullptr, true)); | |
7c673cae | 667 | } |
20effc67 TL |
668 | db_->ReleaseSnapshot(snap); |
669 | } | |
670 | } | |
7c673cae | 671 | |
20effc67 TL |
672 | TEST_F(CorruptionTest, ParanoidFileChecksWithDeleteRange) { |
673 | Options options; | |
674 | options.check_flush_compaction_key_order = false; | |
675 | options.paranoid_file_checks = true; | |
676 | options.create_if_missing = true; | |
677 | for (bool do_flush : {true, false}) { | |
678 | delete db_; | |
679 | db_ = nullptr; | |
680 | ASSERT_OK(DestroyDB(dbname_, options)); | |
681 | ASSERT_OK(DB::Open(options, dbname_, &db_)); | |
682 | assert(db_ != nullptr); | |
683 | Build(10, 0, 0); | |
684 | std::string start, end; | |
685 | ASSERT_OK(db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), | |
686 | Key(5, &start), Key(15, &end))); | |
687 | auto snap = db_->GetSnapshot(); | |
688 | ASSERT_NE(snap, nullptr); | |
689 | ASSERT_OK(db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), | |
690 | Key(8, &start), Key(9, &end))); | |
691 | ASSERT_OK(db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), | |
692 | Key(12, &start), Key(17, &end))); | |
693 | ASSERT_OK(db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), | |
694 | Key(2, &start), Key(4, &end))); | |
695 | Build(10, 10, 0); | |
696 | if (do_flush) { | |
697 | ASSERT_OK(db_->Flush(FlushOptions())); | |
698 | } else { | |
699 | DBImpl* dbi = static_cast_with_check<DBImpl>(db_); | |
700 | ASSERT_OK(dbi->TEST_FlushMemTable()); | |
701 | ASSERT_OK(dbi->TEST_CompactRange(0, nullptr, nullptr, nullptr, true)); | |
702 | } | |
703 | db_->ReleaseSnapshot(snap); | |
7c673cae FG |
704 | } |
705 | } | |
706 | ||
20effc67 TL |
707 | TEST_F(CorruptionTest, ParanoidFileChecksWithDeleteRangeLast) { |
708 | Options options; | |
709 | options.check_flush_compaction_key_order = false; | |
710 | options.paranoid_file_checks = true; | |
711 | options.create_if_missing = true; | |
712 | for (bool do_flush : {true, false}) { | |
713 | delete db_; | |
714 | db_ = nullptr; | |
715 | ASSERT_OK(DestroyDB(dbname_, options)); | |
716 | ASSERT_OK(DB::Open(options, dbname_, &db_)); | |
717 | assert(db_ != nullptr); | |
718 | std::string start, end; | |
719 | Build(10); | |
720 | ASSERT_OK(db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), | |
721 | Key(3, &start), Key(7, &end))); | |
722 | auto snap = db_->GetSnapshot(); | |
723 | ASSERT_NE(snap, nullptr); | |
724 | ASSERT_OK(db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), | |
725 | Key(6, &start), Key(8, &end))); | |
726 | ASSERT_OK(db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), | |
727 | Key(2, &start), Key(5, &end))); | |
728 | if (do_flush) { | |
729 | ASSERT_OK(db_->Flush(FlushOptions())); | |
730 | } else { | |
731 | DBImpl* dbi = static_cast_with_check<DBImpl>(db_); | |
732 | ASSERT_OK(dbi->TEST_FlushMemTable()); | |
733 | ASSERT_OK(dbi->TEST_CompactRange(0, nullptr, nullptr, nullptr, true)); | |
734 | } | |
735 | db_->ReleaseSnapshot(snap); | |
736 | } | |
737 | } | |
738 | ||
739 | TEST_F(CorruptionTest, LogCorruptionErrorsInCompactionIterator) { | |
740 | Options options; | |
741 | options.create_if_missing = true; | |
742 | options.allow_data_in_errors = true; | |
743 | auto mode = mock::MockTableFactory::kCorruptKey; | |
744 | delete db_; | |
745 | db_ = nullptr; | |
746 | ASSERT_OK(DestroyDB(dbname_, options)); | |
747 | ||
748 | std::shared_ptr<mock::MockTableFactory> mock = | |
749 | std::make_shared<mock::MockTableFactory>(); | |
750 | mock->SetCorruptionMode(mode); | |
751 | options.table_factory = mock; | |
752 | ||
753 | ASSERT_OK(DB::Open(options, dbname_, &db_)); | |
754 | assert(db_ != nullptr); | |
755 | Build(100, 2); | |
756 | ||
757 | DBImpl* dbi = static_cast_with_check<DBImpl>(db_); | |
758 | ASSERT_OK(dbi->TEST_FlushMemTable()); | |
759 | Status s = dbi->TEST_CompactRange(0, nullptr, nullptr, nullptr, true); | |
760 | ASSERT_NOK(s); | |
761 | ASSERT_TRUE(s.IsCorruption()); | |
762 | } | |
763 | ||
764 | TEST_F(CorruptionTest, CompactionKeyOrderCheck) { | |
765 | Options options; | |
766 | options.paranoid_file_checks = false; | |
767 | options.create_if_missing = true; | |
768 | options.check_flush_compaction_key_order = false; | |
769 | delete db_; | |
770 | db_ = nullptr; | |
771 | ASSERT_OK(DestroyDB(dbname_, options)); | |
772 | std::shared_ptr<mock::MockTableFactory> mock = | |
773 | std::make_shared<mock::MockTableFactory>(); | |
774 | options.table_factory = mock; | |
775 | ASSERT_OK(DB::Open(options, dbname_, &db_)); | |
776 | assert(db_ != nullptr); | |
777 | mock->SetCorruptionMode(mock::MockTableFactory::kCorruptReorderKey); | |
778 | Build(100, 2); | |
779 | DBImpl* dbi = static_cast_with_check<DBImpl>(db_); | |
780 | ASSERT_OK(dbi->TEST_FlushMemTable()); | |
781 | ||
782 | mock->SetCorruptionMode(mock::MockTableFactory::kCorruptNone); | |
783 | ASSERT_OK(db_->SetOptions({{"check_flush_compaction_key_order", "true"}})); | |
784 | ASSERT_NOK(dbi->TEST_CompactRange(0, nullptr, nullptr, nullptr, true)); | |
785 | } | |
786 | ||
787 | TEST_F(CorruptionTest, FlushKeyOrderCheck) { | |
788 | Options options; | |
789 | options.paranoid_file_checks = false; | |
790 | options.create_if_missing = true; | |
791 | ASSERT_OK(db_->SetOptions({{"check_flush_compaction_key_order", "true"}})); | |
792 | ||
793 | ASSERT_OK(db_->Put(WriteOptions(), "foo1", "v1")); | |
794 | ASSERT_OK(db_->Put(WriteOptions(), "foo2", "v1")); | |
795 | ASSERT_OK(db_->Put(WriteOptions(), "foo3", "v1")); | |
796 | ASSERT_OK(db_->Put(WriteOptions(), "foo4", "v1")); | |
797 | ||
798 | int cnt = 0; | |
799 | // Generate some out of order keys from the memtable | |
800 | SyncPoint::GetInstance()->SetCallBack( | |
801 | "MemTableIterator::Next:0", [&](void* arg) { | |
802 | MemTableRep::Iterator* mem_iter = | |
803 | static_cast<MemTableRep::Iterator*>(arg); | |
804 | if (++cnt == 3) { | |
805 | mem_iter->Prev(); | |
806 | mem_iter->Prev(); | |
807 | } | |
808 | }); | |
809 | ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); | |
810 | Status s = static_cast_with_check<DBImpl>(db_)->TEST_FlushMemTable(); | |
811 | ASSERT_NOK(s); | |
812 | ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); | |
813 | ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->ClearAllCallBacks(); | |
814 | } | |
815 | ||
816 | TEST_F(CorruptionTest, DisableKeyOrderCheck) { | |
817 | Options options; | |
818 | ASSERT_OK(db_->SetOptions({{"check_flush_compaction_key_order", "false"}})); | |
819 | DBImpl* dbi = static_cast_with_check<DBImpl>(db_); | |
820 | ||
821 | SyncPoint::GetInstance()->SetCallBack( | |
822 | "OutputValidator::Add:order_check", | |
823 | [&](void* /*arg*/) { ASSERT_TRUE(false); }); | |
824 | ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); | |
825 | ASSERT_OK(db_->Put(WriteOptions(), "foo1", "v1")); | |
826 | ASSERT_OK(db_->Put(WriteOptions(), "foo3", "v1")); | |
827 | ASSERT_OK(dbi->TEST_FlushMemTable()); | |
828 | ASSERT_OK(db_->Put(WriteOptions(), "foo2", "v1")); | |
829 | ASSERT_OK(db_->Put(WriteOptions(), "foo4", "v1")); | |
830 | ASSERT_OK(dbi->TEST_FlushMemTable()); | |
831 | ASSERT_OK(dbi->TEST_CompactRange(0, nullptr, nullptr, nullptr, true)); | |
832 | ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); | |
833 | ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->ClearAllCallBacks(); | |
834 | } | |
835 | ||
836 | TEST_F(CorruptionTest, VerifyWholeTableChecksum) { | |
837 | CloseDb(); | |
838 | Options options; | |
839 | options.env = &env_; | |
840 | ASSERT_OK(DestroyDB(dbname_, options)); | |
841 | options.create_if_missing = true; | |
842 | options.file_checksum_gen_factory = | |
843 | ROCKSDB_NAMESPACE::GetFileChecksumGenCrc32cFactory(); | |
844 | Reopen(&options); | |
845 | ||
846 | Build(10, 5); | |
847 | ||
848 | ASSERT_OK(db_->VerifyFileChecksums(ReadOptions())); | |
849 | CloseDb(); | |
850 | ||
851 | // Corrupt the first byte of each table file, this must be data block. | |
852 | Corrupt(kTableFile, 0, 1); | |
853 | ||
854 | ASSERT_OK(TryReopen(&options)); | |
855 | ||
856 | SyncPoint::GetInstance()->DisableProcessing(); | |
857 | SyncPoint::GetInstance()->ClearAllCallBacks(); | |
858 | int count{0}; | |
859 | SyncPoint::GetInstance()->SetCallBack( | |
860 | "DBImpl::VerifySstFileChecksum:mismatch", [&](void* arg) { | |
861 | auto* s = reinterpret_cast<Status*>(arg); | |
862 | assert(s); | |
863 | ++count; | |
864 | ASSERT_NOK(*s); | |
865 | }); | |
866 | SyncPoint::GetInstance()->EnableProcessing(); | |
867 | ASSERT_TRUE(db_->VerifyFileChecksums(ReadOptions()).IsCorruption()); | |
868 | ASSERT_EQ(1, count); | |
869 | } | |
870 | ||
f67539c2 | 871 | } // namespace ROCKSDB_NAMESPACE |
7c673cae FG |
872 | |
873 | int main(int argc, char** argv) { | |
874 | ::testing::InitGoogleTest(&argc, argv); | |
875 | return RUN_ALL_TESTS(); | |
876 | } | |
877 | ||
878 | #else | |
879 | #include <stdio.h> | |
880 | ||
11fdf7f2 | 881 | int main(int /*argc*/, char** /*argv*/) { |
7c673cae FG |
882 | fprintf(stderr, "SKIPPED as RepairDB() is not supported in ROCKSDB_LITE\n"); |
883 | return 0; | |
884 | } | |
885 | ||
886 | #endif // !ROCKSDB_LITE |