1 // Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
2 // This source code is licensed under both the GPLv2 (found in the
3 // COPYING file in the root directory) and Apache 2.0 License
4 // (found in the LICENSE.Apache file in the root directory).
6 // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
7 // Use of this source code is governed by a BSD-style license that can be
8 // found in the LICENSE file. See the AUTHORS file for names of contributors.
14 #include <sys/types.h>
18 #include "db/db_impl/db_impl.h"
19 #include "db/db_test_util.h"
20 #include "db/log_format.h"
21 #include "db/version_set.h"
22 #include "env/composite_env_wrapper.h"
23 #include "file/filename.h"
24 #include "rocksdb/cache.h"
25 #include "rocksdb/convenience.h"
26 #include "rocksdb/db.h"
27 #include "rocksdb/env.h"
28 #include "rocksdb/table.h"
29 #include "rocksdb/write_batch.h"
30 #include "table/block_based/block_based_table_builder.h"
31 #include "table/meta_blocks.h"
32 #include "table/mock_table.h"
33 #include "test_util/testharness.h"
34 #include "test_util/testutil.h"
35 #include "util/cast_util.h"
36 #include "util/random.h"
37 #include "util/string_util.h"
39 namespace ROCKSDB_NAMESPACE
{
41 static constexpr int kValueSize
= 1000;
43 class CorruptionTest
: public testing::Test
{
47 std::shared_ptr
<Cache
> tiny_cache_
;
52 // If LRU cache shard bit is smaller than 2 (or -1 which will automatically
53 // set it to 0), test SequenceNumberRecovery will fail, likely because of a
54 // bug in recovery code. Keep it 4 for now to make the test passes.
55 tiny_cache_
= NewLRUCache(100, 4);
56 options_
.wal_recovery_mode
= WALRecoveryMode::kTolerateCorruptedTailRecords
;
58 dbname_
= test::PerThreadDBPath("corruption_test");
59 Status s
= DestroyDB(dbname_
, options_
);
63 options_
.create_if_missing
= true;
64 BlockBasedTableOptions table_options
;
65 table_options
.block_size_deviation
= 0; // make unit test pass for now
66 options_
.table_factory
.reset(NewBlockBasedTableFactory(table_options
));
68 options_
.create_if_missing
= false;
71 ~CorruptionTest() override
{
72 SyncPoint::GetInstance()->DisableProcessing();
73 SyncPoint::GetInstance()->LoadDependency({});
74 SyncPoint::GetInstance()->ClearAllCallBacks();
77 if (getenv("KEEP_DB")) {
78 fprintf(stdout
, "db is still at %s\n", dbname_
.c_str());
80 EXPECT_OK(DestroyDB(dbname_
, Options()));
89 Status
TryReopen(Options
* options
= nullptr) {
92 Options opt
= (options
? *options
: options_
);
93 if (opt
.env
== Options().env
) {
94 // If env is not overridden, replace it with ErrorEnv.
95 // Otherwise, the test already uses a non-default Env.
98 opt
.arena_block_size
= 4096;
99 BlockBasedTableOptions table_options
;
100 table_options
.block_cache
= tiny_cache_
;
101 table_options
.block_size_deviation
= 0;
102 opt
.table_factory
.reset(NewBlockBasedTableFactory(table_options
));
103 return DB::Open(opt
, dbname_
, &db_
);
106 void Reopen(Options
* options
= nullptr) {
107 ASSERT_OK(TryReopen(options
));
113 ASSERT_OK(::ROCKSDB_NAMESPACE::RepairDB(dbname_
, options_
));
116 void Build(int n
, int start
, int flush_every
) {
117 std::string key_space
, value_space
;
119 for (int i
= 0; i
< n
; i
++) {
120 if (flush_every
!= 0 && i
!= 0 && i
% flush_every
== 0) {
121 DBImpl
* dbi
= static_cast_with_check
<DBImpl
>(db_
);
122 ASSERT_OK(dbi
->TEST_FlushMemTable());
124 //if ((i % 100) == 0) fprintf(stderr, "@ %d of %d\n", i, n);
125 Slice key
= Key(i
+ start
, &key_space
);
127 ASSERT_OK(batch
.Put(key
, Value(i
+ start
, &value_space
)));
128 ASSERT_OK(db_
->Write(WriteOptions(), &batch
));
132 void Build(int n
, int flush_every
= 0) { Build(n
, 0, flush_every
); }
134 void Check(int min_expected
, int max_expected
) {
135 uint64_t next_expected
= 0;
140 std::string value_space
;
141 // Do not verify checksums. If we verify checksums then the
142 // db itself will raise errors because data is corrupted.
143 // Instead, we want the reads to be successful and this test
144 // will detect whether the appropriate corruptions have
146 Iterator
* iter
= db_
->NewIterator(ReadOptions(false, true));
147 for (iter
->SeekToFirst(); iter
->Valid(); iter
->Next()) {
148 ASSERT_OK(iter
->status());
150 Slice
in(iter
->key());
151 if (!ConsumeDecimalNumber(&in
, &key
) ||
153 key
< next_expected
) {
157 missed
+= (key
- next_expected
);
158 next_expected
= key
+ 1;
159 if (iter
->value() != Value(static_cast<int>(key
), &value_space
)) {
165 iter
->status().PermitUncheckedError();
169 "expected=%d..%d; got=%d; bad_keys=%d; bad_values=%d; missed=%llu\n",
170 min_expected
, max_expected
, correct
, bad_keys
, bad_values
,
171 static_cast<unsigned long long>(missed
));
172 ASSERT_LE(min_expected
, correct
);
173 ASSERT_GE(max_expected
, correct
);
176 void Corrupt(FileType filetype
, int offset
, int bytes_to_corrupt
) {
177 // Pick file to corrupt
178 std::vector
<std::string
> filenames
;
179 ASSERT_OK(env_
.GetChildren(dbname_
, &filenames
));
183 int picked_number
= -1;
184 for (size_t i
= 0; i
< filenames
.size(); i
++) {
185 if (ParseFileName(filenames
[i
], &number
, &type
) &&
187 static_cast<int>(number
) > picked_number
) { // Pick latest file
188 fname
= dbname_
+ "/" + filenames
[i
];
189 picked_number
= static_cast<int>(number
);
192 ASSERT_TRUE(!fname
.empty()) << filetype
;
194 ASSERT_OK(test::CorruptFile(&env_
, fname
, offset
, bytes_to_corrupt
));
197 // corrupts exactly one file at level `level`. if no file found at level,
199 void CorruptTableFileAtLevel(int level
, int offset
, int bytes_to_corrupt
) {
200 std::vector
<LiveFileMetaData
> metadata
;
201 db_
->GetLiveFilesMetaData(&metadata
);
202 for (const auto& m
: metadata
) {
203 if (m
.level
== level
) {
204 ASSERT_OK(test::CorruptFile(&env_
, dbname_
+ "/" + m
.name
, offset
,
209 FAIL() << "no file found at level";
213 int Property(const std::string
& name
) {
214 std::string property
;
216 if (db_
->GetProperty(name
, &property
) &&
217 sscanf(property
.c_str(), "%d", &result
) == 1) {
224 // Return the ith key
225 Slice
Key(int i
, std::string
* storage
) {
227 snprintf(buf
, sizeof(buf
), "%016d", i
);
228 storage
->assign(buf
, strlen(buf
));
229 return Slice(*storage
);
232 // Return the value to associate with the specified key
233 Slice
Value(int k
, std::string
* storage
) {
235 // Ugh. Random seed of 0 used to produce no entropy. This code
236 // preserves the implementation that was in place when all of the
237 // magic values in this file were picked.
238 *storage
= std::string(kValueSize
, ' ');
241 *storage
= r
.RandomString(kValueSize
);
243 return Slice(*storage
);
247 TEST_F(CorruptionTest
, Recovery
) {
251 // On Wndows OS Disk cache does not behave properly
252 // We do not call FlushBuffers on every Flush. If we do not close
253 // the log file prior to the corruption we end up with the first
254 // block not corrupted but only the second. However, under the debugger
255 // things work just fine but never pass when running normally
256 // For that reason people may want to run with unbuffered I/O. That option
257 // is not available for WAL though.
260 Corrupt(kWalFile
, 19, 1); // WriteBatch tag for first record
261 Corrupt(kWalFile
, log::kBlockSize
+ 1000, 1); // Somewhere in second block
262 ASSERT_TRUE(!TryReopen().ok());
263 options_
.paranoid_checks
= false;
266 // The 64 records in the first two log blocks are completely lost.
270 TEST_F(CorruptionTest
, RecoverWriteError
) {
271 env_
.writable_file_error_
= true;
272 Status s
= TryReopen();
273 ASSERT_TRUE(!s
.ok());
276 TEST_F(CorruptionTest
, NewFileErrorDuringWrite
) {
277 // Do enough writing to force minor compaction
278 env_
.writable_file_error_
= true;
280 static_cast<int>(3 + (Options().write_buffer_size
/ kValueSize
));
281 std::string value_storage
;
284 for (int i
= 0; i
< num
; i
++) {
286 ASSERT_OK(batch
.Put("a", Value(100, &value_storage
)));
287 s
= db_
->Write(WriteOptions(), &batch
);
291 ASSERT_TRUE(!failed
|| !s
.ok());
293 ASSERT_TRUE(!s
.ok());
294 ASSERT_GE(env_
.num_writable_file_errors_
, 1);
295 env_
.writable_file_error_
= false;
299 TEST_F(CorruptionTest
, TableFile
) {
301 DBImpl
* dbi
= static_cast_with_check
<DBImpl
>(db_
);
302 ASSERT_OK(dbi
->TEST_FlushMemTable());
303 ASSERT_OK(dbi
->TEST_CompactRange(0, nullptr, nullptr));
304 ASSERT_OK(dbi
->TEST_CompactRange(1, nullptr, nullptr));
306 Corrupt(kTableFile
, 100, 1);
308 ASSERT_NOK(dbi
->VerifyChecksum());
311 TEST_F(CorruptionTest
, VerifyChecksumReadahead
) {
313 SpecialEnv
senv(Env::Default());
315 // Disable block cache as we are going to check checksum for
316 // the same file twice and measure number of reads.
317 BlockBasedTableOptions table_options_no_bc
;
318 table_options_no_bc
.no_block_cache
= true;
319 options
.table_factory
.reset(NewBlockBasedTableFactory(table_options_no_bc
));
324 DBImpl
* dbi
= static_cast_with_check
<DBImpl
>(db_
);
325 ASSERT_OK(dbi
->TEST_FlushMemTable());
326 ASSERT_OK(dbi
->TEST_CompactRange(0, nullptr, nullptr));
327 ASSERT_OK(dbi
->TEST_CompactRange(1, nullptr, nullptr));
329 senv
.count_random_reads_
= true;
330 senv
.random_read_counter_
.Reset();
331 ASSERT_OK(dbi
->VerifyChecksum());
333 // Make sure the counter is enabled.
334 ASSERT_GT(senv
.random_read_counter_
.Read(), 0);
336 // The SST file is about 10MB. Default readahead size is 256KB.
337 // Give a conservative 20 reads for metadata blocks, The number
338 // of random reads should be within 10 MB / 256KB + 20 = 60.
339 ASSERT_LT(senv
.random_read_counter_
.Read(), 60);
341 senv
.random_read_bytes_counter_
= 0;
343 ro
.readahead_size
= size_t{32 * 1024};
344 ASSERT_OK(dbi
->VerifyChecksum(ro
));
345 // The SST file is about 10MB. We set readahead size to 32KB.
346 // Give 0 to 20 reads for metadata blocks, and allow real read
347 // to range from 24KB to 48KB. The lower bound would be:
348 // 10MB / 48KB + 0 = 213
349 // The higher bound is
350 // 10MB / 24KB + 20 = 447.
351 ASSERT_GE(senv
.random_read_counter_
.Read(), 213);
352 ASSERT_LE(senv
.random_read_counter_
.Read(), 447);
354 // Test readahead shouldn't break mmap mode (where it should be
356 options
.allow_mmap_reads
= true;
358 dbi
= static_cast<DBImpl
*>(db_
);
359 ASSERT_OK(dbi
->VerifyChecksum(ro
));
364 TEST_F(CorruptionTest
, TableFileIndexData
) {
366 // very big, we'll trigger flushes manually
367 options
.write_buffer_size
= 100 * 1024 * 1024;
369 // build 2 tables, flush at 5000
371 DBImpl
* dbi
= static_cast_with_check
<DBImpl
>(db_
);
372 ASSERT_OK(dbi
->TEST_FlushMemTable());
374 // corrupt an index block of an entire file
375 Corrupt(kTableFile
, -2000, 500);
376 options
.paranoid_checks
= false;
378 dbi
= static_cast_with_check
<DBImpl
>(db_
);
379 // one full file may be readable, since only one was corrupted
380 // the other file should be fully non-readable, since index was corrupted
382 ASSERT_NOK(dbi
->VerifyChecksum());
384 // In paranoid mode, the db cannot be opened due to the corrupted file.
385 ASSERT_TRUE(TryReopen().IsCorruption());
388 TEST_F(CorruptionTest
, MissingDescriptor
) {
395 TEST_F(CorruptionTest
, SequenceNumberRecovery
) {
396 ASSERT_OK(db_
->Put(WriteOptions(), "foo", "v1"));
397 ASSERT_OK(db_
->Put(WriteOptions(), "foo", "v2"));
398 ASSERT_OK(db_
->Put(WriteOptions(), "foo", "v3"));
399 ASSERT_OK(db_
->Put(WriteOptions(), "foo", "v4"));
400 ASSERT_OK(db_
->Put(WriteOptions(), "foo", "v5"));
404 ASSERT_OK(db_
->Get(ReadOptions(), "foo", &v
));
406 // Write something. If sequence number was not recovered properly,
407 // it will be hidden by an earlier write.
408 ASSERT_OK(db_
->Put(WriteOptions(), "foo", "v6"));
409 ASSERT_OK(db_
->Get(ReadOptions(), "foo", &v
));
412 ASSERT_OK(db_
->Get(ReadOptions(), "foo", &v
));
416 TEST_F(CorruptionTest
, CorruptedDescriptor
) {
417 ASSERT_OK(db_
->Put(WriteOptions(), "foo", "hello"));
418 DBImpl
* dbi
= static_cast_with_check
<DBImpl
>(db_
);
419 ASSERT_OK(dbi
->TEST_FlushMemTable());
420 ASSERT_OK(dbi
->TEST_CompactRange(0, nullptr, nullptr));
422 Corrupt(kDescriptorFile
, 0, 1000);
423 Status s
= TryReopen();
424 ASSERT_TRUE(!s
.ok());
429 ASSERT_OK(db_
->Get(ReadOptions(), "foo", &v
));
430 ASSERT_EQ("hello", v
);
433 TEST_F(CorruptionTest
, CompactionInputError
) {
437 DBImpl
* dbi
= static_cast_with_check
<DBImpl
>(db_
);
438 ASSERT_OK(dbi
->TEST_FlushMemTable());
439 ASSERT_OK(dbi
->TEST_CompactRange(0, nullptr, nullptr));
440 ASSERT_OK(dbi
->TEST_CompactRange(1, nullptr, nullptr));
441 ASSERT_EQ(1, Property("rocksdb.num-files-at-level2"));
443 Corrupt(kTableFile
, 100, 1);
445 ASSERT_NOK(dbi
->VerifyChecksum());
447 // Force compactions by writing lots of values
450 ASSERT_NOK(dbi
->VerifyChecksum());
453 TEST_F(CorruptionTest
, CompactionInputErrorParanoid
) {
455 options
.paranoid_checks
= true;
456 options
.write_buffer_size
= 131072;
457 options
.max_write_buffer_number
= 2;
459 DBImpl
* dbi
= static_cast_with_check
<DBImpl
>(db_
);
462 for (int level
= 1; level
< dbi
->NumberLevels(); level
++) {
463 ASSERT_OK(dbi
->Put(WriteOptions(), "", "begin"));
464 ASSERT_OK(dbi
->Put(WriteOptions(), "~", "end"));
465 ASSERT_OK(dbi
->TEST_FlushMemTable());
466 for (int comp_level
= 0; comp_level
< dbi
->NumberLevels() - level
;
468 ASSERT_OK(dbi
->TEST_CompactRange(comp_level
, nullptr, nullptr));
474 dbi
= static_cast_with_check
<DBImpl
>(db_
);
476 ASSERT_OK(dbi
->TEST_FlushMemTable());
477 ASSERT_OK(dbi
->TEST_WaitForCompact());
478 ASSERT_EQ(1, Property("rocksdb.num-files-at-level0"));
480 CorruptTableFileAtLevel(0, 100, 1);
482 ASSERT_NOK(dbi
->VerifyChecksum());
484 // Write must eventually fail because of corrupted table
486 std::string tmp1
, tmp2
;
488 for (int i
= 0; i
< 10000; i
++) {
489 s
= db_
->Put(WriteOptions(), Key(i
, &tmp1
), Value(i
, &tmp2
));
493 // if one write failed, every subsequent write must fail, too
494 ASSERT_TRUE(!failed
|| !s
.ok()) << "write did not fail in a corrupted db";
496 ASSERT_TRUE(!s
.ok()) << "write did not fail in corrupted paranoid db";
499 TEST_F(CorruptionTest
, UnrelatedKeys
) {
501 DBImpl
* dbi
= static_cast_with_check
<DBImpl
>(db_
);
502 ASSERT_OK(dbi
->TEST_FlushMemTable());
503 Corrupt(kTableFile
, 100, 1);
504 ASSERT_NOK(dbi
->VerifyChecksum());
506 std::string tmp1
, tmp2
;
507 ASSERT_OK(db_
->Put(WriteOptions(), Key(1000, &tmp1
), Value(1000, &tmp2
)));
509 ASSERT_OK(db_
->Get(ReadOptions(), Key(1000, &tmp1
), &v
));
510 ASSERT_EQ(Value(1000, &tmp2
).ToString(), v
);
511 ASSERT_OK(dbi
->TEST_FlushMemTable());
512 ASSERT_OK(db_
->Get(ReadOptions(), Key(1000, &tmp1
), &v
));
513 ASSERT_EQ(Value(1000, &tmp2
).ToString(), v
);
516 TEST_F(CorruptionTest
, RangeDeletionCorrupted
) {
518 db_
->DeleteRange(WriteOptions(), db_
->DefaultColumnFamily(), "a", "b"));
519 ASSERT_OK(db_
->Flush(FlushOptions()));
520 std::vector
<LiveFileMetaData
> metadata
;
521 db_
->GetLiveFilesMetaData(&metadata
);
522 ASSERT_EQ(static_cast<size_t>(1), metadata
.size());
523 std::string filename
= dbname_
+ metadata
[0].name
;
525 std::unique_ptr
<RandomAccessFile
> file
;
526 ASSERT_OK(options_
.env
->NewRandomAccessFile(filename
, &file
, EnvOptions()));
527 std::unique_ptr
<RandomAccessFileReader
> file_reader(
528 new RandomAccessFileReader(NewLegacyRandomAccessFileWrapper(file
),
532 ASSERT_OK(options_
.env
->GetFileSize(filename
, &file_size
));
534 BlockHandle range_del_handle
;
535 ASSERT_OK(FindMetaBlock(
536 file_reader
.get(), file_size
, kBlockBasedTableMagicNumber
,
537 ImmutableCFOptions(options_
), kRangeDelBlock
, &range_del_handle
));
539 ASSERT_OK(TryReopen());
540 ASSERT_OK(test::CorruptFile(&env_
, filename
,
541 static_cast<int>(range_del_handle
.offset()), 1));
542 ASSERT_TRUE(TryReopen().IsCorruption());
545 TEST_F(CorruptionTest
, FileSystemStateCorrupted
) {
546 for (int iter
= 0; iter
< 2; ++iter
) {
548 options
.paranoid_checks
= true;
549 options
.create_if_missing
= true;
552 ASSERT_OK(db_
->Flush(FlushOptions()));
553 DBImpl
* dbi
= static_cast_with_check
<DBImpl
>(db_
);
554 std::vector
<LiveFileMetaData
> metadata
;
555 dbi
->GetLiveFilesMetaData(&metadata
);
556 ASSERT_GT(metadata
.size(), 0);
557 std::string filename
= dbname_
+ metadata
[0].name
;
562 if (iter
== 0) { // corrupt file size
563 std::unique_ptr
<WritableFile
> file
;
564 env_
.NewWritableFile(filename
, &file
, EnvOptions());
565 ASSERT_OK(file
->Append(Slice("corrupted sst")));
567 Status x
= TryReopen(&options
);
568 ASSERT_TRUE(x
.IsCorruption());
569 } else { // delete the file
570 ASSERT_OK(env_
.DeleteFile(filename
));
571 Status x
= TryReopen(&options
);
572 ASSERT_TRUE(x
.IsCorruption());
575 ASSERT_OK(DestroyDB(dbname_
, options_
));
579 static const auto& corruption_modes
= {
580 mock::MockTableFactory::kCorruptNone
, mock::MockTableFactory::kCorruptKey
,
581 mock::MockTableFactory::kCorruptValue
,
582 mock::MockTableFactory::kCorruptReorderKey
};
584 TEST_F(CorruptionTest
, ParanoidFileChecksOnFlush
) {
586 options
.check_flush_compaction_key_order
= false;
587 options
.paranoid_file_checks
= true;
588 options
.create_if_missing
= true;
590 for (const auto& mode
: corruption_modes
) {
593 s
= DestroyDB(dbname_
, options
);
595 std::shared_ptr
<mock::MockTableFactory
> mock
=
596 std::make_shared
<mock::MockTableFactory
>();
597 options
.table_factory
= mock
;
598 mock
->SetCorruptionMode(mode
);
599 ASSERT_OK(DB::Open(options
, dbname_
, &db_
));
600 assert(db_
!= nullptr);
602 s
= db_
->Flush(FlushOptions());
603 if (mode
== mock::MockTableFactory::kCorruptNone
) {
611 TEST_F(CorruptionTest
, ParanoidFileChecksOnCompact
) {
613 options
.paranoid_file_checks
= true;
614 options
.create_if_missing
= true;
615 options
.check_flush_compaction_key_order
= false;
617 for (const auto& mode
: corruption_modes
) {
620 s
= DestroyDB(dbname_
, options
);
621 std::shared_ptr
<mock::MockTableFactory
> mock
=
622 std::make_shared
<mock::MockTableFactory
>();
623 options
.table_factory
= mock
;
624 ASSERT_OK(DB::Open(options
, dbname_
, &db_
));
625 assert(db_
!= nullptr);
627 // ASSERT_OK(db_->Flush(FlushOptions()));
628 DBImpl
* dbi
= static_cast_with_check
<DBImpl
>(db_
);
629 ASSERT_OK(dbi
->TEST_FlushMemTable());
630 mock
->SetCorruptionMode(mode
);
631 s
= dbi
->TEST_CompactRange(0, nullptr, nullptr, nullptr, true);
632 if (mode
== mock::MockTableFactory::kCorruptNone
) {
640 TEST_F(CorruptionTest
, ParanoidFileChecksWithDeleteRangeFirst
) {
642 options
.check_flush_compaction_key_order
= false;
643 options
.paranoid_file_checks
= true;
644 options
.create_if_missing
= true;
645 for (bool do_flush
: {true, false}) {
648 ASSERT_OK(DestroyDB(dbname_
, options
));
649 ASSERT_OK(DB::Open(options
, dbname_
, &db_
));
650 std::string start
, end
;
651 assert(db_
!= nullptr);
652 ASSERT_OK(db_
->DeleteRange(WriteOptions(), db_
->DefaultColumnFamily(),
653 Key(3, &start
), Key(7, &end
)));
654 auto snap
= db_
->GetSnapshot();
655 ASSERT_NE(snap
, nullptr);
656 ASSERT_OK(db_
->DeleteRange(WriteOptions(), db_
->DefaultColumnFamily(),
657 Key(8, &start
), Key(9, &end
)));
658 ASSERT_OK(db_
->DeleteRange(WriteOptions(), db_
->DefaultColumnFamily(),
659 Key(2, &start
), Key(5, &end
)));
662 ASSERT_OK(db_
->Flush(FlushOptions()));
664 DBImpl
* dbi
= static_cast_with_check
<DBImpl
>(db_
);
665 ASSERT_OK(dbi
->TEST_FlushMemTable());
666 ASSERT_OK(dbi
->TEST_CompactRange(0, nullptr, nullptr, nullptr, true));
668 db_
->ReleaseSnapshot(snap
);
672 TEST_F(CorruptionTest
, ParanoidFileChecksWithDeleteRange
) {
674 options
.check_flush_compaction_key_order
= false;
675 options
.paranoid_file_checks
= true;
676 options
.create_if_missing
= true;
677 for (bool do_flush
: {true, false}) {
680 ASSERT_OK(DestroyDB(dbname_
, options
));
681 ASSERT_OK(DB::Open(options
, dbname_
, &db_
));
682 assert(db_
!= nullptr);
684 std::string start
, end
;
685 ASSERT_OK(db_
->DeleteRange(WriteOptions(), db_
->DefaultColumnFamily(),
686 Key(5, &start
), Key(15, &end
)));
687 auto snap
= db_
->GetSnapshot();
688 ASSERT_NE(snap
, nullptr);
689 ASSERT_OK(db_
->DeleteRange(WriteOptions(), db_
->DefaultColumnFamily(),
690 Key(8, &start
), Key(9, &end
)));
691 ASSERT_OK(db_
->DeleteRange(WriteOptions(), db_
->DefaultColumnFamily(),
692 Key(12, &start
), Key(17, &end
)));
693 ASSERT_OK(db_
->DeleteRange(WriteOptions(), db_
->DefaultColumnFamily(),
694 Key(2, &start
), Key(4, &end
)));
697 ASSERT_OK(db_
->Flush(FlushOptions()));
699 DBImpl
* dbi
= static_cast_with_check
<DBImpl
>(db_
);
700 ASSERT_OK(dbi
->TEST_FlushMemTable());
701 ASSERT_OK(dbi
->TEST_CompactRange(0, nullptr, nullptr, nullptr, true));
703 db_
->ReleaseSnapshot(snap
);
707 TEST_F(CorruptionTest
, ParanoidFileChecksWithDeleteRangeLast
) {
709 options
.check_flush_compaction_key_order
= false;
710 options
.paranoid_file_checks
= true;
711 options
.create_if_missing
= true;
712 for (bool do_flush
: {true, false}) {
715 ASSERT_OK(DestroyDB(dbname_
, options
));
716 ASSERT_OK(DB::Open(options
, dbname_
, &db_
));
717 assert(db_
!= nullptr);
718 std::string start
, end
;
720 ASSERT_OK(db_
->DeleteRange(WriteOptions(), db_
->DefaultColumnFamily(),
721 Key(3, &start
), Key(7, &end
)));
722 auto snap
= db_
->GetSnapshot();
723 ASSERT_NE(snap
, nullptr);
724 ASSERT_OK(db_
->DeleteRange(WriteOptions(), db_
->DefaultColumnFamily(),
725 Key(6, &start
), Key(8, &end
)));
726 ASSERT_OK(db_
->DeleteRange(WriteOptions(), db_
->DefaultColumnFamily(),
727 Key(2, &start
), Key(5, &end
)));
729 ASSERT_OK(db_
->Flush(FlushOptions()));
731 DBImpl
* dbi
= static_cast_with_check
<DBImpl
>(db_
);
732 ASSERT_OK(dbi
->TEST_FlushMemTable());
733 ASSERT_OK(dbi
->TEST_CompactRange(0, nullptr, nullptr, nullptr, true));
735 db_
->ReleaseSnapshot(snap
);
739 TEST_F(CorruptionTest
, LogCorruptionErrorsInCompactionIterator
) {
741 options
.create_if_missing
= true;
742 options
.allow_data_in_errors
= true;
743 auto mode
= mock::MockTableFactory::kCorruptKey
;
746 ASSERT_OK(DestroyDB(dbname_
, options
));
748 std::shared_ptr
<mock::MockTableFactory
> mock
=
749 std::make_shared
<mock::MockTableFactory
>();
750 mock
->SetCorruptionMode(mode
);
751 options
.table_factory
= mock
;
753 ASSERT_OK(DB::Open(options
, dbname_
, &db_
));
754 assert(db_
!= nullptr);
757 DBImpl
* dbi
= static_cast_with_check
<DBImpl
>(db_
);
758 ASSERT_OK(dbi
->TEST_FlushMemTable());
759 Status s
= dbi
->TEST_CompactRange(0, nullptr, nullptr, nullptr, true);
761 ASSERT_TRUE(s
.IsCorruption());
764 TEST_F(CorruptionTest
, CompactionKeyOrderCheck
) {
766 options
.paranoid_file_checks
= false;
767 options
.create_if_missing
= true;
768 options
.check_flush_compaction_key_order
= false;
771 ASSERT_OK(DestroyDB(dbname_
, options
));
772 std::shared_ptr
<mock::MockTableFactory
> mock
=
773 std::make_shared
<mock::MockTableFactory
>();
774 options
.table_factory
= mock
;
775 ASSERT_OK(DB::Open(options
, dbname_
, &db_
));
776 assert(db_
!= nullptr);
777 mock
->SetCorruptionMode(mock::MockTableFactory::kCorruptReorderKey
);
779 DBImpl
* dbi
= static_cast_with_check
<DBImpl
>(db_
);
780 ASSERT_OK(dbi
->TEST_FlushMemTable());
782 mock
->SetCorruptionMode(mock::MockTableFactory::kCorruptNone
);
783 ASSERT_OK(db_
->SetOptions({{"check_flush_compaction_key_order", "true"}}));
784 ASSERT_NOK(dbi
->TEST_CompactRange(0, nullptr, nullptr, nullptr, true));
787 TEST_F(CorruptionTest
, FlushKeyOrderCheck
) {
789 options
.paranoid_file_checks
= false;
790 options
.create_if_missing
= true;
791 ASSERT_OK(db_
->SetOptions({{"check_flush_compaction_key_order", "true"}}));
793 ASSERT_OK(db_
->Put(WriteOptions(), "foo1", "v1"));
794 ASSERT_OK(db_
->Put(WriteOptions(), "foo2", "v1"));
795 ASSERT_OK(db_
->Put(WriteOptions(), "foo3", "v1"));
796 ASSERT_OK(db_
->Put(WriteOptions(), "foo4", "v1"));
799 // Generate some out of order keys from the memtable
800 SyncPoint::GetInstance()->SetCallBack(
801 "MemTableIterator::Next:0", [&](void* arg
) {
802 MemTableRep::Iterator
* mem_iter
=
803 static_cast<MemTableRep::Iterator
*>(arg
);
809 ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing();
810 Status s
= static_cast_with_check
<DBImpl
>(db_
)->TEST_FlushMemTable();
812 ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing();
813 ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->ClearAllCallBacks();
816 TEST_F(CorruptionTest
, DisableKeyOrderCheck
) {
818 ASSERT_OK(db_
->SetOptions({{"check_flush_compaction_key_order", "false"}}));
819 DBImpl
* dbi
= static_cast_with_check
<DBImpl
>(db_
);
821 SyncPoint::GetInstance()->SetCallBack(
822 "OutputValidator::Add:order_check",
823 [&](void* /*arg*/) { ASSERT_TRUE(false); });
824 ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing();
825 ASSERT_OK(db_
->Put(WriteOptions(), "foo1", "v1"));
826 ASSERT_OK(db_
->Put(WriteOptions(), "foo3", "v1"));
827 ASSERT_OK(dbi
->TEST_FlushMemTable());
828 ASSERT_OK(db_
->Put(WriteOptions(), "foo2", "v1"));
829 ASSERT_OK(db_
->Put(WriteOptions(), "foo4", "v1"));
830 ASSERT_OK(dbi
->TEST_FlushMemTable());
831 ASSERT_OK(dbi
->TEST_CompactRange(0, nullptr, nullptr, nullptr, true));
832 ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing();
833 ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->ClearAllCallBacks();
836 TEST_F(CorruptionTest
, VerifyWholeTableChecksum
) {
840 ASSERT_OK(DestroyDB(dbname_
, options
));
841 options
.create_if_missing
= true;
842 options
.file_checksum_gen_factory
=
843 ROCKSDB_NAMESPACE::GetFileChecksumGenCrc32cFactory();
848 ASSERT_OK(db_
->VerifyFileChecksums(ReadOptions()));
851 // Corrupt the first byte of each table file, this must be data block.
852 Corrupt(kTableFile
, 0, 1);
854 ASSERT_OK(TryReopen(&options
));
856 SyncPoint::GetInstance()->DisableProcessing();
857 SyncPoint::GetInstance()->ClearAllCallBacks();
859 SyncPoint::GetInstance()->SetCallBack(
860 "DBImpl::VerifySstFileChecksum:mismatch", [&](void* arg
) {
861 auto* s
= reinterpret_cast<Status
*>(arg
);
866 SyncPoint::GetInstance()->EnableProcessing();
867 ASSERT_TRUE(db_
->VerifyFileChecksums(ReadOptions()).IsCorruption());
871 } // namespace ROCKSDB_NAMESPACE
873 int main(int argc
, char** argv
) {
874 ::testing::InitGoogleTest(&argc
, argv
);
875 return RUN_ALL_TESTS();
881 int main(int /*argc*/, char** /*argv*/) {
882 fprintf(stderr
, "SKIPPED as RepairDB() is not supported in ROCKSDB_LITE\n");
886 #endif // !ROCKSDB_LITE