#include "port/stack_trace.h"
#include "rocksdb/perf_context.h"
#include "rocksdb/sst_file_manager.h"
-#include "util/fault_injection_test_env.h"
+#include "test_util/fault_injection_test_env.h"
#if !defined(ROCKSDB_LITE)
-#include "util/sync_point.h"
+#include "test_util/sync_point.h"
#endif
-namespace rocksdb {
+namespace ROCKSDB_NAMESPACE {
class DBErrorHandlingTest : public DBTestBase {
public:
DBErrorHandlingTest() : DBTestBase("/db_error_handling_test") {}
+
+ std::string GetManifestNameFromLiveFiles() {
+ std::vector<std::string> live_files;
+ uint64_t manifest_size;
+
+ dbfull()->GetLiveFiles(live_files, &manifest_size, false);
+ for (auto& file : live_files) {
+ uint64_t num = 0;
+ FileType type;
+ if (ParseFileName(file, &num, &type) && type == kDescriptorFile) {
+ return file;
+ }
+ }
+ return "";
+ }
};
class DBErrorHandlingEnv : public EnvWrapper {
});
SyncPoint::GetInstance()->EnableProcessing();
s = Flush();
- ASSERT_EQ(s.severity(), rocksdb::Status::Severity::kHardError);
+ ASSERT_EQ(s.severity(), ROCKSDB_NAMESPACE::Status::Severity::kHardError);
SyncPoint::GetInstance()->DisableProcessing();
fault_env->SetFilesystemActive(true);
s = dbfull()->Resume();
Destroy(options);
}
+TEST_F(DBErrorHandlingTest, ManifestWriteError) {
+ std::unique_ptr<FaultInjectionTestEnv> fault_env(
+ new FaultInjectionTestEnv(Env::Default()));
+ std::shared_ptr<ErrorHandlerListener> listener(new ErrorHandlerListener());
+ Options options = GetDefaultOptions();
+ options.create_if_missing = true;
+ options.env = fault_env.get();
+ options.listeners.emplace_back(listener);
+ Status s;
+ std::string old_manifest;
+ std::string new_manifest;
+
+ listener->EnableAutoRecovery(false);
+ DestroyAndReopen(options);
+ old_manifest = GetManifestNameFromLiveFiles();
+
+ Put(Key(0), "val");
+ Flush();
+ Put(Key(1), "val");
+ SyncPoint::GetInstance()->SetCallBack(
+ "VersionSet::LogAndApply:WriteManifest", [&](void *) {
+ fault_env->SetFilesystemActive(false, Status::NoSpace("Out of space"));
+ });
+ SyncPoint::GetInstance()->EnableProcessing();
+ s = Flush();
+ ASSERT_EQ(s.severity(), ROCKSDB_NAMESPACE::Status::Severity::kHardError);
+ SyncPoint::GetInstance()->ClearAllCallBacks();
+ SyncPoint::GetInstance()->DisableProcessing();
+ fault_env->SetFilesystemActive(true);
+ s = dbfull()->Resume();
+ ASSERT_EQ(s, Status::OK());
+
+ new_manifest = GetManifestNameFromLiveFiles();
+ ASSERT_NE(new_manifest, old_manifest);
+
+ Reopen(options);
+ ASSERT_EQ("val", Get(Key(0)));
+ ASSERT_EQ("val", Get(Key(1)));
+ Close();
+}
+
+TEST_F(DBErrorHandlingTest, DoubleManifestWriteError) {
+ std::unique_ptr<FaultInjectionTestEnv> fault_env(
+ new FaultInjectionTestEnv(Env::Default()));
+ std::shared_ptr<ErrorHandlerListener> listener(new ErrorHandlerListener());
+ Options options = GetDefaultOptions();
+ options.create_if_missing = true;
+ options.env = fault_env.get();
+ options.listeners.emplace_back(listener);
+ Status s;
+ std::string old_manifest;
+ std::string new_manifest;
+
+ listener->EnableAutoRecovery(false);
+ DestroyAndReopen(options);
+ old_manifest = GetManifestNameFromLiveFiles();
+
+ Put(Key(0), "val");
+ Flush();
+ Put(Key(1), "val");
+ SyncPoint::GetInstance()->SetCallBack(
+ "VersionSet::LogAndApply:WriteManifest", [&](void *) {
+ fault_env->SetFilesystemActive(false, Status::NoSpace("Out of space"));
+ });
+ SyncPoint::GetInstance()->EnableProcessing();
+ s = Flush();
+ ASSERT_EQ(s.severity(), ROCKSDB_NAMESPACE::Status::Severity::kHardError);
+ fault_env->SetFilesystemActive(true);
+
+ // This Resume() will attempt to create a new manifest file and fail again
+ s = dbfull()->Resume();
+ ASSERT_EQ(s.severity(), ROCKSDB_NAMESPACE::Status::Severity::kHardError);
+ fault_env->SetFilesystemActive(true);
+ SyncPoint::GetInstance()->ClearAllCallBacks();
+ SyncPoint::GetInstance()->DisableProcessing();
+
+ // A successful Resume() will create a new manifest file
+ s = dbfull()->Resume();
+ ASSERT_EQ(s, Status::OK());
+
+ new_manifest = GetManifestNameFromLiveFiles();
+ ASSERT_NE(new_manifest, old_manifest);
+
+ Reopen(options);
+ ASSERT_EQ("val", Get(Key(0)));
+ ASSERT_EQ("val", Get(Key(1)));
+ Close();
+}
+
+TEST_F(DBErrorHandlingTest, CompactionManifestWriteError) {
+ std::unique_ptr<FaultInjectionTestEnv> fault_env(
+ new FaultInjectionTestEnv(Env::Default()));
+ std::shared_ptr<ErrorHandlerListener> listener(new ErrorHandlerListener());
+ Options options = GetDefaultOptions();
+ options.create_if_missing = true;
+ options.level0_file_num_compaction_trigger = 2;
+ options.listeners.emplace_back(listener);
+ options.env = fault_env.get();
+ Status s;
+ std::string old_manifest;
+ std::string new_manifest;
+ std::atomic<bool> fail_manifest(false);
+ DestroyAndReopen(options);
+ old_manifest = GetManifestNameFromLiveFiles();
+
+ Put(Key(0), "val");
+ Put(Key(2), "val");
+ s = Flush();
+ ASSERT_EQ(s, Status::OK());
+
+ ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency(
+ // Wait for flush of 2nd L0 file before starting compaction
+ {{"DBImpl::FlushMemTable:FlushMemTableFinished",
+ "BackgroundCallCompaction:0"},
+ // Wait for compaction to detect manifest write error
+ {"BackgroundCallCompaction:1", "CompactionManifestWriteError:0"},
+ // Make compaction thread wait for error to be cleared
+ {"CompactionManifestWriteError:1",
+ "DBImpl::BackgroundCallCompaction:FoundObsoleteFiles"},
+ // Wait for DB instance to clear bg_error before calling
+ // TEST_WaitForCompact
+ {"SstFileManagerImpl::ErrorCleared", "CompactionManifestWriteError:2"}});
+ // trigger manifest write failure in compaction thread
+ ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
+ "BackgroundCallCompaction:0", [&](void*) { fail_manifest.store(true); });
+ ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
+ "VersionSet::LogAndApply:WriteManifest", [&](void*) {
+ if (fail_manifest.load()) {
+ fault_env->SetFilesystemActive(false,
+ Status::NoSpace("Out of space"));
+ }
+ });
+ ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing();
+
+ Put(Key(1), "val");
+ // This Flush will trigger a compaction, which will fail when appending to
+ // the manifest
+ s = Flush();
+ ASSERT_EQ(s, Status::OK());
+
+ TEST_SYNC_POINT("CompactionManifestWriteError:0");
+ // Clear all errors so when the compaction is retried, it will succeed
+ fault_env->SetFilesystemActive(true);
+ ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->ClearAllCallBacks();
+ TEST_SYNC_POINT("CompactionManifestWriteError:1");
+ TEST_SYNC_POINT("CompactionManifestWriteError:2");
+
+ s = dbfull()->TEST_WaitForCompact();
+ ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing();
+ ASSERT_EQ(s, Status::OK());
+
+ new_manifest = GetManifestNameFromLiveFiles();
+ ASSERT_NE(new_manifest, old_manifest);
+ Reopen(options);
+ ASSERT_EQ("val", Get(Key(0)));
+ ASSERT_EQ("val", Get(Key(1)));
+ ASSERT_EQ("val", Get(Key(2)));
+ Close();
+}
+
TEST_F(DBErrorHandlingTest, CompactionWriteError) {
std::unique_ptr<FaultInjectionTestEnv> fault_env(
new FaultInjectionTestEnv(Env::Default()));
Status(Status::NoSpace(), Status::Severity::kHardError)
);
listener->EnableAutoRecovery(false);
- rocksdb::SyncPoint::GetInstance()->LoadDependency(
- {{"FlushMemTableFinished", "BackgroundCallCompaction:0"}});
- rocksdb::SyncPoint::GetInstance()->SetCallBack(
- "BackgroundCallCompaction:0", [&](void *) {
- fault_env->SetFilesystemActive(false, Status::NoSpace("Out of space"));
+ ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency(
+ {{"DBImpl::FlushMemTable:FlushMemTableFinished",
+ "BackgroundCallCompaction:0"}});
+ ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
+ "BackgroundCallCompaction:0", [&](void*) {
+ fault_env->SetFilesystemActive(false, Status::NoSpace("Out of space"));
});
- rocksdb::SyncPoint::GetInstance()->EnableProcessing();
+ ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing();
Put(Key(1), "val");
s = Flush();
ASSERT_EQ(s, Status::OK());
s = dbfull()->TEST_WaitForCompact();
- ASSERT_EQ(s.severity(), rocksdb::Status::Severity::kHardError);
+ ASSERT_EQ(s.severity(), ROCKSDB_NAMESPACE::Status::Severity::kHardError);
fault_env->SetFilesystemActive(true);
s = dbfull()->Resume();
s = Flush();
ASSERT_EQ(s, Status::OK());
- rocksdb::SyncPoint::GetInstance()->LoadDependency(
- {{"FlushMemTableFinished", "BackgroundCallCompaction:0"}});
- rocksdb::SyncPoint::GetInstance()->SetCallBack(
- "BackgroundCallCompaction:0", [&](void *) {
- fault_env->SetFilesystemActive(false, Status::Corruption("Corruption"));
+ ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency(
+ {{"DBImpl::FlushMemTable:FlushMemTableFinished",
+ "BackgroundCallCompaction:0"}});
+ ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
+ "BackgroundCallCompaction:0", [&](void*) {
+ fault_env->SetFilesystemActive(false, Status::Corruption("Corruption"));
});
- rocksdb::SyncPoint::GetInstance()->EnableProcessing();
+ ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing();
Put(Key(1), "val");
s = Flush();
ASSERT_EQ(s, Status::OK());
s = dbfull()->TEST_WaitForCompact();
- ASSERT_EQ(s.severity(), rocksdb::Status::Severity::kUnrecoverableError);
+ ASSERT_EQ(s.severity(),
+ ROCKSDB_NAMESPACE::Status::Severity::kUnrecoverableError);
fault_env->SetFilesystemActive(true);
s = dbfull()->Resume();
});
SyncPoint::GetInstance()->EnableProcessing();
s = Flush();
- ASSERT_EQ(s.severity(), rocksdb::Status::Severity::kHardError);
+ ASSERT_EQ(s.severity(), ROCKSDB_NAMESPACE::Status::Severity::kHardError);
SyncPoint::GetInstance()->DisableProcessing();
fault_env->SetFilesystemActive(true);
ASSERT_EQ(listener->WaitForRecovery(5000000), true);
});
SyncPoint::GetInstance()->EnableProcessing();
s = Flush();
- ASSERT_EQ(s.severity(), rocksdb::Status::Severity::kHardError);
+ ASSERT_EQ(s.severity(), ROCKSDB_NAMESPACE::Status::Severity::kHardError);
// We should be able to shutdown the database while auto recovery is going
// on in the background
Close();
for (auto i = 0; i < kNumDbInstances; ++i) {
std::string prop;
ASSERT_EQ(listener[i]->WaitForRecovery(5000000), true);
+ ASSERT_EQ(static_cast<DBImpl*>(db[i])->TEST_WaitForCompact(true),
+ Status::OK());
EXPECT_TRUE(db[i]->GetProperty(
"rocksdb.num-files-at-level" + NumberToString(0), &prop));
EXPECT_EQ(atoi(prop.c_str()), 0);
delete def_env;
}
-} // namespace rocksdb
+} // namespace ROCKSDB_NAMESPACE
int main(int argc, char** argv) {
- rocksdb::port::InstallStackTraceHandler();
+ ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
::testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();
}
projects / ceph.git / blobdiff