// COPYING file in the root directory) and Apache 2.0 License
// (found in the LICENSE.Apache file in the root directory).
+#include "rocksdb/options.h"
#ifndef ROCKSDB_LITE
#include <algorithm>
#include "rocksdb/comparator.h"
#include "rocksdb/db.h"
#include "rocksdb/transaction_log.h"
+#include "table/unique_id_impl.h"
#include "util/string_util.h"
namespace ROCKSDB_NAMESPACE {
#ifndef ROCKSDB_LITE
class RepairTest : public DBTestBase {
public:
- RepairTest() : DBTestBase("/repair_test", /*env_do_fsync=*/true) {}
+ RepairTest() : DBTestBase("repair_test", /*env_do_fsync=*/true) {}
Status GetFirstSstPath(std::string* first_sst_path) {
assert(first_sst_path != nullptr);
}
return s;
}
+
+ void ReopenWithSstIdVerify() {
+ std::atomic_int verify_passed{0};
+ SyncPoint::GetInstance()->SetCallBack(
+ "BlockBasedTable::Open::PassedVerifyUniqueId", [&](void* arg) {
+ // override job status
+ auto id = static_cast<UniqueId64x2*>(arg);
+ assert(*id != kNullUniqueId64x2);
+ verify_passed++;
+ });
+ SyncPoint::GetInstance()->EnableProcessing();
+ auto options = CurrentOptions();
+ options.verify_sst_unique_id_in_manifest = true;
+ Reopen(options);
+
+ ASSERT_GT(verify_passed, 0);
+ SyncPoint::GetInstance()->DisableProcessing();
+ }
};
TEST_F(RepairTest, LostManifest) {
ASSERT_OK(env_->FileExists(manifest_path));
ASSERT_OK(env_->DeleteFile(manifest_path));
ASSERT_OK(RepairDB(dbname_, CurrentOptions()));
- Reopen(CurrentOptions());
+ ReopenWithSstIdVerify();
ASSERT_EQ(Get("key"), "val");
ASSERT_EQ(Get("key2"), "val2");
}
+TEST_F(RepairTest, LostManifestMoreDbFeatures) {
+ // Add a couple SST files, delete the manifest, and verify RepairDB() saves
+ // the day.
+ ASSERT_OK(Put("key", "val"));
+ ASSERT_OK(Put("key2", "val2"));
+ ASSERT_OK(Put("key3", "val3"));
+ ASSERT_OK(Put("key4", "val4"));
+ ASSERT_OK(Flush());
+ // Test an SST file containing only a range tombstone
+ ASSERT_OK(db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), "key2",
+ "key3z"));
+ ASSERT_OK(Flush());
+ // Need to get path before Close() deletes db_, but delete it after Close() to
+ // ensure Close() didn't change the manifest.
+ std::string manifest_path =
+ DescriptorFileName(dbname_, dbfull()->TEST_Current_Manifest_FileNo());
+
+ Close();
+ ASSERT_OK(env_->FileExists(manifest_path));
+ ASSERT_OK(env_->DeleteFile(manifest_path));
+ ASSERT_OK(RepairDB(dbname_, CurrentOptions()));
+
+ // repair from sst should work with unique_id verification
+ ReopenWithSstIdVerify();
+
+ ASSERT_EQ(Get("key"), "val");
+ ASSERT_EQ(Get("key2"), "NOT_FOUND");
+ ASSERT_EQ(Get("key3"), "NOT_FOUND");
+ ASSERT_EQ(Get("key4"), "val4");
+}
+
TEST_F(RepairTest, CorruptManifest) {
// Manifest is in an invalid format. Expect a full recovery.
ASSERT_OK(Put("key", "val"));
Close();
ASSERT_OK(env_->FileExists(manifest_path));
- LegacyFileSystemWrapper fs(env_);
- ASSERT_OK(CreateFile(&fs, manifest_path, "blah", false /* use_fsync */));
+ ASSERT_OK(CreateFile(env_->GetFileSystem(), manifest_path, "blah",
+ false /* use_fsync */));
ASSERT_OK(RepairDB(dbname_, CurrentOptions()));
- Reopen(CurrentOptions());
+
+ ReopenWithSstIdVerify();
ASSERT_EQ(Get("key"), "val");
ASSERT_EQ(Get("key2"), "val2");
// Replace the manifest with one that is only aware of the first SST file.
CopyFile(orig_manifest_path + ".tmp", new_manifest_path);
ASSERT_OK(RepairDB(dbname_, CurrentOptions()));
- Reopen(CurrentOptions());
+
+ ReopenWithSstIdVerify();
ASSERT_EQ(Get("key"), "val");
ASSERT_EQ(Get("key2"), "val2");
ASSERT_OK(RepairDB(dbname_, CurrentOptions()));
- Reopen(CurrentOptions());
+ ReopenWithSstIdVerify();
+
uint64_t post_repair_file_num = dbfull()->TEST_Current_Next_FileNo();
ASSERT_GE(post_repair_file_num, pre_repair_file_num);
}
Close();
ASSERT_OK(RepairDB(dbname_, CurrentOptions()));
- Reopen(CurrentOptions());
+ ReopenWithSstIdVerify();
// Exactly one of the key-value pairs should be in the DB now.
ASSERT_TRUE((Get("key") == "val") != (Get("key2") == "val2"));
ASSERT_OK(GetFirstSstPath(&sst_path));
ASSERT_FALSE(sst_path.empty());
- LegacyFileSystemWrapper fs(env_);
- ASSERT_OK(CreateFile(&fs, sst_path, "blah", false /* use_fsync */));
+ ASSERT_OK(CreateFile(env_->GetFileSystem(), sst_path, "blah",
+ false /* use_fsync */));
Close();
ASSERT_OK(RepairDB(dbname_, CurrentOptions()));
- Reopen(CurrentOptions());
+ ReopenWithSstIdVerify();
// Exactly one of the key-value pairs should be in the DB now.
ASSERT_TRUE((Get("key") == "val") != (Get("key2") == "val2"));
{
uint64_t total_ssts_size;
std::unordered_map<std::string, uint64_t> sst_files;
- ASSERT_OK(GetAllSSTFiles(&sst_files, &total_ssts_size));
+ ASSERT_OK(GetAllDataFiles(kTableFile, &sst_files, &total_ssts_size));
ASSERT_EQ(total_ssts_size, 0);
}
// Need to get path before Close() deletes db_, but delete it after Close() to
ASSERT_OK(env_->FileExists(manifest_path));
ASSERT_OK(env_->DeleteFile(manifest_path));
ASSERT_OK(RepairDB(dbname_, CurrentOptions()));
- Reopen(CurrentOptions());
+ ReopenWithSstIdVerify();
ASSERT_OK(dbfull()->GetSortedWalFiles(wal_files));
ASSERT_EQ(wal_files.size(), 0);
{
uint64_t total_ssts_size;
std::unordered_map<std::string, uint64_t> sst_files;
- ASSERT_OK(GetAllSSTFiles(&sst_files, &total_ssts_size));
+ ASSERT_OK(GetAllDataFiles(kTableFile, &sst_files, &total_ssts_size));
ASSERT_GT(total_ssts_size, 0);
}
ASSERT_EQ(Get("key"), "val");
{
uint64_t total_ssts_size;
std::unordered_map<std::string, uint64_t> sst_files;
- ASSERT_OK(GetAllSSTFiles(&sst_files, &total_ssts_size));
+ ASSERT_OK(GetAllDataFiles(kTableFile, &sst_files, &total_ssts_size));
ASSERT_EQ(total_ssts_size, 0);
}
std::string manifest_path =
- DescriptorFileName(dbname_, dbfull()->TEST_Current_Manifest_FileNo());
+ DescriptorFileName(dbname_, dbfull()->TEST_Current_Manifest_FileNo());
Close();
ASSERT_OK(env_->FileExists(manifest_path));
// make sure that all WALs are converted to SSTables.
options.wal_dir = "";
- Reopen(options);
+ ReopenWithSstIdVerify();
ASSERT_OK(dbfull()->GetSortedWalFiles(wal_files));
ASSERT_EQ(wal_files.size(), 0);
{
uint64_t total_ssts_size;
std::unordered_map<std::string, uint64_t> sst_files;
- ASSERT_OK(GetAllSSTFiles(&sst_files, &total_ssts_size));
+ ASSERT_OK(GetAllDataFiles(kTableFile, &sst_files, &total_ssts_size));
ASSERT_GT(total_ssts_size, 0);
}
ASSERT_EQ(Get("key"), "val");
ASSERT_EQ(Get("foo"), "bar");
- } while(ChangeWalOptions());
+ } while (ChangeWalOptions());
}
TEST_F(RepairTest, RepairMultipleColumnFamilies) {
CreateAndReopenWithCF({"pikachu1", "pikachu2"}, CurrentOptions());
for (int i = 0; i < kNumCfs; ++i) {
for (int j = 0; j < kEntriesPerCf; ++j) {
- ASSERT_OK(Put(i, "key" + ToString(j), "val" + ToString(j)));
+ ASSERT_OK(Put(i, "key" + std::to_string(j), "val" + std::to_string(j)));
if (j == kEntriesPerCf - 1 && i == kNumCfs - 1) {
// Leave one unflushed so we can verify WAL entries are properly
// associated with column families.
CurrentOptions());
for (int i = 0; i < kNumCfs; ++i) {
for (int j = 0; j < kEntriesPerCf; ++j) {
- ASSERT_EQ(Get(i, "key" + ToString(j)), "val" + ToString(j));
+ ASSERT_EQ(Get(i, "key" + std::to_string(j)), "val" + std::to_string(j));
}
}
}
std::vector<Options>{opts, rev_opts});
for (int i = 0; i < kNumCfs; ++i) {
for (int j = 0; j < kEntriesPerCf; ++j) {
- ASSERT_OK(Put(i, "key" + ToString(j), "val" + ToString(j)));
+ ASSERT_OK(Put(i, "key" + std::to_string(j), "val" + std::to_string(j)));
if (i == kNumCfs - 1 && j == kEntriesPerCf - 1) {
// Leave one unflushed so we can verify RepairDB's flush logic
continue;
std::vector<Options>{opts, rev_opts}));
for (int i = 0; i < kNumCfs; ++i) {
for (int j = 0; j < kEntriesPerCf; ++j) {
- ASSERT_EQ(Get(i, "key" + ToString(j)), "val" + ToString(j));
+ ASSERT_EQ(Get(i, "key" + std::to_string(j)), "val" + std::to_string(j));
}
}
ASSERT_OK(db_->GetPropertiesOfAllTables(handles_[1], &fname_to_props));
ASSERT_EQ(fname_to_props.size(), 2U);
for (const auto& fname_and_props : fname_to_props) {
- std::string comparator_name (
- InternalKeyComparator(rev_opts.comparator).Name());
- comparator_name = comparator_name.substr(comparator_name.find(':') + 1);
- ASSERT_EQ(comparator_name,
- fname_and_props.second->comparator_name);
+ std::string comparator_name(rev_opts.comparator->Name());
+ ASSERT_EQ(comparator_name, fname_and_props.second->comparator_name);
}
Close();
std::vector<Options>{opts, rev_opts}));
for (int i = 0; i < kNumCfs; ++i) {
for (int j = 0; j < kEntriesPerCf; ++j) {
- ASSERT_EQ(Get(i, "key" + ToString(j)), "val" + ToString(j));
+ ASSERT_EQ(Get(i, "key" + std::to_string(j)), "val" + std::to_string(j));
}
}
}
Close();
ASSERT_OK(RepairDB(dbname_ + "/", CurrentOptions()));
- Reopen(CurrentOptions());
+ ReopenWithSstIdVerify();
ASSERT_EQ(Get("key"), "val");
}
#endif // ROCKSDB_LITE
} // namespace ROCKSDB_NAMESPACE
int main(int argc, char** argv) {
+ ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
::testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();
}
projects / ceph.git / blobdiff