ceph/src/rocksdb/db/repair_test.cc

   1 //  Copyright (c) 2016-present, Facebook, Inc.  All rights reserved.
   2 //  This source code is licensed under the BSD-style license found in the
   3 //  LICENSE file in the root directory of this source tree. An additional grant
   4 //  of patent rights can be found in the PATENTS file in the same directory.
   5
   6 #ifndef ROCKSDB_LITE
   7
   8 #include <algorithm>
   9 #include <string>
  10 #include <vector>
  11
  12 #include "db/db_impl.h"
  13 #include "db/db_test_util.h"
  14 #include "rocksdb/comparator.h"
  15 #include "rocksdb/db.h"
  16 #include "rocksdb/transaction_log.h"
  17 #include "util/file_util.h"
  18 #include "util/string_util.h"
  19
  20 namespace rocksdb {
  21
  22 #ifndef ROCKSDB_LITE
  23 class RepairTest : public DBTestBase {
  24  public:
  25   RepairTest() : DBTestBase("/repair_test") {}
  26
  27   std::string GetFirstSstPath() {
  28     uint64_t manifest_size;
  29     std::vector<std::string> files;
  30     db_->GetLiveFiles(files, &manifest_size);
  31     auto sst_iter =
  32         std::find_if(files.begin(), files.end(), [](const std::string& file) {
  33           uint64_t number;
  34           FileType type;
  35           bool ok = ParseFileName(file, &number, &type);
  36           return ok && type == kTableFile;
  37         });
  38     return sst_iter == files.end() ? "" : dbname_ + *sst_iter;
  39   }
  40 };
  41
  42 TEST_F(RepairTest, LostManifest) {
  43   // Add a couple SST files, delete the manifest, and verify RepairDB() saves
  44   // the day.
  45   Put("key", "val");
  46   Flush();
  47   Put("key2", "val2");
  48   Flush();
  49   // Need to get path before Close() deletes db_, but delete it after Close() to
  50   // ensure Close() didn't change the manifest.
  51   std::string manifest_path =
  52       DescriptorFileName(dbname_, dbfull()->TEST_Current_Manifest_FileNo());
  53
  54   Close();
  55   ASSERT_OK(env_->FileExists(manifest_path));
  56   ASSERT_OK(env_->DeleteFile(manifest_path));
  57   ASSERT_OK(RepairDB(dbname_, CurrentOptions()));
  58   Reopen(CurrentOptions());
  59
  60   ASSERT_EQ(Get("key"), "val");
  61   ASSERT_EQ(Get("key2"), "val2");
  62 }
  63
  64 TEST_F(RepairTest, CorruptManifest) {
  65   // Manifest is in an invalid format. Expect a full recovery.
  66   Put("key", "val");
  67   Flush();
  68   Put("key2", "val2");
  69   Flush();
  70   // Need to get path before Close() deletes db_, but overwrite it after Close()
  71   // to ensure Close() didn't change the manifest.
  72   std::string manifest_path =
  73       DescriptorFileName(dbname_, dbfull()->TEST_Current_Manifest_FileNo());
  74
  75   Close();
  76   ASSERT_OK(env_->FileExists(manifest_path));
  77   CreateFile(env_, manifest_path, "blah");
  78   ASSERT_OK(RepairDB(dbname_, CurrentOptions()));
  79   Reopen(CurrentOptions());
  80
  81   ASSERT_EQ(Get("key"), "val");
  82   ASSERT_EQ(Get("key2"), "val2");
  83 }
  84
  85 TEST_F(RepairTest, IncompleteManifest) {
  86   // In this case, the manifest is valid but does not reference all of the SST
  87   // files. Expect a full recovery.
  88   Put("key", "val");
  89   Flush();
  90   std::string orig_manifest_path =
  91       DescriptorFileName(dbname_, dbfull()->TEST_Current_Manifest_FileNo());
  92   CopyFile(orig_manifest_path, orig_manifest_path + ".tmp");
  93   Put("key2", "val2");
  94   Flush();
  95   // Need to get path before Close() deletes db_, but overwrite it after Close()
  96   // to ensure Close() didn't change the manifest.
  97   std::string new_manifest_path =
  98       DescriptorFileName(dbname_, dbfull()->TEST_Current_Manifest_FileNo());
  99
 100   Close();
 101   ASSERT_OK(env_->FileExists(new_manifest_path));
 102   // Replace the manifest with one that is only aware of the first SST file.
 103   CopyFile(orig_manifest_path + ".tmp", new_manifest_path);
 104   ASSERT_OK(RepairDB(dbname_, CurrentOptions()));
 105   Reopen(CurrentOptions());
 106
 107   ASSERT_EQ(Get("key"), "val");
 108   ASSERT_EQ(Get("key2"), "val2");
 109 }
 110
 111 TEST_F(RepairTest, LostSst) {
 112   // Delete one of the SST files but preserve the manifest that refers to it,
 113   // then verify the DB is still usable for the intact SST.
 114   Put("key", "val");
 115   Flush();
 116   Put("key2", "val2");
 117   Flush();
 118   auto sst_path = GetFirstSstPath();
 119   ASSERT_FALSE(sst_path.empty());
 120   ASSERT_OK(env_->DeleteFile(sst_path));
 121
 122   Close();
 123   ASSERT_OK(RepairDB(dbname_, CurrentOptions()));
 124   Reopen(CurrentOptions());
 125
 126   // Exactly one of the key-value pairs should be in the DB now.
 127   ASSERT_TRUE((Get("key") == "val") != (Get("key2") == "val2"));
 128 }
 129
 130 TEST_F(RepairTest, CorruptSst) {
 131   // Corrupt one of the SST files but preserve the manifest that refers to it,
 132   // then verify the DB is still usable for the intact SST.
 133   Put("key", "val");
 134   Flush();
 135   Put("key2", "val2");
 136   Flush();
 137   auto sst_path = GetFirstSstPath();
 138   ASSERT_FALSE(sst_path.empty());
 139   CreateFile(env_, sst_path, "blah");
 140
 141   Close();
 142   ASSERT_OK(RepairDB(dbname_, CurrentOptions()));
 143   Reopen(CurrentOptions());
 144
 145   // Exactly one of the key-value pairs should be in the DB now.
 146   ASSERT_TRUE((Get("key") == "val") != (Get("key2") == "val2"));
 147 }
 148
 149 TEST_F(RepairTest, UnflushedSst) {
 150   // This test case invokes repair while some data is unflushed, then verifies
 151   // that data is in the db.
 152   Put("key", "val");
 153   VectorLogPtr wal_files;
 154   ASSERT_OK(dbfull()->GetSortedWalFiles(wal_files));
 155   ASSERT_EQ(wal_files.size(), 1);
 156   uint64_t total_ssts_size;
 157   GetAllSSTFiles(&total_ssts_size);
 158   ASSERT_EQ(total_ssts_size, 0);
 159   // Need to get path before Close() deletes db_, but delete it after Close() to
 160   // ensure Close() didn't change the manifest.
 161   std::string manifest_path =
 162       DescriptorFileName(dbname_, dbfull()->TEST_Current_Manifest_FileNo());
 163
 164   Close();
 165   ASSERT_OK(env_->FileExists(manifest_path));
 166   ASSERT_OK(env_->DeleteFile(manifest_path));
 167   ASSERT_OK(RepairDB(dbname_, CurrentOptions()));
 168   Reopen(CurrentOptions());
 169
 170   ASSERT_OK(dbfull()->GetSortedWalFiles(wal_files));
 171   ASSERT_EQ(wal_files.size(), 0);
 172   GetAllSSTFiles(&total_ssts_size);
 173   ASSERT_GT(total_ssts_size, 0);
 174   ASSERT_EQ(Get("key"), "val");
 175 }
 176
 177 TEST_F(RepairTest, RepairMultipleColumnFamilies) {
 178   // Verify repair logic associates SST files with their original column
 179   // families.
 180   const int kNumCfs = 3;
 181   const int kEntriesPerCf = 2;
 182   DestroyAndReopen(CurrentOptions());
 183   CreateAndReopenWithCF({"pikachu1", "pikachu2"}, CurrentOptions());
 184   for (int i = 0; i < kNumCfs; ++i) {
 185     for (int j = 0; j < kEntriesPerCf; ++j) {
 186       Put(i, "key" + ToString(j), "val" + ToString(j));
 187       if (j == kEntriesPerCf - 1 && i == kNumCfs - 1) {
 188         // Leave one unflushed so we can verify WAL entries are properly
 189         // associated with column families.
 190         continue;
 191       }
 192       Flush(i);
 193     }
 194   }
 195
 196   // Need to get path before Close() deletes db_, but delete it after Close() to
 197   // ensure Close() doesn't re-create the manifest.
 198   std::string manifest_path =
 199       DescriptorFileName(dbname_, dbfull()->TEST_Current_Manifest_FileNo());
 200   Close();
 201   ASSERT_OK(env_->FileExists(manifest_path));
 202   ASSERT_OK(env_->DeleteFile(manifest_path));
 203
 204   ASSERT_OK(RepairDB(dbname_, CurrentOptions()));
 205
 206   ReopenWithColumnFamilies({"default", "pikachu1", "pikachu2"},
 207                            CurrentOptions());
 208   for (int i = 0; i < kNumCfs; ++i) {
 209     for (int j = 0; j < kEntriesPerCf; ++j) {
 210       ASSERT_EQ(Get(i, "key" + ToString(j)), "val" + ToString(j));
 211     }
 212   }
 213 }
 214
 215 TEST_F(RepairTest, RepairColumnFamilyOptions) {
 216   // Verify repair logic uses correct ColumnFamilyOptions when repairing a
 217   // database with different options for column families.
 218   const int kNumCfs = 2;
 219   const int kEntriesPerCf = 2;
 220
 221   Options opts(CurrentOptions()), rev_opts(CurrentOptions());
 222   opts.comparator = BytewiseComparator();
 223   rev_opts.comparator = ReverseBytewiseComparator();
 224
 225   DestroyAndReopen(opts);
 226   CreateColumnFamilies({"reverse"}, rev_opts);
 227   ReopenWithColumnFamilies({"default", "reverse"},
 228                            std::vector<Options>{opts, rev_opts});
 229   for (int i = 0; i < kNumCfs; ++i) {
 230     for (int j = 0; j < kEntriesPerCf; ++j) {
 231       Put(i, "key" + ToString(j), "val" + ToString(j));
 232       if (i == kNumCfs - 1 && j == kEntriesPerCf - 1) {
 233         // Leave one unflushed so we can verify RepairDB's flush logic
 234         continue;
 235       }
 236       Flush(i);
 237     }
 238   }
 239   Close();
 240
 241   // RepairDB() records the comparator in the manifest, and DB::Open would fail
 242   // if a different comparator were used.
 243   ASSERT_OK(RepairDB(dbname_, opts, {{"default", opts}, {"reverse", rev_opts}},
 244                      opts /* unknown_cf_opts */));
 245   ASSERT_OK(TryReopenWithColumnFamilies({"default", "reverse"},
 246                                         std::vector<Options>{opts, rev_opts}));
 247   for (int i = 0; i < kNumCfs; ++i) {
 248     for (int j = 0; j < kEntriesPerCf; ++j) {
 249       ASSERT_EQ(Get(i, "key" + ToString(j)), "val" + ToString(j));
 250     }
 251   }
 252
 253   // Examine table properties to verify RepairDB() used the right options when
 254   // converting WAL->SST
 255   TablePropertiesCollection fname_to_props;
 256   db_->GetPropertiesOfAllTables(handles_[1], &fname_to_props);
 257   ASSERT_EQ(fname_to_props.size(), 2U);
 258   for (const auto& fname_and_props : fname_to_props) {
 259     std::string comparator_name (
 260       InternalKeyComparator(rev_opts.comparator).Name());
 261     comparator_name = comparator_name.substr(comparator_name.find(':') + 1);
 262     ASSERT_EQ(comparator_name,
 263               fname_and_props.second->comparator_name);
 264   }
 265
 266   // Also check comparator when it's provided via "unknown" CF options
 267   ASSERT_OK(RepairDB(dbname_, opts, {{"default", opts}},
 268                      rev_opts /* unknown_cf_opts */));
 269   ASSERT_OK(TryReopenWithColumnFamilies({"default", "reverse"},
 270                                         std::vector<Options>{opts, rev_opts}));
 271   for (int i = 0; i < kNumCfs; ++i) {
 272     for (int j = 0; j < kEntriesPerCf; ++j) {
 273       ASSERT_EQ(Get(i, "key" + ToString(j)), "val" + ToString(j));
 274     }
 275   }
 276 }
 277
 278 #endif  // ROCKSDB_LITE
 279 }  // namespace rocksdb
 280
 281 int main(int argc, char** argv) {
 282   ::testing::InitGoogleTest(&argc, argv);
 283   return RUN_ALL_TESTS();
 284 }
 285
 286 #else
 287 #include <stdio.h>
 288
 289 int main(int argc, char** argv) {
 290   fprintf(stderr, "SKIPPED as RepairDB is not supported in ROCKSDB_LITE\n");
 291   return 0;
 292 }
 293
 294 #endif  // ROCKSDB_LITE