ceph/src/rocksdb/utilities/option_change_migration/option_change_migration_test.cc

   1 //  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
   2 //  This source code is licensed under the BSD-style license found in the
   3 //  LICENSE file in the root directory of this source tree. An additional grant
   4 //  of patent rights can be found in the PATENTS file in the same directory.
   5 //
   6 // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
   7 // Use of this source code is governed by a BSD-style license that can be
   8 // found in the LICENSE file. See the AUTHORS file for names of contributors.
   9
  10 #include "rocksdb/utilities/option_change_migration.h"
  11 #include <set>
  12 #include "db/db_test_util.h"
  13 #include "port/stack_trace.h"
  14 namespace rocksdb {
  15
  16 class DBOptionChangeMigrationTests
  17     : public DBTestBase,
  18       public testing::WithParamInterface<
  19           std::tuple<int, int, bool, int, int, bool>> {
  20  public:
  21   DBOptionChangeMigrationTests()
  22       : DBTestBase("/db_option_change_migration_test") {
  23     level1_ = std::get<0>(GetParam());
  24     compaction_style1_ = std::get<1>(GetParam());
  25     is_dynamic1_ = std::get<2>(GetParam());
  26
  27     level2_ = std::get<3>(GetParam());
  28     compaction_style2_ = std::get<4>(GetParam());
  29     is_dynamic2_ = std::get<5>(GetParam());
  30   }
  31
  32   // Required if inheriting from testing::WithParamInterface<>
  33   static void SetUpTestCase() {}
  34   static void TearDownTestCase() {}
  35
  36   int level1_;
  37   int compaction_style1_;
  38   bool is_dynamic1_;
  39
  40   int level2_;
  41   int compaction_style2_;
  42   bool is_dynamic2_;
  43 };
  44
  45 #ifndef ROCKSDB_LITE
  46 TEST_P(DBOptionChangeMigrationTests, Migrate1) {
  47   Options old_options = CurrentOptions();
  48   old_options.compaction_style =
  49       static_cast<CompactionStyle>(compaction_style1_);
  50   if (old_options.compaction_style == CompactionStyle::kCompactionStyleLevel) {
  51     old_options.level_compaction_dynamic_level_bytes = is_dynamic1_;
  52   }
  53
  54   old_options.level0_file_num_compaction_trigger = 3;
  55   old_options.write_buffer_size = 64 * 1024;
  56   old_options.target_file_size_base = 128 * 1024;
  57   // Make level target of L1, L2 to be 200KB and 600KB
  58   old_options.num_levels = level1_;
  59   old_options.max_bytes_for_level_multiplier = 3;
  60   old_options.max_bytes_for_level_base = 200 * 1024;
  61
  62   Reopen(old_options);
  63
  64   Random rnd(301);
  65   int key_idx = 0;
  66
  67   // Generate at least 2MB of data
  68   for (int num = 0; num < 20; num++) {
  69     GenerateNewFile(&rnd, &key_idx);
  70   }
  71   dbfull()->TEST_WaitForFlushMemTable();
  72   dbfull()->TEST_WaitForCompact();
  73
  74   // Will make sure exactly those keys are in the DB after migration.
  75   std::set<std::string> keys;
  76   {
  77     std::unique_ptr<Iterator> it(db_->NewIterator(ReadOptions()));
  78     it->SeekToFirst();
  79     for (; it->Valid(); it->Next()) {
  80       keys.insert(it->key().ToString());
  81     }
  82   }
  83   Close();
  84
  85   Options new_options = old_options;
  86   new_options.compaction_style =
  87       static_cast<CompactionStyle>(compaction_style2_);
  88   if (new_options.compaction_style == CompactionStyle::kCompactionStyleLevel) {
  89     new_options.level_compaction_dynamic_level_bytes = is_dynamic2_;
  90   }
  91   new_options.target_file_size_base = 256 * 1024;
  92   new_options.num_levels = level2_;
  93   new_options.max_bytes_for_level_base = 150 * 1024;
  94   new_options.max_bytes_for_level_multiplier = 4;
  95   ASSERT_OK(OptionChangeMigration(dbname_, old_options, new_options));
  96   Reopen(new_options);
  97
  98   // Wait for compaction to finish and make sure it can reopen
  99   dbfull()->TEST_WaitForFlushMemTable();
 100   dbfull()->TEST_WaitForCompact();
 101   Reopen(new_options);
 102
 103   {
 104     std::unique_ptr<Iterator> it(db_->NewIterator(ReadOptions()));
 105     it->SeekToFirst();
 106     for (std::string key : keys) {
 107       ASSERT_TRUE(it->Valid());
 108       ASSERT_EQ(key, it->key().ToString());
 109       it->Next();
 110     }
 111     ASSERT_TRUE(!it->Valid());
 112   }
 113 }
 114
 115 TEST_P(DBOptionChangeMigrationTests, Migrate2) {
 116   Options old_options = CurrentOptions();
 117   old_options.compaction_style =
 118       static_cast<CompactionStyle>(compaction_style2_);
 119   if (old_options.compaction_style == CompactionStyle::kCompactionStyleLevel) {
 120     old_options.level_compaction_dynamic_level_bytes = is_dynamic2_;
 121   }
 122   old_options.level0_file_num_compaction_trigger = 3;
 123   old_options.write_buffer_size = 64 * 1024;
 124   old_options.target_file_size_base = 128 * 1024;
 125   // Make level target of L1, L2 to be 200KB and 600KB
 126   old_options.num_levels = level2_;
 127   old_options.max_bytes_for_level_multiplier = 3;
 128   old_options.max_bytes_for_level_base = 200 * 1024;
 129
 130   Reopen(old_options);
 131
 132   Random rnd(301);
 133   int key_idx = 0;
 134
 135   // Generate at least 2MB of data
 136   for (int num = 0; num < 20; num++) {
 137     GenerateNewFile(&rnd, &key_idx);
 138   }
 139   dbfull()->TEST_WaitForFlushMemTable();
 140   dbfull()->TEST_WaitForCompact();
 141
 142   // Will make sure exactly those keys are in the DB after migration.
 143   std::set<std::string> keys;
 144   {
 145     std::unique_ptr<Iterator> it(db_->NewIterator(ReadOptions()));
 146     it->SeekToFirst();
 147     for (; it->Valid(); it->Next()) {
 148       keys.insert(it->key().ToString());
 149     }
 150   }
 151
 152   Close();
 153
 154   Options new_options = old_options;
 155   new_options.compaction_style =
 156       static_cast<CompactionStyle>(compaction_style1_);
 157   if (new_options.compaction_style == CompactionStyle::kCompactionStyleLevel) {
 158     new_options.level_compaction_dynamic_level_bytes = is_dynamic1_;
 159   }
 160   new_options.target_file_size_base = 256 * 1024;
 161   new_options.num_levels = level1_;
 162   new_options.max_bytes_for_level_base = 150 * 1024;
 163   new_options.max_bytes_for_level_multiplier = 4;
 164   ASSERT_OK(OptionChangeMigration(dbname_, old_options, new_options));
 165   Reopen(new_options);
 166   // Wait for compaction to finish and make sure it can reopen
 167   dbfull()->TEST_WaitForFlushMemTable();
 168   dbfull()->TEST_WaitForCompact();
 169   Reopen(new_options);
 170
 171   {
 172     std::unique_ptr<Iterator> it(db_->NewIterator(ReadOptions()));
 173     it->SeekToFirst();
 174     for (std::string key : keys) {
 175       ASSERT_TRUE(it->Valid());
 176       ASSERT_EQ(key, it->key().ToString());
 177       it->Next();
 178     }
 179     ASSERT_TRUE(!it->Valid());
 180   }
 181 }
 182
 183 TEST_P(DBOptionChangeMigrationTests, Migrate3) {
 184   Options old_options = CurrentOptions();
 185   old_options.compaction_style =
 186       static_cast<CompactionStyle>(compaction_style1_);
 187   if (old_options.compaction_style == CompactionStyle::kCompactionStyleLevel) {
 188     old_options.level_compaction_dynamic_level_bytes = is_dynamic1_;
 189   }
 190
 191   old_options.level0_file_num_compaction_trigger = 3;
 192   old_options.write_buffer_size = 64 * 1024;
 193   old_options.target_file_size_base = 128 * 1024;
 194   // Make level target of L1, L2 to be 200KB and 600KB
 195   old_options.num_levels = level1_;
 196   old_options.max_bytes_for_level_multiplier = 3;
 197   old_options.max_bytes_for_level_base = 200 * 1024;
 198
 199   Reopen(old_options);
 200   Random rnd(301);
 201   for (int num = 0; num < 20; num++) {
 202     for (int i = 0; i < 50; i++) {
 203       ASSERT_OK(Put(Key(num * 100 + i), RandomString(&rnd, 900)));
 204     }
 205     Flush();
 206     dbfull()->TEST_WaitForCompact();
 207     if (num == 9) {
 208       // Issue a full compaction to generate some zero-out files
 209       CompactRangeOptions cro;
 210       cro.bottommost_level_compaction = BottommostLevelCompaction::kForce;
 211       dbfull()->CompactRange(cro, nullptr, nullptr);
 212     }
 213   }
 214   dbfull()->TEST_WaitForFlushMemTable();
 215   dbfull()->TEST_WaitForCompact();
 216
 217   // Will make sure exactly those keys are in the DB after migration.
 218   std::set<std::string> keys;
 219   {
 220     std::unique_ptr<Iterator> it(db_->NewIterator(ReadOptions()));
 221     it->SeekToFirst();
 222     for (; it->Valid(); it->Next()) {
 223       keys.insert(it->key().ToString());
 224     }
 225   }
 226   Close();
 227
 228   Options new_options = old_options;
 229   new_options.compaction_style =
 230       static_cast<CompactionStyle>(compaction_style2_);
 231   if (new_options.compaction_style == CompactionStyle::kCompactionStyleLevel) {
 232     new_options.level_compaction_dynamic_level_bytes = is_dynamic2_;
 233   }
 234   new_options.target_file_size_base = 256 * 1024;
 235   new_options.num_levels = level2_;
 236   new_options.max_bytes_for_level_base = 150 * 1024;
 237   new_options.max_bytes_for_level_multiplier = 4;
 238   ASSERT_OK(OptionChangeMigration(dbname_, old_options, new_options));
 239   Reopen(new_options);
 240
 241   // Wait for compaction to finish and make sure it can reopen
 242   dbfull()->TEST_WaitForFlushMemTable();
 243   dbfull()->TEST_WaitForCompact();
 244   Reopen(new_options);
 245
 246   {
 247     std::unique_ptr<Iterator> it(db_->NewIterator(ReadOptions()));
 248     it->SeekToFirst();
 249     for (std::string key : keys) {
 250       ASSERT_TRUE(it->Valid());
 251       ASSERT_EQ(key, it->key().ToString());
 252       it->Next();
 253     }
 254     ASSERT_TRUE(!it->Valid());
 255   }
 256 }
 257
 258 TEST_P(DBOptionChangeMigrationTests, Migrate4) {
 259   Options old_options = CurrentOptions();
 260   old_options.compaction_style =
 261       static_cast<CompactionStyle>(compaction_style2_);
 262   if (old_options.compaction_style == CompactionStyle::kCompactionStyleLevel) {
 263     old_options.level_compaction_dynamic_level_bytes = is_dynamic2_;
 264   }
 265   old_options.level0_file_num_compaction_trigger = 3;
 266   old_options.write_buffer_size = 64 * 1024;
 267   old_options.target_file_size_base = 128 * 1024;
 268   // Make level target of L1, L2 to be 200KB and 600KB
 269   old_options.num_levels = level2_;
 270   old_options.max_bytes_for_level_multiplier = 3;
 271   old_options.max_bytes_for_level_base = 200 * 1024;
 272
 273   Reopen(old_options);
 274   Random rnd(301);
 275   for (int num = 0; num < 20; num++) {
 276     for (int i = 0; i < 50; i++) {
 277       ASSERT_OK(Put(Key(num * 100 + i), RandomString(&rnd, 900)));
 278     }
 279     Flush();
 280     dbfull()->TEST_WaitForCompact();
 281     if (num == 9) {
 282       // Issue a full compaction to generate some zero-out files
 283       CompactRangeOptions cro;
 284       cro.bottommost_level_compaction = BottommostLevelCompaction::kForce;
 285       dbfull()->CompactRange(cro, nullptr, nullptr);
 286     }
 287   }
 288   dbfull()->TEST_WaitForFlushMemTable();
 289   dbfull()->TEST_WaitForCompact();
 290
 291   // Will make sure exactly those keys are in the DB after migration.
 292   std::set<std::string> keys;
 293   {
 294     std::unique_ptr<Iterator> it(db_->NewIterator(ReadOptions()));
 295     it->SeekToFirst();
 296     for (; it->Valid(); it->Next()) {
 297       keys.insert(it->key().ToString());
 298     }
 299   }
 300
 301   Close();
 302
 303   Options new_options = old_options;
 304   new_options.compaction_style =
 305       static_cast<CompactionStyle>(compaction_style1_);
 306   if (new_options.compaction_style == CompactionStyle::kCompactionStyleLevel) {
 307     new_options.level_compaction_dynamic_level_bytes = is_dynamic1_;
 308   }
 309   new_options.target_file_size_base = 256 * 1024;
 310   new_options.num_levels = level1_;
 311   new_options.max_bytes_for_level_base = 150 * 1024;
 312   new_options.max_bytes_for_level_multiplier = 4;
 313   ASSERT_OK(OptionChangeMigration(dbname_, old_options, new_options));
 314   Reopen(new_options);
 315   // Wait for compaction to finish and make sure it can reopen
 316   dbfull()->TEST_WaitForFlushMemTable();
 317   dbfull()->TEST_WaitForCompact();
 318   Reopen(new_options);
 319
 320   {
 321     std::unique_ptr<Iterator> it(db_->NewIterator(ReadOptions()));
 322     it->SeekToFirst();
 323     for (std::string key : keys) {
 324       ASSERT_TRUE(it->Valid());
 325       ASSERT_EQ(key, it->key().ToString());
 326       it->Next();
 327     }
 328     ASSERT_TRUE(!it->Valid());
 329   }
 330 }
 331
 332 INSTANTIATE_TEST_CASE_P(
 333     DBOptionChangeMigrationTests, DBOptionChangeMigrationTests,
 334     ::testing::Values(std::make_tuple(3, 0, false, 4, 0, false),
 335                       std::make_tuple(3, 0, true, 4, 0, true),
 336                       std::make_tuple(3, 0, true, 4, 0, false),
 337                       std::make_tuple(3, 0, false, 4, 0, true),
 338                       std::make_tuple(3, 1, false, 4, 1, false),
 339                       std::make_tuple(1, 1, false, 4, 1, false),
 340                       std::make_tuple(3, 0, false, 4, 1, false),
 341                       std::make_tuple(3, 0, false, 1, 1, false),
 342                       std::make_tuple(3, 0, true, 4, 1, false),
 343                       std::make_tuple(3, 0, true, 1, 1, false),
 344                       std::make_tuple(1, 1, false, 4, 0, false),
 345                       std::make_tuple(4, 0, false, 1, 2, false),
 346                       std::make_tuple(3, 0, true, 2, 2, false),
 347                       std::make_tuple(3, 1, false, 3, 2, false),
 348                       std::make_tuple(1, 1, false, 4, 2, false)));
 349
 350 class DBOptionChangeMigrationTest : public DBTestBase {
 351  public:
 352   DBOptionChangeMigrationTest()
 353       : DBTestBase("/db_option_change_migration_test2") {}
 354 };
 355
 356 TEST_F(DBOptionChangeMigrationTest, CompactedSrcToUniversal) {
 357   Options old_options = CurrentOptions();
 358   old_options.compaction_style = CompactionStyle::kCompactionStyleLevel;
 359   old_options.max_compaction_bytes = 200 * 1024;
 360   old_options.level_compaction_dynamic_level_bytes = false;
 361   old_options.level0_file_num_compaction_trigger = 3;
 362   old_options.write_buffer_size = 64 * 1024;
 363   old_options.target_file_size_base = 128 * 1024;
 364   // Make level target of L1, L2 to be 200KB and 600KB
 365   old_options.num_levels = 4;
 366   old_options.max_bytes_for_level_multiplier = 3;
 367   old_options.max_bytes_for_level_base = 200 * 1024;
 368
 369   Reopen(old_options);
 370   Random rnd(301);
 371   for (int num = 0; num < 20; num++) {
 372     for (int i = 0; i < 50; i++) {
 373       ASSERT_OK(Put(Key(num * 100 + i), RandomString(&rnd, 900)));
 374     }
 375   }
 376   Flush();
 377   CompactRangeOptions cro;
 378   cro.bottommost_level_compaction = BottommostLevelCompaction::kForce;
 379   dbfull()->CompactRange(cro, nullptr, nullptr);
 380
 381   // Will make sure exactly those keys are in the DB after migration.
 382   std::set<std::string> keys;
 383   {
 384     std::unique_ptr<Iterator> it(db_->NewIterator(ReadOptions()));
 385     it->SeekToFirst();
 386     for (; it->Valid(); it->Next()) {
 387       keys.insert(it->key().ToString());
 388     }
 389   }
 390
 391   Close();
 392
 393   Options new_options = old_options;
 394   new_options.compaction_style = CompactionStyle::kCompactionStyleUniversal;
 395   new_options.target_file_size_base = 256 * 1024;
 396   new_options.num_levels = 1;
 397   new_options.max_bytes_for_level_base = 150 * 1024;
 398   new_options.max_bytes_for_level_multiplier = 4;
 399   ASSERT_OK(OptionChangeMigration(dbname_, old_options, new_options));
 400   Reopen(new_options);
 401   // Wait for compaction to finish and make sure it can reopen
 402   dbfull()->TEST_WaitForFlushMemTable();
 403   dbfull()->TEST_WaitForCompact();
 404   Reopen(new_options);
 405
 406   {
 407     std::unique_ptr<Iterator> it(db_->NewIterator(ReadOptions()));
 408     it->SeekToFirst();
 409     for (std::string key : keys) {
 410       ASSERT_TRUE(it->Valid());
 411       ASSERT_EQ(key, it->key().ToString());
 412       it->Next();
 413     }
 414     ASSERT_TRUE(!it->Valid());
 415   }
 416 }
 417
 418 #endif  // ROCKSDB_LITE
 419 }  // namespace rocksdb
 420
 421 int main(int argc, char** argv) {
 422   rocksdb::port::InstallStackTraceHandler();
 423   ::testing::InitGoogleTest(&argc, argv);
 424   return RUN_ALL_TESTS();
 425 }