ceph/src/rocksdb/utilities/option_change_migration/option_change_migration.cc

   1 //  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
   2 //  This source code is licensed under both the GPLv2 (found in the
   3 //  COPYING file in the root directory) and Apache 2.0 License
   4 //  (found in the LICENSE.Apache file in the root directory).
   5
   6 #include "rocksdb/utilities/option_change_migration.h"
   7
   8 #ifndef ROCKSDB_LITE
   9 #include "rocksdb/db.h"
  10
  11 namespace ROCKSDB_NAMESPACE {
  12 namespace {
  13 // Return a version of Options `opts` that allow us to open/write into a DB
  14 // without triggering an automatic compaction or stalling. This is guaranteed
  15 // by disabling automatic compactions and using huge values for stalling
  16 // triggers.
  17 Options GetNoCompactionOptions(const Options& opts) {
  18   Options ret_opts = opts;
  19   ret_opts.disable_auto_compactions = true;
  20   ret_opts.level0_slowdown_writes_trigger = 999999;
  21   ret_opts.level0_stop_writes_trigger = 999999;
  22   ret_opts.soft_pending_compaction_bytes_limit = 0;
  23   ret_opts.hard_pending_compaction_bytes_limit = 0;
  24   return ret_opts;
  25 }
  26
  27 Status OpenDb(const Options& options, const std::string& dbname,
  28               std::unique_ptr<DB>* db) {
  29   db->reset();
  30   DB* tmpdb;
  31   Status s = DB::Open(options, dbname, &tmpdb);
  32   if (s.ok()) {
  33     db->reset(tmpdb);
  34   }
  35   return s;
  36 }
  37
  38 // l0_file_size specifies size of file on L0. Files will be range partitioned
  39 // after a full compaction so they are likely qualified to put on L0. If
  40 // left as 0, the files are compacted in a single file and put to L0. Otherwise,
  41 // will try to compact the files as size l0_file_size.
  42 Status CompactToLevel(const Options& options, const std::string& dbname,
  43                       int dest_level, uint64_t l0_file_size, bool need_reopen) {
  44   std::unique_ptr<DB> db;
  45   Options no_compact_opts = GetNoCompactionOptions(options);
  46   if (dest_level == 0) {
  47     if (l0_file_size == 0) {
  48       // Single file.
  49       l0_file_size = 999999999999999;
  50     }
  51     // L0 has strict sequenceID requirements to files to it. It's safer
  52     // to only put one compacted file to there.
  53     // This is only used for converting to universal compaction with
  54     // only one level. In this case, compacting to one file is also
  55     // optimal.
  56     no_compact_opts.target_file_size_base = l0_file_size;
  57     no_compact_opts.max_compaction_bytes = l0_file_size;
  58   }
  59   Status s = OpenDb(no_compact_opts, dbname, &db);
  60   if (!s.ok()) {
  61     return s;
  62   }
  63   CompactRangeOptions cro;
  64   cro.change_level = true;
  65   cro.target_level = dest_level;
  66   if (dest_level == 0) {
  67     // cannot use kForceOptimized because the compaction is expected to
  68     // generate one output file
  69     cro.bottommost_level_compaction = BottommostLevelCompaction::kForce;
  70   }
  71   s = db->CompactRange(cro, nullptr, nullptr);
  72
  73   if (s.ok() && need_reopen) {
  74     // Need to restart DB to rewrite the manifest file.
  75     // In order to open a DB with specific num_levels, the manifest file should
  76     // contain no record that mentiones any level beyond num_levels. Issuing a
  77     // full compaction will move all the data to a level not exceeding
  78     // num_levels, but the manifest may still contain previous record mentioning
  79     // a higher level. Reopening the DB will force the manifest to be rewritten
  80     // so that those records will be cleared.
  81     db.reset();
  82     s = OpenDb(no_compact_opts, dbname, &db);
  83   }
  84   return s;
  85 }
  86
  87 Status MigrateToUniversal(std::string dbname, const Options& old_opts,
  88                           const Options& new_opts) {
  89   if (old_opts.num_levels <= new_opts.num_levels ||
  90       old_opts.compaction_style == CompactionStyle::kCompactionStyleFIFO) {
  91     return Status::OK();
  92   } else {
  93     bool need_compact = false;
  94     {
  95       std::unique_ptr<DB> db;
  96       Options opts = GetNoCompactionOptions(old_opts);
  97       Status s = OpenDb(opts, dbname, &db);
  98       if (!s.ok()) {
  99         return s;
 100       }
 101       ColumnFamilyMetaData metadata;
 102       db->GetColumnFamilyMetaData(&metadata);
 103       if (!metadata.levels.empty() &&
 104           metadata.levels.back().level >= new_opts.num_levels) {
 105         need_compact = true;
 106       }
 107     }
 108     if (need_compact) {
 109       return CompactToLevel(old_opts, dbname, new_opts.num_levels - 1,
 110                             /*l0_file_size=*/0, true);
 111     }
 112     return Status::OK();
 113   }
 114 }
 115
 116 Status MigrateToLevelBase(std::string dbname, const Options& old_opts,
 117                           const Options& new_opts) {
 118   if (!new_opts.level_compaction_dynamic_level_bytes) {
 119     if (old_opts.num_levels == 1) {
 120       return Status::OK();
 121     }
 122     // Compact everything to level 1 to guarantee it can be safely opened.
 123     Options opts = old_opts;
 124     opts.target_file_size_base = new_opts.target_file_size_base;
 125     // Although sometimes we can open the DB with the new option without error,
 126     // We still want to compact the files to avoid the LSM tree to stuck
 127     // in bad shape. For example, if the user changed the level size
 128     // multiplier from 4 to 8, with the same data, we will have fewer
 129     // levels. Unless we issue a full comaction, the LSM tree may stuck
 130     // with more levels than needed and it won't recover automatically.
 131     return CompactToLevel(opts, dbname, 1, /*l0_file_size=*/0, true);
 132   } else {
 133     // Compact everything to the last level to guarantee it can be safely
 134     // opened.
 135     if (old_opts.num_levels == 1) {
 136       return Status::OK();
 137     } else if (new_opts.num_levels > old_opts.num_levels) {
 138       // Dynamic level mode requires data to be put in the last level first.
 139       return CompactToLevel(new_opts, dbname, new_opts.num_levels - 1,
 140                             /*l0_file_size=*/0, false);
 141     } else {
 142       Options opts = old_opts;
 143       opts.target_file_size_base = new_opts.target_file_size_base;
 144       return CompactToLevel(opts, dbname, new_opts.num_levels - 1,
 145                             /*l0_file_size=*/0, true);
 146     }
 147   }
 148 }
 149 }  // namespace
 150
 151 Status OptionChangeMigration(std::string dbname, const Options& old_opts,
 152                              const Options& new_opts) {
 153   if (old_opts.compaction_style == CompactionStyle::kCompactionStyleFIFO) {
 154     // LSM generated by FIFO compaction can be opened by any compaction.
 155     return Status::OK();
 156   } else if (new_opts.compaction_style ==
 157              CompactionStyle::kCompactionStyleUniversal) {
 158     return MigrateToUniversal(dbname, old_opts, new_opts);
 159   } else if (new_opts.compaction_style ==
 160              CompactionStyle::kCompactionStyleLevel) {
 161     return MigrateToLevelBase(dbname, old_opts, new_opts);
 162   } else if (new_opts.compaction_style ==
 163              CompactionStyle::kCompactionStyleFIFO) {
 164     uint64_t l0_file_size = 0;
 165     if (new_opts.compaction_options_fifo.max_table_files_size > 0) {
 166       // Create at least 8 files when max_table_files_size hits, so that the DB
 167       // doesn't just disappear. This in fact violates the FIFO condition, but
 168       // otherwise, the migrated DB is unlikley to be usable.
 169       l0_file_size = new_opts.compaction_options_fifo.max_table_files_size / 8;
 170     }
 171     return CompactToLevel(old_opts, dbname, 0, l0_file_size, true);
 172   } else {
 173     return Status::NotSupported(
 174         "Do not how to migrate to this compaction style");
 175   }
 176 }
 177 }  // namespace ROCKSDB_NAMESPACE
 178 #else
 179 namespace ROCKSDB_NAMESPACE {
 180 Status OptionChangeMigration(std::string /*dbname*/,
 181                              const Options& /*old_opts*/,
 182                              const Options& /*new_opts*/) {
 183   return Status::NotSupported();
 184 }
 185 }  // namespace ROCKSDB_NAMESPACE
 186 #endif  // ROCKSDB_LITE