ceph/src/rocksdb/db/compaction_picker_fifo.cc

   1 //  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
   2 //  This source code is licensed under both the GPLv2 (found in the
   3 //  COPYING file in the root directory) and Apache 2.0 License
   4 //  (found in the LICENSE.Apache file in the root directory).
   5 //
   6 // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
   7 // Use of this source code is governed by a BSD-style license that can be
   8 // found in the LICENSE file. See the AUTHORS file for names of contributors.
   9
  10 #include "db/compaction_picker_fifo.h"
  11 #ifndef ROCKSDB_LITE
  12
  13 #ifndef __STDC_FORMAT_MACROS
  14 #define __STDC_FORMAT_MACROS
  15 #endif
  16
  17 #include <inttypes.h>
  18 #include <string>
  19 #include <vector>
  20 #include "db/column_family.h"
  21 #include "util/log_buffer.h"
  22 #include "util/string_util.h"
  23
  24 namespace rocksdb {
  25 namespace {
  26 uint64_t GetTotalFilesSize(const std::vector<FileMetaData*>& files) {
  27   uint64_t total_size = 0;
  28   for (const auto& f : files) {
  29     total_size += f->fd.file_size;
  30   }
  31   return total_size;
  32 }
  33 }  // anonymous namespace
  34
  35 bool FIFOCompactionPicker::NeedsCompaction(
  36     const VersionStorageInfo* vstorage) const {
  37   const int kLevel0 = 0;
  38   return vstorage->CompactionScore(kLevel0) >= 1;
  39 }
  40
  41 Compaction* FIFOCompactionPicker::PickTTLCompaction(
  42     const std::string& cf_name, const MutableCFOptions& mutable_cf_options,
  43     VersionStorageInfo* vstorage, LogBuffer* log_buffer) {
  44   assert(mutable_cf_options.ttl > 0);
  45
  46   const int kLevel0 = 0;
  47   const std::vector<FileMetaData*>& level_files = vstorage->LevelFiles(kLevel0);
  48   uint64_t total_size = GetTotalFilesSize(level_files);
  49
  50   int64_t _current_time;
  51   auto status = ioptions_.env->GetCurrentTime(&_current_time);
  52   if (!status.ok()) {
  53     ROCKS_LOG_BUFFER(log_buffer,
  54                      "[%s] FIFO compaction: Couldn't get current time: %s. "
  55                      "Not doing compactions based on TTL. ",
  56                      cf_name.c_str(), status.ToString().c_str());
  57     return nullptr;
  58   }
  59   const uint64_t current_time = static_cast<uint64_t>(_current_time);
  60
  61   std::vector<CompactionInputFiles> inputs;
  62   inputs.emplace_back();
  63   inputs[0].level = 0;
  64
  65   // avoid underflow
  66   if (current_time > mutable_cf_options.ttl) {
  67     for (auto ritr = level_files.rbegin(); ritr != level_files.rend(); ++ritr) {
  68       auto f = *ritr;
  69       if (f->fd.table_reader != nullptr &&
  70           f->fd.table_reader->GetTableProperties() != nullptr) {
  71         auto creation_time =
  72             f->fd.table_reader->GetTableProperties()->creation_time;
  73         if (creation_time == 0 ||
  74             creation_time >= (current_time - mutable_cf_options.ttl)) {
  75           break;
  76         }
  77         total_size -= f->compensated_file_size;
  78         inputs[0].files.push_back(f);
  79       }
  80     }
  81   }
  82
  83   // Return a nullptr and proceed to size-based FIFO compaction if:
  84   // 1. there are no files older than ttl OR
  85   // 2. there are a few files older than ttl, but deleting them will not bring
  86   //    the total size to be less than max_table_files_size threshold.
  87   if (inputs[0].files.empty() ||
  88       total_size >
  89           mutable_cf_options.compaction_options_fifo.max_table_files_size) {
  90     return nullptr;
  91   }
  92
  93   for (const auto& f : inputs[0].files) {
  94     ROCKS_LOG_BUFFER(log_buffer,
  95                      "[%s] FIFO compaction: picking file %" PRIu64
  96                      " with creation time %" PRIu64 " for deletion",
  97                      cf_name.c_str(), f->fd.GetNumber(),
  98                      f->fd.table_reader->GetTableProperties()->creation_time);
  99   }
 100
 101   Compaction* c = new Compaction(
 102       vstorage, ioptions_, mutable_cf_options, std::move(inputs), 0, 0, 0, 0,
 103       kNoCompression, ioptions_.compression_opts, /* max_subcompactions */ 0,
 104       {}, /* is manual */ false, vstorage->CompactionScore(0),
 105       /* is deletion compaction */ true, CompactionReason::kFIFOTtl);
 106   return c;
 107 }
 108
 109 Compaction* FIFOCompactionPicker::PickSizeCompaction(
 110     const std::string& cf_name, const MutableCFOptions& mutable_cf_options,
 111     VersionStorageInfo* vstorage, LogBuffer* log_buffer) {
 112   const int kLevel0 = 0;
 113   const std::vector<FileMetaData*>& level_files = vstorage->LevelFiles(kLevel0);
 114   uint64_t total_size = GetTotalFilesSize(level_files);
 115
 116   if (total_size <=
 117           mutable_cf_options.compaction_options_fifo.max_table_files_size ||
 118       level_files.size() == 0) {
 119     // total size not exceeded
 120     if (mutable_cf_options.compaction_options_fifo.allow_compaction &&
 121         level_files.size() > 0) {
 122       CompactionInputFiles comp_inputs;
 123       // try to prevent same files from being compacted multiple times, which
 124       // could produce large files that may never TTL-expire. Achieve this by
 125       // disallowing compactions with files larger than memtable (inflate its
 126       // size by 10% to account for uncompressed L0 files that may have size
 127       // slightly greater than memtable size limit).
 128       size_t max_compact_bytes_per_del_file =
 129           static_cast<size_t>(MultiplyCheckOverflow(
 130               static_cast<uint64_t>(mutable_cf_options.write_buffer_size),
 131               1.1));
 132       if (FindIntraL0Compaction(
 133               level_files,
 134               mutable_cf_options
 135                   .level0_file_num_compaction_trigger /* min_files_to_compact */
 136               ,
 137               max_compact_bytes_per_del_file, &comp_inputs)) {
 138         Compaction* c = new Compaction(
 139             vstorage, ioptions_, mutable_cf_options, {comp_inputs}, 0,
 140             16 * 1024 * 1024 /* output file size limit */,
 141             0 /* max compaction bytes, not applicable */,
 142             0 /* output path ID */, mutable_cf_options.compression,
 143             ioptions_.compression_opts, 0 /* max_subcompactions */, {},
 144             /* is manual */ false, vstorage->CompactionScore(0),
 145             /* is deletion compaction */ false,
 146             CompactionReason::kFIFOReduceNumFiles);
 147         return c;
 148       }
 149     }
 150
 151     ROCKS_LOG_BUFFER(
 152         log_buffer,
 153         "[%s] FIFO compaction: nothing to do. Total size %" PRIu64
 154         ", max size %" PRIu64 "\n",
 155         cf_name.c_str(), total_size,
 156         mutable_cf_options.compaction_options_fifo.max_table_files_size);
 157     return nullptr;
 158   }
 159
 160   if (!level0_compactions_in_progress_.empty()) {
 161     ROCKS_LOG_BUFFER(
 162         log_buffer,
 163         "[%s] FIFO compaction: Already executing compaction. No need "
 164         "to run parallel compactions since compactions are very fast",
 165         cf_name.c_str());
 166     return nullptr;
 167   }
 168
 169   std::vector<CompactionInputFiles> inputs;
 170   inputs.emplace_back();
 171   inputs[0].level = 0;
 172
 173   for (auto ritr = level_files.rbegin(); ritr != level_files.rend(); ++ritr) {
 174     auto f = *ritr;
 175     total_size -= f->compensated_file_size;
 176     inputs[0].files.push_back(f);
 177     char tmp_fsize[16];
 178     AppendHumanBytes(f->fd.GetFileSize(), tmp_fsize, sizeof(tmp_fsize));
 179     ROCKS_LOG_BUFFER(log_buffer,
 180                      "[%s] FIFO compaction: picking file %" PRIu64
 181                      " with size %s for deletion",
 182                      cf_name.c_str(), f->fd.GetNumber(), tmp_fsize);
 183     if (total_size <=
 184         mutable_cf_options.compaction_options_fifo.max_table_files_size) {
 185       break;
 186     }
 187   }
 188
 189   Compaction* c = new Compaction(
 190       vstorage, ioptions_, mutable_cf_options, std::move(inputs), 0, 0, 0, 0,
 191       kNoCompression, ioptions_.compression_opts, /* max_subcompactions */ 0,
 192       {}, /* is manual */ false, vstorage->CompactionScore(0),
 193       /* is deletion compaction */ true, CompactionReason::kFIFOMaxSize);
 194   return c;
 195 }
 196
 197 Compaction* FIFOCompactionPicker::PickCompaction(
 198     const std::string& cf_name, const MutableCFOptions& mutable_cf_options,
 199     VersionStorageInfo* vstorage, LogBuffer* log_buffer) {
 200   assert(vstorage->num_levels() == 1);
 201
 202   Compaction* c = nullptr;
 203   if (mutable_cf_options.ttl > 0) {
 204     c = PickTTLCompaction(cf_name, mutable_cf_options, vstorage, log_buffer);
 205   }
 206   if (c == nullptr) {
 207     c = PickSizeCompaction(cf_name, mutable_cf_options, vstorage, log_buffer);
 208   }
 209   RegisterCompaction(c);
 210   return c;
 211 }
 212
 213 Compaction* FIFOCompactionPicker::CompactRange(
 214     const std::string& cf_name, const MutableCFOptions& mutable_cf_options,
 215     VersionStorageInfo* vstorage, int input_level, int output_level,
 216     uint32_t /*output_path_id*/, uint32_t /*max_subcompactions*/,
 217     const InternalKey* /*begin*/, const InternalKey* /*end*/,
 218     InternalKey** compaction_end, bool* /*manual_conflict*/) {
 219 #ifdef NDEBUG
 220   (void)input_level;
 221   (void)output_level;
 222 #endif
 223   assert(input_level == 0);
 224   assert(output_level == 0);
 225   *compaction_end = nullptr;
 226   LogBuffer log_buffer(InfoLogLevel::INFO_LEVEL, ioptions_.info_log);
 227   Compaction* c =
 228       PickCompaction(cf_name, mutable_cf_options, vstorage, &log_buffer);
 229   log_buffer.FlushBufferToLog();
 230   return c;
 231 }
 232
 233 }  // namespace rocksdb
 234 #endif  // !ROCKSDB_LITE