ceph/src/rocksdb/db/compaction.h

   1 //  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
   2 //  This source code is licensed under both the GPLv2 (found in the
   3 //  COPYING file in the root directory) and Apache 2.0 License
   4 //  (found in the LICENSE.Apache file in the root directory).
   5 //
   6 // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
   7 // Use of this source code is governed by a BSD-style license that can be
   8 // found in the LICENSE file. See the AUTHORS file for names of contributors.
   9
  10 #pragma once
  11 #include "db/version_set.h"
  12 #include "options/cf_options.h"
  13 #include "util/arena.h"
  14 #include "util/autovector.h"
  15
  16 namespace rocksdb {
  17
  18 // The structure that manages compaction input files associated
  19 // with the same physical level.
  20 struct CompactionInputFiles {
  21   int level;
  22   std::vector<FileMetaData*> files;
  23   inline bool empty() const { return files.empty(); }
  24   inline size_t size() const { return files.size(); }
  25   inline void clear() { files.clear(); }
  26   inline FileMetaData* operator[](size_t i) const { return files[i]; }
  27 };
  28
  29 class Version;
  30 class ColumnFamilyData;
  31 class VersionStorageInfo;
  32 class CompactionFilter;
  33
  34 // A Compaction encapsulates information about a compaction.
  35 class Compaction {
  36  public:
  37   Compaction(VersionStorageInfo* input_version,
  38              const ImmutableCFOptions& immutable_cf_options,
  39              const MutableCFOptions& mutable_cf_options,
  40              std::vector<CompactionInputFiles> inputs, int output_level,
  41              uint64_t target_file_size, uint64_t max_compaction_bytes,
  42              uint32_t output_path_id, CompressionType compression,
  43              CompressionOptions compression_opts, uint32_t max_subcompactions,
  44              std::vector<FileMetaData*> grandparents,
  45              bool manual_compaction = false, double score = -1,
  46              bool deletion_compaction = false,
  47              CompactionReason compaction_reason = CompactionReason::kUnknown);
  48
  49   // No copying allowed
  50   Compaction(const Compaction&) = delete;
  51   void operator=(const Compaction&) = delete;
  52
  53   ~Compaction();
  54
  55   // Returns the level associated to the specified compaction input level.
  56   // If compaction_input_level is not specified, then input_level is set to 0.
  57   int level(size_t compaction_input_level = 0) const {
  58     return inputs_[compaction_input_level].level;
  59   }
  60
  61   int start_level() const { return start_level_; }
  62
  63   // Outputs will go to this level
  64   int output_level() const { return output_level_; }
  65
  66   // Returns the number of input levels in this compaction.
  67   size_t num_input_levels() const { return inputs_.size(); }
  68
  69   // Return the object that holds the edits to the descriptor done
  70   // by this compaction.
  71   VersionEdit* edit() { return &edit_; }
  72
  73   // Returns the number of input files associated to the specified
  74   // compaction input level.
  75   // The function will return 0 if when "compaction_input_level" < 0
  76   // or "compaction_input_level" >= "num_input_levels()".
  77   size_t num_input_files(size_t compaction_input_level) const {
  78     if (compaction_input_level < inputs_.size()) {
  79       return inputs_[compaction_input_level].size();
  80     }
  81     return 0;
  82   }
  83
  84   // Returns input version of the compaction
  85   Version* input_version() const { return input_version_; }
  86
  87   // Returns the ColumnFamilyData associated with the compaction.
  88   ColumnFamilyData* column_family_data() const { return cfd_; }
  89
  90   // Returns the file meta data of the 'i'th input file at the
  91   // specified compaction input level.
  92   // REQUIREMENT: "compaction_input_level" must be >= 0 and
  93   //              < "input_levels()"
  94   FileMetaData* input(size_t compaction_input_level, size_t i) const {
  95     assert(compaction_input_level < inputs_.size());
  96     return inputs_[compaction_input_level][i];
  97   }
  98
  99   // Returns the list of file meta data of the specified compaction
 100   // input level.
 101   // REQUIREMENT: "compaction_input_level" must be >= 0 and
 102   //              < "input_levels()"
 103   const std::vector<FileMetaData*>* inputs(
 104       size_t compaction_input_level) const {
 105     assert(compaction_input_level < inputs_.size());
 106     return &inputs_[compaction_input_level].files;
 107   }
 108
 109   const std::vector<CompactionInputFiles>* inputs() { return &inputs_; }
 110
 111   // Returns the LevelFilesBrief of the specified compaction input level.
 112   const LevelFilesBrief* input_levels(size_t compaction_input_level) const {
 113     return &input_levels_[compaction_input_level];
 114   }
 115
 116   // Maximum size of files to build during this compaction.
 117   uint64_t max_output_file_size() const { return max_output_file_size_; }
 118
 119   // What compression for output
 120   CompressionType output_compression() const { return output_compression_; }
 121
 122   // What compression options for output
 123   CompressionOptions output_compression_opts() const {
 124     return output_compression_opts_;
 125   }
 126
 127   // Whether need to write output file to second DB path.
 128   uint32_t output_path_id() const { return output_path_id_; }
 129
 130   // Is this a trivial compaction that can be implemented by just
 131   // moving a single input file to the next level (no merging or splitting)
 132   bool IsTrivialMove() const;
 133
 134   // If true, then the compaction can be done by simply deleting input files.
 135   bool deletion_compaction() const { return deletion_compaction_; }
 136
 137   // Add all inputs to this compaction as delete operations to *edit.
 138   void AddInputDeletions(VersionEdit* edit);
 139
 140   // Returns true if the available information we have guarantees that
 141   // the input "user_key" does not exist in any level beyond "output_level()".
 142   bool KeyNotExistsBeyondOutputLevel(const Slice& user_key,
 143                                      std::vector<size_t>* level_ptrs) const;
 144
 145   // Clear all files to indicate that they are not being compacted
 146   // Delete this compaction from the list of running compactions.
 147   //
 148   // Requirement: DB mutex held
 149   void ReleaseCompactionFiles(Status status);
 150
 151   // Returns the summary of the compaction in "output" with maximum "len"
 152   // in bytes.  The caller is responsible for the memory management of
 153   // "output".
 154   void Summary(char* output, int len);
 155
 156   // Return the score that was used to pick this compaction run.
 157   double score() const { return score_; }
 158
 159   // Is this compaction creating a file in the bottom most level?
 160   bool bottommost_level() const { return bottommost_level_; }
 161
 162   // Does this compaction include all sst files?
 163   bool is_full_compaction() const { return is_full_compaction_; }
 164
 165   // Was this compaction triggered manually by the client?
 166   bool is_manual_compaction() const { return is_manual_compaction_; }
 167
 168   // Used when allow_trivial_move option is set in
 169   // Universal compaction. If all the input files are
 170   // non overlapping, then is_trivial_move_ variable
 171   // will be set true, else false
 172   void set_is_trivial_move(bool trivial_move) {
 173     is_trivial_move_ = trivial_move;
 174   }
 175
 176   // Used when allow_trivial_move option is set in
 177   // Universal compaction. Returns true, if the input files
 178   // are non-overlapping and can be trivially moved.
 179   bool is_trivial_move() const { return is_trivial_move_; }
 180
 181   // How many total levels are there?
 182   int number_levels() const { return number_levels_; }
 183
 184   // Return the ImmutableCFOptions that should be used throughout the compaction
 185   // procedure
 186   const ImmutableCFOptions* immutable_cf_options() const {
 187     return &immutable_cf_options_;
 188   }
 189
 190   // Return the MutableCFOptions that should be used throughout the compaction
 191   // procedure
 192   const MutableCFOptions* mutable_cf_options() const {
 193     return &mutable_cf_options_;
 194   }
 195
 196   // Returns the size in bytes that the output file should be preallocated to.
 197   // In level compaction, that is max_file_size_. In universal compaction, that
 198   // is the sum of all input file sizes.
 199   uint64_t OutputFilePreallocationSize() const;
 200
 201   void SetInputVersion(Version* input_version);
 202
 203   struct InputLevelSummaryBuffer {
 204     char buffer[128];
 205   };
 206
 207   const char* InputLevelSummary(InputLevelSummaryBuffer* scratch) const;
 208
 209   uint64_t CalculateTotalInputSize() const;
 210
 211   // In case of compaction error, reset the nextIndex that is used
 212   // to pick up the next file to be compacted from files_by_size_
 213   void ResetNextCompactionIndex();
 214
 215   // Create a CompactionFilter from compaction_filter_factory
 216   std::unique_ptr<CompactionFilter> CreateCompactionFilter() const;
 217
 218   // Is the input level corresponding to output_level_ empty?
 219   bool IsOutputLevelEmpty() const;
 220
 221   // Should this compaction be broken up into smaller ones run in parallel?
 222   bool ShouldFormSubcompactions() const;
 223
 224   // test function to validate the functionality of IsBottommostLevel()
 225   // function -- determines if compaction with inputs and storage is bottommost
 226   static bool TEST_IsBottommostLevel(
 227       int output_level, VersionStorageInfo* vstorage,
 228       const std::vector<CompactionInputFiles>& inputs);
 229
 230   TablePropertiesCollection GetOutputTableProperties() const {
 231     return output_table_properties_;
 232   }
 233
 234   void SetOutputTableProperties(TablePropertiesCollection tp) {
 235     output_table_properties_ = std::move(tp);
 236   }
 237
 238   Slice GetSmallestUserKey() const { return smallest_user_key_; }
 239
 240   Slice GetLargestUserKey() const { return largest_user_key_; }
 241
 242   int GetInputBaseLevel() const;
 243
 244   CompactionReason compaction_reason() { return compaction_reason_; }
 245
 246   const std::vector<FileMetaData*>& grandparents() const {
 247     return grandparents_;
 248   }
 249
 250   uint64_t max_compaction_bytes() const { return max_compaction_bytes_; }
 251
 252   uint32_t max_subcompactions() const { return max_subcompactions_; }
 253
 254   uint64_t MaxInputFileCreationTime() const;
 255
 256  private:
 257   // mark (or clear) all files that are being compacted
 258   void MarkFilesBeingCompacted(bool mark_as_compacted);
 259
 260   // get the smallest and largest key present in files to be compacted
 261   static void GetBoundaryKeys(VersionStorageInfo* vstorage,
 262                               const std::vector<CompactionInputFiles>& inputs,
 263                               Slice* smallest_key, Slice* largest_key);
 264
 265   // helper function to determine if compaction with inputs and storage is
 266   // bottommost
 267   static bool IsBottommostLevel(
 268       int output_level, VersionStorageInfo* vstorage,
 269       const std::vector<CompactionInputFiles>& inputs);
 270
 271   static bool IsFullCompaction(VersionStorageInfo* vstorage,
 272                                const std::vector<CompactionInputFiles>& inputs);
 273
 274   VersionStorageInfo* input_vstorage_;
 275
 276   const int start_level_;    // the lowest level to be compacted
 277   const int output_level_;  // levels to which output files are stored
 278   uint64_t max_output_file_size_;
 279   uint64_t max_compaction_bytes_;
 280   uint32_t max_subcompactions_;
 281   const ImmutableCFOptions immutable_cf_options_;
 282   const MutableCFOptions mutable_cf_options_;
 283   Version* input_version_;
 284   VersionEdit edit_;
 285   const int number_levels_;
 286   ColumnFamilyData* cfd_;
 287   Arena arena_;          // Arena used to allocate space for file_levels_
 288
 289   const uint32_t output_path_id_;
 290   CompressionType output_compression_;
 291   CompressionOptions output_compression_opts_;
 292   // If true, then the comaction can be done by simply deleting input files.
 293   const bool deletion_compaction_;
 294
 295   // Compaction input files organized by level. Constant after construction
 296   const std::vector<CompactionInputFiles> inputs_;
 297
 298   // A copy of inputs_, organized more closely in memory
 299   autovector<LevelFilesBrief, 2> input_levels_;
 300
 301   // State used to check for number of overlapping grandparent files
 302   // (grandparent == "output_level_ + 1")
 303   std::vector<FileMetaData*> grandparents_;
 304   const double score_;         // score that was used to pick this compaction.
 305
 306   // Is this compaction creating a file in the bottom most level?
 307   const bool bottommost_level_;
 308   // Does this compaction include all sst files?
 309   const bool is_full_compaction_;
 310
 311   // Is this compaction requested by the client?
 312   const bool is_manual_compaction_;
 313
 314   // True if we can do trivial move in Universal multi level
 315   // compaction
 316   bool is_trivial_move_;
 317
 318   // Does input compression match the output compression?
 319   bool InputCompressionMatchesOutput() const;
 320
 321   // table properties of output files
 322   TablePropertiesCollection output_table_properties_;
 323
 324   // smallest user keys in compaction
 325   Slice smallest_user_key_;
 326
 327   // largest user keys in compaction
 328   Slice largest_user_key_;
 329
 330   // Reason for compaction
 331   CompactionReason compaction_reason_;
 332 };
 333
 334 // Utility function
 335 extern uint64_t TotalFileSize(const std::vector<FileMetaData*>& files);
 336
 337 }  // namespace rocksdb