ceph/src/rocksdb/options/cf_options.h

   1 // Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
   2 //  This source code is licensed under both the GPLv2 (found in the
   3 //  COPYING file in the root directory) and Apache 2.0 License
   4 //  (found in the LICENSE.Apache file in the root directory).
   5
   6 #pragma once
   7
   8 #include <string>
   9 #include <vector>
  10
  11 #include "db/dbformat.h"
  12 #include "options/db_options.h"
  13 #include "rocksdb/options.h"
  14 #include "util/compression.h"
  15
  16 namespace ROCKSDB_NAMESPACE {
  17
  18 // ImmutableCFOptions is a data struct used by RocksDB internal. It contains a
  19 // subset of Options that should not be changed during the entire lifetime
  20 // of DB. Raw pointers defined in this struct do not have ownership to the data
  21 // they point to. Options contains std::shared_ptr to these data.
  22 struct ImmutableCFOptions {
  23   static const char* kName() { return "ImmutableCFOptions"; }
  24   explicit ImmutableCFOptions(const Options& options);
  25
  26   ImmutableCFOptions(const ImmutableDBOptions& db_options,
  27                      const ColumnFamilyOptions& cf_options);
  28
  29   CompactionStyle compaction_style;
  30
  31   CompactionPri compaction_pri;
  32
  33   const Comparator* user_comparator;
  34   InternalKeyComparator internal_comparator;
  35
  36   MergeOperator* merge_operator;
  37
  38   const CompactionFilter* compaction_filter;
  39
  40   CompactionFilterFactory* compaction_filter_factory;
  41
  42   int min_write_buffer_number_to_merge;
  43
  44   int max_write_buffer_number_to_maintain;
  45
  46   int64_t max_write_buffer_size_to_maintain;
  47
  48   bool inplace_update_support;
  49
  50   UpdateStatus (*inplace_callback)(char* existing_value,
  51                                    uint32_t* existing_value_size,
  52                                    Slice delta_value,
  53                                    std::string* merged_value);
  54
  55   Logger* info_log;
  56
  57   Statistics* statistics;
  58
  59   RateLimiter* rate_limiter;
  60
  61   InfoLogLevel info_log_level;
  62
  63   Env* env;
  64
  65   FileSystem* fs;
  66
  67   // Allow the OS to mmap file for reading sst tables. Default: false
  68   bool allow_mmap_reads;
  69
  70   // Allow the OS to mmap file for writing. Default: false
  71   bool allow_mmap_writes;
  72
  73   std::vector<DbPath> db_paths;
  74
  75   MemTableRepFactory* memtable_factory;
  76
  77   TableFactory* table_factory;
  78
  79   Options::TablePropertiesCollectorFactories
  80       table_properties_collector_factories;
  81
  82   bool advise_random_on_open;
  83
  84   // This options is required by PlainTableReader. May need to move it
  85   // to PlainTableOptions just like bloom_bits_per_key
  86   uint32_t bloom_locality;
  87
  88   bool purge_redundant_kvs_while_flush;
  89
  90   bool use_fsync;
  91
  92   std::vector<CompressionType> compression_per_level;
  93
  94   bool level_compaction_dynamic_level_bytes;
  95
  96   Options::AccessHint access_hint_on_compaction_start;
  97
  98   bool new_table_reader_for_compaction_inputs;
  99
 100   int num_levels;
 101
 102   bool optimize_filters_for_hits;
 103
 104   bool force_consistency_checks;
 105
 106   bool allow_ingest_behind;
 107
 108   bool preserve_deletes;
 109
 110   // A vector of EventListeners which callback functions will be called
 111   // when specific RocksDB event happens.
 112   std::vector<std::shared_ptr<EventListener>> listeners;
 113
 114   std::shared_ptr<Cache> row_cache;
 115
 116   const SliceTransform* memtable_insert_with_hint_prefix_extractor;
 117
 118   std::vector<DbPath> cf_paths;
 119
 120   std::shared_ptr<ConcurrentTaskLimiter> compaction_thread_limiter;
 121
 122   FileChecksumGenFactory* file_checksum_gen_factory;
 123
 124   std::shared_ptr<SstPartitionerFactory> sst_partitioner_factory;
 125
 126   bool allow_data_in_errors;
 127
 128   std::string db_host_id;
 129 };
 130
 131 struct MutableCFOptions {
 132   static const char* kName() { return "MutableCFOptions"; }
 133   explicit MutableCFOptions(const ColumnFamilyOptions& options)
 134       : write_buffer_size(options.write_buffer_size),
 135         max_write_buffer_number(options.max_write_buffer_number),
 136         arena_block_size(options.arena_block_size),
 137         memtable_prefix_bloom_size_ratio(
 138             options.memtable_prefix_bloom_size_ratio),
 139         memtable_whole_key_filtering(options.memtable_whole_key_filtering),
 140         memtable_huge_page_size(options.memtable_huge_page_size),
 141         max_successive_merges(options.max_successive_merges),
 142         inplace_update_num_locks(options.inplace_update_num_locks),
 143         prefix_extractor(options.prefix_extractor),
 144         disable_auto_compactions(options.disable_auto_compactions),
 145         soft_pending_compaction_bytes_limit(
 146             options.soft_pending_compaction_bytes_limit),
 147         hard_pending_compaction_bytes_limit(
 148             options.hard_pending_compaction_bytes_limit),
 149         level0_file_num_compaction_trigger(
 150             options.level0_file_num_compaction_trigger),
 151         level0_slowdown_writes_trigger(options.level0_slowdown_writes_trigger),
 152         level0_stop_writes_trigger(options.level0_stop_writes_trigger),
 153         max_compaction_bytes(options.max_compaction_bytes),
 154         target_file_size_base(options.target_file_size_base),
 155         target_file_size_multiplier(options.target_file_size_multiplier),
 156         max_bytes_for_level_base(options.max_bytes_for_level_base),
 157         max_bytes_for_level_multiplier(options.max_bytes_for_level_multiplier),
 158         ttl(options.ttl),
 159         periodic_compaction_seconds(options.periodic_compaction_seconds),
 160         max_bytes_for_level_multiplier_additional(
 161             options.max_bytes_for_level_multiplier_additional),
 162         compaction_options_fifo(options.compaction_options_fifo),
 163         compaction_options_universal(options.compaction_options_universal),
 164         enable_blob_files(options.enable_blob_files),
 165         min_blob_size(options.min_blob_size),
 166         blob_file_size(options.blob_file_size),
 167         blob_compression_type(options.blob_compression_type),
 168         enable_blob_garbage_collection(options.enable_blob_garbage_collection),
 169         blob_garbage_collection_age_cutoff(
 170             options.blob_garbage_collection_age_cutoff),
 171         max_sequential_skip_in_iterations(
 172             options.max_sequential_skip_in_iterations),
 173         check_flush_compaction_key_order(
 174             options.check_flush_compaction_key_order),
 175         paranoid_file_checks(options.paranoid_file_checks),
 176         report_bg_io_stats(options.report_bg_io_stats),
 177         compression(options.compression),
 178         bottommost_compression(options.bottommost_compression),
 179         compression_opts(options.compression_opts),
 180         bottommost_compression_opts(options.bottommost_compression_opts),
 181         sample_for_compression(
 182             options.sample_for_compression) {  // TODO: is 0 fine here?
 183     RefreshDerivedOptions(options.num_levels, options.compaction_style);
 184   }
 185
 186   MutableCFOptions()
 187       : write_buffer_size(0),
 188         max_write_buffer_number(0),
 189         arena_block_size(0),
 190         memtable_prefix_bloom_size_ratio(0),
 191         memtable_whole_key_filtering(false),
 192         memtable_huge_page_size(0),
 193         max_successive_merges(0),
 194         inplace_update_num_locks(0),
 195         prefix_extractor(nullptr),
 196         disable_auto_compactions(false),
 197         soft_pending_compaction_bytes_limit(0),
 198         hard_pending_compaction_bytes_limit(0),
 199         level0_file_num_compaction_trigger(0),
 200         level0_slowdown_writes_trigger(0),
 201         level0_stop_writes_trigger(0),
 202         max_compaction_bytes(0),
 203         target_file_size_base(0),
 204         target_file_size_multiplier(0),
 205         max_bytes_for_level_base(0),
 206         max_bytes_for_level_multiplier(0),
 207         ttl(0),
 208         periodic_compaction_seconds(0),
 209         compaction_options_fifo(),
 210         enable_blob_files(false),
 211         min_blob_size(0),
 212         blob_file_size(0),
 213         blob_compression_type(kNoCompression),
 214         enable_blob_garbage_collection(false),
 215         blob_garbage_collection_age_cutoff(0.0),
 216         max_sequential_skip_in_iterations(0),
 217         check_flush_compaction_key_order(true),
 218         paranoid_file_checks(false),
 219         report_bg_io_stats(false),
 220         compression(Snappy_Supported() ? kSnappyCompression : kNoCompression),
 221         bottommost_compression(kDisableCompressionOption),
 222         sample_for_compression(0) {}
 223
 224   explicit MutableCFOptions(const Options& options);
 225
 226   // Must be called after any change to MutableCFOptions
 227   void RefreshDerivedOptions(int num_levels, CompactionStyle compaction_style);
 228
 229   void RefreshDerivedOptions(const ImmutableCFOptions& ioptions) {
 230     RefreshDerivedOptions(ioptions.num_levels, ioptions.compaction_style);
 231   }
 232
 233   int MaxBytesMultiplerAdditional(int level) const {
 234     if (level >=
 235         static_cast<int>(max_bytes_for_level_multiplier_additional.size())) {
 236       return 1;
 237     }
 238     return max_bytes_for_level_multiplier_additional[level];
 239   }
 240
 241   void Dump(Logger* log) const;
 242
 243   // Memtable related options
 244   size_t write_buffer_size;
 245   int max_write_buffer_number;
 246   size_t arena_block_size;
 247   double memtable_prefix_bloom_size_ratio;
 248   bool memtable_whole_key_filtering;
 249   size_t memtable_huge_page_size;
 250   size_t max_successive_merges;
 251   size_t inplace_update_num_locks;
 252   std::shared_ptr<const SliceTransform> prefix_extractor;
 253
 254   // Compaction related options
 255   bool disable_auto_compactions;
 256   uint64_t soft_pending_compaction_bytes_limit;
 257   uint64_t hard_pending_compaction_bytes_limit;
 258   int level0_file_num_compaction_trigger;
 259   int level0_slowdown_writes_trigger;
 260   int level0_stop_writes_trigger;
 261   uint64_t max_compaction_bytes;
 262   uint64_t target_file_size_base;
 263   int target_file_size_multiplier;
 264   uint64_t max_bytes_for_level_base;
 265   double max_bytes_for_level_multiplier;
 266   uint64_t ttl;
 267   uint64_t periodic_compaction_seconds;
 268   std::vector<int> max_bytes_for_level_multiplier_additional;
 269   CompactionOptionsFIFO compaction_options_fifo;
 270   CompactionOptionsUniversal compaction_options_universal;
 271
 272   // Blob file related options
 273   bool enable_blob_files;
 274   uint64_t min_blob_size;
 275   uint64_t blob_file_size;
 276   CompressionType blob_compression_type;
 277   bool enable_blob_garbage_collection;
 278   double blob_garbage_collection_age_cutoff;
 279
 280   // Misc options
 281   uint64_t max_sequential_skip_in_iterations;
 282   bool check_flush_compaction_key_order;
 283   bool paranoid_file_checks;
 284   bool report_bg_io_stats;
 285   CompressionType compression;
 286   CompressionType bottommost_compression;
 287   CompressionOptions compression_opts;
 288   CompressionOptions bottommost_compression_opts;
 289
 290   uint64_t sample_for_compression;
 291
 292   // Derived options
 293   // Per-level target file size.
 294   std::vector<uint64_t> max_file_size;
 295 };
 296
 297 uint64_t MultiplyCheckOverflow(uint64_t op1, double op2);
 298
 299 // Get the max file size in a given level.
 300 uint64_t MaxFileSizeForLevel(const MutableCFOptions& cf_options,
 301     int level, CompactionStyle compaction_style, int base_level = 1,
 302     bool level_compaction_dynamic_level_bytes = false);
 303
 304 // Get the max size of an L0 file for which we will pin its meta-blocks when
 305 // `pin_l0_filter_and_index_blocks_in_cache` is set.
 306 size_t MaxFileSizeForL0MetaPin(const MutableCFOptions& cf_options);
 307
 308 }  // namespace ROCKSDB_NAMESPACE