ceph/src/rocksdb/include/rocksdb/table_properties.h

   1 // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file. See the AUTHORS file for names of contributors.
   4 #pragma once
   5
   6 #include <stdint.h>
   7 #include <map>
   8 #include <string>
   9 #include "rocksdb/status.h"
  10 #include "rocksdb/types.h"
  11
  12 namespace rocksdb {
  13
  14 // -- Table Properties
  15 // Other than basic table properties, each table may also have the user
  16 // collected properties.
  17 // The value of the user-collected properties are encoded as raw bytes --
  18 // users have to interpret these values by themselves.
  19 // Note: To do prefix seek/scan in `UserCollectedProperties`, you can do
  20 // something similar to:
  21 //
  22 // UserCollectedProperties props = ...;
  23 // for (auto pos = props.lower_bound(prefix);
  24 //      pos != props.end() && pos->first.compare(0, prefix.size(), prefix) == 0;
  25 //      ++pos) {
  26 //   ...
  27 // }
  28 typedef std::map<std::string, std::string> UserCollectedProperties;
  29
  30 // table properties' human-readable names in the property block.
  31 struct TablePropertiesNames {
  32   static const std::string kDataSize;
  33   static const std::string kIndexSize;
  34   static const std::string kIndexPartitions;
  35   static const std::string kTopLevelIndexSize;
  36   static const std::string kIndexKeyIsUserKey;
  37   static const std::string kIndexValueIsDeltaEncoded;
  38   static const std::string kFilterSize;
  39   static const std::string kRawKeySize;
  40   static const std::string kRawValueSize;
  41   static const std::string kNumDataBlocks;
  42   static const std::string kNumEntries;
  43   static const std::string kDeletedKeys;
  44   static const std::string kMergeOperands;
  45   static const std::string kNumRangeDeletions;
  46   static const std::string kFormatVersion;
  47   static const std::string kFixedKeyLen;
  48   static const std::string kFilterPolicy;
  49   static const std::string kColumnFamilyName;
  50   static const std::string kColumnFamilyId;
  51   static const std::string kComparator;
  52   static const std::string kMergeOperator;
  53   static const std::string kPrefixExtractorName;
  54   static const std::string kPropertyCollectors;
  55   static const std::string kCompression;
  56   static const std::string kCompressionOptions;
  57   static const std::string kCreationTime;
  58   static const std::string kOldestKeyTime;
  59 };
  60
  61 extern const std::string kPropertiesBlock;
  62 extern const std::string kCompressionDictBlock;
  63 extern const std::string kRangeDelBlock;
  64
  65 // `TablePropertiesCollector` provides the mechanism for users to collect
  66 // their own properties that they are interested in. This class is essentially
  67 // a collection of callback functions that will be invoked during table
  68 // building. It is constructed with TablePropertiesCollectorFactory. The methods
  69 // don't need to be thread-safe, as we will create exactly one
  70 // TablePropertiesCollector object per table and then call it sequentially
  71 class TablePropertiesCollector {
  72  public:
  73   virtual ~TablePropertiesCollector() {}
  74
  75   // DEPRECATE User defined collector should implement AddUserKey(), though
  76   //           this old function still works for backward compatible reason.
  77   // Add() will be called when a new key/value pair is inserted into the table.
  78   // @params key    the user key that is inserted into the table.
  79   // @params value  the value that is inserted into the table.
  80   virtual Status Add(const Slice& /*key*/, const Slice& /*value*/) {
  81     return Status::InvalidArgument(
  82         "TablePropertiesCollector::Add() deprecated.");
  83   }
  84
  85   // AddUserKey() will be called when a new key/value pair is inserted into the
  86   // table.
  87   // @params key    the user key that is inserted into the table.
  88   // @params value  the value that is inserted into the table.
  89   virtual Status AddUserKey(const Slice& key, const Slice& value,
  90                             EntryType /*type*/, SequenceNumber /*seq*/,
  91                             uint64_t /*file_size*/) {
  92     // For backwards-compatibility.
  93     return Add(key, value);
  94   }
  95
  96   // Called after each new block is cut
  97   virtual void BlockAdd(uint64_t /* blockRawBytes */,
  98                         uint64_t /* blockCompressedBytesFast */,
  99                         uint64_t /* blockCompressedBytesSlow */) {
 100     // Nothing to do here. Callback registers can override.
 101     return;
 102   }
 103
 104   // Finish() will be called when a table has already been built and is ready
 105   // for writing the properties block.
 106   // @params properties  User will add their collected statistics to
 107   // `properties`.
 108   virtual Status Finish(UserCollectedProperties* properties) = 0;
 109
 110   // Return the human-readable properties, where the key is property name and
 111   // the value is the human-readable form of value.
 112   virtual UserCollectedProperties GetReadableProperties() const = 0;
 113
 114   // The name of the properties collector can be used for debugging purpose.
 115   virtual const char* Name() const = 0;
 116
 117   // EXPERIMENTAL Return whether the output file should be further compacted
 118   virtual bool NeedCompact() const { return false; }
 119 };
 120
 121 // Constructs TablePropertiesCollector. Internals create a new
 122 // TablePropertiesCollector for each new table
 123 class TablePropertiesCollectorFactory {
 124  public:
 125   struct Context {
 126     uint32_t column_family_id;
 127     static const uint32_t kUnknownColumnFamily;
 128   };
 129
 130   virtual ~TablePropertiesCollectorFactory() {}
 131   // has to be thread-safe
 132   virtual TablePropertiesCollector* CreateTablePropertiesCollector(
 133       TablePropertiesCollectorFactory::Context context) = 0;
 134
 135   // The name of the properties collector can be used for debugging purpose.
 136   virtual const char* Name() const = 0;
 137 };
 138
 139 // TableProperties contains a bunch of read-only properties of its associated
 140 // table.
 141 struct TableProperties {
 142  public:
 143   // the total size of all data blocks.
 144   uint64_t data_size = 0;
 145   // the size of index block.
 146   uint64_t index_size = 0;
 147   // Total number of index partitions if kTwoLevelIndexSearch is used
 148   uint64_t index_partitions = 0;
 149   // Size of the top-level index if kTwoLevelIndexSearch is used
 150   uint64_t top_level_index_size = 0;
 151   // Whether the index key is user key. Otherwise it includes 8 byte of sequence
 152   // number added by internal key format.
 153   uint64_t index_key_is_user_key = 0;
 154   // Whether delta encoding is used to encode the index values.
 155   uint64_t index_value_is_delta_encoded = 0;
 156   // the size of filter block.
 157   uint64_t filter_size = 0;
 158   // total raw key size
 159   uint64_t raw_key_size = 0;
 160   // total raw value size
 161   uint64_t raw_value_size = 0;
 162   // the number of blocks in this table
 163   uint64_t num_data_blocks = 0;
 164   // the number of entries in this table
 165   uint64_t num_entries = 0;
 166   // the number of deletions in the table
 167   uint64_t num_deletions = 0;
 168   // the number of merge operands in the table
 169   uint64_t num_merge_operands = 0;
 170   // the number of range deletions in this table
 171   uint64_t num_range_deletions = 0;
 172   // format version, reserved for backward compatibility
 173   uint64_t format_version = 0;
 174   // If 0, key is variable length. Otherwise number of bytes for each key.
 175   uint64_t fixed_key_len = 0;
 176   // ID of column family for this SST file, corresponding to the CF identified
 177   // by column_family_name.
 178   uint64_t column_family_id =
 179       rocksdb::TablePropertiesCollectorFactory::Context::kUnknownColumnFamily;
 180   // The time when the SST file was created.
 181   // Since SST files are immutable, this is equivalent to last modified time.
 182   uint64_t creation_time = 0;
 183   // Timestamp of the earliest key. 0 means unknown.
 184   uint64_t oldest_key_time = 0;
 185
 186   // Name of the column family with which this SST file is associated.
 187   // If column family is unknown, `column_family_name` will be an empty string.
 188   std::string column_family_name;
 189
 190   // The name of the filter policy used in this table.
 191   // If no filter policy is used, `filter_policy_name` will be an empty string.
 192   std::string filter_policy_name;
 193
 194   // The name of the comparator used in this table.
 195   std::string comparator_name;
 196
 197   // The name of the merge operator used in this table.
 198   // If no merge operator is used, `merge_operator_name` will be "nullptr".
 199   std::string merge_operator_name;
 200
 201   // The name of the prefix extractor used in this table
 202   // If no prefix extractor is used, `prefix_extractor_name` will be "nullptr".
 203   std::string prefix_extractor_name;
 204
 205   // The names of the property collectors factories used in this table
 206   // separated by commas
 207   // {collector_name[1]},{collector_name[2]},{collector_name[3]} ..
 208   std::string property_collectors_names;
 209
 210   // The compression algo used to compress the SST files.
 211   std::string compression_name;
 212
 213   // Compression options used to compress the SST files.
 214   std::string compression_options;
 215
 216   // user collected properties
 217   UserCollectedProperties user_collected_properties;
 218   UserCollectedProperties readable_properties;
 219
 220   // The offset of the value of each property in the file.
 221   std::map<std::string, uint64_t> properties_offsets;
 222
 223   // convert this object to a human readable form
 224   //   @prop_delim: delimiter for each property.
 225   std::string ToString(const std::string& prop_delim = "; ",
 226                        const std::string& kv_delim = "=") const;
 227
 228   // Aggregate the numerical member variables of the specified
 229   // TableProperties.
 230   void Add(const TableProperties& tp);
 231 };
 232
 233 // Extra properties
 234 // Below is a list of non-basic properties that are collected by database
 235 // itself. Especially some properties regarding to the internal keys (which
 236 // is unknown to `table`).
 237 //
 238 // DEPRECATED: these properties now belong as TableProperties members. Please
 239 // use TableProperties::num_deletions and TableProperties::num_merge_operands,
 240 // respectively.
 241 extern uint64_t GetDeletedKeys(const UserCollectedProperties& props);
 242 extern uint64_t GetMergeOperands(const UserCollectedProperties& props,
 243                                  bool* property_present);
 244
 245 }  // namespace rocksdb