ceph/src/rocksdb/table/block_based/block_based_table_builder.h

   1 //  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
   2 //  This source code is licensed under both the GPLv2 (found in the
   3 //  COPYING file in the root directory) and Apache 2.0 License
   4 //  (found in the LICENSE.Apache file in the root directory).
   5 //
   6 // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
   7 // Use of this source code is governed by a BSD-style license that can be
   8 // found in the LICENSE file. See the AUTHORS file for names of contributors.
   9
  10 #pragma once
  11 #include <stdint.h>
  12 #include <limits>
  13 #include <string>
  14 #include <utility>
  15 #include <vector>
  16
  17 #include "db/version_edit.h"
  18 #include "rocksdb/flush_block_policy.h"
  19 #include "rocksdb/listener.h"
  20 #include "rocksdb/options.h"
  21 #include "rocksdb/status.h"
  22 #include "table/meta_blocks.h"
  23 #include "table/table_builder.h"
  24 #include "util/compression.h"
  25
  26 namespace ROCKSDB_NAMESPACE {
  27
  28 class BlockBuilder;
  29 class BlockHandle;
  30 class WritableFile;
  31 struct BlockBasedTableOptions;
  32
  33 extern const uint64_t kBlockBasedTableMagicNumber;
  34 extern const uint64_t kLegacyBlockBasedTableMagicNumber;
  35
  36 class BlockBasedTableBuilder : public TableBuilder {
  37  public:
  38   // Create a builder that will store the contents of the table it is
  39   // building in *file.  Does not close the file.  It is up to the
  40   // caller to close the file after calling Finish().
  41   BlockBasedTableBuilder(
  42       const ImmutableCFOptions& ioptions, const MutableCFOptions& moptions,
  43       const BlockBasedTableOptions& table_options,
  44       const InternalKeyComparator& internal_comparator,
  45       const std::vector<std::unique_ptr<IntTblPropCollectorFactory>>*
  46           int_tbl_prop_collector_factories,
  47       uint32_t column_family_id, WritableFileWriter* file,
  48       const CompressionType compression_type,
  49       const uint64_t sample_for_compression,
  50       const CompressionOptions& compression_opts, const bool skip_filters,
  51       const std::string& column_family_name, const int level_at_creation,
  52       const uint64_t creation_time = 0, const uint64_t oldest_key_time = 0,
  53       const uint64_t target_file_size = 0,
  54       const uint64_t file_creation_time = 0, const std::string& db_id = "",
  55       const std::string& db_session_id = "");
  56
  57   // No copying allowed
  58   BlockBasedTableBuilder(const BlockBasedTableBuilder&) = delete;
  59   BlockBasedTableBuilder& operator=(const BlockBasedTableBuilder&) = delete;
  60
  61   // REQUIRES: Either Finish() or Abandon() has been called.
  62   ~BlockBasedTableBuilder();
  63
  64   // Add key,value to the table being constructed.
  65   // REQUIRES: key is after any previously added key according to comparator.
  66   // REQUIRES: Finish(), Abandon() have not been called
  67   void Add(const Slice& key, const Slice& value) override;
  68
  69   // Return non-ok iff some error has been detected.
  70   Status status() const override;
  71
  72   // Return non-ok iff some error happens during IO.
  73   IOStatus io_status() const override;
  74
  75   // Finish building the table.  Stops using the file passed to the
  76   // constructor after this function returns.
  77   // REQUIRES: Finish(), Abandon() have not been called
  78   Status Finish() override;
  79
  80   // Indicate that the contents of this builder should be abandoned.  Stops
  81   // using the file passed to the constructor after this function returns.
  82   // If the caller is not going to call Finish(), it must call Abandon()
  83   // before destroying this builder.
  84   // REQUIRES: Finish(), Abandon() have not been called
  85   void Abandon() override;
  86
  87   // Number of calls to Add() so far.
  88   uint64_t NumEntries() const override;
  89
  90   bool IsEmpty() const override;
  91
  92   // Size of the file generated so far.  If invoked after a successful
  93   // Finish() call, returns the size of the final generated file.
  94   uint64_t FileSize() const override;
  95
  96   // Estimated size of the file generated so far. This is used when
  97   // FileSize() cannot estimate final SST size, e.g. parallel compression
  98   // is enabled.
  99   uint64_t EstimatedFileSize() const override;
 100
 101   bool NeedCompact() const override;
 102
 103   // Get table properties
 104   TableProperties GetTableProperties() const override;
 105
 106   // Get file checksum
 107   std::string GetFileChecksum() const override;
 108
 109   // Get file checksum function name
 110   const char* GetFileChecksumFuncName() const override;
 111
 112  private:
 113   bool ok() const { return status().ok(); }
 114
 115   // Transition state from buffered to unbuffered. See `Rep::State` API comment
 116   // for details of the states.
 117   // REQUIRES: `rep_->state == kBuffered`
 118   void EnterUnbuffered();
 119
 120   // Call block's Finish() method
 121   // and then write the compressed block contents to file.
 122   void WriteBlock(BlockBuilder* block, BlockHandle* handle, bool is_data_block);
 123
 124   // Compress and write block content to the file.
 125   void WriteBlock(const Slice& block_contents, BlockHandle* handle,
 126                   bool is_data_block);
 127   // Directly write data to the file.
 128   void WriteRawBlock(const Slice& data, CompressionType, BlockHandle* handle,
 129                      bool is_data_block = false);
 130   Status InsertBlockInCache(const Slice& block_contents,
 131                             const CompressionType type,
 132                             const BlockHandle* handle);
 133
 134   void WriteFilterBlock(MetaIndexBuilder* meta_index_builder);
 135   void WriteIndexBlock(MetaIndexBuilder* meta_index_builder,
 136                        BlockHandle* index_block_handle);
 137   void WritePropertiesBlock(MetaIndexBuilder* meta_index_builder);
 138   void WriteCompressionDictBlock(MetaIndexBuilder* meta_index_builder);
 139   void WriteRangeDelBlock(MetaIndexBuilder* meta_index_builder);
 140   void WriteFooter(BlockHandle& metaindex_block_handle,
 141                    BlockHandle& index_block_handle);
 142
 143   struct Rep;
 144   class BlockBasedTablePropertiesCollectorFactory;
 145   class BlockBasedTablePropertiesCollector;
 146   Rep* rep_;
 147
 148   struct ParallelCompressionRep;
 149
 150   // Advanced operation: flush any buffered key/value pairs to file.
 151   // Can be used to ensure that two adjacent entries never live in
 152   // the same data block.  Most clients should not need to use this method.
 153   // REQUIRES: Finish(), Abandon() have not been called
 154   void Flush();
 155
 156   // Some compression libraries fail when the raw size is bigger than int. If
 157   // uncompressed size is bigger than kCompressionSizeLimit, don't compress it
 158   const uint64_t kCompressionSizeLimit = std::numeric_limits<int>::max();
 159
 160   // Get blocks from mem-table walking thread, compress them and
 161   // pass them to the write thread. Used in parallel compression mode only
 162   void BGWorkCompression(const CompressionContext& compression_ctx,
 163                          UncompressionContext* verify_ctx);
 164
 165   // Given raw block content, try to compress it and return result and
 166   // compression type
 167   void CompressAndVerifyBlock(const Slice& raw_block_contents,
 168                               bool is_data_block,
 169                               const CompressionContext& compression_ctx,
 170                               UncompressionContext* verify_ctx,
 171                               std::string* compressed_output,
 172                               Slice* result_block_contents,
 173                               CompressionType* result_compression_type,
 174                               Status* out_status);
 175
 176   // Get compressed blocks from BGWorkCompression and write them into SST
 177   void BGWorkWriteRawBlock();
 178
 179   // Initialize parallel compression context and
 180   // start BGWorkCompression and BGWorkWriteRawBlock threads
 181   void StartParallelCompression();
 182
 183   // Stop BGWorkCompression and BGWorkWriteRawBlock threads
 184   void StopParallelCompression();
 185 };
 186
 187 Slice CompressBlock(const Slice& raw, const CompressionInfo& info,
 188                     CompressionType* type, uint32_t format_version,
 189                     bool do_sample, std::string* compressed_output,
 190                     std::string* sampled_output_fast,
 191                     std::string* sampled_output_slow);
 192
 193 }  // namespace ROCKSDB_NAMESPACE