// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
-// This source code is licensed under the BSD-style license found in the
-// LICENSE file in the root directory of this source tree. An additional grant
-// of patent rights can be found in the PATENTS file in the same directory.
+// This source code is licensed under both the GPLv2 (found in the
+// COPYING file in the root directory) and Apache 2.0 License
+// (found in the LICENSE.Apache file in the root directory).
//
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file. See the AUTHORS file for names of contributors.
-#include <inttypes.h>
#include <stdio.h>
#include <algorithm>
#include <string>
#include <vector>
+#include "cache/lru_cache.h"
#include "db/dbformat.h"
#include "db/memtable.h"
#include "db/write_batch_internal.h"
#include "table/block_based_table_factory.h"
#include "table/block_based_table_reader.h"
#include "table/block_builder.h"
+#include "table/block_fetcher.h"
#include "table/format.h"
#include "table/get_context.h"
#include "table/internal_iterator.h"
public:
const char* Name() const { return ""; }
- Status Finish(UserCollectedProperties* properties) { return Status::OK(); }
+ Status Finish(UserCollectedProperties* /*properties*/) {
+ return Status::OK();
+ }
- Status Add(const Slice& user_key, const Slice& value) { return Status::OK(); }
+ Status Add(const Slice& /*user_key*/, const Slice& /*value*/) {
+ return Status::OK();
+ }
virtual UserCollectedProperties GetReadableProperties() const {
return UserCollectedProperties{};
: public TablePropertiesCollectorFactory {
public:
virtual TablePropertiesCollector* CreateTablePropertiesCollector(
- TablePropertiesCollectorFactory::Context context) {
+ TablePropertiesCollectorFactory::Context /*context*/) {
return new DummyPropertiesCollector();
}
const char* Name() const { return "DummyPropertiesCollector1"; }
: public TablePropertiesCollectorFactory {
public:
virtual TablePropertiesCollector* CreateTablePropertiesCollector(
- TablePropertiesCollectorFactory::Context context) {
+ TablePropertiesCollectorFactory::Context /*context*/) {
return new DummyPropertiesCollector();
}
const char* Name() const { return "DummyPropertiesCollector2"; }
// been added so far. Returns the keys in sorted order in "*keys"
// and stores the key/value pairs in "*kvmap"
void Finish(const Options& options, const ImmutableCFOptions& ioptions,
+ const MutableCFOptions& moptions,
const BlockBasedTableOptions& table_options,
const InternalKeyComparator& internal_comparator,
std::vector<std::string>* keys, stl_wrappers::KVMap* kvmap) {
keys->push_back(kv.first);
}
data_.clear();
- Status s = FinishImpl(options, ioptions, table_options,
+ Status s = FinishImpl(options, ioptions, moptions, table_options,
internal_comparator, *kvmap);
ASSERT_TRUE(s.ok()) << s.ToString();
}
// Construct the data structure from the data in "data"
virtual Status FinishImpl(const Options& options,
const ImmutableCFOptions& ioptions,
+ const MutableCFOptions& moptions,
const BlockBasedTableOptions& table_options,
const InternalKeyComparator& internal_comparator,
const stl_wrappers::KVMap& data) = 0;
- virtual InternalIterator* NewIterator() const = 0;
+ virtual InternalIterator* NewIterator(
+ const SliceTransform* prefix_extractor = nullptr) const = 0;
virtual const stl_wrappers::KVMap& data() { return data_; }
~BlockConstructor() {
delete block_;
}
- virtual Status FinishImpl(const Options& options,
- const ImmutableCFOptions& ioptions,
- const BlockBasedTableOptions& table_options,
- const InternalKeyComparator& internal_comparator,
- const stl_wrappers::KVMap& kv_map) override {
+ virtual Status FinishImpl(
+ const Options& /*options*/, const ImmutableCFOptions& /*ioptions*/,
+ const MutableCFOptions& /*moptions*/,
+ const BlockBasedTableOptions& table_options,
+ const InternalKeyComparator& /*internal_comparator*/,
+ const stl_wrappers::KVMap& kv_map) override {
delete block_;
block_ = nullptr;
BlockBuilder builder(table_options.block_restart_interval);
block_ = new Block(std::move(contents), kDisableGlobalSequenceNumber);
return Status::OK();
}
- virtual InternalIterator* NewIterator() const override {
- return block_->NewIterator(comparator_);
+ virtual InternalIterator* NewIterator(
+ const SliceTransform* /*prefix_extractor*/) const override {
+ return block_->NewIterator<DataBlockIter>(comparator_, comparator_);
}
private:
delete iter_;
}
}
- virtual bool Valid() const override { return iter_->Valid(); }
+ virtual bool Valid() const override { return iter_->Valid() && status_.ok(); }
virtual void Seek(const Slice& target) override {
ParsedInternalKey ikey(target, kMaxSequenceNumber, kTypeValue);
std::string encoded;
class TableConstructor: public Constructor {
public:
explicit TableConstructor(const Comparator* cmp,
- bool convert_to_internal_key = false)
+ bool convert_to_internal_key = false,
+ int level = -1)
: Constructor(cmp),
- convert_to_internal_key_(convert_to_internal_key) {}
+ convert_to_internal_key_(convert_to_internal_key),
+ level_(level) {}
~TableConstructor() { Reset(); }
virtual Status FinishImpl(const Options& options,
const ImmutableCFOptions& ioptions,
- const BlockBasedTableOptions& table_options,
+ const MutableCFOptions& moptions,
+ const BlockBasedTableOptions& /*table_options*/,
const InternalKeyComparator& internal_comparator,
const stl_wrappers::KVMap& kv_map) override {
Reset();
soptions.use_mmap_reads = ioptions.allow_mmap_reads;
- file_writer_.reset(test::GetWritableFileWriter(new test::StringSink()));
+ file_writer_.reset(test::GetWritableFileWriter(new test::StringSink(),
+ "" /* don't care */));
unique_ptr<TableBuilder> builder;
std::vector<std::unique_ptr<IntTblPropCollectorFactory>>
int_tbl_prop_collector_factories;
std::string column_family_name;
- int unknown_level = -1;
builder.reset(ioptions.table_factory->NewTableBuilder(
- TableBuilderOptions(ioptions, internal_comparator,
- &int_tbl_prop_collector_factories,
- options.compression, CompressionOptions(),
- nullptr /* compression_dict */,
- false /* skip_filters */, column_family_name,
- unknown_level),
+ TableBuilderOptions(
+ ioptions, moptions, internal_comparator,
+ &int_tbl_prop_collector_factories, options.compression,
+ CompressionOptions(), nullptr /* compression_dict */,
+ false /* skip_filters */, column_family_name, level_),
TablePropertiesCollectorFactory::Context::kUnknownColumnFamily,
file_writer_.get()));
file_writer_->Flush();
EXPECT_TRUE(s.ok()) << s.ToString();
- EXPECT_EQ(GetSink()->contents().size(), builder->FileSize());
+ EXPECT_EQ(TEST_GetSink()->contents().size(), builder->FileSize());
// Open the table
uniq_id_ = cur_uniq_id_++;
file_reader_.reset(test::GetRandomAccessFileReader(new test::StringSource(
- GetSink()->contents(), uniq_id_, ioptions.allow_mmap_reads)));
+ TEST_GetSink()->contents(), uniq_id_, ioptions.allow_mmap_reads)));
+ const bool kSkipFilters = true;
+ const bool kImmortal = true;
return ioptions.table_factory->NewTableReader(
- TableReaderOptions(ioptions, soptions, internal_comparator),
- std::move(file_reader_), GetSink()->contents().size(), &table_reader_);
+ TableReaderOptions(ioptions, moptions.prefix_extractor.get(), soptions,
+ internal_comparator, !kSkipFilters, !kImmortal,
+ level_),
+ std::move(file_reader_), TEST_GetSink()->contents().size(),
+ &table_reader_);
}
- virtual InternalIterator* NewIterator() const override {
+ virtual InternalIterator* NewIterator(
+ const SliceTransform* prefix_extractor) const override {
ReadOptions ro;
- InternalIterator* iter = table_reader_->NewIterator(ro);
+ InternalIterator* iter = table_reader_->NewIterator(ro, prefix_extractor);
if (convert_to_internal_key_) {
return new KeyConvertingIterator(iter);
} else {
return table_reader_->ApproximateOffsetOf(key);
}
- virtual Status Reopen(const ImmutableCFOptions& ioptions) {
+ virtual Status Reopen(const ImmutableCFOptions& ioptions,
+ const MutableCFOptions& moptions) {
file_reader_.reset(test::GetRandomAccessFileReader(new test::StringSource(
- GetSink()->contents(), uniq_id_, ioptions.allow_mmap_reads)));
+ TEST_GetSink()->contents(), uniq_id_, ioptions.allow_mmap_reads)));
return ioptions.table_factory->NewTableReader(
- TableReaderOptions(ioptions, soptions, *last_internal_key_),
- std::move(file_reader_), GetSink()->contents().size(), &table_reader_);
+ TableReaderOptions(ioptions, moptions.prefix_extractor.get(), soptions,
+ *last_internal_key_),
+ std::move(file_reader_), TEST_GetSink()->contents().size(),
+ &table_reader_);
}
- virtual TableReader* GetTableReader() {
- return table_reader_.get();
- }
+ virtual TableReader* GetTableReader() { return table_reader_.get(); }
virtual bool AnywayDeleteIterator() const override {
return convert_to_internal_key_;
bool ConvertToInternalKey() { return convert_to_internal_key_; }
+ test::StringSink* TEST_GetSink() {
+ return static_cast<test::StringSink*>(file_writer_->writable_file());
+ }
+
private:
void Reset() {
uniq_id_ = 0;
file_reader_.reset();
}
- test::StringSink* GetSink() {
- return static_cast<test::StringSink*>(file_writer_->writable_file());
- }
-
uint64_t uniq_id_;
unique_ptr<WritableFileWriter> file_writer_;
unique_ptr<RandomAccessFileReader> file_reader_;
unique_ptr<TableReader> table_reader_;
bool convert_to_internal_key_;
+ int level_;
TableConstructor();
ImmutableCFOptions ioptions(options_);
memtable_ =
new MemTable(internal_comparator_, ioptions, MutableCFOptions(options_),
- wb, kMaxSequenceNumber);
+ wb, kMaxSequenceNumber, 0 /* column_family_id */);
memtable_->Ref();
}
~MemTableConstructor() {
delete memtable_->Unref();
}
- virtual Status FinishImpl(const Options&, const ImmutableCFOptions& ioptions,
- const BlockBasedTableOptions& table_options,
- const InternalKeyComparator& internal_comparator,
- const stl_wrappers::KVMap& kv_map) override {
+ virtual Status FinishImpl(
+ const Options&, const ImmutableCFOptions& ioptions,
+ const MutableCFOptions& /*moptions*/,
+ const BlockBasedTableOptions& /*table_options*/,
+ const InternalKeyComparator& /*internal_comparator*/,
+ const stl_wrappers::KVMap& kv_map) override {
delete memtable_->Unref();
ImmutableCFOptions mem_ioptions(ioptions);
memtable_ = new MemTable(internal_comparator_, mem_ioptions,
MutableCFOptions(options_), write_buffer_manager_,
- kMaxSequenceNumber);
+ kMaxSequenceNumber, 0 /* column_family_id */);
memtable_->Ref();
int seq = 1;
for (const auto kv : kv_map) {
}
return Status::OK();
}
- virtual InternalIterator* NewIterator() const override {
+ virtual InternalIterator* NewIterator(
+ const SliceTransform* /*prefix_extractor*/) const override {
return new KeyConvertingIterator(
memtable_->NewIterator(ReadOptions(), &arena_), true);
}
~DBConstructor() {
delete db_;
}
- virtual Status FinishImpl(const Options& options,
- const ImmutableCFOptions& ioptions,
- const BlockBasedTableOptions& table_options,
- const InternalKeyComparator& internal_comparator,
- const stl_wrappers::KVMap& kv_map) override {
+ virtual Status FinishImpl(
+ const Options& /*options*/, const ImmutableCFOptions& /*ioptions*/,
+ const MutableCFOptions& /*moptions*/,
+ const BlockBasedTableOptions& /*table_options*/,
+ const InternalKeyComparator& /*internal_comparator*/,
+ const stl_wrappers::KVMap& kv_map) override {
delete db_;
db_ = nullptr;
NewDB();
return Status::OK();
}
- virtual InternalIterator* NewIterator() const override {
+ virtual InternalIterator* NewIterator(
+ const SliceTransform* /*prefix_extractor*/) const override {
return new InternalIteratorFromIterator(db_->NewIterator(ReadOptions()));
}
private:
void NewDB() {
- std::string name = test::TmpDir() + "/table_testdb";
+ std::string name = test::PerThreadDBPath("table_testdb");
Options options;
options.comparator = comparator_;
return Slice(src.data(), prefix_len_);
}
- virtual bool InDomain(const Slice& src) const override { return true; }
+ virtual bool InDomain(const Slice& /*src*/) const override { return true; }
virtual bool InRange(const Slice& dst) const override {
return (dst.size() <= prefix_len_);
}
+ virtual bool FullLengthEnabled(size_t* /*len*/) const override {
+ return false;
+ }
};
class HarnessTest : public testing::Test {
public:
HarnessTest()
: ioptions_(options_),
+ moptions_(options_),
constructor_(nullptr),
write_buffer_(options_.db_write_buffer_size) {}
break;
}
ioptions_ = ImmutableCFOptions(options_);
+ moptions_ = MutableCFOptions(options_);
}
~HarnessTest() { delete constructor_; }
void Test(Random* rnd) {
std::vector<std::string> keys;
stl_wrappers::KVMap data;
- constructor_->Finish(options_, ioptions_, table_options_,
+ constructor_->Finish(options_, ioptions_, moptions_, table_options_,
*internal_comparator_, &keys, &data);
TestForwardScan(keys, data);
TestRandomAccess(rnd, keys, data);
}
- void TestForwardScan(const std::vector<std::string>& keys,
+ void TestForwardScan(const std::vector<std::string>& /*keys*/,
const stl_wrappers::KVMap& data) {
InternalIterator* iter = constructor_->NewIterator();
ASSERT_TRUE(!iter->Valid());
}
}
- void TestBackwardScan(const std::vector<std::string>& keys,
+ void TestBackwardScan(const std::vector<std::string>& /*keys*/,
const stl_wrappers::KVMap& data) {
InternalIterator* iter = constructor_->NewIterator();
ASSERT_TRUE(!iter->Valid());
// Returns nullptr if not running against a DB
DB* db() const { return constructor_->db(); }
+ void RandomizedHarnessTest(size_t part, size_t total) {
+ std::vector<TestArgs> args = GenerateArgList();
+ assert(part);
+ assert(part <= total);
+ for (size_t i = 0; i < args.size(); i++) {
+ if ((i % total) + 1 != part) {
+ continue;
+ }
+ Init(args[i]);
+ Random rnd(test::RandomSeed() + 5);
+ for (int num_entries = 0; num_entries < 2000;
+ num_entries += (num_entries < 50 ? 1 : 200)) {
+ for (int e = 0; e < num_entries; e++) {
+ std::string v;
+ Add(test::RandomKey(&rnd, rnd.Skewed(4)),
+ test::RandomString(&rnd, rnd.Skewed(5), &v).ToString());
+ }
+ Test(&rnd);
+ }
+ }
+ }
+
private:
Options options_ = Options();
ImmutableCFOptions ioptions_;
+ MutableCFOptions moptions_;
BlockBasedTableOptions table_options_ = BlockBasedTableOptions();
Constructor* constructor_;
WriteBufferManager write_buffer_;
};
class GeneralTableTest : public TableTest {};
-class BlockBasedTableTest : public TableTest {};
+class BlockBasedTableTest
+ : public TableTest,
+ virtual public ::testing::WithParamInterface<uint32_t> {
+ public:
+ BlockBasedTableTest() : format_(GetParam()) {}
+
+ BlockBasedTableOptions GetBlockBasedTableOptions() {
+ BlockBasedTableOptions options;
+ options.format_version = format_;
+ return options;
+ }
+
+ protected:
+ uint64_t IndexUncompressedHelper(bool indexCompress);
+
+ private:
+ uint32_t format_;
+};
class PlainTableTest : public TableTest {};
class TablePropertyTest : public testing::Test {};
+class BBTTailPrefetchTest : public TableTest {};
+
+INSTANTIATE_TEST_CASE_P(FormatDef, BlockBasedTableTest,
+ testing::Values(test::kDefaultFormatVersion));
+INSTANTIATE_TEST_CASE_P(FormatLatest, BlockBasedTableTest,
+ testing::Values(test::kLatestFormatVersion));
// This test serves as the living tutorial for the prefix scan of user collected
// properties.
// This test include all the basic checks except those for index size and block
// size, which will be conducted in separated unit tests.
-TEST_F(BlockBasedTableTest, BasicBlockBasedTableProperties) {
+TEST_P(BlockBasedTableTest, BasicBlockBasedTableProperties) {
TableConstructor c(BytewiseComparator(), true /* convert_to_internal_key_ */);
c.Add("a1", "val1");
stl_wrappers::KVMap kvmap;
Options options;
options.compression = kNoCompression;
- BlockBasedTableOptions table_options;
+ options.statistics = CreateDBStatistics();
+ options.statistics->stats_level_ = StatsLevel::kAll;
+ BlockBasedTableOptions table_options = GetBlockBasedTableOptions();
table_options.block_restart_interval = 1;
options.table_factory.reset(NewBlockBasedTableFactory(table_options));
- const ImmutableCFOptions ioptions(options);
- c.Finish(options, ioptions, table_options,
+ ImmutableCFOptions ioptions(options);
+ MutableCFOptions moptions(options);
+ ioptions.statistics = options.statistics.get();
+ c.Finish(options, ioptions, moptions, table_options,
GetPlainInternalComparator(options.comparator), &keys, &kvmap);
+ ASSERT_EQ(options.statistics->getTickerCount(NUMBER_BLOCK_NOT_COMPRESSED), 0);
auto& props = *c.GetTableReader()->GetTableProperties();
ASSERT_EQ(kvmap.size(), props.num_entries);
c.ResetTableReader();
}
-TEST_F(BlockBasedTableTest, BlockBasedTableProperties2) {
+#ifdef SNAPPY
+uint64_t BlockBasedTableTest::IndexUncompressedHelper(bool compressed) {
+ TableConstructor c(BytewiseComparator(), true /* convert_to_internal_key_ */);
+ constexpr size_t kNumKeys = 10000;
+
+ for (size_t k = 0; k < kNumKeys; ++k) {
+ c.Add("key" + ToString(k), "val" + ToString(k));
+ }
+
+ std::vector<std::string> keys;
+ stl_wrappers::KVMap kvmap;
+ Options options;
+ options.compression = kSnappyCompression;
+ options.statistics = CreateDBStatistics();
+ options.statistics->stats_level_ = StatsLevel::kAll;
+ BlockBasedTableOptions table_options = GetBlockBasedTableOptions();
+ table_options.block_restart_interval = 1;
+ table_options.enable_index_compression = compressed;
+ options.table_factory.reset(NewBlockBasedTableFactory(table_options));
+
+ ImmutableCFOptions ioptions(options);
+ MutableCFOptions moptions(options);
+ ioptions.statistics = options.statistics.get();
+ c.Finish(options, ioptions, moptions, table_options,
+ GetPlainInternalComparator(options.comparator), &keys, &kvmap);
+ c.ResetTableReader();
+ return options.statistics->getTickerCount(NUMBER_BLOCK_COMPRESSED);
+}
+TEST_P(BlockBasedTableTest, IndexUncompressed) {
+ uint64_t tbl1_compressed_cnt = IndexUncompressedHelper(true);
+ uint64_t tbl2_compressed_cnt = IndexUncompressedHelper(false);
+ // tbl1_compressed_cnt should include 1 index block
+ EXPECT_EQ(tbl2_compressed_cnt + 1, tbl1_compressed_cnt);
+}
+#endif // SNAPPY
+
+TEST_P(BlockBasedTableTest, BlockBasedTableProperties2) {
TableConstructor c(&reverse_key_comparator);
std::vector<std::string> keys;
stl_wrappers::KVMap kvmap;
{
Options options;
options.compression = CompressionType::kNoCompression;
- BlockBasedTableOptions table_options;
+ BlockBasedTableOptions table_options = GetBlockBasedTableOptions();
options.table_factory.reset(NewBlockBasedTableFactory(table_options));
const ImmutableCFOptions ioptions(options);
- c.Finish(options, ioptions, table_options,
+ const MutableCFOptions moptions(options);
+ c.Finish(options, ioptions, moptions, table_options,
GetPlainInternalComparator(options.comparator), &keys, &kvmap);
auto& props = *c.GetTableReader()->GetTableProperties();
{
Options options;
- BlockBasedTableOptions table_options;
+ BlockBasedTableOptions table_options = GetBlockBasedTableOptions();
options.table_factory.reset(NewBlockBasedTableFactory(table_options));
options.comparator = &reverse_key_comparator;
options.merge_operator = MergeOperators::CreateUInt64AddOperator();
new DummyPropertiesCollectorFactory2());
const ImmutableCFOptions ioptions(options);
- c.Finish(options, ioptions, table_options,
+ const MutableCFOptions moptions(options);
+ c.Finish(options, ioptions, moptions, table_options,
GetPlainInternalComparator(options.comparator), &keys, &kvmap);
auto& props = *c.GetTableReader()->GetTableProperties();
}
}
-TEST_F(BlockBasedTableTest, RangeDelBlock) {
+TEST_P(BlockBasedTableTest, RangeDelBlock) {
TableConstructor c(BytewiseComparator());
std::vector<std::string> keys = {"1pika", "2chu"};
std::vector<std::string> vals = {"p", "c"};
stl_wrappers::KVMap kvmap;
Options options;
options.compression = kNoCompression;
- BlockBasedTableOptions table_options;
+ BlockBasedTableOptions table_options = GetBlockBasedTableOptions();
table_options.block_restart_interval = 1;
options.table_factory.reset(NewBlockBasedTableFactory(table_options));
const ImmutableCFOptions ioptions(options);
+ const MutableCFOptions moptions(options);
std::unique_ptr<InternalKeyComparator> internal_cmp(
new InternalKeyComparator(options.comparator));
- c.Finish(options, ioptions, table_options, *internal_cmp, &sorted_keys,
- &kvmap);
+ c.Finish(options, ioptions, moptions, table_options, *internal_cmp,
+ &sorted_keys, &kvmap);
for (int j = 0; j < 2; ++j) {
std::unique_ptr<InternalIterator> iter(
}
}
-TEST_F(BlockBasedTableTest, FilterPolicyNameProperties) {
+TEST_P(BlockBasedTableTest, FilterPolicyNameProperties) {
TableConstructor c(BytewiseComparator(), true /* convert_to_internal_key_ */);
c.Add("a1", "val1");
std::vector<std::string> keys;
stl_wrappers::KVMap kvmap;
- BlockBasedTableOptions table_options;
+ BlockBasedTableOptions table_options = GetBlockBasedTableOptions();
table_options.filter_policy.reset(NewBloomFilterPolicy(10));
Options options;
options.table_factory.reset(NewBlockBasedTableFactory(table_options));
const ImmutableCFOptions ioptions(options);
- c.Finish(options, ioptions, table_options,
+ const MutableCFOptions moptions(options);
+ c.Finish(options, ioptions, moptions, table_options,
GetPlainInternalComparator(options.comparator), &keys, &kvmap);
auto& props = *c.GetTableReader()->GetTableProperties();
ASSERT_EQ("rocksdb.BuiltinBloomFilter", props.filter_policy_name);
table_options->block_cache = NewLRUCache(16 * 1024 * 1024, 4);
opt->table_factory.reset(NewBlockBasedTableFactory(*table_options));
const ImmutableCFOptions ioptions2(*opt);
- ASSERT_OK(c->Reopen(ioptions2));
+ const MutableCFOptions moptions(*opt);
+ ASSERT_OK(c->Reopen(ioptions2, moptions));
// prefetch
auto* table_reader = dynamic_cast<BlockBasedTable*>(c->GetTableReader());
c->ResetTableReader();
}
-TEST_F(BlockBasedTableTest, PrefetchTest) {
+TEST_P(BlockBasedTableTest, PrefetchTest) {
// The purpose of this test is to test the prefetching operation built into
// BlockBasedTable.
Options opt;
unique_ptr<InternalKeyComparator> ikc;
ikc.reset(new test::PlainInternalKeyComparator(opt.comparator));
opt.compression = kNoCompression;
- BlockBasedTableOptions table_options;
+ BlockBasedTableOptions table_options = GetBlockBasedTableOptions();
table_options.block_size = 1024;
// big enough so we don't ever lose cached values.
table_options.block_cache = NewLRUCache(16 * 1024 * 1024, 4);
std::vector<std::string> keys;
stl_wrappers::KVMap kvmap;
const ImmutableCFOptions ioptions(opt);
- c.Finish(opt, ioptions, table_options, *ikc, &keys, &kvmap);
+ const MutableCFOptions moptions(opt);
+ c.Finish(opt, ioptions, moptions, table_options, *ikc, &keys, &kvmap);
c.ResetTableReader();
// We get the following data spread :
c.ResetTableReader();
}
-TEST_F(BlockBasedTableTest, TotalOrderSeekOnHashIndex) {
- BlockBasedTableOptions table_options;
+TEST_P(BlockBasedTableTest, TotalOrderSeekOnHashIndex) {
+ BlockBasedTableOptions table_options = GetBlockBasedTableOptions();
for (int i = 0; i < 4; ++i) {
Options options;
// Make each key/value an individual block
std::vector<std::string> keys;
stl_wrappers::KVMap kvmap;
const ImmutableCFOptions ioptions(options);
- c.Finish(options, ioptions, table_options,
+ const MutableCFOptions moptions(options);
+ c.Finish(options, ioptions, moptions, table_options,
GetPlainInternalComparator(options.comparator), &keys, &kvmap);
auto props = c.GetTableReader()->GetTableProperties();
ASSERT_EQ(7u, props->num_data_blocks);
auto* reader = c.GetTableReader();
ReadOptions ro;
ro.total_order_seek = true;
- std::unique_ptr<InternalIterator> iter(reader->NewIterator(ro));
+ std::unique_ptr<InternalIterator> iter(
+ reader->NewIterator(ro, moptions.prefix_extractor.get()));
iter->Seek(InternalKey("b", 0, kTypeValue).Encode());
ASSERT_OK(iter->status());
}
}
-TEST_F(BlockBasedTableTest, NoopTransformSeek) {
- BlockBasedTableOptions table_options;
+TEST_P(BlockBasedTableTest, NoopTransformSeek) {
+ BlockBasedTableOptions table_options = GetBlockBasedTableOptions();
table_options.filter_policy.reset(NewBloomFilterPolicy(10));
Options options;
std::vector<std::string> keys;
stl_wrappers::KVMap kvmap;
const ImmutableCFOptions ioptions(options);
+ const MutableCFOptions moptions(options);
const InternalKeyComparator internal_comparator(options.comparator);
- c.Finish(options, ioptions, table_options, internal_comparator, &keys,
- &kvmap);
+ c.Finish(options, ioptions, moptions, table_options, internal_comparator,
+ &keys, &kvmap);
auto* reader = c.GetTableReader();
for (int i = 0; i < 2; ++i) {
ReadOptions ro;
ro.total_order_seek = (i == 0);
- std::unique_ptr<InternalIterator> iter(reader->NewIterator(ro));
+ std::unique_ptr<InternalIterator> iter(
+ reader->NewIterator(ro, moptions.prefix_extractor.get()));
iter->Seek(key.Encode());
ASSERT_OK(iter->status());
}
}
-TEST_F(BlockBasedTableTest, SkipPrefixBloomFilter) {
+TEST_P(BlockBasedTableTest, SkipPrefixBloomFilter) {
// if DB is opened with a prefix extractor of a different name,
// prefix bloom is skipped when read the file
- BlockBasedTableOptions table_options;
+ BlockBasedTableOptions table_options = GetBlockBasedTableOptions();
table_options.filter_policy.reset(NewBloomFilterPolicy(2));
table_options.whole_key_filtering = false;
std::vector<std::string> keys;
stl_wrappers::KVMap kvmap;
const ImmutableCFOptions ioptions(options);
+ const MutableCFOptions moptions(options);
const InternalKeyComparator internal_comparator(options.comparator);
- c.Finish(options, ioptions, table_options, internal_comparator, &keys,
- &kvmap);
+ c.Finish(options, ioptions, moptions, table_options, internal_comparator,
+ &keys, &kvmap);
+ // TODO(Zhongyi): update test to use MutableCFOptions
options.prefix_extractor.reset(NewFixedPrefixTransform(9));
const ImmutableCFOptions new_ioptions(options);
- c.Reopen(new_ioptions);
+ const MutableCFOptions new_moptions(options);
+ c.Reopen(new_ioptions, new_moptions);
auto reader = c.GetTableReader();
- std::unique_ptr<InternalIterator> db_iter(reader->NewIterator(ReadOptions()));
+ std::unique_ptr<InternalIterator> db_iter(
+ reader->NewIterator(ReadOptions(), new_moptions.prefix_extractor.get()));
// Test point lookup
// only one kv
}
void AddInternalKey(TableConstructor* c, const std::string& prefix,
- int suffix_len = 800) {
+ int /*suffix_len*/ = 800) {
static Random rnd(1023);
InternalKey k(prefix + RandomString(&rnd, 800), 0, kTypeValue);
c->Add(k.Encode().ToString(), "v");
std::unique_ptr<InternalKeyComparator> comparator(
new InternalKeyComparator(BytewiseComparator()));
const ImmutableCFOptions ioptions(options);
- c.Finish(options, ioptions, table_options, *comparator, &keys, &kvmap);
+ const MutableCFOptions moptions(options);
+ c.Finish(options, ioptions, moptions, table_options, *comparator, &keys,
+ &kvmap);
auto reader = c.GetTableReader();
auto props = reader->GetTableProperties();
ASSERT_EQ(5u, props->num_data_blocks);
+ // TODO(Zhongyi): update test to use MutableCFOptions
std::unique_ptr<InternalIterator> index_iter(
- reader->NewIterator(ReadOptions()));
+ reader->NewIterator(ReadOptions(), moptions.prefix_extractor.get()));
// -- Find keys do not exist, but have common prefix.
std::vector<std::string> prefixes = {"001", "003", "005", "007", "009"};
c.ResetTableReader();
}
-TEST_F(TableTest, BinaryIndexTest) {
- BlockBasedTableOptions table_options;
+TEST_P(BlockBasedTableTest, BinaryIndexTest) {
+ BlockBasedTableOptions table_options = GetBlockBasedTableOptions();
table_options.index_type = BlockBasedTableOptions::kBinarySearch;
IndexTest(table_options);
}
-TEST_F(TableTest, HashIndexTest) {
- BlockBasedTableOptions table_options;
+TEST_P(BlockBasedTableTest, HashIndexTest) {
+ BlockBasedTableOptions table_options = GetBlockBasedTableOptions();
table_options.index_type = BlockBasedTableOptions::kHashSearch;
IndexTest(table_options);
}
-TEST_F(TableTest, PartitionIndexTest) {
+TEST_P(BlockBasedTableTest, PartitionIndexTest) {
const int max_index_keys = 5;
const int est_max_index_key_value_size = 32;
const int est_max_index_size = max_index_keys * est_max_index_key_value_size;
for (int i = 1; i <= est_max_index_size + 1; i++) {
- BlockBasedTableOptions table_options;
+ BlockBasedTableOptions table_options = GetBlockBasedTableOptions();
table_options.index_type = BlockBasedTableOptions::kTwoLevelIndexSearch;
table_options.metadata_block_size = i;
IndexTest(table_options);
// It's very hard to figure out the index block size of a block accurately.
// To make sure we get the index size, we just make sure as key number
// grows, the filter block size also grows.
-TEST_F(BlockBasedTableTest, IndexSizeStat) {
+TEST_P(BlockBasedTableTest, IndexSizeStat) {
uint64_t last_index_size = 0;
// we need to use random keys since the pure human readable texts
stl_wrappers::KVMap kvmap;
Options options;
options.compression = kNoCompression;
- BlockBasedTableOptions table_options;
+ BlockBasedTableOptions table_options = GetBlockBasedTableOptions();
table_options.block_restart_interval = 1;
options.table_factory.reset(NewBlockBasedTableFactory(table_options));
const ImmutableCFOptions ioptions(options);
- c.Finish(options, ioptions, table_options,
+ const MutableCFOptions moptions(options);
+ c.Finish(options, ioptions, moptions, table_options,
GetPlainInternalComparator(options.comparator), &ks, &kvmap);
auto index_size = c.GetTableReader()->GetTableProperties()->index_size;
ASSERT_GT(index_size, last_index_size);
}
}
-TEST_F(BlockBasedTableTest, NumBlockStat) {
+TEST_P(BlockBasedTableTest, NumBlockStat) {
Random rnd(test::RandomSeed());
TableConstructor c(BytewiseComparator(), true /* convert_to_internal_key_ */);
Options options;
options.compression = kNoCompression;
- BlockBasedTableOptions table_options;
+ BlockBasedTableOptions table_options = GetBlockBasedTableOptions();
table_options.block_restart_interval = 1;
table_options.block_size = 1000;
options.table_factory.reset(NewBlockBasedTableFactory(table_options));
std::vector<std::string> ks;
stl_wrappers::KVMap kvmap;
const ImmutableCFOptions ioptions(options);
- c.Finish(options, ioptions, table_options,
+ const MutableCFOptions moptions(options);
+ c.Finish(options, ioptions, moptions, table_options,
GetPlainInternalComparator(options.comparator), &ks, &kvmap);
ASSERT_EQ(kvmap.size(),
c.GetTableReader()->GetTableProperties()->num_data_blocks);
// Make sure, by default, index/filter blocks were pre-loaded (meaning we won't
// use block cache to store them).
-TEST_F(BlockBasedTableTest, BlockCacheDisabledTest) {
+TEST_P(BlockBasedTableTest, BlockCacheDisabledTest) {
Options options;
options.create_if_missing = true;
options.statistics = CreateDBStatistics();
- BlockBasedTableOptions table_options;
+ BlockBasedTableOptions table_options = GetBlockBasedTableOptions();
table_options.block_cache = NewLRUCache(1024, 4);
table_options.filter_policy.reset(NewBloomFilterPolicy(10));
options.table_factory.reset(new BlockBasedTableFactory(table_options));
TableConstructor c(BytewiseComparator(), true /* convert_to_internal_key_ */);
c.Add("key", "value");
const ImmutableCFOptions ioptions(options);
- c.Finish(options, ioptions, table_options,
+ const MutableCFOptions moptions(options);
+ c.Finish(options, ioptions, moptions, table_options,
GetPlainInternalComparator(options.comparator), &keys, &kvmap);
// preloading filter/index blocks is enabled.
GetContext::kNotFound, Slice(), nullptr, nullptr,
nullptr, nullptr, nullptr);
// a hack that just to trigger BlockBasedTable::GetFilter.
- reader->Get(ReadOptions(), "non-exist-key", &get_context);
+ reader->Get(ReadOptions(), "non-exist-key", &get_context,
+ moptions.prefix_extractor.get());
BlockCachePropertiesSnapshot props(options.statistics.get());
props.AssertIndexBlockStat(0, 0);
props.AssertFilterBlockStat(0, 0);
// Due to the difficulities of the intersaction between statistics, this test
// only tests the case when "index block is put to block cache"
-TEST_F(BlockBasedTableTest, FilterBlockInBlockCache) {
+TEST_P(BlockBasedTableTest, FilterBlockInBlockCache) {
// -- Table construction
Options options;
options.create_if_missing = true;
options.statistics = CreateDBStatistics();
// Enable the cache for index/filter blocks
- BlockBasedTableOptions table_options;
- table_options.block_cache = NewLRUCache(1024, 4);
+ BlockBasedTableOptions table_options = GetBlockBasedTableOptions();
+ table_options.block_cache = NewLRUCache(2048, 2);
table_options.cache_index_and_filter_blocks = true;
options.table_factory.reset(new BlockBasedTableFactory(table_options));
std::vector<std::string> keys;
TableConstructor c(BytewiseComparator(), true /* convert_to_internal_key_ */);
c.Add("key", "value");
const ImmutableCFOptions ioptions(options);
- c.Finish(options, ioptions, table_options,
+ const MutableCFOptions moptions(options);
+ c.Finish(options, ioptions, moptions, table_options,
GetPlainInternalComparator(options.comparator), &keys, &kvmap);
// preloading filter/index blocks is prohibited.
auto* reader = dynamic_cast<BlockBasedTable*>(c.GetTableReader());
// Only index block will be accessed
{
- iter.reset(c.NewIterator());
+ iter.reset(c.NewIterator(moptions.prefix_extractor.get()));
BlockCachePropertiesSnapshot props(options.statistics.get());
// NOTE: to help better highlight the "detla" of each ticker, I use
// <last_value> + <added_value> to indicate the increment of changed
// Data block will be in cache
{
- iter.reset(c.NewIterator());
+ iter.reset(c.NewIterator(moptions.prefix_extractor.get()));
iter->SeekToFirst();
BlockCachePropertiesSnapshot props(options.statistics.get());
props.AssertEqual(1, 1 + 1, /* index block hit */
options.statistics = CreateDBStatistics();
options.table_factory.reset(new BlockBasedTableFactory(table_options));
const ImmutableCFOptions ioptions2(options);
- c.Reopen(ioptions2);
+ const MutableCFOptions moptions2(options);
+ c.Reopen(ioptions2, moptions2);
{
BlockCachePropertiesSnapshot props(options.statistics.get());
props.AssertEqual(1, // index block miss
// Both index and data block get accessed.
// It first cache index block then data block. But since the cache size
// is only 1, index block will be purged after data block is inserted.
- iter.reset(c.NewIterator());
+ iter.reset(c.NewIterator(moptions2.prefix_extractor.get()));
BlockCachePropertiesSnapshot props(options.statistics.get());
props.AssertEqual(1 + 1, // index block miss
0, 0, // data block miss
InternalKey internal_key(user_key, 0, kTypeValue);
c3.Add(internal_key.Encode().ToString(), "hello");
ImmutableCFOptions ioptions3(options);
+ MutableCFOptions moptions3(options);
// Generate table without filter policy
- c3.Finish(options, ioptions3, table_options,
+ c3.Finish(options, ioptions3, moptions3, table_options,
GetPlainInternalComparator(options.comparator), &keys, &kvmap);
c3.ResetTableReader();
options.table_factory.reset(new BlockBasedTableFactory(table_options));
options.statistics = CreateDBStatistics();
ImmutableCFOptions ioptions4(options);
- ASSERT_OK(c3.Reopen(ioptions4));
+ MutableCFOptions moptions4(options);
+ ASSERT_OK(c3.Reopen(ioptions4, moptions4));
reader = dynamic_cast<BlockBasedTable*>(c3.GetTableReader());
ASSERT_TRUE(!reader->TEST_filter_block_preloaded());
PinnableSlice value;
GetContext get_context(options.comparator, nullptr, nullptr, nullptr,
GetContext::kNotFound, user_key, &value, nullptr,
nullptr, nullptr, nullptr);
- ASSERT_OK(reader->Get(ReadOptions(), user_key, &get_context));
+ ASSERT_OK(reader->Get(ReadOptions(), internal_key.Encode(), &get_context,
+ moptions4.prefix_extractor.get()));
ASSERT_STREQ(value.data(), "hello");
BlockCachePropertiesSnapshot props(options.statistics.get());
props.AssertFilterBlockStat(0, 0);
delete factory;
}
-TEST_F(BlockBasedTableTest, InvalidOptions) {
+TEST_P(BlockBasedTableTest, InvalidOptions) {
// invalid values for block_size_deviation (<0 or >100) are silently set to 0
ValidateBlockSizeDeviation(-10, 0);
ValidateBlockSizeDeviation(-1, 0);
ValidateBlockRestartInterval(1000, 1000);
}
-TEST_F(BlockBasedTableTest, BlockReadCountTest) {
+TEST_P(BlockBasedTableTest, BlockReadCountTest) {
// bloom_filter_type = 0 -- block-based filter
// bloom_filter_type = 0 -- full filter
for (int bloom_filter_type = 0; bloom_filter_type < 2; ++bloom_filter_type) {
Options options;
options.create_if_missing = true;
- BlockBasedTableOptions table_options;
+ BlockBasedTableOptions table_options = GetBlockBasedTableOptions();
table_options.block_cache = NewLRUCache(1, 0);
table_options.cache_index_and_filter_blocks = index_and_filter_in_cache;
table_options.filter_policy.reset(
std::string encoded_key = internal_key.Encode().ToString();
c.Add(encoded_key, "hello");
ImmutableCFOptions ioptions(options);
+ MutableCFOptions moptions(options);
// Generate table with filter policy
- c.Finish(options, ioptions, table_options,
+ c.Finish(options, ioptions, moptions, table_options,
GetPlainInternalComparator(options.comparator), &keys, &kvmap);
auto reader = c.GetTableReader();
PinnableSlice value;
GetContext get_context(options.comparator, nullptr, nullptr, nullptr,
GetContext::kNotFound, user_key, &value, nullptr,
nullptr, nullptr, nullptr);
- perf_context.Reset();
- ASSERT_OK(reader->Get(ReadOptions(), encoded_key, &get_context));
+ get_perf_context()->Reset();
+ ASSERT_OK(reader->Get(ReadOptions(), encoded_key, &get_context,
+ moptions.prefix_extractor.get()));
if (index_and_filter_in_cache) {
// data, index and filter block
- ASSERT_EQ(perf_context.block_read_count, 3);
+ ASSERT_EQ(get_perf_context()->block_read_count, 3);
} else {
// just the data block
- ASSERT_EQ(perf_context.block_read_count, 1);
+ ASSERT_EQ(get_perf_context()->block_read_count, 1);
}
ASSERT_EQ(get_context.State(), GetContext::kFound);
ASSERT_STREQ(value.data(), "hello");
get_context = GetContext(options.comparator, nullptr, nullptr, nullptr,
GetContext::kNotFound, user_key, &value, nullptr,
nullptr, nullptr, nullptr);
- perf_context.Reset();
- ASSERT_OK(reader->Get(ReadOptions(), encoded_key, &get_context));
+ get_perf_context()->Reset();
+ ASSERT_OK(reader->Get(ReadOptions(), encoded_key, &get_context,
+ moptions.prefix_extractor.get()));
ASSERT_EQ(get_context.State(), GetContext::kNotFound);
if (index_and_filter_in_cache) {
if (bloom_filter_type == 0) {
// with block-based, we read index and then the filter
- ASSERT_EQ(perf_context.block_read_count, 2);
+ ASSERT_EQ(get_perf_context()->block_read_count, 2);
} else {
// with full-filter, we read filter first and then we stop
- ASSERT_EQ(perf_context.block_read_count, 1);
+ ASSERT_EQ(get_perf_context()->block_read_count, 1);
}
} else {
// filter is already in memory and it figures out that the key doesn't
// exist
- ASSERT_EQ(perf_context.block_read_count, 0);
+ ASSERT_EQ(get_perf_context()->block_read_count, 0);
}
}
}
}
-TEST_F(BlockBasedTableTest, BlockCacheLeak) {
+// A wrapper around LRICache that also keeps track of data blocks (in contrast
+// with the objects) in the cache. The class is very simple and can be used only
+// for trivial tests.
+class MockCache : public LRUCache {
+ public:
+ MockCache(size_t capacity, int num_shard_bits, bool strict_capacity_limit,
+ double high_pri_pool_ratio)
+ : LRUCache(capacity, num_shard_bits, strict_capacity_limit,
+ high_pri_pool_ratio) {}
+ virtual Status Insert(const Slice& key, void* value, size_t charge,
+ void (*deleter)(const Slice& key, void* value),
+ Handle** handle = nullptr,
+ Priority priority = Priority::LOW) override {
+ // Replace the deleter with our own so that we keep track of data blocks
+ // erased from the cache
+ deleters_[key.ToString()] = deleter;
+ return ShardedCache::Insert(key, value, charge, &MockDeleter, handle,
+ priority);
+ }
+ // This is called by the application right after inserting a data block
+ virtual void TEST_mark_as_data_block(const Slice& key,
+ size_t charge) override {
+ marked_data_in_cache_[key.ToString()] = charge;
+ marked_size_ += charge;
+ }
+ using DeleterFunc = void (*)(const Slice& key, void* value);
+ static std::map<std::string, DeleterFunc> deleters_;
+ static std::map<std::string, size_t> marked_data_in_cache_;
+ static size_t marked_size_;
+ static void MockDeleter(const Slice& key, void* value) {
+ // If the item was marked for being data block, decrease its usage from the
+ // total data block usage of the cache
+ if (marked_data_in_cache_.find(key.ToString()) !=
+ marked_data_in_cache_.end()) {
+ marked_size_ -= marked_data_in_cache_[key.ToString()];
+ }
+ // Then call the origianl deleter
+ assert(deleters_.find(key.ToString()) != deleters_.end());
+ auto deleter = deleters_[key.ToString()];
+ deleter(key, value);
+ }
+};
+
+size_t MockCache::marked_size_ = 0;
+std::map<std::string, MockCache::DeleterFunc> MockCache::deleters_;
+std::map<std::string, size_t> MockCache::marked_data_in_cache_;
+
+// Block cache can contain raw data blocks as well as general objects. If an
+// object depends on the table to be live, it then must be destructed before the
+// table is closed. This test makes sure that the only items remains in the
+// cache after the table is closed are raw data blocks.
+TEST_P(BlockBasedTableTest, NoObjectInCacheAfterTableClose) {
+ for (int level: {-1, 0, 1, 10}) {
+ for (auto index_type :
+ {BlockBasedTableOptions::IndexType::kBinarySearch,
+ BlockBasedTableOptions::IndexType::kTwoLevelIndexSearch}) {
+ for (bool block_based_filter : {true, false}) {
+ for (bool partition_filter : {true, false}) {
+ if (partition_filter &&
+ (block_based_filter ||
+ index_type !=
+ BlockBasedTableOptions::IndexType::kTwoLevelIndexSearch)) {
+ continue;
+ }
+ for (bool index_and_filter_in_cache : {true, false}) {
+ for (bool pin_l0 : {true, false}) {
+ for (bool pin_top_level : {true, false}) {
+ if (pin_l0 && !index_and_filter_in_cache) {
+ continue;
+ }
+ // Create a table
+ Options opt;
+ unique_ptr<InternalKeyComparator> ikc;
+ ikc.reset(new test::PlainInternalKeyComparator(opt.comparator));
+ opt.compression = kNoCompression;
+ BlockBasedTableOptions table_options =
+ GetBlockBasedTableOptions();
+ table_options.block_size = 1024;
+ table_options.index_type =
+ BlockBasedTableOptions::IndexType::kTwoLevelIndexSearch;
+ table_options.pin_l0_filter_and_index_blocks_in_cache = pin_l0;
+ table_options.pin_top_level_index_and_filter = pin_top_level;
+ table_options.partition_filters = partition_filter;
+ table_options.cache_index_and_filter_blocks =
+ index_and_filter_in_cache;
+ // big enough so we don't ever lose cached values.
+ table_options.block_cache = std::shared_ptr<rocksdb::Cache>(
+ new MockCache(16 * 1024 * 1024, 4, false, 0.0));
+ table_options.filter_policy.reset(
+ rocksdb::NewBloomFilterPolicy(10, block_based_filter));
+ opt.table_factory.reset(NewBlockBasedTableFactory(table_options));
+
+ bool convert_to_internal_key = false;
+ TableConstructor c(BytewiseComparator(), convert_to_internal_key,
+ level);
+ std::string user_key = "k01";
+ std::string key =
+ InternalKey(user_key, 0, kTypeValue).Encode().ToString();
+ c.Add(key, "hello");
+ std::vector<std::string> keys;
+ stl_wrappers::KVMap kvmap;
+ const ImmutableCFOptions ioptions(opt);
+ const MutableCFOptions moptions(opt);
+ c.Finish(opt, ioptions, moptions, table_options, *ikc, &keys,
+ &kvmap);
+
+ // Doing a read to make index/filter loaded into the cache
+ auto table_reader =
+ dynamic_cast<BlockBasedTable*>(c.GetTableReader());
+ PinnableSlice value;
+ GetContext get_context(opt.comparator, nullptr, nullptr, nullptr,
+ GetContext::kNotFound, user_key, &value,
+ nullptr, nullptr, nullptr, nullptr);
+ InternalKey ikey(user_key, 0, kTypeValue);
+ auto s = table_reader->Get(ReadOptions(), key, &get_context,
+ moptions.prefix_extractor.get());
+ ASSERT_EQ(get_context.State(), GetContext::kFound);
+ ASSERT_STREQ(value.data(), "hello");
+
+ // Close the table
+ c.ResetTableReader();
+
+ auto usage = table_options.block_cache->GetUsage();
+ auto pinned_usage = table_options.block_cache->GetPinnedUsage();
+ // The only usage must be for marked data blocks
+ ASSERT_EQ(usage, MockCache::marked_size_);
+ // There must be some pinned data since PinnableSlice has not
+ // released them yet
+ ASSERT_GT(pinned_usage, 0);
+ // Release pinnable slice reousrces
+ value.Reset();
+ pinned_usage = table_options.block_cache->GetPinnedUsage();
+ ASSERT_EQ(pinned_usage, 0);
+ }
+ }
+ }
+ }
+ }
+ }
+ } // level
+}
+
+TEST_P(BlockBasedTableTest, BlockCacheLeak) {
// Check that when we reopen a table we don't lose access to blocks already
// in the cache. This test checks whether the Table actually makes use of the
// unique ID from the file.
unique_ptr<InternalKeyComparator> ikc;
ikc.reset(new test::PlainInternalKeyComparator(opt.comparator));
opt.compression = kNoCompression;
- BlockBasedTableOptions table_options;
+ BlockBasedTableOptions table_options = GetBlockBasedTableOptions();
table_options.block_size = 1024;
// big enough so we don't ever lose cached values.
table_options.block_cache = NewLRUCache(16 * 1024 * 1024, 4);
std::vector<std::string> keys;
stl_wrappers::KVMap kvmap;
const ImmutableCFOptions ioptions(opt);
- c.Finish(opt, ioptions, table_options, *ikc, &keys, &kvmap);
+ const MutableCFOptions moptions(opt);
+ c.Finish(opt, ioptions, moptions, table_options, *ikc, &keys, &kvmap);
- unique_ptr<InternalIterator> iter(c.NewIterator());
+ unique_ptr<InternalIterator> iter(
+ c.NewIterator(moptions.prefix_extractor.get()));
iter->SeekToFirst();
while (iter->Valid()) {
iter->key();
iter->Next();
}
ASSERT_OK(iter->status());
+ iter.reset();
const ImmutableCFOptions ioptions1(opt);
- ASSERT_OK(c.Reopen(ioptions1));
+ const MutableCFOptions moptions1(opt);
+ ASSERT_OK(c.Reopen(ioptions1, moptions1));
auto table_reader = dynamic_cast<BlockBasedTable*>(c.GetTableReader());
for (const std::string& key : keys) {
- ASSERT_TRUE(table_reader->TEST_KeyInCache(ReadOptions(), key));
+ InternalKey ikey(key, kMaxSequenceNumber, kTypeValue);
+ ASSERT_TRUE(table_reader->TEST_KeyInCache(ReadOptions(), ikey.Encode()));
}
c.ResetTableReader();
table_options.block_cache = NewLRUCache(16 * 1024 * 1024, 4);
opt.table_factory.reset(NewBlockBasedTableFactory(table_options));
const ImmutableCFOptions ioptions2(opt);
- ASSERT_OK(c.Reopen(ioptions2));
+ const MutableCFOptions moptions2(opt);
+ ASSERT_OK(c.Reopen(ioptions2, moptions2));
table_reader = dynamic_cast<BlockBasedTable*>(c.GetTableReader());
for (const std::string& key : keys) {
- ASSERT_TRUE(!table_reader->TEST_KeyInCache(ReadOptions(), key));
+ InternalKey ikey(key, kMaxSequenceNumber, kTypeValue);
+ ASSERT_TRUE(!table_reader->TEST_KeyInCache(ReadOptions(), ikey.Encode()));
}
c.ResetTableReader();
}
-TEST_F(BlockBasedTableTest, NewIndexIteratorLeak) {
+TEST_P(BlockBasedTableTest, NewIndexIteratorLeak) {
// A regression test to avoid data race described in
// https://github.com/facebook/rocksdb/issues/1267
TableConstructor c(BytewiseComparator(), true /* convert_to_internal_key_ */);
c.Add("a1", "val1");
Options options;
options.prefix_extractor.reset(NewFixedPrefixTransform(1));
- BlockBasedTableOptions table_options;
+ BlockBasedTableOptions table_options = GetBlockBasedTableOptions();
table_options.index_type = BlockBasedTableOptions::kHashSearch;
table_options.cache_index_and_filter_blocks = true;
table_options.block_cache = NewLRUCache(0);
options.table_factory.reset(NewBlockBasedTableFactory(table_options));
const ImmutableCFOptions ioptions(options);
- c.Finish(options, ioptions, table_options,
+ const MutableCFOptions moptions(options);
+ c.Finish(options, ioptions, moptions, table_options,
GetPlainInternalComparator(options.comparator), &keys, &kvmap);
rocksdb::SyncPoint::GetInstance()->LoadDependencyAndMarkers(
std::function<void()> func1 = [&]() {
TEST_SYNC_POINT("BlockBasedTableTest::NewIndexIteratorLeak:Thread1Marker");
- std::unique_ptr<InternalIterator> iter(reader->NewIterator(ro));
+ // TODO(Zhongyi): update test to use MutableCFOptions
+ std::unique_ptr<InternalIterator> iter(
+ reader->NewIterator(ro, moptions.prefix_extractor.get()));
iter->Seek(InternalKey("a1", 0, kTypeValue).Encode());
};
std::function<void()> func2 = [&]() {
TEST_SYNC_POINT("BlockBasedTableTest::NewIndexIteratorLeak:Thread2Marker");
- std::unique_ptr<InternalIterator> iter(reader->NewIterator(ro));
+ std::unique_ptr<InternalIterator> iter(
+ reader->NewIterator(ro, moptions.prefix_extractor.get()));
};
auto thread1 = port::Thread(func1);
PlainTableFactory factory(plain_table_options);
test::StringSink sink;
unique_ptr<WritableFileWriter> file_writer(
- test::GetWritableFileWriter(new test::StringSink()));
+ test::GetWritableFileWriter(new test::StringSink(), "" /* don't care */));
Options options;
const ImmutableCFOptions ioptions(options);
+ const MutableCFOptions moptions(options);
InternalKeyComparator ikc(options.comparator);
std::vector<std::unique_ptr<IntTblPropCollectorFactory>>
int_tbl_prop_collector_factories;
std::string column_family_name;
int unknown_level = -1;
std::unique_ptr<TableBuilder> builder(factory.NewTableBuilder(
- TableBuilderOptions(ioptions, ikc, &int_tbl_prop_collector_factories,
- kNoCompression, CompressionOptions(),
- nullptr /* compression_dict */,
- false /* skip_filters */, column_family_name,
- unknown_level),
+ TableBuilderOptions(
+ ioptions, moptions, ikc, &int_tbl_prop_collector_factories,
+ kNoCompression, CompressionOptions(), nullptr /* compression_dict */,
+ false /* skip_filters */, column_family_name, unknown_level),
TablePropertiesCollectorFactory::Context::kUnknownColumnFamily,
file_writer.get()));
TableProperties* props = nullptr;
auto s = ReadTableProperties(file_reader.get(), ss->contents().size(),
kPlainTableMagicNumber, ioptions,
- &props);
+ &props, true /* compression_type_missing */);
std::unique_ptr<TableProperties> props_guard(props);
ASSERT_OK(s);
BlockBasedTableOptions table_options;
table_options.block_size = 1024;
const ImmutableCFOptions ioptions(options);
- c.Finish(options, ioptions, table_options, internal_comparator,
+ const MutableCFOptions moptions(options);
+ c.Finish(options, ioptions, moptions, table_options, internal_comparator,
&keys, &kvmap);
ASSERT_TRUE(Between(c.ApproximateOffsetOf("abc"), 0, 0));
BlockBasedTableOptions table_options;
table_options.block_size = 1024;
const ImmutableCFOptions ioptions(options);
- c.Finish(options, ioptions, table_options, ikc, &keys, &kvmap);
+ const MutableCFOptions moptions(options);
+ c.Finish(options, ioptions, moptions, table_options, ikc, &keys, &kvmap);
ASSERT_TRUE(Between(c.ApproximateOffsetOf("abc"), 0, 0));
ASSERT_TRUE(Between(c.ApproximateOffsetOf("k01"), 0, 0));
}
}
-TEST_F(HarnessTest, Randomized) {
- std::vector<TestArgs> args = GenerateArgList();
- for (unsigned int i = 0; i < args.size(); i++) {
- Init(args[i]);
- Random rnd(test::RandomSeed() + 5);
- for (int num_entries = 0; num_entries < 2000;
- num_entries += (num_entries < 50 ? 1 : 200)) {
- if ((num_entries % 10) == 0) {
- fprintf(stderr, "case %d of %d: num_entries = %d\n", (i + 1),
- static_cast<int>(args.size()), num_entries);
- }
- for (int e = 0; e < num_entries; e++) {
- std::string v;
- Add(test::RandomKey(&rnd, rnd.Skewed(4)),
- test::RandomString(&rnd, rnd.Skewed(5), &v).ToString());
- }
- Test(&rnd);
- }
- }
+// RandomizedHarnessTest is very slow for certain combination of arguments
+// Split into 8 pieces to reduce the time individual tests take.
+TEST_F(HarnessTest, Randomized1) {
+ // part 1 out of 8
+ const size_t part = 1;
+ const size_t total = 8;
+ RandomizedHarnessTest(part, total);
+}
+
+TEST_F(HarnessTest, Randomized2) {
+ // part 2 out of 8
+ const size_t part = 2;
+ const size_t total = 8;
+ RandomizedHarnessTest(part, total);
+}
+
+TEST_F(HarnessTest, Randomized3) {
+ // part 3 out of 8
+ const size_t part = 3;
+ const size_t total = 8;
+ RandomizedHarnessTest(part, total);
+}
+
+TEST_F(HarnessTest, Randomized4) {
+ // part 4 out of 8
+ const size_t part = 4;
+ const size_t total = 8;
+ RandomizedHarnessTest(part, total);
+}
+
+TEST_F(HarnessTest, Randomized5) {
+ // part 5 out of 8
+ const size_t part = 5;
+ const size_t total = 8;
+ RandomizedHarnessTest(part, total);
+}
+
+TEST_F(HarnessTest, Randomized6) {
+ // part 6 out of 8
+ const size_t part = 6;
+ const size_t total = 8;
+ RandomizedHarnessTest(part, total);
+}
+
+TEST_F(HarnessTest, Randomized7) {
+ // part 7 out of 8
+ const size_t part = 7;
+ const size_t total = 8;
+ RandomizedHarnessTest(part, total);
+}
+
+TEST_F(HarnessTest, Randomized8) {
+ // part 8 out of 8
+ const size_t part = 8;
+ const size_t total = 8;
+ RandomizedHarnessTest(part, total);
}
#ifndef ROCKSDB_LITE
options.memtable_factory = table_factory;
ImmutableCFOptions ioptions(options);
WriteBufferManager wb(options.db_write_buffer_size);
- MemTable* memtable = new MemTable(cmp, ioptions, MutableCFOptions(options),
- &wb, kMaxSequenceNumber);
+ MemTable* memtable =
+ new MemTable(cmp, ioptions, MutableCFOptions(options), &wb,
+ kMaxSequenceNumber, 0 /* column_family_id */);
memtable->Ref();
WriteBatch batch;
WriteBatchInternal::SetSequence(&batch, 100);
}
class IndexBlockRestartIntervalTest
- : public BlockBasedTableTest,
- public ::testing::WithParamInterface<int> {
+ : public TableTest,
+ public ::testing::WithParamInterface<std::pair<int, bool>> {
public:
- static std::vector<int> GetRestartValues() { return {-1, 0, 1, 8, 16, 32}; }
+ static std::vector<std::pair<int, bool>> GetRestartValues() {
+ return {{-1, false}, {0, false}, {1, false}, {8, false},
+ {16, false}, {32, false}, {-1, true}, {0, true},
+ {1, true}, {8, true}, {16, true}, {32, true}};
+ }
};
INSTANTIATE_TEST_CASE_P(
const int kKeySize = 100;
const int kValSize = 500;
- int index_block_restart_interval = GetParam();
+ const int index_block_restart_interval = std::get<0>(GetParam());
+ const bool value_delta_encoding = std::get<1>(GetParam());
Options options;
BlockBasedTableOptions table_options;
table_options.block_size = 64; // small block size to get big index block
table_options.index_block_restart_interval = index_block_restart_interval;
+ if (value_delta_encoding) {
+ table_options.format_version = 4;
+ }
options.table_factory.reset(new BlockBasedTableFactory(table_options));
TableConstructor c(BytewiseComparator());
std::unique_ptr<InternalKeyComparator> comparator(
new InternalKeyComparator(BytewiseComparator()));
const ImmutableCFOptions ioptions(options);
- c.Finish(options, ioptions, table_options, *comparator, &keys, &kvmap);
+ const MutableCFOptions moptions(options);
+ c.Finish(options, ioptions, moptions, table_options, *comparator, &keys,
+ &kvmap);
auto reader = c.GetTableReader();
- std::unique_ptr<InternalIterator> db_iter(reader->NewIterator(ReadOptions()));
+ std::unique_ptr<InternalIterator> db_iter(
+ reader->NewIterator(ReadOptions(), moptions.prefix_extractor.get()));
// Test point lookup
for (auto& kv : kvmap) {
return true;
}
- bool InRange(const rocksdb::Slice& dst) const override { return true; }
+ bool InRange(const rocksdb::Slice& /*dst*/) const override { return true; }
bool IsValid(const rocksdb::Slice& src) const {
if (src.size() != 4) {
bbto.block_size = 262144;
bbto.whole_key_filtering = true;
- const std::string kDBPath = test::TmpDir() + "/table_prefix_test";
+ const std::string kDBPath = test::PerThreadDBPath("table_prefix_test");
options.table_factory.reset(NewBlockBasedTableFactory(bbto));
DestroyDB(kDBPath, options);
rocksdb::DB* db;
// rocksdb still works.
}
-TEST_F(BlockBasedTableTest, TableWithGlobalSeqno) {
- BlockBasedTableOptions bbto;
+/*
+ * Disable TableWithGlobalSeqno since RocksDB does not store global_seqno in
+ * the SST file any more. Instead, RocksDB deduces global_seqno from the
+ * MANIFEST while reading from an SST. Therefore, it's not possible to test the
+ * functionality of global_seqno in a single, isolated unit test without the
+ * involvement of Version, VersionSet, etc.
+ */
+TEST_P(BlockBasedTableTest, DISABLED_TableWithGlobalSeqno) {
+ BlockBasedTableOptions bbto = GetBlockBasedTableOptions();
test::StringSink* sink = new test::StringSink();
- unique_ptr<WritableFileWriter> file_writer(test::GetWritableFileWriter(sink));
+ unique_ptr<WritableFileWriter> file_writer(
+ test::GetWritableFileWriter(sink, "" /* don't care */));
Options options;
options.table_factory.reset(NewBlockBasedTableFactory(bbto));
const ImmutableCFOptions ioptions(options);
+ const MutableCFOptions moptions(options);
InternalKeyComparator ikc(options.comparator);
std::vector<std::unique_ptr<IntTblPropCollectorFactory>>
int_tbl_prop_collector_factories;
0 /* global_seqno*/));
std::string column_family_name;
std::unique_ptr<TableBuilder> builder(options.table_factory->NewTableBuilder(
- TableBuilderOptions(ioptions, ikc, &int_tbl_prop_collector_factories,
- kNoCompression, CompressionOptions(),
- nullptr /* compression_dict */,
+ TableBuilderOptions(ioptions, moptions, ikc,
+ &int_tbl_prop_collector_factories, kNoCompression,
+ CompressionOptions(), nullptr /* compression_dict */,
false /* skip_filters */, column_family_name, -1),
TablePropertiesCollectorFactory::Context::kUnknownColumnFamily,
file_writer.get()));
TableProperties* props = nullptr;
ASSERT_OK(ReadTableProperties(file_reader.get(), ss_rw.contents().size(),
kBlockBasedTableMagicNumber, ioptions,
- &props));
+ &props, true /* compression_type_missing */));
UserCollectedProperties user_props = props->user_collected_properties;
version = DecodeFixed32(
new test::StringSource(ss_rw.contents(), 73342, true)));
options.table_factory->NewTableReader(
- TableReaderOptions(ioptions, EnvOptions(), ikc), std::move(file_reader),
- ss_rw.contents().size(), &table_reader);
+ TableReaderOptions(ioptions, moptions.prefix_extractor.get(),
+ EnvOptions(), ikc),
+ std::move(file_reader), ss_rw.contents().size(), &table_reader);
- return table_reader->NewIterator(ReadOptions());
+ return table_reader->NewIterator(ReadOptions(),
+ moptions.prefix_extractor.get());
};
GetVersionAndGlobalSeqno();
delete iter;
}
+TEST_P(BlockBasedTableTest, BlockAlignTest) {
+ BlockBasedTableOptions bbto = GetBlockBasedTableOptions();
+ bbto.block_align = true;
+ test::StringSink* sink = new test::StringSink();
+ unique_ptr<WritableFileWriter> file_writer(
+ test::GetWritableFileWriter(sink, "" /* don't care */));
+ Options options;
+ options.compression = kNoCompression;
+ options.table_factory.reset(NewBlockBasedTableFactory(bbto));
+ const ImmutableCFOptions ioptions(options);
+ const MutableCFOptions moptions(options);
+ InternalKeyComparator ikc(options.comparator);
+ std::vector<std::unique_ptr<IntTblPropCollectorFactory>>
+ int_tbl_prop_collector_factories;
+ std::string column_family_name;
+ std::unique_ptr<TableBuilder> builder(options.table_factory->NewTableBuilder(
+ TableBuilderOptions(ioptions, moptions, ikc,
+ &int_tbl_prop_collector_factories, kNoCompression,
+ CompressionOptions(), nullptr /* compression_dict */,
+ false /* skip_filters */, column_family_name, -1),
+ TablePropertiesCollectorFactory::Context::kUnknownColumnFamily,
+ file_writer.get()));
+
+ for (int i = 1; i <= 10000; ++i) {
+ std::ostringstream ostr;
+ ostr << std::setfill('0') << std::setw(5) << i;
+ std::string key = ostr.str();
+ std::string value = "val";
+ InternalKey ik(key, 0, kTypeValue);
+
+ builder->Add(ik.Encode(), value);
+ }
+ ASSERT_OK(builder->Finish());
+ file_writer->Flush();
+
+ test::RandomRWStringSink ss_rw(sink);
+ unique_ptr<RandomAccessFileReader> file_reader(
+ test::GetRandomAccessFileReader(
+ new test::StringSource(ss_rw.contents(), 73342, true)));
+
+ // Helper function to get version, global_seqno, global_seqno_offset
+ std::function<void()> VerifyBlockAlignment = [&]() {
+ TableProperties* props = nullptr;
+ ASSERT_OK(ReadTableProperties(file_reader.get(), ss_rw.contents().size(),
+ kBlockBasedTableMagicNumber, ioptions,
+ &props, true /* compression_type_missing */));
+
+ uint64_t data_block_size = props->data_size / props->num_data_blocks;
+ ASSERT_EQ(data_block_size, 4096);
+ ASSERT_EQ(props->data_size, data_block_size * props->num_data_blocks);
+ delete props;
+ };
+
+ VerifyBlockAlignment();
+
+ // The below block of code verifies that we can read back the keys. Set
+ // block_align to false when creating the reader to ensure we can flip between
+ // the two modes without any issues
+ std::unique_ptr<TableReader> table_reader;
+ bbto.block_align = false;
+ Options options2;
+ options2.table_factory.reset(NewBlockBasedTableFactory(bbto));
+ ImmutableCFOptions ioptions2(options2);
+ const MutableCFOptions moptions2(options2);
+
+ ASSERT_OK(ioptions.table_factory->NewTableReader(
+ TableReaderOptions(ioptions2, moptions2.prefix_extractor.get(),
+ EnvOptions(),
+ GetPlainInternalComparator(options2.comparator)),
+ std::move(file_reader), ss_rw.contents().size(), &table_reader));
+
+ std::unique_ptr<InternalIterator> db_iter(table_reader->NewIterator(
+ ReadOptions(), moptions2.prefix_extractor.get()));
+
+ int expected_key = 1;
+ for (db_iter->SeekToFirst(); db_iter->Valid(); db_iter->Next()) {
+ std::ostringstream ostr;
+ ostr << std::setfill('0') << std::setw(5) << expected_key++;
+ std::string key = ostr.str();
+ std::string value = "val";
+
+ ASSERT_OK(db_iter->status());
+ ASSERT_EQ(ExtractUserKey(db_iter->key()).ToString(), key);
+ ASSERT_EQ(db_iter->value().ToString(), value);
+ }
+ expected_key--;
+ ASSERT_EQ(expected_key, 10000);
+ table_reader.reset();
+}
+
+TEST_P(BlockBasedTableTest, PropertiesBlockRestartPointTest) {
+ BlockBasedTableOptions bbto = GetBlockBasedTableOptions();
+ bbto.block_align = true;
+ test::StringSink* sink = new test::StringSink();
+ unique_ptr<WritableFileWriter> file_writer(
+ test::GetWritableFileWriter(sink, "" /* don't care */));
+
+ Options options;
+ options.compression = kNoCompression;
+ options.table_factory.reset(NewBlockBasedTableFactory(bbto));
+
+ const ImmutableCFOptions ioptions(options);
+ const MutableCFOptions moptions(options);
+ InternalKeyComparator ikc(options.comparator);
+ std::vector<std::unique_ptr<IntTblPropCollectorFactory>>
+ int_tbl_prop_collector_factories;
+ std::string column_family_name;
+
+ std::unique_ptr<TableBuilder> builder(options.table_factory->NewTableBuilder(
+ TableBuilderOptions(ioptions, moptions, ikc,
+ &int_tbl_prop_collector_factories, kNoCompression,
+ CompressionOptions(), nullptr /* compression_dict */,
+ false /* skip_filters */, column_family_name, -1),
+ TablePropertiesCollectorFactory::Context::kUnknownColumnFamily,
+ file_writer.get()));
+
+ for (int i = 1; i <= 10000; ++i) {
+ std::ostringstream ostr;
+ ostr << std::setfill('0') << std::setw(5) << i;
+ std::string key = ostr.str();
+ std::string value = "val";
+ InternalKey ik(key, 0, kTypeValue);
+
+ builder->Add(ik.Encode(), value);
+ }
+ ASSERT_OK(builder->Finish());
+ file_writer->Flush();
+
+ test::RandomRWStringSink ss_rw(sink);
+ unique_ptr<RandomAccessFileReader> file_reader(
+ test::GetRandomAccessFileReader(
+ new test::StringSource(ss_rw.contents(), 73342, true)));
+
+ {
+ RandomAccessFileReader* file = file_reader.get();
+ uint64_t file_size = ss_rw.contents().size();
+
+ Footer footer;
+ ASSERT_OK(ReadFooterFromFile(file, nullptr /* prefetch_buffer */, file_size,
+ &footer, kBlockBasedTableMagicNumber));
+
+ auto BlockFetchHelper = [&](const BlockHandle& handle,
+ BlockContents* contents) {
+ ReadOptions read_options;
+ read_options.verify_checksums = false;
+ Slice compression_dict;
+ PersistentCacheOptions cache_options;
+
+ BlockFetcher block_fetcher(file, nullptr /* prefetch_buffer */, footer,
+ read_options, handle, contents, ioptions,
+ false /* decompress */, compression_dict,
+ cache_options);
+
+ ASSERT_OK(block_fetcher.ReadBlockContents());
+ };
+
+ // -- Read metaindex block
+ auto metaindex_handle = footer.metaindex_handle();
+ BlockContents metaindex_contents;
+
+ BlockFetchHelper(metaindex_handle, &metaindex_contents);
+ Block metaindex_block(std::move(metaindex_contents),
+ kDisableGlobalSequenceNumber);
+
+ std::unique_ptr<InternalIterator> meta_iter(
+ metaindex_block.NewIterator<DataBlockIter>(BytewiseComparator(),
+ BytewiseComparator()));
+ bool found_properties_block = true;
+ ASSERT_OK(SeekToPropertiesBlock(meta_iter.get(), &found_properties_block));
+ ASSERT_TRUE(found_properties_block);
+
+ // -- Read properties block
+ Slice v = meta_iter->value();
+ BlockHandle properties_handle;
+ ASSERT_OK(properties_handle.DecodeFrom(&v));
+ BlockContents properties_contents;
+
+ BlockFetchHelper(properties_handle, &properties_contents);
+ Block properties_block(std::move(properties_contents),
+ kDisableGlobalSequenceNumber);
+
+ ASSERT_EQ(properties_block.NumRestarts(), 1);
+ }
+}
+
+TEST_P(BlockBasedTableTest, PropertiesMetaBlockLast) {
+ // The properties meta-block should come at the end since we always need to
+ // read it when opening a file, unlike index/filter/other meta-blocks, which
+ // are sometimes read depending on the user's configuration. This ordering
+ // allows us to do a small readahead on the end of the file to read properties
+ // and meta-index blocks with one I/O.
+ TableConstructor c(BytewiseComparator(), true /* convert_to_internal_key_ */);
+ c.Add("a1", "val1");
+ c.Add("b2", "val2");
+ c.Add("c3", "val3");
+ c.Add("d4", "val4");
+ c.Add("e5", "val5");
+ c.Add("f6", "val6");
+ c.Add("g7", "val7");
+ c.Add("h8", "val8");
+ c.Add("j9", "val9");
+
+ // write an SST file
+ Options options;
+ BlockBasedTableOptions table_options = GetBlockBasedTableOptions();
+ table_options.filter_policy.reset(NewBloomFilterPolicy(
+ 8 /* bits_per_key */, false /* use_block_based_filter */));
+ options.table_factory.reset(NewBlockBasedTableFactory(table_options));
+ ImmutableCFOptions ioptions(options);
+ MutableCFOptions moptions(options);
+ std::vector<std::string> keys;
+ stl_wrappers::KVMap kvmap;
+ c.Finish(options, ioptions, moptions, table_options,
+ GetPlainInternalComparator(options.comparator), &keys, &kvmap);
+
+ // get file reader
+ test::StringSink* table_sink = c.TEST_GetSink();
+ std::unique_ptr<RandomAccessFileReader> table_reader{
+ test::GetRandomAccessFileReader(
+ new test::StringSource(table_sink->contents(), 0 /* unique_id */,
+ false /* allow_mmap_reads */))};
+ size_t table_size = table_sink->contents().size();
+
+ // read footer
+ Footer footer;
+ ASSERT_OK(ReadFooterFromFile(table_reader.get(),
+ nullptr /* prefetch_buffer */, table_size,
+ &footer, kBlockBasedTableMagicNumber));
+
+ // read metaindex
+ auto metaindex_handle = footer.metaindex_handle();
+ BlockContents metaindex_contents;
+ Slice compression_dict;
+ PersistentCacheOptions pcache_opts;
+ BlockFetcher block_fetcher(
+ table_reader.get(), nullptr /* prefetch_buffer */, footer, ReadOptions(),
+ metaindex_handle, &metaindex_contents, ioptions, false /* decompress */,
+ compression_dict, pcache_opts);
+ ASSERT_OK(block_fetcher.ReadBlockContents());
+ Block metaindex_block(std::move(metaindex_contents),
+ kDisableGlobalSequenceNumber);
+
+ // verify properties block comes last
+ std::unique_ptr<InternalIterator> metaindex_iter{
+ metaindex_block.NewIterator<DataBlockIter>(options.comparator,
+ options.comparator)};
+ uint64_t max_offset = 0;
+ std::string key_at_max_offset;
+ for (metaindex_iter->SeekToFirst(); metaindex_iter->Valid();
+ metaindex_iter->Next()) {
+ BlockHandle handle;
+ Slice value = metaindex_iter->value();
+ ASSERT_OK(handle.DecodeFrom(&value));
+ if (handle.offset() > max_offset) {
+ max_offset = handle.offset();
+ key_at_max_offset = metaindex_iter->key().ToString();
+ }
+ }
+ ASSERT_EQ(kPropertiesBlock, key_at_max_offset);
+ // index handle is stored in footer rather than metaindex block, so need
+ // separate logic to verify it comes before properties block.
+ ASSERT_GT(max_offset, footer.index_handle().offset());
+ c.ResetTableReader();
+}
+
+TEST_P(BlockBasedTableTest, BadOptions) {
+ rocksdb::Options options;
+ options.compression = kNoCompression;
+ BlockBasedTableOptions bbto = GetBlockBasedTableOptions();
+ bbto.block_size = 4000;
+ bbto.block_align = true;
+
+ const std::string kDBPath =
+ test::PerThreadDBPath("block_based_table_bad_options_test");
+ options.table_factory.reset(NewBlockBasedTableFactory(bbto));
+ DestroyDB(kDBPath, options);
+ rocksdb::DB* db;
+ ASSERT_NOK(rocksdb::DB::Open(options, kDBPath, &db));
+
+ bbto.block_size = 4096;
+ options.compression = kSnappyCompression;
+ options.table_factory.reset(NewBlockBasedTableFactory(bbto));
+ ASSERT_NOK(rocksdb::DB::Open(options, kDBPath, &db));
+}
+
+TEST_F(BBTTailPrefetchTest, TestTailPrefetchStats) {
+ TailPrefetchStats tpstats;
+ ASSERT_EQ(0, tpstats.GetSuggestedPrefetchSize());
+ tpstats.RecordEffectiveSize(size_t{1000});
+ tpstats.RecordEffectiveSize(size_t{1005});
+ tpstats.RecordEffectiveSize(size_t{1002});
+ ASSERT_EQ(1005, tpstats.GetSuggestedPrefetchSize());
+
+ // One single super large value shouldn't influence much
+ tpstats.RecordEffectiveSize(size_t{1002000});
+ tpstats.RecordEffectiveSize(size_t{999});
+ ASSERT_LE(1005, tpstats.GetSuggestedPrefetchSize());
+ ASSERT_GT(1200, tpstats.GetSuggestedPrefetchSize());
+
+ // Only history of 32 is kept
+ for (int i = 0; i < 32; i++) {
+ tpstats.RecordEffectiveSize(size_t{100});
+ }
+ ASSERT_EQ(100, tpstats.GetSuggestedPrefetchSize());
+
+ // 16 large values and 16 small values. The result should be closer
+ // to the small value as the algorithm.
+ for (int i = 0; i < 16; i++) {
+ tpstats.RecordEffectiveSize(size_t{1000});
+ }
+ tpstats.RecordEffectiveSize(size_t{10});
+ tpstats.RecordEffectiveSize(size_t{20});
+ for (int i = 0; i < 6; i++) {
+ tpstats.RecordEffectiveSize(size_t{100});
+ }
+ ASSERT_LE(80, tpstats.GetSuggestedPrefetchSize());
+ ASSERT_GT(200, tpstats.GetSuggestedPrefetchSize());
+}
+
+TEST_F(BBTTailPrefetchTest, FilePrefetchBufferMinOffset) {
+ TailPrefetchStats tpstats;
+ FilePrefetchBuffer buffer(nullptr, 0, 0, false, true);
+ buffer.TryReadFromCache(500, 10, nullptr);
+ buffer.TryReadFromCache(480, 10, nullptr);
+ buffer.TryReadFromCache(490, 10, nullptr);
+ ASSERT_EQ(480, buffer.min_offset_read());
+}
+
+TEST_P(BlockBasedTableTest, DataBlockHashIndex) {
+ const int kNumKeys = 500;
+ const int kKeySize = 8;
+ const int kValSize = 40;
+
+ BlockBasedTableOptions table_options = GetBlockBasedTableOptions();
+ table_options.data_block_index_type =
+ BlockBasedTableOptions::kDataBlockBinaryAndHash;
+
+ Options options;
+ options.comparator = BytewiseComparator();
+
+ options.table_factory.reset(new BlockBasedTableFactory(table_options));
+
+ TableConstructor c(options.comparator);
+
+ static Random rnd(1048);
+ for (int i = 0; i < kNumKeys; i++) {
+ // padding one "0" to mark existent keys.
+ std::string random_key(RandomString(&rnd, kKeySize - 1) + "1");
+ InternalKey k(random_key, 0, kTypeValue);
+ c.Add(k.Encode().ToString(), RandomString(&rnd, kValSize));
+ }
+
+ std::vector<std::string> keys;
+ stl_wrappers::KVMap kvmap;
+ const ImmutableCFOptions ioptions(options);
+ const MutableCFOptions moptions(options);
+ const InternalKeyComparator internal_comparator(options.comparator);
+ c.Finish(options, ioptions, moptions, table_options, internal_comparator,
+ &keys, &kvmap);
+
+ auto reader = c.GetTableReader();
+
+ std::unique_ptr<InternalIterator> seek_iter;
+ seek_iter.reset(
+ reader->NewIterator(ReadOptions(), moptions.prefix_extractor.get()));
+ for (int i = 0; i < 2; ++i) {
+ ReadOptions ro;
+ // for every kv, we seek using two method: Get() and Seek()
+ // Get() will use the SuffixIndexHash in Block. For non-existent key it
+ // will invalidate the iterator
+ // Seek() will use the default BinarySeek() in Block. So for non-existent
+ // key it will land at the closest key that is large than target.
+
+ // Search for existent keys
+ for (auto& kv : kvmap) {
+ if (i == 0) {
+ // Search using Seek()
+ seek_iter->Seek(kv.first);
+ ASSERT_OK(seek_iter->status());
+ ASSERT_TRUE(seek_iter->Valid());
+ ASSERT_EQ(seek_iter->key(), kv.first);
+ ASSERT_EQ(seek_iter->value(), kv.second);
+ } else {
+ // Search using Get()
+ PinnableSlice value;
+ std::string user_key = ExtractUserKey(kv.first).ToString();
+ GetContext get_context(options.comparator, nullptr, nullptr, nullptr,
+ GetContext::kNotFound, user_key, &value, nullptr,
+ nullptr, nullptr, nullptr);
+ ASSERT_OK(reader->Get(ro, kv.first, &get_context,
+ moptions.prefix_extractor.get()));
+ ASSERT_EQ(get_context.State(), GetContext::kFound);
+ ASSERT_EQ(value, Slice(kv.second));
+ value.Reset();
+ }
+ }
+
+ // Search for non-existent keys
+ for (auto& kv : kvmap) {
+ std::string user_key = ExtractUserKey(kv.first).ToString();
+ user_key.back() = '0'; // make it non-existent key
+ InternalKey internal_key(user_key, 0, kTypeValue);
+ std::string encoded_key = internal_key.Encode().ToString();
+ if (i == 0) { // Search using Seek()
+ seek_iter->Seek(encoded_key);
+ ASSERT_OK(seek_iter->status());
+ if (seek_iter->Valid()) {
+ ASSERT_TRUE(BytewiseComparator()->Compare(
+ user_key, ExtractUserKey(seek_iter->key())) < 0);
+ }
+ } else { // Search using Get()
+ PinnableSlice value;
+ GetContext get_context(options.comparator, nullptr, nullptr, nullptr,
+ GetContext::kNotFound, user_key, &value, nullptr,
+ nullptr, nullptr, nullptr);
+ ASSERT_OK(reader->Get(ro, encoded_key, &get_context,
+ moptions.prefix_extractor.get()));
+ ASSERT_EQ(get_context.State(), GetContext::kNotFound);
+ value.Reset();
+ }
+ }
+ }
+}
+
} // namespace rocksdb
int main(int argc, char** argv) {
projects / ceph.git / blobdiff