// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
-// This source code is licensed under the BSD-style license found in the
-// LICENSE file in the root directory of this source tree. An additional grant
-// of patent rights can be found in the PATENTS file in the same directory.
+// This source code is licensed under both the GPLv2 (found in the
+// COPYING file in the root directory) and Apache 2.0 License
+// (found in the LICENSE.Apache file in the root directory).
#include "db/compaction_picker.h"
#include <limits>
class CountingLogger : public Logger {
public:
using Logger::Logv;
- virtual void Logv(const char* format, va_list ap) override { log_count++; }
+ virtual void Logv(const char* /*format*/, va_list /*ap*/) override {
+ log_count++;
+ }
size_t log_count;
};
vstorage_(nullptr) {
fifo_options_.max_table_files_size = 1;
mutable_cf_options_.RefreshDerivedOptions(ioptions_);
- ioptions_.db_paths.emplace_back("dummy",
+ ioptions_.cf_paths.emplace_back("dummy",
std::numeric_limits<uint64_t>::max());
}
f->fd = FileDescriptor(file_number, path_id, file_size);
f->smallest = InternalKey(smallest, smallest_seq, kTypeValue);
f->largest = InternalKey(largest, largest_seq, kTypeValue);
- f->smallest_seqno = smallest_seq;
- f->largest_seqno = largest_seq;
+ f->fd.smallest_seqno = smallest_seq;
+ f->fd.largest_seqno = largest_seq;
f->compensated_file_size = file_size;
f->refs = 0;
vstorage_->AddFile(level, f);
}
TEST_F(CompactionPickerTest, Level1Trigger2) {
+ mutable_cf_options_.target_file_size_base = 10000000000;
+ mutable_cf_options_.RefreshDerivedOptions(ioptions_);
NewVersionStorage(6, kCompactionStyleLevel);
Add(1, 66U, "150", "200", 1000000001U);
Add(1, 88U, "201", "300", 1000000000U);
ASSERT_EQ(66U, compaction->input(0, 0)->fd.GetNumber());
ASSERT_EQ(6U, compaction->input(1, 0)->fd.GetNumber());
ASSERT_EQ(7U, compaction->input(1, 1)->fd.GetNumber());
+ ASSERT_EQ(uint64_t{1073741824}, compaction->OutputFilePreallocationSize());
}
TEST_F(CompactionPickerTest, LevelMaxScore) {
NewVersionStorage(6, kCompactionStyleLevel);
mutable_cf_options_.target_file_size_base = 10000000;
- mutable_cf_options_.target_file_size_multiplier = 10;
mutable_cf_options_.max_bytes_for_level_base = 10 * 1024 * 1024;
+ mutable_cf_options_.RefreshDerivedOptions(ioptions_);
Add(0, 1U, "150", "200", 1000000U);
// Level 1 score 1.2
Add(1, 66U, "150", "200", 6000000U);
ASSERT_TRUE(compaction.get() != nullptr);
ASSERT_EQ(1U, compaction->num_input_files(0));
ASSERT_EQ(7U, compaction->input(0, 0)->fd.GetNumber());
+ ASSERT_EQ(mutable_cf_options_.target_file_size_base +
+ mutable_cf_options_.target_file_size_base / 10,
+ compaction->OutputFilePreallocationSize());
}
TEST_F(CompactionPickerTest, NeedsCompactionLevel) {
NewVersionStorage(1, kCompactionStyleUniversal);
UniversalCompactionPicker universal_compaction_picker(
ioptions_, &icmp_);
+ UpdateVersionStorageInfo();
// must return false when there's no files.
ASSERT_EQ(universal_compaction_picker.NeedsCompaction(vstorage_.get()),
false);
- UpdateVersionStorageInfo();
// verify the trigger given different number of L0 files.
for (int i = 1;
vstorage_->CompactionScore(0) >= 1);
}
}
+
+TEST_F(CompactionPickerTest, CompactionUniversalIngestBehindReservedLevel) {
+ const uint64_t kFileSize = 100000;
+ NewVersionStorage(1, kCompactionStyleUniversal);
+ ioptions_.allow_ingest_behind = true;
+ ioptions_.num_levels = 3;
+ UniversalCompactionPicker universal_compaction_picker(ioptions_, &icmp_);
+ UpdateVersionStorageInfo();
+ // must return false when there's no files.
+ ASSERT_EQ(universal_compaction_picker.NeedsCompaction(vstorage_.get()),
+ false);
+
+ NewVersionStorage(3, kCompactionStyleUniversal);
+
+ Add(0, 1U, "150", "200", kFileSize, 0, 500, 550);
+ Add(0, 2U, "201", "250", kFileSize, 0, 401, 450);
+ Add(0, 4U, "260", "300", kFileSize, 0, 260, 300);
+ Add(1, 5U, "100", "151", kFileSize, 0, 200, 251);
+ Add(1, 3U, "301", "350", kFileSize, 0, 101, 150);
+ Add(2, 6U, "120", "200", kFileSize, 0, 20, 100);
+
+ UpdateVersionStorageInfo();
+
+ std::unique_ptr<Compaction> compaction(
+ universal_compaction_picker.PickCompaction(
+ cf_name_, mutable_cf_options_, vstorage_.get(), &log_buffer_));
+
+ // output level should be the one above the bottom-most
+ ASSERT_EQ(1, compaction->output_level());
+}
// Tests if the files can be trivially moved in multi level
// universal compaction when allow_trivial_move option is set
// In this test as the input files overlaps, they cannot
TEST_F(CompactionPickerTest, CannotTrivialMoveUniversal) {
const uint64_t kFileSize = 100000;
- ioptions_.compaction_options_universal.allow_trivial_move = true;
+ mutable_cf_options_.compaction_options_universal.allow_trivial_move = true;
NewVersionStorage(1, kCompactionStyleUniversal);
UniversalCompactionPicker universal_compaction_picker(ioptions_, &icmp_);
+ UpdateVersionStorageInfo();
// must return false when there's no files.
ASSERT_EQ(universal_compaction_picker.NeedsCompaction(vstorage_.get()),
false);
TEST_F(CompactionPickerTest, AllowsTrivialMoveUniversal) {
const uint64_t kFileSize = 100000;
- ioptions_.compaction_options_universal.allow_trivial_move = true;
+ mutable_cf_options_.compaction_options_universal.allow_trivial_move = true;
UniversalCompactionPicker universal_compaction_picker(ioptions_, &icmp_);
NewVersionStorage(3, kCompactionStyleUniversal);
const uint64_t kMaxSize = kFileSize * kFileCount / 2;
fifo_options_.max_table_files_size = kMaxSize;
- ioptions_.compaction_options_fifo = fifo_options_;
+ mutable_cf_options_.compaction_options_fifo = fifo_options_;
FIFOCompactionPicker fifo_compaction_picker(ioptions_, &icmp_);
UpdateVersionStorageInfo();
// must return false when there's no files.
kFileSize, 0, i * 100, i * 100 + 99);
current_size += kFileSize;
UpdateVersionStorageInfo();
- ASSERT_EQ(level_compaction_picker.NeedsCompaction(vstorage_.get()),
+ ASSERT_EQ(fifo_compaction_picker.NeedsCompaction(vstorage_.get()),
vstorage_->CompactionScore(0) >= 1);
}
}
TEST_F(CompactionPickerTest, CompactionPriMinOverlapping1) {
NewVersionStorage(6, kCompactionStyleLevel);
ioptions_.compaction_pri = kMinOverlappingRatio;
- mutable_cf_options_.target_file_size_base = 10000000;
+ mutable_cf_options_.target_file_size_base = 100000000000;
mutable_cf_options_.target_file_size_multiplier = 10;
mutable_cf_options_.max_bytes_for_level_base = 10 * 1024 * 1024;
+ mutable_cf_options_.RefreshDerivedOptions(ioptions_);
Add(2, 6U, "150", "179", 50000000U);
Add(2, 7U, "180", "220", 50000000U);
ASSERT_EQ(1U, compaction->num_input_files(0));
// Pick file 8 because it overlaps with 0 files on level 3.
ASSERT_EQ(8U, compaction->input(0, 0)->fd.GetNumber());
+ // Compaction input size * 1.1
+ ASSERT_GE(uint64_t{55000000}, compaction->OutputFilePreallocationSize());
}
TEST_F(CompactionPickerTest, CompactionPriMinOverlapping2) {
ASSERT_EQ(8U, compaction->input(1, 1)->fd.GetNumber());
}
+TEST_F(CompactionPickerTest, OverlappingUserKeys10) {
+ // Locked file encountered when pulling in extra input-level files with same
+ // user keys. Verify we pick the next-best file from the same input level.
+ NewVersionStorage(6, kCompactionStyleLevel);
+ mutable_cf_options_.max_compaction_bytes = 100000000000u;
+
+ // file_number 2U is largest and thus first choice. But it overlaps with
+ // file_number 1U which is being compacted. So instead we pick the next-
+ // biggest file, 3U, which is eligible for compaction.
+ Add(1 /* level */, 1U /* file_number */, "100" /* smallest */,
+ "150" /* largest */, 1U /* file_size */);
+ file_map_[1U].first->being_compacted = true;
+ Add(1 /* level */, 2U /* file_number */, "150" /* smallest */,
+ "200" /* largest */, 1000000000U /* file_size */, 0 /* smallest_seq */,
+ 0 /* largest_seq */);
+ Add(1 /* level */, 3U /* file_number */, "201" /* smallest */,
+ "250" /* largest */, 900000000U /* file_size */);
+ Add(2 /* level */, 4U /* file_number */, "100" /* smallest */,
+ "150" /* largest */, 1U /* file_size */);
+ Add(2 /* level */, 5U /* file_number */, "151" /* smallest */,
+ "200" /* largest */, 1U /* file_size */);
+ Add(2 /* level */, 6U /* file_number */, "201" /* smallest */,
+ "250" /* largest */, 1U /* file_size */);
+
+ UpdateVersionStorageInfo();
+
+ std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction(
+ cf_name_, mutable_cf_options_, vstorage_.get(), &log_buffer_));
+ ASSERT_TRUE(compaction.get() != nullptr);
+ ASSERT_EQ(2U, compaction->num_input_levels());
+ ASSERT_EQ(1U, compaction->num_input_files(0));
+ ASSERT_EQ(1U, compaction->num_input_files(1));
+ ASSERT_EQ(3U, compaction->input(0, 0)->fd.GetNumber());
+ ASSERT_EQ(6U, compaction->input(1, 0)->fd.GetNumber());
+}
+
+TEST_F(CompactionPickerTest, OverlappingUserKeys11) {
+ // Locked file encountered when pulling in extra output-level files with same
+ // user keys. Expected to skip that compaction and pick the next-best choice.
+ NewVersionStorage(6, kCompactionStyleLevel);
+ mutable_cf_options_.max_compaction_bytes = 100000000000u;
+
+ // score(L1) = 3.7
+ // score(L2) = 1.85
+ // There is no eligible file in L1 to compact since both candidates pull in
+ // file_number 5U, which overlaps with a file pending compaction (6U). The
+ // first eligible compaction is from L2->L3.
+ Add(1 /* level */, 2U /* file_number */, "151" /* smallest */,
+ "200" /* largest */, 1000000000U /* file_size */);
+ Add(1 /* level */, 3U /* file_number */, "201" /* smallest */,
+ "250" /* largest */, 1U /* file_size */);
+ Add(2 /* level */, 4U /* file_number */, "100" /* smallest */,
+ "149" /* largest */, 5000000000U /* file_size */);
+ Add(2 /* level */, 5U /* file_number */, "150" /* smallest */,
+ "201" /* largest */, 1U /* file_size */);
+ Add(2 /* level */, 6U /* file_number */, "201" /* smallest */,
+ "249" /* largest */, 1U /* file_size */, 0 /* smallest_seq */,
+ 0 /* largest_seq */);
+ file_map_[6U].first->being_compacted = true;
+ Add(3 /* level */, 7U /* file_number */, "100" /* smallest */,
+ "149" /* largest */, 1U /* file_size */);
+
+ UpdateVersionStorageInfo();
+
+ std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction(
+ cf_name_, mutable_cf_options_, vstorage_.get(), &log_buffer_));
+ ASSERT_TRUE(compaction.get() != nullptr);
+ ASSERT_EQ(2U, compaction->num_input_levels());
+ ASSERT_EQ(1U, compaction->num_input_files(0));
+ ASSERT_EQ(1U, compaction->num_input_files(1));
+ ASSERT_EQ(4U, compaction->input(0, 0)->fd.GetNumber());
+ ASSERT_EQ(7U, compaction->input(1, 0)->fd.GetNumber());
+}
+
TEST_F(CompactionPickerTest, NotScheduleL1IfL0WithHigherPri1) {
NewVersionStorage(6, kCompactionStyleLevel);
mutable_cf_options_.level0_file_num_compaction_trigger = 2;
TEST_F(CompactionPickerTest, EstimateCompactionBytesNeeded1) {
int num_levels = ioptions_.num_levels;
ioptions_.level_compaction_dynamic_level_bytes = false;
- mutable_cf_options_.level0_file_num_compaction_trigger = 3;
+ mutable_cf_options_.level0_file_num_compaction_trigger = 4;
mutable_cf_options_.max_bytes_for_level_base = 1000;
mutable_cf_options_.max_bytes_for_level_multiplier = 10;
NewVersionStorage(num_levels, kCompactionStyleLevel);
// Set Last level size 50000
// num_levels - 1 target 5000
- // num_levels - 2 is base level with taret 500
+ // num_levels - 2 is base level with target 1000 (rounded up to
+ // max_bytes_for_level_base).
Add(num_levels - 1, 10U, "400", "500", 50000);
Add(0, 1U, "150", "200", 200);
Add(0, 5U, "150", "200", 200);
Add(0, 6U, "150", "200", 200);
// num_levels - 3 is over target by 100 + 1000
- Add(num_levels - 3, 7U, "400", "500", 300);
- Add(num_levels - 3, 8U, "600", "700", 300);
+ Add(num_levels - 3, 7U, "400", "500", 550);
+ Add(num_levels - 3, 8U, "600", "700", 550);
// num_levels - 2 is over target by 1100 + 200
Add(num_levels - 2, 9U, "150", "200", 5200);
UpdateVersionStorageInfo();
- // Merging to the second last level: (5200 / 1600 + 1) * 1100
+ // Merging to the second last level: (5200 / 2100 + 1) * 1100
// Merging to the last level: (50000 / 6300 + 1) * 1300
- ASSERT_EQ(1600u + 4675u + 11617u,
+ ASSERT_EQ(2100u + 3823u + 11617u,
vstorage_->estimated_compaction_needed_bytes());
}
ASSERT_FALSE(compaction->IsTrivialMove());
}
+TEST_F(CompactionPickerTest, CacheNextCompactionIndex) {
+ NewVersionStorage(6, kCompactionStyleLevel);
+ mutable_cf_options_.max_compaction_bytes = 100000000000u;
+
+ Add(1 /* level */, 1U /* file_number */, "100" /* smallest */,
+ "149" /* largest */, 1000000000U /* file_size */);
+ file_map_[1U].first->being_compacted = true;
+ Add(1 /* level */, 2U /* file_number */, "150" /* smallest */,
+ "199" /* largest */, 900000000U /* file_size */);
+ Add(1 /* level */, 3U /* file_number */, "200" /* smallest */,
+ "249" /* largest */, 800000000U /* file_size */);
+ Add(1 /* level */, 4U /* file_number */, "250" /* smallest */,
+ "299" /* largest */, 700000000U /* file_size */);
+ Add(2 /* level */, 5U /* file_number */, "150" /* smallest */,
+ "199" /* largest */, 1U /* file_size */);
+ file_map_[5U].first->being_compacted = true;
+
+ UpdateVersionStorageInfo();
+
+ std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction(
+ cf_name_, mutable_cf_options_, vstorage_.get(), &log_buffer_));
+ ASSERT_TRUE(compaction.get() != nullptr);
+ ASSERT_EQ(1U, compaction->num_input_levels());
+ ASSERT_EQ(1U, compaction->num_input_files(0));
+ ASSERT_EQ(0U, compaction->num_input_files(1));
+ ASSERT_EQ(3U, compaction->input(0, 0)->fd.GetNumber());
+ ASSERT_EQ(2, vstorage_->NextCompactionIndex(1 /* level */));
+
+ compaction.reset(level_compaction_picker.PickCompaction(
+ cf_name_, mutable_cf_options_, vstorage_.get(), &log_buffer_));
+ ASSERT_TRUE(compaction.get() != nullptr);
+ ASSERT_EQ(1U, compaction->num_input_levels());
+ ASSERT_EQ(1U, compaction->num_input_files(0));
+ ASSERT_EQ(0U, compaction->num_input_files(1));
+ ASSERT_EQ(4U, compaction->input(0, 0)->fd.GetNumber());
+ ASSERT_EQ(3, vstorage_->NextCompactionIndex(1 /* level */));
+
+ compaction.reset(level_compaction_picker.PickCompaction(
+ cf_name_, mutable_cf_options_, vstorage_.get(), &log_buffer_));
+ ASSERT_TRUE(compaction.get() == nullptr);
+ ASSERT_EQ(4, vstorage_->NextCompactionIndex(1 /* level */));
+}
+
} // namespace rocksdb
int main(int argc, char** argv) {
projects / ceph.git / blobdiff