]> git.proxmox.com Git - ceph.git/blame - ceph/src/rocksdb/db/compaction_picker_test.cc
import 14.2.4 nautilus point release
[ceph.git] / ceph / src / rocksdb / db / compaction_picker_test.cc
CommitLineData
7c673cae 1// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
11fdf7f2
TL
2// This source code is licensed under both the GPLv2 (found in the
3// COPYING file in the root directory) and Apache 2.0 License
4// (found in the LICENSE.Apache file in the root directory).
7c673cae
FG
5
6#include "db/compaction_picker.h"
494da23a 7
7c673cae
FG
8#include <limits>
9#include <string>
10#include <utility>
11#include "db/compaction.h"
494da23a 12#include "db/compaction_picker_fifo.h"
7c673cae
FG
13#include "db/compaction_picker_universal.h"
14
15#include "util/logging.h"
16#include "util/string_util.h"
17#include "util/testharness.h"
18#include "util/testutil.h"
19
20namespace rocksdb {
21
22class CountingLogger : public Logger {
23 public:
24 using Logger::Logv;
494da23a 25 void Logv(const char* /*format*/, va_list /*ap*/) override { log_count++; }
7c673cae
FG
26 size_t log_count;
27};
28
29class CompactionPickerTest : public testing::Test {
30 public:
31 const Comparator* ucmp_;
32 InternalKeyComparator icmp_;
33 Options options_;
34 ImmutableCFOptions ioptions_;
35 MutableCFOptions mutable_cf_options_;
36 LevelCompactionPicker level_compaction_picker;
37 std::string cf_name_;
38 CountingLogger logger_;
39 LogBuffer log_buffer_;
40 uint32_t file_num_;
41 CompactionOptionsFIFO fifo_options_;
42 std::unique_ptr<VersionStorageInfo> vstorage_;
43 std::vector<std::unique_ptr<FileMetaData>> files_;
44 // does not own FileMetaData
45 std::unordered_map<uint32_t, std::pair<FileMetaData*, int>> file_map_;
46 // input files to compaction process.
47 std::vector<CompactionInputFiles> input_files_;
48 int compaction_level_start_;
49
50 CompactionPickerTest()
51 : ucmp_(BytewiseComparator()),
52 icmp_(ucmp_),
53 ioptions_(options_),
54 mutable_cf_options_(options_),
55 level_compaction_picker(ioptions_, &icmp_),
56 cf_name_("dummy"),
57 log_buffer_(InfoLogLevel::INFO_LEVEL, &logger_),
58 file_num_(1),
59 vstorage_(nullptr) {
494da23a
TL
60 // ioptions_.compaction_pri = kMinOverlappingRatio has its own set of
61 // tests to cover.
62 ioptions_.compaction_pri = kByCompensatedSize;
7c673cae
FG
63 fifo_options_.max_table_files_size = 1;
64 mutable_cf_options_.RefreshDerivedOptions(ioptions_);
11fdf7f2 65 ioptions_.cf_paths.emplace_back("dummy",
7c673cae
FG
66 std::numeric_limits<uint64_t>::max());
67 }
68
494da23a 69 ~CompactionPickerTest() override {}
7c673cae
FG
70
71 void NewVersionStorage(int num_levels, CompactionStyle style) {
72 DeleteVersionStorage();
73 options_.num_levels = num_levels;
74 vstorage_.reset(new VersionStorageInfo(&icmp_, ucmp_, options_.num_levels,
75 style, nullptr, false));
76 vstorage_->CalculateBaseBytes(ioptions_, mutable_cf_options_);
77 }
78
79 void DeleteVersionStorage() {
80 vstorage_.reset();
81 files_.clear();
82 file_map_.clear();
83 input_files_.clear();
84 }
85
86 void Add(int level, uint32_t file_number, const char* smallest,
87 const char* largest, uint64_t file_size = 1, uint32_t path_id = 0,
494da23a
TL
88 SequenceNumber smallest_seq = 100, SequenceNumber largest_seq = 100,
89 size_t compensated_file_size = 0) {
7c673cae
FG
90 assert(level < vstorage_->num_levels());
91 FileMetaData* f = new FileMetaData;
92 f->fd = FileDescriptor(file_number, path_id, file_size);
93 f->smallest = InternalKey(smallest, smallest_seq, kTypeValue);
94 f->largest = InternalKey(largest, largest_seq, kTypeValue);
11fdf7f2
TL
95 f->fd.smallest_seqno = smallest_seq;
96 f->fd.largest_seqno = largest_seq;
494da23a
TL
97 f->compensated_file_size =
98 (compensated_file_size != 0) ? compensated_file_size : file_size;
7c673cae
FG
99 f->refs = 0;
100 vstorage_->AddFile(level, f);
101 files_.emplace_back(f);
102 file_map_.insert({file_number, {f, level}});
103 }
104
105 void SetCompactionInputFilesLevels(int level_count, int start_level) {
106 input_files_.resize(level_count);
107 for (int i = 0; i < level_count; ++i) {
108 input_files_[i].level = start_level + i;
109 }
110 compaction_level_start_ = start_level;
111 }
112
113 void AddToCompactionFiles(uint32_t file_number) {
114 auto iter = file_map_.find(file_number);
115 assert(iter != file_map_.end());
116 int level = iter->second.second;
117 assert(level < vstorage_->num_levels());
118 input_files_[level - compaction_level_start_].files.emplace_back(
119 iter->second.first);
120 }
121
122 void UpdateVersionStorageInfo() {
123 vstorage_->CalculateBaseBytes(ioptions_, mutable_cf_options_);
124 vstorage_->UpdateFilesByCompactionPri(ioptions_.compaction_pri);
125 vstorage_->UpdateNumNonEmptyLevels();
126 vstorage_->GenerateFileIndexer();
127 vstorage_->GenerateLevelFilesBrief();
128 vstorage_->ComputeCompactionScore(ioptions_, mutable_cf_options_);
129 vstorage_->GenerateLevel0NonOverlapping();
130 vstorage_->ComputeFilesMarkedForCompaction();
131 vstorage_->SetFinalized();
132 }
133};
134
135TEST_F(CompactionPickerTest, Empty) {
136 NewVersionStorage(6, kCompactionStyleLevel);
137 UpdateVersionStorageInfo();
138 std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction(
139 cf_name_, mutable_cf_options_, vstorage_.get(), &log_buffer_));
140 ASSERT_TRUE(compaction.get() == nullptr);
141}
142
143TEST_F(CompactionPickerTest, Single) {
144 NewVersionStorage(6, kCompactionStyleLevel);
145 mutable_cf_options_.level0_file_num_compaction_trigger = 2;
146 Add(0, 1U, "p", "q");
147 UpdateVersionStorageInfo();
148
149 std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction(
150 cf_name_, mutable_cf_options_, vstorage_.get(), &log_buffer_));
151 ASSERT_TRUE(compaction.get() == nullptr);
152}
153
154TEST_F(CompactionPickerTest, Level0Trigger) {
155 NewVersionStorage(6, kCompactionStyleLevel);
156 mutable_cf_options_.level0_file_num_compaction_trigger = 2;
157 Add(0, 1U, "150", "200");
158 Add(0, 2U, "200", "250");
159
160 UpdateVersionStorageInfo();
161
162 std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction(
163 cf_name_, mutable_cf_options_, vstorage_.get(), &log_buffer_));
164 ASSERT_TRUE(compaction.get() != nullptr);
165 ASSERT_EQ(2U, compaction->num_input_files(0));
166 ASSERT_EQ(1U, compaction->input(0, 0)->fd.GetNumber());
167 ASSERT_EQ(2U, compaction->input(0, 1)->fd.GetNumber());
168}
169
170TEST_F(CompactionPickerTest, Level1Trigger) {
171 NewVersionStorage(6, kCompactionStyleLevel);
172 Add(1, 66U, "150", "200", 1000000000U);
173 UpdateVersionStorageInfo();
174
175 std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction(
176 cf_name_, mutable_cf_options_, vstorage_.get(), &log_buffer_));
177 ASSERT_TRUE(compaction.get() != nullptr);
178 ASSERT_EQ(1U, compaction->num_input_files(0));
179 ASSERT_EQ(66U, compaction->input(0, 0)->fd.GetNumber());
180}
181
182TEST_F(CompactionPickerTest, Level1Trigger2) {
11fdf7f2
TL
183 mutable_cf_options_.target_file_size_base = 10000000000;
184 mutable_cf_options_.RefreshDerivedOptions(ioptions_);
7c673cae
FG
185 NewVersionStorage(6, kCompactionStyleLevel);
186 Add(1, 66U, "150", "200", 1000000001U);
187 Add(1, 88U, "201", "300", 1000000000U);
188 Add(2, 6U, "150", "179", 1000000000U);
189 Add(2, 7U, "180", "220", 1000000000U);
190 Add(2, 8U, "221", "300", 1000000000U);
191 UpdateVersionStorageInfo();
192
193 std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction(
194 cf_name_, mutable_cf_options_, vstorage_.get(), &log_buffer_));
195 ASSERT_TRUE(compaction.get() != nullptr);
196 ASSERT_EQ(1U, compaction->num_input_files(0));
197 ASSERT_EQ(2U, compaction->num_input_files(1));
198 ASSERT_EQ(66U, compaction->input(0, 0)->fd.GetNumber());
199 ASSERT_EQ(6U, compaction->input(1, 0)->fd.GetNumber());
200 ASSERT_EQ(7U, compaction->input(1, 1)->fd.GetNumber());
11fdf7f2 201 ASSERT_EQ(uint64_t{1073741824}, compaction->OutputFilePreallocationSize());
7c673cae
FG
202}
203
204TEST_F(CompactionPickerTest, LevelMaxScore) {
205 NewVersionStorage(6, kCompactionStyleLevel);
206 mutable_cf_options_.target_file_size_base = 10000000;
7c673cae 207 mutable_cf_options_.max_bytes_for_level_base = 10 * 1024 * 1024;
11fdf7f2 208 mutable_cf_options_.RefreshDerivedOptions(ioptions_);
7c673cae
FG
209 Add(0, 1U, "150", "200", 1000000U);
210 // Level 1 score 1.2
211 Add(1, 66U, "150", "200", 6000000U);
212 Add(1, 88U, "201", "300", 6000000U);
213 // Level 2 score 1.8. File 7 is the largest. Should be picked
214 Add(2, 6U, "150", "179", 60000000U);
215 Add(2, 7U, "180", "220", 60000001U);
216 Add(2, 8U, "221", "300", 60000000U);
217 // Level 3 score slightly larger than 1
218 Add(3, 26U, "150", "170", 260000000U);
219 Add(3, 27U, "171", "179", 260000000U);
220 Add(3, 28U, "191", "220", 260000000U);
221 Add(3, 29U, "221", "300", 260000000U);
222 UpdateVersionStorageInfo();
223
224 std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction(
225 cf_name_, mutable_cf_options_, vstorage_.get(), &log_buffer_));
226 ASSERT_TRUE(compaction.get() != nullptr);
227 ASSERT_EQ(1U, compaction->num_input_files(0));
228 ASSERT_EQ(7U, compaction->input(0, 0)->fd.GetNumber());
11fdf7f2
TL
229 ASSERT_EQ(mutable_cf_options_.target_file_size_base +
230 mutable_cf_options_.target_file_size_base / 10,
231 compaction->OutputFilePreallocationSize());
7c673cae
FG
232}
233
234TEST_F(CompactionPickerTest, NeedsCompactionLevel) {
235 const int kLevels = 6;
236 const int kFileCount = 20;
237
238 for (int level = 0; level < kLevels - 1; ++level) {
239 NewVersionStorage(kLevels, kCompactionStyleLevel);
240 uint64_t file_size = vstorage_->MaxBytesForLevel(level) * 2 / kFileCount;
241 for (int file_count = 1; file_count <= kFileCount; ++file_count) {
242 // start a brand new version in each test.
243 NewVersionStorage(kLevels, kCompactionStyleLevel);
244 for (int i = 0; i < file_count; ++i) {
245 Add(level, i, ToString((i + 100) * 1000).c_str(),
246 ToString((i + 100) * 1000 + 999).c_str(),
247 file_size, 0, i * 100, i * 100 + 99);
248 }
249 UpdateVersionStorageInfo();
250 ASSERT_EQ(vstorage_->CompactionScoreLevel(0), level);
251 ASSERT_EQ(level_compaction_picker.NeedsCompaction(vstorage_.get()),
252 vstorage_->CompactionScore(0) >= 1);
253 // release the version storage
254 DeleteVersionStorage();
255 }
256 }
257}
258
259TEST_F(CompactionPickerTest, Level0TriggerDynamic) {
260 int num_levels = ioptions_.num_levels;
261 ioptions_.level_compaction_dynamic_level_bytes = true;
262 mutable_cf_options_.level0_file_num_compaction_trigger = 2;
263 mutable_cf_options_.max_bytes_for_level_base = 200;
264 mutable_cf_options_.max_bytes_for_level_multiplier = 10;
265 NewVersionStorage(num_levels, kCompactionStyleLevel);
266 Add(0, 1U, "150", "200");
267 Add(0, 2U, "200", "250");
268
269 UpdateVersionStorageInfo();
270
271 std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction(
272 cf_name_, mutable_cf_options_, vstorage_.get(), &log_buffer_));
273 ASSERT_TRUE(compaction.get() != nullptr);
274 ASSERT_EQ(2U, compaction->num_input_files(0));
275 ASSERT_EQ(1U, compaction->input(0, 0)->fd.GetNumber());
276 ASSERT_EQ(2U, compaction->input(0, 1)->fd.GetNumber());
277 ASSERT_EQ(1, static_cast<int>(compaction->num_input_levels()));
278 ASSERT_EQ(num_levels - 1, compaction->output_level());
279}
280
281TEST_F(CompactionPickerTest, Level0TriggerDynamic2) {
282 int num_levels = ioptions_.num_levels;
283 ioptions_.level_compaction_dynamic_level_bytes = true;
284 mutable_cf_options_.level0_file_num_compaction_trigger = 2;
285 mutable_cf_options_.max_bytes_for_level_base = 200;
286 mutable_cf_options_.max_bytes_for_level_multiplier = 10;
287 NewVersionStorage(num_levels, kCompactionStyleLevel);
288 Add(0, 1U, "150", "200");
289 Add(0, 2U, "200", "250");
290 Add(num_levels - 1, 3U, "200", "250", 300U);
291
292 UpdateVersionStorageInfo();
293 ASSERT_EQ(vstorage_->base_level(), num_levels - 2);
294
295 std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction(
296 cf_name_, mutable_cf_options_, vstorage_.get(), &log_buffer_));
297 ASSERT_TRUE(compaction.get() != nullptr);
298 ASSERT_EQ(2U, compaction->num_input_files(0));
299 ASSERT_EQ(1U, compaction->input(0, 0)->fd.GetNumber());
300 ASSERT_EQ(2U, compaction->input(0, 1)->fd.GetNumber());
301 ASSERT_EQ(1, static_cast<int>(compaction->num_input_levels()));
302 ASSERT_EQ(num_levels - 2, compaction->output_level());
303}
304
305TEST_F(CompactionPickerTest, Level0TriggerDynamic3) {
306 int num_levels = ioptions_.num_levels;
307 ioptions_.level_compaction_dynamic_level_bytes = true;
308 mutable_cf_options_.level0_file_num_compaction_trigger = 2;
309 mutable_cf_options_.max_bytes_for_level_base = 200;
310 mutable_cf_options_.max_bytes_for_level_multiplier = 10;
311 NewVersionStorage(num_levels, kCompactionStyleLevel);
312 Add(0, 1U, "150", "200");
313 Add(0, 2U, "200", "250");
314 Add(num_levels - 1, 3U, "200", "250", 300U);
315 Add(num_levels - 1, 4U, "300", "350", 3000U);
316
317 UpdateVersionStorageInfo();
318 ASSERT_EQ(vstorage_->base_level(), num_levels - 3);
319
320 std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction(
321 cf_name_, mutable_cf_options_, vstorage_.get(), &log_buffer_));
322 ASSERT_TRUE(compaction.get() != nullptr);
323 ASSERT_EQ(2U, compaction->num_input_files(0));
324 ASSERT_EQ(1U, compaction->input(0, 0)->fd.GetNumber());
325 ASSERT_EQ(2U, compaction->input(0, 1)->fd.GetNumber());
326 ASSERT_EQ(1, static_cast<int>(compaction->num_input_levels()));
327 ASSERT_EQ(num_levels - 3, compaction->output_level());
328}
329
330TEST_F(CompactionPickerTest, Level0TriggerDynamic4) {
331 int num_levels = ioptions_.num_levels;
332 ioptions_.level_compaction_dynamic_level_bytes = true;
333 mutable_cf_options_.level0_file_num_compaction_trigger = 2;
334 mutable_cf_options_.max_bytes_for_level_base = 200;
335 mutable_cf_options_.max_bytes_for_level_multiplier = 10;
336
337 NewVersionStorage(num_levels, kCompactionStyleLevel);
338 Add(0, 1U, "150", "200");
339 Add(0, 2U, "200", "250");
340 Add(num_levels - 1, 3U, "200", "250", 300U);
341 Add(num_levels - 1, 4U, "300", "350", 3000U);
342 Add(num_levels - 3, 5U, "150", "180", 3U);
343 Add(num_levels - 3, 6U, "181", "300", 3U);
344 Add(num_levels - 3, 7U, "400", "450", 3U);
345
346 UpdateVersionStorageInfo();
347 ASSERT_EQ(vstorage_->base_level(), num_levels - 3);
348
349 std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction(
350 cf_name_, mutable_cf_options_, vstorage_.get(), &log_buffer_));
351 ASSERT_TRUE(compaction.get() != nullptr);
352 ASSERT_EQ(2U, compaction->num_input_files(0));
353 ASSERT_EQ(1U, compaction->input(0, 0)->fd.GetNumber());
354 ASSERT_EQ(2U, compaction->input(0, 1)->fd.GetNumber());
355 ASSERT_EQ(2U, compaction->num_input_files(1));
356 ASSERT_EQ(num_levels - 3, compaction->level(1));
357 ASSERT_EQ(5U, compaction->input(1, 0)->fd.GetNumber());
358 ASSERT_EQ(6U, compaction->input(1, 1)->fd.GetNumber());
359 ASSERT_EQ(2, static_cast<int>(compaction->num_input_levels()));
360 ASSERT_EQ(num_levels - 3, compaction->output_level());
361}
362
363TEST_F(CompactionPickerTest, LevelTriggerDynamic4) {
364 int num_levels = ioptions_.num_levels;
365 ioptions_.level_compaction_dynamic_level_bytes = true;
366 ioptions_.compaction_pri = kMinOverlappingRatio;
367 mutable_cf_options_.level0_file_num_compaction_trigger = 2;
368 mutable_cf_options_.max_bytes_for_level_base = 200;
369 mutable_cf_options_.max_bytes_for_level_multiplier = 10;
370 NewVersionStorage(num_levels, kCompactionStyleLevel);
371 Add(0, 1U, "150", "200");
372 Add(num_levels - 1, 3U, "200", "250", 300U);
373 Add(num_levels - 1, 4U, "300", "350", 3000U);
374 Add(num_levels - 1, 4U, "400", "450", 3U);
375 Add(num_levels - 2, 5U, "150", "180", 300U);
376 Add(num_levels - 2, 6U, "181", "350", 500U);
377 Add(num_levels - 2, 7U, "400", "450", 200U);
378
379 UpdateVersionStorageInfo();
380
381 std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction(
382 cf_name_, mutable_cf_options_, vstorage_.get(), &log_buffer_));
383 ASSERT_TRUE(compaction.get() != nullptr);
384 ASSERT_EQ(1U, compaction->num_input_files(0));
385 ASSERT_EQ(5U, compaction->input(0, 0)->fd.GetNumber());
386 ASSERT_EQ(0, compaction->num_input_files(1));
387 ASSERT_EQ(1U, compaction->num_input_levels());
388 ASSERT_EQ(num_levels - 1, compaction->output_level());
389}
390
391// Universal and FIFO Compactions are not supported in ROCKSDB_LITE
392#ifndef ROCKSDB_LITE
393TEST_F(CompactionPickerTest, NeedsCompactionUniversal) {
394 NewVersionStorage(1, kCompactionStyleUniversal);
395 UniversalCompactionPicker universal_compaction_picker(
396 ioptions_, &icmp_);
11fdf7f2 397 UpdateVersionStorageInfo();
7c673cae
FG
398 // must return false when there's no files.
399 ASSERT_EQ(universal_compaction_picker.NeedsCompaction(vstorage_.get()),
400 false);
7c673cae
FG
401
402 // verify the trigger given different number of L0 files.
403 for (int i = 1;
404 i <= mutable_cf_options_.level0_file_num_compaction_trigger * 2; ++i) {
405 NewVersionStorage(1, kCompactionStyleUniversal);
406 Add(0, i, ToString((i + 100) * 1000).c_str(),
407 ToString((i + 100) * 1000 + 999).c_str(), 1000000, 0, i * 100,
408 i * 100 + 99);
409 UpdateVersionStorageInfo();
410 ASSERT_EQ(level_compaction_picker.NeedsCompaction(vstorage_.get()),
411 vstorage_->CompactionScore(0) >= 1);
412 }
413}
11fdf7f2
TL
414
415TEST_F(CompactionPickerTest, CompactionUniversalIngestBehindReservedLevel) {
416 const uint64_t kFileSize = 100000;
417 NewVersionStorage(1, kCompactionStyleUniversal);
418 ioptions_.allow_ingest_behind = true;
419 ioptions_.num_levels = 3;
420 UniversalCompactionPicker universal_compaction_picker(ioptions_, &icmp_);
421 UpdateVersionStorageInfo();
422 // must return false when there's no files.
423 ASSERT_EQ(universal_compaction_picker.NeedsCompaction(vstorage_.get()),
424 false);
425
426 NewVersionStorage(3, kCompactionStyleUniversal);
427
428 Add(0, 1U, "150", "200", kFileSize, 0, 500, 550);
429 Add(0, 2U, "201", "250", kFileSize, 0, 401, 450);
430 Add(0, 4U, "260", "300", kFileSize, 0, 260, 300);
431 Add(1, 5U, "100", "151", kFileSize, 0, 200, 251);
432 Add(1, 3U, "301", "350", kFileSize, 0, 101, 150);
433 Add(2, 6U, "120", "200", kFileSize, 0, 20, 100);
434
435 UpdateVersionStorageInfo();
436
437 std::unique_ptr<Compaction> compaction(
438 universal_compaction_picker.PickCompaction(
439 cf_name_, mutable_cf_options_, vstorage_.get(), &log_buffer_));
440
441 // output level should be the one above the bottom-most
442 ASSERT_EQ(1, compaction->output_level());
443}
7c673cae
FG
444// Tests if the files can be trivially moved in multi level
445// universal compaction when allow_trivial_move option is set
446// In this test as the input files overlaps, they cannot
447// be trivially moved.
448
449TEST_F(CompactionPickerTest, CannotTrivialMoveUniversal) {
450 const uint64_t kFileSize = 100000;
451
11fdf7f2 452 mutable_cf_options_.compaction_options_universal.allow_trivial_move = true;
7c673cae
FG
453 NewVersionStorage(1, kCompactionStyleUniversal);
454 UniversalCompactionPicker universal_compaction_picker(ioptions_, &icmp_);
11fdf7f2 455 UpdateVersionStorageInfo();
7c673cae
FG
456 // must return false when there's no files.
457 ASSERT_EQ(universal_compaction_picker.NeedsCompaction(vstorage_.get()),
458 false);
459
460 NewVersionStorage(3, kCompactionStyleUniversal);
461
462 Add(0, 1U, "150", "200", kFileSize, 0, 500, 550);
463 Add(0, 2U, "201", "250", kFileSize, 0, 401, 450);
464 Add(0, 4U, "260", "300", kFileSize, 0, 260, 300);
465 Add(1, 5U, "100", "151", kFileSize, 0, 200, 251);
466 Add(1, 3U, "301", "350", kFileSize, 0, 101, 150);
467 Add(2, 6U, "120", "200", kFileSize, 0, 20, 100);
468
469 UpdateVersionStorageInfo();
470
471 std::unique_ptr<Compaction> compaction(
472 universal_compaction_picker.PickCompaction(
473 cf_name_, mutable_cf_options_, vstorage_.get(), &log_buffer_));
474
475 ASSERT_TRUE(!compaction->is_trivial_move());
476}
477// Tests if the files can be trivially moved in multi level
478// universal compaction when allow_trivial_move option is set
479// In this test as the input files doesn't overlaps, they should
480// be trivially moved.
481TEST_F(CompactionPickerTest, AllowsTrivialMoveUniversal) {
482 const uint64_t kFileSize = 100000;
483
11fdf7f2 484 mutable_cf_options_.compaction_options_universal.allow_trivial_move = true;
7c673cae
FG
485 UniversalCompactionPicker universal_compaction_picker(ioptions_, &icmp_);
486
487 NewVersionStorage(3, kCompactionStyleUniversal);
488
489 Add(0, 1U, "150", "200", kFileSize, 0, 500, 550);
490 Add(0, 2U, "201", "250", kFileSize, 0, 401, 450);
491 Add(0, 4U, "260", "300", kFileSize, 0, 260, 300);
492 Add(1, 5U, "010", "080", kFileSize, 0, 200, 251);
493 Add(2, 3U, "301", "350", kFileSize, 0, 101, 150);
494
495 UpdateVersionStorageInfo();
496
497 std::unique_ptr<Compaction> compaction(
498 universal_compaction_picker.PickCompaction(
499 cf_name_, mutable_cf_options_, vstorage_.get(), &log_buffer_));
500
501 ASSERT_TRUE(compaction->is_trivial_move());
502}
503
504TEST_F(CompactionPickerTest, NeedsCompactionFIFO) {
505 NewVersionStorage(1, kCompactionStyleFIFO);
506 const int kFileCount =
507 mutable_cf_options_.level0_file_num_compaction_trigger * 3;
508 const uint64_t kFileSize = 100000;
509 const uint64_t kMaxSize = kFileSize * kFileCount / 2;
510
511 fifo_options_.max_table_files_size = kMaxSize;
11fdf7f2 512 mutable_cf_options_.compaction_options_fifo = fifo_options_;
7c673cae
FG
513 FIFOCompactionPicker fifo_compaction_picker(ioptions_, &icmp_);
514 UpdateVersionStorageInfo();
515 // must return false when there's no files.
516 ASSERT_EQ(fifo_compaction_picker.NeedsCompaction(vstorage_.get()), false);
517
518 // verify whether compaction is needed based on the current
519 // size of L0 files.
520 uint64_t current_size = 0;
521 for (int i = 1; i <= kFileCount; ++i) {
522 NewVersionStorage(1, kCompactionStyleFIFO);
523 Add(0, i, ToString((i + 100) * 1000).c_str(),
524 ToString((i + 100) * 1000 + 999).c_str(),
525 kFileSize, 0, i * 100, i * 100 + 99);
526 current_size += kFileSize;
527 UpdateVersionStorageInfo();
11fdf7f2 528 ASSERT_EQ(fifo_compaction_picker.NeedsCompaction(vstorage_.get()),
7c673cae
FG
529 vstorage_->CompactionScore(0) >= 1);
530 }
531}
532#endif // ROCKSDB_LITE
533
534TEST_F(CompactionPickerTest, CompactionPriMinOverlapping1) {
535 NewVersionStorage(6, kCompactionStyleLevel);
536 ioptions_.compaction_pri = kMinOverlappingRatio;
11fdf7f2 537 mutable_cf_options_.target_file_size_base = 100000000000;
7c673cae
FG
538 mutable_cf_options_.target_file_size_multiplier = 10;
539 mutable_cf_options_.max_bytes_for_level_base = 10 * 1024 * 1024;
11fdf7f2 540 mutable_cf_options_.RefreshDerivedOptions(ioptions_);
7c673cae
FG
541
542 Add(2, 6U, "150", "179", 50000000U);
543 Add(2, 7U, "180", "220", 50000000U);
544 Add(2, 8U, "321", "400", 50000000U); // File not overlapping
545 Add(2, 9U, "721", "800", 50000000U);
546
547 Add(3, 26U, "150", "170", 260000000U);
548 Add(3, 27U, "171", "179", 260000000U);
549 Add(3, 28U, "191", "220", 260000000U);
550 Add(3, 29U, "221", "300", 260000000U);
551 Add(3, 30U, "750", "900", 260000000U);
552 UpdateVersionStorageInfo();
553
554 std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction(
555 cf_name_, mutable_cf_options_, vstorage_.get(), &log_buffer_));
556 ASSERT_TRUE(compaction.get() != nullptr);
557 ASSERT_EQ(1U, compaction->num_input_files(0));
558 // Pick file 8 because it overlaps with 0 files on level 3.
559 ASSERT_EQ(8U, compaction->input(0, 0)->fd.GetNumber());
11fdf7f2
TL
560 // Compaction input size * 1.1
561 ASSERT_GE(uint64_t{55000000}, compaction->OutputFilePreallocationSize());
7c673cae
FG
562}
563
564TEST_F(CompactionPickerTest, CompactionPriMinOverlapping2) {
565 NewVersionStorage(6, kCompactionStyleLevel);
566 ioptions_.compaction_pri = kMinOverlappingRatio;
567 mutable_cf_options_.target_file_size_base = 10000000;
568 mutable_cf_options_.target_file_size_multiplier = 10;
569 mutable_cf_options_.max_bytes_for_level_base = 10 * 1024 * 1024;
570
571 Add(2, 6U, "150", "175",
572 60000000U); // Overlaps with file 26, 27, total size 521M
573 Add(2, 7U, "176", "200", 60000000U); // Overlaps with file 27, 28, total size
574 // 520M, the smalelst overlapping
575 Add(2, 8U, "201", "300",
576 60000000U); // Overlaps with file 28, 29, total size 521M
577
578 Add(3, 26U, "100", "110", 261000000U);
579 Add(3, 26U, "150", "170", 261000000U);
580 Add(3, 27U, "171", "179", 260000000U);
581 Add(3, 28U, "191", "220", 260000000U);
582 Add(3, 29U, "221", "300", 261000000U);
583 Add(3, 30U, "321", "400", 261000000U);
584 UpdateVersionStorageInfo();
585
586 std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction(
587 cf_name_, mutable_cf_options_, vstorage_.get(), &log_buffer_));
588 ASSERT_TRUE(compaction.get() != nullptr);
589 ASSERT_EQ(1U, compaction->num_input_files(0));
590 // Picking file 7 because overlapping ratio is the biggest.
591 ASSERT_EQ(7U, compaction->input(0, 0)->fd.GetNumber());
592}
593
594TEST_F(CompactionPickerTest, CompactionPriMinOverlapping3) {
595 NewVersionStorage(6, kCompactionStyleLevel);
596 ioptions_.compaction_pri = kMinOverlappingRatio;
597 mutable_cf_options_.max_bytes_for_level_base = 10000000;
598 mutable_cf_options_.max_bytes_for_level_multiplier = 10;
599
600 // file 7 and 8 over lap with the same file, but file 8 is smaller so
601 // it will be picked.
602 Add(2, 6U, "150", "167", 60000000U); // Overlaps with file 26, 27
603 Add(2, 7U, "168", "169", 60000000U); // Overlaps with file 27
604 Add(2, 8U, "201", "300", 61000000U); // Overlaps with file 28, but the file
605 // itself is larger. Should be picked.
606
607 Add(3, 26U, "160", "165", 260000000U);
608 Add(3, 27U, "166", "170", 260000000U);
609 Add(3, 28U, "180", "400", 260000000U);
610 Add(3, 29U, "401", "500", 260000000U);
611 UpdateVersionStorageInfo();
612
613 std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction(
614 cf_name_, mutable_cf_options_, vstorage_.get(), &log_buffer_));
615 ASSERT_TRUE(compaction.get() != nullptr);
616 ASSERT_EQ(1U, compaction->num_input_files(0));
617 // Picking file 8 because overlapping ratio is the biggest.
618 ASSERT_EQ(8U, compaction->input(0, 0)->fd.GetNumber());
619}
620
494da23a
TL
621TEST_F(CompactionPickerTest, CompactionPriMinOverlapping4) {
622 NewVersionStorage(6, kCompactionStyleLevel);
623 ioptions_.compaction_pri = kMinOverlappingRatio;
624 mutable_cf_options_.max_bytes_for_level_base = 10000000;
625 mutable_cf_options_.max_bytes_for_level_multiplier = 10;
626
627 // file 7 and 8 over lap with the same file, but file 8 is smaller so
628 // it will be picked.
629 // Overlaps with file 26, 27. And the file is compensated so will be
630 // picked up.
631 Add(2, 6U, "150", "167", 60000000U, 0, 100, 100, 180000000U);
632 Add(2, 7U, "168", "169", 60000000U); // Overlaps with file 27
633 Add(2, 8U, "201", "300", 61000000U); // Overlaps with file 28
634
635 Add(3, 26U, "160", "165", 60000000U);
636 // Boosted file size in output level is not considered.
637 Add(3, 27U, "166", "170", 60000000U, 0, 100, 100, 260000000U);
638 Add(3, 28U, "180", "400", 60000000U);
639 Add(3, 29U, "401", "500", 60000000U);
640 UpdateVersionStorageInfo();
641
642 std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction(
643 cf_name_, mutable_cf_options_, vstorage_.get(), &log_buffer_));
644 ASSERT_TRUE(compaction.get() != nullptr);
645 ASSERT_EQ(1U, compaction->num_input_files(0));
646 // Picking file 8 because overlapping ratio is the biggest.
647 ASSERT_EQ(6U, compaction->input(0, 0)->fd.GetNumber());
648}
649
7c673cae
FG
650// This test exhibits the bug where we don't properly reset parent_index in
651// PickCompaction()
652TEST_F(CompactionPickerTest, ParentIndexResetBug) {
653 int num_levels = ioptions_.num_levels;
654 mutable_cf_options_.level0_file_num_compaction_trigger = 2;
655 mutable_cf_options_.max_bytes_for_level_base = 200;
656 NewVersionStorage(num_levels, kCompactionStyleLevel);
657 Add(0, 1U, "150", "200"); // <- marked for compaction
658 Add(1, 3U, "400", "500", 600); // <- this one needs compacting
659 Add(2, 4U, "150", "200");
660 Add(2, 5U, "201", "210");
661 Add(2, 6U, "300", "310");
662 Add(2, 7U, "400", "500"); // <- being compacted
663
664 vstorage_->LevelFiles(2)[3]->being_compacted = true;
665 vstorage_->LevelFiles(0)[0]->marked_for_compaction = true;
666
667 UpdateVersionStorageInfo();
668
669 std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction(
670 cf_name_, mutable_cf_options_, vstorage_.get(), &log_buffer_));
671}
672
673// This test checks ExpandWhileOverlapping() by having overlapping user keys
674// ranges (with different sequence numbers) in the input files.
675TEST_F(CompactionPickerTest, OverlappingUserKeys) {
676 NewVersionStorage(6, kCompactionStyleLevel);
677 ioptions_.compaction_pri = kByCompensatedSize;
678
679 Add(1, 1U, "100", "150", 1U);
680 // Overlapping user keys
681 Add(1, 2U, "200", "400", 1U);
682 Add(1, 3U, "400", "500", 1000000000U, 0, 0);
683 Add(2, 4U, "600", "700", 1U);
684 UpdateVersionStorageInfo();
685
686 std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction(
687 cf_name_, mutable_cf_options_, vstorage_.get(), &log_buffer_));
688 ASSERT_TRUE(compaction.get() != nullptr);
689 ASSERT_EQ(1U, compaction->num_input_levels());
690 ASSERT_EQ(2U, compaction->num_input_files(0));
691 ASSERT_EQ(2U, compaction->input(0, 0)->fd.GetNumber());
692 ASSERT_EQ(3U, compaction->input(0, 1)->fd.GetNumber());
693}
694
695TEST_F(CompactionPickerTest, OverlappingUserKeys2) {
696 NewVersionStorage(6, kCompactionStyleLevel);
697 // Overlapping user keys on same level and output level
698 Add(1, 1U, "200", "400", 1000000000U);
699 Add(1, 2U, "400", "500", 1U, 0, 0);
700 Add(2, 3U, "000", "100", 1U);
701 Add(2, 4U, "100", "600", 1U, 0, 0);
702 Add(2, 5U, "600", "700", 1U, 0, 0);
703 UpdateVersionStorageInfo();
704
705 std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction(
706 cf_name_, mutable_cf_options_, vstorage_.get(), &log_buffer_));
707 ASSERT_TRUE(compaction.get() != nullptr);
708 ASSERT_EQ(2U, compaction->num_input_levels());
709 ASSERT_EQ(2U, compaction->num_input_files(0));
710 ASSERT_EQ(3U, compaction->num_input_files(1));
711 ASSERT_EQ(1U, compaction->input(0, 0)->fd.GetNumber());
712 ASSERT_EQ(2U, compaction->input(0, 1)->fd.GetNumber());
713 ASSERT_EQ(3U, compaction->input(1, 0)->fd.GetNumber());
714 ASSERT_EQ(4U, compaction->input(1, 1)->fd.GetNumber());
715 ASSERT_EQ(5U, compaction->input(1, 2)->fd.GetNumber());
716}
717
718TEST_F(CompactionPickerTest, OverlappingUserKeys3) {
719 NewVersionStorage(6, kCompactionStyleLevel);
720 // Chain of overlapping user key ranges (forces ExpandWhileOverlapping() to
721 // expand multiple times)
722 Add(1, 1U, "100", "150", 1U);
723 Add(1, 2U, "150", "200", 1U, 0, 0);
724 Add(1, 3U, "200", "250", 1000000000U, 0, 0);
725 Add(1, 4U, "250", "300", 1U, 0, 0);
726 Add(1, 5U, "300", "350", 1U, 0, 0);
727 // Output level overlaps with the beginning and the end of the chain
728 Add(2, 6U, "050", "100", 1U);
729 Add(2, 7U, "350", "400", 1U);
730 UpdateVersionStorageInfo();
731
732 std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction(
733 cf_name_, mutable_cf_options_, vstorage_.get(), &log_buffer_));
734 ASSERT_TRUE(compaction.get() != nullptr);
735 ASSERT_EQ(2U, compaction->num_input_levels());
736 ASSERT_EQ(5U, compaction->num_input_files(0));
737 ASSERT_EQ(2U, compaction->num_input_files(1));
738 ASSERT_EQ(1U, compaction->input(0, 0)->fd.GetNumber());
739 ASSERT_EQ(2U, compaction->input(0, 1)->fd.GetNumber());
740 ASSERT_EQ(3U, compaction->input(0, 2)->fd.GetNumber());
741 ASSERT_EQ(4U, compaction->input(0, 3)->fd.GetNumber());
742 ASSERT_EQ(5U, compaction->input(0, 4)->fd.GetNumber());
743 ASSERT_EQ(6U, compaction->input(1, 0)->fd.GetNumber());
744 ASSERT_EQ(7U, compaction->input(1, 1)->fd.GetNumber());
745}
746
747TEST_F(CompactionPickerTest, OverlappingUserKeys4) {
748 NewVersionStorage(6, kCompactionStyleLevel);
749 mutable_cf_options_.max_bytes_for_level_base = 1000000;
750
751 Add(1, 1U, "100", "150", 1U);
752 Add(1, 2U, "150", "199", 1U, 0, 0);
753 Add(1, 3U, "200", "250", 1100000U, 0, 0);
754 Add(1, 4U, "251", "300", 1U, 0, 0);
755 Add(1, 5U, "300", "350", 1U, 0, 0);
756
757 Add(2, 6U, "100", "115", 1U);
758 Add(2, 7U, "125", "325", 1U);
759 Add(2, 8U, "350", "400", 1U);
760 UpdateVersionStorageInfo();
761
762 std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction(
763 cf_name_, mutable_cf_options_, vstorage_.get(), &log_buffer_));
764 ASSERT_TRUE(compaction.get() != nullptr);
765 ASSERT_EQ(2U, compaction->num_input_levels());
766 ASSERT_EQ(1U, compaction->num_input_files(0));
767 ASSERT_EQ(1U, compaction->num_input_files(1));
768 ASSERT_EQ(3U, compaction->input(0, 0)->fd.GetNumber());
769 ASSERT_EQ(7U, compaction->input(1, 0)->fd.GetNumber());
770}
771
772TEST_F(CompactionPickerTest, OverlappingUserKeys5) {
773 NewVersionStorage(6, kCompactionStyleLevel);
774 // Overlapping user keys on same level and output level
775 Add(1, 1U, "200", "400", 1000000000U);
776 Add(1, 2U, "400", "500", 1U, 0, 0);
777 Add(2, 3U, "000", "100", 1U);
778 Add(2, 4U, "100", "600", 1U, 0, 0);
779 Add(2, 5U, "600", "700", 1U, 0, 0);
780
781 vstorage_->LevelFiles(2)[2]->being_compacted = true;
782
783 UpdateVersionStorageInfo();
784
785 std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction(
786 cf_name_, mutable_cf_options_, vstorage_.get(), &log_buffer_));
787 ASSERT_TRUE(compaction.get() == nullptr);
788}
789
790TEST_F(CompactionPickerTest, OverlappingUserKeys6) {
791 NewVersionStorage(6, kCompactionStyleLevel);
792 // Overlapping user keys on same level and output level
793 Add(1, 1U, "200", "400", 1U, 0, 0);
794 Add(1, 2U, "401", "500", 1U, 0, 0);
795 Add(2, 3U, "000", "100", 1U);
796 Add(2, 4U, "100", "300", 1U, 0, 0);
797 Add(2, 5U, "305", "450", 1U, 0, 0);
798 Add(2, 6U, "460", "600", 1U, 0, 0);
799 Add(2, 7U, "600", "700", 1U, 0, 0);
800
801 vstorage_->LevelFiles(1)[0]->marked_for_compaction = true;
802 vstorage_->LevelFiles(1)[1]->marked_for_compaction = true;
803
804 UpdateVersionStorageInfo();
805
806 std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction(
807 cf_name_, mutable_cf_options_, vstorage_.get(), &log_buffer_));
808 ASSERT_TRUE(compaction.get() != nullptr);
809 ASSERT_EQ(2U, compaction->num_input_levels());
810 ASSERT_EQ(1U, compaction->num_input_files(0));
811 ASSERT_EQ(3U, compaction->num_input_files(1));
812}
813
814TEST_F(CompactionPickerTest, OverlappingUserKeys7) {
815 NewVersionStorage(6, kCompactionStyleLevel);
816 mutable_cf_options_.max_compaction_bytes = 100000000000u;
817 // Overlapping user keys on same level and output level
818 Add(1, 1U, "200", "400", 1U, 0, 0);
819 Add(1, 2U, "401", "500", 1000000000U, 0, 0);
820 Add(2, 3U, "100", "250", 1U);
821 Add(2, 4U, "300", "600", 1U, 0, 0);
822 Add(2, 5U, "600", "800", 1U, 0, 0);
823
824 UpdateVersionStorageInfo();
825
826 std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction(
827 cf_name_, mutable_cf_options_, vstorage_.get(), &log_buffer_));
828 ASSERT_TRUE(compaction.get() != nullptr);
829 ASSERT_EQ(2U, compaction->num_input_levels());
830 ASSERT_GE(1U, compaction->num_input_files(0));
831 ASSERT_GE(2U, compaction->num_input_files(1));
832 // File 5 has to be included in the compaction
833 ASSERT_EQ(5U, compaction->inputs(1)->back()->fd.GetNumber());
834}
835
836TEST_F(CompactionPickerTest, OverlappingUserKeys8) {
837 NewVersionStorage(6, kCompactionStyleLevel);
838 mutable_cf_options_.max_compaction_bytes = 100000000000u;
839 // grow the number of inputs in "level" without
840 // changing the number of "level+1" files we pick up
841 // Expand input level as much as possible
842 // no overlapping case
843 Add(1, 1U, "101", "150", 1U);
844 Add(1, 2U, "151", "200", 1U);
845 Add(1, 3U, "201", "300", 1000000000U);
846 Add(1, 4U, "301", "400", 1U);
847 Add(1, 5U, "401", "500", 1U);
848 Add(2, 6U, "150", "200", 1U);
849 Add(2, 7U, "200", "450", 1U, 0, 0);
850 Add(2, 8U, "500", "600", 1U);
851
852 UpdateVersionStorageInfo();
853
854 std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction(
855 cf_name_, mutable_cf_options_, vstorage_.get(), &log_buffer_));
856 ASSERT_TRUE(compaction.get() != nullptr);
857 ASSERT_EQ(2U, compaction->num_input_levels());
858 ASSERT_EQ(3U, compaction->num_input_files(0));
859 ASSERT_EQ(2U, compaction->num_input_files(1));
860 ASSERT_EQ(2U, compaction->input(0, 0)->fd.GetNumber());
861 ASSERT_EQ(3U, compaction->input(0, 1)->fd.GetNumber());
862 ASSERT_EQ(4U, compaction->input(0, 2)->fd.GetNumber());
863 ASSERT_EQ(6U, compaction->input(1, 0)->fd.GetNumber());
864 ASSERT_EQ(7U, compaction->input(1, 1)->fd.GetNumber());
865}
866
867TEST_F(CompactionPickerTest, OverlappingUserKeys9) {
868 NewVersionStorage(6, kCompactionStyleLevel);
869 mutable_cf_options_.max_compaction_bytes = 100000000000u;
870 // grow the number of inputs in "level" without
871 // changing the number of "level+1" files we pick up
872 // Expand input level as much as possible
873 // overlapping case
874 Add(1, 1U, "121", "150", 1U);
875 Add(1, 2U, "151", "200", 1U);
876 Add(1, 3U, "201", "300", 1000000000U);
877 Add(1, 4U, "301", "400", 1U);
878 Add(1, 5U, "401", "500", 1U);
879 Add(2, 6U, "100", "120", 1U);
880 Add(2, 7U, "150", "200", 1U);
881 Add(2, 8U, "200", "450", 1U, 0, 0);
882 Add(2, 9U, "501", "600", 1U);
883
884 UpdateVersionStorageInfo();
885
886 std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction(
887 cf_name_, mutable_cf_options_, vstorage_.get(), &log_buffer_));
888 ASSERT_TRUE(compaction.get() != nullptr);
889 ASSERT_EQ(2U, compaction->num_input_levels());
890 ASSERT_EQ(5U, compaction->num_input_files(0));
891 ASSERT_EQ(2U, compaction->num_input_files(1));
892 ASSERT_EQ(1U, compaction->input(0, 0)->fd.GetNumber());
893 ASSERT_EQ(2U, compaction->input(0, 1)->fd.GetNumber());
894 ASSERT_EQ(3U, compaction->input(0, 2)->fd.GetNumber());
895 ASSERT_EQ(4U, compaction->input(0, 3)->fd.GetNumber());
896 ASSERT_EQ(7U, compaction->input(1, 0)->fd.GetNumber());
897 ASSERT_EQ(8U, compaction->input(1, 1)->fd.GetNumber());
898}
899
11fdf7f2
TL
900TEST_F(CompactionPickerTest, OverlappingUserKeys10) {
901 // Locked file encountered when pulling in extra input-level files with same
902 // user keys. Verify we pick the next-best file from the same input level.
903 NewVersionStorage(6, kCompactionStyleLevel);
904 mutable_cf_options_.max_compaction_bytes = 100000000000u;
905
906 // file_number 2U is largest and thus first choice. But it overlaps with
907 // file_number 1U which is being compacted. So instead we pick the next-
908 // biggest file, 3U, which is eligible for compaction.
909 Add(1 /* level */, 1U /* file_number */, "100" /* smallest */,
910 "150" /* largest */, 1U /* file_size */);
911 file_map_[1U].first->being_compacted = true;
912 Add(1 /* level */, 2U /* file_number */, "150" /* smallest */,
913 "200" /* largest */, 1000000000U /* file_size */, 0 /* smallest_seq */,
914 0 /* largest_seq */);
915 Add(1 /* level */, 3U /* file_number */, "201" /* smallest */,
916 "250" /* largest */, 900000000U /* file_size */);
917 Add(2 /* level */, 4U /* file_number */, "100" /* smallest */,
918 "150" /* largest */, 1U /* file_size */);
919 Add(2 /* level */, 5U /* file_number */, "151" /* smallest */,
920 "200" /* largest */, 1U /* file_size */);
921 Add(2 /* level */, 6U /* file_number */, "201" /* smallest */,
922 "250" /* largest */, 1U /* file_size */);
923
924 UpdateVersionStorageInfo();
925
926 std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction(
927 cf_name_, mutable_cf_options_, vstorage_.get(), &log_buffer_));
928 ASSERT_TRUE(compaction.get() != nullptr);
929 ASSERT_EQ(2U, compaction->num_input_levels());
930 ASSERT_EQ(1U, compaction->num_input_files(0));
931 ASSERT_EQ(1U, compaction->num_input_files(1));
932 ASSERT_EQ(3U, compaction->input(0, 0)->fd.GetNumber());
933 ASSERT_EQ(6U, compaction->input(1, 0)->fd.GetNumber());
934}
935
936TEST_F(CompactionPickerTest, OverlappingUserKeys11) {
937 // Locked file encountered when pulling in extra output-level files with same
938 // user keys. Expected to skip that compaction and pick the next-best choice.
939 NewVersionStorage(6, kCompactionStyleLevel);
940 mutable_cf_options_.max_compaction_bytes = 100000000000u;
941
942 // score(L1) = 3.7
943 // score(L2) = 1.85
944 // There is no eligible file in L1 to compact since both candidates pull in
945 // file_number 5U, which overlaps with a file pending compaction (6U). The
946 // first eligible compaction is from L2->L3.
947 Add(1 /* level */, 2U /* file_number */, "151" /* smallest */,
948 "200" /* largest */, 1000000000U /* file_size */);
949 Add(1 /* level */, 3U /* file_number */, "201" /* smallest */,
950 "250" /* largest */, 1U /* file_size */);
951 Add(2 /* level */, 4U /* file_number */, "100" /* smallest */,
952 "149" /* largest */, 5000000000U /* file_size */);
953 Add(2 /* level */, 5U /* file_number */, "150" /* smallest */,
954 "201" /* largest */, 1U /* file_size */);
955 Add(2 /* level */, 6U /* file_number */, "201" /* smallest */,
956 "249" /* largest */, 1U /* file_size */, 0 /* smallest_seq */,
957 0 /* largest_seq */);
958 file_map_[6U].first->being_compacted = true;
959 Add(3 /* level */, 7U /* file_number */, "100" /* smallest */,
960 "149" /* largest */, 1U /* file_size */);
961
962 UpdateVersionStorageInfo();
963
964 std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction(
965 cf_name_, mutable_cf_options_, vstorage_.get(), &log_buffer_));
966 ASSERT_TRUE(compaction.get() != nullptr);
967 ASSERT_EQ(2U, compaction->num_input_levels());
968 ASSERT_EQ(1U, compaction->num_input_files(0));
969 ASSERT_EQ(1U, compaction->num_input_files(1));
970 ASSERT_EQ(4U, compaction->input(0, 0)->fd.GetNumber());
971 ASSERT_EQ(7U, compaction->input(1, 0)->fd.GetNumber());
972}
973
7c673cae
FG
974TEST_F(CompactionPickerTest, NotScheduleL1IfL0WithHigherPri1) {
975 NewVersionStorage(6, kCompactionStyleLevel);
976 mutable_cf_options_.level0_file_num_compaction_trigger = 2;
977 mutable_cf_options_.max_bytes_for_level_base = 900000000U;
978
979 // 6 L0 files, score 3.
980 Add(0, 1U, "000", "400", 1U);
981 Add(0, 2U, "001", "400", 1U, 0, 0);
982 Add(0, 3U, "001", "400", 1000000000U, 0, 0);
983 Add(0, 31U, "001", "400", 1000000000U, 0, 0);
984 Add(0, 32U, "001", "400", 1000000000U, 0, 0);
985 Add(0, 33U, "001", "400", 1000000000U, 0, 0);
986
987 // L1 total size 2GB, score 2.2. If one file being comapcted, score 1.1.
988 Add(1, 4U, "050", "300", 1000000000U, 0, 0);
989 file_map_[4u].first->being_compacted = true;
990 Add(1, 5U, "301", "350", 1000000000U, 0, 0);
991
992 // Output level overlaps with the beginning and the end of the chain
993 Add(2, 6U, "050", "100", 1U);
994 Add(2, 7U, "300", "400", 1U);
995
996 // No compaction should be scheduled, if L0 has higher priority than L1
997 // but L0->L1 compaction is blocked by a file in L1 being compacted.
998 UpdateVersionStorageInfo();
999 ASSERT_EQ(0, vstorage_->CompactionScoreLevel(0));
1000 ASSERT_EQ(1, vstorage_->CompactionScoreLevel(1));
1001 std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction(
1002 cf_name_, mutable_cf_options_, vstorage_.get(), &log_buffer_));
1003 ASSERT_TRUE(compaction.get() == nullptr);
1004}
1005
1006TEST_F(CompactionPickerTest, NotScheduleL1IfL0WithHigherPri2) {
1007 NewVersionStorage(6, kCompactionStyleLevel);
1008 mutable_cf_options_.level0_file_num_compaction_trigger = 2;
1009 mutable_cf_options_.max_bytes_for_level_base = 900000000U;
1010
1011 // 6 L0 files, score 3.
1012 Add(0, 1U, "000", "400", 1U);
1013 Add(0, 2U, "001", "400", 1U, 0, 0);
1014 Add(0, 3U, "001", "400", 1000000000U, 0, 0);
1015 Add(0, 31U, "001", "400", 1000000000U, 0, 0);
1016 Add(0, 32U, "001", "400", 1000000000U, 0, 0);
1017 Add(0, 33U, "001", "400", 1000000000U, 0, 0);
1018
1019 // L1 total size 2GB, score 2.2. If one file being comapcted, score 1.1.
1020 Add(1, 4U, "050", "300", 1000000000U, 0, 0);
1021 Add(1, 5U, "301", "350", 1000000000U, 0, 0);
1022
1023 // Output level overlaps with the beginning and the end of the chain
1024 Add(2, 6U, "050", "100", 1U);
1025 Add(2, 7U, "300", "400", 1U);
1026
1027 // If no file in L1 being compacted, L0->L1 compaction will be scheduled.
1028 UpdateVersionStorageInfo(); // being_compacted flag is cleared here.
1029 ASSERT_EQ(0, vstorage_->CompactionScoreLevel(0));
1030 ASSERT_EQ(1, vstorage_->CompactionScoreLevel(1));
1031 std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction(
1032 cf_name_, mutable_cf_options_, vstorage_.get(), &log_buffer_));
1033 ASSERT_TRUE(compaction.get() != nullptr);
1034}
1035
1036TEST_F(CompactionPickerTest, NotScheduleL1IfL0WithHigherPri3) {
1037 NewVersionStorage(6, kCompactionStyleLevel);
1038 mutable_cf_options_.level0_file_num_compaction_trigger = 2;
1039 mutable_cf_options_.max_bytes_for_level_base = 900000000U;
1040
1041 // 6 L0 files, score 3.
1042 Add(0, 1U, "000", "400", 1U);
1043 Add(0, 2U, "001", "400", 1U, 0, 0);
1044 Add(0, 3U, "001", "400", 1000000000U, 0, 0);
1045 Add(0, 31U, "001", "400", 1000000000U, 0, 0);
1046 Add(0, 32U, "001", "400", 1000000000U, 0, 0);
1047 Add(0, 33U, "001", "400", 1000000000U, 0, 0);
1048
1049 // L1 score more than 6.
1050 Add(1, 4U, "050", "300", 1000000000U, 0, 0);
1051 file_map_[4u].first->being_compacted = true;
1052 Add(1, 5U, "301", "350", 1000000000U, 0, 0);
1053 Add(1, 51U, "351", "400", 6000000000U, 0, 0);
1054
1055 // Output level overlaps with the beginning and the end of the chain
1056 Add(2, 6U, "050", "100", 1U);
1057 Add(2, 7U, "300", "400", 1U);
1058
1059 // If score in L1 is larger than L0, L1 compaction goes through despite
1060 // there is pending L0 compaction.
1061 UpdateVersionStorageInfo();
1062 ASSERT_EQ(1, vstorage_->CompactionScoreLevel(0));
1063 ASSERT_EQ(0, vstorage_->CompactionScoreLevel(1));
1064 std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction(
1065 cf_name_, mutable_cf_options_, vstorage_.get(), &log_buffer_));
1066 ASSERT_TRUE(compaction.get() != nullptr);
1067}
1068
1069TEST_F(CompactionPickerTest, EstimateCompactionBytesNeeded1) {
1070 int num_levels = ioptions_.num_levels;
1071 ioptions_.level_compaction_dynamic_level_bytes = false;
11fdf7f2 1072 mutable_cf_options_.level0_file_num_compaction_trigger = 4;
7c673cae
FG
1073 mutable_cf_options_.max_bytes_for_level_base = 1000;
1074 mutable_cf_options_.max_bytes_for_level_multiplier = 10;
1075 NewVersionStorage(num_levels, kCompactionStyleLevel);
1076 Add(0, 1U, "150", "200", 200);
1077 Add(0, 2U, "150", "200", 200);
1078 Add(0, 3U, "150", "200", 200);
1079 // Level 1 is over target by 200
1080 Add(1, 4U, "400", "500", 600);
1081 Add(1, 5U, "600", "700", 600);
1082 // Level 2 is less than target 10000 even added size of level 1
1083 // Size ratio of L2/L1 is 9600 / 1200 = 8
1084 Add(2, 6U, "150", "200", 2500);
1085 Add(2, 7U, "201", "210", 2000);
1086 Add(2, 8U, "300", "310", 2600);
1087 Add(2, 9U, "400", "500", 2500);
1088 // Level 3 exceeds target 100,000 of 1000
1089 Add(3, 10U, "400", "500", 101000);
1090 // Level 4 exceeds target 1,000,000 by 900 after adding size from level 3
1091 // Size ratio L4/L3 is 9.9
1092 // After merge from L3, L4 size is 1000900
1093 Add(4, 11U, "400", "500", 999900);
1094 Add(5, 11U, "400", "500", 8007200);
1095
1096 UpdateVersionStorageInfo();
1097
1098 ASSERT_EQ(200u * 9u + 10900u + 900u * 9,
1099 vstorage_->estimated_compaction_needed_bytes());
1100}
1101
1102TEST_F(CompactionPickerTest, EstimateCompactionBytesNeeded2) {
1103 int num_levels = ioptions_.num_levels;
1104 ioptions_.level_compaction_dynamic_level_bytes = false;
1105 mutable_cf_options_.level0_file_num_compaction_trigger = 3;
1106 mutable_cf_options_.max_bytes_for_level_base = 1000;
1107 mutable_cf_options_.max_bytes_for_level_multiplier = 10;
1108 NewVersionStorage(num_levels, kCompactionStyleLevel);
1109 Add(0, 1U, "150", "200", 200);
1110 Add(0, 2U, "150", "200", 200);
1111 Add(0, 4U, "150", "200", 200);
1112 Add(0, 5U, "150", "200", 200);
1113 Add(0, 6U, "150", "200", 200);
1114 // Level 1 size will be 1400 after merging with L0
1115 Add(1, 7U, "400", "500", 200);
1116 Add(1, 8U, "600", "700", 200);
1117 // Level 2 is less than target 10000 even added size of level 1
1118 Add(2, 9U, "150", "200", 9100);
1119 // Level 3 over the target, but since level 4 is empty, we assume it will be
1120 // a trivial move.
1121 Add(3, 10U, "400", "500", 101000);
1122
1123 UpdateVersionStorageInfo();
1124
1125 // estimated L1->L2 merge: 400 * (9100.0 / 1400.0 + 1.0)
1126 ASSERT_EQ(1400u + 3000u, vstorage_->estimated_compaction_needed_bytes());
1127}
1128
1129TEST_F(CompactionPickerTest, EstimateCompactionBytesNeeded3) {
1130 int num_levels = ioptions_.num_levels;
1131 ioptions_.level_compaction_dynamic_level_bytes = false;
1132 mutable_cf_options_.level0_file_num_compaction_trigger = 3;
1133 mutable_cf_options_.max_bytes_for_level_base = 1000;
1134 mutable_cf_options_.max_bytes_for_level_multiplier = 10;
1135 NewVersionStorage(num_levels, kCompactionStyleLevel);
1136 Add(0, 1U, "150", "200", 2000);
1137 Add(0, 2U, "150", "200", 2000);
1138 Add(0, 4U, "150", "200", 2000);
1139 Add(0, 5U, "150", "200", 2000);
1140 Add(0, 6U, "150", "200", 1000);
1141 // Level 1 size will be 10000 after merging with L0
1142 Add(1, 7U, "400", "500", 500);
1143 Add(1, 8U, "600", "700", 500);
1144
1145 Add(2, 9U, "150", "200", 10000);
1146
1147 UpdateVersionStorageInfo();
1148
1149 ASSERT_EQ(10000u + 18000u, vstorage_->estimated_compaction_needed_bytes());
1150}
1151
1152TEST_F(CompactionPickerTest, EstimateCompactionBytesNeededDynamicLevel) {
1153 int num_levels = ioptions_.num_levels;
1154 ioptions_.level_compaction_dynamic_level_bytes = true;
1155 mutable_cf_options_.level0_file_num_compaction_trigger = 3;
1156 mutable_cf_options_.max_bytes_for_level_base = 1000;
1157 mutable_cf_options_.max_bytes_for_level_multiplier = 10;
1158 NewVersionStorage(num_levels, kCompactionStyleLevel);
1159
1160 // Set Last level size 50000
1161 // num_levels - 1 target 5000
11fdf7f2
TL
1162 // num_levels - 2 is base level with target 1000 (rounded up to
1163 // max_bytes_for_level_base).
7c673cae
FG
1164 Add(num_levels - 1, 10U, "400", "500", 50000);
1165
1166 Add(0, 1U, "150", "200", 200);
1167 Add(0, 2U, "150", "200", 200);
1168 Add(0, 4U, "150", "200", 200);
1169 Add(0, 5U, "150", "200", 200);
1170 Add(0, 6U, "150", "200", 200);
1171 // num_levels - 3 is over target by 100 + 1000
11fdf7f2
TL
1172 Add(num_levels - 3, 7U, "400", "500", 550);
1173 Add(num_levels - 3, 8U, "600", "700", 550);
7c673cae
FG
1174 // num_levels - 2 is over target by 1100 + 200
1175 Add(num_levels - 2, 9U, "150", "200", 5200);
1176
1177 UpdateVersionStorageInfo();
1178
11fdf7f2 1179 // Merging to the second last level: (5200 / 2100 + 1) * 1100
7c673cae 1180 // Merging to the last level: (50000 / 6300 + 1) * 1300
11fdf7f2 1181 ASSERT_EQ(2100u + 3823u + 11617u,
7c673cae
FG
1182 vstorage_->estimated_compaction_needed_bytes());
1183}
1184
1185TEST_F(CompactionPickerTest, IsBottommostLevelTest) {
1186 // case 1: Higher levels are empty
1187 NewVersionStorage(6, kCompactionStyleLevel);
1188 Add(0, 1U, "a", "m");
1189 Add(0, 2U, "c", "z");
1190 Add(1, 3U, "d", "e");
1191 Add(1, 4U, "l", "p");
1192 Add(2, 5U, "g", "i");
1193 Add(2, 6U, "x", "z");
1194 UpdateVersionStorageInfo();
1195 SetCompactionInputFilesLevels(2, 1);
1196 AddToCompactionFiles(3U);
1197 AddToCompactionFiles(5U);
1198 bool result =
1199 Compaction::TEST_IsBottommostLevel(2, vstorage_.get(), input_files_);
1200 ASSERT_TRUE(result);
1201
1202 // case 2: Higher levels have no overlap
1203 NewVersionStorage(6, kCompactionStyleLevel);
1204 Add(0, 1U, "a", "m");
1205 Add(0, 2U, "c", "z");
1206 Add(1, 3U, "d", "e");
1207 Add(1, 4U, "l", "p");
1208 Add(2, 5U, "g", "i");
1209 Add(2, 6U, "x", "z");
1210 Add(3, 7U, "k", "p");
1211 Add(3, 8U, "t", "w");
1212 Add(4, 9U, "a", "b");
1213 Add(5, 10U, "c", "cc");
1214 UpdateVersionStorageInfo();
1215 SetCompactionInputFilesLevels(2, 1);
1216 AddToCompactionFiles(3U);
1217 AddToCompactionFiles(5U);
1218 result = Compaction::TEST_IsBottommostLevel(2, vstorage_.get(), input_files_);
1219 ASSERT_TRUE(result);
1220
1221 // case 3.1: Higher levels (level 3) have overlap
1222 NewVersionStorage(6, kCompactionStyleLevel);
1223 Add(0, 1U, "a", "m");
1224 Add(0, 2U, "c", "z");
1225 Add(1, 3U, "d", "e");
1226 Add(1, 4U, "l", "p");
1227 Add(2, 5U, "g", "i");
1228 Add(2, 6U, "x", "z");
1229 Add(3, 7U, "e", "g");
1230 Add(3, 8U, "h", "k");
1231 Add(4, 9U, "a", "b");
1232 Add(5, 10U, "c", "cc");
1233 UpdateVersionStorageInfo();
1234 SetCompactionInputFilesLevels(2, 1);
1235 AddToCompactionFiles(3U);
1236 AddToCompactionFiles(5U);
1237 result = Compaction::TEST_IsBottommostLevel(2, vstorage_.get(), input_files_);
1238 ASSERT_FALSE(result);
1239
1240 // case 3.2: Higher levels (level 5) have overlap
1241 DeleteVersionStorage();
1242 NewVersionStorage(6, kCompactionStyleLevel);
1243 Add(0, 1U, "a", "m");
1244 Add(0, 2U, "c", "z");
1245 Add(1, 3U, "d", "e");
1246 Add(1, 4U, "l", "p");
1247 Add(2, 5U, "g", "i");
1248 Add(2, 6U, "x", "z");
1249 Add(3, 7U, "j", "k");
1250 Add(3, 8U, "l", "m");
1251 Add(4, 9U, "a", "b");
1252 Add(5, 10U, "c", "cc");
1253 Add(5, 11U, "h", "k");
1254 Add(5, 12U, "y", "yy");
1255 Add(5, 13U, "z", "zz");
1256 UpdateVersionStorageInfo();
1257 SetCompactionInputFilesLevels(2, 1);
1258 AddToCompactionFiles(3U);
1259 AddToCompactionFiles(5U);
1260 result = Compaction::TEST_IsBottommostLevel(2, vstorage_.get(), input_files_);
1261 ASSERT_FALSE(result);
1262
1263 // case 3.3: Higher levels (level 5) have overlap, but it's only overlapping
1264 // one key ("d")
1265 NewVersionStorage(6, kCompactionStyleLevel);
1266 Add(0, 1U, "a", "m");
1267 Add(0, 2U, "c", "z");
1268 Add(1, 3U, "d", "e");
1269 Add(1, 4U, "l", "p");
1270 Add(2, 5U, "g", "i");
1271 Add(2, 6U, "x", "z");
1272 Add(3, 7U, "j", "k");
1273 Add(3, 8U, "l", "m");
1274 Add(4, 9U, "a", "b");
1275 Add(5, 10U, "c", "cc");
1276 Add(5, 11U, "ccc", "d");
1277 Add(5, 12U, "y", "yy");
1278 Add(5, 13U, "z", "zz");
1279 UpdateVersionStorageInfo();
1280 SetCompactionInputFilesLevels(2, 1);
1281 AddToCompactionFiles(3U);
1282 AddToCompactionFiles(5U);
1283 result = Compaction::TEST_IsBottommostLevel(2, vstorage_.get(), input_files_);
1284 ASSERT_FALSE(result);
1285
1286 // Level 0 files overlap
1287 NewVersionStorage(6, kCompactionStyleLevel);
1288 Add(0, 1U, "s", "t");
1289 Add(0, 2U, "a", "m");
1290 Add(0, 3U, "b", "z");
1291 Add(0, 4U, "e", "f");
1292 Add(5, 10U, "y", "z");
1293 UpdateVersionStorageInfo();
1294 SetCompactionInputFilesLevels(1, 0);
1295 AddToCompactionFiles(1U);
1296 AddToCompactionFiles(2U);
1297 AddToCompactionFiles(3U);
1298 AddToCompactionFiles(4U);
1299 result = Compaction::TEST_IsBottommostLevel(2, vstorage_.get(), input_files_);
1300 ASSERT_FALSE(result);
1301
1302 // Level 0 files don't overlap
1303 NewVersionStorage(6, kCompactionStyleLevel);
1304 Add(0, 1U, "s", "t");
1305 Add(0, 2U, "a", "m");
1306 Add(0, 3U, "b", "k");
1307 Add(0, 4U, "e", "f");
1308 Add(5, 10U, "y", "z");
1309 UpdateVersionStorageInfo();
1310 SetCompactionInputFilesLevels(1, 0);
1311 AddToCompactionFiles(1U);
1312 AddToCompactionFiles(2U);
1313 AddToCompactionFiles(3U);
1314 AddToCompactionFiles(4U);
1315 result = Compaction::TEST_IsBottommostLevel(2, vstorage_.get(), input_files_);
1316 ASSERT_TRUE(result);
1317
1318 // Level 1 files overlap
1319 NewVersionStorage(6, kCompactionStyleLevel);
1320 Add(0, 1U, "s", "t");
1321 Add(0, 2U, "a", "m");
1322 Add(0, 3U, "b", "k");
1323 Add(0, 4U, "e", "f");
1324 Add(1, 5U, "a", "m");
1325 Add(1, 6U, "n", "o");
1326 Add(1, 7U, "w", "y");
1327 Add(5, 10U, "y", "z");
1328 UpdateVersionStorageInfo();
1329 SetCompactionInputFilesLevels(2, 0);
1330 AddToCompactionFiles(1U);
1331 AddToCompactionFiles(2U);
1332 AddToCompactionFiles(3U);
1333 AddToCompactionFiles(4U);
1334 AddToCompactionFiles(5U);
1335 AddToCompactionFiles(6U);
1336 AddToCompactionFiles(7U);
1337 result = Compaction::TEST_IsBottommostLevel(2, vstorage_.get(), input_files_);
1338 ASSERT_FALSE(result);
1339
1340 DeleteVersionStorage();
1341}
1342
1343TEST_F(CompactionPickerTest, MaxCompactionBytesHit) {
1344 mutable_cf_options_.max_bytes_for_level_base = 1000000u;
1345 mutable_cf_options_.max_compaction_bytes = 800000u;
1346 ioptions_.level_compaction_dynamic_level_bytes = false;
1347 NewVersionStorage(6, kCompactionStyleLevel);
1348 // A compaction should be triggered and pick file 2 and 5.
1349 // It can expand because adding file 1 and 3, the compaction size will
1350 // exceed mutable_cf_options_.max_bytes_for_level_base.
1351 Add(1, 1U, "100", "150", 300000U);
1352 Add(1, 2U, "151", "200", 300001U, 0, 0);
1353 Add(1, 3U, "201", "250", 300000U, 0, 0);
1354 Add(1, 4U, "251", "300", 300000U, 0, 0);
1355 Add(2, 5U, "100", "256", 1U);
1356 UpdateVersionStorageInfo();
1357
1358 std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction(
1359 cf_name_, mutable_cf_options_, vstorage_.get(), &log_buffer_));
1360 ASSERT_TRUE(compaction.get() != nullptr);
1361 ASSERT_EQ(2U, compaction->num_input_levels());
1362 ASSERT_EQ(1U, compaction->num_input_files(0));
1363 ASSERT_EQ(1U, compaction->num_input_files(1));
1364 ASSERT_EQ(2U, compaction->input(0, 0)->fd.GetNumber());
1365 ASSERT_EQ(5U, compaction->input(1, 0)->fd.GetNumber());
1366}
1367
1368TEST_F(CompactionPickerTest, MaxCompactionBytesNotHit) {
1369 mutable_cf_options_.max_bytes_for_level_base = 800000u;
1370 mutable_cf_options_.max_compaction_bytes = 1000000u;
1371 ioptions_.level_compaction_dynamic_level_bytes = false;
1372 NewVersionStorage(6, kCompactionStyleLevel);
1373 // A compaction should be triggered and pick file 2 and 5.
1374 // and it expands to file 1 and 3 too.
1375 Add(1, 1U, "100", "150", 300000U);
1376 Add(1, 2U, "151", "200", 300001U, 0, 0);
1377 Add(1, 3U, "201", "250", 300000U, 0, 0);
1378 Add(1, 4U, "251", "300", 300000U, 0, 0);
1379 Add(2, 5U, "000", "251", 1U);
1380 UpdateVersionStorageInfo();
1381
1382 std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction(
1383 cf_name_, mutable_cf_options_, vstorage_.get(), &log_buffer_));
1384 ASSERT_TRUE(compaction.get() != nullptr);
1385 ASSERT_EQ(2U, compaction->num_input_levels());
1386 ASSERT_EQ(3U, compaction->num_input_files(0));
1387 ASSERT_EQ(1U, compaction->num_input_files(1));
1388 ASSERT_EQ(1U, compaction->input(0, 0)->fd.GetNumber());
1389 ASSERT_EQ(2U, compaction->input(0, 1)->fd.GetNumber());
1390 ASSERT_EQ(3U, compaction->input(0, 2)->fd.GetNumber());
1391 ASSERT_EQ(5U, compaction->input(1, 0)->fd.GetNumber());
1392}
1393
1394TEST_F(CompactionPickerTest, IsTrivialMoveOn) {
1395 mutable_cf_options_.max_bytes_for_level_base = 10000u;
1396 mutable_cf_options_.max_compaction_bytes = 10001u;
1397 ioptions_.level_compaction_dynamic_level_bytes = false;
1398 NewVersionStorage(6, kCompactionStyleLevel);
1399 // A compaction should be triggered and pick file 2
1400 Add(1, 1U, "100", "150", 3000U);
1401 Add(1, 2U, "151", "200", 3001U);
1402 Add(1, 3U, "201", "250", 3000U);
1403 Add(1, 4U, "251", "300", 3000U);
1404
1405 Add(3, 5U, "120", "130", 7000U);
1406 Add(3, 6U, "170", "180", 7000U);
1407 Add(3, 5U, "220", "230", 7000U);
1408 Add(3, 5U, "270", "280", 7000U);
1409 UpdateVersionStorageInfo();
1410
1411 std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction(
1412 cf_name_, mutable_cf_options_, vstorage_.get(), &log_buffer_));
1413 ASSERT_TRUE(compaction.get() != nullptr);
1414 ASSERT_TRUE(compaction->IsTrivialMove());
1415}
1416
1417TEST_F(CompactionPickerTest, IsTrivialMoveOff) {
1418 mutable_cf_options_.max_bytes_for_level_base = 1000000u;
1419 mutable_cf_options_.max_compaction_bytes = 10000u;
1420 ioptions_.level_compaction_dynamic_level_bytes = false;
1421 NewVersionStorage(6, kCompactionStyleLevel);
1422 // A compaction should be triggered and pick all files from level 1
1423 Add(1, 1U, "100", "150", 300000U, 0, 0);
1424 Add(1, 2U, "150", "200", 300000U, 0, 0);
1425 Add(1, 3U, "200", "250", 300000U, 0, 0);
1426 Add(1, 4U, "250", "300", 300000U, 0, 0);
1427
1428 Add(3, 5U, "120", "130", 6000U);
1429 Add(3, 6U, "140", "150", 6000U);
1430 UpdateVersionStorageInfo();
1431
1432 std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction(
1433 cf_name_, mutable_cf_options_, vstorage_.get(), &log_buffer_));
1434 ASSERT_TRUE(compaction.get() != nullptr);
1435 ASSERT_FALSE(compaction->IsTrivialMove());
1436}
1437
11fdf7f2
TL
1438TEST_F(CompactionPickerTest, CacheNextCompactionIndex) {
1439 NewVersionStorage(6, kCompactionStyleLevel);
1440 mutable_cf_options_.max_compaction_bytes = 100000000000u;
1441
1442 Add(1 /* level */, 1U /* file_number */, "100" /* smallest */,
1443 "149" /* largest */, 1000000000U /* file_size */);
1444 file_map_[1U].first->being_compacted = true;
1445 Add(1 /* level */, 2U /* file_number */, "150" /* smallest */,
1446 "199" /* largest */, 900000000U /* file_size */);
1447 Add(1 /* level */, 3U /* file_number */, "200" /* smallest */,
1448 "249" /* largest */, 800000000U /* file_size */);
1449 Add(1 /* level */, 4U /* file_number */, "250" /* smallest */,
1450 "299" /* largest */, 700000000U /* file_size */);
1451 Add(2 /* level */, 5U /* file_number */, "150" /* smallest */,
1452 "199" /* largest */, 1U /* file_size */);
1453 file_map_[5U].first->being_compacted = true;
1454
1455 UpdateVersionStorageInfo();
1456
1457 std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction(
1458 cf_name_, mutable_cf_options_, vstorage_.get(), &log_buffer_));
1459 ASSERT_TRUE(compaction.get() != nullptr);
1460 ASSERT_EQ(1U, compaction->num_input_levels());
1461 ASSERT_EQ(1U, compaction->num_input_files(0));
1462 ASSERT_EQ(0U, compaction->num_input_files(1));
1463 ASSERT_EQ(3U, compaction->input(0, 0)->fd.GetNumber());
1464 ASSERT_EQ(2, vstorage_->NextCompactionIndex(1 /* level */));
1465
1466 compaction.reset(level_compaction_picker.PickCompaction(
1467 cf_name_, mutable_cf_options_, vstorage_.get(), &log_buffer_));
1468 ASSERT_TRUE(compaction.get() != nullptr);
1469 ASSERT_EQ(1U, compaction->num_input_levels());
1470 ASSERT_EQ(1U, compaction->num_input_files(0));
1471 ASSERT_EQ(0U, compaction->num_input_files(1));
1472 ASSERT_EQ(4U, compaction->input(0, 0)->fd.GetNumber());
1473 ASSERT_EQ(3, vstorage_->NextCompactionIndex(1 /* level */));
1474
1475 compaction.reset(level_compaction_picker.PickCompaction(
1476 cf_name_, mutable_cf_options_, vstorage_.get(), &log_buffer_));
1477 ASSERT_TRUE(compaction.get() == nullptr);
1478 ASSERT_EQ(4, vstorage_->NextCompactionIndex(1 /* level */));
1479}
1480
7c673cae
FG
1481} // namespace rocksdb
1482
1483int main(int argc, char** argv) {
1484 ::testing::InitGoogleTest(&argc, argv);
1485 return RUN_ALL_TESTS();
1486}