]>
Commit | Line | Data |
---|---|---|
7c673cae | 1 | // Copyright (c) 2011-present, Facebook, Inc. All rights reserved. |
11fdf7f2 TL |
2 | // This source code is licensed under both the GPLv2 (found in the |
3 | // COPYING file in the root directory) and Apache 2.0 License | |
4 | // (found in the LICENSE.Apache file in the root directory). | |
7c673cae FG |
5 | |
6 | #include "db/compaction_picker.h" | |
494da23a | 7 | |
7c673cae FG |
8 | #include <limits> |
9 | #include <string> | |
10 | #include <utility> | |
11 | #include "db/compaction.h" | |
494da23a | 12 | #include "db/compaction_picker_fifo.h" |
7c673cae FG |
13 | #include "db/compaction_picker_universal.h" |
14 | ||
15 | #include "util/logging.h" | |
16 | #include "util/string_util.h" | |
17 | #include "util/testharness.h" | |
18 | #include "util/testutil.h" | |
19 | ||
20 | namespace rocksdb { | |
21 | ||
22 | class CountingLogger : public Logger { | |
23 | public: | |
24 | using Logger::Logv; | |
494da23a | 25 | void Logv(const char* /*format*/, va_list /*ap*/) override { log_count++; } |
7c673cae FG |
26 | size_t log_count; |
27 | }; | |
28 | ||
29 | class CompactionPickerTest : public testing::Test { | |
30 | public: | |
31 | const Comparator* ucmp_; | |
32 | InternalKeyComparator icmp_; | |
33 | Options options_; | |
34 | ImmutableCFOptions ioptions_; | |
35 | MutableCFOptions mutable_cf_options_; | |
36 | LevelCompactionPicker level_compaction_picker; | |
37 | std::string cf_name_; | |
38 | CountingLogger logger_; | |
39 | LogBuffer log_buffer_; | |
40 | uint32_t file_num_; | |
41 | CompactionOptionsFIFO fifo_options_; | |
42 | std::unique_ptr<VersionStorageInfo> vstorage_; | |
43 | std::vector<std::unique_ptr<FileMetaData>> files_; | |
44 | // does not own FileMetaData | |
45 | std::unordered_map<uint32_t, std::pair<FileMetaData*, int>> file_map_; | |
46 | // input files to compaction process. | |
47 | std::vector<CompactionInputFiles> input_files_; | |
48 | int compaction_level_start_; | |
49 | ||
50 | CompactionPickerTest() | |
51 | : ucmp_(BytewiseComparator()), | |
52 | icmp_(ucmp_), | |
53 | ioptions_(options_), | |
54 | mutable_cf_options_(options_), | |
55 | level_compaction_picker(ioptions_, &icmp_), | |
56 | cf_name_("dummy"), | |
57 | log_buffer_(InfoLogLevel::INFO_LEVEL, &logger_), | |
58 | file_num_(1), | |
59 | vstorage_(nullptr) { | |
494da23a TL |
60 | // ioptions_.compaction_pri = kMinOverlappingRatio has its own set of |
61 | // tests to cover. | |
62 | ioptions_.compaction_pri = kByCompensatedSize; | |
7c673cae FG |
63 | fifo_options_.max_table_files_size = 1; |
64 | mutable_cf_options_.RefreshDerivedOptions(ioptions_); | |
11fdf7f2 | 65 | ioptions_.cf_paths.emplace_back("dummy", |
7c673cae FG |
66 | std::numeric_limits<uint64_t>::max()); |
67 | } | |
68 | ||
494da23a | 69 | ~CompactionPickerTest() override {} |
7c673cae FG |
70 | |
71 | void NewVersionStorage(int num_levels, CompactionStyle style) { | |
72 | DeleteVersionStorage(); | |
73 | options_.num_levels = num_levels; | |
74 | vstorage_.reset(new VersionStorageInfo(&icmp_, ucmp_, options_.num_levels, | |
75 | style, nullptr, false)); | |
76 | vstorage_->CalculateBaseBytes(ioptions_, mutable_cf_options_); | |
77 | } | |
78 | ||
79 | void DeleteVersionStorage() { | |
80 | vstorage_.reset(); | |
81 | files_.clear(); | |
82 | file_map_.clear(); | |
83 | input_files_.clear(); | |
84 | } | |
85 | ||
86 | void Add(int level, uint32_t file_number, const char* smallest, | |
87 | const char* largest, uint64_t file_size = 1, uint32_t path_id = 0, | |
494da23a TL |
88 | SequenceNumber smallest_seq = 100, SequenceNumber largest_seq = 100, |
89 | size_t compensated_file_size = 0) { | |
7c673cae FG |
90 | assert(level < vstorage_->num_levels()); |
91 | FileMetaData* f = new FileMetaData; | |
92 | f->fd = FileDescriptor(file_number, path_id, file_size); | |
93 | f->smallest = InternalKey(smallest, smallest_seq, kTypeValue); | |
94 | f->largest = InternalKey(largest, largest_seq, kTypeValue); | |
11fdf7f2 TL |
95 | f->fd.smallest_seqno = smallest_seq; |
96 | f->fd.largest_seqno = largest_seq; | |
494da23a TL |
97 | f->compensated_file_size = |
98 | (compensated_file_size != 0) ? compensated_file_size : file_size; | |
7c673cae FG |
99 | f->refs = 0; |
100 | vstorage_->AddFile(level, f); | |
101 | files_.emplace_back(f); | |
102 | file_map_.insert({file_number, {f, level}}); | |
103 | } | |
104 | ||
105 | void SetCompactionInputFilesLevels(int level_count, int start_level) { | |
106 | input_files_.resize(level_count); | |
107 | for (int i = 0; i < level_count; ++i) { | |
108 | input_files_[i].level = start_level + i; | |
109 | } | |
110 | compaction_level_start_ = start_level; | |
111 | } | |
112 | ||
113 | void AddToCompactionFiles(uint32_t file_number) { | |
114 | auto iter = file_map_.find(file_number); | |
115 | assert(iter != file_map_.end()); | |
116 | int level = iter->second.second; | |
117 | assert(level < vstorage_->num_levels()); | |
118 | input_files_[level - compaction_level_start_].files.emplace_back( | |
119 | iter->second.first); | |
120 | } | |
121 | ||
122 | void UpdateVersionStorageInfo() { | |
123 | vstorage_->CalculateBaseBytes(ioptions_, mutable_cf_options_); | |
124 | vstorage_->UpdateFilesByCompactionPri(ioptions_.compaction_pri); | |
125 | vstorage_->UpdateNumNonEmptyLevels(); | |
126 | vstorage_->GenerateFileIndexer(); | |
127 | vstorage_->GenerateLevelFilesBrief(); | |
128 | vstorage_->ComputeCompactionScore(ioptions_, mutable_cf_options_); | |
129 | vstorage_->GenerateLevel0NonOverlapping(); | |
130 | vstorage_->ComputeFilesMarkedForCompaction(); | |
131 | vstorage_->SetFinalized(); | |
132 | } | |
133 | }; | |
134 | ||
135 | TEST_F(CompactionPickerTest, Empty) { | |
136 | NewVersionStorage(6, kCompactionStyleLevel); | |
137 | UpdateVersionStorageInfo(); | |
138 | std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction( | |
139 | cf_name_, mutable_cf_options_, vstorage_.get(), &log_buffer_)); | |
140 | ASSERT_TRUE(compaction.get() == nullptr); | |
141 | } | |
142 | ||
143 | TEST_F(CompactionPickerTest, Single) { | |
144 | NewVersionStorage(6, kCompactionStyleLevel); | |
145 | mutable_cf_options_.level0_file_num_compaction_trigger = 2; | |
146 | Add(0, 1U, "p", "q"); | |
147 | UpdateVersionStorageInfo(); | |
148 | ||
149 | std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction( | |
150 | cf_name_, mutable_cf_options_, vstorage_.get(), &log_buffer_)); | |
151 | ASSERT_TRUE(compaction.get() == nullptr); | |
152 | } | |
153 | ||
154 | TEST_F(CompactionPickerTest, Level0Trigger) { | |
155 | NewVersionStorage(6, kCompactionStyleLevel); | |
156 | mutable_cf_options_.level0_file_num_compaction_trigger = 2; | |
157 | Add(0, 1U, "150", "200"); | |
158 | Add(0, 2U, "200", "250"); | |
159 | ||
160 | UpdateVersionStorageInfo(); | |
161 | ||
162 | std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction( | |
163 | cf_name_, mutable_cf_options_, vstorage_.get(), &log_buffer_)); | |
164 | ASSERT_TRUE(compaction.get() != nullptr); | |
165 | ASSERT_EQ(2U, compaction->num_input_files(0)); | |
166 | ASSERT_EQ(1U, compaction->input(0, 0)->fd.GetNumber()); | |
167 | ASSERT_EQ(2U, compaction->input(0, 1)->fd.GetNumber()); | |
168 | } | |
169 | ||
170 | TEST_F(CompactionPickerTest, Level1Trigger) { | |
171 | NewVersionStorage(6, kCompactionStyleLevel); | |
172 | Add(1, 66U, "150", "200", 1000000000U); | |
173 | UpdateVersionStorageInfo(); | |
174 | ||
175 | std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction( | |
176 | cf_name_, mutable_cf_options_, vstorage_.get(), &log_buffer_)); | |
177 | ASSERT_TRUE(compaction.get() != nullptr); | |
178 | ASSERT_EQ(1U, compaction->num_input_files(0)); | |
179 | ASSERT_EQ(66U, compaction->input(0, 0)->fd.GetNumber()); | |
180 | } | |
181 | ||
182 | TEST_F(CompactionPickerTest, Level1Trigger2) { | |
11fdf7f2 TL |
183 | mutable_cf_options_.target_file_size_base = 10000000000; |
184 | mutable_cf_options_.RefreshDerivedOptions(ioptions_); | |
7c673cae FG |
185 | NewVersionStorage(6, kCompactionStyleLevel); |
186 | Add(1, 66U, "150", "200", 1000000001U); | |
187 | Add(1, 88U, "201", "300", 1000000000U); | |
188 | Add(2, 6U, "150", "179", 1000000000U); | |
189 | Add(2, 7U, "180", "220", 1000000000U); | |
190 | Add(2, 8U, "221", "300", 1000000000U); | |
191 | UpdateVersionStorageInfo(); | |
192 | ||
193 | std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction( | |
194 | cf_name_, mutable_cf_options_, vstorage_.get(), &log_buffer_)); | |
195 | ASSERT_TRUE(compaction.get() != nullptr); | |
196 | ASSERT_EQ(1U, compaction->num_input_files(0)); | |
197 | ASSERT_EQ(2U, compaction->num_input_files(1)); | |
198 | ASSERT_EQ(66U, compaction->input(0, 0)->fd.GetNumber()); | |
199 | ASSERT_EQ(6U, compaction->input(1, 0)->fd.GetNumber()); | |
200 | ASSERT_EQ(7U, compaction->input(1, 1)->fd.GetNumber()); | |
11fdf7f2 | 201 | ASSERT_EQ(uint64_t{1073741824}, compaction->OutputFilePreallocationSize()); |
7c673cae FG |
202 | } |
203 | ||
204 | TEST_F(CompactionPickerTest, LevelMaxScore) { | |
205 | NewVersionStorage(6, kCompactionStyleLevel); | |
206 | mutable_cf_options_.target_file_size_base = 10000000; | |
7c673cae | 207 | mutable_cf_options_.max_bytes_for_level_base = 10 * 1024 * 1024; |
11fdf7f2 | 208 | mutable_cf_options_.RefreshDerivedOptions(ioptions_); |
7c673cae FG |
209 | Add(0, 1U, "150", "200", 1000000U); |
210 | // Level 1 score 1.2 | |
211 | Add(1, 66U, "150", "200", 6000000U); | |
212 | Add(1, 88U, "201", "300", 6000000U); | |
213 | // Level 2 score 1.8. File 7 is the largest. Should be picked | |
214 | Add(2, 6U, "150", "179", 60000000U); | |
215 | Add(2, 7U, "180", "220", 60000001U); | |
216 | Add(2, 8U, "221", "300", 60000000U); | |
217 | // Level 3 score slightly larger than 1 | |
218 | Add(3, 26U, "150", "170", 260000000U); | |
219 | Add(3, 27U, "171", "179", 260000000U); | |
220 | Add(3, 28U, "191", "220", 260000000U); | |
221 | Add(3, 29U, "221", "300", 260000000U); | |
222 | UpdateVersionStorageInfo(); | |
223 | ||
224 | std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction( | |
225 | cf_name_, mutable_cf_options_, vstorage_.get(), &log_buffer_)); | |
226 | ASSERT_TRUE(compaction.get() != nullptr); | |
227 | ASSERT_EQ(1U, compaction->num_input_files(0)); | |
228 | ASSERT_EQ(7U, compaction->input(0, 0)->fd.GetNumber()); | |
11fdf7f2 TL |
229 | ASSERT_EQ(mutable_cf_options_.target_file_size_base + |
230 | mutable_cf_options_.target_file_size_base / 10, | |
231 | compaction->OutputFilePreallocationSize()); | |
7c673cae FG |
232 | } |
233 | ||
234 | TEST_F(CompactionPickerTest, NeedsCompactionLevel) { | |
235 | const int kLevels = 6; | |
236 | const int kFileCount = 20; | |
237 | ||
238 | for (int level = 0; level < kLevels - 1; ++level) { | |
239 | NewVersionStorage(kLevels, kCompactionStyleLevel); | |
240 | uint64_t file_size = vstorage_->MaxBytesForLevel(level) * 2 / kFileCount; | |
241 | for (int file_count = 1; file_count <= kFileCount; ++file_count) { | |
242 | // start a brand new version in each test. | |
243 | NewVersionStorage(kLevels, kCompactionStyleLevel); | |
244 | for (int i = 0; i < file_count; ++i) { | |
245 | Add(level, i, ToString((i + 100) * 1000).c_str(), | |
246 | ToString((i + 100) * 1000 + 999).c_str(), | |
247 | file_size, 0, i * 100, i * 100 + 99); | |
248 | } | |
249 | UpdateVersionStorageInfo(); | |
250 | ASSERT_EQ(vstorage_->CompactionScoreLevel(0), level); | |
251 | ASSERT_EQ(level_compaction_picker.NeedsCompaction(vstorage_.get()), | |
252 | vstorage_->CompactionScore(0) >= 1); | |
253 | // release the version storage | |
254 | DeleteVersionStorage(); | |
255 | } | |
256 | } | |
257 | } | |
258 | ||
259 | TEST_F(CompactionPickerTest, Level0TriggerDynamic) { | |
260 | int num_levels = ioptions_.num_levels; | |
261 | ioptions_.level_compaction_dynamic_level_bytes = true; | |
262 | mutable_cf_options_.level0_file_num_compaction_trigger = 2; | |
263 | mutable_cf_options_.max_bytes_for_level_base = 200; | |
264 | mutable_cf_options_.max_bytes_for_level_multiplier = 10; | |
265 | NewVersionStorage(num_levels, kCompactionStyleLevel); | |
266 | Add(0, 1U, "150", "200"); | |
267 | Add(0, 2U, "200", "250"); | |
268 | ||
269 | UpdateVersionStorageInfo(); | |
270 | ||
271 | std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction( | |
272 | cf_name_, mutable_cf_options_, vstorage_.get(), &log_buffer_)); | |
273 | ASSERT_TRUE(compaction.get() != nullptr); | |
274 | ASSERT_EQ(2U, compaction->num_input_files(0)); | |
275 | ASSERT_EQ(1U, compaction->input(0, 0)->fd.GetNumber()); | |
276 | ASSERT_EQ(2U, compaction->input(0, 1)->fd.GetNumber()); | |
277 | ASSERT_EQ(1, static_cast<int>(compaction->num_input_levels())); | |
278 | ASSERT_EQ(num_levels - 1, compaction->output_level()); | |
279 | } | |
280 | ||
281 | TEST_F(CompactionPickerTest, Level0TriggerDynamic2) { | |
282 | int num_levels = ioptions_.num_levels; | |
283 | ioptions_.level_compaction_dynamic_level_bytes = true; | |
284 | mutable_cf_options_.level0_file_num_compaction_trigger = 2; | |
285 | mutable_cf_options_.max_bytes_for_level_base = 200; | |
286 | mutable_cf_options_.max_bytes_for_level_multiplier = 10; | |
287 | NewVersionStorage(num_levels, kCompactionStyleLevel); | |
288 | Add(0, 1U, "150", "200"); | |
289 | Add(0, 2U, "200", "250"); | |
290 | Add(num_levels - 1, 3U, "200", "250", 300U); | |
291 | ||
292 | UpdateVersionStorageInfo(); | |
293 | ASSERT_EQ(vstorage_->base_level(), num_levels - 2); | |
294 | ||
295 | std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction( | |
296 | cf_name_, mutable_cf_options_, vstorage_.get(), &log_buffer_)); | |
297 | ASSERT_TRUE(compaction.get() != nullptr); | |
298 | ASSERT_EQ(2U, compaction->num_input_files(0)); | |
299 | ASSERT_EQ(1U, compaction->input(0, 0)->fd.GetNumber()); | |
300 | ASSERT_EQ(2U, compaction->input(0, 1)->fd.GetNumber()); | |
301 | ASSERT_EQ(1, static_cast<int>(compaction->num_input_levels())); | |
302 | ASSERT_EQ(num_levels - 2, compaction->output_level()); | |
303 | } | |
304 | ||
305 | TEST_F(CompactionPickerTest, Level0TriggerDynamic3) { | |
306 | int num_levels = ioptions_.num_levels; | |
307 | ioptions_.level_compaction_dynamic_level_bytes = true; | |
308 | mutable_cf_options_.level0_file_num_compaction_trigger = 2; | |
309 | mutable_cf_options_.max_bytes_for_level_base = 200; | |
310 | mutable_cf_options_.max_bytes_for_level_multiplier = 10; | |
311 | NewVersionStorage(num_levels, kCompactionStyleLevel); | |
312 | Add(0, 1U, "150", "200"); | |
313 | Add(0, 2U, "200", "250"); | |
314 | Add(num_levels - 1, 3U, "200", "250", 300U); | |
315 | Add(num_levels - 1, 4U, "300", "350", 3000U); | |
316 | ||
317 | UpdateVersionStorageInfo(); | |
318 | ASSERT_EQ(vstorage_->base_level(), num_levels - 3); | |
319 | ||
320 | std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction( | |
321 | cf_name_, mutable_cf_options_, vstorage_.get(), &log_buffer_)); | |
322 | ASSERT_TRUE(compaction.get() != nullptr); | |
323 | ASSERT_EQ(2U, compaction->num_input_files(0)); | |
324 | ASSERT_EQ(1U, compaction->input(0, 0)->fd.GetNumber()); | |
325 | ASSERT_EQ(2U, compaction->input(0, 1)->fd.GetNumber()); | |
326 | ASSERT_EQ(1, static_cast<int>(compaction->num_input_levels())); | |
327 | ASSERT_EQ(num_levels - 3, compaction->output_level()); | |
328 | } | |
329 | ||
330 | TEST_F(CompactionPickerTest, Level0TriggerDynamic4) { | |
331 | int num_levels = ioptions_.num_levels; | |
332 | ioptions_.level_compaction_dynamic_level_bytes = true; | |
333 | mutable_cf_options_.level0_file_num_compaction_trigger = 2; | |
334 | mutable_cf_options_.max_bytes_for_level_base = 200; | |
335 | mutable_cf_options_.max_bytes_for_level_multiplier = 10; | |
336 | ||
337 | NewVersionStorage(num_levels, kCompactionStyleLevel); | |
338 | Add(0, 1U, "150", "200"); | |
339 | Add(0, 2U, "200", "250"); | |
340 | Add(num_levels - 1, 3U, "200", "250", 300U); | |
341 | Add(num_levels - 1, 4U, "300", "350", 3000U); | |
342 | Add(num_levels - 3, 5U, "150", "180", 3U); | |
343 | Add(num_levels - 3, 6U, "181", "300", 3U); | |
344 | Add(num_levels - 3, 7U, "400", "450", 3U); | |
345 | ||
346 | UpdateVersionStorageInfo(); | |
347 | ASSERT_EQ(vstorage_->base_level(), num_levels - 3); | |
348 | ||
349 | std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction( | |
350 | cf_name_, mutable_cf_options_, vstorage_.get(), &log_buffer_)); | |
351 | ASSERT_TRUE(compaction.get() != nullptr); | |
352 | ASSERT_EQ(2U, compaction->num_input_files(0)); | |
353 | ASSERT_EQ(1U, compaction->input(0, 0)->fd.GetNumber()); | |
354 | ASSERT_EQ(2U, compaction->input(0, 1)->fd.GetNumber()); | |
355 | ASSERT_EQ(2U, compaction->num_input_files(1)); | |
356 | ASSERT_EQ(num_levels - 3, compaction->level(1)); | |
357 | ASSERT_EQ(5U, compaction->input(1, 0)->fd.GetNumber()); | |
358 | ASSERT_EQ(6U, compaction->input(1, 1)->fd.GetNumber()); | |
359 | ASSERT_EQ(2, static_cast<int>(compaction->num_input_levels())); | |
360 | ASSERT_EQ(num_levels - 3, compaction->output_level()); | |
361 | } | |
362 | ||
363 | TEST_F(CompactionPickerTest, LevelTriggerDynamic4) { | |
364 | int num_levels = ioptions_.num_levels; | |
365 | ioptions_.level_compaction_dynamic_level_bytes = true; | |
366 | ioptions_.compaction_pri = kMinOverlappingRatio; | |
367 | mutable_cf_options_.level0_file_num_compaction_trigger = 2; | |
368 | mutable_cf_options_.max_bytes_for_level_base = 200; | |
369 | mutable_cf_options_.max_bytes_for_level_multiplier = 10; | |
370 | NewVersionStorage(num_levels, kCompactionStyleLevel); | |
371 | Add(0, 1U, "150", "200"); | |
372 | Add(num_levels - 1, 3U, "200", "250", 300U); | |
373 | Add(num_levels - 1, 4U, "300", "350", 3000U); | |
374 | Add(num_levels - 1, 4U, "400", "450", 3U); | |
375 | Add(num_levels - 2, 5U, "150", "180", 300U); | |
376 | Add(num_levels - 2, 6U, "181", "350", 500U); | |
377 | Add(num_levels - 2, 7U, "400", "450", 200U); | |
378 | ||
379 | UpdateVersionStorageInfo(); | |
380 | ||
381 | std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction( | |
382 | cf_name_, mutable_cf_options_, vstorage_.get(), &log_buffer_)); | |
383 | ASSERT_TRUE(compaction.get() != nullptr); | |
384 | ASSERT_EQ(1U, compaction->num_input_files(0)); | |
385 | ASSERT_EQ(5U, compaction->input(0, 0)->fd.GetNumber()); | |
386 | ASSERT_EQ(0, compaction->num_input_files(1)); | |
387 | ASSERT_EQ(1U, compaction->num_input_levels()); | |
388 | ASSERT_EQ(num_levels - 1, compaction->output_level()); | |
389 | } | |
390 | ||
391 | // Universal and FIFO Compactions are not supported in ROCKSDB_LITE | |
392 | #ifndef ROCKSDB_LITE | |
393 | TEST_F(CompactionPickerTest, NeedsCompactionUniversal) { | |
394 | NewVersionStorage(1, kCompactionStyleUniversal); | |
395 | UniversalCompactionPicker universal_compaction_picker( | |
396 | ioptions_, &icmp_); | |
11fdf7f2 | 397 | UpdateVersionStorageInfo(); |
7c673cae FG |
398 | // must return false when there's no files. |
399 | ASSERT_EQ(universal_compaction_picker.NeedsCompaction(vstorage_.get()), | |
400 | false); | |
7c673cae FG |
401 | |
402 | // verify the trigger given different number of L0 files. | |
403 | for (int i = 1; | |
404 | i <= mutable_cf_options_.level0_file_num_compaction_trigger * 2; ++i) { | |
405 | NewVersionStorage(1, kCompactionStyleUniversal); | |
406 | Add(0, i, ToString((i + 100) * 1000).c_str(), | |
407 | ToString((i + 100) * 1000 + 999).c_str(), 1000000, 0, i * 100, | |
408 | i * 100 + 99); | |
409 | UpdateVersionStorageInfo(); | |
410 | ASSERT_EQ(level_compaction_picker.NeedsCompaction(vstorage_.get()), | |
411 | vstorage_->CompactionScore(0) >= 1); | |
412 | } | |
413 | } | |
11fdf7f2 TL |
414 | |
415 | TEST_F(CompactionPickerTest, CompactionUniversalIngestBehindReservedLevel) { | |
416 | const uint64_t kFileSize = 100000; | |
417 | NewVersionStorage(1, kCompactionStyleUniversal); | |
418 | ioptions_.allow_ingest_behind = true; | |
419 | ioptions_.num_levels = 3; | |
420 | UniversalCompactionPicker universal_compaction_picker(ioptions_, &icmp_); | |
421 | UpdateVersionStorageInfo(); | |
422 | // must return false when there's no files. | |
423 | ASSERT_EQ(universal_compaction_picker.NeedsCompaction(vstorage_.get()), | |
424 | false); | |
425 | ||
426 | NewVersionStorage(3, kCompactionStyleUniversal); | |
427 | ||
428 | Add(0, 1U, "150", "200", kFileSize, 0, 500, 550); | |
429 | Add(0, 2U, "201", "250", kFileSize, 0, 401, 450); | |
430 | Add(0, 4U, "260", "300", kFileSize, 0, 260, 300); | |
431 | Add(1, 5U, "100", "151", kFileSize, 0, 200, 251); | |
432 | Add(1, 3U, "301", "350", kFileSize, 0, 101, 150); | |
433 | Add(2, 6U, "120", "200", kFileSize, 0, 20, 100); | |
434 | ||
435 | UpdateVersionStorageInfo(); | |
436 | ||
437 | std::unique_ptr<Compaction> compaction( | |
438 | universal_compaction_picker.PickCompaction( | |
439 | cf_name_, mutable_cf_options_, vstorage_.get(), &log_buffer_)); | |
440 | ||
441 | // output level should be the one above the bottom-most | |
442 | ASSERT_EQ(1, compaction->output_level()); | |
443 | } | |
7c673cae FG |
444 | // Tests if the files can be trivially moved in multi level |
445 | // universal compaction when allow_trivial_move option is set | |
446 | // In this test as the input files overlaps, they cannot | |
447 | // be trivially moved. | |
448 | ||
449 | TEST_F(CompactionPickerTest, CannotTrivialMoveUniversal) { | |
450 | const uint64_t kFileSize = 100000; | |
451 | ||
11fdf7f2 | 452 | mutable_cf_options_.compaction_options_universal.allow_trivial_move = true; |
7c673cae FG |
453 | NewVersionStorage(1, kCompactionStyleUniversal); |
454 | UniversalCompactionPicker universal_compaction_picker(ioptions_, &icmp_); | |
11fdf7f2 | 455 | UpdateVersionStorageInfo(); |
7c673cae FG |
456 | // must return false when there's no files. |
457 | ASSERT_EQ(universal_compaction_picker.NeedsCompaction(vstorage_.get()), | |
458 | false); | |
459 | ||
460 | NewVersionStorage(3, kCompactionStyleUniversal); | |
461 | ||
462 | Add(0, 1U, "150", "200", kFileSize, 0, 500, 550); | |
463 | Add(0, 2U, "201", "250", kFileSize, 0, 401, 450); | |
464 | Add(0, 4U, "260", "300", kFileSize, 0, 260, 300); | |
465 | Add(1, 5U, "100", "151", kFileSize, 0, 200, 251); | |
466 | Add(1, 3U, "301", "350", kFileSize, 0, 101, 150); | |
467 | Add(2, 6U, "120", "200", kFileSize, 0, 20, 100); | |
468 | ||
469 | UpdateVersionStorageInfo(); | |
470 | ||
471 | std::unique_ptr<Compaction> compaction( | |
472 | universal_compaction_picker.PickCompaction( | |
473 | cf_name_, mutable_cf_options_, vstorage_.get(), &log_buffer_)); | |
474 | ||
475 | ASSERT_TRUE(!compaction->is_trivial_move()); | |
476 | } | |
477 | // Tests if the files can be trivially moved in multi level | |
478 | // universal compaction when allow_trivial_move option is set | |
479 | // In this test as the input files doesn't overlaps, they should | |
480 | // be trivially moved. | |
481 | TEST_F(CompactionPickerTest, AllowsTrivialMoveUniversal) { | |
482 | const uint64_t kFileSize = 100000; | |
483 | ||
11fdf7f2 | 484 | mutable_cf_options_.compaction_options_universal.allow_trivial_move = true; |
7c673cae FG |
485 | UniversalCompactionPicker universal_compaction_picker(ioptions_, &icmp_); |
486 | ||
487 | NewVersionStorage(3, kCompactionStyleUniversal); | |
488 | ||
489 | Add(0, 1U, "150", "200", kFileSize, 0, 500, 550); | |
490 | Add(0, 2U, "201", "250", kFileSize, 0, 401, 450); | |
491 | Add(0, 4U, "260", "300", kFileSize, 0, 260, 300); | |
492 | Add(1, 5U, "010", "080", kFileSize, 0, 200, 251); | |
493 | Add(2, 3U, "301", "350", kFileSize, 0, 101, 150); | |
494 | ||
495 | UpdateVersionStorageInfo(); | |
496 | ||
497 | std::unique_ptr<Compaction> compaction( | |
498 | universal_compaction_picker.PickCompaction( | |
499 | cf_name_, mutable_cf_options_, vstorage_.get(), &log_buffer_)); | |
500 | ||
501 | ASSERT_TRUE(compaction->is_trivial_move()); | |
502 | } | |
503 | ||
504 | TEST_F(CompactionPickerTest, NeedsCompactionFIFO) { | |
505 | NewVersionStorage(1, kCompactionStyleFIFO); | |
506 | const int kFileCount = | |
507 | mutable_cf_options_.level0_file_num_compaction_trigger * 3; | |
508 | const uint64_t kFileSize = 100000; | |
509 | const uint64_t kMaxSize = kFileSize * kFileCount / 2; | |
510 | ||
511 | fifo_options_.max_table_files_size = kMaxSize; | |
11fdf7f2 | 512 | mutable_cf_options_.compaction_options_fifo = fifo_options_; |
7c673cae FG |
513 | FIFOCompactionPicker fifo_compaction_picker(ioptions_, &icmp_); |
514 | UpdateVersionStorageInfo(); | |
515 | // must return false when there's no files. | |
516 | ASSERT_EQ(fifo_compaction_picker.NeedsCompaction(vstorage_.get()), false); | |
517 | ||
518 | // verify whether compaction is needed based on the current | |
519 | // size of L0 files. | |
520 | uint64_t current_size = 0; | |
521 | for (int i = 1; i <= kFileCount; ++i) { | |
522 | NewVersionStorage(1, kCompactionStyleFIFO); | |
523 | Add(0, i, ToString((i + 100) * 1000).c_str(), | |
524 | ToString((i + 100) * 1000 + 999).c_str(), | |
525 | kFileSize, 0, i * 100, i * 100 + 99); | |
526 | current_size += kFileSize; | |
527 | UpdateVersionStorageInfo(); | |
11fdf7f2 | 528 | ASSERT_EQ(fifo_compaction_picker.NeedsCompaction(vstorage_.get()), |
7c673cae FG |
529 | vstorage_->CompactionScore(0) >= 1); |
530 | } | |
531 | } | |
532 | #endif // ROCKSDB_LITE | |
533 | ||
534 | TEST_F(CompactionPickerTest, CompactionPriMinOverlapping1) { | |
535 | NewVersionStorage(6, kCompactionStyleLevel); | |
536 | ioptions_.compaction_pri = kMinOverlappingRatio; | |
11fdf7f2 | 537 | mutable_cf_options_.target_file_size_base = 100000000000; |
7c673cae FG |
538 | mutable_cf_options_.target_file_size_multiplier = 10; |
539 | mutable_cf_options_.max_bytes_for_level_base = 10 * 1024 * 1024; | |
11fdf7f2 | 540 | mutable_cf_options_.RefreshDerivedOptions(ioptions_); |
7c673cae FG |
541 | |
542 | Add(2, 6U, "150", "179", 50000000U); | |
543 | Add(2, 7U, "180", "220", 50000000U); | |
544 | Add(2, 8U, "321", "400", 50000000U); // File not overlapping | |
545 | Add(2, 9U, "721", "800", 50000000U); | |
546 | ||
547 | Add(3, 26U, "150", "170", 260000000U); | |
548 | Add(3, 27U, "171", "179", 260000000U); | |
549 | Add(3, 28U, "191", "220", 260000000U); | |
550 | Add(3, 29U, "221", "300", 260000000U); | |
551 | Add(3, 30U, "750", "900", 260000000U); | |
552 | UpdateVersionStorageInfo(); | |
553 | ||
554 | std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction( | |
555 | cf_name_, mutable_cf_options_, vstorage_.get(), &log_buffer_)); | |
556 | ASSERT_TRUE(compaction.get() != nullptr); | |
557 | ASSERT_EQ(1U, compaction->num_input_files(0)); | |
558 | // Pick file 8 because it overlaps with 0 files on level 3. | |
559 | ASSERT_EQ(8U, compaction->input(0, 0)->fd.GetNumber()); | |
11fdf7f2 TL |
560 | // Compaction input size * 1.1 |
561 | ASSERT_GE(uint64_t{55000000}, compaction->OutputFilePreallocationSize()); | |
7c673cae FG |
562 | } |
563 | ||
564 | TEST_F(CompactionPickerTest, CompactionPriMinOverlapping2) { | |
565 | NewVersionStorage(6, kCompactionStyleLevel); | |
566 | ioptions_.compaction_pri = kMinOverlappingRatio; | |
567 | mutable_cf_options_.target_file_size_base = 10000000; | |
568 | mutable_cf_options_.target_file_size_multiplier = 10; | |
569 | mutable_cf_options_.max_bytes_for_level_base = 10 * 1024 * 1024; | |
570 | ||
571 | Add(2, 6U, "150", "175", | |
572 | 60000000U); // Overlaps with file 26, 27, total size 521M | |
573 | Add(2, 7U, "176", "200", 60000000U); // Overlaps with file 27, 28, total size | |
574 | // 520M, the smalelst overlapping | |
575 | Add(2, 8U, "201", "300", | |
576 | 60000000U); // Overlaps with file 28, 29, total size 521M | |
577 | ||
578 | Add(3, 26U, "100", "110", 261000000U); | |
579 | Add(3, 26U, "150", "170", 261000000U); | |
580 | Add(3, 27U, "171", "179", 260000000U); | |
581 | Add(3, 28U, "191", "220", 260000000U); | |
582 | Add(3, 29U, "221", "300", 261000000U); | |
583 | Add(3, 30U, "321", "400", 261000000U); | |
584 | UpdateVersionStorageInfo(); | |
585 | ||
586 | std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction( | |
587 | cf_name_, mutable_cf_options_, vstorage_.get(), &log_buffer_)); | |
588 | ASSERT_TRUE(compaction.get() != nullptr); | |
589 | ASSERT_EQ(1U, compaction->num_input_files(0)); | |
590 | // Picking file 7 because overlapping ratio is the biggest. | |
591 | ASSERT_EQ(7U, compaction->input(0, 0)->fd.GetNumber()); | |
592 | } | |
593 | ||
594 | TEST_F(CompactionPickerTest, CompactionPriMinOverlapping3) { | |
595 | NewVersionStorage(6, kCompactionStyleLevel); | |
596 | ioptions_.compaction_pri = kMinOverlappingRatio; | |
597 | mutable_cf_options_.max_bytes_for_level_base = 10000000; | |
598 | mutable_cf_options_.max_bytes_for_level_multiplier = 10; | |
599 | ||
600 | // file 7 and 8 over lap with the same file, but file 8 is smaller so | |
601 | // it will be picked. | |
602 | Add(2, 6U, "150", "167", 60000000U); // Overlaps with file 26, 27 | |
603 | Add(2, 7U, "168", "169", 60000000U); // Overlaps with file 27 | |
604 | Add(2, 8U, "201", "300", 61000000U); // Overlaps with file 28, but the file | |
605 | // itself is larger. Should be picked. | |
606 | ||
607 | Add(3, 26U, "160", "165", 260000000U); | |
608 | Add(3, 27U, "166", "170", 260000000U); | |
609 | Add(3, 28U, "180", "400", 260000000U); | |
610 | Add(3, 29U, "401", "500", 260000000U); | |
611 | UpdateVersionStorageInfo(); | |
612 | ||
613 | std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction( | |
614 | cf_name_, mutable_cf_options_, vstorage_.get(), &log_buffer_)); | |
615 | ASSERT_TRUE(compaction.get() != nullptr); | |
616 | ASSERT_EQ(1U, compaction->num_input_files(0)); | |
617 | // Picking file 8 because overlapping ratio is the biggest. | |
618 | ASSERT_EQ(8U, compaction->input(0, 0)->fd.GetNumber()); | |
619 | } | |
620 | ||
494da23a TL |
621 | TEST_F(CompactionPickerTest, CompactionPriMinOverlapping4) { |
622 | NewVersionStorage(6, kCompactionStyleLevel); | |
623 | ioptions_.compaction_pri = kMinOverlappingRatio; | |
624 | mutable_cf_options_.max_bytes_for_level_base = 10000000; | |
625 | mutable_cf_options_.max_bytes_for_level_multiplier = 10; | |
626 | ||
627 | // file 7 and 8 over lap with the same file, but file 8 is smaller so | |
628 | // it will be picked. | |
629 | // Overlaps with file 26, 27. And the file is compensated so will be | |
630 | // picked up. | |
631 | Add(2, 6U, "150", "167", 60000000U, 0, 100, 100, 180000000U); | |
632 | Add(2, 7U, "168", "169", 60000000U); // Overlaps with file 27 | |
633 | Add(2, 8U, "201", "300", 61000000U); // Overlaps with file 28 | |
634 | ||
635 | Add(3, 26U, "160", "165", 60000000U); | |
636 | // Boosted file size in output level is not considered. | |
637 | Add(3, 27U, "166", "170", 60000000U, 0, 100, 100, 260000000U); | |
638 | Add(3, 28U, "180", "400", 60000000U); | |
639 | Add(3, 29U, "401", "500", 60000000U); | |
640 | UpdateVersionStorageInfo(); | |
641 | ||
642 | std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction( | |
643 | cf_name_, mutable_cf_options_, vstorage_.get(), &log_buffer_)); | |
644 | ASSERT_TRUE(compaction.get() != nullptr); | |
645 | ASSERT_EQ(1U, compaction->num_input_files(0)); | |
646 | // Picking file 8 because overlapping ratio is the biggest. | |
647 | ASSERT_EQ(6U, compaction->input(0, 0)->fd.GetNumber()); | |
648 | } | |
649 | ||
7c673cae FG |
650 | // This test exhibits the bug where we don't properly reset parent_index in |
651 | // PickCompaction() | |
652 | TEST_F(CompactionPickerTest, ParentIndexResetBug) { | |
653 | int num_levels = ioptions_.num_levels; | |
654 | mutable_cf_options_.level0_file_num_compaction_trigger = 2; | |
655 | mutable_cf_options_.max_bytes_for_level_base = 200; | |
656 | NewVersionStorage(num_levels, kCompactionStyleLevel); | |
657 | Add(0, 1U, "150", "200"); // <- marked for compaction | |
658 | Add(1, 3U, "400", "500", 600); // <- this one needs compacting | |
659 | Add(2, 4U, "150", "200"); | |
660 | Add(2, 5U, "201", "210"); | |
661 | Add(2, 6U, "300", "310"); | |
662 | Add(2, 7U, "400", "500"); // <- being compacted | |
663 | ||
664 | vstorage_->LevelFiles(2)[3]->being_compacted = true; | |
665 | vstorage_->LevelFiles(0)[0]->marked_for_compaction = true; | |
666 | ||
667 | UpdateVersionStorageInfo(); | |
668 | ||
669 | std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction( | |
670 | cf_name_, mutable_cf_options_, vstorage_.get(), &log_buffer_)); | |
671 | } | |
672 | ||
673 | // This test checks ExpandWhileOverlapping() by having overlapping user keys | |
674 | // ranges (with different sequence numbers) in the input files. | |
675 | TEST_F(CompactionPickerTest, OverlappingUserKeys) { | |
676 | NewVersionStorage(6, kCompactionStyleLevel); | |
677 | ioptions_.compaction_pri = kByCompensatedSize; | |
678 | ||
679 | Add(1, 1U, "100", "150", 1U); | |
680 | // Overlapping user keys | |
681 | Add(1, 2U, "200", "400", 1U); | |
682 | Add(1, 3U, "400", "500", 1000000000U, 0, 0); | |
683 | Add(2, 4U, "600", "700", 1U); | |
684 | UpdateVersionStorageInfo(); | |
685 | ||
686 | std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction( | |
687 | cf_name_, mutable_cf_options_, vstorage_.get(), &log_buffer_)); | |
688 | ASSERT_TRUE(compaction.get() != nullptr); | |
689 | ASSERT_EQ(1U, compaction->num_input_levels()); | |
690 | ASSERT_EQ(2U, compaction->num_input_files(0)); | |
691 | ASSERT_EQ(2U, compaction->input(0, 0)->fd.GetNumber()); | |
692 | ASSERT_EQ(3U, compaction->input(0, 1)->fd.GetNumber()); | |
693 | } | |
694 | ||
695 | TEST_F(CompactionPickerTest, OverlappingUserKeys2) { | |
696 | NewVersionStorage(6, kCompactionStyleLevel); | |
697 | // Overlapping user keys on same level and output level | |
698 | Add(1, 1U, "200", "400", 1000000000U); | |
699 | Add(1, 2U, "400", "500", 1U, 0, 0); | |
700 | Add(2, 3U, "000", "100", 1U); | |
701 | Add(2, 4U, "100", "600", 1U, 0, 0); | |
702 | Add(2, 5U, "600", "700", 1U, 0, 0); | |
703 | UpdateVersionStorageInfo(); | |
704 | ||
705 | std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction( | |
706 | cf_name_, mutable_cf_options_, vstorage_.get(), &log_buffer_)); | |
707 | ASSERT_TRUE(compaction.get() != nullptr); | |
708 | ASSERT_EQ(2U, compaction->num_input_levels()); | |
709 | ASSERT_EQ(2U, compaction->num_input_files(0)); | |
710 | ASSERT_EQ(3U, compaction->num_input_files(1)); | |
711 | ASSERT_EQ(1U, compaction->input(0, 0)->fd.GetNumber()); | |
712 | ASSERT_EQ(2U, compaction->input(0, 1)->fd.GetNumber()); | |
713 | ASSERT_EQ(3U, compaction->input(1, 0)->fd.GetNumber()); | |
714 | ASSERT_EQ(4U, compaction->input(1, 1)->fd.GetNumber()); | |
715 | ASSERT_EQ(5U, compaction->input(1, 2)->fd.GetNumber()); | |
716 | } | |
717 | ||
718 | TEST_F(CompactionPickerTest, OverlappingUserKeys3) { | |
719 | NewVersionStorage(6, kCompactionStyleLevel); | |
720 | // Chain of overlapping user key ranges (forces ExpandWhileOverlapping() to | |
721 | // expand multiple times) | |
722 | Add(1, 1U, "100", "150", 1U); | |
723 | Add(1, 2U, "150", "200", 1U, 0, 0); | |
724 | Add(1, 3U, "200", "250", 1000000000U, 0, 0); | |
725 | Add(1, 4U, "250", "300", 1U, 0, 0); | |
726 | Add(1, 5U, "300", "350", 1U, 0, 0); | |
727 | // Output level overlaps with the beginning and the end of the chain | |
728 | Add(2, 6U, "050", "100", 1U); | |
729 | Add(2, 7U, "350", "400", 1U); | |
730 | UpdateVersionStorageInfo(); | |
731 | ||
732 | std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction( | |
733 | cf_name_, mutable_cf_options_, vstorage_.get(), &log_buffer_)); | |
734 | ASSERT_TRUE(compaction.get() != nullptr); | |
735 | ASSERT_EQ(2U, compaction->num_input_levels()); | |
736 | ASSERT_EQ(5U, compaction->num_input_files(0)); | |
737 | ASSERT_EQ(2U, compaction->num_input_files(1)); | |
738 | ASSERT_EQ(1U, compaction->input(0, 0)->fd.GetNumber()); | |
739 | ASSERT_EQ(2U, compaction->input(0, 1)->fd.GetNumber()); | |
740 | ASSERT_EQ(3U, compaction->input(0, 2)->fd.GetNumber()); | |
741 | ASSERT_EQ(4U, compaction->input(0, 3)->fd.GetNumber()); | |
742 | ASSERT_EQ(5U, compaction->input(0, 4)->fd.GetNumber()); | |
743 | ASSERT_EQ(6U, compaction->input(1, 0)->fd.GetNumber()); | |
744 | ASSERT_EQ(7U, compaction->input(1, 1)->fd.GetNumber()); | |
745 | } | |
746 | ||
747 | TEST_F(CompactionPickerTest, OverlappingUserKeys4) { | |
748 | NewVersionStorage(6, kCompactionStyleLevel); | |
749 | mutable_cf_options_.max_bytes_for_level_base = 1000000; | |
750 | ||
751 | Add(1, 1U, "100", "150", 1U); | |
752 | Add(1, 2U, "150", "199", 1U, 0, 0); | |
753 | Add(1, 3U, "200", "250", 1100000U, 0, 0); | |
754 | Add(1, 4U, "251", "300", 1U, 0, 0); | |
755 | Add(1, 5U, "300", "350", 1U, 0, 0); | |
756 | ||
757 | Add(2, 6U, "100", "115", 1U); | |
758 | Add(2, 7U, "125", "325", 1U); | |
759 | Add(2, 8U, "350", "400", 1U); | |
760 | UpdateVersionStorageInfo(); | |
761 | ||
762 | std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction( | |
763 | cf_name_, mutable_cf_options_, vstorage_.get(), &log_buffer_)); | |
764 | ASSERT_TRUE(compaction.get() != nullptr); | |
765 | ASSERT_EQ(2U, compaction->num_input_levels()); | |
766 | ASSERT_EQ(1U, compaction->num_input_files(0)); | |
767 | ASSERT_EQ(1U, compaction->num_input_files(1)); | |
768 | ASSERT_EQ(3U, compaction->input(0, 0)->fd.GetNumber()); | |
769 | ASSERT_EQ(7U, compaction->input(1, 0)->fd.GetNumber()); | |
770 | } | |
771 | ||
772 | TEST_F(CompactionPickerTest, OverlappingUserKeys5) { | |
773 | NewVersionStorage(6, kCompactionStyleLevel); | |
774 | // Overlapping user keys on same level and output level | |
775 | Add(1, 1U, "200", "400", 1000000000U); | |
776 | Add(1, 2U, "400", "500", 1U, 0, 0); | |
777 | Add(2, 3U, "000", "100", 1U); | |
778 | Add(2, 4U, "100", "600", 1U, 0, 0); | |
779 | Add(2, 5U, "600", "700", 1U, 0, 0); | |
780 | ||
781 | vstorage_->LevelFiles(2)[2]->being_compacted = true; | |
782 | ||
783 | UpdateVersionStorageInfo(); | |
784 | ||
785 | std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction( | |
786 | cf_name_, mutable_cf_options_, vstorage_.get(), &log_buffer_)); | |
787 | ASSERT_TRUE(compaction.get() == nullptr); | |
788 | } | |
789 | ||
790 | TEST_F(CompactionPickerTest, OverlappingUserKeys6) { | |
791 | NewVersionStorage(6, kCompactionStyleLevel); | |
792 | // Overlapping user keys on same level and output level | |
793 | Add(1, 1U, "200", "400", 1U, 0, 0); | |
794 | Add(1, 2U, "401", "500", 1U, 0, 0); | |
795 | Add(2, 3U, "000", "100", 1U); | |
796 | Add(2, 4U, "100", "300", 1U, 0, 0); | |
797 | Add(2, 5U, "305", "450", 1U, 0, 0); | |
798 | Add(2, 6U, "460", "600", 1U, 0, 0); | |
799 | Add(2, 7U, "600", "700", 1U, 0, 0); | |
800 | ||
801 | vstorage_->LevelFiles(1)[0]->marked_for_compaction = true; | |
802 | vstorage_->LevelFiles(1)[1]->marked_for_compaction = true; | |
803 | ||
804 | UpdateVersionStorageInfo(); | |
805 | ||
806 | std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction( | |
807 | cf_name_, mutable_cf_options_, vstorage_.get(), &log_buffer_)); | |
808 | ASSERT_TRUE(compaction.get() != nullptr); | |
809 | ASSERT_EQ(2U, compaction->num_input_levels()); | |
810 | ASSERT_EQ(1U, compaction->num_input_files(0)); | |
811 | ASSERT_EQ(3U, compaction->num_input_files(1)); | |
812 | } | |
813 | ||
814 | TEST_F(CompactionPickerTest, OverlappingUserKeys7) { | |
815 | NewVersionStorage(6, kCompactionStyleLevel); | |
816 | mutable_cf_options_.max_compaction_bytes = 100000000000u; | |
817 | // Overlapping user keys on same level and output level | |
818 | Add(1, 1U, "200", "400", 1U, 0, 0); | |
819 | Add(1, 2U, "401", "500", 1000000000U, 0, 0); | |
820 | Add(2, 3U, "100", "250", 1U); | |
821 | Add(2, 4U, "300", "600", 1U, 0, 0); | |
822 | Add(2, 5U, "600", "800", 1U, 0, 0); | |
823 | ||
824 | UpdateVersionStorageInfo(); | |
825 | ||
826 | std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction( | |
827 | cf_name_, mutable_cf_options_, vstorage_.get(), &log_buffer_)); | |
828 | ASSERT_TRUE(compaction.get() != nullptr); | |
829 | ASSERT_EQ(2U, compaction->num_input_levels()); | |
830 | ASSERT_GE(1U, compaction->num_input_files(0)); | |
831 | ASSERT_GE(2U, compaction->num_input_files(1)); | |
832 | // File 5 has to be included in the compaction | |
833 | ASSERT_EQ(5U, compaction->inputs(1)->back()->fd.GetNumber()); | |
834 | } | |
835 | ||
836 | TEST_F(CompactionPickerTest, OverlappingUserKeys8) { | |
837 | NewVersionStorage(6, kCompactionStyleLevel); | |
838 | mutable_cf_options_.max_compaction_bytes = 100000000000u; | |
839 | // grow the number of inputs in "level" without | |
840 | // changing the number of "level+1" files we pick up | |
841 | // Expand input level as much as possible | |
842 | // no overlapping case | |
843 | Add(1, 1U, "101", "150", 1U); | |
844 | Add(1, 2U, "151", "200", 1U); | |
845 | Add(1, 3U, "201", "300", 1000000000U); | |
846 | Add(1, 4U, "301", "400", 1U); | |
847 | Add(1, 5U, "401", "500", 1U); | |
848 | Add(2, 6U, "150", "200", 1U); | |
849 | Add(2, 7U, "200", "450", 1U, 0, 0); | |
850 | Add(2, 8U, "500", "600", 1U); | |
851 | ||
852 | UpdateVersionStorageInfo(); | |
853 | ||
854 | std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction( | |
855 | cf_name_, mutable_cf_options_, vstorage_.get(), &log_buffer_)); | |
856 | ASSERT_TRUE(compaction.get() != nullptr); | |
857 | ASSERT_EQ(2U, compaction->num_input_levels()); | |
858 | ASSERT_EQ(3U, compaction->num_input_files(0)); | |
859 | ASSERT_EQ(2U, compaction->num_input_files(1)); | |
860 | ASSERT_EQ(2U, compaction->input(0, 0)->fd.GetNumber()); | |
861 | ASSERT_EQ(3U, compaction->input(0, 1)->fd.GetNumber()); | |
862 | ASSERT_EQ(4U, compaction->input(0, 2)->fd.GetNumber()); | |
863 | ASSERT_EQ(6U, compaction->input(1, 0)->fd.GetNumber()); | |
864 | ASSERT_EQ(7U, compaction->input(1, 1)->fd.GetNumber()); | |
865 | } | |
866 | ||
867 | TEST_F(CompactionPickerTest, OverlappingUserKeys9) { | |
868 | NewVersionStorage(6, kCompactionStyleLevel); | |
869 | mutable_cf_options_.max_compaction_bytes = 100000000000u; | |
870 | // grow the number of inputs in "level" without | |
871 | // changing the number of "level+1" files we pick up | |
872 | // Expand input level as much as possible | |
873 | // overlapping case | |
874 | Add(1, 1U, "121", "150", 1U); | |
875 | Add(1, 2U, "151", "200", 1U); | |
876 | Add(1, 3U, "201", "300", 1000000000U); | |
877 | Add(1, 4U, "301", "400", 1U); | |
878 | Add(1, 5U, "401", "500", 1U); | |
879 | Add(2, 6U, "100", "120", 1U); | |
880 | Add(2, 7U, "150", "200", 1U); | |
881 | Add(2, 8U, "200", "450", 1U, 0, 0); | |
882 | Add(2, 9U, "501", "600", 1U); | |
883 | ||
884 | UpdateVersionStorageInfo(); | |
885 | ||
886 | std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction( | |
887 | cf_name_, mutable_cf_options_, vstorage_.get(), &log_buffer_)); | |
888 | ASSERT_TRUE(compaction.get() != nullptr); | |
889 | ASSERT_EQ(2U, compaction->num_input_levels()); | |
890 | ASSERT_EQ(5U, compaction->num_input_files(0)); | |
891 | ASSERT_EQ(2U, compaction->num_input_files(1)); | |
892 | ASSERT_EQ(1U, compaction->input(0, 0)->fd.GetNumber()); | |
893 | ASSERT_EQ(2U, compaction->input(0, 1)->fd.GetNumber()); | |
894 | ASSERT_EQ(3U, compaction->input(0, 2)->fd.GetNumber()); | |
895 | ASSERT_EQ(4U, compaction->input(0, 3)->fd.GetNumber()); | |
896 | ASSERT_EQ(7U, compaction->input(1, 0)->fd.GetNumber()); | |
897 | ASSERT_EQ(8U, compaction->input(1, 1)->fd.GetNumber()); | |
898 | } | |
899 | ||
11fdf7f2 TL |
900 | TEST_F(CompactionPickerTest, OverlappingUserKeys10) { |
901 | // Locked file encountered when pulling in extra input-level files with same | |
902 | // user keys. Verify we pick the next-best file from the same input level. | |
903 | NewVersionStorage(6, kCompactionStyleLevel); | |
904 | mutable_cf_options_.max_compaction_bytes = 100000000000u; | |
905 | ||
906 | // file_number 2U is largest and thus first choice. But it overlaps with | |
907 | // file_number 1U which is being compacted. So instead we pick the next- | |
908 | // biggest file, 3U, which is eligible for compaction. | |
909 | Add(1 /* level */, 1U /* file_number */, "100" /* smallest */, | |
910 | "150" /* largest */, 1U /* file_size */); | |
911 | file_map_[1U].first->being_compacted = true; | |
912 | Add(1 /* level */, 2U /* file_number */, "150" /* smallest */, | |
913 | "200" /* largest */, 1000000000U /* file_size */, 0 /* smallest_seq */, | |
914 | 0 /* largest_seq */); | |
915 | Add(1 /* level */, 3U /* file_number */, "201" /* smallest */, | |
916 | "250" /* largest */, 900000000U /* file_size */); | |
917 | Add(2 /* level */, 4U /* file_number */, "100" /* smallest */, | |
918 | "150" /* largest */, 1U /* file_size */); | |
919 | Add(2 /* level */, 5U /* file_number */, "151" /* smallest */, | |
920 | "200" /* largest */, 1U /* file_size */); | |
921 | Add(2 /* level */, 6U /* file_number */, "201" /* smallest */, | |
922 | "250" /* largest */, 1U /* file_size */); | |
923 | ||
924 | UpdateVersionStorageInfo(); | |
925 | ||
926 | std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction( | |
927 | cf_name_, mutable_cf_options_, vstorage_.get(), &log_buffer_)); | |
928 | ASSERT_TRUE(compaction.get() != nullptr); | |
929 | ASSERT_EQ(2U, compaction->num_input_levels()); | |
930 | ASSERT_EQ(1U, compaction->num_input_files(0)); | |
931 | ASSERT_EQ(1U, compaction->num_input_files(1)); | |
932 | ASSERT_EQ(3U, compaction->input(0, 0)->fd.GetNumber()); | |
933 | ASSERT_EQ(6U, compaction->input(1, 0)->fd.GetNumber()); | |
934 | } | |
935 | ||
936 | TEST_F(CompactionPickerTest, OverlappingUserKeys11) { | |
937 | // Locked file encountered when pulling in extra output-level files with same | |
938 | // user keys. Expected to skip that compaction and pick the next-best choice. | |
939 | NewVersionStorage(6, kCompactionStyleLevel); | |
940 | mutable_cf_options_.max_compaction_bytes = 100000000000u; | |
941 | ||
942 | // score(L1) = 3.7 | |
943 | // score(L2) = 1.85 | |
944 | // There is no eligible file in L1 to compact since both candidates pull in | |
945 | // file_number 5U, which overlaps with a file pending compaction (6U). The | |
946 | // first eligible compaction is from L2->L3. | |
947 | Add(1 /* level */, 2U /* file_number */, "151" /* smallest */, | |
948 | "200" /* largest */, 1000000000U /* file_size */); | |
949 | Add(1 /* level */, 3U /* file_number */, "201" /* smallest */, | |
950 | "250" /* largest */, 1U /* file_size */); | |
951 | Add(2 /* level */, 4U /* file_number */, "100" /* smallest */, | |
952 | "149" /* largest */, 5000000000U /* file_size */); | |
953 | Add(2 /* level */, 5U /* file_number */, "150" /* smallest */, | |
954 | "201" /* largest */, 1U /* file_size */); | |
955 | Add(2 /* level */, 6U /* file_number */, "201" /* smallest */, | |
956 | "249" /* largest */, 1U /* file_size */, 0 /* smallest_seq */, | |
957 | 0 /* largest_seq */); | |
958 | file_map_[6U].first->being_compacted = true; | |
959 | Add(3 /* level */, 7U /* file_number */, "100" /* smallest */, | |
960 | "149" /* largest */, 1U /* file_size */); | |
961 | ||
962 | UpdateVersionStorageInfo(); | |
963 | ||
964 | std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction( | |
965 | cf_name_, mutable_cf_options_, vstorage_.get(), &log_buffer_)); | |
966 | ASSERT_TRUE(compaction.get() != nullptr); | |
967 | ASSERT_EQ(2U, compaction->num_input_levels()); | |
968 | ASSERT_EQ(1U, compaction->num_input_files(0)); | |
969 | ASSERT_EQ(1U, compaction->num_input_files(1)); | |
970 | ASSERT_EQ(4U, compaction->input(0, 0)->fd.GetNumber()); | |
971 | ASSERT_EQ(7U, compaction->input(1, 0)->fd.GetNumber()); | |
972 | } | |
973 | ||
7c673cae FG |
974 | TEST_F(CompactionPickerTest, NotScheduleL1IfL0WithHigherPri1) { |
975 | NewVersionStorage(6, kCompactionStyleLevel); | |
976 | mutable_cf_options_.level0_file_num_compaction_trigger = 2; | |
977 | mutable_cf_options_.max_bytes_for_level_base = 900000000U; | |
978 | ||
979 | // 6 L0 files, score 3. | |
980 | Add(0, 1U, "000", "400", 1U); | |
981 | Add(0, 2U, "001", "400", 1U, 0, 0); | |
982 | Add(0, 3U, "001", "400", 1000000000U, 0, 0); | |
983 | Add(0, 31U, "001", "400", 1000000000U, 0, 0); | |
984 | Add(0, 32U, "001", "400", 1000000000U, 0, 0); | |
985 | Add(0, 33U, "001", "400", 1000000000U, 0, 0); | |
986 | ||
987 | // L1 total size 2GB, score 2.2. If one file being comapcted, score 1.1. | |
988 | Add(1, 4U, "050", "300", 1000000000U, 0, 0); | |
989 | file_map_[4u].first->being_compacted = true; | |
990 | Add(1, 5U, "301", "350", 1000000000U, 0, 0); | |
991 | ||
992 | // Output level overlaps with the beginning and the end of the chain | |
993 | Add(2, 6U, "050", "100", 1U); | |
994 | Add(2, 7U, "300", "400", 1U); | |
995 | ||
996 | // No compaction should be scheduled, if L0 has higher priority than L1 | |
997 | // but L0->L1 compaction is blocked by a file in L1 being compacted. | |
998 | UpdateVersionStorageInfo(); | |
999 | ASSERT_EQ(0, vstorage_->CompactionScoreLevel(0)); | |
1000 | ASSERT_EQ(1, vstorage_->CompactionScoreLevel(1)); | |
1001 | std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction( | |
1002 | cf_name_, mutable_cf_options_, vstorage_.get(), &log_buffer_)); | |
1003 | ASSERT_TRUE(compaction.get() == nullptr); | |
1004 | } | |
1005 | ||
1006 | TEST_F(CompactionPickerTest, NotScheduleL1IfL0WithHigherPri2) { | |
1007 | NewVersionStorage(6, kCompactionStyleLevel); | |
1008 | mutable_cf_options_.level0_file_num_compaction_trigger = 2; | |
1009 | mutable_cf_options_.max_bytes_for_level_base = 900000000U; | |
1010 | ||
1011 | // 6 L0 files, score 3. | |
1012 | Add(0, 1U, "000", "400", 1U); | |
1013 | Add(0, 2U, "001", "400", 1U, 0, 0); | |
1014 | Add(0, 3U, "001", "400", 1000000000U, 0, 0); | |
1015 | Add(0, 31U, "001", "400", 1000000000U, 0, 0); | |
1016 | Add(0, 32U, "001", "400", 1000000000U, 0, 0); | |
1017 | Add(0, 33U, "001", "400", 1000000000U, 0, 0); | |
1018 | ||
1019 | // L1 total size 2GB, score 2.2. If one file being comapcted, score 1.1. | |
1020 | Add(1, 4U, "050", "300", 1000000000U, 0, 0); | |
1021 | Add(1, 5U, "301", "350", 1000000000U, 0, 0); | |
1022 | ||
1023 | // Output level overlaps with the beginning and the end of the chain | |
1024 | Add(2, 6U, "050", "100", 1U); | |
1025 | Add(2, 7U, "300", "400", 1U); | |
1026 | ||
1027 | // If no file in L1 being compacted, L0->L1 compaction will be scheduled. | |
1028 | UpdateVersionStorageInfo(); // being_compacted flag is cleared here. | |
1029 | ASSERT_EQ(0, vstorage_->CompactionScoreLevel(0)); | |
1030 | ASSERT_EQ(1, vstorage_->CompactionScoreLevel(1)); | |
1031 | std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction( | |
1032 | cf_name_, mutable_cf_options_, vstorage_.get(), &log_buffer_)); | |
1033 | ASSERT_TRUE(compaction.get() != nullptr); | |
1034 | } | |
1035 | ||
1036 | TEST_F(CompactionPickerTest, NotScheduleL1IfL0WithHigherPri3) { | |
1037 | NewVersionStorage(6, kCompactionStyleLevel); | |
1038 | mutable_cf_options_.level0_file_num_compaction_trigger = 2; | |
1039 | mutable_cf_options_.max_bytes_for_level_base = 900000000U; | |
1040 | ||
1041 | // 6 L0 files, score 3. | |
1042 | Add(0, 1U, "000", "400", 1U); | |
1043 | Add(0, 2U, "001", "400", 1U, 0, 0); | |
1044 | Add(0, 3U, "001", "400", 1000000000U, 0, 0); | |
1045 | Add(0, 31U, "001", "400", 1000000000U, 0, 0); | |
1046 | Add(0, 32U, "001", "400", 1000000000U, 0, 0); | |
1047 | Add(0, 33U, "001", "400", 1000000000U, 0, 0); | |
1048 | ||
1049 | // L1 score more than 6. | |
1050 | Add(1, 4U, "050", "300", 1000000000U, 0, 0); | |
1051 | file_map_[4u].first->being_compacted = true; | |
1052 | Add(1, 5U, "301", "350", 1000000000U, 0, 0); | |
1053 | Add(1, 51U, "351", "400", 6000000000U, 0, 0); | |
1054 | ||
1055 | // Output level overlaps with the beginning and the end of the chain | |
1056 | Add(2, 6U, "050", "100", 1U); | |
1057 | Add(2, 7U, "300", "400", 1U); | |
1058 | ||
1059 | // If score in L1 is larger than L0, L1 compaction goes through despite | |
1060 | // there is pending L0 compaction. | |
1061 | UpdateVersionStorageInfo(); | |
1062 | ASSERT_EQ(1, vstorage_->CompactionScoreLevel(0)); | |
1063 | ASSERT_EQ(0, vstorage_->CompactionScoreLevel(1)); | |
1064 | std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction( | |
1065 | cf_name_, mutable_cf_options_, vstorage_.get(), &log_buffer_)); | |
1066 | ASSERT_TRUE(compaction.get() != nullptr); | |
1067 | } | |
1068 | ||
1069 | TEST_F(CompactionPickerTest, EstimateCompactionBytesNeeded1) { | |
1070 | int num_levels = ioptions_.num_levels; | |
1071 | ioptions_.level_compaction_dynamic_level_bytes = false; | |
11fdf7f2 | 1072 | mutable_cf_options_.level0_file_num_compaction_trigger = 4; |
7c673cae FG |
1073 | mutable_cf_options_.max_bytes_for_level_base = 1000; |
1074 | mutable_cf_options_.max_bytes_for_level_multiplier = 10; | |
1075 | NewVersionStorage(num_levels, kCompactionStyleLevel); | |
1076 | Add(0, 1U, "150", "200", 200); | |
1077 | Add(0, 2U, "150", "200", 200); | |
1078 | Add(0, 3U, "150", "200", 200); | |
1079 | // Level 1 is over target by 200 | |
1080 | Add(1, 4U, "400", "500", 600); | |
1081 | Add(1, 5U, "600", "700", 600); | |
1082 | // Level 2 is less than target 10000 even added size of level 1 | |
1083 | // Size ratio of L2/L1 is 9600 / 1200 = 8 | |
1084 | Add(2, 6U, "150", "200", 2500); | |
1085 | Add(2, 7U, "201", "210", 2000); | |
1086 | Add(2, 8U, "300", "310", 2600); | |
1087 | Add(2, 9U, "400", "500", 2500); | |
1088 | // Level 3 exceeds target 100,000 of 1000 | |
1089 | Add(3, 10U, "400", "500", 101000); | |
1090 | // Level 4 exceeds target 1,000,000 by 900 after adding size from level 3 | |
1091 | // Size ratio L4/L3 is 9.9 | |
1092 | // After merge from L3, L4 size is 1000900 | |
1093 | Add(4, 11U, "400", "500", 999900); | |
1094 | Add(5, 11U, "400", "500", 8007200); | |
1095 | ||
1096 | UpdateVersionStorageInfo(); | |
1097 | ||
1098 | ASSERT_EQ(200u * 9u + 10900u + 900u * 9, | |
1099 | vstorage_->estimated_compaction_needed_bytes()); | |
1100 | } | |
1101 | ||
1102 | TEST_F(CompactionPickerTest, EstimateCompactionBytesNeeded2) { | |
1103 | int num_levels = ioptions_.num_levels; | |
1104 | ioptions_.level_compaction_dynamic_level_bytes = false; | |
1105 | mutable_cf_options_.level0_file_num_compaction_trigger = 3; | |
1106 | mutable_cf_options_.max_bytes_for_level_base = 1000; | |
1107 | mutable_cf_options_.max_bytes_for_level_multiplier = 10; | |
1108 | NewVersionStorage(num_levels, kCompactionStyleLevel); | |
1109 | Add(0, 1U, "150", "200", 200); | |
1110 | Add(0, 2U, "150", "200", 200); | |
1111 | Add(0, 4U, "150", "200", 200); | |
1112 | Add(0, 5U, "150", "200", 200); | |
1113 | Add(0, 6U, "150", "200", 200); | |
1114 | // Level 1 size will be 1400 after merging with L0 | |
1115 | Add(1, 7U, "400", "500", 200); | |
1116 | Add(1, 8U, "600", "700", 200); | |
1117 | // Level 2 is less than target 10000 even added size of level 1 | |
1118 | Add(2, 9U, "150", "200", 9100); | |
1119 | // Level 3 over the target, but since level 4 is empty, we assume it will be | |
1120 | // a trivial move. | |
1121 | Add(3, 10U, "400", "500", 101000); | |
1122 | ||
1123 | UpdateVersionStorageInfo(); | |
1124 | ||
1125 | // estimated L1->L2 merge: 400 * (9100.0 / 1400.0 + 1.0) | |
1126 | ASSERT_EQ(1400u + 3000u, vstorage_->estimated_compaction_needed_bytes()); | |
1127 | } | |
1128 | ||
1129 | TEST_F(CompactionPickerTest, EstimateCompactionBytesNeeded3) { | |
1130 | int num_levels = ioptions_.num_levels; | |
1131 | ioptions_.level_compaction_dynamic_level_bytes = false; | |
1132 | mutable_cf_options_.level0_file_num_compaction_trigger = 3; | |
1133 | mutable_cf_options_.max_bytes_for_level_base = 1000; | |
1134 | mutable_cf_options_.max_bytes_for_level_multiplier = 10; | |
1135 | NewVersionStorage(num_levels, kCompactionStyleLevel); | |
1136 | Add(0, 1U, "150", "200", 2000); | |
1137 | Add(0, 2U, "150", "200", 2000); | |
1138 | Add(0, 4U, "150", "200", 2000); | |
1139 | Add(0, 5U, "150", "200", 2000); | |
1140 | Add(0, 6U, "150", "200", 1000); | |
1141 | // Level 1 size will be 10000 after merging with L0 | |
1142 | Add(1, 7U, "400", "500", 500); | |
1143 | Add(1, 8U, "600", "700", 500); | |
1144 | ||
1145 | Add(2, 9U, "150", "200", 10000); | |
1146 | ||
1147 | UpdateVersionStorageInfo(); | |
1148 | ||
1149 | ASSERT_EQ(10000u + 18000u, vstorage_->estimated_compaction_needed_bytes()); | |
1150 | } | |
1151 | ||
1152 | TEST_F(CompactionPickerTest, EstimateCompactionBytesNeededDynamicLevel) { | |
1153 | int num_levels = ioptions_.num_levels; | |
1154 | ioptions_.level_compaction_dynamic_level_bytes = true; | |
1155 | mutable_cf_options_.level0_file_num_compaction_trigger = 3; | |
1156 | mutable_cf_options_.max_bytes_for_level_base = 1000; | |
1157 | mutable_cf_options_.max_bytes_for_level_multiplier = 10; | |
1158 | NewVersionStorage(num_levels, kCompactionStyleLevel); | |
1159 | ||
1160 | // Set Last level size 50000 | |
1161 | // num_levels - 1 target 5000 | |
11fdf7f2 TL |
1162 | // num_levels - 2 is base level with target 1000 (rounded up to |
1163 | // max_bytes_for_level_base). | |
7c673cae FG |
1164 | Add(num_levels - 1, 10U, "400", "500", 50000); |
1165 | ||
1166 | Add(0, 1U, "150", "200", 200); | |
1167 | Add(0, 2U, "150", "200", 200); | |
1168 | Add(0, 4U, "150", "200", 200); | |
1169 | Add(0, 5U, "150", "200", 200); | |
1170 | Add(0, 6U, "150", "200", 200); | |
1171 | // num_levels - 3 is over target by 100 + 1000 | |
11fdf7f2 TL |
1172 | Add(num_levels - 3, 7U, "400", "500", 550); |
1173 | Add(num_levels - 3, 8U, "600", "700", 550); | |
7c673cae FG |
1174 | // num_levels - 2 is over target by 1100 + 200 |
1175 | Add(num_levels - 2, 9U, "150", "200", 5200); | |
1176 | ||
1177 | UpdateVersionStorageInfo(); | |
1178 | ||
11fdf7f2 | 1179 | // Merging to the second last level: (5200 / 2100 + 1) * 1100 |
7c673cae | 1180 | // Merging to the last level: (50000 / 6300 + 1) * 1300 |
11fdf7f2 | 1181 | ASSERT_EQ(2100u + 3823u + 11617u, |
7c673cae FG |
1182 | vstorage_->estimated_compaction_needed_bytes()); |
1183 | } | |
1184 | ||
1185 | TEST_F(CompactionPickerTest, IsBottommostLevelTest) { | |
1186 | // case 1: Higher levels are empty | |
1187 | NewVersionStorage(6, kCompactionStyleLevel); | |
1188 | Add(0, 1U, "a", "m"); | |
1189 | Add(0, 2U, "c", "z"); | |
1190 | Add(1, 3U, "d", "e"); | |
1191 | Add(1, 4U, "l", "p"); | |
1192 | Add(2, 5U, "g", "i"); | |
1193 | Add(2, 6U, "x", "z"); | |
1194 | UpdateVersionStorageInfo(); | |
1195 | SetCompactionInputFilesLevels(2, 1); | |
1196 | AddToCompactionFiles(3U); | |
1197 | AddToCompactionFiles(5U); | |
1198 | bool result = | |
1199 | Compaction::TEST_IsBottommostLevel(2, vstorage_.get(), input_files_); | |
1200 | ASSERT_TRUE(result); | |
1201 | ||
1202 | // case 2: Higher levels have no overlap | |
1203 | NewVersionStorage(6, kCompactionStyleLevel); | |
1204 | Add(0, 1U, "a", "m"); | |
1205 | Add(0, 2U, "c", "z"); | |
1206 | Add(1, 3U, "d", "e"); | |
1207 | Add(1, 4U, "l", "p"); | |
1208 | Add(2, 5U, "g", "i"); | |
1209 | Add(2, 6U, "x", "z"); | |
1210 | Add(3, 7U, "k", "p"); | |
1211 | Add(3, 8U, "t", "w"); | |
1212 | Add(4, 9U, "a", "b"); | |
1213 | Add(5, 10U, "c", "cc"); | |
1214 | UpdateVersionStorageInfo(); | |
1215 | SetCompactionInputFilesLevels(2, 1); | |
1216 | AddToCompactionFiles(3U); | |
1217 | AddToCompactionFiles(5U); | |
1218 | result = Compaction::TEST_IsBottommostLevel(2, vstorage_.get(), input_files_); | |
1219 | ASSERT_TRUE(result); | |
1220 | ||
1221 | // case 3.1: Higher levels (level 3) have overlap | |
1222 | NewVersionStorage(6, kCompactionStyleLevel); | |
1223 | Add(0, 1U, "a", "m"); | |
1224 | Add(0, 2U, "c", "z"); | |
1225 | Add(1, 3U, "d", "e"); | |
1226 | Add(1, 4U, "l", "p"); | |
1227 | Add(2, 5U, "g", "i"); | |
1228 | Add(2, 6U, "x", "z"); | |
1229 | Add(3, 7U, "e", "g"); | |
1230 | Add(3, 8U, "h", "k"); | |
1231 | Add(4, 9U, "a", "b"); | |
1232 | Add(5, 10U, "c", "cc"); | |
1233 | UpdateVersionStorageInfo(); | |
1234 | SetCompactionInputFilesLevels(2, 1); | |
1235 | AddToCompactionFiles(3U); | |
1236 | AddToCompactionFiles(5U); | |
1237 | result = Compaction::TEST_IsBottommostLevel(2, vstorage_.get(), input_files_); | |
1238 | ASSERT_FALSE(result); | |
1239 | ||
1240 | // case 3.2: Higher levels (level 5) have overlap | |
1241 | DeleteVersionStorage(); | |
1242 | NewVersionStorage(6, kCompactionStyleLevel); | |
1243 | Add(0, 1U, "a", "m"); | |
1244 | Add(0, 2U, "c", "z"); | |
1245 | Add(1, 3U, "d", "e"); | |
1246 | Add(1, 4U, "l", "p"); | |
1247 | Add(2, 5U, "g", "i"); | |
1248 | Add(2, 6U, "x", "z"); | |
1249 | Add(3, 7U, "j", "k"); | |
1250 | Add(3, 8U, "l", "m"); | |
1251 | Add(4, 9U, "a", "b"); | |
1252 | Add(5, 10U, "c", "cc"); | |
1253 | Add(5, 11U, "h", "k"); | |
1254 | Add(5, 12U, "y", "yy"); | |
1255 | Add(5, 13U, "z", "zz"); | |
1256 | UpdateVersionStorageInfo(); | |
1257 | SetCompactionInputFilesLevels(2, 1); | |
1258 | AddToCompactionFiles(3U); | |
1259 | AddToCompactionFiles(5U); | |
1260 | result = Compaction::TEST_IsBottommostLevel(2, vstorage_.get(), input_files_); | |
1261 | ASSERT_FALSE(result); | |
1262 | ||
1263 | // case 3.3: Higher levels (level 5) have overlap, but it's only overlapping | |
1264 | // one key ("d") | |
1265 | NewVersionStorage(6, kCompactionStyleLevel); | |
1266 | Add(0, 1U, "a", "m"); | |
1267 | Add(0, 2U, "c", "z"); | |
1268 | Add(1, 3U, "d", "e"); | |
1269 | Add(1, 4U, "l", "p"); | |
1270 | Add(2, 5U, "g", "i"); | |
1271 | Add(2, 6U, "x", "z"); | |
1272 | Add(3, 7U, "j", "k"); | |
1273 | Add(3, 8U, "l", "m"); | |
1274 | Add(4, 9U, "a", "b"); | |
1275 | Add(5, 10U, "c", "cc"); | |
1276 | Add(5, 11U, "ccc", "d"); | |
1277 | Add(5, 12U, "y", "yy"); | |
1278 | Add(5, 13U, "z", "zz"); | |
1279 | UpdateVersionStorageInfo(); | |
1280 | SetCompactionInputFilesLevels(2, 1); | |
1281 | AddToCompactionFiles(3U); | |
1282 | AddToCompactionFiles(5U); | |
1283 | result = Compaction::TEST_IsBottommostLevel(2, vstorage_.get(), input_files_); | |
1284 | ASSERT_FALSE(result); | |
1285 | ||
1286 | // Level 0 files overlap | |
1287 | NewVersionStorage(6, kCompactionStyleLevel); | |
1288 | Add(0, 1U, "s", "t"); | |
1289 | Add(0, 2U, "a", "m"); | |
1290 | Add(0, 3U, "b", "z"); | |
1291 | Add(0, 4U, "e", "f"); | |
1292 | Add(5, 10U, "y", "z"); | |
1293 | UpdateVersionStorageInfo(); | |
1294 | SetCompactionInputFilesLevels(1, 0); | |
1295 | AddToCompactionFiles(1U); | |
1296 | AddToCompactionFiles(2U); | |
1297 | AddToCompactionFiles(3U); | |
1298 | AddToCompactionFiles(4U); | |
1299 | result = Compaction::TEST_IsBottommostLevel(2, vstorage_.get(), input_files_); | |
1300 | ASSERT_FALSE(result); | |
1301 | ||
1302 | // Level 0 files don't overlap | |
1303 | NewVersionStorage(6, kCompactionStyleLevel); | |
1304 | Add(0, 1U, "s", "t"); | |
1305 | Add(0, 2U, "a", "m"); | |
1306 | Add(0, 3U, "b", "k"); | |
1307 | Add(0, 4U, "e", "f"); | |
1308 | Add(5, 10U, "y", "z"); | |
1309 | UpdateVersionStorageInfo(); | |
1310 | SetCompactionInputFilesLevels(1, 0); | |
1311 | AddToCompactionFiles(1U); | |
1312 | AddToCompactionFiles(2U); | |
1313 | AddToCompactionFiles(3U); | |
1314 | AddToCompactionFiles(4U); | |
1315 | result = Compaction::TEST_IsBottommostLevel(2, vstorage_.get(), input_files_); | |
1316 | ASSERT_TRUE(result); | |
1317 | ||
1318 | // Level 1 files overlap | |
1319 | NewVersionStorage(6, kCompactionStyleLevel); | |
1320 | Add(0, 1U, "s", "t"); | |
1321 | Add(0, 2U, "a", "m"); | |
1322 | Add(0, 3U, "b", "k"); | |
1323 | Add(0, 4U, "e", "f"); | |
1324 | Add(1, 5U, "a", "m"); | |
1325 | Add(1, 6U, "n", "o"); | |
1326 | Add(1, 7U, "w", "y"); | |
1327 | Add(5, 10U, "y", "z"); | |
1328 | UpdateVersionStorageInfo(); | |
1329 | SetCompactionInputFilesLevels(2, 0); | |
1330 | AddToCompactionFiles(1U); | |
1331 | AddToCompactionFiles(2U); | |
1332 | AddToCompactionFiles(3U); | |
1333 | AddToCompactionFiles(4U); | |
1334 | AddToCompactionFiles(5U); | |
1335 | AddToCompactionFiles(6U); | |
1336 | AddToCompactionFiles(7U); | |
1337 | result = Compaction::TEST_IsBottommostLevel(2, vstorage_.get(), input_files_); | |
1338 | ASSERT_FALSE(result); | |
1339 | ||
1340 | DeleteVersionStorage(); | |
1341 | } | |
1342 | ||
1343 | TEST_F(CompactionPickerTest, MaxCompactionBytesHit) { | |
1344 | mutable_cf_options_.max_bytes_for_level_base = 1000000u; | |
1345 | mutable_cf_options_.max_compaction_bytes = 800000u; | |
1346 | ioptions_.level_compaction_dynamic_level_bytes = false; | |
1347 | NewVersionStorage(6, kCompactionStyleLevel); | |
1348 | // A compaction should be triggered and pick file 2 and 5. | |
1349 | // It can expand because adding file 1 and 3, the compaction size will | |
1350 | // exceed mutable_cf_options_.max_bytes_for_level_base. | |
1351 | Add(1, 1U, "100", "150", 300000U); | |
1352 | Add(1, 2U, "151", "200", 300001U, 0, 0); | |
1353 | Add(1, 3U, "201", "250", 300000U, 0, 0); | |
1354 | Add(1, 4U, "251", "300", 300000U, 0, 0); | |
1355 | Add(2, 5U, "100", "256", 1U); | |
1356 | UpdateVersionStorageInfo(); | |
1357 | ||
1358 | std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction( | |
1359 | cf_name_, mutable_cf_options_, vstorage_.get(), &log_buffer_)); | |
1360 | ASSERT_TRUE(compaction.get() != nullptr); | |
1361 | ASSERT_EQ(2U, compaction->num_input_levels()); | |
1362 | ASSERT_EQ(1U, compaction->num_input_files(0)); | |
1363 | ASSERT_EQ(1U, compaction->num_input_files(1)); | |
1364 | ASSERT_EQ(2U, compaction->input(0, 0)->fd.GetNumber()); | |
1365 | ASSERT_EQ(5U, compaction->input(1, 0)->fd.GetNumber()); | |
1366 | } | |
1367 | ||
1368 | TEST_F(CompactionPickerTest, MaxCompactionBytesNotHit) { | |
1369 | mutable_cf_options_.max_bytes_for_level_base = 800000u; | |
1370 | mutable_cf_options_.max_compaction_bytes = 1000000u; | |
1371 | ioptions_.level_compaction_dynamic_level_bytes = false; | |
1372 | NewVersionStorage(6, kCompactionStyleLevel); | |
1373 | // A compaction should be triggered and pick file 2 and 5. | |
1374 | // and it expands to file 1 and 3 too. | |
1375 | Add(1, 1U, "100", "150", 300000U); | |
1376 | Add(1, 2U, "151", "200", 300001U, 0, 0); | |
1377 | Add(1, 3U, "201", "250", 300000U, 0, 0); | |
1378 | Add(1, 4U, "251", "300", 300000U, 0, 0); | |
1379 | Add(2, 5U, "000", "251", 1U); | |
1380 | UpdateVersionStorageInfo(); | |
1381 | ||
1382 | std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction( | |
1383 | cf_name_, mutable_cf_options_, vstorage_.get(), &log_buffer_)); | |
1384 | ASSERT_TRUE(compaction.get() != nullptr); | |
1385 | ASSERT_EQ(2U, compaction->num_input_levels()); | |
1386 | ASSERT_EQ(3U, compaction->num_input_files(0)); | |
1387 | ASSERT_EQ(1U, compaction->num_input_files(1)); | |
1388 | ASSERT_EQ(1U, compaction->input(0, 0)->fd.GetNumber()); | |
1389 | ASSERT_EQ(2U, compaction->input(0, 1)->fd.GetNumber()); | |
1390 | ASSERT_EQ(3U, compaction->input(0, 2)->fd.GetNumber()); | |
1391 | ASSERT_EQ(5U, compaction->input(1, 0)->fd.GetNumber()); | |
1392 | } | |
1393 | ||
1394 | TEST_F(CompactionPickerTest, IsTrivialMoveOn) { | |
1395 | mutable_cf_options_.max_bytes_for_level_base = 10000u; | |
1396 | mutable_cf_options_.max_compaction_bytes = 10001u; | |
1397 | ioptions_.level_compaction_dynamic_level_bytes = false; | |
1398 | NewVersionStorage(6, kCompactionStyleLevel); | |
1399 | // A compaction should be triggered and pick file 2 | |
1400 | Add(1, 1U, "100", "150", 3000U); | |
1401 | Add(1, 2U, "151", "200", 3001U); | |
1402 | Add(1, 3U, "201", "250", 3000U); | |
1403 | Add(1, 4U, "251", "300", 3000U); | |
1404 | ||
1405 | Add(3, 5U, "120", "130", 7000U); | |
1406 | Add(3, 6U, "170", "180", 7000U); | |
1407 | Add(3, 5U, "220", "230", 7000U); | |
1408 | Add(3, 5U, "270", "280", 7000U); | |
1409 | UpdateVersionStorageInfo(); | |
1410 | ||
1411 | std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction( | |
1412 | cf_name_, mutable_cf_options_, vstorage_.get(), &log_buffer_)); | |
1413 | ASSERT_TRUE(compaction.get() != nullptr); | |
1414 | ASSERT_TRUE(compaction->IsTrivialMove()); | |
1415 | } | |
1416 | ||
1417 | TEST_F(CompactionPickerTest, IsTrivialMoveOff) { | |
1418 | mutable_cf_options_.max_bytes_for_level_base = 1000000u; | |
1419 | mutable_cf_options_.max_compaction_bytes = 10000u; | |
1420 | ioptions_.level_compaction_dynamic_level_bytes = false; | |
1421 | NewVersionStorage(6, kCompactionStyleLevel); | |
1422 | // A compaction should be triggered and pick all files from level 1 | |
1423 | Add(1, 1U, "100", "150", 300000U, 0, 0); | |
1424 | Add(1, 2U, "150", "200", 300000U, 0, 0); | |
1425 | Add(1, 3U, "200", "250", 300000U, 0, 0); | |
1426 | Add(1, 4U, "250", "300", 300000U, 0, 0); | |
1427 | ||
1428 | Add(3, 5U, "120", "130", 6000U); | |
1429 | Add(3, 6U, "140", "150", 6000U); | |
1430 | UpdateVersionStorageInfo(); | |
1431 | ||
1432 | std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction( | |
1433 | cf_name_, mutable_cf_options_, vstorage_.get(), &log_buffer_)); | |
1434 | ASSERT_TRUE(compaction.get() != nullptr); | |
1435 | ASSERT_FALSE(compaction->IsTrivialMove()); | |
1436 | } | |
1437 | ||
11fdf7f2 TL |
1438 | TEST_F(CompactionPickerTest, CacheNextCompactionIndex) { |
1439 | NewVersionStorage(6, kCompactionStyleLevel); | |
1440 | mutable_cf_options_.max_compaction_bytes = 100000000000u; | |
1441 | ||
1442 | Add(1 /* level */, 1U /* file_number */, "100" /* smallest */, | |
1443 | "149" /* largest */, 1000000000U /* file_size */); | |
1444 | file_map_[1U].first->being_compacted = true; | |
1445 | Add(1 /* level */, 2U /* file_number */, "150" /* smallest */, | |
1446 | "199" /* largest */, 900000000U /* file_size */); | |
1447 | Add(1 /* level */, 3U /* file_number */, "200" /* smallest */, | |
1448 | "249" /* largest */, 800000000U /* file_size */); | |
1449 | Add(1 /* level */, 4U /* file_number */, "250" /* smallest */, | |
1450 | "299" /* largest */, 700000000U /* file_size */); | |
1451 | Add(2 /* level */, 5U /* file_number */, "150" /* smallest */, | |
1452 | "199" /* largest */, 1U /* file_size */); | |
1453 | file_map_[5U].first->being_compacted = true; | |
1454 | ||
1455 | UpdateVersionStorageInfo(); | |
1456 | ||
1457 | std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction( | |
1458 | cf_name_, mutable_cf_options_, vstorage_.get(), &log_buffer_)); | |
1459 | ASSERT_TRUE(compaction.get() != nullptr); | |
1460 | ASSERT_EQ(1U, compaction->num_input_levels()); | |
1461 | ASSERT_EQ(1U, compaction->num_input_files(0)); | |
1462 | ASSERT_EQ(0U, compaction->num_input_files(1)); | |
1463 | ASSERT_EQ(3U, compaction->input(0, 0)->fd.GetNumber()); | |
1464 | ASSERT_EQ(2, vstorage_->NextCompactionIndex(1 /* level */)); | |
1465 | ||
1466 | compaction.reset(level_compaction_picker.PickCompaction( | |
1467 | cf_name_, mutable_cf_options_, vstorage_.get(), &log_buffer_)); | |
1468 | ASSERT_TRUE(compaction.get() != nullptr); | |
1469 | ASSERT_EQ(1U, compaction->num_input_levels()); | |
1470 | ASSERT_EQ(1U, compaction->num_input_files(0)); | |
1471 | ASSERT_EQ(0U, compaction->num_input_files(1)); | |
1472 | ASSERT_EQ(4U, compaction->input(0, 0)->fd.GetNumber()); | |
1473 | ASSERT_EQ(3, vstorage_->NextCompactionIndex(1 /* level */)); | |
1474 | ||
1475 | compaction.reset(level_compaction_picker.PickCompaction( | |
1476 | cf_name_, mutable_cf_options_, vstorage_.get(), &log_buffer_)); | |
1477 | ASSERT_TRUE(compaction.get() == nullptr); | |
1478 | ASSERT_EQ(4, vstorage_->NextCompactionIndex(1 /* level */)); | |
1479 | } | |
1480 | ||
7c673cae FG |
1481 | } // namespace rocksdb |
1482 | ||
1483 | int main(int argc, char** argv) { | |
1484 | ::testing::InitGoogleTest(&argc, argv); | |
1485 | return RUN_ALL_TESTS(); | |
1486 | } |