1 // Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
2 // This source code is licensed under both the GPLv2 (found in the
3 // COPYING file in the root directory) and Apache 2.0 License
4 // (found in the LICENSE.Apache file in the root directory).
6 // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
7 // Use of this source code is governed by a BSD-style license that can be
8 // found in the LICENSE file. See the AUTHORS file for names of contributors.
10 // Introduction of SyncPoint effectively disabled building and running this test
12 // which is a pity, it is a good test
13 #if !defined(ROCKSDB_LITE)
15 #include "db/db_test_util.h"
16 #include "port/port.h"
17 #include "port/stack_trace.h"
18 #include "util/random.h"
20 namespace ROCKSDB_NAMESPACE
{
21 class DBTestDynamicLevel
: public DBTestBase
{
24 : DBTestBase("/db_dynamic_level_test", /*env_do_fsync=*/true) {}
27 TEST_F(DBTestDynamicLevel
, DynamicLevelMaxBytesBase
) {
28 if (!Snappy_Supported() || !LZ4_Supported()) {
31 // Use InMemoryEnv, or it would be too slow.
32 std::unique_ptr
<Env
> env(new MockEnv(env_
));
34 const int kNKeys
= 1000;
37 auto verify_func
= [&]() {
38 for (int i
= 0; i
< kNKeys
; i
++) {
39 ASSERT_NE("NOT_FOUND", Get(Key(i
)));
40 ASSERT_NE("NOT_FOUND", Get(Key(kNKeys
* 2 + i
)));
41 if (i
< kNKeys
/ 10) {
42 ASSERT_EQ("NOT_FOUND", Get(Key(kNKeys
+ keys
[i
])));
44 ASSERT_NE("NOT_FOUND", Get(Key(kNKeys
+ keys
[i
])));
50 for (int ordered_insert
= 0; ordered_insert
<= 1; ordered_insert
++) {
51 for (int i
= 0; i
< kNKeys
; i
++) {
54 if (ordered_insert
== 0) {
55 RandomShuffle(std::begin(keys
), std::end(keys
), rnd
.Next());
57 for (int max_background_compactions
= 1; max_background_compactions
< 4;
58 max_background_compactions
+= 2) {
60 options
.env
= env
.get();
61 options
.create_if_missing
= true;
62 options
.write_buffer_size
= 2048;
63 options
.max_write_buffer_number
= 2;
64 options
.level0_file_num_compaction_trigger
= 2;
65 options
.level0_slowdown_writes_trigger
= 2;
66 options
.level0_stop_writes_trigger
= 2;
67 options
.target_file_size_base
= 2048;
68 options
.level_compaction_dynamic_level_bytes
= true;
69 options
.max_bytes_for_level_base
= 10240;
70 options
.max_bytes_for_level_multiplier
= 4;
71 options
.soft_rate_limit
= 1.1;
72 options
.max_background_compactions
= max_background_compactions
;
73 options
.num_levels
= 5;
75 options
.compression_per_level
.resize(3);
76 options
.compression_per_level
[0] = kNoCompression
;
77 options
.compression_per_level
[1] = kLZ4Compression
;
78 options
.compression_per_level
[2] = kSnappyCompression
;
81 DestroyAndReopen(options
);
83 for (int i
= 0; i
< kNKeys
; i
++) {
85 ASSERT_OK(Put(Key(kNKeys
+ key
), rnd
.RandomString(102)));
86 ASSERT_OK(Put(Key(key
), rnd
.RandomString(102)));
87 ASSERT_OK(Put(Key(kNKeys
* 2 + key
), rnd
.RandomString(102)));
88 ASSERT_OK(Delete(Key(kNKeys
+ keys
[i
/ 10])));
89 env_
->SleepForMicroseconds(5000);
93 ASSERT_TRUE(db_
->GetIntProperty("rocksdb.background-errors", &int_prop
));
94 ASSERT_EQ(0U, int_prop
);
97 for (int j
= 0; j
< 2; j
++) {
104 // Test compact range works
105 dbfull()->CompactRange(CompactRangeOptions(), nullptr, nullptr);
106 // All data should be in the last level.
107 ColumnFamilyMetaData cf_meta
;
108 db_
->GetColumnFamilyMetaData(&cf_meta
);
109 ASSERT_EQ(5U, cf_meta
.levels
.size());
110 for (int i
= 0; i
< 4; i
++) {
111 ASSERT_EQ(0U, cf_meta
.levels
[i
].files
.size());
113 ASSERT_GT(cf_meta
.levels
[4U].files
.size(), 0U);
120 env_
->SetBackgroundThreads(1, Env::LOW
);
121 env_
->SetBackgroundThreads(1, Env::HIGH
);
124 // Test specific cases in dynamic max bytes
125 TEST_F(DBTestDynamicLevel
, DynamicLevelMaxBytesBase2
) {
127 int kMaxKey
= 1000000;
129 Options options
= CurrentOptions();
130 options
.compression
= kNoCompression
;
131 options
.create_if_missing
= true;
132 options
.write_buffer_size
= 20480;
133 options
.max_write_buffer_number
= 2;
134 options
.level0_file_num_compaction_trigger
= 2;
135 options
.level0_slowdown_writes_trigger
= 9999;
136 options
.level0_stop_writes_trigger
= 9999;
137 options
.target_file_size_base
= 9102;
138 options
.level_compaction_dynamic_level_bytes
= true;
139 options
.max_bytes_for_level_base
= 40960;
140 options
.max_bytes_for_level_multiplier
= 4;
141 options
.max_background_compactions
= 2;
142 options
.num_levels
= 5;
143 options
.max_compaction_bytes
= 0; // Force not expanding in compactions
144 options
.db_host_id
= ""; // Setting this messes up the file size calculation
145 BlockBasedTableOptions table_options
;
146 table_options
.block_size
= 1024;
147 options
.table_factory
.reset(NewBlockBasedTableFactory(table_options
));
149 DestroyAndReopen(options
);
150 ASSERT_OK(dbfull()->SetOptions({
151 {"disable_auto_compactions", "true"},
155 std::string str_prop
;
157 // Initial base level is the last level
158 ASSERT_TRUE(db_
->GetIntProperty("rocksdb.base-level", &int_prop
));
159 ASSERT_EQ(4U, int_prop
);
161 // Put about 28K to L0
162 for (int i
= 0; i
< 70; i
++) {
163 ASSERT_OK(Put(Key(static_cast<int>(rnd
.Uniform(kMaxKey
))),
164 rnd
.RandomString(380)));
166 ASSERT_OK(dbfull()->SetOptions({
167 {"disable_auto_compactions", "false"},
170 dbfull()->TEST_WaitForCompact();
171 ASSERT_TRUE(db_
->GetIntProperty("rocksdb.base-level", &int_prop
));
172 ASSERT_EQ(4U, int_prop
);
174 // Insert extra about 28K to L0. After they are compacted to L4, the base
175 // level should be changed to L3.
176 ASSERT_OK(dbfull()->SetOptions({
177 {"disable_auto_compactions", "true"},
179 for (int i
= 0; i
< 70; i
++) {
180 ASSERT_OK(Put(Key(static_cast<int>(rnd
.Uniform(kMaxKey
))),
181 rnd
.RandomString(380)));
184 ASSERT_OK(dbfull()->SetOptions({
185 {"disable_auto_compactions", "false"},
188 dbfull()->TEST_WaitForCompact();
189 ASSERT_TRUE(db_
->GetIntProperty("rocksdb.base-level", &int_prop
));
190 ASSERT_EQ(3U, int_prop
);
191 ASSERT_TRUE(db_
->GetProperty("rocksdb.num-files-at-level1", &str_prop
));
192 ASSERT_EQ("0", str_prop
);
193 ASSERT_TRUE(db_
->GetProperty("rocksdb.num-files-at-level2", &str_prop
));
194 ASSERT_EQ("0", str_prop
);
196 // Write even more data while leaving the base level at L3.
197 ASSERT_OK(dbfull()->SetOptions({
198 {"disable_auto_compactions", "true"},
200 // Write about 40K more
201 for (int i
= 0; i
< 100; i
++) {
202 ASSERT_OK(Put(Key(static_cast<int>(rnd
.Uniform(kMaxKey
))),
203 rnd
.RandomString(380)));
205 ASSERT_OK(dbfull()->SetOptions({
206 {"disable_auto_compactions", "false"},
209 dbfull()->TEST_WaitForCompact();
210 ASSERT_TRUE(db_
->GetIntProperty("rocksdb.base-level", &int_prop
));
211 ASSERT_EQ(3U, int_prop
);
213 // Fill up L0, and then run an (auto) L0->Lmax compaction to raise the base
215 ASSERT_OK(dbfull()->SetOptions({
216 {"disable_auto_compactions", "true"},
218 // Write about 650K more.
219 // Each file is about 11KB, with 9KB of data.
220 for (int i
= 0; i
< 1300; i
++) {
221 ASSERT_OK(Put(Key(static_cast<int>(rnd
.Uniform(kMaxKey
))),
222 rnd
.RandomString(380)));
225 // Make sure that the compaction starts before the last bit of data is
226 // flushed, so that the base level isn't raised to L1.
227 ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency({
228 {"CompactionJob::Run():Start", "DynamicLevelMaxBytesBase2:0"},
230 ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing();
232 ASSERT_OK(dbfull()->SetOptions({
233 {"disable_auto_compactions", "false"},
236 TEST_SYNC_POINT("DynamicLevelMaxBytesBase2:0");
238 dbfull()->TEST_WaitForCompact();
239 ASSERT_TRUE(db_
->GetIntProperty("rocksdb.base-level", &int_prop
));
240 ASSERT_EQ(2U, int_prop
);
241 ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing();
242 ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->ClearAllCallBacks();
244 // Write more data until the base level changes to L1. There will be
245 // a manual compaction going on at the same time.
246 ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency({
247 {"CompactionJob::Run():Start", "DynamicLevelMaxBytesBase2:1"},
248 {"DynamicLevelMaxBytesBase2:2", "CompactionJob::Run():End"},
249 {"DynamicLevelMaxBytesBase2:compact_range_finish",
250 "FlushJob::WriteLevel0Table"},
252 ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing();
254 ROCKSDB_NAMESPACE::port::Thread
thread([this] {
255 TEST_SYNC_POINT("DynamicLevelMaxBytesBase2:compact_range_start");
256 ASSERT_OK(db_
->CompactRange(CompactRangeOptions(), nullptr, nullptr));
257 TEST_SYNC_POINT("DynamicLevelMaxBytesBase2:compact_range_finish");
260 TEST_SYNC_POINT("DynamicLevelMaxBytesBase2:1");
261 for (int i
= 0; i
< 2; i
++) {
262 ASSERT_OK(Put(Key(static_cast<int>(rnd
.Uniform(kMaxKey
))),
263 rnd
.RandomString(380)));
265 TEST_SYNC_POINT("DynamicLevelMaxBytesBase2:2");
271 ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing();
272 ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->ClearAllCallBacks();
274 ASSERT_TRUE(db_
->GetIntProperty("rocksdb.base-level", &int_prop
));
275 ASSERT_EQ(1U, int_prop
);
278 // Test specific cases in dynamic max bytes
279 TEST_F(DBTestDynamicLevel
, DynamicLevelMaxBytesCompactRange
) {
281 int kMaxKey
= 1000000;
283 Options options
= CurrentOptions();
284 options
.create_if_missing
= true;
285 options
.write_buffer_size
= 2048;
286 options
.max_write_buffer_number
= 2;
287 options
.level0_file_num_compaction_trigger
= 2;
288 options
.level0_slowdown_writes_trigger
= 9999;
289 options
.level0_stop_writes_trigger
= 9999;
290 options
.target_file_size_base
= 2;
291 options
.level_compaction_dynamic_level_bytes
= true;
292 options
.max_bytes_for_level_base
= 10240;
293 options
.max_bytes_for_level_multiplier
= 4;
294 options
.max_background_compactions
= 1;
295 const int kNumLevels
= 5;
296 options
.num_levels
= kNumLevels
;
297 options
.max_compaction_bytes
= 1; // Force not expanding in compactions
298 BlockBasedTableOptions table_options
;
299 table_options
.block_size
= 1024;
300 options
.table_factory
.reset(NewBlockBasedTableFactory(table_options
));
302 DestroyAndReopen(options
);
304 // Compact against empty DB
305 dbfull()->CompactRange(CompactRangeOptions(), nullptr, nullptr);
308 std::string str_prop
;
310 // Initial base level is the last level
311 ASSERT_TRUE(db_
->GetIntProperty("rocksdb.base-level", &int_prop
));
312 ASSERT_EQ(4U, int_prop
);
314 // Put about 7K to L0
315 for (int i
= 0; i
< 140; i
++) {
317 Put(Key(static_cast<int>(rnd
.Uniform(kMaxKey
))), rnd
.RandomString(80)));
320 dbfull()->TEST_WaitForCompact();
321 if (NumTableFilesAtLevel(0) == 0) {
322 // Make sure level 0 is not empty
324 Put(Key(static_cast<int>(rnd
.Uniform(kMaxKey
))), rnd
.RandomString(80)));
328 ASSERT_TRUE(db_
->GetIntProperty("rocksdb.base-level", &int_prop
));
329 ASSERT_EQ(3U, int_prop
);
330 ASSERT_TRUE(db_
->GetProperty("rocksdb.num-files-at-level1", &str_prop
));
331 ASSERT_EQ("0", str_prop
);
332 ASSERT_TRUE(db_
->GetProperty("rocksdb.num-files-at-level2", &str_prop
));
333 ASSERT_EQ("0", str_prop
);
335 ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing();
336 ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->ClearAllCallBacks();
338 std::set
<int> output_levels
;
339 ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
340 "CompactionPicker::CompactRange:Return", [&](void* arg
) {
341 Compaction
* compaction
= reinterpret_cast<Compaction
*>(arg
);
342 output_levels
.insert(compaction
->output_level());
344 ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing();
346 dbfull()->CompactRange(CompactRangeOptions(), nullptr, nullptr);
347 ASSERT_EQ(output_levels
.size(), 2);
348 ASSERT_TRUE(output_levels
.find(3) != output_levels
.end());
349 ASSERT_TRUE(output_levels
.find(4) != output_levels
.end());
350 ASSERT_TRUE(db_
->GetProperty("rocksdb.num-files-at-level0", &str_prop
));
351 ASSERT_EQ("0", str_prop
);
352 ASSERT_TRUE(db_
->GetProperty("rocksdb.num-files-at-level3", &str_prop
));
353 ASSERT_EQ("0", str_prop
);
354 // Base level is still level 3.
355 ASSERT_TRUE(db_
->GetIntProperty("rocksdb.base-level", &int_prop
));
356 ASSERT_EQ(3U, int_prop
);
359 TEST_F(DBTestDynamicLevel
, DynamicLevelMaxBytesBaseInc
) {
360 Options options
= CurrentOptions();
361 options
.create_if_missing
= true;
362 options
.write_buffer_size
= 2048;
363 options
.max_write_buffer_number
= 2;
364 options
.level0_file_num_compaction_trigger
= 2;
365 options
.level0_slowdown_writes_trigger
= 2;
366 options
.level0_stop_writes_trigger
= 2;
367 options
.target_file_size_base
= 2048;
368 options
.level_compaction_dynamic_level_bytes
= true;
369 options
.max_bytes_for_level_base
= 10240;
370 options
.max_bytes_for_level_multiplier
= 4;
371 options
.soft_rate_limit
= 1.1;
372 options
.max_background_compactions
= 2;
373 options
.num_levels
= 5;
374 options
.max_compaction_bytes
= 100000000;
376 DestroyAndReopen(options
);
379 ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
380 "DBImpl::BackgroundCompaction:NonTrivial",
381 [&](void* /*arg*/) { non_trivial
++; });
382 ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing();
385 const int total_keys
= 3000;
386 const int random_part_size
= 100;
387 for (int i
= 0; i
< total_keys
; i
++) {
388 std::string value
= rnd
.RandomString(random_part_size
);
389 PutFixed32(&value
, static_cast<uint32_t>(i
));
390 ASSERT_OK(Put(Key(i
), value
));
393 dbfull()->TEST_WaitForCompact();
394 ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing();
396 ASSERT_EQ(non_trivial
, 0);
398 for (int i
= 0; i
< total_keys
; i
++) {
399 std::string value
= Get(Key(i
));
400 ASSERT_EQ(DecodeFixed32(value
.c_str() + random_part_size
),
401 static_cast<uint32_t>(i
));
404 env_
->SetBackgroundThreads(1, Env::LOW
);
405 env_
->SetBackgroundThreads(1, Env::HIGH
);
408 TEST_F(DBTestDynamicLevel
, DISABLED_MigrateToDynamicLevelMaxBytesBase
) {
410 const int kMaxKey
= 2000;
413 options
.create_if_missing
= true;
414 options
.write_buffer_size
= 2048;
415 options
.max_write_buffer_number
= 8;
416 options
.level0_file_num_compaction_trigger
= 4;
417 options
.level0_slowdown_writes_trigger
= 4;
418 options
.level0_stop_writes_trigger
= 8;
419 options
.target_file_size_base
= 2048;
420 options
.level_compaction_dynamic_level_bytes
= false;
421 options
.max_bytes_for_level_base
= 10240;
422 options
.max_bytes_for_level_multiplier
= 4;
423 options
.soft_rate_limit
= 1.1;
424 options
.num_levels
= 8;
426 DestroyAndReopen(options
);
428 auto verify_func
= [&](int num_keys
, bool if_sleep
) {
429 for (int i
= 0; i
< num_keys
; i
++) {
430 ASSERT_NE("NOT_FOUND", Get(Key(kMaxKey
+ i
)));
431 if (i
< num_keys
/ 10) {
432 ASSERT_EQ("NOT_FOUND", Get(Key(i
)));
434 ASSERT_NE("NOT_FOUND", Get(Key(i
)));
436 if (if_sleep
&& i
% 1000 == 0) {
437 // Without it, valgrind may choose not to give another
438 // thread a chance to run before finishing the function,
439 // causing the test to be extremely slow.
440 env_
->SleepForMicroseconds(1);
445 int total_keys
= 1000;
446 for (int i
= 0; i
< total_keys
; i
++) {
447 ASSERT_OK(Put(Key(i
), rnd
.RandomString(102)));
448 ASSERT_OK(Put(Key(kMaxKey
+ i
), rnd
.RandomString(102)));
449 ASSERT_OK(Delete(Key(i
/ 10)));
451 verify_func(total_keys
, false);
452 dbfull()->TEST_WaitForCompact();
454 options
.level_compaction_dynamic_level_bytes
= true;
455 options
.disable_auto_compactions
= true;
457 verify_func(total_keys
, false);
459 std::atomic_bool compaction_finished
;
460 compaction_finished
= false;
461 // Issue manual compaction in one thread and still verify DB state
463 ROCKSDB_NAMESPACE::port::Thread
t([&]() {
464 CompactRangeOptions compact_options
;
465 compact_options
.change_level
= true;
466 compact_options
.target_level
= options
.num_levels
- 1;
467 dbfull()->CompactRange(compact_options
, nullptr, nullptr);
468 compaction_finished
.store(true);
471 verify_func(total_keys
, true);
472 } while (!compaction_finished
.load());
475 ASSERT_OK(dbfull()->SetOptions({
476 {"disable_auto_compactions", "false"},
479 int total_keys2
= 2000;
480 for (int i
= total_keys
; i
< total_keys2
; i
++) {
481 ASSERT_OK(Put(Key(i
), rnd
.RandomString(102)));
482 ASSERT_OK(Put(Key(kMaxKey
+ i
), rnd
.RandomString(102)));
483 ASSERT_OK(Delete(Key(i
/ 10)));
486 verify_func(total_keys2
, false);
487 dbfull()->TEST_WaitForCompact();
488 verify_func(total_keys2
, false);
490 // Base level is not level 1
491 ASSERT_EQ(NumTableFilesAtLevel(1), 0);
492 ASSERT_EQ(NumTableFilesAtLevel(2), 0);
494 } // namespace ROCKSDB_NAMESPACE
496 #endif // !defined(ROCKSDB_LITE)
498 int main(int argc
, char** argv
) {
499 #if !defined(ROCKSDB_LITE)
500 ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
501 ::testing::InitGoogleTest(&argc
, argv
);
502 return RUN_ALL_TESTS();