1 // Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
2 // This source code is licensed under both the GPLv2 (found in the
3 // COPYING file in the root directory) and Apache 2.0 License
4 // (found in the LICENSE.Apache file in the root directory).
6 // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
7 // Use of this source code is governed by a BSD-style license that can be
8 // found in the LICENSE file. See the AUTHORS file for names of contributors.
10 #include "rocksdb/utilities/option_change_migration.h"
14 #include "db/db_test_util.h"
15 #include "port/stack_trace.h"
16 #include "util/random.h"
18 namespace ROCKSDB_NAMESPACE
{
20 class DBOptionChangeMigrationTests
22 public testing::WithParamInterface
<
23 std::tuple
<int, int, bool, int, int, bool, uint64_t>> {
25 DBOptionChangeMigrationTests()
26 : DBTestBase("db_option_change_migration_test", /*env_do_fsync=*/true) {
27 level1_
= std::get
<0>(GetParam());
28 compaction_style1_
= std::get
<1>(GetParam());
29 is_dynamic1_
= std::get
<2>(GetParam());
31 level2_
= std::get
<3>(GetParam());
32 compaction_style2_
= std::get
<4>(GetParam());
33 is_dynamic2_
= std::get
<5>(GetParam());
34 fifo_max_table_files_size_
= std::get
<6>(GetParam());
37 // Required if inheriting from testing::WithParamInterface<>
38 static void SetUpTestCase() {}
39 static void TearDownTestCase() {}
42 int compaction_style1_
;
46 int compaction_style2_
;
49 uint64_t fifo_max_table_files_size_
;
53 TEST_P(DBOptionChangeMigrationTests
, Migrate1
) {
54 Options old_options
= CurrentOptions();
55 old_options
.compaction_style
=
56 static_cast<CompactionStyle
>(compaction_style1_
);
57 if (old_options
.compaction_style
== CompactionStyle::kCompactionStyleLevel
) {
58 old_options
.level_compaction_dynamic_level_bytes
= is_dynamic1_
;
60 if (old_options
.compaction_style
== CompactionStyle::kCompactionStyleFIFO
) {
61 old_options
.max_open_files
= -1;
63 old_options
.level0_file_num_compaction_trigger
= 3;
64 old_options
.write_buffer_size
= 64 * 1024;
65 old_options
.target_file_size_base
= 128 * 1024;
66 // Make level target of L1, L2 to be 200KB and 600KB
67 old_options
.num_levels
= level1_
;
68 old_options
.max_bytes_for_level_multiplier
= 3;
69 old_options
.max_bytes_for_level_base
= 200 * 1024;
76 // Generate at least 2MB of data
77 for (int num
= 0; num
< 20; num
++) {
78 GenerateNewFile(&rnd
, &key_idx
);
80 ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable());
81 ASSERT_OK(dbfull()->TEST_WaitForCompact());
83 // Will make sure exactly those keys are in the DB after migration.
84 std::set
<std::string
> keys
;
86 std::unique_ptr
<Iterator
> it(db_
->NewIterator(ReadOptions()));
88 for (; it
->Valid(); it
->Next()) {
89 keys
.insert(it
->key().ToString());
94 Options new_options
= old_options
;
95 new_options
.compaction_style
=
96 static_cast<CompactionStyle
>(compaction_style2_
);
97 if (new_options
.compaction_style
== CompactionStyle::kCompactionStyleLevel
) {
98 new_options
.level_compaction_dynamic_level_bytes
= is_dynamic2_
;
100 if (new_options
.compaction_style
== CompactionStyle::kCompactionStyleFIFO
) {
101 new_options
.max_open_files
= -1;
103 if (fifo_max_table_files_size_
!= 0) {
104 new_options
.compaction_options_fifo
.max_table_files_size
=
105 fifo_max_table_files_size_
;
107 new_options
.target_file_size_base
= 256 * 1024;
108 new_options
.num_levels
= level2_
;
109 new_options
.max_bytes_for_level_base
= 150 * 1024;
110 new_options
.max_bytes_for_level_multiplier
= 4;
111 ASSERT_OK(OptionChangeMigration(dbname_
, old_options
, new_options
));
114 // Wait for compaction to finish and make sure it can reopen
115 ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable());
116 ASSERT_OK(dbfull()->TEST_WaitForCompact());
120 std::unique_ptr
<Iterator
> it(db_
->NewIterator(ReadOptions()));
122 for (std::string key
: keys
) {
123 ASSERT_TRUE(it
->Valid());
124 ASSERT_EQ(key
, it
->key().ToString());
127 ASSERT_TRUE(!it
->Valid());
131 TEST_P(DBOptionChangeMigrationTests
, Migrate2
) {
132 Options old_options
= CurrentOptions();
133 old_options
.compaction_style
=
134 static_cast<CompactionStyle
>(compaction_style2_
);
135 if (old_options
.compaction_style
== CompactionStyle::kCompactionStyleLevel
) {
136 old_options
.level_compaction_dynamic_level_bytes
= is_dynamic2_
;
138 if (old_options
.compaction_style
== CompactionStyle::kCompactionStyleFIFO
) {
139 old_options
.max_open_files
= -1;
141 old_options
.level0_file_num_compaction_trigger
= 3;
142 old_options
.write_buffer_size
= 64 * 1024;
143 old_options
.target_file_size_base
= 128 * 1024;
144 // Make level target of L1, L2 to be 200KB and 600KB
145 old_options
.num_levels
= level2_
;
146 old_options
.max_bytes_for_level_multiplier
= 3;
147 old_options
.max_bytes_for_level_base
= 200 * 1024;
154 // Generate at least 2MB of data
155 for (int num
= 0; num
< 20; num
++) {
156 GenerateNewFile(&rnd
, &key_idx
);
158 ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable());
159 ASSERT_OK(dbfull()->TEST_WaitForCompact());
161 // Will make sure exactly those keys are in the DB after migration.
162 std::set
<std::string
> keys
;
164 std::unique_ptr
<Iterator
> it(db_
->NewIterator(ReadOptions()));
166 for (; it
->Valid(); it
->Next()) {
167 keys
.insert(it
->key().ToString());
173 Options new_options
= old_options
;
174 new_options
.compaction_style
=
175 static_cast<CompactionStyle
>(compaction_style1_
);
176 if (new_options
.compaction_style
== CompactionStyle::kCompactionStyleLevel
) {
177 new_options
.level_compaction_dynamic_level_bytes
= is_dynamic1_
;
179 if (new_options
.compaction_style
== CompactionStyle::kCompactionStyleFIFO
) {
180 new_options
.max_open_files
= -1;
182 if (fifo_max_table_files_size_
!= 0) {
183 new_options
.compaction_options_fifo
.max_table_files_size
=
184 fifo_max_table_files_size_
;
186 new_options
.target_file_size_base
= 256 * 1024;
187 new_options
.num_levels
= level1_
;
188 new_options
.max_bytes_for_level_base
= 150 * 1024;
189 new_options
.max_bytes_for_level_multiplier
= 4;
190 ASSERT_OK(OptionChangeMigration(dbname_
, old_options
, new_options
));
192 // Wait for compaction to finish and make sure it can reopen
193 ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable());
194 ASSERT_OK(dbfull()->TEST_WaitForCompact());
198 std::unique_ptr
<Iterator
> it(db_
->NewIterator(ReadOptions()));
200 for (std::string key
: keys
) {
201 ASSERT_TRUE(it
->Valid());
202 ASSERT_EQ(key
, it
->key().ToString());
205 ASSERT_TRUE(!it
->Valid());
209 TEST_P(DBOptionChangeMigrationTests
, Migrate3
) {
210 Options old_options
= CurrentOptions();
211 old_options
.compaction_style
=
212 static_cast<CompactionStyle
>(compaction_style1_
);
213 if (old_options
.compaction_style
== CompactionStyle::kCompactionStyleLevel
) {
214 old_options
.level_compaction_dynamic_level_bytes
= is_dynamic1_
;
216 if (old_options
.compaction_style
== CompactionStyle::kCompactionStyleFIFO
) {
217 old_options
.max_open_files
= -1;
219 old_options
.level0_file_num_compaction_trigger
= 3;
220 old_options
.write_buffer_size
= 64 * 1024;
221 old_options
.target_file_size_base
= 128 * 1024;
222 // Make level target of L1, L2 to be 200KB and 600KB
223 old_options
.num_levels
= level1_
;
224 old_options
.max_bytes_for_level_multiplier
= 3;
225 old_options
.max_bytes_for_level_base
= 200 * 1024;
229 for (int num
= 0; num
< 20; num
++) {
230 for (int i
= 0; i
< 50; i
++) {
231 ASSERT_OK(Put(Key(num
* 100 + i
), rnd
.RandomString(900)));
234 ASSERT_OK(dbfull()->TEST_WaitForCompact());
236 // Issue a full compaction to generate some zero-out files
237 CompactRangeOptions cro
;
238 cro
.bottommost_level_compaction
= BottommostLevelCompaction::kForce
;
239 ASSERT_OK(dbfull()->CompactRange(cro
, nullptr, nullptr));
242 ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable());
243 ASSERT_OK(dbfull()->TEST_WaitForCompact());
245 // Will make sure exactly those keys are in the DB after migration.
246 std::set
<std::string
> keys
;
248 std::unique_ptr
<Iterator
> it(db_
->NewIterator(ReadOptions()));
250 for (; it
->Valid(); it
->Next()) {
251 keys
.insert(it
->key().ToString());
256 Options new_options
= old_options
;
257 new_options
.compaction_style
=
258 static_cast<CompactionStyle
>(compaction_style2_
);
259 if (new_options
.compaction_style
== CompactionStyle::kCompactionStyleLevel
) {
260 new_options
.level_compaction_dynamic_level_bytes
= is_dynamic2_
;
262 if (new_options
.compaction_style
== CompactionStyle::kCompactionStyleFIFO
) {
263 new_options
.max_open_files
= -1;
265 if (fifo_max_table_files_size_
!= 0) {
266 new_options
.compaction_options_fifo
.max_table_files_size
=
267 fifo_max_table_files_size_
;
269 new_options
.target_file_size_base
= 256 * 1024;
270 new_options
.num_levels
= level2_
;
271 new_options
.max_bytes_for_level_base
= 150 * 1024;
272 new_options
.max_bytes_for_level_multiplier
= 4;
273 ASSERT_OK(OptionChangeMigration(dbname_
, old_options
, new_options
));
276 // Wait for compaction to finish and make sure it can reopen
277 ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable());
278 ASSERT_OK(dbfull()->TEST_WaitForCompact());
282 std::unique_ptr
<Iterator
> it(db_
->NewIterator(ReadOptions()));
284 for (std::string key
: keys
) {
285 ASSERT_TRUE(it
->Valid());
286 ASSERT_EQ(key
, it
->key().ToString());
289 ASSERT_TRUE(!it
->Valid());
293 TEST_P(DBOptionChangeMigrationTests
, Migrate4
) {
294 Options old_options
= CurrentOptions();
295 old_options
.compaction_style
=
296 static_cast<CompactionStyle
>(compaction_style2_
);
297 if (old_options
.compaction_style
== CompactionStyle::kCompactionStyleLevel
) {
298 old_options
.level_compaction_dynamic_level_bytes
= is_dynamic2_
;
300 if (old_options
.compaction_style
== CompactionStyle::kCompactionStyleFIFO
) {
301 old_options
.max_open_files
= -1;
303 old_options
.level0_file_num_compaction_trigger
= 3;
304 old_options
.write_buffer_size
= 64 * 1024;
305 old_options
.target_file_size_base
= 128 * 1024;
306 // Make level target of L1, L2 to be 200KB and 600KB
307 old_options
.num_levels
= level2_
;
308 old_options
.max_bytes_for_level_multiplier
= 3;
309 old_options
.max_bytes_for_level_base
= 200 * 1024;
313 for (int num
= 0; num
< 20; num
++) {
314 for (int i
= 0; i
< 50; i
++) {
315 ASSERT_OK(Put(Key(num
* 100 + i
), rnd
.RandomString(900)));
318 ASSERT_OK(dbfull()->TEST_WaitForCompact());
320 // Issue a full compaction to generate some zero-out files
321 CompactRangeOptions cro
;
322 cro
.bottommost_level_compaction
= BottommostLevelCompaction::kForce
;
323 ASSERT_OK(dbfull()->CompactRange(cro
, nullptr, nullptr));
326 ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable());
327 ASSERT_OK(dbfull()->TEST_WaitForCompact());
329 // Will make sure exactly those keys are in the DB after migration.
330 std::set
<std::string
> keys
;
332 std::unique_ptr
<Iterator
> it(db_
->NewIterator(ReadOptions()));
334 for (; it
->Valid(); it
->Next()) {
335 keys
.insert(it
->key().ToString());
341 Options new_options
= old_options
;
342 new_options
.compaction_style
=
343 static_cast<CompactionStyle
>(compaction_style1_
);
344 if (new_options
.compaction_style
== CompactionStyle::kCompactionStyleLevel
) {
345 new_options
.level_compaction_dynamic_level_bytes
= is_dynamic1_
;
347 if (new_options
.compaction_style
== CompactionStyle::kCompactionStyleFIFO
) {
348 new_options
.max_open_files
= -1;
350 if (fifo_max_table_files_size_
!= 0) {
351 new_options
.compaction_options_fifo
.max_table_files_size
=
352 fifo_max_table_files_size_
;
354 new_options
.target_file_size_base
= 256 * 1024;
355 new_options
.num_levels
= level1_
;
356 new_options
.max_bytes_for_level_base
= 150 * 1024;
357 new_options
.max_bytes_for_level_multiplier
= 4;
358 ASSERT_OK(OptionChangeMigration(dbname_
, old_options
, new_options
));
360 // Wait for compaction to finish and make sure it can reopen
361 ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable());
362 ASSERT_OK(dbfull()->TEST_WaitForCompact());
366 std::unique_ptr
<Iterator
> it(db_
->NewIterator(ReadOptions()));
368 for (std::string key
: keys
) {
369 ASSERT_TRUE(it
->Valid());
370 ASSERT_EQ(key
, it
->key().ToString());
373 ASSERT_TRUE(!it
->Valid());
377 INSTANTIATE_TEST_CASE_P(
378 DBOptionChangeMigrationTests
, DBOptionChangeMigrationTests
,
380 std::make_tuple(3 /* old num_levels */, 0 /* old compaction style */,
381 false /* is dynamic leveling in old option */,
382 4 /* old num_levels */, 0 /* new compaction style */,
383 false /* is dynamic leveling in new option */,
384 0 /*fifo max_table_files_size*/),
385 std::make_tuple(3 /* old num_levels */, 0 /* old compaction style */,
386 true /* is dynamic leveling in old option */,
387 4 /* old num_levels */, 0 /* new compaction style */,
388 true /* is dynamic leveling in new option */,
389 0 /*fifo max_table_files_size*/),
390 std::make_tuple(3 /* old num_levels */, 0 /* old compaction style */,
391 true /* is dynamic leveling in old option */,
392 4 /* old num_levels */, 0 /* new compaction style */,
393 false, 0 /*fifo max_table_files_size*/),
394 std::make_tuple(3 /* old num_levels */, 0 /* old compaction style */,
395 false /* is dynamic leveling in old option */,
396 4 /* old num_levels */, 0 /* new compaction style */,
397 true /* is dynamic leveling in new option */,
398 0 /*fifo max_table_files_size*/),
399 std::make_tuple(3 /* old num_levels */, 1 /* old compaction style */,
400 false /* is dynamic leveling in old option */,
401 4 /* old num_levels */, 1 /* new compaction style */,
402 false /* is dynamic leveling in new option */,
403 0 /*fifo max_table_files_size*/),
404 std::make_tuple(1 /* old num_levels */, 1 /* old compaction style */,
405 false /* is dynamic leveling in old option */,
406 4 /* old num_levels */, 1 /* new compaction style */,
407 false /* is dynamic leveling in new option */,
408 0 /*fifo max_table_files_size*/),
409 std::make_tuple(3 /* old num_levels */, 0 /* old compaction style */,
410 false /* is dynamic leveling in old option */,
411 4 /* old num_levels */, 1 /* new compaction style */,
412 false /* is dynamic leveling in new option */,
413 0 /*fifo max_table_files_size*/),
414 std::make_tuple(3 /* old num_levels */, 0 /* old compaction style */,
415 false /* is dynamic leveling in old option */,
416 1 /* old num_levels */, 1 /* new compaction style */,
417 false /* is dynamic leveling in new option */,
418 0 /*fifo max_table_files_size*/),
419 std::make_tuple(3 /* old num_levels */, 0 /* old compaction style */,
420 true /* is dynamic leveling in old option */,
421 4 /* old num_levels */, 1 /* new compaction style */,
422 false /* is dynamic leveling in new option */,
423 0 /*fifo max_table_files_size*/),
424 std::make_tuple(3 /* old num_levels */, 0 /* old compaction style */,
425 true /* is dynamic leveling in old option */,
426 1 /* old num_levels */, 1 /* new compaction style */,
427 false /* is dynamic leveling in new option */,
428 0 /*fifo max_table_files_size*/),
429 std::make_tuple(1 /* old num_levels */, 1 /* old compaction style */,
430 false /* is dynamic leveling in old option */,
431 4 /* old num_levels */, 0 /* new compaction style */,
432 false /* is dynamic leveling in new option */,
433 0 /*fifo max_table_files_size*/),
434 std::make_tuple(4 /* old num_levels */, 0 /* old compaction style */,
435 false /* is dynamic leveling in old option */,
436 1 /* old num_levels */, 2 /* new compaction style */,
437 false /* is dynamic leveling in new option */,
438 0 /*fifo max_table_files_size*/),
439 std::make_tuple(3 /* old num_levels */, 0 /* old compaction style */,
440 true /* is dynamic leveling in old option */,
441 2 /* old num_levels */, 2 /* new compaction style */,
442 false /* is dynamic leveling in new option */,
443 0 /*fifo max_table_files_size*/),
444 std::make_tuple(3 /* old num_levels */, 1 /* old compaction style */,
445 false /* is dynamic leveling in old option */,
446 3 /* old num_levels */, 2 /* new compaction style */,
447 false /* is dynamic leveling in new option */,
448 0 /*fifo max_table_files_size*/),
449 std::make_tuple(1 /* old num_levels */, 1 /* old compaction style */,
450 false /* is dynamic leveling in old option */,
451 4 /* old num_levels */, 2 /* new compaction style */,
452 false /* is dynamic leveling in new option */, 0),
453 std::make_tuple(4 /* old num_levels */, 0 /* old compaction style */,
454 false /* is dynamic leveling in old option */,
455 1 /* old num_levels */, 2 /* new compaction style */,
456 false /* is dynamic leveling in new option */,
457 5 * 1024 * 1024 /*fifo max_table_files_size*/),
458 std::make_tuple(3 /* old num_levels */, 0 /* old compaction style */,
459 true /* is dynamic leveling in old option */,
460 2 /* old num_levels */, 2 /* new compaction style */,
461 false /* is dynamic leveling in new option */,
462 5 * 1024 * 1024 /*fifo max_table_files_size*/),
463 std::make_tuple(3 /* old num_levels */, 1 /* old compaction style */,
464 false /* is dynamic leveling in old option */,
465 3 /* old num_levels */, 2 /* new compaction style */,
466 false /* is dynamic leveling in new option */,
467 5 * 1024 * 1024 /*fifo max_table_files_size*/),
468 std::make_tuple(1 /* old num_levels */, 1 /* old compaction style */,
469 false /* is dynamic leveling in old option */,
470 4 /* old num_levels */, 2 /* new compaction style */,
471 false /* is dynamic leveling in new option */,
472 5 * 1024 * 1024 /*fifo max_table_files_size*/)));
474 class DBOptionChangeMigrationTest
: public DBTestBase
{
476 DBOptionChangeMigrationTest()
477 : DBTestBase("db_option_change_migration_test2", /*env_do_fsync=*/true) {}
480 TEST_F(DBOptionChangeMigrationTest
, CompactedSrcToUniversal
) {
481 Options old_options
= CurrentOptions();
482 old_options
.compaction_style
= CompactionStyle::kCompactionStyleLevel
;
483 old_options
.max_compaction_bytes
= 200 * 1024;
484 old_options
.level_compaction_dynamic_level_bytes
= false;
485 old_options
.level0_file_num_compaction_trigger
= 3;
486 old_options
.write_buffer_size
= 64 * 1024;
487 old_options
.target_file_size_base
= 128 * 1024;
488 // Make level target of L1, L2 to be 200KB and 600KB
489 old_options
.num_levels
= 4;
490 old_options
.max_bytes_for_level_multiplier
= 3;
491 old_options
.max_bytes_for_level_base
= 200 * 1024;
495 for (int num
= 0; num
< 20; num
++) {
496 for (int i
= 0; i
< 50; i
++) {
497 ASSERT_OK(Put(Key(num
* 100 + i
), rnd
.RandomString(900)));
501 CompactRangeOptions cro
;
502 cro
.bottommost_level_compaction
= BottommostLevelCompaction::kForce
;
503 ASSERT_OK(dbfull()->CompactRange(cro
, nullptr, nullptr));
505 // Will make sure exactly those keys are in the DB after migration.
506 std::set
<std::string
> keys
;
508 std::unique_ptr
<Iterator
> it(db_
->NewIterator(ReadOptions()));
510 for (; it
->Valid(); it
->Next()) {
511 keys
.insert(it
->key().ToString());
517 Options new_options
= old_options
;
518 new_options
.compaction_style
= CompactionStyle::kCompactionStyleUniversal
;
519 new_options
.target_file_size_base
= 256 * 1024;
520 new_options
.num_levels
= 1;
521 new_options
.max_bytes_for_level_base
= 150 * 1024;
522 new_options
.max_bytes_for_level_multiplier
= 4;
523 ASSERT_OK(OptionChangeMigration(dbname_
, old_options
, new_options
));
525 // Wait for compaction to finish and make sure it can reopen
526 ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable());
527 ASSERT_OK(dbfull()->TEST_WaitForCompact());
531 std::unique_ptr
<Iterator
> it(db_
->NewIterator(ReadOptions()));
533 for (std::string key
: keys
) {
534 ASSERT_TRUE(it
->Valid());
535 ASSERT_EQ(key
, it
->key().ToString());
538 ASSERT_TRUE(!it
->Valid());
539 ASSERT_OK(it
->status());
543 #endif // ROCKSDB_LITE
544 } // namespace ROCKSDB_NAMESPACE
546 int main(int argc
, char** argv
) {
547 ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
548 ::testing::InitGoogleTest(&argc
, argv
);
549 return RUN_ALL_TESTS();