#if !defined(ROCKSDB_LITE)
#include "rocksdb/utilities/table_properties_collectors.h"
#include "test_util/sync_point.h"
+#include "test_util/testutil.h"
#include "util/random.h"
namespace ROCKSDB_NAMESPACE {
class DBTestUniversalCompaction : public DBTestUniversalCompactionBase {
public:
- DBTestUniversalCompaction() :
- DBTestUniversalCompactionBase("/db_universal_compaction_test") {}
+ DBTestUniversalCompaction()
+ : DBTestUniversalCompactionBase("/db_universal_compaction_test") {}
};
class DBTestUniversalCompaction2 : public DBTestBase {
public:
DBTestUniversalCompaction2()
- : DBTestBase("/db_universal_compaction_test2", /*env_do_fsync=*/false) {}
+ : DBTestBase("db_universal_compaction_test2", /*env_do_fsync=*/false) {}
};
namespace {
std::atomic_bool expect_full_compaction_;
std::atomic_bool expect_manual_compaction_;
};
-} // namespace
+} // anonymous namespace
// Make sure we don't trigger a problem if the trigger condtion is given
// to be 0, which is invalid.
options.table_factory.reset(NewBlockBasedTableFactory(bbto));
options.optimize_filters_for_hits = true;
options.statistics = ROCKSDB_NAMESPACE::CreateDBStatistics();
- options.memtable_factory.reset(new SpecialSkipListFactory(3));
+ options.memtable_factory.reset(test::NewSpecialSkipListFactory(3));
DestroyAndReopen(options);
ASSERT_EQ(options.compaction_style, kCompactionStyleUniversal);
Random rnd(301);
for (int key = 1024 * kEntriesPerBuffer; key >= 0; --key) {
- ASSERT_OK(Put(1, ToString(key), rnd.RandomString(kTestValueSize)));
+ ASSERT_OK(Put(1, std::to_string(key), rnd.RandomString(kTestValueSize)));
}
ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable(handles_[1]));
ASSERT_OK(dbfull()->TEST_WaitForCompact());
}
if (compaction_input_file_names.size() == 0) {
- compaction_input_file_names.push_back(
- cf_meta.levels[0].files[0].name);
+ compaction_input_file_names.push_back(cf_meta.levels[0].files[0].name);
}
// expect fail since universal compaction only allow L0 output
.ok());
// expect ok and verify the compacted files no longer exist.
- ASSERT_OK(dbfull()->CompactFiles(
- CompactionOptions(), handles_[1],
- compaction_input_file_names, 0));
+ ASSERT_OK(dbfull()->CompactFiles(CompactionOptions(), handles_[1],
+ compaction_input_file_names, 0));
dbfull()->GetColumnFamilyMetaData(handles_[1], &cf_meta);
VerifyCompactionResult(
- cf_meta,
- std::set<std::string>(compaction_input_file_names.begin(),
- compaction_input_file_names.end()));
+ cf_meta, std::set<std::string>(compaction_input_file_names.begin(),
+ compaction_input_file_names.end()));
compaction_input_file_names.clear();
// Pick the first and the last file, expect everything is
// compacted into one single file.
+ compaction_input_file_names.push_back(cf_meta.levels[0].files[0].name);
compaction_input_file_names.push_back(
- cf_meta.levels[0].files[0].name);
- compaction_input_file_names.push_back(
- cf_meta.levels[0].files[
- cf_meta.levels[0].files.size() - 1].name);
- ASSERT_OK(dbfull()->CompactFiles(
- CompactionOptions(), handles_[1],
- compaction_input_file_names, 0));
+ cf_meta.levels[0].files[cf_meta.levels[0].files.size() - 1].name);
+ ASSERT_OK(dbfull()->CompactFiles(CompactionOptions(), handles_[1],
+ compaction_input_file_names, 0));
dbfull()->GetColumnFamilyMetaData(handles_[1], &cf_meta);
ASSERT_EQ(cf_meta.levels[0].files.size(), 1U);
TEST_P(DBTestUniversalCompaction, UniversalCompactionTargetLevel) {
Options options = CurrentOptions();
options.compaction_style = kCompactionStyleUniversal;
- options.write_buffer_size = 100 << 10; // 100KB
+ options.write_buffer_size = 100 << 10; // 100KB
options.num_levels = 7;
options.disable_auto_compactions = true;
DestroyAndReopen(options);
ASSERT_EQ("0,0,0,0,1", FilesPerLevel(0));
}
-#ifndef ROCKSDB_VALGRIND_RUN
+#if !defined(ROCKSDB_VALGRIND_RUN) || defined(ROCKSDB_FULL_VALGRIND_RUN)
class DBTestUniversalCompactionMultiLevels
: public DBTestUniversalCompactionBase {
public:
- DBTestUniversalCompactionMultiLevels() :
- DBTestUniversalCompactionBase(
- "/db_universal_compaction_multi_levels_test") {}
+ DBTestUniversalCompactionMultiLevels()
+ : DBTestUniversalCompactionBase(
+ "/db_universal_compaction_multi_levels_test") {}
};
TEST_P(DBTestUniversalCompactionMultiLevels, UniversalCompactionMultiLevels) {
::testing::Combine(::testing::Values(3, 20),
::testing::Bool()));
-class DBTestUniversalCompactionParallel :
- public DBTestUniversalCompactionBase {
+class DBTestUniversalCompactionParallel : public DBTestUniversalCompactionBase {
public:
- DBTestUniversalCompactionParallel() :
- DBTestUniversalCompactionBase(
- "/db_universal_compaction_prallel_test") {}
+ DBTestUniversalCompactionParallel()
+ : DBTestUniversalCompactionBase("/db_universal_compaction_prallel_test") {
+ }
};
TEST_P(DBTestUniversalCompactionParallel, UniversalCompactionParallel) {
Options options = CurrentOptions();
options.compaction_style = kCompactionStyleUniversal;
options.num_levels = num_levels_;
+ options.env = env_;
options.write_buffer_size = 1 << 10; // 1KB
options.level0_file_num_compaction_trigger = 3;
options.max_background_compactions = 3;
INSTANTIATE_TEST_CASE_P(Parallel, DBTestUniversalCompactionParallel,
::testing::Combine(::testing::Values(1, 10),
::testing::Values(false)));
-#endif // ROCKSDB_VALGRIND_RUN
+#endif // !defined(ROCKSDB_VALGRIND_RUN) || defined(ROCKSDB_FULL_VALGRIND_RUN)
TEST_P(DBTestUniversalCompaction, UniversalCompactionOptions) {
Options options = CurrentOptions();
options.compaction_style = kCompactionStyleUniversal;
- options.write_buffer_size = 105 << 10; // 105KB
- options.arena_block_size = 4 << 10; // 4KB
+ options.write_buffer_size = 105 << 10; // 105KB
+ options.arena_block_size = 4 << 10; // 4KB
options.target_file_size_base = 32 << 10; // 32KB
options.level0_file_num_compaction_trigger = 4;
options.num_levels = num_levels_;
TEST_P(DBTestUniversalCompaction, UniversalCompactionStopStyleSimilarSize) {
Options options = CurrentOptions();
options.compaction_style = kCompactionStyleUniversal;
- options.write_buffer_size = 105 << 10; // 105KB
- options.arena_block_size = 4 << 10; // 4KB
+ options.write_buffer_size = 105 << 10; // 105KB
+ options.arena_block_size = 4 << 10; // 4KB
options.target_file_size_base = 32 << 10; // 32KB
// trigger compaction if there are >= 4 files
options.level0_file_num_compaction_trigger = 4;
// First, clean up memtable before inserting new data. This will generate
// a level-0 file, with size around 0.4 (according to previously written
// data amount).
- dbfull()->Flush(FlushOptions());
+ ASSERT_OK(dbfull()->Flush(FlushOptions()));
for (int num = 0; num < options.level0_file_num_compaction_trigger - 3;
num++) {
// Write 110KB (11 values, each 10K)
ASSERT_LT(TotalSize(), 120000U * 12 * 0.82 + 120000 * 2);
}
-#ifndef ROCKSDB_VALGRIND_RUN
+#if !defined(ROCKSDB_VALGRIND_RUN) || defined(ROCKSDB_FULL_VALGRIND_RUN)
// Test that checks trivial move in universal compaction
TEST_P(DBTestUniversalCompaction, UniversalCompactionTrivialMoveTest1) {
int32_t trivial_move = 0;
ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing();
}
-#endif // ROCKSDB_VALGRIND_RUN
+#endif // !defined(ROCKSDB_VALGRIND_RUN) || defined(ROCKSDB_FULL_VALGRIND_RUN)
TEST_P(DBTestUniversalCompaction, UniversalCompactionFourPaths) {
Options options = CurrentOptions();
options.db_paths.emplace_back(dbname_ + "_3", 500 * 1024);
options.db_paths.emplace_back(dbname_ + "_4", 1024 * 1024 * 1024);
options.memtable_factory.reset(
- new SpecialSkipListFactory(KNumKeysByGenerateNewFile - 1));
+ test::NewSpecialSkipListFactory(KNumKeysByGenerateNewFile - 1));
options.compaction_style = kCompactionStyleUniversal;
options.compaction_options_universal.size_ratio = 5;
options.write_buffer_size = 111 << 10; // 114KB
options.db_paths.emplace_back(dbname_ + "_3", 500 * 1024);
options.db_paths.emplace_back(dbname_ + "_4", 1024 * 1024 * 1024);
options.memtable_factory.reset(
- new SpecialSkipListFactory(KNumKeysByGenerateNewFile - 1));
+ test::NewSpecialSkipListFactory(KNumKeysByGenerateNewFile - 1));
options.compaction_style = kCompactionStyleUniversal;
options.compaction_options_universal.size_ratio = 10;
options.write_buffer_size = 111 << 10; // 114KB
cf_opt1.cf_paths.emplace_back(dbname_ + "cf1_3", 500 * 1024);
cf_opt1.cf_paths.emplace_back(dbname_ + "cf1_4", 1024 * 1024 * 1024);
option_vector.emplace_back(DBOptions(options), cf_opt1);
- CreateColumnFamilies({"one"},option_vector[1]);
+ CreateColumnFamilies({"one"}, option_vector[1]);
// Configura CF2 specific paths.
cf_opt2.cf_paths.emplace_back(dbname_ + "cf2", 300 * 1024);
cf_opt2.cf_paths.emplace_back(dbname_ + "cf2_3", 500 * 1024);
cf_opt2.cf_paths.emplace_back(dbname_ + "cf2_4", 1024 * 1024 * 1024);
option_vector.emplace_back(DBOptions(options), cf_opt2);
- CreateColumnFamilies({"two"},option_vector[2]);
+ CreateColumnFamilies({"two"}, option_vector[2]);
ReopenWithColumnFamilies({"default", "one", "two"}, option_vector);
options.num_levels = 1;
options.write_buffer_size = 200 << 10; // 200KB
options.level0_file_num_compaction_trigger = 3;
- options.memtable_factory.reset(new SpecialSkipListFactory(KNumKeysPerFile));
+ options.memtable_factory.reset(
+ test::NewSpecialSkipListFactory(KNumKeysPerFile));
options = CurrentOptions(options);
CreateAndReopenWithCF({"pikachu"}, options);
verify_func(max_key3);
}
-
TEST_P(DBTestUniversalCompaction, UniversalCompactionSecondPathRatio) {
if (!Snappy_Supported()) {
return;
options.level0_file_num_compaction_trigger = 2;
options.num_levels = 1;
options.memtable_factory.reset(
- new SpecialSkipListFactory(KNumKeysByGenerateNewFile - 1));
+ test::NewSpecialSkipListFactory(KNumKeysByGenerateNewFile - 1));
std::vector<std::string> filenames;
if (env_->GetChildren(options.db_paths[1].path, &filenames).ok()) {
Env::Default()->SetBackgroundThreads(1, Env::Priority::BOTTOM);
Options options = CurrentOptions();
options.compaction_style = kCompactionStyleUniversal;
+ options.max_background_compactions = 2;
options.num_levels = num_levels_;
options.write_buffer_size = 100 << 10; // 100KB
options.target_file_size_base = 32 << 10; // 32KB
options.compaction_options_universal.max_size_amplification_percent = 110;
DestroyAndReopen(options);
+ // Need to get a token to enable compaction parallelism up to
+ // `max_background_compactions` jobs.
+ auto pressure_token =
+ dbfull()->TEST_write_controler().GetCompactionPressureToken();
ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency(
{// wait for the full compaction to be picked before adding files intended
// for the second one.
const int kNumFilesTrigger = 8;
Options options = CurrentOptions();
options.memtable_factory.reset(
- new SpecialSkipListFactory(KNumKeysByGenerateNewFile - 1));
+ test::NewSpecialSkipListFactory(KNumKeysByGenerateNewFile - 1));
options.compaction_options_universal.max_merge_width = kNumFilesTrigger / 2;
options.compaction_options_universal.max_size_amplification_percent =
static_cast<unsigned int>(-1);
dbfull()->TEST_write_controler().GetCompactionPressureToken();
ASSERT_OK(Put("key", "val"));
- Flush();
+ ASSERT_OK(Flush());
ASSERT_OK(dbfull()->CompactRange(CompactRangeOptions(), nullptr, nullptr));
ASSERT_EQ(NumTableFilesAtLevel(num_levels_ - 1), 1);
ColumnFamilyMetaData cf_meta;
"DBTestUniversalCompaction:FinalSortedRunCompactFilesConflict:0");
for (int i = 0; i < 2; ++i) {
ASSERT_OK(Put("key", "val"));
- Flush();
+ ASSERT_OK(Flush());
}
ASSERT_OK(dbfull()->TEST_WaitForCompact());
class DBTestUniversalManualCompactionOutputPathId
: public DBTestUniversalCompactionBase {
public:
- DBTestUniversalManualCompactionOutputPathId() :
- DBTestUniversalCompactionBase(
- "/db_universal_compaction_manual_pid_test") {}
+ DBTestUniversalManualCompactionOutputPathId()
+ : DBTestUniversalCompactionBase(
+ "/db_universal_compaction_manual_pid_test") {}
};
TEST_P(DBTestUniversalManualCompactionOutputPathId,
for (i = 0; i < 2000; ++i) {
ASSERT_OK(Put(Key(i), "val"));
}
- Flush();
+ ASSERT_OK(Flush());
// MoveFilesToLevel(6);
ASSERT_OK(dbfull()->CompactRange(CompactRangeOptions(), nullptr, nullptr));
ASSERT_OK(Put(Key(i), "val"));
}
}
- Flush();
+ ASSERT_OK(Flush());
ASSERT_OK(dbfull()->TEST_WaitForCompact());
ASSERT_EQ(0, NumTableFilesAtLevel(0));
for (i = 0; i < 2000; ++i) {
ASSERT_OK(Put(Key(i), "val"));
}
- Flush();
+ ASSERT_OK(Flush());
for (i = 1999; i < kNumKeys; ++i) {
if (i >= kNumKeys - kWindowSize &&
ASSERT_OK(Put(Key(i), "val"));
}
}
- Flush();
+ ASSERT_OK(Flush()(;
ASSERT_OK(dbfull()->TEST_WaitForCompact());
ASSERT_EQ(1, NumTableFilesAtLevel(0));
for (i = 0; i < 500; ++i) {
ASSERT_OK(Put(Key(i), "val"));
}
- Flush();
+ ASSERT_OK(Flush());
for (i = 500; i < 1000; ++i) {
ASSERT_OK(Put(Key(i), "val"));
}
- Flush();
+ ASSERT_OK(Flush());
for (i = 1000; i < 1500; ++i) {
ASSERT_OK(Put(Key(i), "val"));
}
- Flush();
+ ASSERT_OK(Flush());
for (i = 1500; i < 2000; ++i) {
ASSERT_OK(Put(Key(i), "val"));
}
- Flush();
+ ASSERT_OK(Flush());
ASSERT_OK(dbfull()->TEST_WaitForCompact());
ASSERT_EQ(0, NumTableFilesAtLevel(0));
for (i = 1999; i < 2333; ++i) {
ASSERT_OK(Put(Key(i), "val"));
}
- Flush();
+ ASSERT_OK(Flush());
for (i = 2333; i < 2666; ++i) {
ASSERT_OK(Put(Key(i), "val"));
}
- Flush();
+ ASSERT_OK(Flush());
for (i = 2666; i < 2999; ++i) {
ASSERT_OK(Put(Key(i), "val"));
}
- Flush();
+ ASSERT_OK(Flush());
ASSERT_OK(dbfull()->TEST_WaitForCompact());
ASSERT_EQ(0, NumTableFilesAtLevel(0));
for (i = 1900; i < 2100; ++i) {
ASSERT_OK(Delete(Key(i)));
}
- Flush();
+ ASSERT_OK(Flush());
ASSERT_OK(dbfull()->TEST_WaitForCompact());
ASSERT_EQ(0, NumTableFilesAtLevel(0));
for (i = 0; i < 2000; ++i) {
ASSERT_OK(Put(Key(i), "val"));
}
- Flush();
+ ASSERT_OK(Flush());
for (i = 2000; i < 3000; ++i) {
ASSERT_OK(Put(Key(i), "val"));
}
- Flush();
+ ASSERT_OK(Flush());
for (i = 3500; i < 4000; ++i) {
ASSERT_OK(Put(Key(i), "val"));
}
- Flush();
+ ASSERT_OK(Flush());
for (i = 2900; i < 3100; ++i) {
ASSERT_OK(Delete(Key(i)));
}
- Flush();
+ ASSERT_OK(Flush());
ASSERT_OK(dbfull()->TEST_WaitForCompact());
ASSERT_EQ(2, NumTableFilesAtLevel(0));
for (i = 0; i < 2000; ++i) {
ASSERT_OK(Put(Key(i), "val"));
}
- Flush();
+ ASSERT_OK(Flush());
// MoveFilesToLevel(6);
ASSERT_OK(dbfull()->CompactRange(CompactRangeOptions(), nullptr, nullptr));
ASSERT_OK(Put(Key(i), "val"));
}
}
- Flush();
+ ASSERT_OK(Flush());
ASSERT_OK(dbfull()->TEST_WaitForCompact());
ASSERT_EQ(0, NumTableFilesAtLevel(0));
// Case 1: Oldest flushed file excceeds periodic compaction threshold.
ASSERT_OK(Put("foo", "bar"));
- Flush();
+ ASSERT_OK(Flush());
ASSERT_EQ(0, periodic_compactions);
// Move clock forward so that the flushed file would qualify periodic
// compaction.
// Another flush would trigger compaction the oldest file.
ASSERT_OK(Put("foo", "bar2"));
- Flush();
+ ASSERT_OK(Flush());
ASSERT_OK(dbfull()->TEST_WaitForCompact());
ASSERT_EQ(1, periodic_compactions);
periodic_compactions = 0;
// A flush doesn't trigger a periodic compaction when threshold not hit
ASSERT_OK(Put("foo", "bar2"));
- Flush();
+ ASSERT_OK(Flush());
ASSERT_OK(dbfull()->TEST_WaitForCompact());
ASSERT_EQ(0, periodic_compactions);
// a compaction
ASSERT_OK(Put("foo", "bar2"));
env_->MockSleepForSeconds(48 * 60 * 60 + 100);
- Flush();
+ ASSERT_OK(Flush());
ASSERT_OK(dbfull()->TEST_WaitForCompact());
ASSERT_EQ(1, periodic_compactions);
ASSERT_EQ(0, start_level);
::testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();
#else
- (void) argc;
- (void) argv;
+ (void)argc;
+ (void)argv;
return 0;
#endif
}
projects / ceph.git / blobdiff