// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
-// This source code is licensed under the BSD-style license found in the
-// LICENSE file in the root directory of this source tree. An additional grant
-// of patent rights can be found in the PATENTS file in the same directory.
+// This source code is licensed under both the GPLv2 (found in the
+// COPYING file in the root directory) and Apache 2.0 License
+// (found in the LICENSE.Apache file in the root directory).
//
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
#include "db/db_test_util.h"
#include "options/options_helper.h"
#include "port/stack_trace.h"
+#include "rocksdb/cache.h"
#include "rocksdb/convenience.h"
+#include "rocksdb/rate_limiter.h"
#include "util/random.h"
#include "util/sync_point.h"
#include "util/testutil.h"
GetMutableCFOptionsMap(dbfull()->GetOptions(handles_[1])));
}
+TEST_F(DBOptionsTest, SetBytesPerSync) {
+ const size_t kValueSize = 1024 * 1024; // 1MB
+ Options options;
+ options.create_if_missing = true;
+ options.bytes_per_sync = 1024 * 1024;
+ options.use_direct_reads = false;
+ options.write_buffer_size = 400 * kValueSize;
+ options.disable_auto_compactions = true;
+ options.compression = kNoCompression;
+ options.env = env_;
+ Reopen(options);
+ int counter = 0;
+ int low_bytes_per_sync = 0;
+ int i = 0;
+ const std::string kValue(kValueSize, 'v');
+ ASSERT_EQ(options.bytes_per_sync, dbfull()->GetDBOptions().bytes_per_sync);
+ rocksdb::SyncPoint::GetInstance()->SetCallBack(
+ "WritableFileWriter::RangeSync:0", [&](void* /*arg*/) {
+ counter++;
+ });
+
+ WriteOptions write_opts;
+ // should sync approximately 40MB/1MB ~= 40 times.
+ for (i = 0; i < 40; i++) {
+ Put(Key(i), kValue, write_opts);
+ }
+ rocksdb::SyncPoint::GetInstance()->EnableProcessing();
+ ASSERT_OK(dbfull()->CompactRange(CompactRangeOptions(), nullptr, nullptr));
+ rocksdb::SyncPoint::GetInstance()->DisableProcessing();
+ low_bytes_per_sync = counter;
+ ASSERT_GT(low_bytes_per_sync, 35);
+ ASSERT_LT(low_bytes_per_sync, 45);
+
+ counter = 0;
+ // 8388608 = 8 * 1024 * 1024
+ ASSERT_OK(dbfull()->SetDBOptions({{"bytes_per_sync", "8388608"}}));
+ ASSERT_EQ(8388608, dbfull()->GetDBOptions().bytes_per_sync);
+ // should sync approximately 40MB*2/8MB ~= 10 times.
+ // data will be 40*2MB because of previous Puts too.
+ for (i = 0; i < 40; i++) {
+ Put(Key(i), kValue, write_opts);
+ }
+ rocksdb::SyncPoint::GetInstance()->EnableProcessing();
+ ASSERT_OK(dbfull()->CompactRange(CompactRangeOptions(), nullptr, nullptr));
+ ASSERT_GT(counter, 5);
+ ASSERT_LT(counter, 15);
+
+ // Redundant assert. But leaving it here just to get the point across that
+ // low_bytes_per_sync > counter.
+ ASSERT_GT(low_bytes_per_sync, counter);
+}
+
+TEST_F(DBOptionsTest, SetWalBytesPerSync) {
+ const size_t kValueSize = 1024 * 1024 * 3;
+ Options options;
+ options.create_if_missing = true;
+ options.wal_bytes_per_sync = 512;
+ options.write_buffer_size = 100 * kValueSize;
+ options.disable_auto_compactions = true;
+ options.compression = kNoCompression;
+ options.env = env_;
+ Reopen(options);
+ ASSERT_EQ(512, dbfull()->GetDBOptions().wal_bytes_per_sync);
+ int counter = 0;
+ int low_bytes_per_sync = 0;
+ rocksdb::SyncPoint::GetInstance()->SetCallBack(
+ "WritableFileWriter::RangeSync:0", [&](void* /*arg*/) {
+ counter++;
+ });
+ rocksdb::SyncPoint::GetInstance()->EnableProcessing();
+ const std::string kValue(kValueSize, 'v');
+ int i = 0;
+ for (; i < 10; i++) {
+ Put(Key(i), kValue);
+ }
+ // Do not flush. If we flush here, SwitchWAL will reuse old WAL file since its
+ // empty and will not get the new wal_bytes_per_sync value.
+ low_bytes_per_sync = counter;
+ //5242880 = 1024 * 1024 * 5
+ ASSERT_OK(dbfull()->SetDBOptions({{"wal_bytes_per_sync", "5242880"}}));
+ ASSERT_EQ(5242880, dbfull()->GetDBOptions().wal_bytes_per_sync);
+ counter = 0;
+ i = 0;
+ for (; i < 10; i++) {
+ Put(Key(i), kValue);
+ }
+ ASSERT_GT(counter, 0);
+ ASSERT_GT(low_bytes_per_sync, 0);
+ ASSERT_GT(low_bytes_per_sync, counter);
+}
+
+TEST_F(DBOptionsTest, WritableFileMaxBufferSize) {
+ Options options;
+ options.create_if_missing = true;
+ options.writable_file_max_buffer_size = 1024 * 1024;
+ options.level0_file_num_compaction_trigger = 3;
+ options.max_manifest_file_size = 1;
+ options.env = env_;
+ int buffer_size = 1024 * 1024;
+ Reopen(options);
+ ASSERT_EQ(buffer_size,
+ dbfull()->GetDBOptions().writable_file_max_buffer_size);
+
+ std::atomic<int> match_cnt(0);
+ std::atomic<int> unmatch_cnt(0);
+ rocksdb::SyncPoint::GetInstance()->SetCallBack(
+ "WritableFileWriter::WritableFileWriter:0", [&](void* arg) {
+ int value = static_cast<int>(reinterpret_cast<uintptr_t>(arg));
+ if (value == buffer_size) {
+ match_cnt++;
+ } else {
+ unmatch_cnt++;
+ }
+ });
+ rocksdb::SyncPoint::GetInstance()->EnableProcessing();
+ int i = 0;
+ for (; i < 3; i++) {
+ ASSERT_OK(Put("foo", ToString(i)));
+ ASSERT_OK(Put("bar", ToString(i)));
+ Flush();
+ }
+ dbfull()->TEST_WaitForCompact();
+ ASSERT_EQ(unmatch_cnt, 0);
+ ASSERT_GE(match_cnt, 11);
+
+ ASSERT_OK(
+ dbfull()->SetDBOptions({{"writable_file_max_buffer_size", "524288"}}));
+ buffer_size = 512 * 1024;
+ match_cnt = 0;
+ unmatch_cnt = 0; // SetDBOptions() will create a WriteableFileWriter
+
+ ASSERT_EQ(buffer_size,
+ dbfull()->GetDBOptions().writable_file_max_buffer_size);
+ i = 0;
+ for (; i < 3; i++) {
+ ASSERT_OK(Put("foo", ToString(i)));
+ ASSERT_OK(Put("bar", ToString(i)));
+ Flush();
+ }
+ dbfull()->TEST_WaitForCompact();
+ ASSERT_EQ(unmatch_cnt, 0);
+ ASSERT_GE(match_cnt, 11);
+}
+
TEST_F(DBOptionsTest, SetOptionsAndReopen) {
Random rnd(1044);
auto rand_opts = GetRandomizedMutableCFOptionsMap(&rnd);
TEST_F(DBOptionsTest, SetBackgroundCompactionThreads) {
Options options;
options.create_if_missing = true;
- options.base_background_compactions = 1; // default value
options.max_background_compactions = 1; // default value
options.env = env_;
Reopen(options);
ASSERT_EQ(1, dbfull()->TEST_BGCompactionsAllowed());
- ASSERT_OK(dbfull()->SetDBOptions({{"base_background_compactions", "2"},
- {"max_background_compactions", "3"}}));
- ASSERT_EQ(2, dbfull()->TEST_BGCompactionsAllowed());
+ ASSERT_OK(dbfull()->SetDBOptions({{"max_background_compactions", "3"}}));
+ ASSERT_EQ(1, dbfull()->TEST_BGCompactionsAllowed());
auto stop_token = dbfull()->TEST_write_controler().GetStopToken();
ASSERT_EQ(3, dbfull()->TEST_BGCompactionsAllowed());
}
+TEST_F(DBOptionsTest, SetBackgroundJobs) {
+ Options options;
+ options.create_if_missing = true;
+ options.max_background_jobs = 8;
+ options.env = env_;
+ Reopen(options);
+
+ for (int i = 0; i < 2; ++i) {
+ if (i > 0) {
+ options.max_background_jobs = 12;
+ ASSERT_OK(dbfull()->SetDBOptions(
+ {{"max_background_jobs",
+ std::to_string(options.max_background_jobs)}}));
+ }
+
+ ASSERT_EQ(options.max_background_jobs / 4,
+ dbfull()->TEST_BGFlushesAllowed());
+ ASSERT_EQ(1, dbfull()->TEST_BGCompactionsAllowed());
+
+ auto stop_token = dbfull()->TEST_write_controler().GetStopToken();
+
+ ASSERT_EQ(options.max_background_jobs / 4,
+ dbfull()->TEST_BGFlushesAllowed());
+ ASSERT_EQ(3 * options.max_background_jobs / 4,
+ dbfull()->TEST_BGCompactionsAllowed());
+ }
+}
+
TEST_F(DBOptionsTest, AvoidFlushDuringShutdown) {
Options options;
options.create_if_missing = true;
JobContext job_context(0);
dbfull->FindObsoleteFiles(&job_context, false);
ASSERT_EQ(empty, job_context.full_scan_candidate_files.empty());
- job_context.Clean();
dbfull->TEST_UnlockMutex();
+ if (job_context.HaveSomethingToDelete()) {
+ // fulfill the contract of FindObsoleteFiles by calling PurgeObsoleteFiles
+ // afterwards; otherwise the test may hang on shutdown
+ dbfull->PurgeObsoleteFiles(job_context);
+ }
+ job_context.Clean();
}
TEST_F(DBOptionsTest, DeleteObsoleteFilesPeriodChange) {
Close();
}
+TEST_F(DBOptionsTest, MaxOpenFilesChange) {
+ SpecialEnv env(env_);
+ Options options;
+ options.env = CurrentOptions().env;
+ options.max_open_files = -1;
+
+ Reopen(options);
+
+ Cache* tc = dbfull()->TEST_table_cache();
+
+ ASSERT_EQ(-1, dbfull()->GetDBOptions().max_open_files);
+ ASSERT_LT(2000, tc->GetCapacity());
+ ASSERT_OK(dbfull()->SetDBOptions({{"max_open_files", "1024"}}));
+ ASSERT_EQ(1024, dbfull()->GetDBOptions().max_open_files);
+ // examine the table cache (actual size should be 1014)
+ ASSERT_GT(1500, tc->GetCapacity());
+ Close();
+}
+
+TEST_F(DBOptionsTest, SanitizeDelayedWriteRate) {
+ Options options;
+ options.delayed_write_rate = 0;
+ Reopen(options);
+ ASSERT_EQ(16 * 1024 * 1024, dbfull()->GetDBOptions().delayed_write_rate);
+
+ options.rate_limiter.reset(NewGenericRateLimiter(31 * 1024 * 1024));
+ Reopen(options);
+ ASSERT_EQ(31 * 1024 * 1024, dbfull()->GetDBOptions().delayed_write_rate);
+}
+
+TEST_F(DBOptionsTest, SetFIFOCompactionOptions) {
+ Options options;
+ options.compaction_style = kCompactionStyleFIFO;
+ options.write_buffer_size = 10 << 10; // 10KB
+ options.arena_block_size = 4096;
+ options.compression = kNoCompression;
+ options.create_if_missing = true;
+ options.compaction_options_fifo.allow_compaction = false;
+ env_->time_elapse_only_sleep_ = false;
+ options.env = env_;
+
+ // Test dynamically changing compaction_options_fifo.ttl
+ env_->addon_time_.store(0);
+ options.compaction_options_fifo.ttl = 1 * 60 * 60; // 1 hour
+ ASSERT_OK(TryReopen(options));
+
+ Random rnd(301);
+ for (int i = 0; i < 10; i++) {
+ // Generate and flush a file about 10KB.
+ for (int j = 0; j < 10; j++) {
+ ASSERT_OK(Put(ToString(i * 20 + j), RandomString(&rnd, 980)));
+ }
+ Flush();
+ }
+ ASSERT_OK(dbfull()->TEST_WaitForCompact());
+ ASSERT_EQ(NumTableFilesAtLevel(0), 10);
+
+ // Add 61 seconds to the time.
+ env_->addon_time_.fetch_add(61);
+
+ // No files should be compacted as ttl is set to 1 hour.
+ ASSERT_EQ(dbfull()->GetOptions().compaction_options_fifo.ttl, 3600);
+ dbfull()->CompactRange(CompactRangeOptions(), nullptr, nullptr);
+ ASSERT_EQ(NumTableFilesAtLevel(0), 10);
+
+ // Set ttl to 1 minute. So all files should get deleted.
+ ASSERT_OK(dbfull()->SetOptions({{"compaction_options_fifo", "{ttl=60;}"}}));
+ ASSERT_EQ(dbfull()->GetOptions().compaction_options_fifo.ttl, 60);
+ dbfull()->CompactRange(CompactRangeOptions(), nullptr, nullptr);
+ ASSERT_OK(dbfull()->TEST_WaitForCompact());
+ ASSERT_EQ(NumTableFilesAtLevel(0), 0);
+
+ // Test dynamically changing compaction_options_fifo.max_table_files_size
+ env_->addon_time_.store(0);
+ options.compaction_options_fifo.max_table_files_size = 500 << 10; // 00KB
+ options.compaction_options_fifo.ttl = 0;
+ DestroyAndReopen(options);
+
+ for (int i = 0; i < 10; i++) {
+ // Generate and flush a file about 10KB.
+ for (int j = 0; j < 10; j++) {
+ ASSERT_OK(Put(ToString(i * 20 + j), RandomString(&rnd, 980)));
+ }
+ Flush();
+ }
+ ASSERT_OK(dbfull()->TEST_WaitForCompact());
+ ASSERT_EQ(NumTableFilesAtLevel(0), 10);
+
+ // No files should be compacted as max_table_files_size is set to 500 KB.
+ ASSERT_EQ(dbfull()->GetOptions().compaction_options_fifo.max_table_files_size,
+ 500 << 10);
+ dbfull()->CompactRange(CompactRangeOptions(), nullptr, nullptr);
+ ASSERT_EQ(NumTableFilesAtLevel(0), 10);
+
+ // Set max_table_files_size to 12 KB. So only 1 file should remain now.
+ ASSERT_OK(dbfull()->SetOptions(
+ {{"compaction_options_fifo", "{max_table_files_size=12288;}"}}));
+ ASSERT_EQ(dbfull()->GetOptions().compaction_options_fifo.max_table_files_size,
+ 12 << 10);
+ dbfull()->CompactRange(CompactRangeOptions(), nullptr, nullptr);
+ ASSERT_OK(dbfull()->TEST_WaitForCompact());
+ ASSERT_EQ(NumTableFilesAtLevel(0), 1);
+
+ // Test dynamically changing compaction_options_fifo.allow_compaction
+ options.compaction_options_fifo.max_table_files_size = 500 << 10; // 500KB
+ options.compaction_options_fifo.ttl = 0;
+ options.compaction_options_fifo.allow_compaction = false;
+ options.level0_file_num_compaction_trigger = 6;
+ DestroyAndReopen(options);
+
+ for (int i = 0; i < 10; i++) {
+ // Generate and flush a file about 10KB.
+ for (int j = 0; j < 10; j++) {
+ ASSERT_OK(Put(ToString(i * 20 + j), RandomString(&rnd, 980)));
+ }
+ Flush();
+ }
+ ASSERT_OK(dbfull()->TEST_WaitForCompact());
+ ASSERT_EQ(NumTableFilesAtLevel(0), 10);
+
+ // No files should be compacted as max_table_files_size is set to 500 KB and
+ // allow_compaction is false
+ ASSERT_EQ(dbfull()->GetOptions().compaction_options_fifo.allow_compaction,
+ false);
+ dbfull()->CompactRange(CompactRangeOptions(), nullptr, nullptr);
+ ASSERT_EQ(NumTableFilesAtLevel(0), 10);
+
+ // Set allow_compaction to true. So number of files should be between 1 and 5.
+ ASSERT_OK(dbfull()->SetOptions(
+ {{"compaction_options_fifo", "{allow_compaction=true;}"}}));
+ ASSERT_EQ(dbfull()->GetOptions().compaction_options_fifo.allow_compaction,
+ true);
+ dbfull()->CompactRange(CompactRangeOptions(), nullptr, nullptr);
+ ASSERT_OK(dbfull()->TEST_WaitForCompact());
+ ASSERT_GE(NumTableFilesAtLevel(0), 1);
+ ASSERT_LE(NumTableFilesAtLevel(0), 5);
+}
+
+TEST_F(DBOptionsTest, CompactionReadaheadSizeChange) {
+ SpecialEnv env(env_);
+ Options options;
+ options.env = &env;
+
+ options.compaction_readahead_size = 0;
+ options.new_table_reader_for_compaction_inputs = true;
+ options.level0_file_num_compaction_trigger = 2;
+ const std::string kValue(1024, 'v');
+ Reopen(options);
+
+ ASSERT_EQ(0, dbfull()->GetDBOptions().compaction_readahead_size);
+ ASSERT_OK(dbfull()->SetDBOptions({{"compaction_readahead_size", "256"}}));
+ ASSERT_EQ(256, dbfull()->GetDBOptions().compaction_readahead_size);
+ for (int i = 0; i < 1024; i++) {
+ Put(Key(i), kValue);
+ }
+ Flush();
+ for (int i = 0; i < 1024 * 2; i++) {
+ Put(Key(i), kValue);
+ }
+ Flush();
+ dbfull()->TEST_WaitForCompact();
+ ASSERT_EQ(256, env_->compaction_readahead_size_);
+ Close();
+}
#endif // ROCKSDB_LITE
} // namespace rocksdb
projects / ceph.git / blobdiff