]>
Commit | Line | Data |
---|---|---|
7c673cae | 1 | // Copyright (c) 2011-present, Facebook, Inc. All rights reserved. |
11fdf7f2 TL |
2 | // This source code is licensed under both the GPLv2 (found in the |
3 | // COPYING file in the root directory) and Apache 2.0 License | |
4 | // (found in the LICENSE.Apache file in the root directory). | |
7c673cae FG |
5 | // |
6 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. | |
7 | // Use of this source code is governed by a BSD-style license that can be | |
8 | // found in the LICENSE file. See the AUTHORS file for names of contributors. | |
9 | #include <limits> | |
10 | #include <string> | |
11 | #include <unordered_map> | |
12 | ||
13 | #include "db/column_family.h" | |
f67539c2 | 14 | #include "db/db_impl/db_impl.h" |
7c673cae FG |
15 | #include "db/db_test_util.h" |
16 | #include "options/options_helper.h" | |
17 | #include "port/stack_trace.h" | |
11fdf7f2 | 18 | #include "rocksdb/cache.h" |
7c673cae | 19 | #include "rocksdb/convenience.h" |
11fdf7f2 | 20 | #include "rocksdb/rate_limiter.h" |
494da23a | 21 | #include "rocksdb/stats_history.h" |
f67539c2 TL |
22 | #include "test_util/sync_point.h" |
23 | #include "test_util/testutil.h" | |
7c673cae | 24 | #include "util/random.h" |
7c673cae | 25 | |
f67539c2 | 26 | namespace ROCKSDB_NAMESPACE { |
494da23a | 27 | |
7c673cae FG |
28 | class DBOptionsTest : public DBTestBase { |
29 | public: | |
1e59de90 | 30 | DBOptionsTest() : DBTestBase("db_options_test", /*env_do_fsync=*/true) {} |
7c673cae FG |
31 | |
32 | #ifndef ROCKSDB_LITE | |
33 | std::unordered_map<std::string, std::string> GetMutableDBOptionsMap( | |
34 | const DBOptions& options) { | |
35 | std::string options_str; | |
7c673cae | 36 | std::unordered_map<std::string, std::string> mutable_map; |
1e59de90 | 37 | ConfigOptions config_options(options); |
20effc67 TL |
38 | config_options.delimiter = "; "; |
39 | ||
40 | EXPECT_OK(GetStringFromMutableDBOptions( | |
41 | config_options, MutableDBOptions(options), &options_str)); | |
42 | EXPECT_OK(StringToMap(options_str, &mutable_map)); | |
43 | ||
7c673cae FG |
44 | return mutable_map; |
45 | } | |
46 | ||
47 | std::unordered_map<std::string, std::string> GetMutableCFOptionsMap( | |
48 | const ColumnFamilyOptions& options) { | |
49 | std::string options_str; | |
20effc67 TL |
50 | ConfigOptions config_options; |
51 | config_options.delimiter = "; "; | |
52 | ||
7c673cae | 53 | std::unordered_map<std::string, std::string> mutable_map; |
20effc67 TL |
54 | EXPECT_OK(GetStringFromMutableCFOptions( |
55 | config_options, MutableCFOptions(options), &options_str)); | |
56 | EXPECT_OK(StringToMap(options_str, &mutable_map)); | |
7c673cae FG |
57 | return mutable_map; |
58 | } | |
59 | ||
60 | std::unordered_map<std::string, std::string> GetRandomizedMutableCFOptionsMap( | |
61 | Random* rnd) { | |
f67539c2 | 62 | Options options = CurrentOptions(); |
7c673cae FG |
63 | options.env = env_; |
64 | ImmutableDBOptions db_options(options); | |
f67539c2 | 65 | test::RandomInitCFOptions(&options, options, rnd); |
7c673cae FG |
66 | auto sanitized_options = SanitizeOptions(db_options, options); |
67 | auto opt_map = GetMutableCFOptionsMap(sanitized_options); | |
68 | delete options.compaction_filter; | |
69 | return opt_map; | |
70 | } | |
71 | ||
72 | std::unordered_map<std::string, std::string> GetRandomizedMutableDBOptionsMap( | |
73 | Random* rnd) { | |
74 | DBOptions db_options; | |
75 | test::RandomInitDBOptions(&db_options, rnd); | |
76 | auto sanitized_options = SanitizeOptions(dbname_, db_options); | |
77 | return GetMutableDBOptionsMap(sanitized_options); | |
78 | } | |
79 | #endif // ROCKSDB_LITE | |
80 | }; | |
81 | ||
20effc67 TL |
82 | TEST_F(DBOptionsTest, ImmutableTrackAndVerifyWalsInManifest) { |
83 | Options options; | |
84 | options.env = env_; | |
85 | options.track_and_verify_wals_in_manifest = true; | |
86 | ||
87 | ImmutableDBOptions db_options(options); | |
88 | ASSERT_TRUE(db_options.track_and_verify_wals_in_manifest); | |
89 | ||
90 | Reopen(options); | |
91 | ASSERT_TRUE(dbfull()->GetDBOptions().track_and_verify_wals_in_manifest); | |
92 | ||
93 | Status s = | |
94 | dbfull()->SetDBOptions({{"track_and_verify_wals_in_manifest", "false"}}); | |
95 | ASSERT_FALSE(s.ok()); | |
96 | } | |
97 | ||
1e59de90 TL |
98 | TEST_F(DBOptionsTest, ImmutableVerifySstUniqueIdInManifest) { |
99 | Options options; | |
100 | options.env = env_; | |
101 | options.verify_sst_unique_id_in_manifest = true; | |
102 | ||
103 | ImmutableDBOptions db_options(options); | |
104 | ASSERT_TRUE(db_options.verify_sst_unique_id_in_manifest); | |
105 | ||
106 | Reopen(options); | |
107 | ASSERT_TRUE(dbfull()->GetDBOptions().verify_sst_unique_id_in_manifest); | |
108 | ||
109 | Status s = | |
110 | dbfull()->SetDBOptions({{"verify_sst_unique_id_in_manifest", "false"}}); | |
111 | ASSERT_FALSE(s.ok()); | |
112 | } | |
113 | ||
7c673cae FG |
114 | // RocksDB lite don't support dynamic options. |
115 | #ifndef ROCKSDB_LITE | |
116 | ||
1e59de90 TL |
117 | TEST_F(DBOptionsTest, AvoidUpdatingOptions) { |
118 | Options options; | |
119 | options.env = env_; | |
120 | options.max_background_jobs = 4; | |
121 | options.delayed_write_rate = 1024; | |
122 | ||
123 | Reopen(options); | |
124 | ||
125 | SyncPoint::GetInstance()->DisableProcessing(); | |
126 | SyncPoint::GetInstance()->ClearAllCallBacks(); | |
127 | bool is_changed_stats = false; | |
128 | SyncPoint::GetInstance()->SetCallBack( | |
129 | "DBImpl::WriteOptionsFile:PersistOptions", [&](void* /*arg*/) { | |
130 | ASSERT_FALSE(is_changed_stats); // should only save options file once | |
131 | is_changed_stats = true; | |
132 | }); | |
133 | SyncPoint::GetInstance()->EnableProcessing(); | |
134 | ||
135 | // helper function to check the status and reset after each check | |
136 | auto is_changed = [&] { | |
137 | bool ret = is_changed_stats; | |
138 | is_changed_stats = false; | |
139 | return ret; | |
140 | }; | |
141 | ||
142 | // without changing the value, but it's sanitized to a different value | |
143 | ASSERT_OK(dbfull()->SetDBOptions({{"bytes_per_sync", "0"}})); | |
144 | ASSERT_TRUE(is_changed()); | |
145 | ||
146 | // without changing the value | |
147 | ASSERT_OK(dbfull()->SetDBOptions({{"max_background_jobs", "4"}})); | |
148 | ASSERT_FALSE(is_changed()); | |
149 | ||
150 | // changing the value | |
151 | ASSERT_OK(dbfull()->SetDBOptions({{"bytes_per_sync", "123"}})); | |
152 | ASSERT_TRUE(is_changed()); | |
153 | ||
154 | // update again | |
155 | ASSERT_OK(dbfull()->SetDBOptions({{"bytes_per_sync", "123"}})); | |
156 | ASSERT_FALSE(is_changed()); | |
157 | ||
158 | // without changing a default value | |
159 | ASSERT_OK(dbfull()->SetDBOptions({{"strict_bytes_per_sync", "false"}})); | |
160 | ASSERT_FALSE(is_changed()); | |
161 | ||
162 | // now change | |
163 | ASSERT_OK(dbfull()->SetDBOptions({{"strict_bytes_per_sync", "true"}})); | |
164 | ASSERT_TRUE(is_changed()); | |
165 | ||
166 | // multiple values without change | |
167 | ASSERT_OK(dbfull()->SetDBOptions( | |
168 | {{"max_total_wal_size", "0"}, {"stats_dump_period_sec", "600"}})); | |
169 | ASSERT_FALSE(is_changed()); | |
170 | ||
171 | // multiple values with change | |
172 | ASSERT_OK(dbfull()->SetDBOptions( | |
173 | {{"max_open_files", "100"}, {"stats_dump_period_sec", "600"}})); | |
174 | ASSERT_TRUE(is_changed()); | |
175 | } | |
176 | ||
7c673cae FG |
177 | TEST_F(DBOptionsTest, GetLatestDBOptions) { |
178 | // GetOptions should be able to get latest option changed by SetOptions. | |
179 | Options options; | |
180 | options.create_if_missing = true; | |
181 | options.env = env_; | |
182 | Random rnd(228); | |
183 | Reopen(options); | |
184 | auto new_options = GetRandomizedMutableDBOptionsMap(&rnd); | |
185 | ASSERT_OK(dbfull()->SetDBOptions(new_options)); | |
186 | ASSERT_EQ(new_options, GetMutableDBOptionsMap(dbfull()->GetDBOptions())); | |
187 | } | |
188 | ||
189 | TEST_F(DBOptionsTest, GetLatestCFOptions) { | |
190 | // GetOptions should be able to get latest option changed by SetOptions. | |
191 | Options options; | |
192 | options.create_if_missing = true; | |
193 | options.env = env_; | |
194 | Random rnd(228); | |
195 | Reopen(options); | |
196 | CreateColumnFamilies({"foo"}, options); | |
197 | ReopenWithColumnFamilies({"default", "foo"}, options); | |
198 | auto options_default = GetRandomizedMutableCFOptionsMap(&rnd); | |
199 | auto options_foo = GetRandomizedMutableCFOptionsMap(&rnd); | |
200 | ASSERT_OK(dbfull()->SetOptions(handles_[0], options_default)); | |
201 | ASSERT_OK(dbfull()->SetOptions(handles_[1], options_foo)); | |
202 | ASSERT_EQ(options_default, | |
203 | GetMutableCFOptionsMap(dbfull()->GetOptions(handles_[0]))); | |
204 | ASSERT_EQ(options_foo, | |
205 | GetMutableCFOptionsMap(dbfull()->GetOptions(handles_[1]))); | |
206 | } | |
207 | ||
1e59de90 TL |
208 | TEST_F(DBOptionsTest, SetMutableTableOptions) { |
209 | Options options; | |
210 | options.create_if_missing = true; | |
211 | options.env = env_; | |
212 | options.blob_file_size = 16384; | |
213 | BlockBasedTableOptions bbto; | |
214 | bbto.no_block_cache = true; | |
215 | bbto.block_size = 8192; | |
216 | bbto.block_restart_interval = 7; | |
217 | ||
218 | options.table_factory.reset(NewBlockBasedTableFactory(bbto)); | |
219 | Reopen(options); | |
220 | ||
221 | ColumnFamilyHandle* cfh = dbfull()->DefaultColumnFamily(); | |
222 | Options c_opts = dbfull()->GetOptions(cfh); | |
223 | ||
224 | const auto* c_bbto = | |
225 | c_opts.table_factory->GetOptions<BlockBasedTableOptions>(); | |
226 | ASSERT_NE(c_bbto, nullptr); | |
227 | ASSERT_EQ(c_opts.blob_file_size, 16384); | |
228 | ASSERT_EQ(c_bbto->no_block_cache, true); | |
229 | ASSERT_EQ(c_bbto->block_size, 8192); | |
230 | ASSERT_EQ(c_bbto->block_restart_interval, 7); | |
231 | ASSERT_OK(dbfull()->SetOptions( | |
232 | cfh, {{"table_factory.block_size", "16384"}, | |
233 | {"table_factory.block_restart_interval", "11"}})); | |
234 | ASSERT_EQ(c_bbto->block_size, 16384); | |
235 | ASSERT_EQ(c_bbto->block_restart_interval, 11); | |
236 | ||
237 | // Now set an option that is not mutable - options should not change | |
238 | ASSERT_NOK( | |
239 | dbfull()->SetOptions(cfh, {{"table_factory.no_block_cache", "false"}})); | |
240 | ASSERT_EQ(c_bbto->no_block_cache, true); | |
241 | ASSERT_EQ(c_bbto->block_size, 16384); | |
242 | ASSERT_EQ(c_bbto->block_restart_interval, 11); | |
243 | ||
244 | // Set some that are mutable and some that are not - options should not change | |
245 | ASSERT_NOK(dbfull()->SetOptions( | |
246 | cfh, {{"table_factory.no_block_cache", "false"}, | |
247 | {"table_factory.block_size", "8192"}, | |
248 | {"table_factory.block_restart_interval", "7"}})); | |
249 | ASSERT_EQ(c_bbto->no_block_cache, true); | |
250 | ASSERT_EQ(c_bbto->block_size, 16384); | |
251 | ASSERT_EQ(c_bbto->block_restart_interval, 11); | |
252 | ||
253 | // Set some that are mutable and some that do not exist - options should not | |
254 | // change | |
255 | ASSERT_NOK(dbfull()->SetOptions( | |
256 | cfh, {{"table_factory.block_size", "8192"}, | |
257 | {"table_factory.does_not_exist", "true"}, | |
258 | {"table_factory.block_restart_interval", "7"}})); | |
259 | ASSERT_EQ(c_bbto->no_block_cache, true); | |
260 | ASSERT_EQ(c_bbto->block_size, 16384); | |
261 | ASSERT_EQ(c_bbto->block_restart_interval, 11); | |
262 | ||
263 | // Trying to change the table factory fails | |
264 | ASSERT_NOK(dbfull()->SetOptions( | |
265 | cfh, {{"table_factory", TableFactory::kPlainTableName()}})); | |
266 | ||
267 | // Set some on the table and some on the Column Family | |
268 | ASSERT_OK(dbfull()->SetOptions( | |
269 | cfh, {{"table_factory.block_size", "16384"}, | |
270 | {"blob_file_size", "32768"}, | |
271 | {"table_factory.block_restart_interval", "13"}})); | |
272 | c_opts = dbfull()->GetOptions(cfh); | |
273 | ASSERT_EQ(c_opts.blob_file_size, 32768); | |
274 | ASSERT_EQ(c_bbto->block_size, 16384); | |
275 | ASSERT_EQ(c_bbto->block_restart_interval, 13); | |
276 | // Set some on the table and a bad one on the ColumnFamily - options should | |
277 | // not change | |
278 | ASSERT_NOK(dbfull()->SetOptions( | |
279 | cfh, {{"table_factory.block_size", "1024"}, | |
280 | {"no_such_option", "32768"}, | |
281 | {"table_factory.block_restart_interval", "7"}})); | |
282 | ASSERT_EQ(c_bbto->block_size, 16384); | |
283 | ASSERT_EQ(c_bbto->block_restart_interval, 13); | |
284 | } | |
285 | ||
286 | TEST_F(DBOptionsTest, SetWithCustomMemTableFactory) { | |
287 | class DummySkipListFactory : public SkipListFactory { | |
288 | public: | |
289 | static const char* kClassName() { return "DummySkipListFactory"; } | |
290 | const char* Name() const override { return kClassName(); } | |
291 | explicit DummySkipListFactory() : SkipListFactory(2) {} | |
292 | }; | |
293 | { | |
294 | // Verify the DummySkipList cannot be created | |
295 | ConfigOptions config_options; | |
296 | config_options.ignore_unsupported_options = false; | |
297 | std::unique_ptr<MemTableRepFactory> factory; | |
298 | ASSERT_NOK(MemTableRepFactory::CreateFromString( | |
299 | config_options, DummySkipListFactory::kClassName(), &factory)); | |
300 | } | |
301 | Options options; | |
302 | options.create_if_missing = true; | |
303 | // Try with fail_if_options_file_error=false/true to update the options | |
304 | for (bool on_error : {false, true}) { | |
305 | options.fail_if_options_file_error = on_error; | |
306 | options.env = env_; | |
307 | options.disable_auto_compactions = false; | |
308 | ||
309 | options.memtable_factory.reset(new DummySkipListFactory()); | |
310 | Reopen(options); | |
311 | ||
312 | ColumnFamilyHandle* cfh = dbfull()->DefaultColumnFamily(); | |
313 | ASSERT_OK( | |
314 | dbfull()->SetOptions(cfh, {{"disable_auto_compactions", "true"}})); | |
315 | ColumnFamilyDescriptor cfd; | |
316 | ASSERT_OK(cfh->GetDescriptor(&cfd)); | |
317 | ASSERT_STREQ(cfd.options.memtable_factory->Name(), | |
318 | DummySkipListFactory::kClassName()); | |
319 | ColumnFamilyHandle* test = nullptr; | |
320 | ASSERT_OK(dbfull()->CreateColumnFamily(options, "test", &test)); | |
321 | ASSERT_OK(test->GetDescriptor(&cfd)); | |
322 | ASSERT_STREQ(cfd.options.memtable_factory->Name(), | |
323 | DummySkipListFactory::kClassName()); | |
324 | ||
325 | ASSERT_OK(dbfull()->DropColumnFamily(test)); | |
326 | delete test; | |
327 | } | |
328 | } | |
329 | ||
11fdf7f2 TL |
330 | TEST_F(DBOptionsTest, SetBytesPerSync) { |
331 | const size_t kValueSize = 1024 * 1024; // 1MB | |
332 | Options options; | |
333 | options.create_if_missing = true; | |
334 | options.bytes_per_sync = 1024 * 1024; | |
335 | options.use_direct_reads = false; | |
336 | options.write_buffer_size = 400 * kValueSize; | |
337 | options.disable_auto_compactions = true; | |
338 | options.compression = kNoCompression; | |
339 | options.env = env_; | |
340 | Reopen(options); | |
341 | int counter = 0; | |
342 | int low_bytes_per_sync = 0; | |
343 | int i = 0; | |
344 | const std::string kValue(kValueSize, 'v'); | |
345 | ASSERT_EQ(options.bytes_per_sync, dbfull()->GetDBOptions().bytes_per_sync); | |
f67539c2 TL |
346 | ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( |
347 | "WritableFileWriter::RangeSync:0", [&](void* /*arg*/) { counter++; }); | |
11fdf7f2 TL |
348 | |
349 | WriteOptions write_opts; | |
350 | // should sync approximately 40MB/1MB ~= 40 times. | |
351 | for (i = 0; i < 40; i++) { | |
20effc67 | 352 | ASSERT_OK(Put(Key(i), kValue, write_opts)); |
11fdf7f2 | 353 | } |
f67539c2 | 354 | ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); |
11fdf7f2 | 355 | ASSERT_OK(dbfull()->CompactRange(CompactRangeOptions(), nullptr, nullptr)); |
f67539c2 | 356 | ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); |
11fdf7f2 TL |
357 | low_bytes_per_sync = counter; |
358 | ASSERT_GT(low_bytes_per_sync, 35); | |
359 | ASSERT_LT(low_bytes_per_sync, 45); | |
360 | ||
361 | counter = 0; | |
362 | // 8388608 = 8 * 1024 * 1024 | |
363 | ASSERT_OK(dbfull()->SetDBOptions({{"bytes_per_sync", "8388608"}})); | |
364 | ASSERT_EQ(8388608, dbfull()->GetDBOptions().bytes_per_sync); | |
365 | // should sync approximately 40MB*2/8MB ~= 10 times. | |
366 | // data will be 40*2MB because of previous Puts too. | |
367 | for (i = 0; i < 40; i++) { | |
20effc67 | 368 | ASSERT_OK(Put(Key(i), kValue, write_opts)); |
11fdf7f2 | 369 | } |
f67539c2 | 370 | ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); |
11fdf7f2 TL |
371 | ASSERT_OK(dbfull()->CompactRange(CompactRangeOptions(), nullptr, nullptr)); |
372 | ASSERT_GT(counter, 5); | |
373 | ASSERT_LT(counter, 15); | |
374 | ||
375 | // Redundant assert. But leaving it here just to get the point across that | |
376 | // low_bytes_per_sync > counter. | |
377 | ASSERT_GT(low_bytes_per_sync, counter); | |
378 | } | |
379 | ||
380 | TEST_F(DBOptionsTest, SetWalBytesPerSync) { | |
381 | const size_t kValueSize = 1024 * 1024 * 3; | |
382 | Options options; | |
383 | options.create_if_missing = true; | |
384 | options.wal_bytes_per_sync = 512; | |
385 | options.write_buffer_size = 100 * kValueSize; | |
386 | options.disable_auto_compactions = true; | |
387 | options.compression = kNoCompression; | |
388 | options.env = env_; | |
389 | Reopen(options); | |
390 | ASSERT_EQ(512, dbfull()->GetDBOptions().wal_bytes_per_sync); | |
1e59de90 | 391 | std::atomic_int counter{0}; |
11fdf7f2 | 392 | int low_bytes_per_sync = 0; |
f67539c2 | 393 | ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( |
1e59de90 TL |
394 | "WritableFileWriter::RangeSync:0", |
395 | [&](void* /*arg*/) { counter.fetch_add(1); }); | |
f67539c2 | 396 | ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); |
11fdf7f2 TL |
397 | const std::string kValue(kValueSize, 'v'); |
398 | int i = 0; | |
399 | for (; i < 10; i++) { | |
20effc67 | 400 | ASSERT_OK(Put(Key(i), kValue)); |
11fdf7f2 TL |
401 | } |
402 | // Do not flush. If we flush here, SwitchWAL will reuse old WAL file since its | |
403 | // empty and will not get the new wal_bytes_per_sync value. | |
404 | low_bytes_per_sync = counter; | |
1e59de90 | 405 | // 5242880 = 1024 * 1024 * 5 |
11fdf7f2 TL |
406 | ASSERT_OK(dbfull()->SetDBOptions({{"wal_bytes_per_sync", "5242880"}})); |
407 | ASSERT_EQ(5242880, dbfull()->GetDBOptions().wal_bytes_per_sync); | |
408 | counter = 0; | |
409 | i = 0; | |
410 | for (; i < 10; i++) { | |
20effc67 | 411 | ASSERT_OK(Put(Key(i), kValue)); |
11fdf7f2 TL |
412 | } |
413 | ASSERT_GT(counter, 0); | |
414 | ASSERT_GT(low_bytes_per_sync, 0); | |
415 | ASSERT_GT(low_bytes_per_sync, counter); | |
416 | } | |
417 | ||
418 | TEST_F(DBOptionsTest, WritableFileMaxBufferSize) { | |
419 | Options options; | |
420 | options.create_if_missing = true; | |
421 | options.writable_file_max_buffer_size = 1024 * 1024; | |
422 | options.level0_file_num_compaction_trigger = 3; | |
423 | options.max_manifest_file_size = 1; | |
424 | options.env = env_; | |
425 | int buffer_size = 1024 * 1024; | |
426 | Reopen(options); | |
427 | ASSERT_EQ(buffer_size, | |
428 | dbfull()->GetDBOptions().writable_file_max_buffer_size); | |
429 | ||
430 | std::atomic<int> match_cnt(0); | |
431 | std::atomic<int> unmatch_cnt(0); | |
f67539c2 | 432 | ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( |
11fdf7f2 TL |
433 | "WritableFileWriter::WritableFileWriter:0", [&](void* arg) { |
434 | int value = static_cast<int>(reinterpret_cast<uintptr_t>(arg)); | |
435 | if (value == buffer_size) { | |
436 | match_cnt++; | |
437 | } else { | |
438 | unmatch_cnt++; | |
439 | } | |
440 | }); | |
f67539c2 | 441 | ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); |
11fdf7f2 TL |
442 | int i = 0; |
443 | for (; i < 3; i++) { | |
1e59de90 TL |
444 | ASSERT_OK(Put("foo", std::to_string(i))); |
445 | ASSERT_OK(Put("bar", std::to_string(i))); | |
20effc67 | 446 | ASSERT_OK(Flush()); |
11fdf7f2 | 447 | } |
20effc67 | 448 | ASSERT_OK(dbfull()->TEST_WaitForCompact()); |
11fdf7f2 TL |
449 | ASSERT_EQ(unmatch_cnt, 0); |
450 | ASSERT_GE(match_cnt, 11); | |
451 | ||
452 | ASSERT_OK( | |
453 | dbfull()->SetDBOptions({{"writable_file_max_buffer_size", "524288"}})); | |
454 | buffer_size = 512 * 1024; | |
455 | match_cnt = 0; | |
1e59de90 | 456 | unmatch_cnt = 0; // SetDBOptions() will create a WritableFileWriter |
11fdf7f2 TL |
457 | |
458 | ASSERT_EQ(buffer_size, | |
459 | dbfull()->GetDBOptions().writable_file_max_buffer_size); | |
460 | i = 0; | |
461 | for (; i < 3; i++) { | |
1e59de90 TL |
462 | ASSERT_OK(Put("foo", std::to_string(i))); |
463 | ASSERT_OK(Put("bar", std::to_string(i))); | |
20effc67 | 464 | ASSERT_OK(Flush()); |
11fdf7f2 | 465 | } |
20effc67 | 466 | ASSERT_OK(dbfull()->TEST_WaitForCompact()); |
11fdf7f2 TL |
467 | ASSERT_EQ(unmatch_cnt, 0); |
468 | ASSERT_GE(match_cnt, 11); | |
469 | } | |
470 | ||
7c673cae FG |
471 | TEST_F(DBOptionsTest, SetOptionsAndReopen) { |
472 | Random rnd(1044); | |
473 | auto rand_opts = GetRandomizedMutableCFOptionsMap(&rnd); | |
474 | ASSERT_OK(dbfull()->SetOptions(rand_opts)); | |
475 | // Verify if DB can be reopen after setting options. | |
476 | Options options; | |
477 | options.env = env_; | |
478 | ASSERT_OK(TryReopen(options)); | |
479 | } | |
480 | ||
481 | TEST_F(DBOptionsTest, EnableAutoCompactionAndTriggerStall) { | |
482 | const std::string kValue(1024, 'v'); | |
483 | for (int method_type = 0; method_type < 2; method_type++) { | |
484 | for (int option_type = 0; option_type < 4; option_type++) { | |
485 | Options options; | |
486 | options.create_if_missing = true; | |
487 | options.disable_auto_compactions = true; | |
488 | options.write_buffer_size = 1024 * 1024 * 10; | |
489 | options.compression = CompressionType::kNoCompression; | |
490 | options.level0_file_num_compaction_trigger = 1; | |
491 | options.level0_stop_writes_trigger = std::numeric_limits<int>::max(); | |
492 | options.level0_slowdown_writes_trigger = std::numeric_limits<int>::max(); | |
493 | options.hard_pending_compaction_bytes_limit = | |
494 | std::numeric_limits<uint64_t>::max(); | |
495 | options.soft_pending_compaction_bytes_limit = | |
496 | std::numeric_limits<uint64_t>::max(); | |
497 | options.env = env_; | |
498 | ||
499 | DestroyAndReopen(options); | |
500 | int i = 0; | |
501 | for (; i < 1024; i++) { | |
20effc67 | 502 | ASSERT_OK(Put(Key(i), kValue)); |
7c673cae | 503 | } |
20effc67 | 504 | ASSERT_OK(Flush()); |
7c673cae | 505 | for (; i < 1024 * 2; i++) { |
20effc67 | 506 | ASSERT_OK(Put(Key(i), kValue)); |
7c673cae | 507 | } |
20effc67 TL |
508 | ASSERT_OK(Flush()); |
509 | ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable()); | |
7c673cae FG |
510 | ASSERT_EQ(2, NumTableFilesAtLevel(0)); |
511 | uint64_t l0_size = SizeAtLevel(0); | |
512 | ||
513 | switch (option_type) { | |
514 | case 0: | |
515 | // test with level0_stop_writes_trigger | |
516 | options.level0_stop_writes_trigger = 2; | |
517 | options.level0_slowdown_writes_trigger = 2; | |
518 | break; | |
519 | case 1: | |
520 | options.level0_slowdown_writes_trigger = 2; | |
521 | break; | |
522 | case 2: | |
523 | options.hard_pending_compaction_bytes_limit = l0_size; | |
524 | options.soft_pending_compaction_bytes_limit = l0_size; | |
525 | break; | |
526 | case 3: | |
527 | options.soft_pending_compaction_bytes_limit = l0_size; | |
528 | break; | |
529 | } | |
530 | Reopen(options); | |
20effc67 | 531 | ASSERT_OK(dbfull()->TEST_WaitForCompact()); |
7c673cae FG |
532 | ASSERT_FALSE(dbfull()->TEST_write_controler().IsStopped()); |
533 | ASSERT_FALSE(dbfull()->TEST_write_controler().NeedsDelay()); | |
534 | ||
535 | SyncPoint::GetInstance()->LoadDependency( | |
536 | {{"DBOptionsTest::EnableAutoCompactionAndTriggerStall:1", | |
537 | "BackgroundCallCompaction:0"}, | |
538 | {"DBImpl::BackgroundCompaction():BeforePickCompaction", | |
539 | "DBOptionsTest::EnableAutoCompactionAndTriggerStall:2"}, | |
540 | {"DBOptionsTest::EnableAutoCompactionAndTriggerStall:3", | |
541 | "DBImpl::BackgroundCompaction():AfterPickCompaction"}}); | |
542 | // Block background compaction. | |
543 | SyncPoint::GetInstance()->EnableProcessing(); | |
544 | ||
545 | switch (method_type) { | |
546 | case 0: | |
547 | ASSERT_OK( | |
548 | dbfull()->SetOptions({{"disable_auto_compactions", "false"}})); | |
549 | break; | |
550 | case 1: | |
551 | ASSERT_OK(dbfull()->EnableAutoCompaction( | |
552 | {dbfull()->DefaultColumnFamily()})); | |
553 | break; | |
554 | } | |
555 | TEST_SYNC_POINT("DBOptionsTest::EnableAutoCompactionAndTriggerStall:1"); | |
556 | // Wait for stall condition recalculate. | |
557 | TEST_SYNC_POINT("DBOptionsTest::EnableAutoCompactionAndTriggerStall:2"); | |
558 | ||
559 | switch (option_type) { | |
560 | case 0: | |
561 | ASSERT_TRUE(dbfull()->TEST_write_controler().IsStopped()); | |
562 | break; | |
563 | case 1: | |
564 | ASSERT_FALSE(dbfull()->TEST_write_controler().IsStopped()); | |
565 | ASSERT_TRUE(dbfull()->TEST_write_controler().NeedsDelay()); | |
566 | break; | |
567 | case 2: | |
568 | ASSERT_TRUE(dbfull()->TEST_write_controler().IsStopped()); | |
569 | break; | |
570 | case 3: | |
571 | ASSERT_FALSE(dbfull()->TEST_write_controler().IsStopped()); | |
572 | ASSERT_TRUE(dbfull()->TEST_write_controler().NeedsDelay()); | |
573 | break; | |
574 | } | |
575 | TEST_SYNC_POINT("DBOptionsTest::EnableAutoCompactionAndTriggerStall:3"); | |
576 | ||
577 | // Background compaction executed. | |
20effc67 | 578 | ASSERT_OK(dbfull()->TEST_WaitForCompact()); |
7c673cae FG |
579 | ASSERT_FALSE(dbfull()->TEST_write_controler().IsStopped()); |
580 | ASSERT_FALSE(dbfull()->TEST_write_controler().NeedsDelay()); | |
581 | } | |
582 | } | |
583 | } | |
584 | ||
585 | TEST_F(DBOptionsTest, SetOptionsMayTriggerCompaction) { | |
586 | Options options; | |
587 | options.create_if_missing = true; | |
588 | options.level0_file_num_compaction_trigger = 1000; | |
589 | options.env = env_; | |
590 | Reopen(options); | |
591 | for (int i = 0; i < 3; i++) { | |
592 | // Need to insert two keys to avoid trivial move. | |
1e59de90 TL |
593 | ASSERT_OK(Put("foo", std::to_string(i))); |
594 | ASSERT_OK(Put("bar", std::to_string(i))); | |
20effc67 | 595 | ASSERT_OK(Flush()); |
7c673cae FG |
596 | } |
597 | ASSERT_EQ("3", FilesPerLevel()); | |
598 | ASSERT_OK( | |
599 | dbfull()->SetOptions({{"level0_file_num_compaction_trigger", "3"}})); | |
20effc67 | 600 | ASSERT_OK(dbfull()->TEST_WaitForCompact()); |
7c673cae FG |
601 | ASSERT_EQ("0,1", FilesPerLevel()); |
602 | } | |
603 | ||
604 | TEST_F(DBOptionsTest, SetBackgroundCompactionThreads) { | |
605 | Options options; | |
606 | options.create_if_missing = true; | |
1e59de90 | 607 | options.max_background_compactions = 1; // default value |
7c673cae FG |
608 | options.env = env_; |
609 | Reopen(options); | |
610 | ASSERT_EQ(1, dbfull()->TEST_BGCompactionsAllowed()); | |
11fdf7f2 TL |
611 | ASSERT_OK(dbfull()->SetDBOptions({{"max_background_compactions", "3"}})); |
612 | ASSERT_EQ(1, dbfull()->TEST_BGCompactionsAllowed()); | |
7c673cae FG |
613 | auto stop_token = dbfull()->TEST_write_controler().GetStopToken(); |
614 | ASSERT_EQ(3, dbfull()->TEST_BGCompactionsAllowed()); | |
615 | } | |
616 | ||
20effc67 TL |
617 | TEST_F(DBOptionsTest, SetBackgroundFlushThreads) { |
618 | Options options; | |
619 | options.create_if_missing = true; | |
620 | options.max_background_flushes = 1; | |
621 | options.env = env_; | |
622 | Reopen(options); | |
623 | ASSERT_EQ(1, dbfull()->TEST_BGFlushesAllowed()); | |
624 | ASSERT_EQ(1, env_->GetBackgroundThreads(Env::Priority::HIGH)); | |
625 | ASSERT_OK(dbfull()->SetDBOptions({{"max_background_flushes", "3"}})); | |
626 | ASSERT_EQ(3, env_->GetBackgroundThreads(Env::Priority::HIGH)); | |
627 | ASSERT_EQ(3, dbfull()->TEST_BGFlushesAllowed()); | |
628 | } | |
629 | ||
11fdf7f2 TL |
630 | TEST_F(DBOptionsTest, SetBackgroundJobs) { |
631 | Options options; | |
632 | options.create_if_missing = true; | |
633 | options.max_background_jobs = 8; | |
634 | options.env = env_; | |
635 | Reopen(options); | |
636 | ||
637 | for (int i = 0; i < 2; ++i) { | |
638 | if (i > 0) { | |
639 | options.max_background_jobs = 12; | |
640 | ASSERT_OK(dbfull()->SetDBOptions( | |
641 | {{"max_background_jobs", | |
642 | std::to_string(options.max_background_jobs)}})); | |
643 | } | |
644 | ||
f67539c2 TL |
645 | const int expected_max_flushes = options.max_background_jobs / 4; |
646 | ||
647 | ASSERT_EQ(expected_max_flushes, dbfull()->TEST_BGFlushesAllowed()); | |
11fdf7f2 TL |
648 | ASSERT_EQ(1, dbfull()->TEST_BGCompactionsAllowed()); |
649 | ||
650 | auto stop_token = dbfull()->TEST_write_controler().GetStopToken(); | |
651 | ||
f67539c2 TL |
652 | const int expected_max_compactions = 3 * expected_max_flushes; |
653 | ||
654 | ASSERT_EQ(expected_max_flushes, dbfull()->TEST_BGFlushesAllowed()); | |
655 | ASSERT_EQ(expected_max_compactions, dbfull()->TEST_BGCompactionsAllowed()); | |
656 | ||
657 | ASSERT_EQ(expected_max_flushes, | |
658 | env_->GetBackgroundThreads(Env::Priority::HIGH)); | |
659 | ASSERT_EQ(expected_max_compactions, | |
660 | env_->GetBackgroundThreads(Env::Priority::LOW)); | |
11fdf7f2 TL |
661 | } |
662 | } | |
663 | ||
7c673cae FG |
664 | TEST_F(DBOptionsTest, AvoidFlushDuringShutdown) { |
665 | Options options; | |
666 | options.create_if_missing = true; | |
667 | options.disable_auto_compactions = true; | |
668 | options.env = env_; | |
669 | WriteOptions write_without_wal; | |
670 | write_without_wal.disableWAL = true; | |
671 | ||
672 | ASSERT_FALSE(options.avoid_flush_during_shutdown); | |
673 | DestroyAndReopen(options); | |
674 | ASSERT_OK(Put("foo", "v1", write_without_wal)); | |
675 | Reopen(options); | |
676 | ASSERT_EQ("v1", Get("foo")); | |
677 | ASSERT_EQ("1", FilesPerLevel()); | |
678 | ||
679 | DestroyAndReopen(options); | |
680 | ASSERT_OK(Put("foo", "v2", write_without_wal)); | |
681 | ASSERT_OK(dbfull()->SetDBOptions({{"avoid_flush_during_shutdown", "true"}})); | |
682 | Reopen(options); | |
683 | ASSERT_EQ("NOT_FOUND", Get("foo")); | |
684 | ASSERT_EQ("", FilesPerLevel()); | |
685 | } | |
686 | ||
687 | TEST_F(DBOptionsTest, SetDelayedWriteRateOption) { | |
688 | Options options; | |
689 | options.create_if_missing = true; | |
690 | options.delayed_write_rate = 2 * 1024U * 1024U; | |
691 | options.env = env_; | |
692 | Reopen(options); | |
1e59de90 TL |
693 | ASSERT_EQ(2 * 1024U * 1024U, |
694 | dbfull()->TEST_write_controler().max_delayed_write_rate()); | |
7c673cae FG |
695 | |
696 | ASSERT_OK(dbfull()->SetDBOptions({{"delayed_write_rate", "20000"}})); | |
697 | ASSERT_EQ(20000, dbfull()->TEST_write_controler().max_delayed_write_rate()); | |
698 | } | |
699 | ||
700 | TEST_F(DBOptionsTest, MaxTotalWalSizeChange) { | |
701 | Random rnd(1044); | |
702 | const auto value_size = size_t(1024); | |
20effc67 | 703 | std::string value = rnd.RandomString(value_size); |
7c673cae FG |
704 | |
705 | Options options; | |
706 | options.create_if_missing = true; | |
707 | options.env = env_; | |
708 | CreateColumnFamilies({"1", "2", "3"}, options); | |
709 | ReopenWithColumnFamilies({"default", "1", "2", "3"}, options); | |
710 | ||
711 | WriteOptions write_options; | |
712 | ||
713 | const int key_count = 100; | |
714 | for (int i = 0; i < key_count; ++i) { | |
715 | for (size_t cf = 0; cf < handles_.size(); ++cf) { | |
716 | ASSERT_OK(Put(static_cast<int>(cf), Key(i), value)); | |
717 | } | |
718 | } | |
719 | ASSERT_OK(dbfull()->SetDBOptions({{"max_total_wal_size", "10"}})); | |
720 | ||
721 | for (size_t cf = 0; cf < handles_.size(); ++cf) { | |
20effc67 | 722 | ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable(handles_[cf])); |
7c673cae FG |
723 | ASSERT_EQ("1", FilesPerLevel(static_cast<int>(cf))); |
724 | } | |
725 | } | |
726 | ||
727 | TEST_F(DBOptionsTest, SetStatsDumpPeriodSec) { | |
728 | Options options; | |
729 | options.create_if_missing = true; | |
730 | options.stats_dump_period_sec = 5; | |
731 | options.env = env_; | |
732 | Reopen(options); | |
f67539c2 | 733 | ASSERT_EQ(5u, dbfull()->GetDBOptions().stats_dump_period_sec); |
7c673cae FG |
734 | |
735 | for (int i = 0; i < 20; i++) { | |
f67539c2 | 736 | unsigned int num = rand() % 5000 + 1; |
1e59de90 TL |
737 | ASSERT_OK(dbfull()->SetDBOptions( |
738 | {{"stats_dump_period_sec", std::to_string(num)}})); | |
7c673cae FG |
739 | ASSERT_EQ(num, dbfull()->GetDBOptions().stats_dump_period_sec); |
740 | } | |
494da23a TL |
741 | Close(); |
742 | } | |
743 | ||
494da23a TL |
744 | TEST_F(DBOptionsTest, SetOptionsStatsPersistPeriodSec) { |
745 | Options options; | |
746 | options.create_if_missing = true; | |
747 | options.stats_persist_period_sec = 5; | |
748 | options.env = env_; | |
749 | Reopen(options); | |
f67539c2 | 750 | ASSERT_EQ(5u, dbfull()->GetDBOptions().stats_persist_period_sec); |
494da23a TL |
751 | |
752 | ASSERT_OK(dbfull()->SetDBOptions({{"stats_persist_period_sec", "12345"}})); | |
f67539c2 | 753 | ASSERT_EQ(12345u, dbfull()->GetDBOptions().stats_persist_period_sec); |
494da23a | 754 | ASSERT_NOK(dbfull()->SetDBOptions({{"stats_persist_period_sec", "abcde"}})); |
f67539c2 | 755 | ASSERT_EQ(12345u, dbfull()->GetDBOptions().stats_persist_period_sec); |
7c673cae FG |
756 | } |
757 | ||
758 | static void assert_candidate_files_empty(DBImpl* dbfull, const bool empty) { | |
759 | dbfull->TEST_LockMutex(); | |
760 | JobContext job_context(0); | |
761 | dbfull->FindObsoleteFiles(&job_context, false); | |
762 | ASSERT_EQ(empty, job_context.full_scan_candidate_files.empty()); | |
7c673cae | 763 | dbfull->TEST_UnlockMutex(); |
11fdf7f2 TL |
764 | if (job_context.HaveSomethingToDelete()) { |
765 | // fulfill the contract of FindObsoleteFiles by calling PurgeObsoleteFiles | |
766 | // afterwards; otherwise the test may hang on shutdown | |
767 | dbfull->PurgeObsoleteFiles(job_context); | |
768 | } | |
769 | job_context.Clean(); | |
7c673cae FG |
770 | } |
771 | ||
772 | TEST_F(DBOptionsTest, DeleteObsoleteFilesPeriodChange) { | |
7c673cae | 773 | Options options; |
20effc67 TL |
774 | options.env = env_; |
775 | SetTimeElapseOnlySleepOnReopen(&options); | |
7c673cae FG |
776 | options.create_if_missing = true; |
777 | ASSERT_OK(TryReopen(options)); | |
778 | ||
779 | // Verify that candidate files set is empty when no full scan requested. | |
780 | assert_candidate_files_empty(dbfull(), true); | |
781 | ||
782 | ASSERT_OK( | |
783 | dbfull()->SetDBOptions({{"delete_obsolete_files_period_micros", "0"}})); | |
784 | ||
785 | // After delete_obsolete_files_period_micros updated to 0, the next call | |
786 | // to FindObsoleteFiles should make a full scan | |
787 | assert_candidate_files_empty(dbfull(), false); | |
788 | ||
789 | ASSERT_OK( | |
790 | dbfull()->SetDBOptions({{"delete_obsolete_files_period_micros", "20"}})); | |
791 | ||
792 | assert_candidate_files_empty(dbfull(), true); | |
793 | ||
20effc67 | 794 | env_->MockSleepForMicroseconds(20); |
7c673cae FG |
795 | assert_candidate_files_empty(dbfull(), true); |
796 | ||
20effc67 | 797 | env_->MockSleepForMicroseconds(1); |
7c673cae FG |
798 | assert_candidate_files_empty(dbfull(), false); |
799 | ||
800 | Close(); | |
801 | } | |
802 | ||
11fdf7f2 TL |
803 | TEST_F(DBOptionsTest, MaxOpenFilesChange) { |
804 | SpecialEnv env(env_); | |
805 | Options options; | |
806 | options.env = CurrentOptions().env; | |
807 | options.max_open_files = -1; | |
808 | ||
809 | Reopen(options); | |
810 | ||
811 | Cache* tc = dbfull()->TEST_table_cache(); | |
812 | ||
813 | ASSERT_EQ(-1, dbfull()->GetDBOptions().max_open_files); | |
814 | ASSERT_LT(2000, tc->GetCapacity()); | |
815 | ASSERT_OK(dbfull()->SetDBOptions({{"max_open_files", "1024"}})); | |
816 | ASSERT_EQ(1024, dbfull()->GetDBOptions().max_open_files); | |
817 | // examine the table cache (actual size should be 1014) | |
818 | ASSERT_GT(1500, tc->GetCapacity()); | |
819 | Close(); | |
820 | } | |
821 | ||
822 | TEST_F(DBOptionsTest, SanitizeDelayedWriteRate) { | |
823 | Options options; | |
20effc67 | 824 | options.env = CurrentOptions().env; |
11fdf7f2 TL |
825 | options.delayed_write_rate = 0; |
826 | Reopen(options); | |
827 | ASSERT_EQ(16 * 1024 * 1024, dbfull()->GetDBOptions().delayed_write_rate); | |
828 | ||
829 | options.rate_limiter.reset(NewGenericRateLimiter(31 * 1024 * 1024)); | |
830 | Reopen(options); | |
831 | ASSERT_EQ(31 * 1024 * 1024, dbfull()->GetDBOptions().delayed_write_rate); | |
832 | } | |
833 | ||
f67539c2 TL |
834 | TEST_F(DBOptionsTest, SanitizeUniversalTTLCompaction) { |
835 | Options options; | |
20effc67 | 836 | options.env = CurrentOptions().env; |
f67539c2 TL |
837 | options.compaction_style = kCompactionStyleUniversal; |
838 | ||
839 | options.ttl = 0; | |
840 | options.periodic_compaction_seconds = 0; | |
841 | Reopen(options); | |
842 | ASSERT_EQ(0, dbfull()->GetOptions().ttl); | |
843 | ASSERT_EQ(0, dbfull()->GetOptions().periodic_compaction_seconds); | |
844 | ||
845 | options.ttl = 0; | |
846 | options.periodic_compaction_seconds = 100; | |
847 | Reopen(options); | |
848 | ASSERT_EQ(0, dbfull()->GetOptions().ttl); | |
849 | ASSERT_EQ(100, dbfull()->GetOptions().periodic_compaction_seconds); | |
850 | ||
851 | options.ttl = 100; | |
852 | options.periodic_compaction_seconds = 0; | |
853 | Reopen(options); | |
854 | ASSERT_EQ(100, dbfull()->GetOptions().ttl); | |
855 | ASSERT_EQ(100, dbfull()->GetOptions().periodic_compaction_seconds); | |
856 | ||
857 | options.ttl = 100; | |
858 | options.periodic_compaction_seconds = 500; | |
859 | Reopen(options); | |
860 | ASSERT_EQ(100, dbfull()->GetOptions().ttl); | |
861 | ASSERT_EQ(100, dbfull()->GetOptions().periodic_compaction_seconds); | |
862 | } | |
863 | ||
864 | TEST_F(DBOptionsTest, SanitizeTtlDefault) { | |
865 | Options options; | |
20effc67 | 866 | options.env = CurrentOptions().env; |
f67539c2 TL |
867 | Reopen(options); |
868 | ASSERT_EQ(30 * 24 * 60 * 60, dbfull()->GetOptions().ttl); | |
869 | ||
870 | options.compaction_style = kCompactionStyleLevel; | |
871 | options.ttl = 0; | |
872 | Reopen(options); | |
873 | ASSERT_EQ(0, dbfull()->GetOptions().ttl); | |
874 | ||
875 | options.ttl = 100; | |
876 | Reopen(options); | |
877 | ASSERT_EQ(100, dbfull()->GetOptions().ttl); | |
878 | } | |
879 | ||
880 | TEST_F(DBOptionsTest, SanitizeFIFOPeriodicCompaction) { | |
881 | Options options; | |
882 | options.compaction_style = kCompactionStyleFIFO; | |
20effc67 | 883 | options.env = CurrentOptions().env; |
f67539c2 TL |
884 | options.ttl = 0; |
885 | Reopen(options); | |
886 | ASSERT_EQ(30 * 24 * 60 * 60, dbfull()->GetOptions().ttl); | |
887 | ||
888 | options.ttl = 100; | |
889 | Reopen(options); | |
890 | ASSERT_EQ(100, dbfull()->GetOptions().ttl); | |
891 | ||
892 | options.ttl = 100 * 24 * 60 * 60; | |
893 | Reopen(options); | |
894 | ASSERT_EQ(100 * 24 * 60 * 60, dbfull()->GetOptions().ttl); | |
895 | ||
896 | options.ttl = 200; | |
897 | options.periodic_compaction_seconds = 300; | |
898 | Reopen(options); | |
899 | ASSERT_EQ(200, dbfull()->GetOptions().ttl); | |
900 | ||
901 | options.ttl = 500; | |
902 | options.periodic_compaction_seconds = 300; | |
903 | Reopen(options); | |
904 | ASSERT_EQ(300, dbfull()->GetOptions().ttl); | |
905 | } | |
906 | ||
11fdf7f2 TL |
907 | TEST_F(DBOptionsTest, SetFIFOCompactionOptions) { |
908 | Options options; | |
20effc67 | 909 | options.env = CurrentOptions().env; |
11fdf7f2 TL |
910 | options.compaction_style = kCompactionStyleFIFO; |
911 | options.write_buffer_size = 10 << 10; // 10KB | |
912 | options.arena_block_size = 4096; | |
913 | options.compression = kNoCompression; | |
914 | options.create_if_missing = true; | |
915 | options.compaction_options_fifo.allow_compaction = false; | |
20effc67 | 916 | env_->SetMockSleep(); |
11fdf7f2 TL |
917 | options.env = env_; |
918 | ||
20effc67 TL |
919 | // NOTE: Presumed unnecessary and removed: resetting mock time in env |
920 | ||
494da23a | 921 | // Test dynamically changing ttl. |
494da23a | 922 | options.ttl = 1 * 60 * 60; // 1 hour |
11fdf7f2 TL |
923 | ASSERT_OK(TryReopen(options)); |
924 | ||
925 | Random rnd(301); | |
926 | for (int i = 0; i < 10; i++) { | |
927 | // Generate and flush a file about 10KB. | |
928 | for (int j = 0; j < 10; j++) { | |
1e59de90 | 929 | ASSERT_OK(Put(std::to_string(i * 20 + j), rnd.RandomString(980))); |
11fdf7f2 | 930 | } |
1e59de90 | 931 | ASSERT_OK(Flush()); |
11fdf7f2 TL |
932 | } |
933 | ASSERT_OK(dbfull()->TEST_WaitForCompact()); | |
934 | ASSERT_EQ(NumTableFilesAtLevel(0), 10); | |
935 | ||
20effc67 | 936 | env_->MockSleepForSeconds(61); |
11fdf7f2 TL |
937 | |
938 | // No files should be compacted as ttl is set to 1 hour. | |
494da23a | 939 | ASSERT_EQ(dbfull()->GetOptions().ttl, 3600); |
20effc67 | 940 | ASSERT_OK(dbfull()->CompactRange(CompactRangeOptions(), nullptr, nullptr)); |
11fdf7f2 TL |
941 | ASSERT_EQ(NumTableFilesAtLevel(0), 10); |
942 | ||
943 | // Set ttl to 1 minute. So all files should get deleted. | |
494da23a TL |
944 | ASSERT_OK(dbfull()->SetOptions({{"ttl", "60"}})); |
945 | ASSERT_EQ(dbfull()->GetOptions().ttl, 60); | |
20effc67 | 946 | ASSERT_OK(dbfull()->CompactRange(CompactRangeOptions(), nullptr, nullptr)); |
11fdf7f2 TL |
947 | ASSERT_OK(dbfull()->TEST_WaitForCompact()); |
948 | ASSERT_EQ(NumTableFilesAtLevel(0), 0); | |
949 | ||
20effc67 TL |
950 | // NOTE: Presumed unnecessary and removed: resetting mock time in env |
951 | ||
11fdf7f2 | 952 | // Test dynamically changing compaction_options_fifo.max_table_files_size |
11fdf7f2 | 953 | options.compaction_options_fifo.max_table_files_size = 500 << 10; // 00KB |
494da23a | 954 | options.ttl = 0; |
11fdf7f2 TL |
955 | DestroyAndReopen(options); |
956 | ||
957 | for (int i = 0; i < 10; i++) { | |
958 | // Generate and flush a file about 10KB. | |
959 | for (int j = 0; j < 10; j++) { | |
1e59de90 | 960 | ASSERT_OK(Put(std::to_string(i * 20 + j), rnd.RandomString(980))); |
11fdf7f2 | 961 | } |
20effc67 | 962 | ASSERT_OK(Flush()); |
11fdf7f2 TL |
963 | } |
964 | ASSERT_OK(dbfull()->TEST_WaitForCompact()); | |
965 | ASSERT_EQ(NumTableFilesAtLevel(0), 10); | |
966 | ||
967 | // No files should be compacted as max_table_files_size is set to 500 KB. | |
968 | ASSERT_EQ(dbfull()->GetOptions().compaction_options_fifo.max_table_files_size, | |
969 | 500 << 10); | |
20effc67 | 970 | ASSERT_OK(dbfull()->CompactRange(CompactRangeOptions(), nullptr, nullptr)); |
11fdf7f2 TL |
971 | ASSERT_EQ(NumTableFilesAtLevel(0), 10); |
972 | ||
973 | // Set max_table_files_size to 12 KB. So only 1 file should remain now. | |
974 | ASSERT_OK(dbfull()->SetOptions( | |
975 | {{"compaction_options_fifo", "{max_table_files_size=12288;}"}})); | |
976 | ASSERT_EQ(dbfull()->GetOptions().compaction_options_fifo.max_table_files_size, | |
977 | 12 << 10); | |
20effc67 | 978 | ASSERT_OK(dbfull()->CompactRange(CompactRangeOptions(), nullptr, nullptr)); |
11fdf7f2 TL |
979 | ASSERT_OK(dbfull()->TEST_WaitForCompact()); |
980 | ASSERT_EQ(NumTableFilesAtLevel(0), 1); | |
981 | ||
982 | // Test dynamically changing compaction_options_fifo.allow_compaction | |
983 | options.compaction_options_fifo.max_table_files_size = 500 << 10; // 500KB | |
494da23a | 984 | options.ttl = 0; |
11fdf7f2 TL |
985 | options.compaction_options_fifo.allow_compaction = false; |
986 | options.level0_file_num_compaction_trigger = 6; | |
987 | DestroyAndReopen(options); | |
988 | ||
989 | for (int i = 0; i < 10; i++) { | |
990 | // Generate and flush a file about 10KB. | |
991 | for (int j = 0; j < 10; j++) { | |
1e59de90 | 992 | ASSERT_OK(Put(std::to_string(i * 20 + j), rnd.RandomString(980))); |
11fdf7f2 | 993 | } |
20effc67 | 994 | ASSERT_OK(Flush()); |
11fdf7f2 TL |
995 | } |
996 | ASSERT_OK(dbfull()->TEST_WaitForCompact()); | |
997 | ASSERT_EQ(NumTableFilesAtLevel(0), 10); | |
998 | ||
999 | // No files should be compacted as max_table_files_size is set to 500 KB and | |
1000 | // allow_compaction is false | |
1001 | ASSERT_EQ(dbfull()->GetOptions().compaction_options_fifo.allow_compaction, | |
1002 | false); | |
20effc67 | 1003 | ASSERT_OK(dbfull()->CompactRange(CompactRangeOptions(), nullptr, nullptr)); |
11fdf7f2 TL |
1004 | ASSERT_EQ(NumTableFilesAtLevel(0), 10); |
1005 | ||
1006 | // Set allow_compaction to true. So number of files should be between 1 and 5. | |
1007 | ASSERT_OK(dbfull()->SetOptions( | |
1008 | {{"compaction_options_fifo", "{allow_compaction=true;}"}})); | |
1009 | ASSERT_EQ(dbfull()->GetOptions().compaction_options_fifo.allow_compaction, | |
1010 | true); | |
20effc67 | 1011 | ASSERT_OK(dbfull()->CompactRange(CompactRangeOptions(), nullptr, nullptr)); |
11fdf7f2 TL |
1012 | ASSERT_OK(dbfull()->TEST_WaitForCompact()); |
1013 | ASSERT_GE(NumTableFilesAtLevel(0), 1); | |
1014 | ASSERT_LE(NumTableFilesAtLevel(0), 5); | |
1015 | } | |
1016 | ||
1017 | TEST_F(DBOptionsTest, CompactionReadaheadSizeChange) { | |
1018 | SpecialEnv env(env_); | |
1019 | Options options; | |
1020 | options.env = &env; | |
1021 | ||
1022 | options.compaction_readahead_size = 0; | |
11fdf7f2 TL |
1023 | options.level0_file_num_compaction_trigger = 2; |
1024 | const std::string kValue(1024, 'v'); | |
1025 | Reopen(options); | |
1026 | ||
1027 | ASSERT_EQ(0, dbfull()->GetDBOptions().compaction_readahead_size); | |
1028 | ASSERT_OK(dbfull()->SetDBOptions({{"compaction_readahead_size", "256"}})); | |
1029 | ASSERT_EQ(256, dbfull()->GetDBOptions().compaction_readahead_size); | |
1030 | for (int i = 0; i < 1024; i++) { | |
20effc67 | 1031 | ASSERT_OK(Put(Key(i), kValue)); |
11fdf7f2 | 1032 | } |
20effc67 | 1033 | ASSERT_OK(Flush()); |
11fdf7f2 | 1034 | for (int i = 0; i < 1024 * 2; i++) { |
20effc67 | 1035 | ASSERT_OK(Put(Key(i), kValue)); |
11fdf7f2 | 1036 | } |
20effc67 TL |
1037 | ASSERT_OK(Flush()); |
1038 | ASSERT_OK(dbfull()->TEST_WaitForCompact()); | |
11fdf7f2 TL |
1039 | ASSERT_EQ(256, env_->compaction_readahead_size_); |
1040 | Close(); | |
1041 | } | |
f67539c2 TL |
1042 | |
1043 | TEST_F(DBOptionsTest, FIFOTtlBackwardCompatible) { | |
1044 | Options options; | |
1045 | options.compaction_style = kCompactionStyleFIFO; | |
1046 | options.write_buffer_size = 10 << 10; // 10KB | |
1047 | options.create_if_missing = true; | |
20effc67 | 1048 | options.env = CurrentOptions().env; |
f67539c2 TL |
1049 | |
1050 | ASSERT_OK(TryReopen(options)); | |
1051 | ||
1052 | Random rnd(301); | |
1053 | for (int i = 0; i < 10; i++) { | |
1054 | // Generate and flush a file about 10KB. | |
1055 | for (int j = 0; j < 10; j++) { | |
1e59de90 | 1056 | ASSERT_OK(Put(std::to_string(i * 20 + j), rnd.RandomString(980))); |
f67539c2 | 1057 | } |
20effc67 | 1058 | ASSERT_OK(Flush()); |
f67539c2 TL |
1059 | } |
1060 | ASSERT_OK(dbfull()->TEST_WaitForCompact()); | |
1061 | ASSERT_EQ(NumTableFilesAtLevel(0), 10); | |
1062 | ||
1063 | // In release 6.0, ttl was promoted from a secondary level option under | |
1064 | // compaction_options_fifo to a top level option under ColumnFamilyOptions. | |
1065 | // We still need to handle old SetOptions calls but should ignore | |
1066 | // ttl under compaction_options_fifo. | |
1067 | ASSERT_OK(dbfull()->SetOptions( | |
1068 | {{"compaction_options_fifo", | |
1069 | "{allow_compaction=true;max_table_files_size=1024;ttl=731;}"}, | |
1070 | {"ttl", "60"}})); | |
1071 | ASSERT_EQ(dbfull()->GetOptions().compaction_options_fifo.allow_compaction, | |
1072 | true); | |
1073 | ASSERT_EQ(dbfull()->GetOptions().compaction_options_fifo.max_table_files_size, | |
1074 | 1024); | |
1075 | ASSERT_EQ(dbfull()->GetOptions().ttl, 60); | |
1076 | ||
1077 | // Put ttl as the first option inside compaction_options_fifo. That works as | |
1078 | // it doesn't overwrite any other option. | |
1079 | ASSERT_OK(dbfull()->SetOptions( | |
1080 | {{"compaction_options_fifo", | |
1081 | "{ttl=985;allow_compaction=true;max_table_files_size=1024;}"}, | |
1082 | {"ttl", "191"}})); | |
1083 | ASSERT_EQ(dbfull()->GetOptions().compaction_options_fifo.allow_compaction, | |
1084 | true); | |
1085 | ASSERT_EQ(dbfull()->GetOptions().compaction_options_fifo.max_table_files_size, | |
1086 | 1024); | |
1087 | ASSERT_EQ(dbfull()->GetOptions().ttl, 191); | |
1088 | } | |
1089 | ||
20effc67 TL |
1090 | TEST_F(DBOptionsTest, ChangeCompression) { |
1091 | if (!Snappy_Supported() || !LZ4_Supported()) { | |
1092 | return; | |
1093 | } | |
1094 | Options options; | |
1095 | options.write_buffer_size = 10 << 10; // 10KB | |
1096 | options.level0_file_num_compaction_trigger = 2; | |
1097 | options.create_if_missing = true; | |
1098 | options.compression = CompressionType::kLZ4Compression; | |
1099 | options.bottommost_compression = CompressionType::kNoCompression; | |
1100 | options.bottommost_compression_opts.level = 2; | |
1101 | options.bottommost_compression_opts.parallel_threads = 1; | |
1102 | options.env = CurrentOptions().env; | |
1103 | ||
1104 | ASSERT_OK(TryReopen(options)); | |
1105 | ||
1106 | CompressionType compression_used = CompressionType::kLZ4Compression; | |
1107 | CompressionOptions compression_opt_used; | |
1108 | bool compacted = false; | |
1109 | SyncPoint::GetInstance()->SetCallBack( | |
1110 | "LevelCompactionPicker::PickCompaction:Return", [&](void* arg) { | |
1111 | Compaction* c = reinterpret_cast<Compaction*>(arg); | |
1112 | compression_used = c->output_compression(); | |
1113 | compression_opt_used = c->output_compression_opts(); | |
1114 | compacted = true; | |
1115 | }); | |
1116 | SyncPoint::GetInstance()->EnableProcessing(); | |
1117 | ||
1118 | ASSERT_OK(Put("foo", "foofoofoo")); | |
1119 | ASSERT_OK(Put("bar", "foofoofoo")); | |
1120 | ASSERT_OK(Flush()); | |
1121 | ASSERT_OK(Put("foo", "foofoofoo")); | |
1122 | ASSERT_OK(Put("bar", "foofoofoo")); | |
1123 | ASSERT_OK(Flush()); | |
1124 | ASSERT_OK(dbfull()->TEST_WaitForCompact()); | |
1125 | ASSERT_TRUE(compacted); | |
1126 | ASSERT_EQ(CompressionType::kNoCompression, compression_used); | |
1127 | ASSERT_EQ(options.compression_opts.level, compression_opt_used.level); | |
1128 | ASSERT_EQ(options.compression_opts.parallel_threads, | |
1129 | compression_opt_used.parallel_threads); | |
1130 | ||
1131 | compression_used = CompressionType::kLZ4Compression; | |
1132 | compacted = false; | |
1133 | ASSERT_OK(dbfull()->SetOptions( | |
1134 | {{"bottommost_compression", "kSnappyCompression"}, | |
1135 | {"bottommost_compression_opts", "0:6:0:0:4:true"}})); | |
1136 | ASSERT_OK(Put("foo", "foofoofoo")); | |
1137 | ASSERT_OK(Put("bar", "foofoofoo")); | |
1138 | ASSERT_OK(Flush()); | |
1139 | ASSERT_OK(Put("foo", "foofoofoo")); | |
1140 | ASSERT_OK(Put("bar", "foofoofoo")); | |
1141 | ASSERT_OK(Flush()); | |
1142 | ASSERT_OK(dbfull()->TEST_WaitForCompact()); | |
1143 | ASSERT_TRUE(compacted); | |
1144 | ASSERT_EQ(CompressionType::kSnappyCompression, compression_used); | |
1145 | ASSERT_EQ(6, compression_opt_used.level); | |
1146 | // Right now parallel_level is not yet allowed to be changed. | |
1147 | ||
1148 | SyncPoint::GetInstance()->DisableProcessing(); | |
1149 | } | |
1150 | ||
7c673cae FG |
1151 | #endif // ROCKSDB_LITE |
1152 | ||
20effc67 TL |
1153 | TEST_F(DBOptionsTest, BottommostCompressionOptsWithFallbackType) { |
1154 | // Verify the bottommost compression options still take effect even when the | |
1155 | // bottommost compression type is left at its default value. Verify for both | |
1156 | // automatic and manual compaction. | |
1157 | if (!Snappy_Supported() || !LZ4_Supported()) { | |
1158 | return; | |
1159 | } | |
1160 | ||
1161 | constexpr int kUpperCompressionLevel = 1; | |
1162 | constexpr int kBottommostCompressionLevel = 2; | |
1163 | constexpr int kNumL0Files = 2; | |
1164 | ||
1165 | Options options = CurrentOptions(); | |
1166 | options.level0_file_num_compaction_trigger = kNumL0Files; | |
1167 | options.compression = CompressionType::kLZ4Compression; | |
1168 | options.compression_opts.level = kUpperCompressionLevel; | |
1169 | options.bottommost_compression_opts.level = kBottommostCompressionLevel; | |
1170 | options.bottommost_compression_opts.enabled = true; | |
1171 | Reopen(options); | |
1172 | ||
1173 | CompressionType compression_used = CompressionType::kDisableCompressionOption; | |
1174 | CompressionOptions compression_opt_used; | |
1175 | bool compacted = false; | |
1176 | SyncPoint::GetInstance()->SetCallBack( | |
1177 | "CompactionPicker::RegisterCompaction:Registered", [&](void* arg) { | |
1178 | Compaction* c = static_cast<Compaction*>(arg); | |
1179 | compression_used = c->output_compression(); | |
1180 | compression_opt_used = c->output_compression_opts(); | |
1181 | compacted = true; | |
1182 | }); | |
1183 | SyncPoint::GetInstance()->EnableProcessing(); | |
1184 | ||
1185 | // First, verify for automatic compaction. | |
1186 | for (int i = 0; i < kNumL0Files; ++i) { | |
1187 | ASSERT_OK(Put("foo", "foofoofoo")); | |
1188 | ASSERT_OK(Put("bar", "foofoofoo")); | |
1189 | ASSERT_OK(Flush()); | |
1190 | } | |
1191 | ASSERT_OK(dbfull()->TEST_WaitForCompact()); | |
1192 | ||
1193 | ASSERT_TRUE(compacted); | |
1194 | ASSERT_EQ(CompressionType::kLZ4Compression, compression_used); | |
1195 | ASSERT_EQ(kBottommostCompressionLevel, compression_opt_used.level); | |
1196 | ||
1197 | // Second, verify for manual compaction. | |
1198 | compacted = false; | |
1199 | compression_used = CompressionType::kDisableCompressionOption; | |
1200 | compression_opt_used = CompressionOptions(); | |
1201 | CompactRangeOptions cro; | |
1202 | cro.bottommost_level_compaction = BottommostLevelCompaction::kForceOptimized; | |
1203 | ASSERT_OK(dbfull()->CompactRange(cro, nullptr, nullptr)); | |
1204 | ||
1205 | ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); | |
1206 | ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->ClearAllCallBacks(); | |
1207 | ||
1208 | ASSERT_TRUE(compacted); | |
1209 | ASSERT_EQ(CompressionType::kLZ4Compression, compression_used); | |
1210 | ASSERT_EQ(kBottommostCompressionLevel, compression_opt_used.level); | |
1211 | } | |
1212 | ||
f67539c2 | 1213 | } // namespace ROCKSDB_NAMESPACE |
7c673cae FG |
1214 | |
1215 | int main(int argc, char** argv) { | |
f67539c2 | 1216 | ROCKSDB_NAMESPACE::port::InstallStackTraceHandler(); |
7c673cae FG |
1217 | ::testing::InitGoogleTest(&argc, argv); |
1218 | return RUN_ALL_TESTS(); | |
1219 | } |