1 // Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
2 // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
3 // Use of this source code is governed by a BSD-style license that can be
4 // found in the LICENSE file. See the AUTHORS file for names of contributors.
11 #include "rocksdb/compaction_filter.h"
12 #include "rocksdb/convenience.h"
13 #include "rocksdb/merge_operator.h"
14 #include "rocksdb/utilities/db_ttl.h"
15 #include "rocksdb/utilities/object_registry.h"
16 #include "test_util/testharness.h"
17 #include "util/string_util.h"
18 #include "utilities/merge_operators/bytesxor.h"
19 #include "utilities/ttl/db_ttl_impl.h"
24 namespace ROCKSDB_NAMESPACE
{
28 using KVMap
= std::map
<std::string
, std::string
>;
30 enum BatchOperation
{ OP_PUT
= 0, OP_DELETE
= 1 };
33 class SpecialTimeEnv
: public EnvWrapper
{
35 explicit SpecialTimeEnv(Env
* base
) : EnvWrapper(base
) {
36 EXPECT_OK(base
->GetCurrentTime(¤t_time_
));
38 const char* Name() const override
{ return "SpecialTimeEnv"; }
39 void Sleep(int64_t sleep_time
) { current_time_
+= sleep_time
; }
40 Status
GetCurrentTime(int64_t* current_time
) override
{
41 *current_time
= current_time_
;
46 int64_t current_time_
= 0;
49 class TtlTest
: public testing::Test
{
52 env_
.reset(new SpecialTimeEnv(Env::Default()));
53 dbname_
= test::PerThreadDBPath("db_ttl");
54 options_
.create_if_missing
= true;
55 options_
.env
= env_
.get();
56 // ensure that compaction is kicked in to always strip timestamp from kvs
57 options_
.max_compaction_bytes
= 1;
58 // compaction should take place always from level0 for determinism
60 EXPECT_OK(DestroyDB(dbname_
, Options()));
65 EXPECT_OK(DestroyDB(dbname_
, Options()));
68 // Open database with TTL support when TTL not provided with db_ttl_ pointer
70 ASSERT_TRUE(db_ttl_
==
71 nullptr); // db should be closed before opening again
72 ASSERT_OK(DBWithTTL::Open(options_
, dbname_
, &db_ttl_
));
75 // Open database with TTL support when TTL provided with db_ttl_ pointer
76 void OpenTtl(int32_t ttl
) {
77 ASSERT_TRUE(db_ttl_
== nullptr);
78 ASSERT_OK(DBWithTTL::Open(options_
, dbname_
, &db_ttl_
, ttl
));
81 // Open with TestFilter compaction filter
82 void OpenTtlWithTestCompaction(int32_t ttl
) {
83 options_
.compaction_filter_factory
=
84 std::shared_ptr
<CompactionFilterFactory
>(
85 new TestFilterFactory(kSampleSize_
, kNewValue_
));
89 // Open database with TTL support in read_only mode
90 void OpenReadOnlyTtl(int32_t ttl
) {
91 ASSERT_TRUE(db_ttl_
== nullptr);
92 ASSERT_OK(DBWithTTL::Open(options_
, dbname_
, &db_ttl_
, ttl
, true));
95 // Call db_ttl_->Close() before delete db_ttl_
96 void CloseTtl() { CloseTtlHelper(true); }
98 // No db_ttl_->Close() before delete db_ttl_
99 void CloseTtlNoDBClose() { CloseTtlHelper(false); }
101 void CloseTtlHelper(bool close_db
) {
102 if (db_ttl_
!= nullptr) {
104 EXPECT_OK(db_ttl_
->Close());
111 // Populates and returns a kv-map
112 void MakeKVMap(int64_t num_entries
) {
115 for (int64_t dummy
= num_entries
; dummy
/= 10; ++digits
) {
118 for (int64_t i
= 0; i
< num_entries
; i
++) {
119 std::string key
= "key";
120 std::string value
= "value";
124 for (int j
= digits_in_i
; j
< digits
; j
++) {
128 AppendNumberTo(&key
, i
);
129 AppendNumberTo(&value
, i
);
132 ASSERT_EQ(static_cast<int64_t>(kvmap_
.size()),
133 num_entries
); // check all insertions done
136 // Makes a write-batch with key-vals from kvmap_ and 'Write''s it
137 void MakePutWriteBatch(const BatchOperation
* batch_ops
, int64_t num_ops
) {
138 ASSERT_LE(num_ops
, static_cast<int64_t>(kvmap_
.size()));
139 static WriteOptions wopts
;
140 static FlushOptions flush_opts
;
142 kv_it_
= kvmap_
.begin();
143 for (int64_t i
= 0; i
< num_ops
&& kv_it_
!= kvmap_
.end(); i
++, ++kv_it_
) {
144 switch (batch_ops
[i
]) {
146 ASSERT_OK(batch
.Put(kv_it_
->first
, kv_it_
->second
));
149 ASSERT_OK(batch
.Delete(kv_it_
->first
));
155 ASSERT_OK(db_ttl_
->Write(wopts
, &batch
));
156 ASSERT_OK(db_ttl_
->Flush(flush_opts
));
159 // Puts num_entries starting from start_pos_map from kvmap_ into the database
160 void PutValues(int64_t start_pos_map
, int64_t num_entries
, bool flush
= true,
161 ColumnFamilyHandle
* cf
= nullptr) {
162 ASSERT_TRUE(db_ttl_
);
163 ASSERT_LE(start_pos_map
+ num_entries
, static_cast<int64_t>(kvmap_
.size()));
164 static WriteOptions wopts
;
165 static FlushOptions flush_opts
;
166 kv_it_
= kvmap_
.begin();
167 advance(kv_it_
, start_pos_map
);
168 for (int64_t i
= 0; kv_it_
!= kvmap_
.end() && i
< num_entries
;
170 ASSERT_OK(cf
== nullptr
171 ? db_ttl_
->Put(wopts
, kv_it_
->first
, kv_it_
->second
)
172 : db_ttl_
->Put(wopts
, cf
, kv_it_
->first
, kv_it_
->second
));
174 // Put a mock kv at the end because CompactionFilter doesn't delete last key
175 ASSERT_OK(cf
== nullptr ? db_ttl_
->Put(wopts
, "keymock", "valuemock")
176 : db_ttl_
->Put(wopts
, cf
, "keymock", "valuemock"));
179 ASSERT_OK(db_ttl_
->Flush(flush_opts
));
181 ASSERT_OK(db_ttl_
->Flush(flush_opts
, cf
));
186 // Runs a manual compaction
187 Status
ManualCompact(ColumnFamilyHandle
* cf
= nullptr) {
190 return db_ttl_
->CompactRange(CompactRangeOptions(), nullptr, nullptr);
192 return db_ttl_
->CompactRange(CompactRangeOptions(), cf
, nullptr, nullptr);
196 // Runs a DeleteRange
197 void MakeDeleteRange(std::string start
, std::string end
,
198 ColumnFamilyHandle
* cf
= nullptr) {
199 ASSERT_TRUE(db_ttl_
);
200 static WriteOptions wops
;
202 ASSERT_OK(cf
== nullptr
203 ? wb
.DeleteRange(db_ttl_
->DefaultColumnFamily(), start
, end
)
204 : wb
.DeleteRange(cf
, start
, end
));
205 ASSERT_OK(db_ttl_
->Write(wops
, &wb
));
208 // checks the whole kvmap_ to return correct values using KeyMayExist
209 void SimpleKeyMayExistCheck() {
210 static ReadOptions ropts
;
213 for (auto& kv
: kvmap_
) {
214 bool ret
= db_ttl_
->KeyMayExist(ropts
, kv
.first
, &val
, &value_found
);
215 if (ret
== false || value_found
== false) {
217 "KeyMayExist could not find key=%s in the database but"
221 } else if (val
.compare(kv
.second
) != 0) {
223 " value for key=%s present in database is %s but"
225 kv
.first
.c_str(), val
.c_str(), kv
.second
.c_str());
231 // checks the whole kvmap_ to return correct values using MultiGet
232 void SimpleMultiGetTest() {
233 static ReadOptions ropts
;
234 std::vector
<Slice
> keys
;
235 std::vector
<std::string
> values
;
237 for (auto& kv
: kvmap_
) {
238 keys
.emplace_back(kv
.first
);
241 auto statuses
= db_ttl_
->MultiGet(ropts
, keys
, &values
);
243 for (auto& kv
: kvmap_
) {
244 ASSERT_OK(statuses
[i
]);
245 ASSERT_EQ(values
[i
], kv
.second
);
250 void CompactCheck(int64_t st_pos
, int64_t span
, bool check
= true,
251 bool test_compaction_change
= false,
252 ColumnFamilyHandle
* cf
= nullptr) {
253 static ReadOptions ropts
;
254 kv_it_
= kvmap_
.begin();
255 advance(kv_it_
, st_pos
);
257 for (int64_t i
= 0; kv_it_
!= kvmap_
.end() && i
< span
; i
++, ++kv_it_
) {
258 Status s
= (cf
== nullptr) ? db_ttl_
->Get(ropts
, kv_it_
->first
, &v
)
259 : db_ttl_
->Get(ropts
, cf
, kv_it_
->first
, &v
);
260 if (s
.ok() != check
) {
261 fprintf(stderr
, "key=%s ", kv_it_
->first
.c_str());
263 fprintf(stderr
, "is absent from db but was expected to be present\n");
265 fprintf(stderr
, "is present in db but was expected to be absent\n");
269 if (test_compaction_change
&& v
.compare(kNewValue_
) != 0) {
271 " value for key=%s present in database is %s but "
273 kv_it_
->first
.c_str(), v
.c_str(), kNewValue_
.c_str());
275 } else if (!test_compaction_change
&& v
.compare(kv_it_
->second
) != 0) {
277 " value for key=%s present in database is %s but "
279 kv_it_
->first
.c_str(), v
.c_str(), kv_it_
->second
.c_str());
285 // Sleeps for slp_tim then runs a manual compaction
286 // Checks span starting from st_pos from kvmap_ in the db and
287 // Gets should return true if check is true and false otherwise
288 // Also checks that value that we got is the same as inserted; and =kNewValue
289 // if test_compaction_change is true
290 void SleepCompactCheck(int slp_tim
, int64_t st_pos
, int64_t span
,
291 bool check
= true, bool test_compaction_change
= false,
292 ColumnFamilyHandle
* cf
= nullptr) {
293 ASSERT_TRUE(db_ttl_
);
295 env_
->Sleep(slp_tim
);
296 ASSERT_OK(ManualCompact(cf
));
297 CompactCheck(st_pos
, span
, check
, test_compaction_change
, cf
);
300 // Similar as SleepCompactCheck but uses TtlIterator to read from db
301 void SleepCompactCheckIter(int slp
, int st_pos
, int64_t span
,
303 ASSERT_TRUE(db_ttl_
);
305 ASSERT_OK(ManualCompact());
306 static ReadOptions ropts
;
307 Iterator
* dbiter
= db_ttl_
->NewIterator(ropts
);
308 kv_it_
= kvmap_
.begin();
309 advance(kv_it_
, st_pos
);
311 dbiter
->Seek(kv_it_
->first
);
313 if (dbiter
->Valid()) {
314 ASSERT_NE(dbiter
->value().compare(kv_it_
->second
), 0);
316 } else { // dbiter should have found out kvmap_[st_pos]
317 for (int64_t i
= st_pos
; kv_it_
!= kvmap_
.end() && i
< st_pos
+ span
;
319 ASSERT_TRUE(dbiter
->Valid());
320 ASSERT_EQ(dbiter
->value().compare(kv_it_
->second
), 0);
324 ASSERT_OK(dbiter
->status());
328 // Set ttl on open db
329 void SetTtl(int32_t ttl
, ColumnFamilyHandle
* cf
= nullptr) {
330 ASSERT_TRUE(db_ttl_
);
331 cf
== nullptr ? db_ttl_
->SetTtl(ttl
) : db_ttl_
->SetTtl(cf
, ttl
);
334 class TestFilter
: public CompactionFilter
{
336 TestFilter(const int64_t kSampleSize
, const std::string
& kNewValue
)
337 : kSampleSize_(kSampleSize
), kNewValue_(kNewValue
) {}
339 // Works on keys of the form "key<number>"
340 // Drops key if number at the end of key is in [0, kSampleSize_/3),
341 // Keeps key if it is in [kSampleSize_/3, 2*kSampleSize_/3),
342 // Change value if it is in [2*kSampleSize_/3, kSampleSize_)
343 // Eg. kSampleSize_=6. Drop:key0-1...Keep:key2-3...Change:key4-5...
344 bool Filter(int /*level*/, const Slice
& key
, const Slice
& /*value*/,
345 std::string
* new_value
, bool* value_changed
) const override
{
346 assert(new_value
!= nullptr);
348 std::string search_str
= "0123456789";
349 std::string key_string
= key
.ToString();
350 size_t pos
= key_string
.find_first_of(search_str
);
352 if (pos
!= std::string::npos
) {
353 auto key_substr
= key_string
.substr(pos
, key
.size() - pos
);
355 num_key_end
= std::stoi(key_substr
);
357 num_key_end
= std::strtol(key_substr
.c_str(), 0, 10);
361 return false; // Keep keys not matching the format "key<NUMBER>"
364 int64_t partition
= kSampleSize_
/ 3;
365 if (num_key_end
< partition
) {
367 } else if (num_key_end
< partition
* 2) {
370 *new_value
= kNewValue_
;
371 *value_changed
= true;
376 const char* Name() const override
{ return "TestFilter"; }
379 const int64_t kSampleSize_
;
380 const std::string kNewValue_
;
383 class TestFilterFactory
: public CompactionFilterFactory
{
385 TestFilterFactory(const int64_t kSampleSize
, const std::string
& kNewValue
)
386 : kSampleSize_(kSampleSize
), kNewValue_(kNewValue
) {}
388 std::unique_ptr
<CompactionFilter
> CreateCompactionFilter(
389 const CompactionFilter::Context
& /*context*/) override
{
390 return std::unique_ptr
<CompactionFilter
>(
391 new TestFilter(kSampleSize_
, kNewValue_
));
394 const char* Name() const override
{ return "TestFilterFactory"; }
397 const int64_t kSampleSize_
;
398 const std::string kNewValue_
;
401 // Choose carefully so that Put, Gets & Compaction complete in 1 second buffer
402 static const int64_t kSampleSize_
= 100;
405 std::unique_ptr
<SpecialTimeEnv
> env_
;
410 KVMap::iterator kv_it_
;
411 const std::string kNewValue_
= "new_value";
412 std::unique_ptr
<CompactionFilter
> test_comp_filter_
;
415 // If TTL is non positive or not provided, the behaviour is TTL = infinity
416 // This test opens the db 3 times with such default behavior and inserts a
417 // bunch of kvs each time. All kvs should accumulate in the db till the end
418 // Partitions the sample-size provided into 3 sets over boundary1 and boundary2
419 TEST_F(TtlTest
, NoEffect
) {
420 MakeKVMap(kSampleSize_
);
421 int64_t boundary1
= kSampleSize_
/ 3;
422 int64_t boundary2
= 2 * boundary1
;
425 PutValues(0, boundary1
); // T=0: Set1 never deleted
426 SleepCompactCheck(1, 0, boundary1
); // T=1: Set1 still there
430 PutValues(boundary1
, boundary2
- boundary1
); // T=1: Set2 never deleted
431 SleepCompactCheck(1, 0, boundary2
); // T=2: Sets1 & 2 still there
435 PutValues(boundary2
, kSampleSize_
- boundary2
); // T=3: Set3 never deleted
436 SleepCompactCheck(1, 0, kSampleSize_
, true); // T=4: Sets 1,2,3 still there
440 // Rerun the NoEffect test with a different version of CloseTtl
441 // function, where db is directly deleted without close.
442 TEST_F(TtlTest
, DestructWithoutClose
) {
443 MakeKVMap(kSampleSize_
);
444 int64_t boundary1
= kSampleSize_
/ 3;
445 int64_t boundary2
= 2 * boundary1
;
448 PutValues(0, boundary1
); // T=0: Set1 never deleted
449 SleepCompactCheck(1, 0, boundary1
); // T=1: Set1 still there
453 PutValues(boundary1
, boundary2
- boundary1
); // T=1: Set2 never deleted
454 SleepCompactCheck(1, 0, boundary2
); // T=2: Sets1 & 2 still there
458 PutValues(boundary2
, kSampleSize_
- boundary2
); // T=3: Set3 never deleted
459 SleepCompactCheck(1, 0, kSampleSize_
, true); // T=4: Sets 1,2,3 still there
463 // Puts a set of values and checks its presence using Get during ttl
464 TEST_F(TtlTest
, PresentDuringTTL
) {
465 MakeKVMap(kSampleSize_
);
467 OpenTtl(2); // T=0:Open the db with ttl = 2
468 PutValues(0, kSampleSize_
); // T=0:Insert Set1. Delete at t=2
469 SleepCompactCheck(1, 0, kSampleSize_
,
470 true); // T=1:Set1 should still be there
474 // Puts a set of values and checks its absence using Get after ttl
475 TEST_F(TtlTest
, AbsentAfterTTL
) {
476 MakeKVMap(kSampleSize_
);
478 OpenTtl(1); // T=0:Open the db with ttl = 2
479 PutValues(0, kSampleSize_
); // T=0:Insert Set1. Delete at t=2
480 SleepCompactCheck(2, 0, kSampleSize_
, false); // T=2:Set1 should not be there
484 // Resets the timestamp of a set of kvs by updating them and checks that they
485 // are not deleted according to the old timestamp
486 TEST_F(TtlTest
, ResetTimestamp
) {
487 MakeKVMap(kSampleSize_
);
490 PutValues(0, kSampleSize_
); // T=0: Insert Set1. Delete at t=3
491 env_
->Sleep(2); // T=2
492 PutValues(0, kSampleSize_
); // T=2: Insert Set1. Delete at t=5
493 SleepCompactCheck(2, 0, kSampleSize_
); // T=4: Set1 should still be there
497 // Similar to PresentDuringTTL but uses Iterator
498 TEST_F(TtlTest
, IterPresentDuringTTL
) {
499 MakeKVMap(kSampleSize_
);
502 PutValues(0, kSampleSize_
); // T=0: Insert. Delete at t=2
503 SleepCompactCheckIter(1, 0, kSampleSize_
); // T=1: Set should be there
507 // Similar to AbsentAfterTTL but uses Iterator
508 TEST_F(TtlTest
, IterAbsentAfterTTL
) {
509 MakeKVMap(kSampleSize_
);
512 PutValues(0, kSampleSize_
); // T=0: Insert. Delete at t=1
513 SleepCompactCheckIter(2, 0, kSampleSize_
, false); // T=2: Should not be there
517 // Checks presence while opening the same db more than once with the same ttl
518 // Note: The second open will open the same db
519 TEST_F(TtlTest
, MultiOpenSamePresent
) {
520 MakeKVMap(kSampleSize_
);
523 PutValues(0, kSampleSize_
); // T=0: Insert. Delete at t=2
526 OpenTtl(2); // T=0. Delete at t=2
527 SleepCompactCheck(1, 0, kSampleSize_
); // T=1: Set should be there
531 // Checks absence while opening the same db more than once with the same ttl
532 // Note: The second open will open the same db
533 TEST_F(TtlTest
, MultiOpenSameAbsent
) {
534 MakeKVMap(kSampleSize_
);
537 PutValues(0, kSampleSize_
); // T=0: Insert. Delete at t=1
540 OpenTtl(1); // T=0.Delete at t=1
541 SleepCompactCheck(2, 0, kSampleSize_
, false); // T=2: Set should not be there
545 // Checks presence while opening the same db more than once with bigger ttl
546 TEST_F(TtlTest
, MultiOpenDifferent
) {
547 MakeKVMap(kSampleSize_
);
550 PutValues(0, kSampleSize_
); // T=0: Insert. Delete at t=1
553 OpenTtl(3); // T=0: Set deleted at t=3
554 SleepCompactCheck(2, 0, kSampleSize_
); // T=2: Set should be there
558 // Checks presence during ttl in read_only mode
559 TEST_F(TtlTest
, ReadOnlyPresentForever
) {
560 MakeKVMap(kSampleSize_
);
562 OpenTtl(1); // T=0:Open the db normally
563 PutValues(0, kSampleSize_
); // T=0:Insert Set1. Delete at t=1
567 ASSERT_TRUE(db_ttl_
);
570 Status s
= ManualCompact(); // T=2:Set1 should still be there
571 ASSERT_TRUE(s
.IsNotSupported());
572 CompactCheck(0, kSampleSize_
);
576 // Checks whether WriteBatch works well with TTL
577 // Puts all kvs in kvmap_ in a batch and writes first, then deletes first half
578 TEST_F(TtlTest
, WriteBatchTest
) {
579 MakeKVMap(kSampleSize_
);
580 BatchOperation batch_ops
[kSampleSize_
];
581 for (int i
= 0; i
< kSampleSize_
; i
++) {
582 batch_ops
[i
] = OP_PUT
;
586 MakePutWriteBatch(batch_ops
, kSampleSize_
);
587 for (int i
= 0; i
< kSampleSize_
/ 2; i
++) {
588 batch_ops
[i
] = OP_DELETE
;
590 MakePutWriteBatch(batch_ops
, kSampleSize_
/ 2);
591 SleepCompactCheck(0, 0, kSampleSize_
/ 2, false);
592 SleepCompactCheck(0, kSampleSize_
/ 2, kSampleSize_
- kSampleSize_
/ 2);
596 // Checks user's compaction filter for correctness with TTL logic
597 TEST_F(TtlTest
, CompactionFilter
) {
598 MakeKVMap(kSampleSize_
);
600 OpenTtlWithTestCompaction(1);
601 PutValues(0, kSampleSize_
); // T=0:Insert Set1. Delete at t=1
602 // T=2: TTL logic takes precedence over TestFilter:-Set1 should not be there
603 SleepCompactCheck(2, 0, kSampleSize_
, false);
606 OpenTtlWithTestCompaction(3);
607 PutValues(0, kSampleSize_
); // T=0:Insert Set1.
608 int64_t partition
= kSampleSize_
/ 3;
609 SleepCompactCheck(1, 0, partition
, false); // Part dropped
610 SleepCompactCheck(0, partition
, partition
); // Part kept
611 SleepCompactCheck(0, 2 * partition
, partition
, true, true); // Part changed
615 // Insert some key-values which KeyMayExist should be able to get and check that
616 // values returned are fine
617 TEST_F(TtlTest
, KeyMayExist
) {
618 MakeKVMap(kSampleSize_
);
621 PutValues(0, kSampleSize_
, false);
623 SimpleKeyMayExistCheck();
628 TEST_F(TtlTest
, MultiGetTest
) {
629 MakeKVMap(kSampleSize_
);
632 PutValues(0, kSampleSize_
, false);
634 SimpleMultiGetTest();
639 TEST_F(TtlTest
, ColumnFamiliesTest
) {
642 options
.create_if_missing
= true;
643 options
.env
= env_
.get();
645 DB::Open(options
, dbname_
, &db
);
646 ColumnFamilyHandle
* handle
;
647 ASSERT_OK(db
->CreateColumnFamily(ColumnFamilyOptions(options
),
648 "ttl_column_family", &handle
));
653 std::vector
<ColumnFamilyDescriptor
> column_families
;
654 column_families
.push_back(ColumnFamilyDescriptor(
655 kDefaultColumnFamilyName
, ColumnFamilyOptions(options
)));
656 column_families
.push_back(ColumnFamilyDescriptor(
657 "ttl_column_family", ColumnFamilyOptions(options
)));
659 std::vector
<ColumnFamilyHandle
*> handles
;
661 ASSERT_OK(DBWithTTL::Open(DBOptions(options
), dbname_
, column_families
,
662 &handles
, &db_ttl_
, {3, 5}, false));
663 ASSERT_EQ(handles
.size(), 2U);
664 ColumnFamilyHandle
* new_handle
;
665 ASSERT_OK(db_ttl_
->CreateColumnFamilyWithTtl(options
, "ttl_column_family_2",
667 handles
.push_back(new_handle
);
669 MakeKVMap(kSampleSize_
);
670 PutValues(0, kSampleSize_
, false, handles
[0]);
671 PutValues(0, kSampleSize_
, false, handles
[1]);
672 PutValues(0, kSampleSize_
, false, handles
[2]);
674 // everything should be there after 1 second
675 SleepCompactCheck(1, 0, kSampleSize_
, true, false, handles
[0]);
676 SleepCompactCheck(0, 0, kSampleSize_
, true, false, handles
[1]);
677 SleepCompactCheck(0, 0, kSampleSize_
, true, false, handles
[2]);
679 // only column family 1 should be alive after 4 seconds
680 SleepCompactCheck(3, 0, kSampleSize_
, false, false, handles
[0]);
681 SleepCompactCheck(0, 0, kSampleSize_
, true, false, handles
[1]);
682 SleepCompactCheck(0, 0, kSampleSize_
, false, false, handles
[2]);
684 // nothing should be there after 6 seconds
685 SleepCompactCheck(2, 0, kSampleSize_
, false, false, handles
[0]);
686 SleepCompactCheck(0, 0, kSampleSize_
, false, false, handles
[1]);
687 SleepCompactCheck(0, 0, kSampleSize_
, false, false, handles
[2]);
689 for (auto h
: handles
) {
696 // Puts a set of values and checks its absence using Get after ttl
697 TEST_F(TtlTest
, ChangeTtlOnOpenDb
) {
698 MakeKVMap(kSampleSize_
);
700 OpenTtl(1); // T=0:Open the db with ttl = 2
702 PutValues(0, kSampleSize_
); // T=0:Insert Set1. Delete at t=2
703 SleepCompactCheck(2, 0, kSampleSize_
, true); // T=2:Set1 should be there
707 // Test DeleteRange for DBWithTtl
708 TEST_F(TtlTest
, DeleteRangeTest
) {
710 ASSERT_OK(db_ttl_
->Put(WriteOptions(), "a", "val"));
711 MakeDeleteRange("a", "b");
712 ASSERT_OK(db_ttl_
->Put(WriteOptions(), "c", "val"));
713 MakeDeleteRange("b", "d");
714 ASSERT_OK(db_ttl_
->Put(WriteOptions(), "e", "val"));
715 MakeDeleteRange("d", "e");
716 // first iteration verifies query correctness in memtable, second verifies
717 // query correctness for a single SST file
718 for (int i
= 0; i
< 2; i
++) {
720 ASSERT_OK(db_ttl_
->Flush(FlushOptions()));
723 ASSERT_TRUE(db_ttl_
->Get(ReadOptions(), "a", &value
).IsNotFound());
724 ASSERT_TRUE(db_ttl_
->Get(ReadOptions(), "c", &value
).IsNotFound());
725 ASSERT_OK(db_ttl_
->Get(ReadOptions(), "e", &value
));
730 class DummyFilter
: public CompactionFilter
{
732 bool Filter(int /*level*/, const Slice
& /*key*/, const Slice
& /*value*/,
733 std::string
* /*new_value*/,
734 bool* /*value_changed*/) const override
{
738 const char* Name() const override
{ return kClassName(); }
739 static const char* kClassName() { return "DummyFilter"; }
742 class DummyFilterFactory
: public CompactionFilterFactory
{
744 const char* Name() const override
{ return kClassName(); }
745 static const char* kClassName() { return "DummyFilterFactory"; }
747 std::unique_ptr
<CompactionFilter
> CreateCompactionFilter(
748 const CompactionFilter::Context
&) override
{
749 std::unique_ptr
<CompactionFilter
> f(new DummyFilter());
754 static int RegisterTestObjects(ObjectLibrary
& library
,
755 const std::string
& /*arg*/) {
756 library
.AddFactory
<CompactionFilter
>(
757 "DummyFilter", [](const std::string
& /*uri*/,
758 std::unique_ptr
<CompactionFilter
>* /*guard*/,
759 std::string
* /* errmsg */) {
760 static DummyFilter dummy
;
763 library
.AddFactory
<CompactionFilterFactory
>(
764 "DummyFilterFactory", [](const std::string
& /*uri*/,
765 std::unique_ptr
<CompactionFilterFactory
>* guard
,
766 std::string
* /* errmsg */) {
767 guard
->reset(new DummyFilterFactory());
773 class TtlOptionsTest
: public testing::Test
{
776 config_options_
.registry
->AddLibrary("RegisterTtlObjects",
777 RegisterTtlObjects
, "");
778 config_options_
.registry
->AddLibrary("RegisterTtlTestObjects",
779 RegisterTestObjects
, "");
781 ConfigOptions config_options_
;
784 TEST_F(TtlOptionsTest
, LoadTtlCompactionFilter
) {
785 const CompactionFilter
* filter
= nullptr;
787 ASSERT_OK(CompactionFilter::CreateFromString(
788 config_options_
, TtlCompactionFilter::kClassName(), &filter
));
789 ASSERT_NE(filter
, nullptr);
790 ASSERT_STREQ(filter
->Name(), TtlCompactionFilter::kClassName());
791 auto ttl
= filter
->GetOptions
<int32_t>("TTL");
792 ASSERT_NE(ttl
, nullptr);
794 ASSERT_OK(filter
->ValidateOptions(DBOptions(), ColumnFamilyOptions()));
798 ASSERT_OK(CompactionFilter::CreateFromString(
799 config_options_
, "id=TtlCompactionFilter; ttl=123", &filter
));
800 ASSERT_NE(filter
, nullptr);
801 ttl
= filter
->GetOptions
<int32_t>("TTL");
802 ASSERT_NE(ttl
, nullptr);
803 ASSERT_EQ(*ttl
, 123);
804 ASSERT_OK(filter
->ValidateOptions(DBOptions(), ColumnFamilyOptions()));
808 ASSERT_OK(CompactionFilter::CreateFromString(
810 "id=TtlCompactionFilter; ttl=456; user_filter=DummyFilter;", &filter
));
811 ASSERT_NE(filter
, nullptr);
812 auto inner
= filter
->CheckedCast
<DummyFilter
>();
813 ASSERT_NE(inner
, nullptr);
814 ASSERT_OK(filter
->ValidateOptions(DBOptions(), ColumnFamilyOptions()));
815 std::string mismatch
;
816 std::string opts_str
= filter
->ToString(config_options_
);
817 const CompactionFilter
* copy
= nullptr;
819 CompactionFilter::CreateFromString(config_options_
, opts_str
, ©
));
820 ASSERT_TRUE(filter
->AreEquivalent(config_options_
, copy
, &mismatch
));
825 TEST_F(TtlOptionsTest
, LoadTtlCompactionFilterFactory
) {
826 std::shared_ptr
<CompactionFilterFactory
> cff
;
828 ASSERT_OK(CompactionFilterFactory::CreateFromString(
829 config_options_
, TtlCompactionFilterFactory::kClassName(), &cff
));
830 ASSERT_NE(cff
.get(), nullptr);
831 ASSERT_STREQ(cff
->Name(), TtlCompactionFilterFactory::kClassName());
832 auto ttl
= cff
->GetOptions
<int32_t>("TTL");
833 ASSERT_NE(ttl
, nullptr);
835 ASSERT_OK(cff
->ValidateOptions(DBOptions(), ColumnFamilyOptions()));
837 ASSERT_OK(CompactionFilterFactory::CreateFromString(
838 config_options_
, "id=TtlCompactionFilterFactory; ttl=123", &cff
));
839 ASSERT_NE(cff
.get(), nullptr);
840 ASSERT_STREQ(cff
->Name(), TtlCompactionFilterFactory::kClassName());
841 ttl
= cff
->GetOptions
<int32_t>("TTL");
842 ASSERT_NE(ttl
, nullptr);
843 ASSERT_EQ(*ttl
, 123);
844 ASSERT_OK(cff
->ValidateOptions(DBOptions(), ColumnFamilyOptions()));
846 ASSERT_OK(CompactionFilterFactory::CreateFromString(
848 "id=TtlCompactionFilterFactory; ttl=456; "
849 "user_filter_factory=DummyFilterFactory;",
851 ASSERT_NE(cff
.get(), nullptr);
852 auto filter
= cff
->CreateCompactionFilter(CompactionFilter::Context());
853 ASSERT_NE(filter
.get(), nullptr);
854 auto ttlf
= filter
->CheckedCast
<TtlCompactionFilter
>();
855 ASSERT_EQ(filter
.get(), ttlf
);
856 auto user
= filter
->CheckedCast
<DummyFilter
>();
857 ASSERT_NE(user
, nullptr);
858 ASSERT_OK(cff
->ValidateOptions(DBOptions(), ColumnFamilyOptions()));
860 std::string opts_str
= cff
->ToString(config_options_
);
861 std::string mismatch
;
862 std::shared_ptr
<CompactionFilterFactory
> copy
;
863 ASSERT_OK(CompactionFilterFactory::CreateFromString(config_options_
, opts_str
,
865 ASSERT_TRUE(cff
->AreEquivalent(config_options_
, copy
.get(), &mismatch
));
868 TEST_F(TtlOptionsTest
, LoadTtlMergeOperator
) {
869 std::shared_ptr
<MergeOperator
> mo
;
871 config_options_
.invoke_prepare_options
= false;
872 ASSERT_OK(MergeOperator::CreateFromString(
873 config_options_
, TtlMergeOperator::kClassName(), &mo
));
874 ASSERT_NE(mo
.get(), nullptr);
875 ASSERT_STREQ(mo
->Name(), TtlMergeOperator::kClassName());
876 ASSERT_NOK(mo
->ValidateOptions(DBOptions(), ColumnFamilyOptions()));
878 config_options_
.invoke_prepare_options
= true;
879 ASSERT_OK(MergeOperator::CreateFromString(
880 config_options_
, "id=TtlMergeOperator; user_operator=bytesxor", &mo
));
881 ASSERT_NE(mo
.get(), nullptr);
882 ASSERT_STREQ(mo
->Name(), TtlMergeOperator::kClassName());
883 ASSERT_OK(mo
->ValidateOptions(DBOptions(), ColumnFamilyOptions()));
884 auto ttl_mo
= mo
->CheckedCast
<TtlMergeOperator
>();
885 ASSERT_EQ(mo
.get(), ttl_mo
);
886 auto user
= ttl_mo
->CheckedCast
<BytesXOROperator
>();
887 ASSERT_NE(user
, nullptr);
889 std::string mismatch
;
890 std::string opts_str
= mo
->ToString(config_options_
);
891 std::shared_ptr
<MergeOperator
> copy
;
892 ASSERT_OK(MergeOperator::CreateFromString(config_options_
, opts_str
, ©
));
893 ASSERT_TRUE(mo
->AreEquivalent(config_options_
, copy
.get(), &mismatch
));
895 } // namespace ROCKSDB_NAMESPACE
897 // A black-box test for the ttl wrapper around rocksdb
898 int main(int argc
, char** argv
) {
899 ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
900 ::testing::InitGoogleTest(&argc
, argv
);
901 return RUN_ALL_TESTS();
907 int main(int /*argc*/, char** /*argv*/) {
908 fprintf(stderr
, "SKIPPED as DBWithTTL is not supported in ROCKSDB_LITE\n");
912 #endif // !ROCKSDB_LITE