1 // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file. See the AUTHORS file for names of contributors.
9 #include "rocksdb/compaction_filter.h"
10 #include "rocksdb/utilities/db_ttl.h"
11 #include "util/string_util.h"
12 #include "util/testharness.h"
21 typedef std::map
<std::string
, std::string
> KVMap
;
23 enum BatchOperation
{ OP_PUT
= 0, OP_DELETE
= 1 };
26 class SpecialTimeEnv
: public EnvWrapper
{
28 explicit SpecialTimeEnv(Env
* base
) : EnvWrapper(base
) {
29 base
->GetCurrentTime(¤t_time_
);
32 void Sleep(int64_t sleep_time
) { current_time_
+= sleep_time
; }
33 Status
GetCurrentTime(int64_t* current_time
) override
{
34 *current_time
= current_time_
;
39 int64_t current_time_
= 0;
42 class TtlTest
: public testing::Test
{
45 env_
.reset(new SpecialTimeEnv(Env::Default()));
46 dbname_
= test::PerThreadDBPath("db_ttl");
47 options_
.create_if_missing
= true;
48 options_
.env
= env_
.get();
49 // ensure that compaction is kicked in to always strip timestamp from kvs
50 options_
.max_compaction_bytes
= 1;
51 // compaction should take place always from level0 for determinism
53 DestroyDB(dbname_
, Options());
58 DestroyDB(dbname_
, Options());
61 // Open database with TTL support when TTL not provided with db_ttl_ pointer
63 ASSERT_TRUE(db_ttl_
==
64 nullptr); // db should be closed before opening again
65 ASSERT_OK(DBWithTTL::Open(options_
, dbname_
, &db_ttl_
));
68 // Open database with TTL support when TTL provided with db_ttl_ pointer
69 void OpenTtl(int32_t ttl
) {
70 ASSERT_TRUE(db_ttl_
== nullptr);
71 ASSERT_OK(DBWithTTL::Open(options_
, dbname_
, &db_ttl_
, ttl
));
74 // Open with TestFilter compaction filter
75 void OpenTtlWithTestCompaction(int32_t ttl
) {
76 options_
.compaction_filter_factory
=
77 std::shared_ptr
<CompactionFilterFactory
>(
78 new TestFilterFactory(kSampleSize_
, kNewValue_
));
82 // Open database with TTL support in read_only mode
83 void OpenReadOnlyTtl(int32_t ttl
) {
84 ASSERT_TRUE(db_ttl_
== nullptr);
85 ASSERT_OK(DBWithTTL::Open(options_
, dbname_
, &db_ttl_
, ttl
, true));
93 // Populates and returns a kv-map
94 void MakeKVMap(int64_t num_entries
) {
97 for (int64_t dummy
= num_entries
; dummy
/= 10; ++digits
) {
100 for (int64_t i
= 0; i
< num_entries
; i
++) {
101 std::string key
= "key";
102 std::string value
= "value";
106 for(int j
= digits_in_i
; j
< digits
; j
++) {
110 AppendNumberTo(&key
, i
);
111 AppendNumberTo(&value
, i
);
114 ASSERT_EQ(static_cast<int64_t>(kvmap_
.size()),
115 num_entries
); // check all insertions done
118 // Makes a write-batch with key-vals from kvmap_ and 'Write''s it
119 void MakePutWriteBatch(const BatchOperation
* batch_ops
, int64_t num_ops
) {
120 ASSERT_LE(num_ops
, static_cast<int64_t>(kvmap_
.size()));
121 static WriteOptions wopts
;
122 static FlushOptions flush_opts
;
124 kv_it_
= kvmap_
.begin();
125 for (int64_t i
= 0; i
< num_ops
&& kv_it_
!= kvmap_
.end(); i
++, ++kv_it_
) {
126 switch (batch_ops
[i
]) {
128 batch
.Put(kv_it_
->first
, kv_it_
->second
);
131 batch
.Delete(kv_it_
->first
);
137 db_ttl_
->Write(wopts
, &batch
);
138 db_ttl_
->Flush(flush_opts
);
141 // Puts num_entries starting from start_pos_map from kvmap_ into the database
142 void PutValues(int64_t start_pos_map
, int64_t num_entries
, bool flush
= true,
143 ColumnFamilyHandle
* cf
= nullptr) {
144 ASSERT_TRUE(db_ttl_
);
145 ASSERT_LE(start_pos_map
+ num_entries
, static_cast<int64_t>(kvmap_
.size()));
146 static WriteOptions wopts
;
147 static FlushOptions flush_opts
;
148 kv_it_
= kvmap_
.begin();
149 advance(kv_it_
, start_pos_map
);
150 for (int64_t i
= 0; kv_it_
!= kvmap_
.end() && i
< num_entries
;
152 ASSERT_OK(cf
== nullptr
153 ? db_ttl_
->Put(wopts
, kv_it_
->first
, kv_it_
->second
)
154 : db_ttl_
->Put(wopts
, cf
, kv_it_
->first
, kv_it_
->second
));
156 // Put a mock kv at the end because CompactionFilter doesn't delete last key
157 ASSERT_OK(cf
== nullptr ? db_ttl_
->Put(wopts
, "keymock", "valuemock")
158 : db_ttl_
->Put(wopts
, cf
, "keymock", "valuemock"));
161 db_ttl_
->Flush(flush_opts
);
163 db_ttl_
->Flush(flush_opts
, cf
);
168 // Runs a manual compaction
169 void ManualCompact(ColumnFamilyHandle
* cf
= nullptr) {
171 db_ttl_
->CompactRange(CompactRangeOptions(), nullptr, nullptr);
173 db_ttl_
->CompactRange(CompactRangeOptions(), cf
, nullptr, nullptr);
177 // checks the whole kvmap_ to return correct values using KeyMayExist
178 void SimpleKeyMayExistCheck() {
179 static ReadOptions ropts
;
182 for(auto &kv
: kvmap_
) {
183 bool ret
= db_ttl_
->KeyMayExist(ropts
, kv
.first
, &val
, &value_found
);
184 if (ret
== false || value_found
== false) {
185 fprintf(stderr
, "KeyMayExist could not find key=%s in the database but"
186 " should have\n", kv
.first
.c_str());
188 } else if (val
.compare(kv
.second
) != 0) {
189 fprintf(stderr
, " value for key=%s present in database is %s but"
190 " should be %s\n", kv
.first
.c_str(), val
.c_str(),
197 // checks the whole kvmap_ to return correct values using MultiGet
198 void SimpleMultiGetTest() {
199 static ReadOptions ropts
;
200 std::vector
<Slice
> keys
;
201 std::vector
<std::string
> values
;
203 for (auto& kv
: kvmap_
) {
204 keys
.emplace_back(kv
.first
);
207 auto statuses
= db_ttl_
->MultiGet(ropts
, keys
, &values
);
209 for (auto& kv
: kvmap_
) {
210 ASSERT_OK(statuses
[i
]);
211 ASSERT_EQ(values
[i
], kv
.second
);
216 // Sleeps for slp_tim then runs a manual compaction
217 // Checks span starting from st_pos from kvmap_ in the db and
218 // Gets should return true if check is true and false otherwise
219 // Also checks that value that we got is the same as inserted; and =kNewValue
220 // if test_compaction_change is true
221 void SleepCompactCheck(int slp_tim
, int64_t st_pos
, int64_t span
,
222 bool check
= true, bool test_compaction_change
= false,
223 ColumnFamilyHandle
* cf
= nullptr) {
224 ASSERT_TRUE(db_ttl_
);
226 env_
->Sleep(slp_tim
);
228 static ReadOptions ropts
;
229 kv_it_
= kvmap_
.begin();
230 advance(kv_it_
, st_pos
);
232 for (int64_t i
= 0; kv_it_
!= kvmap_
.end() && i
< span
; i
++, ++kv_it_
) {
233 Status s
= (cf
== nullptr) ? db_ttl_
->Get(ropts
, kv_it_
->first
, &v
)
234 : db_ttl_
->Get(ropts
, cf
, kv_it_
->first
, &v
);
235 if (s
.ok() != check
) {
236 fprintf(stderr
, "key=%s ", kv_it_
->first
.c_str());
238 fprintf(stderr
, "is absent from db but was expected to be present\n");
240 fprintf(stderr
, "is present in db but was expected to be absent\n");
244 if (test_compaction_change
&& v
.compare(kNewValue_
) != 0) {
245 fprintf(stderr
, " value for key=%s present in database is %s but "
246 " should be %s\n", kv_it_
->first
.c_str(), v
.c_str(),
249 } else if (!test_compaction_change
&& v
.compare(kv_it_
->second
) !=0) {
250 fprintf(stderr
, " value for key=%s present in database is %s but "
251 " should be %s\n", kv_it_
->first
.c_str(), v
.c_str(),
252 kv_it_
->second
.c_str());
259 // Similar as SleepCompactCheck but uses TtlIterator to read from db
260 void SleepCompactCheckIter(int slp
, int st_pos
, int64_t span
,
262 ASSERT_TRUE(db_ttl_
);
265 static ReadOptions ropts
;
266 Iterator
*dbiter
= db_ttl_
->NewIterator(ropts
);
267 kv_it_
= kvmap_
.begin();
268 advance(kv_it_
, st_pos
);
270 dbiter
->Seek(kv_it_
->first
);
272 if (dbiter
->Valid()) {
273 ASSERT_NE(dbiter
->value().compare(kv_it_
->second
), 0);
275 } else { // dbiter should have found out kvmap_[st_pos]
276 for (int64_t i
= st_pos
; kv_it_
!= kvmap_
.end() && i
< st_pos
+ span
;
278 ASSERT_TRUE(dbiter
->Valid());
279 ASSERT_EQ(dbiter
->value().compare(kv_it_
->second
), 0);
286 // Set ttl on open db
287 void SetTtl(int32_t ttl
, ColumnFamilyHandle
* cf
= nullptr) {
288 ASSERT_TRUE(db_ttl_
);
289 cf
== nullptr ? db_ttl_
->SetTtl(ttl
) : db_ttl_
->SetTtl(cf
, ttl
);
292 class TestFilter
: public CompactionFilter
{
294 TestFilter(const int64_t kSampleSize
, const std::string
& kNewValue
)
295 : kSampleSize_(kSampleSize
),
296 kNewValue_(kNewValue
) {
299 // Works on keys of the form "key<number>"
300 // Drops key if number at the end of key is in [0, kSampleSize_/3),
301 // Keeps key if it is in [kSampleSize_/3, 2*kSampleSize_/3),
302 // Change value if it is in [2*kSampleSize_/3, kSampleSize_)
303 // Eg. kSampleSize_=6. Drop:key0-1...Keep:key2-3...Change:key4-5...
304 bool Filter(int /*level*/, const Slice
& key
, const Slice
& /*value*/,
305 std::string
* new_value
, bool* value_changed
) const override
{
306 assert(new_value
!= nullptr);
308 std::string search_str
= "0123456789";
309 std::string key_string
= key
.ToString();
310 size_t pos
= key_string
.find_first_of(search_str
);
312 if (pos
!= std::string::npos
) {
313 auto key_substr
= key_string
.substr(pos
, key
.size() - pos
);
315 num_key_end
= std::stoi(key_substr
);
317 num_key_end
= std::strtol(key_substr
.c_str(), 0, 10);
321 return false; // Keep keys not matching the format "key<NUMBER>"
324 int64_t partition
= kSampleSize_
/ 3;
325 if (num_key_end
< partition
) {
327 } else if (num_key_end
< partition
* 2) {
330 *new_value
= kNewValue_
;
331 *value_changed
= true;
336 const char* Name() const override
{ return "TestFilter"; }
339 const int64_t kSampleSize_
;
340 const std::string kNewValue_
;
343 class TestFilterFactory
: public CompactionFilterFactory
{
345 TestFilterFactory(const int64_t kSampleSize
, const std::string
& kNewValue
)
346 : kSampleSize_(kSampleSize
),
347 kNewValue_(kNewValue
) {
350 std::unique_ptr
<CompactionFilter
> CreateCompactionFilter(
351 const CompactionFilter::Context
& /*context*/) override
{
352 return std::unique_ptr
<CompactionFilter
>(
353 new TestFilter(kSampleSize_
, kNewValue_
));
356 const char* Name() const override
{ return "TestFilterFactory"; }
359 const int64_t kSampleSize_
;
360 const std::string kNewValue_
;
364 // Choose carefully so that Put, Gets & Compaction complete in 1 second buffer
365 static const int64_t kSampleSize_
= 100;
368 std::unique_ptr
<SpecialTimeEnv
> env_
;
373 KVMap::iterator kv_it_
;
374 const std::string kNewValue_
= "new_value";
375 std::unique_ptr
<CompactionFilter
> test_comp_filter_
;
378 // If TTL is non positive or not provided, the behaviour is TTL = infinity
379 // This test opens the db 3 times with such default behavior and inserts a
380 // bunch of kvs each time. All kvs should accumulate in the db till the end
381 // Partitions the sample-size provided into 3 sets over boundary1 and boundary2
382 TEST_F(TtlTest
, NoEffect
) {
383 MakeKVMap(kSampleSize_
);
384 int64_t boundary1
= kSampleSize_
/ 3;
385 int64_t boundary2
= 2 * boundary1
;
388 PutValues(0, boundary1
); //T=0: Set1 never deleted
389 SleepCompactCheck(1, 0, boundary1
); //T=1: Set1 still there
393 PutValues(boundary1
, boundary2
- boundary1
); //T=1: Set2 never deleted
394 SleepCompactCheck(1, 0, boundary2
); //T=2: Sets1 & 2 still there
398 PutValues(boundary2
, kSampleSize_
- boundary2
); //T=3: Set3 never deleted
399 SleepCompactCheck(1, 0, kSampleSize_
, true); //T=4: Sets 1,2,3 still there
403 // Puts a set of values and checks its presence using Get during ttl
404 TEST_F(TtlTest
, PresentDuringTTL
) {
405 MakeKVMap(kSampleSize_
);
407 OpenTtl(2); // T=0:Open the db with ttl = 2
408 PutValues(0, kSampleSize_
); // T=0:Insert Set1. Delete at t=2
409 SleepCompactCheck(1, 0, kSampleSize_
, true); // T=1:Set1 should still be there
413 // Puts a set of values and checks its absence using Get after ttl
414 TEST_F(TtlTest
, AbsentAfterTTL
) {
415 MakeKVMap(kSampleSize_
);
417 OpenTtl(1); // T=0:Open the db with ttl = 2
418 PutValues(0, kSampleSize_
); // T=0:Insert Set1. Delete at t=2
419 SleepCompactCheck(2, 0, kSampleSize_
, false); // T=2:Set1 should not be there
423 // Resets the timestamp of a set of kvs by updating them and checks that they
424 // are not deleted according to the old timestamp
425 TEST_F(TtlTest
, ResetTimestamp
) {
426 MakeKVMap(kSampleSize_
);
429 PutValues(0, kSampleSize_
); // T=0: Insert Set1. Delete at t=3
430 env_
->Sleep(2); // T=2
431 PutValues(0, kSampleSize_
); // T=2: Insert Set1. Delete at t=5
432 SleepCompactCheck(2, 0, kSampleSize_
); // T=4: Set1 should still be there
436 // Similar to PresentDuringTTL but uses Iterator
437 TEST_F(TtlTest
, IterPresentDuringTTL
) {
438 MakeKVMap(kSampleSize_
);
441 PutValues(0, kSampleSize_
); // T=0: Insert. Delete at t=2
442 SleepCompactCheckIter(1, 0, kSampleSize_
); // T=1: Set should be there
446 // Similar to AbsentAfterTTL but uses Iterator
447 TEST_F(TtlTest
, IterAbsentAfterTTL
) {
448 MakeKVMap(kSampleSize_
);
451 PutValues(0, kSampleSize_
); // T=0: Insert. Delete at t=1
452 SleepCompactCheckIter(2, 0, kSampleSize_
, false); // T=2: Should not be there
456 // Checks presence while opening the same db more than once with the same ttl
457 // Note: The second open will open the same db
458 TEST_F(TtlTest
, MultiOpenSamePresent
) {
459 MakeKVMap(kSampleSize_
);
462 PutValues(0, kSampleSize_
); // T=0: Insert. Delete at t=2
465 OpenTtl(2); // T=0. Delete at t=2
466 SleepCompactCheck(1, 0, kSampleSize_
); // T=1: Set should be there
470 // Checks absence while opening the same db more than once with the same ttl
471 // Note: The second open will open the same db
472 TEST_F(TtlTest
, MultiOpenSameAbsent
) {
473 MakeKVMap(kSampleSize_
);
476 PutValues(0, kSampleSize_
); // T=0: Insert. Delete at t=1
479 OpenTtl(1); // T=0.Delete at t=1
480 SleepCompactCheck(2, 0, kSampleSize_
, false); // T=2: Set should not be there
484 // Checks presence while opening the same db more than once with bigger ttl
485 TEST_F(TtlTest
, MultiOpenDifferent
) {
486 MakeKVMap(kSampleSize_
);
489 PutValues(0, kSampleSize_
); // T=0: Insert. Delete at t=1
492 OpenTtl(3); // T=0: Set deleted at t=3
493 SleepCompactCheck(2, 0, kSampleSize_
); // T=2: Set should be there
497 // Checks presence during ttl in read_only mode
498 TEST_F(TtlTest
, ReadOnlyPresentForever
) {
499 MakeKVMap(kSampleSize_
);
501 OpenTtl(1); // T=0:Open the db normally
502 PutValues(0, kSampleSize_
); // T=0:Insert Set1. Delete at t=1
506 SleepCompactCheck(2, 0, kSampleSize_
); // T=2:Set1 should still be there
510 // Checks whether WriteBatch works well with TTL
511 // Puts all kvs in kvmap_ in a batch and writes first, then deletes first half
512 TEST_F(TtlTest
, WriteBatchTest
) {
513 MakeKVMap(kSampleSize_
);
514 BatchOperation batch_ops
[kSampleSize_
];
515 for (int i
= 0; i
< kSampleSize_
; i
++) {
516 batch_ops
[i
] = OP_PUT
;
520 MakePutWriteBatch(batch_ops
, kSampleSize_
);
521 for (int i
= 0; i
< kSampleSize_
/ 2; i
++) {
522 batch_ops
[i
] = OP_DELETE
;
524 MakePutWriteBatch(batch_ops
, kSampleSize_
/ 2);
525 SleepCompactCheck(0, 0, kSampleSize_
/ 2, false);
526 SleepCompactCheck(0, kSampleSize_
/ 2, kSampleSize_
- kSampleSize_
/ 2);
530 // Checks user's compaction filter for correctness with TTL logic
531 TEST_F(TtlTest
, CompactionFilter
) {
532 MakeKVMap(kSampleSize_
);
534 OpenTtlWithTestCompaction(1);
535 PutValues(0, kSampleSize_
); // T=0:Insert Set1. Delete at t=1
536 // T=2: TTL logic takes precedence over TestFilter:-Set1 should not be there
537 SleepCompactCheck(2, 0, kSampleSize_
, false);
540 OpenTtlWithTestCompaction(3);
541 PutValues(0, kSampleSize_
); // T=0:Insert Set1.
542 int64_t partition
= kSampleSize_
/ 3;
543 SleepCompactCheck(1, 0, partition
, false); // Part dropped
544 SleepCompactCheck(0, partition
, partition
); // Part kept
545 SleepCompactCheck(0, 2 * partition
, partition
, true, true); // Part changed
549 // Insert some key-values which KeyMayExist should be able to get and check that
550 // values returned are fine
551 TEST_F(TtlTest
, KeyMayExist
) {
552 MakeKVMap(kSampleSize_
);
555 PutValues(0, kSampleSize_
, false);
557 SimpleKeyMayExistCheck();
562 TEST_F(TtlTest
, MultiGetTest
) {
563 MakeKVMap(kSampleSize_
);
566 PutValues(0, kSampleSize_
, false);
568 SimpleMultiGetTest();
573 TEST_F(TtlTest
, ColumnFamiliesTest
) {
576 options
.create_if_missing
= true;
577 options
.env
= env_
.get();
579 DB::Open(options
, dbname_
, &db
);
580 ColumnFamilyHandle
* handle
;
581 ASSERT_OK(db
->CreateColumnFamily(ColumnFamilyOptions(options
),
582 "ttl_column_family", &handle
));
587 std::vector
<ColumnFamilyDescriptor
> column_families
;
588 column_families
.push_back(ColumnFamilyDescriptor(
589 kDefaultColumnFamilyName
, ColumnFamilyOptions(options
)));
590 column_families
.push_back(ColumnFamilyDescriptor(
591 "ttl_column_family", ColumnFamilyOptions(options
)));
593 std::vector
<ColumnFamilyHandle
*> handles
;
595 ASSERT_OK(DBWithTTL::Open(DBOptions(options
), dbname_
, column_families
,
596 &handles
, &db_ttl_
, {3, 5}, false));
597 ASSERT_EQ(handles
.size(), 2U);
598 ColumnFamilyHandle
* new_handle
;
599 ASSERT_OK(db_ttl_
->CreateColumnFamilyWithTtl(options
, "ttl_column_family_2",
601 handles
.push_back(new_handle
);
603 MakeKVMap(kSampleSize_
);
604 PutValues(0, kSampleSize_
, false, handles
[0]);
605 PutValues(0, kSampleSize_
, false, handles
[1]);
606 PutValues(0, kSampleSize_
, false, handles
[2]);
608 // everything should be there after 1 second
609 SleepCompactCheck(1, 0, kSampleSize_
, true, false, handles
[0]);
610 SleepCompactCheck(0, 0, kSampleSize_
, true, false, handles
[1]);
611 SleepCompactCheck(0, 0, kSampleSize_
, true, false, handles
[2]);
613 // only column family 1 should be alive after 4 seconds
614 SleepCompactCheck(3, 0, kSampleSize_
, false, false, handles
[0]);
615 SleepCompactCheck(0, 0, kSampleSize_
, true, false, handles
[1]);
616 SleepCompactCheck(0, 0, kSampleSize_
, false, false, handles
[2]);
618 // nothing should be there after 6 seconds
619 SleepCompactCheck(2, 0, kSampleSize_
, false, false, handles
[0]);
620 SleepCompactCheck(0, 0, kSampleSize_
, false, false, handles
[1]);
621 SleepCompactCheck(0, 0, kSampleSize_
, false, false, handles
[2]);
623 for (auto h
: handles
) {
630 // Puts a set of values and checks its absence using Get after ttl
631 TEST_F(TtlTest
, ChangeTtlOnOpenDb
) {
632 MakeKVMap(kSampleSize_
);
634 OpenTtl(1); // T=0:Open the db with ttl = 2
636 // @lint-ignore TXT2 T25377293 Grandfathered in
637 PutValues(0, kSampleSize_
); // T=0:Insert Set1. Delete at t=2
638 SleepCompactCheck(2, 0, kSampleSize_
, true); // T=2:Set1 should be there
642 } // namespace rocksdb
644 // A black-box test for the ttl wrapper around rocksdb
645 int main(int argc
, char** argv
) {
646 ::testing::InitGoogleTest(&argc
, argv
);
647 return RUN_ALL_TESTS();
653 int main(int /*argc*/, char** /*argv*/) {
654 fprintf(stderr
, "SKIPPED as DBWithTTL is not supported in ROCKSDB_LITE\n");
658 #endif // !ROCKSDB_LITE