1 // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file. See the AUTHORS file for names of contributors.
9 #include "rocksdb/compaction_filter.h"
10 #include "rocksdb/utilities/db_ttl.h"
11 #include "util/string_util.h"
12 #include "util/testharness.h"
21 typedef std::map
<std::string
, std::string
> KVMap
;
23 enum BatchOperation
{ OP_PUT
= 0, OP_DELETE
= 1 };
26 class SpecialTimeEnv
: public EnvWrapper
{
28 explicit SpecialTimeEnv(Env
* base
) : EnvWrapper(base
) {
29 base
->GetCurrentTime(¤t_time_
);
32 void Sleep(int64_t sleep_time
) { current_time_
+= sleep_time
; }
33 virtual Status
GetCurrentTime(int64_t* current_time
) override
{
34 *current_time
= current_time_
;
39 int64_t current_time_
;
42 class TtlTest
: public testing::Test
{
45 env_
.reset(new SpecialTimeEnv(Env::Default()));
46 dbname_
= test::TmpDir() + "/db_ttl";
47 options_
.create_if_missing
= true;
48 options_
.env
= env_
.get();
49 // ensure that compaction is kicked in to always strip timestamp from kvs
50 options_
.max_compaction_bytes
= 1;
51 // compaction should take place always from level0 for determinism
53 DestroyDB(dbname_
, Options());
58 DestroyDB(dbname_
, Options());
61 // Open database with TTL support when TTL not provided with db_ttl_ pointer
63 ASSERT_TRUE(db_ttl_
==
64 nullptr); // db should be closed before opening again
65 ASSERT_OK(DBWithTTL::Open(options_
, dbname_
, &db_ttl_
));
68 // Open database with TTL support when TTL provided with db_ttl_ pointer
69 void OpenTtl(int32_t ttl
) {
70 ASSERT_TRUE(db_ttl_
== nullptr);
71 ASSERT_OK(DBWithTTL::Open(options_
, dbname_
, &db_ttl_
, ttl
));
74 // Open with TestFilter compaction filter
75 void OpenTtlWithTestCompaction(int32_t ttl
) {
76 options_
.compaction_filter_factory
=
77 std::shared_ptr
<CompactionFilterFactory
>(
78 new TestFilterFactory(kSampleSize_
, kNewValue_
));
82 // Open database with TTL support in read_only mode
83 void OpenReadOnlyTtl(int32_t ttl
) {
84 ASSERT_TRUE(db_ttl_
== nullptr);
85 ASSERT_OK(DBWithTTL::Open(options_
, dbname_
, &db_ttl_
, ttl
, true));
93 // Populates and returns a kv-map
94 void MakeKVMap(int64_t num_entries
) {
97 for (int64_t dummy
= num_entries
; dummy
/= 10; ++digits
) {
100 for (int64_t i
= 0; i
< num_entries
; i
++) {
101 std::string key
= "key";
102 std::string value
= "value";
106 for(int j
= digits_in_i
; j
< digits
; j
++) {
110 AppendNumberTo(&key
, i
);
111 AppendNumberTo(&value
, i
);
114 ASSERT_EQ(static_cast<int64_t>(kvmap_
.size()),
115 num_entries
); // check all insertions done
118 // Makes a write-batch with key-vals from kvmap_ and 'Write''s it
119 void MakePutWriteBatch(const BatchOperation
* batch_ops
, int64_t num_ops
) {
120 ASSERT_LE(num_ops
, static_cast<int64_t>(kvmap_
.size()));
121 static WriteOptions wopts
;
122 static FlushOptions flush_opts
;
124 kv_it_
= kvmap_
.begin();
125 for (int64_t i
= 0; i
< num_ops
&& kv_it_
!= kvmap_
.end(); i
++, ++kv_it_
) {
126 switch (batch_ops
[i
]) {
128 batch
.Put(kv_it_
->first
, kv_it_
->second
);
131 batch
.Delete(kv_it_
->first
);
137 db_ttl_
->Write(wopts
, &batch
);
138 db_ttl_
->Flush(flush_opts
);
141 // Puts num_entries starting from start_pos_map from kvmap_ into the database
142 void PutValues(int64_t start_pos_map
, int64_t num_entries
, bool flush
= true,
143 ColumnFamilyHandle
* cf
= nullptr) {
144 ASSERT_TRUE(db_ttl_
);
145 ASSERT_LE(start_pos_map
+ num_entries
, static_cast<int64_t>(kvmap_
.size()));
146 static WriteOptions wopts
;
147 static FlushOptions flush_opts
;
148 kv_it_
= kvmap_
.begin();
149 advance(kv_it_
, start_pos_map
);
150 for (int64_t i
= 0; kv_it_
!= kvmap_
.end() && i
< num_entries
;
152 ASSERT_OK(cf
== nullptr
153 ? db_ttl_
->Put(wopts
, kv_it_
->first
, kv_it_
->second
)
154 : db_ttl_
->Put(wopts
, cf
, kv_it_
->first
, kv_it_
->second
));
156 // Put a mock kv at the end because CompactionFilter doesn't delete last key
157 ASSERT_OK(cf
== nullptr ? db_ttl_
->Put(wopts
, "keymock", "valuemock")
158 : db_ttl_
->Put(wopts
, cf
, "keymock", "valuemock"));
161 db_ttl_
->Flush(flush_opts
);
163 db_ttl_
->Flush(flush_opts
, cf
);
168 // Runs a manual compaction
169 void ManualCompact(ColumnFamilyHandle
* cf
= nullptr) {
171 db_ttl_
->CompactRange(CompactRangeOptions(), nullptr, nullptr);
173 db_ttl_
->CompactRange(CompactRangeOptions(), cf
, nullptr, nullptr);
177 // checks the whole kvmap_ to return correct values using KeyMayExist
178 void SimpleKeyMayExistCheck() {
179 static ReadOptions ropts
;
182 for(auto &kv
: kvmap_
) {
183 bool ret
= db_ttl_
->KeyMayExist(ropts
, kv
.first
, &val
, &value_found
);
184 if (ret
== false || value_found
== false) {
185 fprintf(stderr
, "KeyMayExist could not find key=%s in the database but"
186 " should have\n", kv
.first
.c_str());
188 } else if (val
.compare(kv
.second
) != 0) {
189 fprintf(stderr
, " value for key=%s present in database is %s but"
190 " should be %s\n", kv
.first
.c_str(), val
.c_str(),
197 // checks the whole kvmap_ to return correct values using MultiGet
198 void SimpleMultiGetTest() {
199 static ReadOptions ropts
;
200 std::vector
<Slice
> keys
;
201 std::vector
<std::string
> values
;
203 for (auto& kv
: kvmap_
) {
204 keys
.emplace_back(kv
.first
);
207 auto statuses
= db_ttl_
->MultiGet(ropts
, keys
, &values
);
209 for (auto& kv
: kvmap_
) {
210 ASSERT_OK(statuses
[i
]);
211 ASSERT_EQ(values
[i
], kv
.second
);
216 // Sleeps for slp_tim then runs a manual compaction
217 // Checks span starting from st_pos from kvmap_ in the db and
218 // Gets should return true if check is true and false otherwise
219 // Also checks that value that we got is the same as inserted; and =kNewValue
220 // if test_compaction_change is true
221 void SleepCompactCheck(int slp_tim
, int64_t st_pos
, int64_t span
,
222 bool check
= true, bool test_compaction_change
= false,
223 ColumnFamilyHandle
* cf
= nullptr) {
224 ASSERT_TRUE(db_ttl_
);
226 env_
->Sleep(slp_tim
);
228 static ReadOptions ropts
;
229 kv_it_
= kvmap_
.begin();
230 advance(kv_it_
, st_pos
);
232 for (int64_t i
= 0; kv_it_
!= kvmap_
.end() && i
< span
; i
++, ++kv_it_
) {
233 Status s
= (cf
== nullptr) ? db_ttl_
->Get(ropts
, kv_it_
->first
, &v
)
234 : db_ttl_
->Get(ropts
, cf
, kv_it_
->first
, &v
);
235 if (s
.ok() != check
) {
236 fprintf(stderr
, "key=%s ", kv_it_
->first
.c_str());
238 fprintf(stderr
, "is absent from db but was expected to be present\n");
240 fprintf(stderr
, "is present in db but was expected to be absent\n");
244 if (test_compaction_change
&& v
.compare(kNewValue_
) != 0) {
245 fprintf(stderr
, " value for key=%s present in database is %s but "
246 " should be %s\n", kv_it_
->first
.c_str(), v
.c_str(),
249 } else if (!test_compaction_change
&& v
.compare(kv_it_
->second
) !=0) {
250 fprintf(stderr
, " value for key=%s present in database is %s but "
251 " should be %s\n", kv_it_
->first
.c_str(), v
.c_str(),
252 kv_it_
->second
.c_str());
259 // Similar as SleepCompactCheck but uses TtlIterator to read from db
260 void SleepCompactCheckIter(int slp
, int st_pos
, int64_t span
,
262 ASSERT_TRUE(db_ttl_
);
265 static ReadOptions ropts
;
266 Iterator
*dbiter
= db_ttl_
->NewIterator(ropts
);
267 kv_it_
= kvmap_
.begin();
268 advance(kv_it_
, st_pos
);
270 dbiter
->Seek(kv_it_
->first
);
272 if (dbiter
->Valid()) {
273 ASSERT_NE(dbiter
->value().compare(kv_it_
->second
), 0);
275 } else { // dbiter should have found out kvmap_[st_pos]
276 for (int64_t i
= st_pos
; kv_it_
!= kvmap_
.end() && i
< st_pos
+ span
;
278 ASSERT_TRUE(dbiter
->Valid());
279 ASSERT_EQ(dbiter
->value().compare(kv_it_
->second
), 0);
286 class TestFilter
: public CompactionFilter
{
288 TestFilter(const int64_t kSampleSize
, const std::string
& kNewValue
)
289 : kSampleSize_(kSampleSize
),
290 kNewValue_(kNewValue
) {
293 // Works on keys of the form "key<number>"
294 // Drops key if number at the end of key is in [0, kSampleSize_/3),
295 // Keeps key if it is in [kSampleSize_/3, 2*kSampleSize_/3),
296 // Change value if it is in [2*kSampleSize_/3, kSampleSize_)
297 // Eg. kSampleSize_=6. Drop:key0-1...Keep:key2-3...Change:key4-5...
298 virtual bool Filter(int level
, const Slice
& key
,
299 const Slice
& value
, std::string
* new_value
,
300 bool* value_changed
) const override
{
301 assert(new_value
!= nullptr);
303 std::string search_str
= "0123456789";
304 std::string key_string
= key
.ToString();
305 size_t pos
= key_string
.find_first_of(search_str
);
307 if (pos
!= std::string::npos
) {
308 auto key_substr
= key_string
.substr(pos
, key
.size() - pos
);
310 num_key_end
= std::stoi(key_substr
);
312 num_key_end
= std::strtol(key_substr
.c_str(), 0, 10);
316 return false; // Keep keys not matching the format "key<NUMBER>"
319 int64_t partition
= kSampleSize_
/ 3;
320 if (num_key_end
< partition
) {
322 } else if (num_key_end
< partition
* 2) {
325 *new_value
= kNewValue_
;
326 *value_changed
= true;
331 virtual const char* Name() const override
{
336 const int64_t kSampleSize_
;
337 const std::string kNewValue_
;
340 class TestFilterFactory
: public CompactionFilterFactory
{
342 TestFilterFactory(const int64_t kSampleSize
, const std::string
& kNewValue
)
343 : kSampleSize_(kSampleSize
),
344 kNewValue_(kNewValue
) {
347 virtual std::unique_ptr
<CompactionFilter
> CreateCompactionFilter(
348 const CompactionFilter::Context
& context
) override
{
349 return std::unique_ptr
<CompactionFilter
>(
350 new TestFilter(kSampleSize_
, kNewValue_
));
353 virtual const char* Name() const override
{
354 return "TestFilterFactory";
358 const int64_t kSampleSize_
;
359 const std::string kNewValue_
;
363 // Choose carefully so that Put, Gets & Compaction complete in 1 second buffer
364 static const int64_t kSampleSize_
= 100;
367 unique_ptr
<SpecialTimeEnv
> env_
;
372 KVMap::iterator kv_it_
;
373 const std::string kNewValue_
= "new_value";
374 unique_ptr
<CompactionFilter
> test_comp_filter_
;
377 // If TTL is non positive or not provided, the behaviour is TTL = infinity
378 // This test opens the db 3 times with such default behavior and inserts a
379 // bunch of kvs each time. All kvs should accumulate in the db till the end
380 // Partitions the sample-size provided into 3 sets over boundary1 and boundary2
381 TEST_F(TtlTest
, NoEffect
) {
382 MakeKVMap(kSampleSize_
);
383 int64_t boundary1
= kSampleSize_
/ 3;
384 int64_t boundary2
= 2 * boundary1
;
387 PutValues(0, boundary1
); //T=0: Set1 never deleted
388 SleepCompactCheck(1, 0, boundary1
); //T=1: Set1 still there
392 PutValues(boundary1
, boundary2
- boundary1
); //T=1: Set2 never deleted
393 SleepCompactCheck(1, 0, boundary2
); //T=2: Sets1 & 2 still there
397 PutValues(boundary2
, kSampleSize_
- boundary2
); //T=3: Set3 never deleted
398 SleepCompactCheck(1, 0, kSampleSize_
, true); //T=4: Sets 1,2,3 still there
402 // Puts a set of values and checks its presence using Get during ttl
403 TEST_F(TtlTest
, PresentDuringTTL
) {
404 MakeKVMap(kSampleSize_
);
406 OpenTtl(2); // T=0:Open the db with ttl = 2
407 PutValues(0, kSampleSize_
); // T=0:Insert Set1. Delete at t=2
408 SleepCompactCheck(1, 0, kSampleSize_
, true); // T=1:Set1 should still be there
412 // Puts a set of values and checks its absence using Get after ttl
413 TEST_F(TtlTest
, AbsentAfterTTL
) {
414 MakeKVMap(kSampleSize_
);
416 OpenTtl(1); // T=0:Open the db with ttl = 2
417 PutValues(0, kSampleSize_
); // T=0:Insert Set1. Delete at t=2
418 SleepCompactCheck(2, 0, kSampleSize_
, false); // T=2:Set1 should not be there
422 // Resets the timestamp of a set of kvs by updating them and checks that they
423 // are not deleted according to the old timestamp
424 TEST_F(TtlTest
, ResetTimestamp
) {
425 MakeKVMap(kSampleSize_
);
428 PutValues(0, kSampleSize_
); // T=0: Insert Set1. Delete at t=3
429 env_
->Sleep(2); // T=2
430 PutValues(0, kSampleSize_
); // T=2: Insert Set1. Delete at t=5
431 SleepCompactCheck(2, 0, kSampleSize_
); // T=4: Set1 should still be there
435 // Similar to PresentDuringTTL but uses Iterator
436 TEST_F(TtlTest
, IterPresentDuringTTL
) {
437 MakeKVMap(kSampleSize_
);
440 PutValues(0, kSampleSize_
); // T=0: Insert. Delete at t=2
441 SleepCompactCheckIter(1, 0, kSampleSize_
); // T=1: Set should be there
445 // Similar to AbsentAfterTTL but uses Iterator
446 TEST_F(TtlTest
, IterAbsentAfterTTL
) {
447 MakeKVMap(kSampleSize_
);
450 PutValues(0, kSampleSize_
); // T=0: Insert. Delete at t=1
451 SleepCompactCheckIter(2, 0, kSampleSize_
, false); // T=2: Should not be there
455 // Checks presence while opening the same db more than once with the same ttl
456 // Note: The second open will open the same db
457 TEST_F(TtlTest
, MultiOpenSamePresent
) {
458 MakeKVMap(kSampleSize_
);
461 PutValues(0, kSampleSize_
); // T=0: Insert. Delete at t=2
464 OpenTtl(2); // T=0. Delete at t=2
465 SleepCompactCheck(1, 0, kSampleSize_
); // T=1: Set should be there
469 // Checks absence while opening the same db more than once with the same ttl
470 // Note: The second open will open the same db
471 TEST_F(TtlTest
, MultiOpenSameAbsent
) {
472 MakeKVMap(kSampleSize_
);
475 PutValues(0, kSampleSize_
); // T=0: Insert. Delete at t=1
478 OpenTtl(1); // T=0.Delete at t=1
479 SleepCompactCheck(2, 0, kSampleSize_
, false); // T=2: Set should not be there
483 // Checks presence while opening the same db more than once with bigger ttl
484 TEST_F(TtlTest
, MultiOpenDifferent
) {
485 MakeKVMap(kSampleSize_
);
488 PutValues(0, kSampleSize_
); // T=0: Insert. Delete at t=1
491 OpenTtl(3); // T=0: Set deleted at t=3
492 SleepCompactCheck(2, 0, kSampleSize_
); // T=2: Set should be there
496 // Checks presence during ttl in read_only mode
497 TEST_F(TtlTest
, ReadOnlyPresentForever
) {
498 MakeKVMap(kSampleSize_
);
500 OpenTtl(1); // T=0:Open the db normally
501 PutValues(0, kSampleSize_
); // T=0:Insert Set1. Delete at t=1
505 SleepCompactCheck(2, 0, kSampleSize_
); // T=2:Set1 should still be there
509 // Checks whether WriteBatch works well with TTL
510 // Puts all kvs in kvmap_ in a batch and writes first, then deletes first half
511 TEST_F(TtlTest
, WriteBatchTest
) {
512 MakeKVMap(kSampleSize_
);
513 BatchOperation batch_ops
[kSampleSize_
];
514 for (int i
= 0; i
< kSampleSize_
; i
++) {
515 batch_ops
[i
] = OP_PUT
;
519 MakePutWriteBatch(batch_ops
, kSampleSize_
);
520 for (int i
= 0; i
< kSampleSize_
/ 2; i
++) {
521 batch_ops
[i
] = OP_DELETE
;
523 MakePutWriteBatch(batch_ops
, kSampleSize_
/ 2);
524 SleepCompactCheck(0, 0, kSampleSize_
/ 2, false);
525 SleepCompactCheck(0, kSampleSize_
/ 2, kSampleSize_
- kSampleSize_
/ 2);
529 // Checks user's compaction filter for correctness with TTL logic
530 TEST_F(TtlTest
, CompactionFilter
) {
531 MakeKVMap(kSampleSize_
);
533 OpenTtlWithTestCompaction(1);
534 PutValues(0, kSampleSize_
); // T=0:Insert Set1. Delete at t=1
535 // T=2: TTL logic takes precedence over TestFilter:-Set1 should not be there
536 SleepCompactCheck(2, 0, kSampleSize_
, false);
539 OpenTtlWithTestCompaction(3);
540 PutValues(0, kSampleSize_
); // T=0:Insert Set1.
541 int64_t partition
= kSampleSize_
/ 3;
542 SleepCompactCheck(1, 0, partition
, false); // Part dropped
543 SleepCompactCheck(0, partition
, partition
); // Part kept
544 SleepCompactCheck(0, 2 * partition
, partition
, true, true); // Part changed
548 // Insert some key-values which KeyMayExist should be able to get and check that
549 // values returned are fine
550 TEST_F(TtlTest
, KeyMayExist
) {
551 MakeKVMap(kSampleSize_
);
554 PutValues(0, kSampleSize_
, false);
556 SimpleKeyMayExistCheck();
561 TEST_F(TtlTest
, MultiGetTest
) {
562 MakeKVMap(kSampleSize_
);
565 PutValues(0, kSampleSize_
, false);
567 SimpleMultiGetTest();
572 TEST_F(TtlTest
, ColumnFamiliesTest
) {
575 options
.create_if_missing
= true;
576 options
.env
= env_
.get();
578 DB::Open(options
, dbname_
, &db
);
579 ColumnFamilyHandle
* handle
;
580 ASSERT_OK(db
->CreateColumnFamily(ColumnFamilyOptions(options
),
581 "ttl_column_family", &handle
));
586 std::vector
<ColumnFamilyDescriptor
> column_families
;
587 column_families
.push_back(ColumnFamilyDescriptor(
588 kDefaultColumnFamilyName
, ColumnFamilyOptions(options
)));
589 column_families
.push_back(ColumnFamilyDescriptor(
590 "ttl_column_family", ColumnFamilyOptions(options
)));
592 std::vector
<ColumnFamilyHandle
*> handles
;
594 ASSERT_OK(DBWithTTL::Open(DBOptions(options
), dbname_
, column_families
,
595 &handles
, &db_ttl_
, {3, 5}, false));
596 ASSERT_EQ(handles
.size(), 2U);
597 ColumnFamilyHandle
* new_handle
;
598 ASSERT_OK(db_ttl_
->CreateColumnFamilyWithTtl(options
, "ttl_column_family_2",
600 handles
.push_back(new_handle
);
602 MakeKVMap(kSampleSize_
);
603 PutValues(0, kSampleSize_
, false, handles
[0]);
604 PutValues(0, kSampleSize_
, false, handles
[1]);
605 PutValues(0, kSampleSize_
, false, handles
[2]);
607 // everything should be there after 1 second
608 SleepCompactCheck(1, 0, kSampleSize_
, true, false, handles
[0]);
609 SleepCompactCheck(0, 0, kSampleSize_
, true, false, handles
[1]);
610 SleepCompactCheck(0, 0, kSampleSize_
, true, false, handles
[2]);
612 // only column family 1 should be alive after 4 seconds
613 SleepCompactCheck(3, 0, kSampleSize_
, false, false, handles
[0]);
614 SleepCompactCheck(0, 0, kSampleSize_
, true, false, handles
[1]);
615 SleepCompactCheck(0, 0, kSampleSize_
, false, false, handles
[2]);
617 // nothing should be there after 6 seconds
618 SleepCompactCheck(2, 0, kSampleSize_
, false, false, handles
[0]);
619 SleepCompactCheck(0, 0, kSampleSize_
, false, false, handles
[1]);
620 SleepCompactCheck(0, 0, kSampleSize_
, false, false, handles
[2]);
622 for (auto h
: handles
) {
629 } // namespace rocksdb
631 // A black-box test for the ttl wrapper around rocksdb
632 int main(int argc
, char** argv
) {
633 ::testing::InitGoogleTest(&argc
, argv
);
634 return RUN_ALL_TESTS();
640 int main(int argc
, char** argv
) {
641 fprintf(stderr
, "SKIPPED as DBWithTTL is not supported in ROCKSDB_LITE\n");
645 #endif // !ROCKSDB_LITE