1 // Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
2 // This source code is licensed under both the GPLv2 (found in the
3 // COPYING file in the root directory) and Apache 2.0 License
4 // (found in the LICENSE.Apache file in the root directory).
9 #include "memtable/stl_wrappers.h"
10 #include "rocksdb/db.h"
11 #include "rocksdb/env.h"
12 #include "util/hash.h"
13 #include "util/kv_map.h"
14 #include "util/string_util.h"
15 #include "util/testharness.h"
16 #include "util/testutil.h"
17 #include "utilities/merge_operators.h"
19 using std::unique_ptr
;
24 static const Comparator
* comparator
;
26 class KVIter
: public Iterator
{
28 explicit KVIter(const stl_wrappers::KVMap
* map
)
29 : map_(map
), iter_(map_
->end()) {}
30 virtual bool Valid() const override
{ return iter_
!= map_
->end(); }
31 virtual void SeekToFirst() override
{ iter_
= map_
->begin(); }
32 virtual void SeekToLast() override
{
36 iter_
= map_
->find(map_
->rbegin()->first
);
39 virtual void Seek(const Slice
& k
) override
{
40 iter_
= map_
->lower_bound(k
.ToString());
42 virtual void SeekForPrev(const Slice
& k
) override
{
43 iter_
= map_
->upper_bound(k
.ToString());
46 virtual void Next() override
{ ++iter_
; }
47 virtual void Prev() override
{
48 if (iter_
== map_
->begin()) {
55 virtual Slice
key() const override
{ return iter_
->first
; }
56 virtual Slice
value() const override
{ return iter_
->second
; }
57 virtual Status
status() const override
{ return Status::OK(); }
60 const stl_wrappers::KVMap
* const map_
;
61 stl_wrappers::KVMap::const_iterator iter_
;
64 void AssertItersEqual(Iterator
* iter1
, Iterator
* iter2
) {
65 ASSERT_EQ(iter1
->Valid(), iter2
->Valid());
67 ASSERT_EQ(iter1
->key().ToString(), iter2
->key().ToString());
68 ASSERT_EQ(iter1
->value().ToString(), iter2
->value().ToString());
72 // Measuring operations on DB (expect to be empty).
73 // source_strings are candidate keys
74 void DoRandomIteraratorTest(DB
* db
, std::vector
<std::string
> source_strings
,
75 Random
* rnd
, int num_writes
, int num_iter_ops
,
76 int num_trigger_flush
) {
77 stl_wrappers::KVMap
map((stl_wrappers::LessOfComparator(comparator
)));
79 for (int i
= 0; i
< num_writes
; i
++) {
80 if (num_trigger_flush
> 0 && i
!= 0 && i
% num_trigger_flush
== 0) {
81 db
->Flush(FlushOptions());
84 int type
= rnd
->Uniform(2);
85 int index
= rnd
->Uniform(static_cast<int>(source_strings
.size()));
86 auto& key
= source_strings
[index
];
91 ASSERT_OK(db
->Put(WriteOptions(), key
, key
));
95 if (map
.find(key
) != map
.end()) {
98 ASSERT_OK(db
->Delete(WriteOptions(), key
));
105 std::unique_ptr
<Iterator
> iter(db
->NewIterator(ReadOptions()));
106 std::unique_ptr
<Iterator
> result_iter(new KVIter(&map
));
108 bool is_valid
= false;
109 for (int i
= 0; i
< num_iter_ops
; i
++) {
110 // Random walk and make sure iter and result_iter returns the
111 // same key and value
112 int type
= rnd
->Uniform(6);
113 ASSERT_OK(iter
->status());
118 result_iter
->SeekToFirst();
123 result_iter
->SeekToLast();
126 // Seek to random key
127 auto key_idx
= rnd
->Uniform(static_cast<int>(source_strings
.size()));
128 auto key
= source_strings
[key_idx
];
130 result_iter
->Seek(key
);
153 auto key_idx
= rnd
->Uniform(static_cast<int>(source_strings
.size()));
154 auto key
= source_strings
[key_idx
];
156 auto status
= db
->Get(ReadOptions(), key
, &result
);
157 if (map
.find(key
) == map
.end()) {
158 ASSERT_TRUE(status
.IsNotFound());
160 ASSERT_EQ(map
[key
], result
);
165 AssertItersEqual(iter
.get(), result_iter
.get());
166 is_valid
= iter
->Valid();
170 class DoubleComparator
: public Comparator
{
172 DoubleComparator() {}
174 virtual const char* Name() const override
{ return "DoubleComparator"; }
176 virtual int Compare(const Slice
& a
, const Slice
& b
) const override
{
178 double da
= std::stod(a
.ToString());
179 double db
= std::stod(b
.ToString());
181 double da
= std::strtod(a
.ToString().c_str(), 0 /* endptr */);
182 double db
= std::strtod(a
.ToString().c_str(), 0 /* endptr */);
186 } else if (da
> db
) {
192 virtual void FindShortestSeparator(std::string
* /*start*/,
193 const Slice
& /*limit*/) const override
{}
195 virtual void FindShortSuccessor(std::string
* /*key*/) const override
{}
198 class HashComparator
: public Comparator
{
202 virtual const char* Name() const override
{ return "HashComparator"; }
204 virtual int Compare(const Slice
& a
, const Slice
& b
) const override
{
205 uint32_t ha
= Hash(a
.data(), a
.size(), 66);
206 uint32_t hb
= Hash(b
.data(), b
.size(), 66);
209 } else if (ha
> hb
) {
215 virtual void FindShortestSeparator(std::string
* /*start*/,
216 const Slice
& /*limit*/) const override
{}
218 virtual void FindShortSuccessor(std::string
* /*key*/) const override
{}
221 class TwoStrComparator
: public Comparator
{
223 TwoStrComparator() {}
225 virtual const char* Name() const override
{ return "TwoStrComparator"; }
227 virtual int Compare(const Slice
& a
, const Slice
& b
) const override
{
228 assert(a
.size() >= 2);
229 assert(b
.size() >= 2);
230 size_t size_a1
= static_cast<size_t>(a
[0]);
231 size_t size_b1
= static_cast<size_t>(b
[0]);
232 size_t size_a2
= static_cast<size_t>(a
[1]);
233 size_t size_b2
= static_cast<size_t>(b
[1]);
234 assert(size_a1
+ size_a2
+ 2 == a
.size());
235 assert(size_b1
+ size_b2
+ 2 == b
.size());
237 Slice a1
= Slice(a
.data() + 2, size_a1
);
238 Slice b1
= Slice(b
.data() + 2, size_b1
);
239 Slice a2
= Slice(a
.data() + 2 + size_a1
, size_a2
);
240 Slice b2
= Slice(b
.data() + 2 + size_b1
, size_b2
);
243 return a1
.compare(b1
);
245 return a2
.compare(b2
);
247 virtual void FindShortestSeparator(std::string
* /*start*/,
248 const Slice
& /*limit*/) const override
{}
250 virtual void FindShortSuccessor(std::string
* /*key*/) const override
{}
254 class ComparatorDBTest
255 : public testing::Test
,
256 virtual public ::testing::WithParamInterface
<uint32_t> {
261 Options last_options_
;
262 std::unique_ptr
<const Comparator
> comparator_guard
;
265 ComparatorDBTest() : env_(Env::Default()), db_(nullptr) {
266 comparator
= BytewiseComparator();
267 dbname_
= test::PerThreadDBPath("comparator_db_test");
268 BlockBasedTableOptions toptions
;
269 toptions
.format_version
= GetParam();
270 last_options_
.table_factory
.reset(
271 rocksdb::NewBlockBasedTableFactory(toptions
));
272 EXPECT_OK(DestroyDB(dbname_
, last_options_
));
275 ~ComparatorDBTest() {
277 EXPECT_OK(DestroyDB(dbname_
, last_options_
));
278 comparator
= BytewiseComparator();
281 DB
* GetDB() { return db_
; }
283 void SetOwnedComparator(const Comparator
* cmp
, bool owner
= true) {
285 comparator_guard
.reset(cmp
);
287 comparator_guard
.reset();
290 last_options_
.comparator
= cmp
;
293 // Return the current option configuration.
294 Options
* GetOptions() { return &last_options_
; }
296 void DestroyAndReopen() {
297 // Destroy using last options
299 ASSERT_OK(TryReopen());
305 ASSERT_OK(DestroyDB(dbname_
, last_options_
));
311 last_options_
.create_if_missing
= true;
313 return DB::Open(last_options_
, dbname_
, &db_
);
317 INSTANTIATE_TEST_CASE_P(FormatDef
, ComparatorDBTest
,
318 testing::Values(test::kDefaultFormatVersion
));
319 INSTANTIATE_TEST_CASE_P(FormatLatest
, ComparatorDBTest
,
320 testing::Values(test::kLatestFormatVersion
));
322 TEST_P(ComparatorDBTest
, Bytewise
) {
323 for (int rand_seed
= 301; rand_seed
< 306; rand_seed
++) {
325 Random
rnd(rand_seed
);
326 DoRandomIteraratorTest(GetDB(),
327 {"a", "b", "c", "d", "e", "f", "g", "h", "i"}, &rnd
,
332 TEST_P(ComparatorDBTest
, SimpleSuffixReverseComparator
) {
333 SetOwnedComparator(new test::SimpleSuffixReverseComparator());
335 for (int rnd_seed
= 301; rnd_seed
< 316; rnd_seed
++) {
336 Options
* opt
= GetOptions();
337 opt
->comparator
= comparator
;
339 Random
rnd(rnd_seed
);
341 std::vector
<std::string
> source_strings
;
342 std::vector
<std::string
> source_prefixes
;
343 // Randomly generate 5 prefixes
344 for (int i
= 0; i
< 5; i
++) {
345 source_prefixes
.push_back(test::RandomHumanReadableString(&rnd
, 8));
347 for (int j
= 0; j
< 20; j
++) {
348 int prefix_index
= rnd
.Uniform(static_cast<int>(source_prefixes
.size()));
349 std::string key
= source_prefixes
[prefix_index
] +
350 test::RandomHumanReadableString(&rnd
, rnd
.Uniform(8));
351 source_strings
.push_back(key
);
354 DoRandomIteraratorTest(GetDB(), source_strings
, &rnd
, 30, 600, 66);
358 TEST_P(ComparatorDBTest
, Uint64Comparator
) {
359 SetOwnedComparator(test::Uint64Comparator(), false /* owner */);
361 for (int rnd_seed
= 301; rnd_seed
< 316; rnd_seed
++) {
362 Options
* opt
= GetOptions();
363 opt
->comparator
= comparator
;
365 Random
rnd(rnd_seed
);
366 Random64
rnd64(rnd_seed
);
368 std::vector
<std::string
> source_strings
;
369 // Randomly generate source keys
370 for (int i
= 0; i
< 100; i
++) {
371 uint64_t r
= rnd64
.Next();
374 memcpy(&str
[0], static_cast<void*>(&r
), 8);
375 source_strings
.push_back(str
);
378 DoRandomIteraratorTest(GetDB(), source_strings
, &rnd
, 200, 1000, 66);
382 TEST_P(ComparatorDBTest
, DoubleComparator
) {
383 SetOwnedComparator(new DoubleComparator());
385 for (int rnd_seed
= 301; rnd_seed
< 316; rnd_seed
++) {
386 Options
* opt
= GetOptions();
387 opt
->comparator
= comparator
;
389 Random
rnd(rnd_seed
);
391 std::vector
<std::string
> source_strings
;
392 // Randomly generate source keys
393 for (int i
= 0; i
< 100; i
++) {
394 uint32_t r
= rnd
.Next();
395 uint32_t divide_order
= rnd
.Uniform(8);
396 double to_divide
= 1.0;
397 for (uint32_t j
= 0; j
< divide_order
; j
++) {
400 source_strings
.push_back(ToString(r
/ to_divide
));
403 DoRandomIteraratorTest(GetDB(), source_strings
, &rnd
, 200, 1000, 66);
407 TEST_P(ComparatorDBTest
, HashComparator
) {
408 SetOwnedComparator(new HashComparator());
410 for (int rnd_seed
= 301; rnd_seed
< 316; rnd_seed
++) {
411 Options
* opt
= GetOptions();
412 opt
->comparator
= comparator
;
414 Random
rnd(rnd_seed
);
416 std::vector
<std::string
> source_strings
;
417 // Randomly generate source keys
418 for (int i
= 0; i
< 100; i
++) {
419 source_strings
.push_back(test::RandomKey(&rnd
, 8));
422 DoRandomIteraratorTest(GetDB(), source_strings
, &rnd
, 200, 1000, 66);
426 TEST_P(ComparatorDBTest
, TwoStrComparator
) {
427 SetOwnedComparator(new TwoStrComparator());
429 for (int rnd_seed
= 301; rnd_seed
< 316; rnd_seed
++) {
430 Options
* opt
= GetOptions();
431 opt
->comparator
= comparator
;
433 Random
rnd(rnd_seed
);
435 std::vector
<std::string
> source_strings
;
436 // Randomly generate source keys
437 for (int i
= 0; i
< 100; i
++) {
439 uint32_t size1
= rnd
.Uniform(8);
440 uint32_t size2
= rnd
.Uniform(8);
441 str
.append(1, static_cast<char>(size1
));
442 str
.append(1, static_cast<char>(size2
));
443 str
.append(test::RandomKey(&rnd
, size1
));
444 str
.append(test::RandomKey(&rnd
, size2
));
445 source_strings
.push_back(str
);
448 DoRandomIteraratorTest(GetDB(), source_strings
, &rnd
, 200, 1000, 66);
452 TEST_P(ComparatorDBTest
, IsSameLengthImmediateSuccessor
) {
457 ASSERT_FALSE(BytewiseComparator()->IsSameLengthImmediateSuccessor(s
, t
));
462 ASSERT_FALSE(BytewiseComparator()->IsSameLengthImmediateSuccessor(s
, t
));
465 // not last byte different
468 ASSERT_FALSE(BytewiseComparator()->IsSameLengthImmediateSuccessor(s
, t
));
474 ASSERT_FALSE(BytewiseComparator()->IsSameLengthImmediateSuccessor(s
, t
));
479 ASSERT_TRUE(BytewiseComparator()->IsSameLengthImmediateSuccessor(s
, t
));
484 ASSERT_FALSE(BytewiseComparator()->IsSameLengthImmediateSuccessor(s
, t
));
487 const char s_array
[] = "\x50\x8a\xac";
488 const char t_array
[] = "\x50\x8a\xad";
491 ASSERT_TRUE(BytewiseComparator()->IsSameLengthImmediateSuccessor(s
, t
));
494 const char s_array
[] = "\x50\x8a\xff";
495 const char t_array
[] = "\x50\x8b\x00";
498 ASSERT_TRUE(BytewiseComparator()->IsSameLengthImmediateSuccessor(s
, t
));
501 const char s_array
[] = "\x50\x8a\xff\xff";
502 const char t_array
[] = "\x50\x8b\x00\x00";
505 ASSERT_TRUE(BytewiseComparator()->IsSameLengthImmediateSuccessor(s
, t
));
508 const char s_array
[] = "\x50\x8a\xff\xff";
509 const char t_array
[] = "\x50\x8b\x00\x01";
512 ASSERT_FALSE(BytewiseComparator()->IsSameLengthImmediateSuccessor(s
, t
));
516 TEST_P(ComparatorDBTest
, FindShortestSeparator
) {
517 std::string s1
= "abc1xyz";
518 std::string s2
= "abc3xy";
520 BytewiseComparator()->FindShortestSeparator(&s1
, s2
);
521 ASSERT_EQ("abc2", s1
);
525 ReverseBytewiseComparator()->FindShortestSeparator(&s1
, s2
);
526 ASSERT_EQ("abc5", s1
);
530 ReverseBytewiseComparator()->FindShortestSeparator(&s1
, s2
);
531 ASSERT_EQ("abc3", s1
);
535 ReverseBytewiseComparator()->FindShortestSeparator(&s1
, s2
);
536 ASSERT_EQ("abc3", s1
);
540 ReverseBytewiseComparator()->FindShortestSeparator(&s1
, s2
);
541 ASSERT_EQ("abc3", s1
);
543 std::string old_s1
= s1
= "abc2xy";
545 ReverseBytewiseComparator()->FindShortestSeparator(&s1
, s2
);
546 ASSERT_TRUE(old_s1
>= s1
);
547 ASSERT_TRUE(s1
> s2
);
550 TEST_P(ComparatorDBTest
, SeparatorSuccessorRandomizeTest
) {
551 // Char list for boundary cases.
552 std::array
<unsigned char, 6> char_list
{{0, 1, 2, 253, 254, 255}};
555 for (int attempts
= 0; attempts
< 1000; attempts
++) {
556 uint32_t size1
= rnd
.Skewed(4);
560 // size2 to be random size
561 size2
= rnd
.Skewed(4);
563 // size1 is within [-2, +2] of size1
564 int diff
= static_cast<int>(rnd
.Uniform(5)) - 2;
565 int tmp_size2
= static_cast<int>(size1
) + diff
;
569 size2
= static_cast<uint32_t>(tmp_size2
);
574 for (uint32_t i
= 0; i
< size1
; i
++) {
577 s1
+= static_cast<char>(rnd
.Uniform(256));
579 // Use one byte in char_list
580 char c
= static_cast<char>(char_list
[rnd
.Uniform(sizeof(char_list
))]);
585 // First set s2 to be the same as s1, and then modify s2.
588 // We start from the back of the string
590 uint32_t pos
= size2
- 1;
592 if (pos
>= size1
|| rnd
.OneIn(4)) {
593 // For 1/4 chance, use random byte
594 s2
[pos
] = static_cast<char>(rnd
.Uniform(256));
595 } else if (rnd
.OneIn(4)) {
596 // In 1/4 chance, stop here.
599 // Create a char within [-2, +2] of the matching char of s1.
600 int diff
= static_cast<int>(rnd
.Uniform(5)) - 2;
601 // char may be signed or unsigned based on platform.
602 int s1_char
= static_cast<int>(static_cast<unsigned char>(s1
[pos
]));
603 int s2_char
= s1_char
+ diff
;
610 s2
[pos
] = static_cast<char>(s2_char
);
612 } while (pos
-- != 0);
616 for (int rev
= 0; rev
< 2; rev
++) {
623 std::string separator
= s1
;
624 BytewiseComparator()->FindShortestSeparator(&separator
, s2
);
625 std::string rev_separator
= s1
;
626 ReverseBytewiseComparator()->FindShortestSeparator(&rev_separator
, s2
);
629 ASSERT_EQ(s1
, separator
);
630 ASSERT_EQ(s2
, rev_separator
);
631 } else if (s1
< s2
) {
632 ASSERT_TRUE(s1
<= separator
);
633 ASSERT_TRUE(s2
> separator
);
634 ASSERT_LE(separator
.size(), std::max(s1
.size(), s2
.size()));
635 ASSERT_EQ(s1
, rev_separator
);
637 ASSERT_TRUE(s1
>= rev_separator
);
638 ASSERT_TRUE(s2
< rev_separator
);
639 ASSERT_LE(rev_separator
.size(), std::max(s1
.size(), s2
.size()));
640 ASSERT_EQ(s1
, separator
);
645 std::string succ
= s1
;
646 BytewiseComparator()->FindShortSuccessor(&succ
);
647 ASSERT_TRUE(succ
>= s1
);
650 ReverseBytewiseComparator()->FindShortSuccessor(&succ
);
651 ASSERT_TRUE(succ
<= s1
);
655 } // namespace rocksdb
657 int main(int argc
, char** argv
) {
658 ::testing::InitGoogleTest(&argc
, argv
);
659 return RUN_ALL_TESTS();