]> git.proxmox.com Git - ceph.git/blob - ceph/src/rocksdb/db/comparator_db_test.cc
update sources to ceph Nautilus 14.2.1
[ceph.git] / ceph / src / rocksdb / db / comparator_db_test.cc
1 // Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
2 // This source code is licensed under both the GPLv2 (found in the
3 // COPYING file in the root directory) and Apache 2.0 License
4 // (found in the LICENSE.Apache file in the root directory).
5 #include <array>
6 #include <map>
7 #include <string>
8
9 #include "memtable/stl_wrappers.h"
10 #include "rocksdb/db.h"
11 #include "rocksdb/env.h"
12 #include "util/hash.h"
13 #include "util/kv_map.h"
14 #include "util/string_util.h"
15 #include "util/testharness.h"
16 #include "util/testutil.h"
17 #include "utilities/merge_operators.h"
18
19 using std::unique_ptr;
20
21 namespace rocksdb {
22 namespace {
23
24 static const Comparator* comparator;
25
26 class KVIter : public Iterator {
27 public:
28 explicit KVIter(const stl_wrappers::KVMap* map)
29 : map_(map), iter_(map_->end()) {}
30 virtual bool Valid() const override { return iter_ != map_->end(); }
31 virtual void SeekToFirst() override { iter_ = map_->begin(); }
32 virtual void SeekToLast() override {
33 if (map_->empty()) {
34 iter_ = map_->end();
35 } else {
36 iter_ = map_->find(map_->rbegin()->first);
37 }
38 }
39 virtual void Seek(const Slice& k) override {
40 iter_ = map_->lower_bound(k.ToString());
41 }
42 virtual void SeekForPrev(const Slice& k) override {
43 iter_ = map_->upper_bound(k.ToString());
44 Prev();
45 }
46 virtual void Next() override { ++iter_; }
47 virtual void Prev() override {
48 if (iter_ == map_->begin()) {
49 iter_ = map_->end();
50 return;
51 }
52 --iter_;
53 }
54
55 virtual Slice key() const override { return iter_->first; }
56 virtual Slice value() const override { return iter_->second; }
57 virtual Status status() const override { return Status::OK(); }
58
59 private:
60 const stl_wrappers::KVMap* const map_;
61 stl_wrappers::KVMap::const_iterator iter_;
62 };
63
64 void AssertItersEqual(Iterator* iter1, Iterator* iter2) {
65 ASSERT_EQ(iter1->Valid(), iter2->Valid());
66 if (iter1->Valid()) {
67 ASSERT_EQ(iter1->key().ToString(), iter2->key().ToString());
68 ASSERT_EQ(iter1->value().ToString(), iter2->value().ToString());
69 }
70 }
71
72 // Measuring operations on DB (expect to be empty).
73 // source_strings are candidate keys
74 void DoRandomIteraratorTest(DB* db, std::vector<std::string> source_strings,
75 Random* rnd, int num_writes, int num_iter_ops,
76 int num_trigger_flush) {
77 stl_wrappers::KVMap map((stl_wrappers::LessOfComparator(comparator)));
78
79 for (int i = 0; i < num_writes; i++) {
80 if (num_trigger_flush > 0 && i != 0 && i % num_trigger_flush == 0) {
81 db->Flush(FlushOptions());
82 }
83
84 int type = rnd->Uniform(2);
85 int index = rnd->Uniform(static_cast<int>(source_strings.size()));
86 auto& key = source_strings[index];
87 switch (type) {
88 case 0:
89 // put
90 map[key] = key;
91 ASSERT_OK(db->Put(WriteOptions(), key, key));
92 break;
93 case 1:
94 // delete
95 if (map.find(key) != map.end()) {
96 map.erase(key);
97 }
98 ASSERT_OK(db->Delete(WriteOptions(), key));
99 break;
100 default:
101 assert(false);
102 }
103 }
104
105 std::unique_ptr<Iterator> iter(db->NewIterator(ReadOptions()));
106 std::unique_ptr<Iterator> result_iter(new KVIter(&map));
107
108 bool is_valid = false;
109 for (int i = 0; i < num_iter_ops; i++) {
110 // Random walk and make sure iter and result_iter returns the
111 // same key and value
112 int type = rnd->Uniform(6);
113 ASSERT_OK(iter->status());
114 switch (type) {
115 case 0:
116 // Seek to First
117 iter->SeekToFirst();
118 result_iter->SeekToFirst();
119 break;
120 case 1:
121 // Seek to last
122 iter->SeekToLast();
123 result_iter->SeekToLast();
124 break;
125 case 2: {
126 // Seek to random key
127 auto key_idx = rnd->Uniform(static_cast<int>(source_strings.size()));
128 auto key = source_strings[key_idx];
129 iter->Seek(key);
130 result_iter->Seek(key);
131 break;
132 }
133 case 3:
134 // Next
135 if (is_valid) {
136 iter->Next();
137 result_iter->Next();
138 } else {
139 continue;
140 }
141 break;
142 case 4:
143 // Prev
144 if (is_valid) {
145 iter->Prev();
146 result_iter->Prev();
147 } else {
148 continue;
149 }
150 break;
151 default: {
152 assert(type == 5);
153 auto key_idx = rnd->Uniform(static_cast<int>(source_strings.size()));
154 auto key = source_strings[key_idx];
155 std::string result;
156 auto status = db->Get(ReadOptions(), key, &result);
157 if (map.find(key) == map.end()) {
158 ASSERT_TRUE(status.IsNotFound());
159 } else {
160 ASSERT_EQ(map[key], result);
161 }
162 break;
163 }
164 }
165 AssertItersEqual(iter.get(), result_iter.get());
166 is_valid = iter->Valid();
167 }
168 }
169
170 class DoubleComparator : public Comparator {
171 public:
172 DoubleComparator() {}
173
174 virtual const char* Name() const override { return "DoubleComparator"; }
175
176 virtual int Compare(const Slice& a, const Slice& b) const override {
177 #ifndef CYGWIN
178 double da = std::stod(a.ToString());
179 double db = std::stod(b.ToString());
180 #else
181 double da = std::strtod(a.ToString().c_str(), 0 /* endptr */);
182 double db = std::strtod(a.ToString().c_str(), 0 /* endptr */);
183 #endif
184 if (da == db) {
185 return a.compare(b);
186 } else if (da > db) {
187 return 1;
188 } else {
189 return -1;
190 }
191 }
192 virtual void FindShortestSeparator(std::string* /*start*/,
193 const Slice& /*limit*/) const override {}
194
195 virtual void FindShortSuccessor(std::string* /*key*/) const override {}
196 };
197
198 class HashComparator : public Comparator {
199 public:
200 HashComparator() {}
201
202 virtual const char* Name() const override { return "HashComparator"; }
203
204 virtual int Compare(const Slice& a, const Slice& b) const override {
205 uint32_t ha = Hash(a.data(), a.size(), 66);
206 uint32_t hb = Hash(b.data(), b.size(), 66);
207 if (ha == hb) {
208 return a.compare(b);
209 } else if (ha > hb) {
210 return 1;
211 } else {
212 return -1;
213 }
214 }
215 virtual void FindShortestSeparator(std::string* /*start*/,
216 const Slice& /*limit*/) const override {}
217
218 virtual void FindShortSuccessor(std::string* /*key*/) const override {}
219 };
220
221 class TwoStrComparator : public Comparator {
222 public:
223 TwoStrComparator() {}
224
225 virtual const char* Name() const override { return "TwoStrComparator"; }
226
227 virtual int Compare(const Slice& a, const Slice& b) const override {
228 assert(a.size() >= 2);
229 assert(b.size() >= 2);
230 size_t size_a1 = static_cast<size_t>(a[0]);
231 size_t size_b1 = static_cast<size_t>(b[0]);
232 size_t size_a2 = static_cast<size_t>(a[1]);
233 size_t size_b2 = static_cast<size_t>(b[1]);
234 assert(size_a1 + size_a2 + 2 == a.size());
235 assert(size_b1 + size_b2 + 2 == b.size());
236
237 Slice a1 = Slice(a.data() + 2, size_a1);
238 Slice b1 = Slice(b.data() + 2, size_b1);
239 Slice a2 = Slice(a.data() + 2 + size_a1, size_a2);
240 Slice b2 = Slice(b.data() + 2 + size_b1, size_b2);
241
242 if (a1 != b1) {
243 return a1.compare(b1);
244 }
245 return a2.compare(b2);
246 }
247 virtual void FindShortestSeparator(std::string* /*start*/,
248 const Slice& /*limit*/) const override {}
249
250 virtual void FindShortSuccessor(std::string* /*key*/) const override {}
251 };
252 } // namespace
253
254 class ComparatorDBTest
255 : public testing::Test,
256 virtual public ::testing::WithParamInterface<uint32_t> {
257 private:
258 std::string dbname_;
259 Env* env_;
260 DB* db_;
261 Options last_options_;
262 std::unique_ptr<const Comparator> comparator_guard;
263
264 public:
265 ComparatorDBTest() : env_(Env::Default()), db_(nullptr) {
266 comparator = BytewiseComparator();
267 dbname_ = test::PerThreadDBPath("comparator_db_test");
268 BlockBasedTableOptions toptions;
269 toptions.format_version = GetParam();
270 last_options_.table_factory.reset(
271 rocksdb::NewBlockBasedTableFactory(toptions));
272 EXPECT_OK(DestroyDB(dbname_, last_options_));
273 }
274
275 ~ComparatorDBTest() {
276 delete db_;
277 EXPECT_OK(DestroyDB(dbname_, last_options_));
278 comparator = BytewiseComparator();
279 }
280
281 DB* GetDB() { return db_; }
282
283 void SetOwnedComparator(const Comparator* cmp, bool owner = true) {
284 if (owner) {
285 comparator_guard.reset(cmp);
286 } else {
287 comparator_guard.reset();
288 }
289 comparator = cmp;
290 last_options_.comparator = cmp;
291 }
292
293 // Return the current option configuration.
294 Options* GetOptions() { return &last_options_; }
295
296 void DestroyAndReopen() {
297 // Destroy using last options
298 Destroy();
299 ASSERT_OK(TryReopen());
300 }
301
302 void Destroy() {
303 delete db_;
304 db_ = nullptr;
305 ASSERT_OK(DestroyDB(dbname_, last_options_));
306 }
307
308 Status TryReopen() {
309 delete db_;
310 db_ = nullptr;
311 last_options_.create_if_missing = true;
312
313 return DB::Open(last_options_, dbname_, &db_);
314 }
315 };
316
317 INSTANTIATE_TEST_CASE_P(FormatDef, ComparatorDBTest,
318 testing::Values(test::kDefaultFormatVersion));
319 INSTANTIATE_TEST_CASE_P(FormatLatest, ComparatorDBTest,
320 testing::Values(test::kLatestFormatVersion));
321
322 TEST_P(ComparatorDBTest, Bytewise) {
323 for (int rand_seed = 301; rand_seed < 306; rand_seed++) {
324 DestroyAndReopen();
325 Random rnd(rand_seed);
326 DoRandomIteraratorTest(GetDB(),
327 {"a", "b", "c", "d", "e", "f", "g", "h", "i"}, &rnd,
328 8, 100, 3);
329 }
330 }
331
332 TEST_P(ComparatorDBTest, SimpleSuffixReverseComparator) {
333 SetOwnedComparator(new test::SimpleSuffixReverseComparator());
334
335 for (int rnd_seed = 301; rnd_seed < 316; rnd_seed++) {
336 Options* opt = GetOptions();
337 opt->comparator = comparator;
338 DestroyAndReopen();
339 Random rnd(rnd_seed);
340
341 std::vector<std::string> source_strings;
342 std::vector<std::string> source_prefixes;
343 // Randomly generate 5 prefixes
344 for (int i = 0; i < 5; i++) {
345 source_prefixes.push_back(test::RandomHumanReadableString(&rnd, 8));
346 }
347 for (int j = 0; j < 20; j++) {
348 int prefix_index = rnd.Uniform(static_cast<int>(source_prefixes.size()));
349 std::string key = source_prefixes[prefix_index] +
350 test::RandomHumanReadableString(&rnd, rnd.Uniform(8));
351 source_strings.push_back(key);
352 }
353
354 DoRandomIteraratorTest(GetDB(), source_strings, &rnd, 30, 600, 66);
355 }
356 }
357
358 TEST_P(ComparatorDBTest, Uint64Comparator) {
359 SetOwnedComparator(test::Uint64Comparator(), false /* owner */);
360
361 for (int rnd_seed = 301; rnd_seed < 316; rnd_seed++) {
362 Options* opt = GetOptions();
363 opt->comparator = comparator;
364 DestroyAndReopen();
365 Random rnd(rnd_seed);
366 Random64 rnd64(rnd_seed);
367
368 std::vector<std::string> source_strings;
369 // Randomly generate source keys
370 for (int i = 0; i < 100; i++) {
371 uint64_t r = rnd64.Next();
372 std::string str;
373 str.resize(8);
374 memcpy(&str[0], static_cast<void*>(&r), 8);
375 source_strings.push_back(str);
376 }
377
378 DoRandomIteraratorTest(GetDB(), source_strings, &rnd, 200, 1000, 66);
379 }
380 }
381
382 TEST_P(ComparatorDBTest, DoubleComparator) {
383 SetOwnedComparator(new DoubleComparator());
384
385 for (int rnd_seed = 301; rnd_seed < 316; rnd_seed++) {
386 Options* opt = GetOptions();
387 opt->comparator = comparator;
388 DestroyAndReopen();
389 Random rnd(rnd_seed);
390
391 std::vector<std::string> source_strings;
392 // Randomly generate source keys
393 for (int i = 0; i < 100; i++) {
394 uint32_t r = rnd.Next();
395 uint32_t divide_order = rnd.Uniform(8);
396 double to_divide = 1.0;
397 for (uint32_t j = 0; j < divide_order; j++) {
398 to_divide *= 10.0;
399 }
400 source_strings.push_back(ToString(r / to_divide));
401 }
402
403 DoRandomIteraratorTest(GetDB(), source_strings, &rnd, 200, 1000, 66);
404 }
405 }
406
407 TEST_P(ComparatorDBTest, HashComparator) {
408 SetOwnedComparator(new HashComparator());
409
410 for (int rnd_seed = 301; rnd_seed < 316; rnd_seed++) {
411 Options* opt = GetOptions();
412 opt->comparator = comparator;
413 DestroyAndReopen();
414 Random rnd(rnd_seed);
415
416 std::vector<std::string> source_strings;
417 // Randomly generate source keys
418 for (int i = 0; i < 100; i++) {
419 source_strings.push_back(test::RandomKey(&rnd, 8));
420 }
421
422 DoRandomIteraratorTest(GetDB(), source_strings, &rnd, 200, 1000, 66);
423 }
424 }
425
426 TEST_P(ComparatorDBTest, TwoStrComparator) {
427 SetOwnedComparator(new TwoStrComparator());
428
429 for (int rnd_seed = 301; rnd_seed < 316; rnd_seed++) {
430 Options* opt = GetOptions();
431 opt->comparator = comparator;
432 DestroyAndReopen();
433 Random rnd(rnd_seed);
434
435 std::vector<std::string> source_strings;
436 // Randomly generate source keys
437 for (int i = 0; i < 100; i++) {
438 std::string str;
439 uint32_t size1 = rnd.Uniform(8);
440 uint32_t size2 = rnd.Uniform(8);
441 str.append(1, static_cast<char>(size1));
442 str.append(1, static_cast<char>(size2));
443 str.append(test::RandomKey(&rnd, size1));
444 str.append(test::RandomKey(&rnd, size2));
445 source_strings.push_back(str);
446 }
447
448 DoRandomIteraratorTest(GetDB(), source_strings, &rnd, 200, 1000, 66);
449 }
450 }
451
452 TEST_P(ComparatorDBTest, IsSameLengthImmediateSuccessor) {
453 {
454 // different length
455 Slice s("abcxy");
456 Slice t("abcxyz");
457 ASSERT_FALSE(BytewiseComparator()->IsSameLengthImmediateSuccessor(s, t));
458 }
459 {
460 Slice s("abcxyz");
461 Slice t("abcxy");
462 ASSERT_FALSE(BytewiseComparator()->IsSameLengthImmediateSuccessor(s, t));
463 }
464 {
465 // not last byte different
466 Slice s("abc1xyz");
467 Slice t("abc2xyz");
468 ASSERT_FALSE(BytewiseComparator()->IsSameLengthImmediateSuccessor(s, t));
469 }
470 {
471 // same string
472 Slice s("abcxyz");
473 Slice t("abcxyz");
474 ASSERT_FALSE(BytewiseComparator()->IsSameLengthImmediateSuccessor(s, t));
475 }
476 {
477 Slice s("abcxy");
478 Slice t("abcxz");
479 ASSERT_TRUE(BytewiseComparator()->IsSameLengthImmediateSuccessor(s, t));
480 }
481 {
482 Slice s("abcxz");
483 Slice t("abcxy");
484 ASSERT_FALSE(BytewiseComparator()->IsSameLengthImmediateSuccessor(s, t));
485 }
486 {
487 const char s_array[] = "\x50\x8a\xac";
488 const char t_array[] = "\x50\x8a\xad";
489 Slice s(s_array);
490 Slice t(t_array);
491 ASSERT_TRUE(BytewiseComparator()->IsSameLengthImmediateSuccessor(s, t));
492 }
493 {
494 const char s_array[] = "\x50\x8a\xff";
495 const char t_array[] = "\x50\x8b\x00";
496 Slice s(s_array, 3);
497 Slice t(t_array, 3);
498 ASSERT_TRUE(BytewiseComparator()->IsSameLengthImmediateSuccessor(s, t));
499 }
500 {
501 const char s_array[] = "\x50\x8a\xff\xff";
502 const char t_array[] = "\x50\x8b\x00\x00";
503 Slice s(s_array, 4);
504 Slice t(t_array, 4);
505 ASSERT_TRUE(BytewiseComparator()->IsSameLengthImmediateSuccessor(s, t));
506 }
507 {
508 const char s_array[] = "\x50\x8a\xff\xff";
509 const char t_array[] = "\x50\x8b\x00\x01";
510 Slice s(s_array, 4);
511 Slice t(t_array, 4);
512 ASSERT_FALSE(BytewiseComparator()->IsSameLengthImmediateSuccessor(s, t));
513 }
514 }
515
516 TEST_P(ComparatorDBTest, FindShortestSeparator) {
517 std::string s1 = "abc1xyz";
518 std::string s2 = "abc3xy";
519
520 BytewiseComparator()->FindShortestSeparator(&s1, s2);
521 ASSERT_EQ("abc2", s1);
522
523 s1 = "abc5xyztt";
524
525 ReverseBytewiseComparator()->FindShortestSeparator(&s1, s2);
526 ASSERT_EQ("abc5", s1);
527
528 s1 = "abc3";
529 s2 = "abc2xy";
530 ReverseBytewiseComparator()->FindShortestSeparator(&s1, s2);
531 ASSERT_EQ("abc3", s1);
532
533 s1 = "abc3xyz";
534 s2 = "abc2xy";
535 ReverseBytewiseComparator()->FindShortestSeparator(&s1, s2);
536 ASSERT_EQ("abc3", s1);
537
538 s1 = "abc3xyz";
539 s2 = "abc2";
540 ReverseBytewiseComparator()->FindShortestSeparator(&s1, s2);
541 ASSERT_EQ("abc3", s1);
542
543 std::string old_s1 = s1 = "abc2xy";
544 s2 = "abc2";
545 ReverseBytewiseComparator()->FindShortestSeparator(&s1, s2);
546 ASSERT_TRUE(old_s1 >= s1);
547 ASSERT_TRUE(s1 > s2);
548 }
549
550 TEST_P(ComparatorDBTest, SeparatorSuccessorRandomizeTest) {
551 // Char list for boundary cases.
552 std::array<unsigned char, 6> char_list{{0, 1, 2, 253, 254, 255}};
553 Random rnd(301);
554
555 for (int attempts = 0; attempts < 1000; attempts++) {
556 uint32_t size1 = rnd.Skewed(4);
557 uint32_t size2;
558
559 if (rnd.OneIn(2)) {
560 // size2 to be random size
561 size2 = rnd.Skewed(4);
562 } else {
563 // size1 is within [-2, +2] of size1
564 int diff = static_cast<int>(rnd.Uniform(5)) - 2;
565 int tmp_size2 = static_cast<int>(size1) + diff;
566 if (tmp_size2 < 0) {
567 tmp_size2 = 0;
568 }
569 size2 = static_cast<uint32_t>(tmp_size2);
570 }
571
572 std::string s1;
573 std::string s2;
574 for (uint32_t i = 0; i < size1; i++) {
575 if (rnd.OneIn(2)) {
576 // Use random byte
577 s1 += static_cast<char>(rnd.Uniform(256));
578 } else {
579 // Use one byte in char_list
580 char c = static_cast<char>(char_list[rnd.Uniform(sizeof(char_list))]);
581 s1 += c;
582 }
583 }
584
585 // First set s2 to be the same as s1, and then modify s2.
586 s2 = s1;
587 s2.resize(size2);
588 // We start from the back of the string
589 if (size2 > 0) {
590 uint32_t pos = size2 - 1;
591 do {
592 if (pos >= size1 || rnd.OneIn(4)) {
593 // For 1/4 chance, use random byte
594 s2[pos] = static_cast<char>(rnd.Uniform(256));
595 } else if (rnd.OneIn(4)) {
596 // In 1/4 chance, stop here.
597 break;
598 } else {
599 // Create a char within [-2, +2] of the matching char of s1.
600 int diff = static_cast<int>(rnd.Uniform(5)) - 2;
601 // char may be signed or unsigned based on platform.
602 int s1_char = static_cast<int>(static_cast<unsigned char>(s1[pos]));
603 int s2_char = s1_char + diff;
604 if (s2_char < 0) {
605 s2_char = 0;
606 }
607 if (s2_char > 255) {
608 s2_char = 255;
609 }
610 s2[pos] = static_cast<char>(s2_char);
611 }
612 } while (pos-- != 0);
613 }
614
615 // Test separators
616 for (int rev = 0; rev < 2; rev++) {
617 if (rev == 1) {
618 // switch s1 and s2
619 std::string t = s1;
620 s1 = s2;
621 s2 = t;
622 }
623 std::string separator = s1;
624 BytewiseComparator()->FindShortestSeparator(&separator, s2);
625 std::string rev_separator = s1;
626 ReverseBytewiseComparator()->FindShortestSeparator(&rev_separator, s2);
627
628 if (s1 == s2) {
629 ASSERT_EQ(s1, separator);
630 ASSERT_EQ(s2, rev_separator);
631 } else if (s1 < s2) {
632 ASSERT_TRUE(s1 <= separator);
633 ASSERT_TRUE(s2 > separator);
634 ASSERT_LE(separator.size(), std::max(s1.size(), s2.size()));
635 ASSERT_EQ(s1, rev_separator);
636 } else {
637 ASSERT_TRUE(s1 >= rev_separator);
638 ASSERT_TRUE(s2 < rev_separator);
639 ASSERT_LE(rev_separator.size(), std::max(s1.size(), s2.size()));
640 ASSERT_EQ(s1, separator);
641 }
642 }
643
644 // Test successors
645 std::string succ = s1;
646 BytewiseComparator()->FindShortSuccessor(&succ);
647 ASSERT_TRUE(succ >= s1);
648
649 succ = s1;
650 ReverseBytewiseComparator()->FindShortSuccessor(&succ);
651 ASSERT_TRUE(succ <= s1);
652 }
653 }
654
655 } // namespace rocksdb
656
657 int main(int argc, char** argv) {
658 ::testing::InitGoogleTest(&argc, argv);
659 return RUN_ALL_TESTS();
660 }