ceph/src/rocksdb/utilities/merge_operators/string_append/stringappend_test.cc

   1 /**
   2  * An persistent map : key -> (list of strings), using rocksdb merge.
   3  * This file is a test-harness / use-case for the StringAppendOperator.
   4  *
   5  * @author Deon Nicholas (dnicholas@fb.com)
   6  * Copyright 2013 Facebook, Inc.
   7 */
   8
   9 #include <iostream>
  10 #include <map>
  11
  12 #include "rocksdb/db.h"
  13 #include "rocksdb/merge_operator.h"
  14 #include "rocksdb/utilities/db_ttl.h"
  15 #include "test_util/testharness.h"
  16 #include "util/random.h"
  17 #include "utilities/merge_operators.h"
  18 #include "utilities/merge_operators/string_append/stringappend.h"
  19 #include "utilities/merge_operators/string_append/stringappend2.h"
  20
  21 using namespace ROCKSDB_NAMESPACE;
  22
  23 namespace ROCKSDB_NAMESPACE {
  24
  25 // Path to the database on file system
  26 const std::string kDbName = test::PerThreadDBPath("stringappend_test");
  27
  28 namespace {
  29 // OpenDb opens a (possibly new) rocksdb database with a StringAppendOperator
  30 std::shared_ptr<DB> OpenNormalDb(char delim_char) {
  31   DB* db;
  32   Options options;
  33   options.create_if_missing = true;
  34   options.merge_operator.reset(new StringAppendOperator(delim_char));
  35   EXPECT_OK(DB::Open(options, kDbName, &db));
  36   return std::shared_ptr<DB>(db);
  37 }
  38
  39 #ifndef ROCKSDB_LITE  // TtlDb is not supported in Lite
  40 // Open a TtlDB with a non-associative StringAppendTESTOperator
  41 std::shared_ptr<DB> OpenTtlDb(char delim_char) {
  42   DBWithTTL* db;
  43   Options options;
  44   options.create_if_missing = true;
  45   options.merge_operator.reset(new StringAppendTESTOperator(delim_char));
  46   EXPECT_OK(DBWithTTL::Open(options, kDbName, &db, 123456));
  47   return std::shared_ptr<DB>(db);
  48 }
  49 #endif  // !ROCKSDB_LITE
  50 }  // namespace
  51
  52 /// StringLists represents a set of string-lists, each with a key-index.
  53 /// Supports Append(list, string) and Get(list)
  54 class StringLists {
  55  public:
  56
  57   //Constructor: specifies the rocksdb db
  58   /* implicit */
  59   StringLists(std::shared_ptr<DB> db)
  60       : db_(db),
  61         merge_option_(),
  62         get_option_() {
  63     assert(db);
  64   }
  65
  66   // Append string val onto the list defined by key; return true on success
  67   bool Append(const std::string& key, const std::string& val){
  68     Slice valSlice(val.data(), val.size());
  69     auto s = db_->Merge(merge_option_, key, valSlice);
  70
  71     if (s.ok()) {
  72       return true;
  73     } else {
  74       std::cerr << "ERROR " << s.ToString() << std::endl;
  75       return false;
  76     }
  77   }
  78
  79   // Returns the list of strings associated with key (or "" if does not exist)
  80   bool Get(const std::string& key, std::string* const result){
  81     assert(result != nullptr); // we should have a place to store the result
  82     auto s = db_->Get(get_option_, key, result);
  83
  84     if (s.ok()) {
  85       return true;
  86     }
  87
  88     // Either key does not exist, or there is some error.
  89     *result = "";       // Always return empty string (just for convention)
  90
  91     //NotFound is okay; just return empty (similar to std::map)
  92     //But network or db errors, etc, should fail the test (or at least yell)
  93     if (!s.IsNotFound()) {
  94       std::cerr << "ERROR " << s.ToString() << std::endl;
  95     }
  96
  97     // Always return false if s.ok() was not true
  98     return false;
  99   }
 100
 101
 102  private:
 103   std::shared_ptr<DB> db_;
 104   WriteOptions merge_option_;
 105   ReadOptions get_option_;
 106
 107 };
 108
 109
 110 // The class for unit-testing
 111 class StringAppendOperatorTest : public testing::Test {
 112  public:
 113   StringAppendOperatorTest() {
 114     DestroyDB(kDbName, Options());    // Start each test with a fresh DB
 115   }
 116
 117   typedef std::shared_ptr<DB> (* OpenFuncPtr)(char);
 118
 119   // Allows user to open databases with different configurations.
 120   // e.g.: Can open a DB or a TtlDB, etc.
 121   static void SetOpenDbFunction(OpenFuncPtr func) {
 122     OpenDb = func;
 123   }
 124
 125  protected:
 126   static OpenFuncPtr OpenDb;
 127 };
 128 StringAppendOperatorTest::OpenFuncPtr StringAppendOperatorTest::OpenDb = nullptr;
 129
 130 // THE TEST CASES BEGIN HERE
 131
 132 TEST_F(StringAppendOperatorTest, IteratorTest) {
 133   auto db_ = OpenDb(',');
 134   StringLists slists(db_);
 135
 136   slists.Append("k1", "v1");
 137   slists.Append("k1", "v2");
 138   slists.Append("k1", "v3");
 139
 140   slists.Append("k2", "a1");
 141   slists.Append("k2", "a2");
 142   slists.Append("k2", "a3");
 143
 144   std::string res;
 145   std::unique_ptr<ROCKSDB_NAMESPACE::Iterator> it(
 146       db_->NewIterator(ReadOptions()));
 147   std::string k1("k1");
 148   std::string k2("k2");
 149   bool first = true;
 150   for (it->Seek(k1); it->Valid(); it->Next()) {
 151     res = it->value().ToString();
 152     if (first) {
 153       ASSERT_EQ(res, "v1,v2,v3");
 154       first = false;
 155     } else {
 156       ASSERT_EQ(res, "a1,a2,a3");
 157     }
 158   }
 159   slists.Append("k2", "a4");
 160   slists.Append("k1", "v4");
 161
 162   // Snapshot should still be the same. Should ignore a4 and v4.
 163   first = true;
 164   for (it->Seek(k1); it->Valid(); it->Next()) {
 165     res = it->value().ToString();
 166     if (first) {
 167       ASSERT_EQ(res, "v1,v2,v3");
 168       first = false;
 169     } else {
 170       ASSERT_EQ(res, "a1,a2,a3");
 171     }
 172   }
 173
 174
 175   // Should release the snapshot and be aware of the new stuff now
 176   it.reset(db_->NewIterator(ReadOptions()));
 177   first = true;
 178   for (it->Seek(k1); it->Valid(); it->Next()) {
 179     res = it->value().ToString();
 180     if (first) {
 181       ASSERT_EQ(res, "v1,v2,v3,v4");
 182       first = false;
 183     } else {
 184       ASSERT_EQ(res, "a1,a2,a3,a4");
 185     }
 186   }
 187
 188   // start from k2 this time.
 189   for (it->Seek(k2); it->Valid(); it->Next()) {
 190     res = it->value().ToString();
 191     if (first) {
 192       ASSERT_EQ(res, "v1,v2,v3,v4");
 193       first = false;
 194     } else {
 195       ASSERT_EQ(res, "a1,a2,a3,a4");
 196     }
 197   }
 198
 199   slists.Append("k3", "g1");
 200
 201   it.reset(db_->NewIterator(ReadOptions()));
 202   first = true;
 203   std::string k3("k3");
 204   for(it->Seek(k2); it->Valid(); it->Next()) {
 205     res = it->value().ToString();
 206     if (first) {
 207       ASSERT_EQ(res, "a1,a2,a3,a4");
 208       first = false;
 209     } else {
 210       ASSERT_EQ(res, "g1");
 211     }
 212   }
 213   for(it->Seek(k3); it->Valid(); it->Next()) {
 214     res = it->value().ToString();
 215     if (first) {
 216       // should not be hit
 217       ASSERT_EQ(res, "a1,a2,a3,a4");
 218       first = false;
 219     } else {
 220       ASSERT_EQ(res, "g1");
 221     }
 222   }
 223
 224 }
 225
 226 TEST_F(StringAppendOperatorTest, SimpleTest) {
 227   auto db = OpenDb(',');
 228   StringLists slists(db);
 229
 230   slists.Append("k1", "v1");
 231   slists.Append("k1", "v2");
 232   slists.Append("k1", "v3");
 233
 234   std::string res;
 235   bool status = slists.Get("k1", &res);
 236
 237   ASSERT_TRUE(status);
 238   ASSERT_EQ(res, "v1,v2,v3");
 239 }
 240
 241 TEST_F(StringAppendOperatorTest, SimpleDelimiterTest) {
 242   auto db = OpenDb('|');
 243   StringLists slists(db);
 244
 245   slists.Append("k1", "v1");
 246   slists.Append("k1", "v2");
 247   slists.Append("k1", "v3");
 248
 249   std::string res;
 250   slists.Get("k1", &res);
 251   ASSERT_EQ(res, "v1|v2|v3");
 252 }
 253
 254 TEST_F(StringAppendOperatorTest, OneValueNoDelimiterTest) {
 255   auto db = OpenDb('!');
 256   StringLists slists(db);
 257
 258   slists.Append("random_key", "single_val");
 259
 260   std::string res;
 261   slists.Get("random_key", &res);
 262   ASSERT_EQ(res, "single_val");
 263 }
 264
 265 TEST_F(StringAppendOperatorTest, VariousKeys) {
 266   auto db = OpenDb('\n');
 267   StringLists slists(db);
 268
 269   slists.Append("c", "asdasd");
 270   slists.Append("a", "x");
 271   slists.Append("b", "y");
 272   slists.Append("a", "t");
 273   slists.Append("a", "r");
 274   slists.Append("b", "2");
 275   slists.Append("c", "asdasd");
 276
 277   std::string a, b, c;
 278   bool sa, sb, sc;
 279   sa = slists.Get("a", &a);
 280   sb = slists.Get("b", &b);
 281   sc = slists.Get("c", &c);
 282
 283   ASSERT_TRUE(sa && sb && sc); // All three keys should have been found
 284
 285   ASSERT_EQ(a, "x\nt\nr");
 286   ASSERT_EQ(b, "y\n2");
 287   ASSERT_EQ(c, "asdasd\nasdasd");
 288 }
 289
 290 // Generate semi random keys/words from a small distribution.
 291 TEST_F(StringAppendOperatorTest, RandomMixGetAppend) {
 292   auto db = OpenDb(' ');
 293   StringLists slists(db);
 294
 295   // Generate a list of random keys and values
 296   const int kWordCount = 15;
 297   std::string words[] = {"sdasd", "triejf", "fnjsdfn", "dfjisdfsf", "342839",
 298                          "dsuha", "mabuais", "sadajsid", "jf9834hf", "2d9j89",
 299                          "dj9823jd", "a", "dk02ed2dh", "$(jd4h984$(*", "mabz"};
 300   const int kKeyCount = 6;
 301   std::string keys[] = {"dhaiusdhu", "denidw", "daisda", "keykey", "muki",
 302                         "shzassdianmd"};
 303
 304   // Will store a local copy of all data in order to verify correctness
 305   std::map<std::string, std::string> parallel_copy;
 306
 307   // Generate a bunch of random queries (Append and Get)!
 308   enum query_t  { APPEND_OP, GET_OP, NUM_OPS };
 309   Random randomGen(1337);       //deterministic seed; always get same results!
 310
 311   const int kNumQueries = 30;
 312   for (int q=0; q<kNumQueries; ++q) {
 313     // Generate a random query (Append or Get) and random parameters
 314     query_t query = (query_t)randomGen.Uniform((int)NUM_OPS);
 315     std::string key = keys[randomGen.Uniform((int)kKeyCount)];
 316     std::string word = words[randomGen.Uniform((int)kWordCount)];
 317
 318     // Apply the query and any checks.
 319     if (query == APPEND_OP) {
 320
 321       // Apply the rocksdb test-harness Append defined above
 322       slists.Append(key, word);  //apply the rocksdb append
 323
 324       // Apply the similar "Append" to the parallel copy
 325       if (parallel_copy[key].size() > 0) {
 326         parallel_copy[key] += " " + word;
 327       } else {
 328         parallel_copy[key] = word;
 329       }
 330
 331     } else if (query == GET_OP) {
 332       // Assumes that a non-existent key just returns <empty>
 333       std::string res;
 334       slists.Get(key, &res);
 335       ASSERT_EQ(res, parallel_copy[key]);
 336     }
 337
 338   }
 339
 340 }
 341
 342 TEST_F(StringAppendOperatorTest, BIGRandomMixGetAppend) {
 343   auto db = OpenDb(' ');
 344   StringLists slists(db);
 345
 346   // Generate a list of random keys and values
 347   const int kWordCount = 15;
 348   std::string words[] = {"sdasd", "triejf", "fnjsdfn", "dfjisdfsf", "342839",
 349                          "dsuha", "mabuais", "sadajsid", "jf9834hf", "2d9j89",
 350                          "dj9823jd", "a", "dk02ed2dh", "$(jd4h984$(*", "mabz"};
 351   const int kKeyCount = 6;
 352   std::string keys[] = {"dhaiusdhu", "denidw", "daisda", "keykey", "muki",
 353                         "shzassdianmd"};
 354
 355   // Will store a local copy of all data in order to verify correctness
 356   std::map<std::string, std::string> parallel_copy;
 357
 358   // Generate a bunch of random queries (Append and Get)!
 359   enum query_t  { APPEND_OP, GET_OP, NUM_OPS };
 360   Random randomGen(9138204);       // deterministic seed
 361
 362   const int kNumQueries = 1000;
 363   for (int q=0; q<kNumQueries; ++q) {
 364     // Generate a random query (Append or Get) and random parameters
 365     query_t query = (query_t)randomGen.Uniform((int)NUM_OPS);
 366     std::string key = keys[randomGen.Uniform((int)kKeyCount)];
 367     std::string word = words[randomGen.Uniform((int)kWordCount)];
 368
 369     //Apply the query and any checks.
 370     if (query == APPEND_OP) {
 371
 372       // Apply the rocksdb test-harness Append defined above
 373       slists.Append(key, word);  //apply the rocksdb append
 374
 375       // Apply the similar "Append" to the parallel copy
 376       if (parallel_copy[key].size() > 0) {
 377         parallel_copy[key] += " " + word;
 378       } else {
 379         parallel_copy[key] = word;
 380       }
 381
 382     } else if (query == GET_OP) {
 383       // Assumes that a non-existent key just returns <empty>
 384       std::string res;
 385       slists.Get(key, &res);
 386       ASSERT_EQ(res, parallel_copy[key]);
 387     }
 388
 389   }
 390
 391 }
 392
 393 TEST_F(StringAppendOperatorTest, PersistentVariousKeys) {
 394   // Perform the following operations in limited scope
 395   {
 396     auto db = OpenDb('\n');
 397     StringLists slists(db);
 398
 399     slists.Append("c", "asdasd");
 400     slists.Append("a", "x");
 401     slists.Append("b", "y");
 402     slists.Append("a", "t");
 403     slists.Append("a", "r");
 404     slists.Append("b", "2");
 405     slists.Append("c", "asdasd");
 406
 407     std::string a, b, c;
 408     slists.Get("a", &a);
 409     slists.Get("b", &b);
 410     slists.Get("c", &c);
 411
 412     ASSERT_EQ(a, "x\nt\nr");
 413     ASSERT_EQ(b, "y\n2");
 414     ASSERT_EQ(c, "asdasd\nasdasd");
 415   }
 416
 417   // Reopen the database (the previous changes should persist / be remembered)
 418   {
 419     auto db = OpenDb('\n');
 420     StringLists slists(db);
 421
 422     slists.Append("c", "bbnagnagsx");
 423     slists.Append("a", "sa");
 424     slists.Append("b", "df");
 425     slists.Append("a", "gh");
 426     slists.Append("a", "jk");
 427     slists.Append("b", "l;");
 428     slists.Append("c", "rogosh");
 429
 430     // The previous changes should be on disk (L0)
 431     // The most recent changes should be in memory (MemTable)
 432     // Hence, this will test both Get() paths.
 433     std::string a, b, c;
 434     slists.Get("a", &a);
 435     slists.Get("b", &b);
 436     slists.Get("c", &c);
 437
 438     ASSERT_EQ(a, "x\nt\nr\nsa\ngh\njk");
 439     ASSERT_EQ(b, "y\n2\ndf\nl;");
 440     ASSERT_EQ(c, "asdasd\nasdasd\nbbnagnagsx\nrogosh");
 441   }
 442
 443   // Reopen the database (the previous changes should persist / be remembered)
 444   {
 445     auto db = OpenDb('\n');
 446     StringLists slists(db);
 447
 448     // All changes should be on disk. This will test VersionSet Get()
 449     std::string a, b, c;
 450     slists.Get("a", &a);
 451     slists.Get("b", &b);
 452     slists.Get("c", &c);
 453
 454     ASSERT_EQ(a, "x\nt\nr\nsa\ngh\njk");
 455     ASSERT_EQ(b, "y\n2\ndf\nl;");
 456     ASSERT_EQ(c, "asdasd\nasdasd\nbbnagnagsx\nrogosh");
 457   }
 458 }
 459
 460 TEST_F(StringAppendOperatorTest, PersistentFlushAndCompaction) {
 461   // Perform the following operations in limited scope
 462   {
 463     auto db = OpenDb('\n');
 464     StringLists slists(db);
 465     std::string a, b, c;
 466     bool success;
 467
 468     // Append, Flush, Get
 469     slists.Append("c", "asdasd");
 470     db->Flush(ROCKSDB_NAMESPACE::FlushOptions());
 471     success = slists.Get("c", &c);
 472     ASSERT_TRUE(success);
 473     ASSERT_EQ(c, "asdasd");
 474
 475     // Append, Flush, Append, Get
 476     slists.Append("a", "x");
 477     slists.Append("b", "y");
 478     db->Flush(ROCKSDB_NAMESPACE::FlushOptions());
 479     slists.Append("a", "t");
 480     slists.Append("a", "r");
 481     slists.Append("b", "2");
 482
 483     success = slists.Get("a", &a);
 484     assert(success == true);
 485     ASSERT_EQ(a, "x\nt\nr");
 486
 487     success = slists.Get("b", &b);
 488     assert(success == true);
 489     ASSERT_EQ(b, "y\n2");
 490
 491     // Append, Get
 492     success = slists.Append("c", "asdasd");
 493     assert(success);
 494     success = slists.Append("b", "monkey");
 495     assert(success);
 496
 497     // I omit the "assert(success)" checks here.
 498     slists.Get("a", &a);
 499     slists.Get("b", &b);
 500     slists.Get("c", &c);
 501
 502     ASSERT_EQ(a, "x\nt\nr");
 503     ASSERT_EQ(b, "y\n2\nmonkey");
 504     ASSERT_EQ(c, "asdasd\nasdasd");
 505   }
 506
 507   // Reopen the database (the previous changes should persist / be remembered)
 508   {
 509     auto db = OpenDb('\n');
 510     StringLists slists(db);
 511     std::string a, b, c;
 512
 513     // Get (Quick check for persistence of previous database)
 514     slists.Get("a", &a);
 515     ASSERT_EQ(a, "x\nt\nr");
 516
 517     //Append, Compact, Get
 518     slists.Append("c", "bbnagnagsx");
 519     slists.Append("a", "sa");
 520     slists.Append("b", "df");
 521     db->CompactRange(CompactRangeOptions(), nullptr, nullptr);
 522     slists.Get("a", &a);
 523     slists.Get("b", &b);
 524     slists.Get("c", &c);
 525     ASSERT_EQ(a, "x\nt\nr\nsa");
 526     ASSERT_EQ(b, "y\n2\nmonkey\ndf");
 527     ASSERT_EQ(c, "asdasd\nasdasd\nbbnagnagsx");
 528
 529     // Append, Get
 530     slists.Append("a", "gh");
 531     slists.Append("a", "jk");
 532     slists.Append("b", "l;");
 533     slists.Append("c", "rogosh");
 534     slists.Get("a", &a);
 535     slists.Get("b", &b);
 536     slists.Get("c", &c);
 537     ASSERT_EQ(a, "x\nt\nr\nsa\ngh\njk");
 538     ASSERT_EQ(b, "y\n2\nmonkey\ndf\nl;");
 539     ASSERT_EQ(c, "asdasd\nasdasd\nbbnagnagsx\nrogosh");
 540
 541     // Compact, Get
 542     db->CompactRange(CompactRangeOptions(), nullptr, nullptr);
 543     ASSERT_EQ(a, "x\nt\nr\nsa\ngh\njk");
 544     ASSERT_EQ(b, "y\n2\nmonkey\ndf\nl;");
 545     ASSERT_EQ(c, "asdasd\nasdasd\nbbnagnagsx\nrogosh");
 546
 547     // Append, Flush, Compact, Get
 548     slists.Append("b", "afcg");
 549     db->Flush(ROCKSDB_NAMESPACE::FlushOptions());
 550     db->CompactRange(CompactRangeOptions(), nullptr, nullptr);
 551     slists.Get("b", &b);
 552     ASSERT_EQ(b, "y\n2\nmonkey\ndf\nl;\nafcg");
 553   }
 554 }
 555
 556 TEST_F(StringAppendOperatorTest, SimpleTestNullDelimiter) {
 557   auto db = OpenDb('\0');
 558   StringLists slists(db);
 559
 560   slists.Append("k1", "v1");
 561   slists.Append("k1", "v2");
 562   slists.Append("k1", "v3");
 563
 564   std::string res;
 565   bool status = slists.Get("k1", &res);
 566   ASSERT_TRUE(status);
 567
 568   // Construct the desired string. Default constructor doesn't like '\0' chars.
 569   std::string checker("v1,v2,v3");    // Verify that the string is right size.
 570   checker[2] = '\0';                  // Use null delimiter instead of comma.
 571   checker[5] = '\0';
 572   assert(checker.size() == 8);        // Verify it is still the correct size
 573
 574   // Check that the rocksdb result string matches the desired string
 575   assert(res.size() == checker.size());
 576   ASSERT_EQ(res, checker);
 577 }
 578
 579 }  // namespace ROCKSDB_NAMESPACE
 580
 581 int main(int argc, char** argv) {
 582   ::testing::InitGoogleTest(&argc, argv);
 583   // Run with regular database
 584   int result;
 585   {
 586     fprintf(stderr, "Running tests with regular db and operator.\n");
 587     StringAppendOperatorTest::SetOpenDbFunction(&OpenNormalDb);
 588     result = RUN_ALL_TESTS();
 589   }
 590
 591 #ifndef ROCKSDB_LITE  // TtlDb is not supported in Lite
 592   // Run with TTL
 593   {
 594     fprintf(stderr, "Running tests with ttl db and generic operator.\n");
 595     StringAppendOperatorTest::SetOpenDbFunction(&OpenTtlDb);
 596     result |= RUN_ALL_TESTS();
 597   }
 598 #endif  // !ROCKSDB_LITE
 599
 600   return result;
 601 }