ceph/src/rocksdb/db/version_set_test.cc

   1 //  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
   2 //  This source code is licensed under both the GPLv2 (found in the
   3 //  COPYING file in the root directory) and Apache 2.0 License
   4 //  (found in the LICENSE.Apache file in the root directory).
   5 //
   6 // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
   7 // Use of this source code is governed by a BSD-style license that can be
   8 // found in the LICENSE file. See the AUTHORS file for names of contributors.
   9
  10 #include "db/version_set.h"
  11 #include "db/log_writer.h"
  12 #include "table/mock_table.h"
  13 #include "util/logging.h"
  14 #include "util/string_util.h"
  15 #include "util/testharness.h"
  16 #include "util/testutil.h"
  17
  18 namespace rocksdb {
  19
  20 class GenerateLevelFilesBriefTest : public testing::Test {
  21  public:
  22   std::vector<FileMetaData*> files_;
  23   LevelFilesBrief file_level_;
  24   Arena arena_;
  25
  26   GenerateLevelFilesBriefTest() { }
  27
  28   ~GenerateLevelFilesBriefTest() {
  29     for (size_t i = 0; i < files_.size(); i++) {
  30       delete files_[i];
  31     }
  32   }
  33
  34   void Add(const char* smallest, const char* largest,
  35            SequenceNumber smallest_seq = 100,
  36            SequenceNumber largest_seq = 100) {
  37     FileMetaData* f = new FileMetaData;
  38     f->fd = FileDescriptor(files_.size() + 1, 0, 0);
  39     f->smallest = InternalKey(smallest, smallest_seq, kTypeValue);
  40     f->largest = InternalKey(largest, largest_seq, kTypeValue);
  41     files_.push_back(f);
  42   }
  43
  44   int Compare() {
  45     int diff = 0;
  46     for (size_t i = 0; i < files_.size(); i++) {
  47       if (file_level_.files[i].fd.GetNumber() != files_[i]->fd.GetNumber()) {
  48         diff++;
  49       }
  50     }
  51     return diff;
  52   }
  53 };
  54
  55 TEST_F(GenerateLevelFilesBriefTest, Empty) {
  56   DoGenerateLevelFilesBrief(&file_level_, files_, &arena_);
  57   ASSERT_EQ(0u, file_level_.num_files);
  58   ASSERT_EQ(0, Compare());
  59 }
  60
  61 TEST_F(GenerateLevelFilesBriefTest, Single) {
  62   Add("p", "q");
  63   DoGenerateLevelFilesBrief(&file_level_, files_, &arena_);
  64   ASSERT_EQ(1u, file_level_.num_files);
  65   ASSERT_EQ(0, Compare());
  66 }
  67
  68 TEST_F(GenerateLevelFilesBriefTest, Multiple) {
  69   Add("150", "200");
  70   Add("200", "250");
  71   Add("300", "350");
  72   Add("400", "450");
  73   DoGenerateLevelFilesBrief(&file_level_, files_, &arena_);
  74   ASSERT_EQ(4u, file_level_.num_files);
  75   ASSERT_EQ(0, Compare());
  76 }
  77
  78 class CountingLogger : public Logger {
  79  public:
  80   CountingLogger() : log_count(0) {}
  81   using Logger::Logv;
  82   virtual void Logv(const char* /*format*/, va_list /*ap*/) override {
  83     log_count++;
  84   }
  85   int log_count;
  86 };
  87
  88 Options GetOptionsWithNumLevels(int num_levels,
  89                                 std::shared_ptr<CountingLogger> logger) {
  90   Options opt;
  91   opt.num_levels = num_levels;
  92   opt.info_log = logger;
  93   return opt;
  94 }
  95
  96 class VersionStorageInfoTest : public testing::Test {
  97  public:
  98   const Comparator* ucmp_;
  99   InternalKeyComparator icmp_;
 100   std::shared_ptr<CountingLogger> logger_;
 101   Options options_;
 102   ImmutableCFOptions ioptions_;
 103   MutableCFOptions mutable_cf_options_;
 104   VersionStorageInfo vstorage_;
 105
 106   InternalKey GetInternalKey(const char* ukey,
 107                              SequenceNumber smallest_seq = 100) {
 108     return InternalKey(ukey, smallest_seq, kTypeValue);
 109   }
 110
 111   VersionStorageInfoTest()
 112       : ucmp_(BytewiseComparator()),
 113         icmp_(ucmp_),
 114         logger_(new CountingLogger()),
 115         options_(GetOptionsWithNumLevels(6, logger_)),
 116         ioptions_(options_),
 117         mutable_cf_options_(options_),
 118         vstorage_(&icmp_, ucmp_, 6, kCompactionStyleLevel, nullptr, false) {}
 119
 120   ~VersionStorageInfoTest() {
 121     for (int i = 0; i < vstorage_.num_levels(); i++) {
 122       for (auto* f : vstorage_.LevelFiles(i)) {
 123         if (--f->refs == 0) {
 124           delete f;
 125         }
 126       }
 127     }
 128   }
 129
 130   void Add(int level, uint32_t file_number, const char* smallest,
 131            const char* largest, uint64_t file_size = 0) {
 132     assert(level < vstorage_.num_levels());
 133     FileMetaData* f = new FileMetaData;
 134     f->fd = FileDescriptor(file_number, 0, file_size);
 135     f->smallest = GetInternalKey(smallest, 0);
 136     f->largest = GetInternalKey(largest, 0);
 137     f->compensated_file_size = file_size;
 138     f->refs = 0;
 139     f->num_entries = 0;
 140     f->num_deletions = 0;
 141     vstorage_.AddFile(level, f);
 142   }
 143
 144   void Add(int level, uint32_t file_number, const InternalKey& smallest,
 145            const InternalKey& largest, uint64_t file_size = 0) {
 146     assert(level < vstorage_.num_levels());
 147     FileMetaData* f = new FileMetaData;
 148     f->fd = FileDescriptor(file_number, 0, file_size);
 149     f->smallest = smallest;
 150     f->largest = largest;
 151     f->compensated_file_size = file_size;
 152     f->refs = 0;
 153     f->num_entries = 0;
 154     f->num_deletions = 0;
 155     vstorage_.AddFile(level, f);
 156   }
 157
 158   std::string GetOverlappingFiles(int level, const InternalKey& begin,
 159                                   const InternalKey& end) {
 160     std::vector<FileMetaData*> inputs;
 161     vstorage_.GetOverlappingInputs(level, &begin, &end, &inputs);
 162
 163     std::string result;
 164     for (size_t i = 0; i < inputs.size(); ++i) {
 165       if (i > 0) {
 166         result += ",";
 167       }
 168       AppendNumberTo(&result, inputs[i]->fd.GetNumber());
 169     }
 170     return result;
 171   }
 172 };
 173
 174 TEST_F(VersionStorageInfoTest, MaxBytesForLevelStatic) {
 175   ioptions_.level_compaction_dynamic_level_bytes = false;
 176   mutable_cf_options_.max_bytes_for_level_base = 10;
 177   mutable_cf_options_.max_bytes_for_level_multiplier = 5;
 178   Add(4, 100U, "1", "2");
 179   Add(5, 101U, "1", "2");
 180
 181   vstorage_.CalculateBaseBytes(ioptions_, mutable_cf_options_);
 182   ASSERT_EQ(vstorage_.MaxBytesForLevel(1), 10U);
 183   ASSERT_EQ(vstorage_.MaxBytesForLevel(2), 50U);
 184   ASSERT_EQ(vstorage_.MaxBytesForLevel(3), 250U);
 185   ASSERT_EQ(vstorage_.MaxBytesForLevel(4), 1250U);
 186
 187   ASSERT_EQ(0, logger_->log_count);
 188 }
 189
 190 TEST_F(VersionStorageInfoTest, MaxBytesForLevelDynamic) {
 191   ioptions_.level_compaction_dynamic_level_bytes = true;
 192   mutable_cf_options_.max_bytes_for_level_base = 1000;
 193   mutable_cf_options_.max_bytes_for_level_multiplier = 5;
 194   Add(5, 1U, "1", "2", 500U);
 195
 196   vstorage_.CalculateBaseBytes(ioptions_, mutable_cf_options_);
 197   ASSERT_EQ(0, logger_->log_count);
 198   ASSERT_EQ(vstorage_.base_level(), 5);
 199
 200   Add(5, 2U, "3", "4", 550U);
 201   vstorage_.CalculateBaseBytes(ioptions_, mutable_cf_options_);
 202   ASSERT_EQ(0, logger_->log_count);
 203   ASSERT_EQ(vstorage_.MaxBytesForLevel(4), 1000U);
 204   ASSERT_EQ(vstorage_.base_level(), 4);
 205
 206   Add(4, 3U, "3", "4", 550U);
 207   vstorage_.CalculateBaseBytes(ioptions_, mutable_cf_options_);
 208   ASSERT_EQ(0, logger_->log_count);
 209   ASSERT_EQ(vstorage_.MaxBytesForLevel(4), 1000U);
 210   ASSERT_EQ(vstorage_.base_level(), 4);
 211
 212   Add(3, 4U, "3", "4", 250U);
 213   Add(3, 5U, "5", "7", 300U);
 214   vstorage_.CalculateBaseBytes(ioptions_, mutable_cf_options_);
 215   ASSERT_EQ(1, logger_->log_count);
 216   ASSERT_EQ(vstorage_.MaxBytesForLevel(4), 1005U);
 217   ASSERT_EQ(vstorage_.MaxBytesForLevel(3), 1000U);
 218   ASSERT_EQ(vstorage_.base_level(), 3);
 219
 220   Add(1, 6U, "3", "4", 5U);
 221   Add(1, 7U, "8", "9", 5U);
 222   logger_->log_count = 0;
 223   vstorage_.CalculateBaseBytes(ioptions_, mutable_cf_options_);
 224   ASSERT_EQ(1, logger_->log_count);
 225   ASSERT_GT(vstorage_.MaxBytesForLevel(4), 1005U);
 226   ASSERT_GT(vstorage_.MaxBytesForLevel(3), 1005U);
 227   ASSERT_EQ(vstorage_.MaxBytesForLevel(2), 1005U);
 228   ASSERT_EQ(vstorage_.MaxBytesForLevel(1), 1000U);
 229   ASSERT_EQ(vstorage_.base_level(), 1);
 230 }
 231
 232 TEST_F(VersionStorageInfoTest, MaxBytesForLevelDynamicLotsOfData) {
 233   ioptions_.level_compaction_dynamic_level_bytes = true;
 234   mutable_cf_options_.max_bytes_for_level_base = 100;
 235   mutable_cf_options_.max_bytes_for_level_multiplier = 2;
 236   Add(0, 1U, "1", "2", 50U);
 237   Add(1, 2U, "1", "2", 50U);
 238   Add(2, 3U, "1", "2", 500U);
 239   Add(3, 4U, "1", "2", 500U);
 240   Add(4, 5U, "1", "2", 1700U);
 241   Add(5, 6U, "1", "2", 500U);
 242
 243   vstorage_.CalculateBaseBytes(ioptions_, mutable_cf_options_);
 244   ASSERT_EQ(vstorage_.MaxBytesForLevel(4), 800U);
 245   ASSERT_EQ(vstorage_.MaxBytesForLevel(3), 400U);
 246   ASSERT_EQ(vstorage_.MaxBytesForLevel(2), 200U);
 247   ASSERT_EQ(vstorage_.MaxBytesForLevel(1), 100U);
 248   ASSERT_EQ(vstorage_.base_level(), 1);
 249   ASSERT_EQ(0, logger_->log_count);
 250 }
 251
 252 TEST_F(VersionStorageInfoTest, MaxBytesForLevelDynamicLargeLevel) {
 253   uint64_t kOneGB = 1000U * 1000U * 1000U;
 254   ioptions_.level_compaction_dynamic_level_bytes = true;
 255   mutable_cf_options_.max_bytes_for_level_base = 10U * kOneGB;
 256   mutable_cf_options_.max_bytes_for_level_multiplier = 10;
 257   Add(0, 1U, "1", "2", 50U);
 258   Add(3, 4U, "1", "2", 32U * kOneGB);
 259   Add(4, 5U, "1", "2", 500U * kOneGB);
 260   Add(5, 6U, "1", "2", 3000U * kOneGB);
 261
 262   vstorage_.CalculateBaseBytes(ioptions_, mutable_cf_options_);
 263   ASSERT_EQ(vstorage_.MaxBytesForLevel(5), 3000U * kOneGB);
 264   ASSERT_EQ(vstorage_.MaxBytesForLevel(4), 300U * kOneGB);
 265   ASSERT_EQ(vstorage_.MaxBytesForLevel(3), 30U * kOneGB);
 266   ASSERT_EQ(vstorage_.MaxBytesForLevel(2), 10U * kOneGB);
 267   ASSERT_EQ(vstorage_.base_level(), 2);
 268   ASSERT_EQ(0, logger_->log_count);
 269 }
 270
 271 TEST_F(VersionStorageInfoTest, EstimateLiveDataSize) {
 272   // Test whether the overlaps are detected as expected
 273   Add(1, 1U, "4", "7", 1U);  // Perfect overlap with last level
 274   Add(2, 2U, "3", "5", 1U);  // Partial overlap with last level
 275   Add(2, 3U, "6", "8", 1U);  // Partial overlap with last level
 276   Add(3, 4U, "1", "9", 1U);  // Contains range of last level
 277   Add(4, 5U, "4", "5", 1U);  // Inside range of last level
 278   Add(4, 5U, "6", "7", 1U);  // Inside range of last level
 279   Add(5, 6U, "4", "7", 10U);
 280   ASSERT_EQ(10U, vstorage_.EstimateLiveDataSize());
 281 }
 282
 283 TEST_F(VersionStorageInfoTest, EstimateLiveDataSize2) {
 284   Add(0, 1U, "9", "9", 1U);  // Level 0 is not ordered
 285   Add(0, 1U, "5", "6", 1U);  // Ignored because of [5,6] in l1
 286   Add(1, 1U, "1", "2", 1U);  // Ignored because of [2,3] in l2
 287   Add(1, 2U, "3", "4", 1U);  // Ignored because of [2,3] in l2
 288   Add(1, 3U, "5", "6", 1U);
 289   Add(2, 4U, "2", "3", 1U);
 290   Add(3, 5U, "7", "8", 1U);
 291   ASSERT_EQ(4U, vstorage_.EstimateLiveDataSize());
 292 }
 293
 294 TEST_F(VersionStorageInfoTest, GetOverlappingInputs) {
 295   // Two files that overlap at the range deletion tombstone sentinel.
 296   Add(1, 1U, {"a", 0, kTypeValue}, {"b", kMaxSequenceNumber, kTypeRangeDeletion}, 1);
 297   Add(1, 2U, {"b", 0, kTypeValue}, {"c", 0, kTypeValue}, 1);
 298   // Two files that overlap at the same user key.
 299   Add(1, 3U, {"d", 0, kTypeValue}, {"e", kMaxSequenceNumber, kTypeValue}, 1);
 300   Add(1, 4U, {"e", 0, kTypeValue}, {"f", 0, kTypeValue}, 1);
 301   // Two files that do not overlap.
 302   Add(1, 5U, {"g", 0, kTypeValue}, {"h", 0, kTypeValue}, 1);
 303   Add(1, 6U, {"i", 0, kTypeValue}, {"j", 0, kTypeValue}, 1);
 304   vstorage_.UpdateNumNonEmptyLevels();
 305   vstorage_.GenerateLevelFilesBrief();
 306
 307   ASSERT_EQ("1,2", GetOverlappingFiles(
 308       1, {"a", 0, kTypeValue}, {"b", 0, kTypeValue}));
 309   ASSERT_EQ("1", GetOverlappingFiles(
 310       1, {"a", 0, kTypeValue}, {"b", kMaxSequenceNumber, kTypeRangeDeletion}));
 311   ASSERT_EQ("2", GetOverlappingFiles(
 312       1, {"b", kMaxSequenceNumber, kTypeValue}, {"c", 0, kTypeValue}));
 313   ASSERT_EQ("3,4", GetOverlappingFiles(
 314       1, {"d", 0, kTypeValue}, {"e", 0, kTypeValue}));
 315   ASSERT_EQ("3", GetOverlappingFiles(
 316       1, {"d", 0, kTypeValue}, {"e", kMaxSequenceNumber, kTypeRangeDeletion}));
 317   ASSERT_EQ("3,4", GetOverlappingFiles(
 318       1, {"e", kMaxSequenceNumber, kTypeValue}, {"f", 0, kTypeValue}));
 319   ASSERT_EQ("3,4", GetOverlappingFiles(
 320       1, {"e", 0, kTypeValue}, {"f", 0, kTypeValue}));
 321   ASSERT_EQ("5", GetOverlappingFiles(
 322       1, {"g", 0, kTypeValue}, {"h", 0, kTypeValue}));
 323   ASSERT_EQ("6", GetOverlappingFiles(
 324       1, {"i", 0, kTypeValue}, {"j", 0, kTypeValue}));
 325 }
 326
 327
 328 class FindLevelFileTest : public testing::Test {
 329  public:
 330   LevelFilesBrief file_level_;
 331   bool disjoint_sorted_files_;
 332   Arena arena_;
 333
 334   FindLevelFileTest() : disjoint_sorted_files_(true) { }
 335
 336   ~FindLevelFileTest() {
 337   }
 338
 339   void LevelFileInit(size_t num = 0) {
 340     char* mem = arena_.AllocateAligned(num * sizeof(FdWithKeyRange));
 341     file_level_.files = new (mem)FdWithKeyRange[num];
 342     file_level_.num_files = 0;
 343   }
 344
 345   void Add(const char* smallest, const char* largest,
 346            SequenceNumber smallest_seq = 100,
 347            SequenceNumber largest_seq = 100) {
 348     InternalKey smallest_key = InternalKey(smallest, smallest_seq, kTypeValue);
 349     InternalKey largest_key = InternalKey(largest, largest_seq, kTypeValue);
 350
 351     Slice smallest_slice = smallest_key.Encode();
 352     Slice largest_slice = largest_key.Encode();
 353
 354     char* mem = arena_.AllocateAligned(
 355         smallest_slice.size() + largest_slice.size());
 356     memcpy(mem, smallest_slice.data(), smallest_slice.size());
 357     memcpy(mem + smallest_slice.size(), largest_slice.data(),
 358         largest_slice.size());
 359
 360     // add to file_level_
 361     size_t num = file_level_.num_files;
 362     auto& file = file_level_.files[num];
 363     file.fd = FileDescriptor(num + 1, 0, 0);
 364     file.smallest_key = Slice(mem, smallest_slice.size());
 365     file.largest_key = Slice(mem + smallest_slice.size(),
 366         largest_slice.size());
 367     file_level_.num_files++;
 368   }
 369
 370   int Find(const char* key) {
 371     InternalKey target(key, 100, kTypeValue);
 372     InternalKeyComparator cmp(BytewiseComparator());
 373     return FindFile(cmp, file_level_, target.Encode());
 374   }
 375
 376   bool Overlaps(const char* smallest, const char* largest) {
 377     InternalKeyComparator cmp(BytewiseComparator());
 378     Slice s(smallest != nullptr ? smallest : "");
 379     Slice l(largest != nullptr ? largest : "");
 380     return SomeFileOverlapsRange(cmp, disjoint_sorted_files_, file_level_,
 381                                  (smallest != nullptr ? &s : nullptr),
 382                                  (largest != nullptr ? &l : nullptr));
 383   }
 384 };
 385
 386 TEST_F(FindLevelFileTest, LevelEmpty) {
 387   LevelFileInit(0);
 388
 389   ASSERT_EQ(0, Find("foo"));
 390   ASSERT_TRUE(! Overlaps("a", "z"));
 391   ASSERT_TRUE(! Overlaps(nullptr, "z"));
 392   ASSERT_TRUE(! Overlaps("a", nullptr));
 393   ASSERT_TRUE(! Overlaps(nullptr, nullptr));
 394 }
 395
 396 TEST_F(FindLevelFileTest, LevelSingle) {
 397   LevelFileInit(1);
 398
 399   Add("p", "q");
 400   ASSERT_EQ(0, Find("a"));
 401   ASSERT_EQ(0, Find("p"));
 402   ASSERT_EQ(0, Find("p1"));
 403   ASSERT_EQ(0, Find("q"));
 404   ASSERT_EQ(1, Find("q1"));
 405   ASSERT_EQ(1, Find("z"));
 406
 407   ASSERT_TRUE(! Overlaps("a", "b"));
 408   ASSERT_TRUE(! Overlaps("z1", "z2"));
 409   ASSERT_TRUE(Overlaps("a", "p"));
 410   ASSERT_TRUE(Overlaps("a", "q"));
 411   ASSERT_TRUE(Overlaps("a", "z"));
 412   ASSERT_TRUE(Overlaps("p", "p1"));
 413   ASSERT_TRUE(Overlaps("p", "q"));
 414   ASSERT_TRUE(Overlaps("p", "z"));
 415   ASSERT_TRUE(Overlaps("p1", "p2"));
 416   ASSERT_TRUE(Overlaps("p1", "z"));
 417   ASSERT_TRUE(Overlaps("q", "q"));
 418   ASSERT_TRUE(Overlaps("q", "q1"));
 419
 420   ASSERT_TRUE(! Overlaps(nullptr, "j"));
 421   ASSERT_TRUE(! Overlaps("r", nullptr));
 422   ASSERT_TRUE(Overlaps(nullptr, "p"));
 423   ASSERT_TRUE(Overlaps(nullptr, "p1"));
 424   ASSERT_TRUE(Overlaps("q", nullptr));
 425   ASSERT_TRUE(Overlaps(nullptr, nullptr));
 426 }
 427
 428 TEST_F(FindLevelFileTest, LevelMultiple) {
 429   LevelFileInit(4);
 430
 431   Add("150", "200");
 432   Add("200", "250");
 433   Add("300", "350");
 434   Add("400", "450");
 435   ASSERT_EQ(0, Find("100"));
 436   ASSERT_EQ(0, Find("150"));
 437   ASSERT_EQ(0, Find("151"));
 438   ASSERT_EQ(0, Find("199"));
 439   ASSERT_EQ(0, Find("200"));
 440   ASSERT_EQ(1, Find("201"));
 441   ASSERT_EQ(1, Find("249"));
 442   ASSERT_EQ(1, Find("250"));
 443   ASSERT_EQ(2, Find("251"));
 444   ASSERT_EQ(2, Find("299"));
 445   ASSERT_EQ(2, Find("300"));
 446   ASSERT_EQ(2, Find("349"));
 447   ASSERT_EQ(2, Find("350"));
 448   ASSERT_EQ(3, Find("351"));
 449   ASSERT_EQ(3, Find("400"));
 450   ASSERT_EQ(3, Find("450"));
 451   ASSERT_EQ(4, Find("451"));
 452
 453   ASSERT_TRUE(! Overlaps("100", "149"));
 454   ASSERT_TRUE(! Overlaps("251", "299"));
 455   ASSERT_TRUE(! Overlaps("451", "500"));
 456   ASSERT_TRUE(! Overlaps("351", "399"));
 457
 458   ASSERT_TRUE(Overlaps("100", "150"));
 459   ASSERT_TRUE(Overlaps("100", "200"));
 460   ASSERT_TRUE(Overlaps("100", "300"));
 461   ASSERT_TRUE(Overlaps("100", "400"));
 462   ASSERT_TRUE(Overlaps("100", "500"));
 463   ASSERT_TRUE(Overlaps("375", "400"));
 464   ASSERT_TRUE(Overlaps("450", "450"));
 465   ASSERT_TRUE(Overlaps("450", "500"));
 466 }
 467
 468 TEST_F(FindLevelFileTest, LevelMultipleNullBoundaries) {
 469   LevelFileInit(4);
 470
 471   Add("150", "200");
 472   Add("200", "250");
 473   Add("300", "350");
 474   Add("400", "450");
 475   ASSERT_TRUE(! Overlaps(nullptr, "149"));
 476   ASSERT_TRUE(! Overlaps("451", nullptr));
 477   ASSERT_TRUE(Overlaps(nullptr, nullptr));
 478   ASSERT_TRUE(Overlaps(nullptr, "150"));
 479   ASSERT_TRUE(Overlaps(nullptr, "199"));
 480   ASSERT_TRUE(Overlaps(nullptr, "200"));
 481   ASSERT_TRUE(Overlaps(nullptr, "201"));
 482   ASSERT_TRUE(Overlaps(nullptr, "400"));
 483   ASSERT_TRUE(Overlaps(nullptr, "800"));
 484   ASSERT_TRUE(Overlaps("100", nullptr));
 485   ASSERT_TRUE(Overlaps("200", nullptr));
 486   ASSERT_TRUE(Overlaps("449", nullptr));
 487   ASSERT_TRUE(Overlaps("450", nullptr));
 488 }
 489
 490 TEST_F(FindLevelFileTest, LevelOverlapSequenceChecks) {
 491   LevelFileInit(1);
 492
 493   Add("200", "200", 5000, 3000);
 494   ASSERT_TRUE(! Overlaps("199", "199"));
 495   ASSERT_TRUE(! Overlaps("201", "300"));
 496   ASSERT_TRUE(Overlaps("200", "200"));
 497   ASSERT_TRUE(Overlaps("190", "200"));
 498   ASSERT_TRUE(Overlaps("200", "210"));
 499 }
 500
 501 TEST_F(FindLevelFileTest, LevelOverlappingFiles) {
 502   LevelFileInit(2);
 503
 504   Add("150", "600");
 505   Add("400", "500");
 506   disjoint_sorted_files_ = false;
 507   ASSERT_TRUE(! Overlaps("100", "149"));
 508   ASSERT_TRUE(! Overlaps("601", "700"));
 509   ASSERT_TRUE(Overlaps("100", "150"));
 510   ASSERT_TRUE(Overlaps("100", "200"));
 511   ASSERT_TRUE(Overlaps("100", "300"));
 512   ASSERT_TRUE(Overlaps("100", "400"));
 513   ASSERT_TRUE(Overlaps("100", "500"));
 514   ASSERT_TRUE(Overlaps("375", "400"));
 515   ASSERT_TRUE(Overlaps("450", "450"));
 516   ASSERT_TRUE(Overlaps("450", "500"));
 517   ASSERT_TRUE(Overlaps("450", "700"));
 518   ASSERT_TRUE(Overlaps("600", "700"));
 519 }
 520
 521 class ManifestWriterTest : public testing::Test {
 522  public:
 523   ManifestWriterTest()
 524       : env_(Env::Default()),
 525         dbname_(test::PerThreadDBPath("version_set_test")),
 526         db_options_(),
 527         mutable_cf_options_(cf_options_),
 528         table_cache_(NewLRUCache(50000, 16)),
 529         write_buffer_manager_(db_options_.db_write_buffer_size),
 530         versions_(new VersionSet(dbname_, &db_options_, env_options_,
 531                                  table_cache_.get(), &write_buffer_manager_,
 532                                  &write_controller_)),
 533         shutting_down_(false),
 534         mock_table_factory_(std::make_shared<mock::MockTableFactory>()) {
 535     EXPECT_OK(env_->CreateDirIfMissing(dbname_));
 536     db_options_.db_paths.emplace_back(dbname_,
 537                                       std::numeric_limits<uint64_t>::max());
 538   }
 539
 540   // Create DB with 3 column families.
 541   void NewDB() {
 542     VersionEdit new_db;
 543     new_db.SetLogNumber(0);
 544     new_db.SetNextFile(2);
 545     new_db.SetLastSequence(0);
 546
 547     const std::vector<std::string> cf_names = {kDefaultColumnFamilyName,
 548                                                "alice", "bob"};
 549     const int kInitialNumOfCfs = static_cast<int>(cf_names.size());
 550     autovector<VersionEdit> new_cfs;
 551     uint64_t last_seq = 1;
 552     uint32_t cf_id = 1;
 553     for (int i = 1; i != kInitialNumOfCfs; ++i) {
 554       VersionEdit new_cf;
 555       new_cf.AddColumnFamily(cf_names[i]);
 556       new_cf.SetColumnFamily(cf_id++);
 557       new_cf.SetLogNumber(0);
 558       new_cf.SetNextFile(2);
 559       new_cf.SetLastSequence(last_seq++);
 560       new_cfs.emplace_back(new_cf);
 561     }
 562
 563     const std::string manifest = DescriptorFileName(dbname_, 1);
 564     unique_ptr<WritableFile> file;
 565     Status s = env_->NewWritableFile(
 566         manifest, &file, env_->OptimizeForManifestWrite(env_options_));
 567     ASSERT_OK(s);
 568     unique_ptr<WritableFileWriter> file_writer(
 569         new WritableFileWriter(std::move(file), manifest, env_options_));
 570     {
 571       log::Writer log(std::move(file_writer), 0, false);
 572       std::string record;
 573       new_db.EncodeTo(&record);
 574       s = log.AddRecord(record);
 575       for (const auto& e : new_cfs) {
 576         e.EncodeTo(&record);
 577         s = log.AddRecord(record);
 578         ASSERT_OK(s);
 579       }
 580     }
 581     ASSERT_OK(s);
 582     // Make "CURRENT" file point to the new manifest file.
 583     s = SetCurrentFile(env_, dbname_, 1, nullptr);
 584
 585     std::vector<ColumnFamilyDescriptor> column_families;
 586     cf_options_.table_factory = mock_table_factory_;
 587     for (const auto& cf_name : cf_names) {
 588       column_families.emplace_back(cf_name, cf_options_);
 589     }
 590
 591     EXPECT_OK(versions_->Recover(column_families, false));
 592     EXPECT_EQ(kInitialNumOfCfs,
 593               versions_->GetColumnFamilySet()->NumberOfColumnFamilies());
 594     for (auto cfd : *versions_->GetColumnFamilySet()) {
 595       cfds_.emplace_back(cfd);
 596     }
 597   }
 598
 599   Env* env_;
 600   const std::string dbname_;
 601   EnvOptions env_options_;
 602   ImmutableDBOptions db_options_;
 603   ColumnFamilyOptions cf_options_;
 604   MutableCFOptions mutable_cf_options_;
 605   std::shared_ptr<Cache> table_cache_;
 606   WriteController write_controller_;
 607   WriteBufferManager write_buffer_manager_;
 608   std::unique_ptr<VersionSet> versions_;
 609   InstrumentedMutex mutex_;
 610   std::atomic<bool> shutting_down_;
 611   std::shared_ptr<mock::MockTableFactory> mock_table_factory_;
 612   std::vector<ColumnFamilyData*> cfds_;
 613 };
 614
 615 TEST_F(ManifestWriterTest, SameColumnFamilyGroupCommit) {
 616   NewDB();
 617   const int kGroupSize = 5;
 618   std::vector<VersionEdit> edits(kGroupSize);
 619   std::vector<ColumnFamilyData*> cfds(kGroupSize, cfds_[0]);
 620   std::vector<MutableCFOptions> all_mutable_cf_options(kGroupSize,
 621                                                        mutable_cf_options_);
 622   std::vector<autovector<VersionEdit*>> edit_lists(kGroupSize);
 623   for (int i = 0; i != kGroupSize; ++i) {
 624     edit_lists[i].emplace_back(&edits[i]);
 625   }
 626
 627   int count = 0;
 628   SyncPoint::GetInstance()->SetCallBack(
 629       "VersionSet::ProcessManifestWrites:SameColumnFamily", [&](void* arg) {
 630         uint32_t* cf_id = reinterpret_cast<uint32_t*>(arg);
 631         EXPECT_EQ(0, *cf_id);
 632         ++count;
 633       });
 634   SyncPoint::GetInstance()->EnableProcessing();
 635   mutex_.Lock();
 636   Status s =
 637       versions_->LogAndApply(cfds, all_mutable_cf_options, edit_lists, &mutex_);
 638   mutex_.Unlock();
 639   EXPECT_OK(s);
 640   EXPECT_EQ(kGroupSize - 1, count);
 641 }
 642 }  // namespace rocksdb
 643
 644 int main(int argc, char** argv) {
 645   ::testing::InitGoogleTest(&argc, argv);
 646   return RUN_ALL_TESTS();
 647 }