ceph/src/rocksdb/monitoring/stats_history_test.cc

   1 //  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
   2 //  This source code is licensed under both the GPLv2 (found in the
   3 //  COPYING file in the root directory) and Apache 2.0 License
   4 //  (found in the LICENSE.Apache file in the root directory).
   5 //
   6 // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
   7 // Use of this source code is governed by a BSD-style license that can be
   8 // found in the LICENSE file. See the AUTHORS file for names of contributors.
   9 #include "rocksdb/stats_history.h"
  10
  11 #include <limits>
  12 #include <string>
  13 #include <unordered_map>
  14
  15 #include "db/column_family.h"
  16 #include "db/db_impl/db_impl.h"
  17 #include "db/db_test_util.h"
  18 #include "db/periodic_task_scheduler.h"
  19 #include "monitoring/persistent_stats_history.h"
  20 #include "options/options_helper.h"
  21 #include "port/stack_trace.h"
  22 #include "rocksdb/cache.h"
  23 #include "rocksdb/convenience.h"
  24 #include "rocksdb/rate_limiter.h"
  25 #include "test_util/mock_time_env.h"
  26 #include "test_util/sync_point.h"
  27 #include "test_util/testutil.h"
  28 #include "util/random.h"
  29
  30 namespace ROCKSDB_NAMESPACE {
  31
  32 #ifndef ROCKSDB_LITE
  33 class StatsHistoryTest : public DBTestBase {
  34  public:
  35   StatsHistoryTest() : DBTestBase("stats_history_test", /*env_do_fsync=*/true) {
  36     mock_clock_ = std::make_shared<MockSystemClock>(env_->GetSystemClock());
  37     mock_env_.reset(new CompositeEnvWrapper(env_, mock_clock_));
  38   }
  39
  40  protected:
  41   std::shared_ptr<MockSystemClock> mock_clock_;
  42   std::unique_ptr<Env> mock_env_;
  43
  44   void SetUp() override {
  45     mock_clock_->InstallTimedWaitFixCallback();
  46     SyncPoint::GetInstance()->SetCallBack(
  47         "DBImpl::StartPeriodicTaskScheduler:Init", [&](void* arg) {
  48           auto periodic_task_scheduler_ptr =
  49               reinterpret_cast<PeriodicTaskScheduler*>(arg);
  50           periodic_task_scheduler_ptr->TEST_OverrideTimer(mock_clock_.get());
  51         });
  52   }
  53 };
  54
  55 TEST_F(StatsHistoryTest, RunStatsDumpPeriodSec) {
  56   constexpr int kPeriodSec = 5;
  57   Options options;
  58   options.create_if_missing = true;
  59   options.stats_dump_period_sec = kPeriodSec;
  60   options.env = mock_env_.get();
  61   int counter = 0;
  62   SyncPoint::GetInstance()->SetCallBack("DBImpl::DumpStats:1",
  63                                         [&](void* /*arg*/) { counter++; });
  64   Reopen(options);
  65   ASSERT_EQ(5u, dbfull()->GetDBOptions().stats_dump_period_sec);
  66
  67   // Wait for the first stats persist to finish, as the initial delay could be
  68   // different.
  69   dbfull()->TEST_WaitForPeridicTaskRun(
  70       [&] { mock_clock_->MockSleepForSeconds(kPeriodSec - 1); });
  71
  72   dbfull()->TEST_WaitForPeridicTaskRun(
  73       [&] { mock_clock_->MockSleepForSeconds(kPeriodSec); });
  74   ASSERT_GE(counter, 1);
  75
  76   // Test cancel job through SetOptions
  77   ASSERT_OK(dbfull()->SetDBOptions({{"stats_dump_period_sec", "0"}}));
  78   int old_val = counter;
  79   for (int i = 1; i < 20; ++i) {
  80     mock_clock_->MockSleepForSeconds(kPeriodSec);
  81   }
  82   ASSERT_EQ(counter, old_val);
  83   Close();
  84 }
  85
  86 // Test persistent stats background thread scheduling and cancelling
  87 TEST_F(StatsHistoryTest, StatsPersistScheduling) {
  88   constexpr int kPeriodSec = 5;
  89   Options options;
  90   options.create_if_missing = true;
  91   options.stats_persist_period_sec = kPeriodSec;
  92   options.env = mock_env_.get();
  93   int counter = 0;
  94   SyncPoint::GetInstance()->SetCallBack("DBImpl::PersistStats:Entry",
  95                                         [&](void* /*arg*/) { counter++; });
  96   Reopen(options);
  97   ASSERT_EQ(5u, dbfull()->GetDBOptions().stats_persist_period_sec);
  98
  99   // Wait for the first stats persist to finish, as the initial delay could be
 100   // different.
 101   dbfull()->TEST_WaitForPeridicTaskRun(
 102       [&] { mock_clock_->MockSleepForSeconds(kPeriodSec - 1); });
 103
 104   dbfull()->TEST_WaitForPeridicTaskRun(
 105       [&] { mock_clock_->MockSleepForSeconds(kPeriodSec); });
 106   ASSERT_GE(counter, 1);
 107
 108   // Test cancel job through SetOptions
 109   ASSERT_OK(dbfull()->SetDBOptions({{"stats_persist_period_sec", "0"}}));
 110   int old_val = counter;
 111   dbfull()->TEST_WaitForPeridicTaskRun(
 112       [&] { mock_clock_->MockSleepForSeconds(kPeriodSec * 2); });
 113   ASSERT_EQ(counter, old_val);
 114
 115   Close();
 116 }
 117
 118 // Test enabling persistent stats for the first time
 119 TEST_F(StatsHistoryTest, PersistentStatsFreshInstall) {
 120   constexpr unsigned int kPeriodSec = 5;
 121   Options options;
 122   options.create_if_missing = true;
 123   options.stats_persist_period_sec = 0;
 124   options.env = mock_env_.get();
 125   int counter = 0;
 126   SyncPoint::GetInstance()->SetCallBack("DBImpl::PersistStats:Entry",
 127                                         [&](void* /*arg*/) { counter++; });
 128   Reopen(options);
 129   ASSERT_OK(dbfull()->SetDBOptions(
 130       {{"stats_persist_period_sec", std::to_string(kPeriodSec)}}));
 131   ASSERT_EQ(kPeriodSec, dbfull()->GetDBOptions().stats_persist_period_sec);
 132
 133   dbfull()->TEST_WaitForPeridicTaskRun(
 134       [&] { mock_clock_->MockSleepForSeconds(kPeriodSec); });
 135   ASSERT_GE(counter, 1);
 136   Close();
 137 }
 138
 139 // TODO(Zhongyi): Move persistent stats related tests to a separate file
 140 TEST_F(StatsHistoryTest, GetStatsHistoryInMemory) {
 141   constexpr int kPeriodSec = 5;
 142   Options options;
 143   options.create_if_missing = true;
 144   options.stats_persist_period_sec = kPeriodSec;
 145   options.statistics = CreateDBStatistics();
 146   options.env = mock_env_.get();
 147   CreateColumnFamilies({"pikachu"}, options);
 148   ASSERT_OK(Put("foo", "bar"));
 149   ReopenWithColumnFamilies({"default", "pikachu"}, options);
 150
 151   // make sure the first stats persist to finish
 152   dbfull()->TEST_WaitForPeridicTaskRun(
 153       [&] { mock_clock_->MockSleepForSeconds(kPeriodSec - 1); });
 154
 155   // Wait for stats persist to finish
 156   dbfull()->TEST_WaitForPeridicTaskRun(
 157       [&] { mock_clock_->MockSleepForSeconds(kPeriodSec); });
 158
 159   std::unique_ptr<StatsHistoryIterator> stats_iter;
 160   ASSERT_OK(
 161       db_->GetStatsHistory(0, mock_clock_->NowSeconds() + 1, &stats_iter));
 162   ASSERT_TRUE(stats_iter != nullptr);
 163   // disabled stats snapshots
 164   ASSERT_OK(dbfull()->SetDBOptions({{"stats_persist_period_sec", "0"}}));
 165   size_t stats_count = 0;
 166   for (; stats_iter->Valid(); stats_iter->Next()) {
 167     auto stats_map = stats_iter->GetStatsMap();
 168     ASSERT_EQ(stats_iter->GetStatsTime(), mock_clock_->NowSeconds());
 169     stats_count += stats_map.size();
 170   }
 171   ASSERT_GT(stats_count, 0);
 172   // Wait a bit and verify no more stats are found
 173   for (int i = 0; i < 10; ++i) {
 174     dbfull()->TEST_WaitForPeridicTaskRun(
 175         [&] { mock_clock_->MockSleepForSeconds(1); });
 176   }
 177   ASSERT_OK(db_->GetStatsHistory(0, mock_clock_->NowSeconds(), &stats_iter));
 178   ASSERT_TRUE(stats_iter != nullptr);
 179   size_t stats_count_new = 0;
 180   for (; stats_iter->Valid(); stats_iter->Next()) {
 181     stats_count_new += stats_iter->GetStatsMap().size();
 182   }
 183   ASSERT_EQ(stats_count_new, stats_count);
 184   Close();
 185 }
 186
 187 TEST_F(StatsHistoryTest, InMemoryStatsHistoryPurging) {
 188   constexpr int kPeriodSec = 1;
 189   Options options;
 190   options.create_if_missing = true;
 191   options.statistics = CreateDBStatistics();
 192   options.stats_persist_period_sec = kPeriodSec;
 193   options.env = mock_env_.get();
 194
 195   CreateColumnFamilies({"pikachu"}, options);
 196   ASSERT_OK(Put("foo", "bar"));
 197   ReopenWithColumnFamilies({"default", "pikachu"}, options);
 198   // some random operation to populate statistics
 199   ASSERT_OK(Delete("foo"));
 200   ASSERT_OK(Put("sol", "sol"));
 201   ASSERT_OK(Put("epic", "epic"));
 202   ASSERT_OK(Put("ltd", "ltd"));
 203   ASSERT_EQ("sol", Get("sol"));
 204   ASSERT_EQ("epic", Get("epic"));
 205   ASSERT_EQ("ltd", Get("ltd"));
 206   Iterator* iterator = db_->NewIterator(ReadOptions());
 207   for (iterator->SeekToFirst(); iterator->Valid(); iterator->Next()) {
 208     ASSERT_TRUE(iterator->key() == iterator->value());
 209   }
 210   delete iterator;
 211   ASSERT_OK(Flush());
 212   ASSERT_OK(Delete("sol"));
 213   ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr));
 214
 215   // second round of ops
 216   ASSERT_OK(Put("saigon", "saigon"));
 217   ASSERT_OK(Put("noodle talk", "noodle talk"));
 218   ASSERT_OK(Put("ping bistro", "ping bistro"));
 219   iterator = db_->NewIterator(ReadOptions());
 220   for (iterator->SeekToFirst(); iterator->Valid(); iterator->Next()) {
 221     ASSERT_TRUE(iterator->key() == iterator->value());
 222   }
 223   delete iterator;
 224   ASSERT_OK(Flush());
 225   ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr));
 226
 227   const int kIterations = 10;
 228   for (int i = 0; i < kIterations; ++i) {
 229     dbfull()->TEST_WaitForPeridicTaskRun(
 230         [&] { mock_clock_->MockSleepForSeconds(kPeriodSec); });
 231   }
 232
 233   std::unique_ptr<StatsHistoryIterator> stats_iter;
 234   ASSERT_OK(
 235       db_->GetStatsHistory(0, mock_clock_->NowSeconds() + 1, &stats_iter));
 236   ASSERT_TRUE(stats_iter != nullptr);
 237   size_t stats_count = 0;
 238   int slice_count = 0;
 239   for (; stats_iter->Valid(); stats_iter->Next()) {
 240     slice_count++;
 241     auto stats_map = stats_iter->GetStatsMap();
 242     stats_count += stats_map.size();
 243   }
 244   size_t stats_history_size = dbfull()->TEST_EstimateInMemoryStatsHistorySize();
 245   ASSERT_GE(slice_count, kIterations - 1);
 246   ASSERT_GE(stats_history_size, 15000);
 247   // capping memory cost at 15000 bytes since one slice is around 10000~15000
 248   ASSERT_OK(dbfull()->SetDBOptions({{"stats_history_buffer_size", "15000"}}));
 249   ASSERT_EQ(15000, dbfull()->GetDBOptions().stats_history_buffer_size);
 250
 251   // Wait for stats persist to finish
 252   for (int i = 0; i < kIterations; ++i) {
 253     dbfull()->TEST_WaitForPeridicTaskRun(
 254         [&] { mock_clock_->MockSleepForSeconds(kPeriodSec); });
 255   }
 256
 257   ASSERT_OK(
 258       db_->GetStatsHistory(0, mock_clock_->NowSeconds() + 1, &stats_iter));
 259   ASSERT_TRUE(stats_iter != nullptr);
 260   size_t stats_count_reopen = 0;
 261   slice_count = 0;
 262   for (; stats_iter->Valid(); stats_iter->Next()) {
 263     slice_count++;
 264     auto stats_map = stats_iter->GetStatsMap();
 265     stats_count_reopen += stats_map.size();
 266   }
 267   size_t stats_history_size_reopen =
 268       dbfull()->TEST_EstimateInMemoryStatsHistorySize();
 269   // only one slice can fit under the new stats_history_buffer_size
 270   ASSERT_LT(slice_count, 2);
 271   ASSERT_TRUE(stats_history_size_reopen < 15000 &&
 272               stats_history_size_reopen > 0);
 273   ASSERT_TRUE(stats_count_reopen < stats_count && stats_count_reopen > 0);
 274   Close();
 275   // TODO: may also want to verify stats timestamp to make sure we are purging
 276   // the correct stats snapshot
 277 }
 278
 279 int countkeys(Iterator* iter) {
 280   int count = 0;
 281   for (iter->SeekToFirst(); iter->Valid(); iter->Next()) {
 282     count++;
 283   }
 284   return count;
 285 }
 286
 287 TEST_F(StatsHistoryTest, GetStatsHistoryFromDisk) {
 288   constexpr int kPeriodSec = 5;
 289   Options options;
 290   options.create_if_missing = true;
 291   options.stats_persist_period_sec = kPeriodSec;
 292   options.statistics = CreateDBStatistics();
 293   options.persist_stats_to_disk = true;
 294   options.env = mock_env_.get();
 295   CreateColumnFamilies({"pikachu"}, options);
 296   ASSERT_OK(Put("foo", "bar"));
 297   ReopenWithColumnFamilies({"default", "pikachu"}, options);
 298   ASSERT_EQ(Get("foo"), "bar");
 299
 300   // Wait for the first stats persist to finish, as the initial delay could be
 301   // different.
 302   dbfull()->TEST_WaitForPeridicTaskRun(
 303       [&] { mock_clock_->MockSleepForSeconds(kPeriodSec - 1); });
 304
 305   // Wait for stats persist to finish
 306   dbfull()->TEST_WaitForPeridicTaskRun(
 307       [&] { mock_clock_->MockSleepForSeconds(kPeriodSec); });
 308
 309   auto iter =
 310       db_->NewIterator(ReadOptions(), dbfull()->PersistentStatsColumnFamily());
 311   int key_count1 = countkeys(iter);
 312   delete iter;
 313
 314   dbfull()->TEST_WaitForPeridicTaskRun(
 315       [&] { mock_clock_->MockSleepForSeconds(kPeriodSec); });
 316   iter =
 317       db_->NewIterator(ReadOptions(), dbfull()->PersistentStatsColumnFamily());
 318   int key_count2 = countkeys(iter);
 319   delete iter;
 320
 321   dbfull()->TEST_WaitForPeridicTaskRun(
 322       [&] { mock_clock_->MockSleepForSeconds(kPeriodSec); });
 323   iter =
 324       db_->NewIterator(ReadOptions(), dbfull()->PersistentStatsColumnFamily());
 325   int key_count3 = countkeys(iter);
 326   delete iter;
 327   ASSERT_GE(key_count2, key_count1);
 328   ASSERT_GE(key_count3, key_count2);
 329   ASSERT_EQ(key_count3 - key_count2, key_count2 - key_count1);
 330   std::unique_ptr<StatsHistoryIterator> stats_iter;
 331   ASSERT_OK(
 332       db_->GetStatsHistory(0, mock_clock_->NowSeconds() + 1, &stats_iter));
 333   ASSERT_TRUE(stats_iter != nullptr);
 334   size_t stats_count = 0;
 335   int slice_count = 0;
 336   int non_zero_count = 0;
 337   for (int i = 2; stats_iter->Valid(); stats_iter->Next(), i++) {
 338     slice_count++;
 339     auto stats_map = stats_iter->GetStatsMap();
 340     ASSERT_EQ(stats_iter->GetStatsTime(), kPeriodSec * i - 1);
 341     for (auto& stat : stats_map) {
 342       if (stat.second != 0) {
 343         non_zero_count++;
 344       }
 345     }
 346     stats_count += stats_map.size();
 347   }
 348   ASSERT_EQ(slice_count, 3);
 349   // 2 extra keys for format version
 350   ASSERT_EQ(stats_count, key_count3 - 2);
 351   // verify reopen will not cause data loss
 352   ReopenWithColumnFamilies({"default", "pikachu"}, options);
 353   ASSERT_OK(
 354       db_->GetStatsHistory(0, mock_clock_->NowSeconds() + 1, &stats_iter));
 355   ASSERT_TRUE(stats_iter != nullptr);
 356   size_t stats_count_reopen = 0;
 357   int slice_count_reopen = 0;
 358   int non_zero_count_recover = 0;
 359   for (; stats_iter->Valid(); stats_iter->Next()) {
 360     slice_count_reopen++;
 361     auto stats_map = stats_iter->GetStatsMap();
 362     for (auto& stat : stats_map) {
 363       if (stat.second != 0) {
 364         non_zero_count_recover++;
 365       }
 366     }
 367     stats_count_reopen += stats_map.size();
 368   }
 369
 370   ASSERT_EQ(non_zero_count, non_zero_count_recover);
 371   ASSERT_EQ(slice_count, slice_count_reopen);
 372   ASSERT_EQ(stats_count, stats_count_reopen);
 373   Close();
 374 }
 375
 376 // Test persisted stats matches the value found in options.statistics and
 377 // the stats value retains after DB reopen
 378 TEST_F(StatsHistoryTest, PersitentStatsVerifyValue) {
 379   constexpr int kPeriodSec = 5;
 380   Options options;
 381   options.create_if_missing = true;
 382   options.stats_persist_period_sec = kPeriodSec;
 383   options.statistics = CreateDBStatistics();
 384   options.persist_stats_to_disk = true;
 385   std::map<std::string, uint64_t> stats_map_before;
 386   ASSERT_TRUE(options.statistics->getTickerMap(&stats_map_before));
 387   options.env = mock_env_.get();
 388   CreateColumnFamilies({"pikachu"}, options);
 389   ASSERT_OK(Put("foo", "bar"));
 390   ReopenWithColumnFamilies({"default", "pikachu"}, options);
 391   ASSERT_EQ(Get("foo"), "bar");
 392
 393   // Wait for the first stats persist to finish, as the initial delay could be
 394   // different.
 395   dbfull()->TEST_WaitForPeridicTaskRun(
 396       [&] { mock_clock_->MockSleepForSeconds(kPeriodSec - 1); });
 397
 398   // Wait for stats persist to finish
 399   dbfull()->TEST_WaitForPeridicTaskRun(
 400       [&] { mock_clock_->MockSleepForSeconds(kPeriodSec); });
 401   auto iter =
 402       db_->NewIterator(ReadOptions(), dbfull()->PersistentStatsColumnFamily());
 403   countkeys(iter);
 404   delete iter;
 405
 406   dbfull()->TEST_WaitForPeridicTaskRun(
 407       [&] { mock_clock_->MockSleepForSeconds(kPeriodSec); });
 408   iter =
 409       db_->NewIterator(ReadOptions(), dbfull()->PersistentStatsColumnFamily());
 410   countkeys(iter);
 411   delete iter;
 412
 413   dbfull()->TEST_WaitForPeridicTaskRun(
 414       [&] { mock_clock_->MockSleepForSeconds(kPeriodSec); });
 415   iter =
 416       db_->NewIterator(ReadOptions(), dbfull()->PersistentStatsColumnFamily());
 417   countkeys(iter);
 418   delete iter;
 419
 420   dbfull()->TEST_WaitForPeridicTaskRun(
 421       [&] { mock_clock_->MockSleepForSeconds(kPeriodSec); });
 422
 423   std::map<std::string, uint64_t> stats_map_after;
 424   ASSERT_TRUE(options.statistics->getTickerMap(&stats_map_after));
 425   std::unique_ptr<StatsHistoryIterator> stats_iter;
 426   ASSERT_OK(
 427       db_->GetStatsHistory(0, mock_clock_->NowSeconds() + 1, &stats_iter));
 428   ASSERT_TRUE(stats_iter != nullptr);
 429   std::string sample = "rocksdb.num.iterator.deleted";
 430   uint64_t recovered_value = 0;
 431   for (int i = 2; stats_iter->Valid(); stats_iter->Next(), ++i) {
 432     auto stats_map = stats_iter->GetStatsMap();
 433     ASSERT_EQ(stats_iter->GetStatsTime(), kPeriodSec * i - 1);
 434     for (const auto& stat : stats_map) {
 435       if (sample.compare(stat.first) == 0) {
 436         recovered_value += stat.second;
 437       }
 438     }
 439   }
 440   ASSERT_EQ(recovered_value, stats_map_after[sample]);
 441
 442   // test stats value retains after recovery
 443   ReopenWithColumnFamilies({"default", "pikachu"}, options);
 444   ASSERT_OK(
 445       db_->GetStatsHistory(0, mock_clock_->NowSeconds() + 1, &stats_iter));
 446   ASSERT_TRUE(stats_iter != nullptr);
 447   uint64_t new_recovered_value = 0;
 448   for (int i = 2; stats_iter->Valid(); stats_iter->Next(), i++) {
 449     auto stats_map = stats_iter->GetStatsMap();
 450     ASSERT_EQ(stats_iter->GetStatsTime(), kPeriodSec * i - 1);
 451     for (const auto& stat : stats_map) {
 452       if (sample.compare(stat.first) == 0) {
 453         new_recovered_value += stat.second;
 454       }
 455     }
 456   }
 457   ASSERT_EQ(recovered_value, new_recovered_value);
 458
 459   // TODO(Zhongyi): also add test to read raw values from disk and verify
 460   // correctness
 461   Close();
 462 }
 463
 464 // TODO(Zhongyi): add test for different format versions
 465
 466 TEST_F(StatsHistoryTest, PersistentStatsCreateColumnFamilies) {
 467   constexpr int kPeriodSec = 5;
 468   Options options;
 469   options.create_if_missing = true;
 470   options.stats_persist_period_sec = kPeriodSec;
 471   options.statistics = CreateDBStatistics();
 472   options.persist_stats_to_disk = true;
 473   options.env = mock_env_.get();
 474   ASSERT_OK(TryReopen(options));
 475   CreateColumnFamilies({"one", "two", "three"}, options);
 476   ASSERT_OK(Put(1, "foo", "bar"));
 477   ReopenWithColumnFamilies({"default", "one", "two", "three"}, options);
 478   ASSERT_EQ(Get(2, "foo"), "bar");
 479   CreateColumnFamilies({"four"}, options);
 480   ReopenWithColumnFamilies({"default", "one", "two", "three", "four"}, options);
 481   ASSERT_EQ(Get(2, "foo"), "bar");
 482
 483   // make sure the first stats persist to finish
 484   dbfull()->TEST_WaitForPeridicTaskRun(
 485       [&] { mock_clock_->MockSleepForSeconds(kPeriodSec - 1); });
 486
 487   dbfull()->TEST_WaitForPeridicTaskRun(
 488       [&] { mock_clock_->MockSleepForSeconds(kPeriodSec); });
 489   auto iter =
 490       db_->NewIterator(ReadOptions(), dbfull()->PersistentStatsColumnFamily());
 491   int key_count = countkeys(iter);
 492   delete iter;
 493   ASSERT_GE(key_count, 0);
 494   uint64_t num_write_wal = 0;
 495   std::string sample = "rocksdb.write.wal";
 496   std::unique_ptr<StatsHistoryIterator> stats_iter;
 497   ASSERT_OK(db_->GetStatsHistory(0, mock_clock_->NowSeconds(), &stats_iter));
 498   ASSERT_TRUE(stats_iter != nullptr);
 499   for (; stats_iter->Valid(); stats_iter->Next()) {
 500     auto stats_map = stats_iter->GetStatsMap();
 501     for (const auto& stat : stats_map) {
 502       if (sample.compare(stat.first) == 0) {
 503         num_write_wal += stat.second;
 504       }
 505     }
 506   }
 507   stats_iter.reset();
 508   ASSERT_EQ(num_write_wal, 1);
 509
 510   options.persist_stats_to_disk = false;
 511   ReopenWithColumnFamilies({"default", "one", "two", "three", "four"}, options);
 512   int cf_count = 0;
 513   for (auto cfd : *dbfull()->versions_->GetColumnFamilySet()) {
 514     (void)cfd;
 515     cf_count++;
 516   }
 517   // persistent stats cf will be implicitly opened even if
 518   // persist_stats_to_disk is false
 519   ASSERT_EQ(cf_count, 6);
 520   ASSERT_EQ(Get(2, "foo"), "bar");
 521
 522   // attempt to create column family using same name, should fail
 523   ColumnFamilyOptions cf_opts(options);
 524   ColumnFamilyHandle* handle;
 525   ASSERT_NOK(db_->CreateColumnFamily(cf_opts, kPersistentStatsColumnFamilyName,
 526                                      &handle));
 527
 528   options.persist_stats_to_disk = true;
 529   ReopenWithColumnFamilies({"default", "one", "two", "three", "four"}, options);
 530   ASSERT_NOK(db_->CreateColumnFamily(cf_opts, kPersistentStatsColumnFamilyName,
 531                                      &handle));
 532   // verify stats is not affected by prior failed CF creation
 533   ASSERT_OK(db_->GetStatsHistory(0, mock_clock_->NowSeconds(), &stats_iter));
 534   ASSERT_TRUE(stats_iter != nullptr);
 535   num_write_wal = 0;
 536   for (; stats_iter->Valid(); stats_iter->Next()) {
 537     auto stats_map = stats_iter->GetStatsMap();
 538     for (const auto& stat : stats_map) {
 539       if (sample.compare(stat.first) == 0) {
 540         num_write_wal += stat.second;
 541       }
 542     }
 543   }
 544   ASSERT_EQ(num_write_wal, 1);
 545
 546   Close();
 547   Destroy(options);
 548 }
 549
 550 TEST_F(StatsHistoryTest, PersistentStatsReadOnly) {
 551   ASSERT_OK(Put("bar", "v2"));
 552   Close();
 553
 554   auto options = CurrentOptions();
 555   options.stats_persist_period_sec = 5;
 556   options.persist_stats_to_disk = true;
 557   assert(options.env == env_);
 558   ASSERT_OK(ReadOnlyReopen(options));
 559   ASSERT_EQ("v2", Get("bar"));
 560   Close();
 561
 562   // Reopen and flush memtable.
 563   ASSERT_OK(TryReopen(options));
 564   ASSERT_OK(Flush());
 565   Close();
 566   // Now check keys in read only mode.
 567   ASSERT_OK(ReadOnlyReopen(options));
 568 }
 569
 570 TEST_F(StatsHistoryTest, ForceManualFlushStatsCF) {
 571   constexpr int kPeriodSec = 5;
 572   Options options;
 573   options.create_if_missing = true;
 574   options.write_buffer_size = 1024 * 1024 * 10;  // 10 Mb
 575   options.stats_persist_period_sec = kPeriodSec;
 576   options.statistics = CreateDBStatistics();
 577   options.persist_stats_to_disk = true;
 578   options.env = mock_env_.get();
 579   CreateColumnFamilies({"pikachu"}, options);
 580   ReopenWithColumnFamilies({"default", "pikachu"}, options);
 581
 582   // Wait for the first stats persist to finish, as the initial delay could be
 583   // different.
 584   dbfull()->TEST_WaitForPeridicTaskRun(
 585       [&] { mock_clock_->MockSleepForSeconds(kPeriodSec - 1); });
 586
 587   ColumnFamilyData* cfd_default =
 588       static_cast<ColumnFamilyHandleImpl*>(dbfull()->DefaultColumnFamily())
 589           ->cfd();
 590   ColumnFamilyData* cfd_stats = static_cast<ColumnFamilyHandleImpl*>(
 591                                     dbfull()->PersistentStatsColumnFamily())
 592                                     ->cfd();
 593   ColumnFamilyData* cfd_test =
 594       static_cast<ColumnFamilyHandleImpl*>(handles_[1])->cfd();
 595
 596   ASSERT_OK(Put("foo", "v0"));
 597   ASSERT_OK(Put("bar", "v0"));
 598   ASSERT_EQ("v0", Get("bar"));
 599   ASSERT_EQ("v0", Get("foo"));
 600   ASSERT_OK(Put(1, "Eevee", "v0"));
 601   ASSERT_EQ("v0", Get(1, "Eevee"));
 602
 603   dbfull()->TEST_WaitForPeridicTaskRun(
 604       [&] { mock_clock_->MockSleepForSeconds(kPeriodSec); });
 605   // writing to all three cf, flush default cf
 606   // LogNumbers: default: 16, stats: 10, pikachu: 5
 607   // Since in recovery process, cfd_stats column is created after WAL is
 608   // created, synced and MANIFEST is persisted, its log number which depends on
 609   // logfile_number_ will be different. Since "pikachu" is never flushed, thus
 610   // its log_number should be the smallest of the three.
 611   ASSERT_OK(Flush());
 612   ASSERT_LT(cfd_test->GetLogNumber(), cfd_stats->GetLogNumber());
 613   ASSERT_LT(cfd_test->GetLogNumber(), cfd_default->GetLogNumber());
 614
 615   ASSERT_OK(Put("foo1", "v1"));
 616   ASSERT_OK(Put("bar1", "v1"));
 617   ASSERT_EQ("v1", Get("bar1"));
 618   ASSERT_EQ("v1", Get("foo1"));
 619   ASSERT_OK(Put(1, "Vaporeon", "v1"));
 620   ASSERT_EQ("v1", Get(1, "Vaporeon"));
 621   // writing to default and test cf, flush test cf
 622   // LogNumbers: default: 14, stats: 16, pikachu: 16
 623   ASSERT_OK(Flush(1));
 624   ASSERT_EQ(cfd_stats->GetLogNumber(), cfd_test->GetLogNumber());
 625   ASSERT_GT(cfd_stats->GetLogNumber(), cfd_default->GetLogNumber());
 626
 627   ASSERT_OK(Put("foo2", "v2"));
 628   ASSERT_OK(Put("bar2", "v2"));
 629   ASSERT_EQ("v2", Get("bar2"));
 630   ASSERT_EQ("v2", Get("foo2"));
 631
 632   dbfull()->TEST_WaitForPeridicTaskRun(
 633       [&] { mock_clock_->MockSleepForSeconds(kPeriodSec); });
 634   // writing to default and stats cf, flushing default cf
 635   // LogNumbers: default: 19, stats: 19, pikachu: 19
 636   ASSERT_OK(Flush());
 637   ASSERT_EQ(cfd_stats->GetLogNumber(), cfd_test->GetLogNumber());
 638   ASSERT_EQ(cfd_stats->GetLogNumber(), cfd_default->GetLogNumber());
 639
 640   ASSERT_OK(Put("foo3", "v3"));
 641   ASSERT_OK(Put("bar3", "v3"));
 642   ASSERT_EQ("v3", Get("bar3"));
 643   ASSERT_EQ("v3", Get("foo3"));
 644   ASSERT_OK(Put(1, "Jolteon", "v3"));
 645   ASSERT_EQ("v3", Get(1, "Jolteon"));
 646
 647   dbfull()->TEST_WaitForPeridicTaskRun(
 648       [&] { mock_clock_->MockSleepForSeconds(kPeriodSec); });
 649   // writing to all three cf, flushing test cf
 650   // LogNumbers: default: 19, stats: 19, pikachu: 22
 651   ASSERT_OK(Flush(1));
 652   ASSERT_LT(cfd_stats->GetLogNumber(), cfd_test->GetLogNumber());
 653   ASSERT_EQ(cfd_stats->GetLogNumber(), cfd_default->GetLogNumber());
 654   Close();
 655 }
 656
 657 #endif  // !ROCKSDB_LITE
 658 }  // namespace ROCKSDB_NAMESPACE
 659
 660 int main(int argc, char** argv) {
 661   ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
 662   ::testing::InitGoogleTest(&argc, argv);
 663   return RUN_ALL_TESTS();
 664 }