ceph/src/rocksdb/memtable/write_buffer_manager_test.cc

   1 //  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
   2 //  This source code is licensed under both the GPLv2 (found in the
   3 //  COPYING file in the root directory) and Apache 2.0 License
   4 //  (found in the LICENSE.Apache file in the root directory).
   5 //
   6 // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
   7 // Use of this source code is governed by a BSD-style license that can be
   8 // found in the LICENSE file. See the AUTHORS file for names of contributors.
   9
  10 #include "rocksdb/write_buffer_manager.h"
  11 #include "test_util/testharness.h"
  12
  13 namespace ROCKSDB_NAMESPACE {
  14
  15 class WriteBufferManagerTest : public testing::Test {};
  16
  17 #ifndef ROCKSDB_LITE
  18 TEST_F(WriteBufferManagerTest, ShouldFlush) {
  19   // A write buffer manager of size 10MB
  20   std::unique_ptr<WriteBufferManager> wbf(
  21       new WriteBufferManager(10 * 1024 * 1024));
  22
  23   wbf->ReserveMem(8 * 1024 * 1024);
  24   ASSERT_FALSE(wbf->ShouldFlush());
  25   // 90% of the hard limit will hit the condition
  26   wbf->ReserveMem(1 * 1024 * 1024);
  27   ASSERT_TRUE(wbf->ShouldFlush());
  28   // Scheduling for freeing will release the condition
  29   wbf->ScheduleFreeMem(1 * 1024 * 1024);
  30   ASSERT_FALSE(wbf->ShouldFlush());
  31
  32   wbf->ReserveMem(2 * 1024 * 1024);
  33   ASSERT_TRUE(wbf->ShouldFlush());
  34
  35   wbf->ScheduleFreeMem(4 * 1024 * 1024);
  36   // 11MB total, 6MB mutable. hard limit still hit
  37   ASSERT_TRUE(wbf->ShouldFlush());
  38
  39   wbf->ScheduleFreeMem(2 * 1024 * 1024);
  40   // 11MB total, 4MB mutable. hard limit stills but won't flush because more
  41   // than half data is already being flushed.
  42   ASSERT_FALSE(wbf->ShouldFlush());
  43
  44   wbf->ReserveMem(4 * 1024 * 1024);
  45   // 15 MB total, 8MB mutable.
  46   ASSERT_TRUE(wbf->ShouldFlush());
  47
  48   wbf->FreeMem(7 * 1024 * 1024);
  49   // 9MB total, 8MB mutable.
  50   ASSERT_FALSE(wbf->ShouldFlush());
  51 }
  52
  53 TEST_F(WriteBufferManagerTest, CacheCost) {
  54   LRUCacheOptions co;
  55   // 1GB cache
  56   co.capacity = 1024 * 1024 * 1024;
  57   co.num_shard_bits = 4;
  58   co.metadata_charge_policy = kDontChargeCacheMetadata;
  59   std::shared_ptr<Cache> cache = NewLRUCache(co);
  60   // A write buffer manager of size 50MB
  61   std::unique_ptr<WriteBufferManager> wbf(
  62       new WriteBufferManager(50 * 1024 * 1024, cache));
  63
  64   // Allocate 333KB will allocate 512KB
  65   wbf->ReserveMem(333 * 1024);
  66   ASSERT_GE(cache->GetPinnedUsage(), 2 * 256 * 1024);
  67   ASSERT_LT(cache->GetPinnedUsage(), 2 * 256 * 1024 + 10000);
  68
  69   // Allocate another 512KB
  70   wbf->ReserveMem(512 * 1024);
  71   ASSERT_GE(cache->GetPinnedUsage(), 4 * 256 * 1024);
  72   ASSERT_LT(cache->GetPinnedUsage(), 4 * 256 * 1024 + 10000);
  73
  74   // Allocate another 10MB
  75   wbf->ReserveMem(10 * 1024 * 1024);
  76   ASSERT_GE(cache->GetPinnedUsage(), 11 * 1024 * 1024);
  77   ASSERT_LT(cache->GetPinnedUsage(), 11 * 1024 * 1024 + 10000);
  78
  79   // Free 1MB will not cause any change in cache cost
  80   wbf->FreeMem(1024 * 1024);
  81   ASSERT_GE(cache->GetPinnedUsage(), 11 * 1024 * 1024);
  82   ASSERT_LT(cache->GetPinnedUsage(), 11 * 1024 * 1024 + 10000);
  83
  84   ASSERT_FALSE(wbf->ShouldFlush());
  85
  86   // Allocate another 41MB
  87   wbf->ReserveMem(41 * 1024 * 1024);
  88   ASSERT_GE(cache->GetPinnedUsage(), 51 * 1024 * 1024);
  89   ASSERT_LT(cache->GetPinnedUsage(), 51 * 1024 * 1024 + 10000);
  90   ASSERT_TRUE(wbf->ShouldFlush());
  91
  92   ASSERT_TRUE(wbf->ShouldFlush());
  93
  94   wbf->ScheduleFreeMem(20 * 1024 * 1024);
  95   ASSERT_GE(cache->GetPinnedUsage(), 51 * 1024 * 1024);
  96   ASSERT_LT(cache->GetPinnedUsage(), 51 * 1024 * 1024 + 10000);
  97
  98   // Still need flush as the hard limit hits
  99   ASSERT_TRUE(wbf->ShouldFlush());
 100
 101   // Free 20MB will releae 256KB from cache
 102   wbf->FreeMem(20 * 1024 * 1024);
 103   ASSERT_GE(cache->GetPinnedUsage(), 51 * 1024 * 1024 - 256 * 1024);
 104   ASSERT_LT(cache->GetPinnedUsage(), 51 * 1024 * 1024 - 256 * 1024 + 10000);
 105
 106   ASSERT_FALSE(wbf->ShouldFlush());
 107
 108   // Every free will release 256KB if still not hit 3/4
 109   wbf->FreeMem(16 * 1024);
 110   ASSERT_GE(cache->GetPinnedUsage(), 51 * 1024 * 1024 - 2 * 256 * 1024);
 111   ASSERT_LT(cache->GetPinnedUsage(), 51 * 1024 * 1024 - 2 * 256 * 1024 + 10000);
 112
 113   wbf->FreeMem(16 * 1024);
 114   ASSERT_GE(cache->GetPinnedUsage(), 51 * 1024 * 1024 - 3 * 256 * 1024);
 115   ASSERT_LT(cache->GetPinnedUsage(), 51 * 1024 * 1024 - 3 * 256 * 1024 + 10000);
 116
 117   // Reserve 512KB will not cause any change in cache cost
 118   wbf->ReserveMem(512 * 1024);
 119   ASSERT_GE(cache->GetPinnedUsage(), 51 * 1024 * 1024 - 3 * 256 * 1024);
 120   ASSERT_LT(cache->GetPinnedUsage(), 51 * 1024 * 1024 - 3 * 256 * 1024 + 10000);
 121
 122   wbf->FreeMem(16 * 1024);
 123   ASSERT_GE(cache->GetPinnedUsage(), 51 * 1024 * 1024 - 4 * 256 * 1024);
 124   ASSERT_LT(cache->GetPinnedUsage(), 51 * 1024 * 1024 - 4 * 256 * 1024 + 10000);
 125
 126   // Destory write buffer manger should free everything
 127   wbf.reset();
 128   ASSERT_LT(cache->GetPinnedUsage(), 1024 * 1024);
 129 }
 130
 131 TEST_F(WriteBufferManagerTest, NoCapCacheCost) {
 132   // 1GB cache
 133   std::shared_ptr<Cache> cache = NewLRUCache(1024 * 1024 * 1024, 4);
 134   // A write buffer manager of size 256MB
 135   std::unique_ptr<WriteBufferManager> wbf(new WriteBufferManager(0, cache));
 136   // Allocate 1.5MB will allocate 2MB
 137   wbf->ReserveMem(10 * 1024 * 1024);
 138   ASSERT_GE(cache->GetPinnedUsage(), 10 * 1024 * 1024);
 139   ASSERT_LT(cache->GetPinnedUsage(), 10 * 1024 * 1024 + 10000);
 140   ASSERT_FALSE(wbf->ShouldFlush());
 141
 142   wbf->FreeMem(9 * 1024 * 1024);
 143   for (int i = 0; i < 40; i++) {
 144     wbf->FreeMem(4 * 1024);
 145   }
 146   ASSERT_GE(cache->GetPinnedUsage(), 1024 * 1024);
 147   ASSERT_LT(cache->GetPinnedUsage(), 1024 * 1024 + 10000);
 148 }
 149
 150 TEST_F(WriteBufferManagerTest, CacheFull) {
 151   // 15MB cache size with strict capacity
 152   LRUCacheOptions lo;
 153   lo.capacity = 12 * 1024 * 1024;
 154   lo.num_shard_bits = 0;
 155   lo.strict_capacity_limit = true;
 156   std::shared_ptr<Cache> cache = NewLRUCache(lo);
 157   std::unique_ptr<WriteBufferManager> wbf(new WriteBufferManager(0, cache));
 158   wbf->ReserveMem(10 * 1024 * 1024);
 159   size_t prev_pinned = cache->GetPinnedUsage();
 160   ASSERT_GE(prev_pinned, 10 * 1024 * 1024);
 161   // Some insert will fail
 162   wbf->ReserveMem(10 * 1024 * 1024);
 163   ASSERT_LE(cache->GetPinnedUsage(), 12 * 1024 * 1024);
 164
 165   // Increase capacity so next insert will succeed
 166   cache->SetCapacity(30 * 1024 * 1024);
 167   wbf->ReserveMem(10 * 1024 * 1024);
 168   ASSERT_GT(cache->GetPinnedUsage(), 20 * 1024 * 1024);
 169
 170   // Gradually release 20 MB
 171   for (int i = 0; i < 40; i++) {
 172     wbf->FreeMem(512 * 1024);
 173   }
 174   ASSERT_GE(cache->GetPinnedUsage(), 10 * 1024 * 1024);
 175   ASSERT_LT(cache->GetPinnedUsage(), 20 * 1024 * 1024);
 176 }
 177
 178 #endif  // ROCKSDB_LITE
 179 }  // namespace ROCKSDB_NAMESPACE
 180
 181 int main(int argc, char** argv) {
 182   ::testing::InitGoogleTest(&argc, argv);
 183   return RUN_ALL_TESTS();
 184 }