]>
Commit | Line | Data |
---|---|---|
7c673cae | 1 | // Copyright (c) 2011-present, Facebook, Inc. All rights reserved. |
11fdf7f2 TL |
2 | // This source code is licensed under both the GPLv2 (found in the |
3 | // COPYING file in the root directory) and Apache 2.0 License | |
4 | // (found in the LICENSE.Apache file in the root directory). | |
7c673cae FG |
5 | |
6 | #include "rocksdb/utilities/sim_cache.h" | |
1e59de90 | 7 | |
7c673cae | 8 | #include <cstdlib> |
1e59de90 | 9 | |
7c673cae FG |
10 | #include "db/db_test_util.h" |
11 | #include "port/stack_trace.h" | |
12 | ||
f67539c2 | 13 | namespace ROCKSDB_NAMESPACE { |
7c673cae FG |
14 | |
15 | class SimCacheTest : public DBTestBase { | |
16 | private: | |
17 | size_t miss_count_ = 0; | |
18 | size_t hit_count_ = 0; | |
19 | size_t insert_count_ = 0; | |
20 | size_t failure_count_ = 0; | |
21 | ||
22 | public: | |
23 | const size_t kNumBlocks = 5; | |
24 | const size_t kValueSize = 1000; | |
25 | ||
1e59de90 | 26 | SimCacheTest() : DBTestBase("sim_cache_test", /*env_do_fsync=*/true) {} |
7c673cae FG |
27 | |
28 | BlockBasedTableOptions GetTableOptions() { | |
29 | BlockBasedTableOptions table_options; | |
30 | // Set a small enough block size so that each key-value get its own block. | |
31 | table_options.block_size = 1; | |
32 | return table_options; | |
33 | } | |
34 | ||
35 | Options GetOptions(const BlockBasedTableOptions& table_options) { | |
36 | Options options = CurrentOptions(); | |
37 | options.create_if_missing = true; | |
38 | // options.compression = kNoCompression; | |
f67539c2 | 39 | options.statistics = ROCKSDB_NAMESPACE::CreateDBStatistics(); |
20effc67 | 40 | options.table_factory.reset(NewBlockBasedTableFactory(table_options)); |
7c673cae FG |
41 | return options; |
42 | } | |
43 | ||
11fdf7f2 | 44 | void InitTable(const Options& /*options*/) { |
7c673cae FG |
45 | std::string value(kValueSize, 'a'); |
46 | for (size_t i = 0; i < kNumBlocks * 2; i++) { | |
1e59de90 | 47 | ASSERT_OK(Put(std::to_string(i), value.c_str())); |
7c673cae FG |
48 | } |
49 | } | |
50 | ||
51 | void RecordCacheCounters(const Options& options) { | |
52 | miss_count_ = TestGetTickerCount(options, BLOCK_CACHE_MISS); | |
53 | hit_count_ = TestGetTickerCount(options, BLOCK_CACHE_HIT); | |
54 | insert_count_ = TestGetTickerCount(options, BLOCK_CACHE_ADD); | |
55 | failure_count_ = TestGetTickerCount(options, BLOCK_CACHE_ADD_FAILURES); | |
56 | } | |
57 | ||
58 | void CheckCacheCounters(const Options& options, size_t expected_misses, | |
59 | size_t expected_hits, size_t expected_inserts, | |
60 | size_t expected_failures) { | |
61 | size_t new_miss_count = TestGetTickerCount(options, BLOCK_CACHE_MISS); | |
62 | size_t new_hit_count = TestGetTickerCount(options, BLOCK_CACHE_HIT); | |
63 | size_t new_insert_count = TestGetTickerCount(options, BLOCK_CACHE_ADD); | |
64 | size_t new_failure_count = | |
65 | TestGetTickerCount(options, BLOCK_CACHE_ADD_FAILURES); | |
66 | ASSERT_EQ(miss_count_ + expected_misses, new_miss_count); | |
67 | ASSERT_EQ(hit_count_ + expected_hits, new_hit_count); | |
68 | ASSERT_EQ(insert_count_ + expected_inserts, new_insert_count); | |
69 | ASSERT_EQ(failure_count_ + expected_failures, new_failure_count); | |
70 | miss_count_ = new_miss_count; | |
71 | hit_count_ = new_hit_count; | |
72 | insert_count_ = new_insert_count; | |
73 | failure_count_ = new_failure_count; | |
74 | } | |
75 | }; | |
76 | ||
77 | TEST_F(SimCacheTest, SimCache) { | |
78 | ReadOptions read_options; | |
79 | auto table_options = GetTableOptions(); | |
80 | auto options = GetOptions(table_options); | |
81 | InitTable(options); | |
f67539c2 TL |
82 | LRUCacheOptions co; |
83 | co.capacity = 0; | |
84 | co.num_shard_bits = 0; | |
85 | co.strict_capacity_limit = false; | |
86 | co.metadata_charge_policy = kDontChargeCacheMetadata; | |
87 | std::shared_ptr<SimCache> simCache = NewSimCache(NewLRUCache(co), 20000, 0); | |
7c673cae | 88 | table_options.block_cache = simCache; |
20effc67 | 89 | options.table_factory.reset(NewBlockBasedTableFactory(table_options)); |
7c673cae FG |
90 | Reopen(options); |
91 | RecordCacheCounters(options); | |
1e59de90 TL |
92 | // due to cache entry stats collector |
93 | uint64_t base_misses = simCache->get_miss_counter(); | |
7c673cae FG |
94 | |
95 | std::vector<std::unique_ptr<Iterator>> iterators(kNumBlocks); | |
96 | Iterator* iter = nullptr; | |
97 | ||
98 | // Load blocks into cache. | |
99 | for (size_t i = 0; i < kNumBlocks; i++) { | |
100 | iter = db_->NewIterator(read_options); | |
1e59de90 | 101 | iter->Seek(std::to_string(i)); |
7c673cae FG |
102 | ASSERT_OK(iter->status()); |
103 | CheckCacheCounters(options, 1, 0, 1, 0); | |
104 | iterators[i].reset(iter); | |
105 | } | |
1e59de90 TL |
106 | ASSERT_EQ(kNumBlocks, simCache->get_hit_counter() + |
107 | simCache->get_miss_counter() - base_misses); | |
7c673cae FG |
108 | ASSERT_EQ(0, simCache->get_hit_counter()); |
109 | size_t usage = simCache->GetUsage(); | |
110 | ASSERT_LT(0, usage); | |
111 | ASSERT_EQ(usage, simCache->GetSimUsage()); | |
112 | simCache->SetCapacity(usage); | |
113 | ASSERT_EQ(usage, simCache->GetPinnedUsage()); | |
114 | ||
115 | // Test with strict capacity limit. | |
116 | simCache->SetStrictCapacityLimit(true); | |
117 | iter = db_->NewIterator(read_options); | |
1e59de90 TL |
118 | iter->Seek(std::to_string(kNumBlocks * 2 - 1)); |
119 | ASSERT_TRUE(iter->status().IsMemoryLimit()); | |
7c673cae FG |
120 | CheckCacheCounters(options, 1, 0, 0, 1); |
121 | delete iter; | |
122 | iter = nullptr; | |
123 | ||
124 | // Release iterators and access cache again. | |
125 | for (size_t i = 0; i < kNumBlocks; i++) { | |
126 | iterators[i].reset(); | |
127 | CheckCacheCounters(options, 0, 0, 0, 0); | |
128 | } | |
129 | // Add kNumBlocks again | |
130 | for (size_t i = 0; i < kNumBlocks; i++) { | |
131 | std::unique_ptr<Iterator> it(db_->NewIterator(read_options)); | |
1e59de90 | 132 | it->Seek(std::to_string(i)); |
7c673cae FG |
133 | ASSERT_OK(it->status()); |
134 | CheckCacheCounters(options, 0, 1, 0, 0); | |
135 | } | |
136 | ASSERT_EQ(5, simCache->get_hit_counter()); | |
137 | for (size_t i = kNumBlocks; i < kNumBlocks * 2; i++) { | |
138 | std::unique_ptr<Iterator> it(db_->NewIterator(read_options)); | |
1e59de90 | 139 | it->Seek(std::to_string(i)); |
7c673cae FG |
140 | ASSERT_OK(it->status()); |
141 | CheckCacheCounters(options, 1, 0, 1, 0); | |
142 | } | |
143 | ASSERT_EQ(0, simCache->GetPinnedUsage()); | |
1e59de90 TL |
144 | ASSERT_EQ(3 * kNumBlocks + 1, simCache->get_hit_counter() + |
145 | simCache->get_miss_counter() - base_misses); | |
7c673cae FG |
146 | ASSERT_EQ(6, simCache->get_hit_counter()); |
147 | } | |
148 | ||
11fdf7f2 TL |
149 | TEST_F(SimCacheTest, SimCacheLogging) { |
150 | auto table_options = GetTableOptions(); | |
151 | auto options = GetOptions(table_options); | |
152 | options.disable_auto_compactions = true; | |
f67539c2 TL |
153 | LRUCacheOptions co; |
154 | co.capacity = 1024 * 1024; | |
155 | co.metadata_charge_policy = kDontChargeCacheMetadata; | |
156 | std::shared_ptr<SimCache> sim_cache = NewSimCache(NewLRUCache(co), 20000, 0); | |
11fdf7f2 | 157 | table_options.block_cache = sim_cache; |
20effc67 | 158 | options.table_factory.reset(NewBlockBasedTableFactory(table_options)); |
11fdf7f2 TL |
159 | Reopen(options); |
160 | ||
161 | int num_block_entries = 20; | |
162 | for (int i = 0; i < num_block_entries; i++) { | |
20effc67 TL |
163 | ASSERT_OK(Put(Key(i), "val")); |
164 | ASSERT_OK(Flush()); | |
11fdf7f2 TL |
165 | } |
166 | ||
167 | std::string log_file = test::PerThreadDBPath(env_, "cache_log.txt"); | |
168 | ASSERT_OK(sim_cache->StartActivityLogging(log_file, env_)); | |
169 | for (int i = 0; i < num_block_entries; i++) { | |
170 | ASSERT_EQ(Get(Key(i)), "val"); | |
171 | } | |
172 | for (int i = 0; i < num_block_entries; i++) { | |
173 | ASSERT_EQ(Get(Key(i)), "val"); | |
174 | } | |
175 | sim_cache->StopActivityLogging(); | |
176 | ASSERT_OK(sim_cache->GetActivityLoggingStatus()); | |
177 | ||
178 | std::string file_contents = ""; | |
20effc67 | 179 | ASSERT_OK(ReadFileToString(env_, log_file, &file_contents)); |
1e59de90 | 180 | std::istringstream contents(file_contents); |
11fdf7f2 TL |
181 | |
182 | int lookup_num = 0; | |
183 | int add_num = 0; | |
11fdf7f2 | 184 | |
1e59de90 TL |
185 | std::string line; |
186 | // count number of lookups and additions | |
187 | while (std::getline(contents, line)) { | |
188 | // check if the line starts with LOOKUP or ADD | |
189 | if (line.rfind("LOOKUP -", 0) == 0) { | |
190 | ++lookup_num; | |
191 | } | |
192 | if (line.rfind("ADD -", 0) == 0) { | |
193 | ++add_num; | |
194 | } | |
11fdf7f2 TL |
195 | } |
196 | ||
197 | // We asked for every block twice | |
198 | ASSERT_EQ(lookup_num, num_block_entries * 2); | |
199 | ||
200 | // We added every block only once, since the cache can hold all blocks | |
201 | ASSERT_EQ(add_num, num_block_entries); | |
202 | ||
203 | // Log things again but stop logging automatically after reaching 512 bytes | |
20effc67 | 204 | int max_size = 512; |
11fdf7f2 TL |
205 | ASSERT_OK(sim_cache->StartActivityLogging(log_file, env_, max_size)); |
206 | for (int it = 0; it < 10; it++) { | |
207 | for (int i = 0; i < num_block_entries; i++) { | |
208 | ASSERT_EQ(Get(Key(i)), "val"); | |
209 | } | |
210 | } | |
211 | ASSERT_OK(sim_cache->GetActivityLoggingStatus()); | |
212 | ||
213 | uint64_t fsize = 0; | |
214 | ASSERT_OK(env_->GetFileSize(log_file, &fsize)); | |
20effc67 | 215 | // error margin of 100 bytes |
11fdf7f2 | 216 | ASSERT_LT(fsize, max_size + 100); |
20effc67 | 217 | ASSERT_GT(fsize, max_size - 100); |
11fdf7f2 TL |
218 | } |
219 | ||
f67539c2 | 220 | } // namespace ROCKSDB_NAMESPACE |
7c673cae FG |
221 | |
222 | int main(int argc, char** argv) { | |
f67539c2 | 223 | ROCKSDB_NAMESPACE::port::InstallStackTraceHandler(); |
7c673cae FG |
224 | ::testing::InitGoogleTest(&argc, argv); |
225 | return RUN_ALL_TESTS(); | |
226 | } |