]>
Commit | Line | Data |
---|---|---|
7c673cae FG |
1 | // Copyright (c) 2011-present, Facebook, Inc. All rights reserved. |
2 | // This source code is licensed under the BSD-style license found in the | |
3 | // LICENSE file in the root directory of this source tree. An additional grant | |
4 | // of patent rights can be found in the PATENTS file in the same directory. | |
5 | // | |
6 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. | |
7 | // Use of this source code is governed by a BSD-style license that can be | |
8 | // found in the LICENSE file. See the AUTHORS file for names of contributors. | |
9 | #include <cstdlib> | |
10 | #include "cache/lru_cache.h" | |
11 | #include "db/db_test_util.h" | |
12 | #include "port/stack_trace.h" | |
13 | ||
14 | namespace rocksdb { | |
15 | ||
16 | class DBBlockCacheTest : public DBTestBase { | |
17 | private: | |
18 | size_t miss_count_ = 0; | |
19 | size_t hit_count_ = 0; | |
20 | size_t insert_count_ = 0; | |
21 | size_t failure_count_ = 0; | |
22 | size_t compressed_miss_count_ = 0; | |
23 | size_t compressed_hit_count_ = 0; | |
24 | size_t compressed_insert_count_ = 0; | |
25 | size_t compressed_failure_count_ = 0; | |
26 | ||
27 | public: | |
28 | const size_t kNumBlocks = 10; | |
29 | const size_t kValueSize = 100; | |
30 | ||
31 | DBBlockCacheTest() : DBTestBase("/db_block_cache_test") {} | |
32 | ||
33 | BlockBasedTableOptions GetTableOptions() { | |
34 | BlockBasedTableOptions table_options; | |
35 | // Set a small enough block size so that each key-value get its own block. | |
36 | table_options.block_size = 1; | |
37 | return table_options; | |
38 | } | |
39 | ||
40 | Options GetOptions(const BlockBasedTableOptions& table_options) { | |
41 | Options options = CurrentOptions(); | |
42 | options.create_if_missing = true; | |
43 | options.avoid_flush_during_recovery = false; | |
44 | // options.compression = kNoCompression; | |
45 | options.statistics = rocksdb::CreateDBStatistics(); | |
46 | options.table_factory.reset(new BlockBasedTableFactory(table_options)); | |
47 | return options; | |
48 | } | |
49 | ||
50 | void InitTable(const Options& options) { | |
51 | std::string value(kValueSize, 'a'); | |
52 | for (size_t i = 0; i < kNumBlocks; i++) { | |
53 | ASSERT_OK(Put(ToString(i), value.c_str())); | |
54 | } | |
55 | } | |
56 | ||
57 | void RecordCacheCounters(const Options& options) { | |
58 | miss_count_ = TestGetTickerCount(options, BLOCK_CACHE_MISS); | |
59 | hit_count_ = TestGetTickerCount(options, BLOCK_CACHE_HIT); | |
60 | insert_count_ = TestGetTickerCount(options, BLOCK_CACHE_ADD); | |
61 | failure_count_ = TestGetTickerCount(options, BLOCK_CACHE_ADD_FAILURES); | |
62 | compressed_miss_count_ = | |
63 | TestGetTickerCount(options, BLOCK_CACHE_COMPRESSED_MISS); | |
64 | compressed_hit_count_ = | |
65 | TestGetTickerCount(options, BLOCK_CACHE_COMPRESSED_HIT); | |
66 | compressed_insert_count_ = | |
67 | TestGetTickerCount(options, BLOCK_CACHE_COMPRESSED_ADD); | |
68 | compressed_failure_count_ = | |
69 | TestGetTickerCount(options, BLOCK_CACHE_COMPRESSED_ADD_FAILURES); | |
70 | } | |
71 | ||
72 | void CheckCacheCounters(const Options& options, size_t expected_misses, | |
73 | size_t expected_hits, size_t expected_inserts, | |
74 | size_t expected_failures) { | |
75 | size_t new_miss_count = TestGetTickerCount(options, BLOCK_CACHE_MISS); | |
76 | size_t new_hit_count = TestGetTickerCount(options, BLOCK_CACHE_HIT); | |
77 | size_t new_insert_count = TestGetTickerCount(options, BLOCK_CACHE_ADD); | |
78 | size_t new_failure_count = | |
79 | TestGetTickerCount(options, BLOCK_CACHE_ADD_FAILURES); | |
80 | ASSERT_EQ(miss_count_ + expected_misses, new_miss_count); | |
81 | ASSERT_EQ(hit_count_ + expected_hits, new_hit_count); | |
82 | ASSERT_EQ(insert_count_ + expected_inserts, new_insert_count); | |
83 | ASSERT_EQ(failure_count_ + expected_failures, new_failure_count); | |
84 | miss_count_ = new_miss_count; | |
85 | hit_count_ = new_hit_count; | |
86 | insert_count_ = new_insert_count; | |
87 | failure_count_ = new_failure_count; | |
88 | } | |
89 | ||
90 | void CheckCompressedCacheCounters(const Options& options, | |
91 | size_t expected_misses, | |
92 | size_t expected_hits, | |
93 | size_t expected_inserts, | |
94 | size_t expected_failures) { | |
95 | size_t new_miss_count = | |
96 | TestGetTickerCount(options, BLOCK_CACHE_COMPRESSED_MISS); | |
97 | size_t new_hit_count = | |
98 | TestGetTickerCount(options, BLOCK_CACHE_COMPRESSED_HIT); | |
99 | size_t new_insert_count = | |
100 | TestGetTickerCount(options, BLOCK_CACHE_COMPRESSED_ADD); | |
101 | size_t new_failure_count = | |
102 | TestGetTickerCount(options, BLOCK_CACHE_COMPRESSED_ADD_FAILURES); | |
103 | ASSERT_EQ(compressed_miss_count_ + expected_misses, new_miss_count); | |
104 | ASSERT_EQ(compressed_hit_count_ + expected_hits, new_hit_count); | |
105 | ASSERT_EQ(compressed_insert_count_ + expected_inserts, new_insert_count); | |
106 | ASSERT_EQ(compressed_failure_count_ + expected_failures, new_failure_count); | |
107 | compressed_miss_count_ = new_miss_count; | |
108 | compressed_hit_count_ = new_hit_count; | |
109 | compressed_insert_count_ = new_insert_count; | |
110 | compressed_failure_count_ = new_failure_count; | |
111 | } | |
112 | }; | |
113 | ||
114 | TEST_F(DBBlockCacheTest, TestWithoutCompressedBlockCache) { | |
115 | ReadOptions read_options; | |
116 | auto table_options = GetTableOptions(); | |
117 | auto options = GetOptions(table_options); | |
118 | InitTable(options); | |
119 | ||
120 | std::shared_ptr<Cache> cache = NewLRUCache(0, 0, false); | |
121 | table_options.block_cache = cache; | |
122 | options.table_factory.reset(new BlockBasedTableFactory(table_options)); | |
123 | Reopen(options); | |
124 | RecordCacheCounters(options); | |
125 | ||
126 | std::vector<std::unique_ptr<Iterator>> iterators(kNumBlocks - 1); | |
127 | Iterator* iter = nullptr; | |
128 | ||
129 | // Load blocks into cache. | |
130 | for (size_t i = 0; i < kNumBlocks - 1; i++) { | |
131 | iter = db_->NewIterator(read_options); | |
132 | iter->Seek(ToString(i)); | |
133 | ASSERT_OK(iter->status()); | |
134 | CheckCacheCounters(options, 1, 0, 1, 0); | |
135 | iterators[i].reset(iter); | |
136 | } | |
137 | size_t usage = cache->GetUsage(); | |
138 | ASSERT_LT(0, usage); | |
139 | cache->SetCapacity(usage); | |
140 | ASSERT_EQ(usage, cache->GetPinnedUsage()); | |
141 | ||
142 | // Test with strict capacity limit. | |
143 | cache->SetStrictCapacityLimit(true); | |
144 | iter = db_->NewIterator(read_options); | |
145 | iter->Seek(ToString(kNumBlocks - 1)); | |
146 | ASSERT_TRUE(iter->status().IsIncomplete()); | |
147 | CheckCacheCounters(options, 1, 0, 0, 1); | |
148 | delete iter; | |
149 | iter = nullptr; | |
150 | ||
151 | // Release interators and access cache again. | |
152 | for (size_t i = 0; i < kNumBlocks - 1; i++) { | |
153 | iterators[i].reset(); | |
154 | CheckCacheCounters(options, 0, 0, 0, 0); | |
155 | } | |
156 | ASSERT_EQ(0, cache->GetPinnedUsage()); | |
157 | for (size_t i = 0; i < kNumBlocks - 1; i++) { | |
158 | iter = db_->NewIterator(read_options); | |
159 | iter->Seek(ToString(i)); | |
160 | ASSERT_OK(iter->status()); | |
161 | CheckCacheCounters(options, 0, 1, 0, 0); | |
162 | iterators[i].reset(iter); | |
163 | } | |
164 | } | |
165 | ||
166 | #ifdef SNAPPY | |
167 | TEST_F(DBBlockCacheTest, TestWithCompressedBlockCache) { | |
168 | ReadOptions read_options; | |
169 | auto table_options = GetTableOptions(); | |
170 | auto options = GetOptions(table_options); | |
171 | options.compression = CompressionType::kSnappyCompression; | |
172 | InitTable(options); | |
173 | ||
174 | std::shared_ptr<Cache> cache = NewLRUCache(0, 0, false); | |
175 | std::shared_ptr<Cache> compressed_cache = NewLRUCache(1 << 25, 0, false); | |
176 | table_options.block_cache = cache; | |
177 | table_options.block_cache_compressed = compressed_cache; | |
178 | options.table_factory.reset(new BlockBasedTableFactory(table_options)); | |
179 | Reopen(options); | |
180 | RecordCacheCounters(options); | |
181 | ||
182 | std::vector<std::unique_ptr<Iterator>> iterators(kNumBlocks - 1); | |
183 | Iterator* iter = nullptr; | |
184 | ||
185 | // Load blocks into cache. | |
186 | for (size_t i = 0; i < kNumBlocks - 1; i++) { | |
187 | iter = db_->NewIterator(read_options); | |
188 | iter->Seek(ToString(i)); | |
189 | ASSERT_OK(iter->status()); | |
190 | CheckCacheCounters(options, 1, 0, 1, 0); | |
191 | CheckCompressedCacheCounters(options, 1, 0, 1, 0); | |
192 | iterators[i].reset(iter); | |
193 | } | |
194 | size_t usage = cache->GetUsage(); | |
195 | ASSERT_LT(0, usage); | |
196 | ASSERT_EQ(usage, cache->GetPinnedUsage()); | |
197 | size_t compressed_usage = compressed_cache->GetUsage(); | |
198 | ASSERT_LT(0, compressed_usage); | |
199 | // Compressed block cache cannot be pinned. | |
200 | ASSERT_EQ(0, compressed_cache->GetPinnedUsage()); | |
201 | ||
202 | // Set strict capacity limit flag. Now block will only load into compressed | |
203 | // block cache. | |
204 | cache->SetCapacity(usage); | |
205 | cache->SetStrictCapacityLimit(true); | |
206 | ASSERT_EQ(usage, cache->GetPinnedUsage()); | |
207 | iter = db_->NewIterator(read_options); | |
208 | iter->Seek(ToString(kNumBlocks - 1)); | |
209 | ASSERT_TRUE(iter->status().IsIncomplete()); | |
210 | CheckCacheCounters(options, 1, 0, 0, 1); | |
211 | CheckCompressedCacheCounters(options, 1, 0, 1, 0); | |
212 | delete iter; | |
213 | iter = nullptr; | |
214 | ||
215 | // Clear strict capacity limit flag. This time we shall hit compressed block | |
216 | // cache. | |
217 | cache->SetStrictCapacityLimit(false); | |
218 | iter = db_->NewIterator(read_options); | |
219 | iter->Seek(ToString(kNumBlocks - 1)); | |
220 | ASSERT_OK(iter->status()); | |
221 | CheckCacheCounters(options, 1, 0, 1, 0); | |
222 | CheckCompressedCacheCounters(options, 0, 1, 0, 0); | |
223 | delete iter; | |
224 | iter = nullptr; | |
225 | } | |
226 | #endif // SNAPPY | |
227 | ||
228 | #ifndef ROCKSDB_LITE | |
229 | ||
230 | // Make sure that when options.block_cache is set, after a new table is | |
231 | // created its index/filter blocks are added to block cache. | |
232 | TEST_F(DBBlockCacheTest, IndexAndFilterBlocksOfNewTableAddedToCache) { | |
233 | Options options = CurrentOptions(); | |
234 | options.create_if_missing = true; | |
235 | options.statistics = rocksdb::CreateDBStatistics(); | |
236 | BlockBasedTableOptions table_options; | |
237 | table_options.cache_index_and_filter_blocks = true; | |
238 | table_options.filter_policy.reset(NewBloomFilterPolicy(20)); | |
239 | options.table_factory.reset(new BlockBasedTableFactory(table_options)); | |
240 | CreateAndReopenWithCF({"pikachu"}, options); | |
241 | ||
242 | ASSERT_OK(Put(1, "key", "val")); | |
243 | // Create a new table. | |
244 | ASSERT_OK(Flush(1)); | |
245 | ||
246 | // index/filter blocks added to block cache right after table creation. | |
247 | ASSERT_EQ(1, TestGetTickerCount(options, BLOCK_CACHE_INDEX_MISS)); | |
248 | ASSERT_EQ(1, TestGetTickerCount(options, BLOCK_CACHE_FILTER_MISS)); | |
249 | ASSERT_EQ(2, /* only index/filter were added */ | |
250 | TestGetTickerCount(options, BLOCK_CACHE_ADD)); | |
251 | ASSERT_EQ(0, TestGetTickerCount(options, BLOCK_CACHE_DATA_MISS)); | |
252 | uint64_t int_num; | |
253 | ASSERT_TRUE( | |
254 | dbfull()->GetIntProperty("rocksdb.estimate-table-readers-mem", &int_num)); | |
255 | ASSERT_EQ(int_num, 0U); | |
256 | ||
257 | // Make sure filter block is in cache. | |
258 | std::string value; | |
259 | ReadOptions ropt; | |
260 | db_->KeyMayExist(ReadOptions(), handles_[1], "key", &value); | |
261 | ||
262 | // Miss count should remain the same. | |
263 | ASSERT_EQ(1, TestGetTickerCount(options, BLOCK_CACHE_FILTER_MISS)); | |
264 | ASSERT_EQ(1, TestGetTickerCount(options, BLOCK_CACHE_FILTER_HIT)); | |
265 | ||
266 | db_->KeyMayExist(ReadOptions(), handles_[1], "key", &value); | |
267 | ASSERT_EQ(1, TestGetTickerCount(options, BLOCK_CACHE_FILTER_MISS)); | |
268 | ASSERT_EQ(2, TestGetTickerCount(options, BLOCK_CACHE_FILTER_HIT)); | |
269 | ||
270 | // Make sure index block is in cache. | |
271 | auto index_block_hit = TestGetTickerCount(options, BLOCK_CACHE_INDEX_HIT); | |
272 | value = Get(1, "key"); | |
273 | ASSERT_EQ(1, TestGetTickerCount(options, BLOCK_CACHE_INDEX_MISS)); | |
274 | ASSERT_EQ(index_block_hit + 1, | |
275 | TestGetTickerCount(options, BLOCK_CACHE_INDEX_HIT)); | |
276 | ||
277 | value = Get(1, "key"); | |
278 | ASSERT_EQ(1, TestGetTickerCount(options, BLOCK_CACHE_INDEX_MISS)); | |
279 | ASSERT_EQ(index_block_hit + 2, | |
280 | TestGetTickerCount(options, BLOCK_CACHE_INDEX_HIT)); | |
281 | } | |
282 | ||
283 | TEST_F(DBBlockCacheTest, IndexAndFilterBlocksStats) { | |
284 | Options options = CurrentOptions(); | |
285 | options.create_if_missing = true; | |
286 | options.statistics = rocksdb::CreateDBStatistics(); | |
287 | BlockBasedTableOptions table_options; | |
288 | table_options.cache_index_and_filter_blocks = true; | |
289 | // 200 bytes are enough to hold the first two blocks | |
290 | std::shared_ptr<Cache> cache = NewLRUCache(200, 0, false); | |
291 | table_options.block_cache = cache; | |
292 | table_options.filter_policy.reset(NewBloomFilterPolicy(20)); | |
293 | options.table_factory.reset(new BlockBasedTableFactory(table_options)); | |
294 | CreateAndReopenWithCF({"pikachu"}, options); | |
295 | ||
296 | ASSERT_OK(Put(1, "key", "val")); | |
297 | // Create a new table | |
298 | ASSERT_OK(Flush(1)); | |
299 | size_t index_bytes_insert = | |
300 | TestGetTickerCount(options, BLOCK_CACHE_INDEX_BYTES_INSERT); | |
301 | size_t filter_bytes_insert = | |
302 | TestGetTickerCount(options, BLOCK_CACHE_FILTER_BYTES_INSERT); | |
303 | ASSERT_GT(index_bytes_insert, 0); | |
304 | ASSERT_GT(filter_bytes_insert, 0); | |
305 | ASSERT_EQ(cache->GetUsage(), index_bytes_insert + filter_bytes_insert); | |
306 | // set the cache capacity to the current usage | |
307 | cache->SetCapacity(index_bytes_insert + filter_bytes_insert); | |
308 | ASSERT_EQ(TestGetTickerCount(options, BLOCK_CACHE_INDEX_BYTES_EVICT), 0); | |
309 | ASSERT_EQ(TestGetTickerCount(options, BLOCK_CACHE_FILTER_BYTES_EVICT), 0); | |
310 | ASSERT_OK(Put(1, "key2", "val")); | |
311 | // Create a new table | |
312 | ASSERT_OK(Flush(1)); | |
313 | // cache evicted old index and block entries | |
314 | ASSERT_GT(TestGetTickerCount(options, BLOCK_CACHE_INDEX_BYTES_INSERT), | |
315 | index_bytes_insert); | |
316 | ASSERT_GT(TestGetTickerCount(options, BLOCK_CACHE_FILTER_BYTES_INSERT), | |
317 | filter_bytes_insert); | |
318 | ASSERT_EQ(TestGetTickerCount(options, BLOCK_CACHE_INDEX_BYTES_EVICT), | |
319 | index_bytes_insert); | |
320 | ASSERT_EQ(TestGetTickerCount(options, BLOCK_CACHE_FILTER_BYTES_EVICT), | |
321 | filter_bytes_insert); | |
322 | } | |
323 | ||
324 | namespace { | |
325 | ||
326 | // A mock cache wraps LRUCache, and record how many entries have been | |
327 | // inserted for each priority. | |
328 | class MockCache : public LRUCache { | |
329 | public: | |
330 | static uint32_t high_pri_insert_count; | |
331 | static uint32_t low_pri_insert_count; | |
332 | ||
333 | MockCache() : LRUCache(1 << 25, 0, false, 0.0) {} | |
334 | ||
335 | virtual Status Insert(const Slice& key, void* value, size_t charge, | |
336 | void (*deleter)(const Slice& key, void* value), | |
337 | Handle** handle, Priority priority) override { | |
338 | if (priority == Priority::LOW) { | |
339 | low_pri_insert_count++; | |
340 | } else { | |
341 | high_pri_insert_count++; | |
342 | } | |
343 | return LRUCache::Insert(key, value, charge, deleter, handle, priority); | |
344 | } | |
345 | }; | |
346 | ||
347 | uint32_t MockCache::high_pri_insert_count = 0; | |
348 | uint32_t MockCache::low_pri_insert_count = 0; | |
349 | ||
350 | } // anonymous namespace | |
351 | ||
352 | TEST_F(DBBlockCacheTest, IndexAndFilterBlocksCachePriority) { | |
353 | for (auto priority : {Cache::Priority::LOW, Cache::Priority::HIGH}) { | |
354 | Options options = CurrentOptions(); | |
355 | options.create_if_missing = true; | |
356 | options.statistics = rocksdb::CreateDBStatistics(); | |
357 | BlockBasedTableOptions table_options; | |
358 | table_options.cache_index_and_filter_blocks = true; | |
359 | table_options.block_cache.reset(new MockCache()); | |
360 | table_options.filter_policy.reset(NewBloomFilterPolicy(20)); | |
361 | table_options.cache_index_and_filter_blocks_with_high_priority = | |
362 | priority == Cache::Priority::HIGH ? true : false; | |
363 | options.table_factory.reset(new BlockBasedTableFactory(table_options)); | |
364 | DestroyAndReopen(options); | |
365 | ||
366 | MockCache::high_pri_insert_count = 0; | |
367 | MockCache::low_pri_insert_count = 0; | |
368 | ||
369 | // Create a new table. | |
370 | ASSERT_OK(Put("foo", "value")); | |
371 | ASSERT_OK(Put("bar", "value")); | |
372 | ASSERT_OK(Flush()); | |
373 | ASSERT_EQ(1, NumTableFilesAtLevel(0)); | |
374 | ||
375 | // index/filter blocks added to block cache right after table creation. | |
376 | ASSERT_EQ(1, TestGetTickerCount(options, BLOCK_CACHE_INDEX_MISS)); | |
377 | ASSERT_EQ(1, TestGetTickerCount(options, BLOCK_CACHE_FILTER_MISS)); | |
378 | ASSERT_EQ(2, /* only index/filter were added */ | |
379 | TestGetTickerCount(options, BLOCK_CACHE_ADD)); | |
380 | ASSERT_EQ(0, TestGetTickerCount(options, BLOCK_CACHE_DATA_MISS)); | |
381 | if (priority == Cache::Priority::LOW) { | |
382 | ASSERT_EQ(0, MockCache::high_pri_insert_count); | |
383 | ASSERT_EQ(2, MockCache::low_pri_insert_count); | |
384 | } else { | |
385 | ASSERT_EQ(2, MockCache::high_pri_insert_count); | |
386 | ASSERT_EQ(0, MockCache::low_pri_insert_count); | |
387 | } | |
388 | ||
389 | // Access data block. | |
390 | ASSERT_EQ("value", Get("foo")); | |
391 | ||
392 | ASSERT_EQ(1, TestGetTickerCount(options, BLOCK_CACHE_INDEX_MISS)); | |
393 | ASSERT_EQ(1, TestGetTickerCount(options, BLOCK_CACHE_FILTER_MISS)); | |
394 | ASSERT_EQ(3, /*adding data block*/ | |
395 | TestGetTickerCount(options, BLOCK_CACHE_ADD)); | |
396 | ASSERT_EQ(1, TestGetTickerCount(options, BLOCK_CACHE_DATA_MISS)); | |
397 | ||
398 | // Data block should be inserted with low priority. | |
399 | if (priority == Cache::Priority::LOW) { | |
400 | ASSERT_EQ(0, MockCache::high_pri_insert_count); | |
401 | ASSERT_EQ(3, MockCache::low_pri_insert_count); | |
402 | } else { | |
403 | ASSERT_EQ(2, MockCache::high_pri_insert_count); | |
404 | ASSERT_EQ(1, MockCache::low_pri_insert_count); | |
405 | } | |
406 | } | |
407 | } | |
408 | ||
409 | TEST_F(DBBlockCacheTest, ParanoidFileChecks) { | |
410 | Options options = CurrentOptions(); | |
411 | options.create_if_missing = true; | |
412 | options.statistics = rocksdb::CreateDBStatistics(); | |
413 | options.level0_file_num_compaction_trigger = 2; | |
414 | options.paranoid_file_checks = true; | |
415 | BlockBasedTableOptions table_options; | |
416 | table_options.cache_index_and_filter_blocks = false; | |
417 | table_options.filter_policy.reset(NewBloomFilterPolicy(20)); | |
418 | options.table_factory.reset(new BlockBasedTableFactory(table_options)); | |
419 | CreateAndReopenWithCF({"pikachu"}, options); | |
420 | ||
421 | ASSERT_OK(Put(1, "1_key", "val")); | |
422 | ASSERT_OK(Put(1, "9_key", "val")); | |
423 | // Create a new table. | |
424 | ASSERT_OK(Flush(1)); | |
425 | ASSERT_EQ(1, /* read and cache data block */ | |
426 | TestGetTickerCount(options, BLOCK_CACHE_ADD)); | |
427 | ||
428 | ASSERT_OK(Put(1, "1_key2", "val2")); | |
429 | ASSERT_OK(Put(1, "9_key2", "val2")); | |
430 | // Create a new SST file. This will further trigger a compaction | |
431 | // and generate another file. | |
432 | ASSERT_OK(Flush(1)); | |
433 | dbfull()->TEST_WaitForCompact(); | |
434 | ASSERT_EQ(3, /* Totally 3 files created up to now */ | |
435 | TestGetTickerCount(options, BLOCK_CACHE_ADD)); | |
436 | ||
437 | // After disabling options.paranoid_file_checks. NO further block | |
438 | // is added after generating a new file. | |
439 | ASSERT_OK( | |
440 | dbfull()->SetOptions(handles_[1], {{"paranoid_file_checks", "false"}})); | |
441 | ||
442 | ASSERT_OK(Put(1, "1_key3", "val3")); | |
443 | ASSERT_OK(Put(1, "9_key3", "val3")); | |
444 | ASSERT_OK(Flush(1)); | |
445 | ASSERT_OK(Put(1, "1_key4", "val4")); | |
446 | ASSERT_OK(Put(1, "9_key4", "val4")); | |
447 | ASSERT_OK(Flush(1)); | |
448 | dbfull()->TEST_WaitForCompact(); | |
449 | ASSERT_EQ(3, /* Totally 3 files created up to now */ | |
450 | TestGetTickerCount(options, BLOCK_CACHE_ADD)); | |
451 | } | |
452 | ||
453 | TEST_F(DBBlockCacheTest, CompressedCache) { | |
454 | if (!Snappy_Supported()) { | |
455 | return; | |
456 | } | |
457 | int num_iter = 80; | |
458 | ||
459 | // Run this test three iterations. | |
460 | // Iteration 1: only a uncompressed block cache | |
461 | // Iteration 2: only a compressed block cache | |
462 | // Iteration 3: both block cache and compressed cache | |
463 | // Iteration 4: both block cache and compressed cache, but DB is not | |
464 | // compressed | |
465 | for (int iter = 0; iter < 4; iter++) { | |
466 | Options options = CurrentOptions(); | |
467 | options.write_buffer_size = 64 * 1024; // small write buffer | |
468 | options.statistics = rocksdb::CreateDBStatistics(); | |
469 | ||
470 | BlockBasedTableOptions table_options; | |
471 | switch (iter) { | |
472 | case 0: | |
473 | // only uncompressed block cache | |
474 | table_options.block_cache = NewLRUCache(8 * 1024); | |
475 | table_options.block_cache_compressed = nullptr; | |
476 | options.table_factory.reset(NewBlockBasedTableFactory(table_options)); | |
477 | break; | |
478 | case 1: | |
479 | // no block cache, only compressed cache | |
480 | table_options.no_block_cache = true; | |
481 | table_options.block_cache = nullptr; | |
482 | table_options.block_cache_compressed = NewLRUCache(8 * 1024); | |
483 | options.table_factory.reset(NewBlockBasedTableFactory(table_options)); | |
484 | break; | |
485 | case 2: | |
486 | // both compressed and uncompressed block cache | |
487 | table_options.block_cache = NewLRUCache(1024); | |
488 | table_options.block_cache_compressed = NewLRUCache(8 * 1024); | |
489 | options.table_factory.reset(NewBlockBasedTableFactory(table_options)); | |
490 | break; | |
491 | case 3: | |
492 | // both block cache and compressed cache, but DB is not compressed | |
493 | // also, make block cache sizes bigger, to trigger block cache hits | |
494 | table_options.block_cache = NewLRUCache(1024 * 1024); | |
495 | table_options.block_cache_compressed = NewLRUCache(8 * 1024 * 1024); | |
496 | options.table_factory.reset(NewBlockBasedTableFactory(table_options)); | |
497 | options.compression = kNoCompression; | |
498 | break; | |
499 | default: | |
500 | ASSERT_TRUE(false); | |
501 | } | |
502 | CreateAndReopenWithCF({"pikachu"}, options); | |
503 | // default column family doesn't have block cache | |
504 | Options no_block_cache_opts; | |
505 | no_block_cache_opts.statistics = options.statistics; | |
506 | no_block_cache_opts = CurrentOptions(no_block_cache_opts); | |
507 | BlockBasedTableOptions table_options_no_bc; | |
508 | table_options_no_bc.no_block_cache = true; | |
509 | no_block_cache_opts.table_factory.reset( | |
510 | NewBlockBasedTableFactory(table_options_no_bc)); | |
511 | ReopenWithColumnFamilies( | |
512 | {"default", "pikachu"}, | |
513 | std::vector<Options>({no_block_cache_opts, options})); | |
514 | ||
515 | Random rnd(301); | |
516 | ||
517 | // Write 8MB (80 values, each 100K) | |
518 | ASSERT_EQ(NumTableFilesAtLevel(0, 1), 0); | |
519 | std::vector<std::string> values; | |
520 | std::string str; | |
521 | for (int i = 0; i < num_iter; i++) { | |
522 | if (i % 4 == 0) { // high compression ratio | |
523 | str = RandomString(&rnd, 1000); | |
524 | } | |
525 | values.push_back(str); | |
526 | ASSERT_OK(Put(1, Key(i), values[i])); | |
527 | } | |
528 | ||
529 | // flush all data from memtable so that reads are from block cache | |
530 | ASSERT_OK(Flush(1)); | |
531 | ||
532 | for (int i = 0; i < num_iter; i++) { | |
533 | ASSERT_EQ(Get(1, Key(i)), values[i]); | |
534 | } | |
535 | ||
536 | // check that we triggered the appropriate code paths in the cache | |
537 | switch (iter) { | |
538 | case 0: | |
539 | // only uncompressed block cache | |
540 | ASSERT_GT(TestGetTickerCount(options, BLOCK_CACHE_MISS), 0); | |
541 | ASSERT_EQ(TestGetTickerCount(options, BLOCK_CACHE_COMPRESSED_MISS), 0); | |
542 | break; | |
543 | case 1: | |
544 | // no block cache, only compressed cache | |
545 | ASSERT_EQ(TestGetTickerCount(options, BLOCK_CACHE_MISS), 0); | |
546 | ASSERT_GT(TestGetTickerCount(options, BLOCK_CACHE_COMPRESSED_MISS), 0); | |
547 | break; | |
548 | case 2: | |
549 | // both compressed and uncompressed block cache | |
550 | ASSERT_GT(TestGetTickerCount(options, BLOCK_CACHE_MISS), 0); | |
551 | ASSERT_GT(TestGetTickerCount(options, BLOCK_CACHE_COMPRESSED_MISS), 0); | |
552 | break; | |
553 | case 3: | |
554 | // both compressed and uncompressed block cache | |
555 | ASSERT_GT(TestGetTickerCount(options, BLOCK_CACHE_MISS), 0); | |
556 | ASSERT_GT(TestGetTickerCount(options, BLOCK_CACHE_HIT), 0); | |
557 | ASSERT_GT(TestGetTickerCount(options, BLOCK_CACHE_COMPRESSED_MISS), 0); | |
558 | // compressed doesn't have any hits since blocks are not compressed on | |
559 | // storage | |
560 | ASSERT_EQ(TestGetTickerCount(options, BLOCK_CACHE_COMPRESSED_HIT), 0); | |
561 | break; | |
562 | default: | |
563 | ASSERT_TRUE(false); | |
564 | } | |
565 | ||
566 | options.create_if_missing = true; | |
567 | DestroyAndReopen(options); | |
568 | } | |
569 | } | |
570 | ||
571 | #endif // ROCKSDB_LITE | |
572 | ||
573 | } // namespace rocksdb | |
574 | ||
575 | int main(int argc, char** argv) { | |
576 | rocksdb::port::InstallStackTraceHandler(); | |
577 | ::testing::InitGoogleTest(&argc, argv); | |
578 | return RUN_ALL_TESTS(); | |
579 | } |