]>
Commit | Line | Data |
---|---|---|
f67539c2 TL |
1 | // Copyright (c) 2011-present, Facebook, Inc. All rights reserved. |
2 | // This source code is licensed under both the GPLv2 (found in the | |
3 | // COPYING file in the root directory) and Apache 2.0 License | |
4 | // (found in the LICENSE.Apache file in the root directory). | |
5 | ||
6 | #ifndef ROCKSDB_LITE | |
7 | #ifndef GFLAGS | |
8 | #include <cstdio> | |
9 | int main() { | |
10 | fprintf(stderr, | |
11 | "Please install gflags to run block_cache_trace_analyzer_test\n"); | |
1e59de90 | 12 | return 0; |
f67539c2 TL |
13 | } |
14 | #else | |
15 | ||
16 | #include <fstream> | |
17 | #include <iostream> | |
18 | #include <map> | |
19 | #include <vector> | |
20 | ||
1e59de90 | 21 | #include "rocksdb/db.h" |
f67539c2 TL |
22 | #include "rocksdb/env.h" |
23 | #include "rocksdb/status.h" | |
24 | #include "rocksdb/trace_reader_writer.h" | |
1e59de90 | 25 | #include "rocksdb/trace_record.h" |
f67539c2 TL |
26 | #include "test_util/testharness.h" |
27 | #include "test_util/testutil.h" | |
28 | #include "tools/block_cache_analyzer/block_cache_trace_analyzer.h" | |
29 | #include "trace_replay/block_cache_tracer.h" | |
30 | ||
31 | namespace ROCKSDB_NAMESPACE { | |
32 | ||
33 | namespace { | |
34 | const uint64_t kBlockSize = 1024; | |
35 | const std::string kBlockKeyPrefix = "test-block-"; | |
36 | const uint32_t kCFId = 0; | |
37 | const uint32_t kLevel = 1; | |
38 | const uint64_t kSSTStoringEvenKeys = 100; | |
39 | const uint64_t kSSTStoringOddKeys = 101; | |
40 | const std::string kRefKeyPrefix = "test-get-"; | |
41 | const uint64_t kNumKeysInBlock = 1024; | |
42 | const int kMaxArgCount = 100; | |
43 | const size_t kArgBufferSize = 100000; | |
44 | } // namespace | |
45 | ||
46 | class BlockCacheTracerTest : public testing::Test { | |
47 | public: | |
48 | BlockCacheTracerTest() { | |
1e59de90 | 49 | test_path_ = test::PerThreadDBPath("block_cache_trace_analyzer_test"); |
f67539c2 TL |
50 | env_ = ROCKSDB_NAMESPACE::Env::Default(); |
51 | EXPECT_OK(env_->CreateDir(test_path_)); | |
52 | trace_file_path_ = test_path_ + "/block_cache_trace"; | |
53 | block_cache_sim_config_path_ = test_path_ + "/block_cache_sim_config"; | |
54 | timeline_labels_ = | |
55 | "block,all,cf,sst,level,bt,caller,cf_sst,cf_level,cf_bt,cf_caller"; | |
56 | reuse_distance_labels_ = | |
57 | "block,all,cf,sst,level,bt,caller,cf_sst,cf_level,cf_bt,cf_caller"; | |
58 | reuse_distance_buckets_ = "1,1K,1M,1G"; | |
59 | reuse_interval_labels_ = "block,all,cf,sst,level,bt,cf_sst,cf_level,cf_bt"; | |
60 | reuse_interval_buckets_ = "1,10,100,1000"; | |
61 | reuse_lifetime_labels_ = "block,all,cf,sst,level,bt,cf_sst,cf_level,cf_bt"; | |
62 | reuse_lifetime_buckets_ = "1,10,100,1000"; | |
63 | analyzing_callers_ = "Get,Iterator"; | |
64 | access_count_buckets_ = "2,3,4,5,10"; | |
65 | analyze_get_spatial_locality_labels_ = "all"; | |
66 | analyze_get_spatial_locality_buckets_ = "10,20,30,40,50,60,70,80,90,100"; | |
67 | } | |
68 | ||
69 | ~BlockCacheTracerTest() override { | |
70 | if (getenv("KEEP_DB")) { | |
71 | printf("The trace file is still at %s\n", trace_file_path_.c_str()); | |
72 | return; | |
73 | } | |
74 | EXPECT_OK(env_->DeleteFile(trace_file_path_)); | |
75 | EXPECT_OK(env_->DeleteDir(test_path_)); | |
76 | } | |
77 | ||
78 | TableReaderCaller GetCaller(uint32_t key_id) { | |
79 | uint32_t n = key_id % 5; | |
80 | switch (n) { | |
81 | case 0: | |
82 | return TableReaderCaller::kPrefetch; | |
83 | case 1: | |
84 | return TableReaderCaller::kCompaction; | |
85 | case 2: | |
86 | return TableReaderCaller::kUserGet; | |
87 | case 3: | |
88 | return TableReaderCaller::kUserMultiGet; | |
89 | case 4: | |
90 | return TableReaderCaller::kUserIterator; | |
91 | } | |
92 | // This cannot happend. | |
93 | assert(false); | |
94 | return TableReaderCaller::kMaxBlockCacheLookupCaller; | |
95 | } | |
96 | ||
97 | void WriteBlockAccess(BlockCacheTraceWriter* writer, uint32_t from_key_id, | |
98 | TraceType block_type, uint32_t nblocks) { | |
99 | assert(writer); | |
100 | for (uint32_t i = 0; i < nblocks; i++) { | |
101 | uint32_t key_id = from_key_id + i; | |
102 | uint64_t timestamp = (key_id + 1) * kMicrosInSecond; | |
103 | BlockCacheTraceRecord record; | |
104 | record.block_type = block_type; | |
105 | record.block_size = kBlockSize + key_id; | |
106 | record.block_key = kBlockKeyPrefix + std::to_string(key_id); | |
107 | record.access_timestamp = timestamp; | |
108 | record.cf_id = kCFId; | |
109 | record.cf_name = kDefaultColumnFamilyName; | |
110 | record.caller = GetCaller(key_id); | |
111 | record.level = kLevel; | |
112 | if (key_id % 2 == 0) { | |
113 | record.sst_fd_number = kSSTStoringEvenKeys; | |
114 | } else { | |
115 | record.sst_fd_number = kSSTStoringOddKeys; | |
116 | } | |
1e59de90 TL |
117 | record.is_cache_hit = false; |
118 | record.no_insert = false; | |
f67539c2 TL |
119 | // Provide these fields for all block types. |
120 | // The writer should only write these fields for data blocks and the | |
121 | // caller is either GET or MGET. | |
122 | record.referenced_key = | |
123 | kRefKeyPrefix + std::to_string(key_id) + std::string(8, 0); | |
1e59de90 | 124 | record.referenced_key_exist_in_block = true; |
f67539c2 TL |
125 | record.num_keys_in_block = kNumKeysInBlock; |
126 | ASSERT_OK(writer->WriteBlockAccess( | |
127 | record, record.block_key, record.cf_name, record.referenced_key)); | |
128 | } | |
129 | } | |
130 | ||
131 | void AssertBlockAccessInfo( | |
132 | uint32_t key_id, TraceType type, | |
133 | const std::map<std::string, BlockAccessInfo>& block_access_info_map) { | |
134 | auto key_id_str = kBlockKeyPrefix + std::to_string(key_id); | |
135 | ASSERT_TRUE(block_access_info_map.find(key_id_str) != | |
136 | block_access_info_map.end()); | |
137 | auto& block_access_info = block_access_info_map.find(key_id_str)->second; | |
138 | ASSERT_EQ(1, block_access_info.num_accesses); | |
139 | ASSERT_EQ(kBlockSize + key_id, block_access_info.block_size); | |
140 | ASSERT_GT(block_access_info.first_access_time, 0); | |
141 | ASSERT_GT(block_access_info.last_access_time, 0); | |
142 | ASSERT_EQ(1, block_access_info.caller_num_access_map.size()); | |
143 | TableReaderCaller expected_caller = GetCaller(key_id); | |
144 | ASSERT_TRUE(block_access_info.caller_num_access_map.find(expected_caller) != | |
145 | block_access_info.caller_num_access_map.end()); | |
146 | ASSERT_EQ( | |
147 | 1, | |
148 | block_access_info.caller_num_access_map.find(expected_caller)->second); | |
149 | ||
150 | if ((expected_caller == TableReaderCaller::kUserGet || | |
151 | expected_caller == TableReaderCaller::kUserMultiGet) && | |
152 | type == TraceType::kBlockTraceDataBlock) { | |
153 | ASSERT_EQ(kNumKeysInBlock, block_access_info.num_keys); | |
154 | ASSERT_EQ(1, block_access_info.key_num_access_map.size()); | |
155 | ASSERT_EQ(0, block_access_info.non_exist_key_num_access_map.size()); | |
156 | ASSERT_EQ(1, block_access_info.num_referenced_key_exist_in_block); | |
157 | } | |
158 | } | |
159 | ||
160 | void RunBlockCacheTraceAnalyzer() { | |
161 | std::vector<std::string> params = { | |
162 | "./block_cache_trace_analyzer", | |
163 | "-block_cache_trace_path=" + trace_file_path_, | |
164 | "-block_cache_sim_config_path=" + block_cache_sim_config_path_, | |
165 | "-block_cache_analysis_result_dir=" + test_path_, | |
166 | "-print_block_size_stats", | |
167 | "-print_access_count_stats", | |
168 | "-print_data_block_access_count_stats", | |
169 | "-cache_sim_warmup_seconds=0", | |
170 | "-analyze_bottom_k_access_count_blocks=5", | |
171 | "-analyze_top_k_access_count_blocks=5", | |
172 | "-analyze_blocks_reuse_k_reuse_window=5", | |
173 | "-timeline_labels=" + timeline_labels_, | |
174 | "-reuse_distance_labels=" + reuse_distance_labels_, | |
175 | "-reuse_distance_buckets=" + reuse_distance_buckets_, | |
176 | "-reuse_interval_labels=" + reuse_interval_labels_, | |
177 | "-reuse_interval_buckets=" + reuse_interval_buckets_, | |
178 | "-reuse_lifetime_labels=" + reuse_lifetime_labels_, | |
179 | "-reuse_lifetime_buckets=" + reuse_lifetime_buckets_, | |
180 | "-analyze_callers=" + analyzing_callers_, | |
181 | "-access_count_buckets=" + access_count_buckets_, | |
182 | "-analyze_get_spatial_locality_labels=" + | |
183 | analyze_get_spatial_locality_labels_, | |
184 | "-analyze_get_spatial_locality_buckets=" + | |
185 | analyze_get_spatial_locality_buckets_, | |
186 | "-analyze_correlation_coefficients_labels=all", | |
187 | "-skew_labels=all", | |
188 | "-skew_buckets=10,50,100"}; | |
189 | char arg_buffer[kArgBufferSize]; | |
190 | char* argv[kMaxArgCount]; | |
191 | int argc = 0; | |
192 | int cursor = 0; | |
193 | for (const auto& arg : params) { | |
194 | ASSERT_LE(cursor + arg.size() + 1, kArgBufferSize); | |
195 | ASSERT_LE(argc + 1, kMaxArgCount); | |
196 | snprintf(arg_buffer + cursor, arg.size() + 1, "%s", arg.c_str()); | |
197 | ||
198 | argv[argc++] = arg_buffer + cursor; | |
199 | cursor += static_cast<int>(arg.size()) + 1; | |
200 | } | |
201 | ASSERT_EQ(0, | |
202 | ROCKSDB_NAMESPACE::block_cache_trace_analyzer_tool(argc, argv)); | |
203 | } | |
204 | ||
205 | Env* env_; | |
206 | EnvOptions env_options_; | |
207 | std::string block_cache_sim_config_path_; | |
208 | std::string trace_file_path_; | |
209 | std::string test_path_; | |
210 | std::string timeline_labels_; | |
211 | std::string reuse_distance_labels_; | |
212 | std::string reuse_distance_buckets_; | |
213 | std::string reuse_interval_labels_; | |
214 | std::string reuse_interval_buckets_; | |
215 | std::string reuse_lifetime_labels_; | |
216 | std::string reuse_lifetime_buckets_; | |
217 | std::string analyzing_callers_; | |
218 | std::string access_count_buckets_; | |
219 | std::string analyze_get_spatial_locality_labels_; | |
220 | std::string analyze_get_spatial_locality_buckets_; | |
221 | }; | |
222 | ||
223 | TEST_F(BlockCacheTracerTest, BlockCacheAnalyzer) { | |
224 | { | |
225 | // Generate a trace file. | |
1e59de90 | 226 | BlockCacheTraceWriterOptions trace_writer_opt; |
f67539c2 TL |
227 | std::unique_ptr<TraceWriter> trace_writer; |
228 | ASSERT_OK(NewFileTraceWriter(env_, env_options_, trace_file_path_, | |
229 | &trace_writer)); | |
1e59de90 TL |
230 | const auto& clock = env_->GetSystemClock(); |
231 | std::unique_ptr<BlockCacheTraceWriter> block_cache_trace_writer = | |
232 | NewBlockCacheTraceWriter(clock.get(), trace_writer_opt, | |
233 | std::move(trace_writer)); | |
234 | ASSERT_NE(block_cache_trace_writer, nullptr); | |
235 | ASSERT_OK(block_cache_trace_writer->WriteHeader()); | |
236 | WriteBlockAccess(block_cache_trace_writer.get(), 0, | |
237 | TraceType::kBlockTraceDataBlock, 50); | |
f67539c2 TL |
238 | ASSERT_OK(env_->FileExists(trace_file_path_)); |
239 | } | |
240 | { | |
241 | // Generate a cache sim config. | |
242 | std::string config = "lru,1,0,1K,1M,1G"; | |
243 | std::ofstream out(block_cache_sim_config_path_); | |
244 | ASSERT_TRUE(out.is_open()); | |
245 | out << config << std::endl; | |
246 | out.close(); | |
247 | } | |
248 | RunBlockCacheTraceAnalyzer(); | |
249 | { | |
250 | // Validate the cache miss ratios. | |
251 | std::vector<uint64_t> expected_capacities{1024, 1024 * 1024, | |
252 | 1024 * 1024 * 1024}; | |
253 | const std::string mrc_path = test_path_ + "/49_50_mrc"; | |
254 | std::ifstream infile(mrc_path); | |
255 | uint32_t config_index = 0; | |
256 | std::string line; | |
257 | // Read header. | |
258 | ASSERT_TRUE(getline(infile, line)); | |
259 | while (getline(infile, line)) { | |
260 | std::stringstream ss(line); | |
261 | std::vector<std::string> result_strs; | |
262 | while (ss.good()) { | |
263 | std::string substr; | |
264 | getline(ss, substr, ','); | |
265 | result_strs.push_back(substr); | |
266 | } | |
267 | ASSERT_EQ(6, result_strs.size()); | |
268 | ASSERT_LT(config_index, expected_capacities.size()); | |
269 | ASSERT_EQ("lru", result_strs[0]); // cache_name | |
270 | ASSERT_EQ("1", result_strs[1]); // num_shard_bits | |
271 | ASSERT_EQ("0", result_strs[2]); // ghost_cache_capacity | |
272 | ASSERT_EQ(std::to_string(expected_capacities[config_index]), | |
273 | result_strs[3]); // cache_capacity | |
274 | ASSERT_EQ("100.0000", result_strs[4]); // miss_ratio | |
275 | ASSERT_EQ("50", result_strs[5]); // number of accesses. | |
276 | config_index++; | |
277 | } | |
278 | ASSERT_EQ(expected_capacities.size(), config_index); | |
279 | infile.close(); | |
280 | ASSERT_OK(env_->DeleteFile(mrc_path)); | |
281 | ||
282 | const std::vector<std::string> time_units{"1", "60", "3600"}; | |
1e59de90 | 283 | expected_capacities.push_back(std::numeric_limits<uint64_t>::max()); |
f67539c2 TL |
284 | for (auto const& expected_capacity : expected_capacities) { |
285 | for (auto const& time_unit : time_units) { | |
286 | const std::string miss_ratio_timeline_path = | |
287 | test_path_ + "/" + std::to_string(expected_capacity) + "_" + | |
288 | time_unit + "_miss_ratio_timeline"; | |
289 | std::ifstream mrt_file(miss_ratio_timeline_path); | |
290 | // Read header. | |
291 | ASSERT_TRUE(getline(mrt_file, line)); | |
292 | ASSERT_TRUE(getline(mrt_file, line)); | |
293 | std::stringstream ss(line); | |
294 | bool read_header = false; | |
295 | while (ss.good()) { | |
296 | std::string substr; | |
297 | getline(ss, substr, ','); | |
298 | if (!read_header) { | |
1e59de90 | 299 | if (expected_capacity == std::numeric_limits<uint64_t>::max()) { |
f67539c2 TL |
300 | ASSERT_EQ("trace", substr); |
301 | } else { | |
302 | ASSERT_EQ("lru-1-0", substr); | |
303 | } | |
304 | read_header = true; | |
305 | continue; | |
306 | } | |
307 | ASSERT_DOUBLE_EQ(100.0, ParseDouble(substr)); | |
308 | } | |
309 | ASSERT_FALSE(getline(mrt_file, line)); | |
310 | mrt_file.close(); | |
311 | ASSERT_OK(env_->DeleteFile(miss_ratio_timeline_path)); | |
312 | } | |
313 | for (auto const& time_unit : time_units) { | |
314 | const std::string miss_timeline_path = | |
315 | test_path_ + "/" + std::to_string(expected_capacity) + "_" + | |
316 | time_unit + "_miss_timeline"; | |
317 | std::ifstream mt_file(miss_timeline_path); | |
318 | // Read header. | |
319 | ASSERT_TRUE(getline(mt_file, line)); | |
320 | ASSERT_TRUE(getline(mt_file, line)); | |
321 | std::stringstream ss(line); | |
322 | uint32_t num_misses = 0; | |
323 | while (ss.good()) { | |
324 | std::string substr; | |
325 | getline(ss, substr, ','); | |
326 | if (num_misses == 0) { | |
1e59de90 | 327 | if (expected_capacity == std::numeric_limits<uint64_t>::max()) { |
f67539c2 TL |
328 | ASSERT_EQ("trace", substr); |
329 | } else { | |
330 | ASSERT_EQ("lru-1-0", substr); | |
331 | } | |
332 | num_misses++; | |
333 | continue; | |
334 | } | |
335 | num_misses += ParseInt(substr); | |
336 | } | |
20effc67 | 337 | ASSERT_EQ(51u, num_misses); |
f67539c2 TL |
338 | ASSERT_FALSE(getline(mt_file, line)); |
339 | mt_file.close(); | |
340 | ASSERT_OK(env_->DeleteFile(miss_timeline_path)); | |
341 | } | |
342 | } | |
343 | } | |
344 | { | |
345 | // Validate the skewness csv file. | |
346 | const std::string skewness_file_path = test_path_ + "/all_skewness"; | |
347 | std::ifstream skew_file(skewness_file_path); | |
348 | // Read header. | |
349 | std::string line; | |
350 | ASSERT_TRUE(getline(skew_file, line)); | |
351 | std::stringstream ss(line); | |
352 | double sum_percent = 0; | |
353 | while (getline(skew_file, line)) { | |
354 | std::stringstream ss_naccess(line); | |
355 | std::string substr; | |
356 | bool read_label = false; | |
357 | while (ss_naccess.good()) { | |
358 | ASSERT_TRUE(getline(ss_naccess, substr, ',')); | |
359 | if (!read_label) { | |
360 | read_label = true; | |
361 | continue; | |
362 | } | |
363 | sum_percent += ParseDouble(substr); | |
364 | } | |
365 | } | |
366 | ASSERT_EQ(100.0, sum_percent); | |
367 | ASSERT_FALSE(getline(skew_file, line)); | |
368 | skew_file.close(); | |
369 | ASSERT_OK(env_->DeleteFile(skewness_file_path)); | |
370 | } | |
371 | { | |
372 | // Validate the timeline csv files. | |
373 | const std::vector<std::string> time_units{"_60", "_3600"}; | |
374 | const std::vector<std::string> user_access_only_flags{"user_access_only_", | |
375 | "all_access_"}; | |
376 | for (auto const& user_access_only : user_access_only_flags) { | |
377 | for (auto const& unit : time_units) { | |
378 | std::stringstream ss(timeline_labels_); | |
379 | while (ss.good()) { | |
380 | std::string l; | |
381 | ASSERT_TRUE(getline(ss, l, ',')); | |
382 | if (l.find("block") == std::string::npos) { | |
383 | if (user_access_only != "all_access_") { | |
384 | continue; | |
385 | } | |
386 | } | |
387 | const std::string timeline_file = test_path_ + "/" + | |
388 | user_access_only + l + unit + | |
389 | "_access_timeline"; | |
390 | std::ifstream infile(timeline_file); | |
391 | std::string line; | |
392 | const uint64_t expected_naccesses = 50; | |
393 | const uint64_t expected_user_accesses = 30; | |
394 | ASSERT_TRUE(getline(infile, line)) << timeline_file; | |
395 | uint32_t naccesses = 0; | |
396 | while (getline(infile, line)) { | |
397 | std::stringstream ss_naccess(line); | |
398 | std::string substr; | |
399 | bool read_label = false; | |
400 | while (ss_naccess.good()) { | |
401 | ASSERT_TRUE(getline(ss_naccess, substr, ',')); | |
402 | if (!read_label) { | |
403 | read_label = true; | |
404 | continue; | |
405 | } | |
406 | naccesses += ParseUint32(substr); | |
407 | } | |
408 | } | |
409 | if (user_access_only == "user_access_only_") { | |
410 | ASSERT_EQ(expected_user_accesses, naccesses) << timeline_file; | |
411 | } else { | |
412 | ASSERT_EQ(expected_naccesses, naccesses) << timeline_file; | |
413 | } | |
414 | ASSERT_OK(env_->DeleteFile(timeline_file)); | |
415 | } | |
416 | } | |
417 | } | |
418 | } | |
419 | { | |
420 | // Validate the reuse_interval and reuse_distance csv files. | |
421 | std::map<std::string, std::string> test_reuse_csv_files; | |
422 | test_reuse_csv_files["_access_reuse_interval"] = reuse_interval_labels_; | |
423 | test_reuse_csv_files["_reuse_distance"] = reuse_distance_labels_; | |
424 | test_reuse_csv_files["_reuse_lifetime"] = reuse_lifetime_labels_; | |
425 | test_reuse_csv_files["_avg_reuse_interval"] = reuse_interval_labels_; | |
426 | test_reuse_csv_files["_avg_reuse_interval_naccesses"] = | |
427 | reuse_interval_labels_; | |
428 | for (auto const& test : test_reuse_csv_files) { | |
429 | const std::string& file_suffix = test.first; | |
430 | const std::string& labels = test.second; | |
431 | const uint32_t expected_num_rows = 5; | |
432 | std::stringstream ss(labels); | |
433 | while (ss.good()) { | |
434 | std::string l; | |
435 | ASSERT_TRUE(getline(ss, l, ',')); | |
436 | const std::string reuse_csv_file = test_path_ + "/" + l + file_suffix; | |
437 | std::ifstream infile(reuse_csv_file); | |
438 | std::string line; | |
439 | ASSERT_TRUE(getline(infile, line)); | |
440 | double npercentage = 0; | |
441 | uint32_t nrows = 0; | |
442 | while (getline(infile, line)) { | |
443 | std::stringstream ss_naccess(line); | |
444 | bool label_read = false; | |
445 | nrows++; | |
446 | while (ss_naccess.good()) { | |
447 | std::string substr; | |
448 | ASSERT_TRUE(getline(ss_naccess, substr, ',')); | |
449 | if (!label_read) { | |
450 | label_read = true; | |
451 | continue; | |
452 | } | |
453 | npercentage += ParseDouble(substr); | |
454 | } | |
455 | } | |
456 | ASSERT_EQ(expected_num_rows, nrows); | |
457 | if ("_reuse_lifetime" == test.first || | |
458 | "_avg_reuse_interval" == test.first || | |
459 | "_avg_reuse_interval_naccesses" == test.first) { | |
460 | ASSERT_EQ(100, npercentage) << reuse_csv_file; | |
461 | } else { | |
462 | ASSERT_LT(npercentage, 0); | |
463 | } | |
464 | ASSERT_OK(env_->DeleteFile(reuse_csv_file)); | |
465 | } | |
466 | } | |
467 | } | |
468 | ||
469 | { | |
470 | // Validate the percentage of accesses summary. | |
471 | const std::string percent_access_summary_file = | |
472 | test_path_ + "/percentage_of_accesses_summary"; | |
473 | std::ifstream infile(percent_access_summary_file); | |
474 | std::string line; | |
475 | ASSERT_TRUE(getline(infile, line)); | |
476 | std::set<std::string> callers; | |
477 | std::set<std::string> expected_callers{"Get", "MultiGet", "Iterator", | |
478 | "Prefetch", "Compaction"}; | |
479 | while (getline(infile, line)) { | |
480 | std::stringstream caller_percent(line); | |
481 | std::string caller; | |
482 | ASSERT_TRUE(getline(caller_percent, caller, ',')); | |
483 | std::string percent; | |
484 | ASSERT_TRUE(getline(caller_percent, percent, ',')); | |
485 | ASSERT_FALSE(caller_percent.good()); | |
486 | callers.insert(caller); | |
487 | ASSERT_EQ(20, ParseDouble(percent)); | |
488 | } | |
489 | ASSERT_EQ(expected_callers.size(), callers.size()); | |
490 | for (auto caller : callers) { | |
491 | ASSERT_TRUE(expected_callers.find(caller) != expected_callers.end()); | |
492 | } | |
493 | ASSERT_OK(env_->DeleteFile(percent_access_summary_file)); | |
494 | } | |
495 | { | |
496 | // Validate the percentage of accesses summary by analyzing callers. | |
497 | std::stringstream analyzing_callers(analyzing_callers_); | |
498 | while (analyzing_callers.good()) { | |
499 | std::string caller; | |
500 | ASSERT_TRUE(getline(analyzing_callers, caller, ',')); | |
501 | std::vector<std::string> breakdowns{"level", "bt"}; | |
502 | for (auto breakdown : breakdowns) { | |
503 | const std::string file_name = test_path_ + "/" + caller + "_" + | |
504 | breakdown + | |
505 | "_percentage_of_accesses_summary"; | |
506 | std::ifstream infile(file_name); | |
507 | std::string line; | |
508 | ASSERT_TRUE(getline(infile, line)); | |
509 | double sum = 0; | |
510 | while (getline(infile, line)) { | |
511 | std::stringstream label_percent(line); | |
512 | std::string label; | |
513 | ASSERT_TRUE(getline(label_percent, label, ',')); | |
514 | std::string percent; | |
515 | ASSERT_TRUE(getline(label_percent, percent, ',')); | |
516 | ASSERT_FALSE(label_percent.good()); | |
517 | sum += ParseDouble(percent); | |
518 | } | |
519 | ASSERT_EQ(100, sum); | |
520 | ASSERT_OK(env_->DeleteFile(file_name)); | |
521 | } | |
522 | } | |
523 | } | |
524 | const std::vector<std::string> access_types{"user_access_only", "all_access"}; | |
525 | const std::vector<std::string> prefix{"bt", "cf"}; | |
526 | for (auto const& pre : prefix) { | |
527 | for (auto const& access_type : access_types) { | |
528 | { | |
529 | // Validate the access count summary. | |
530 | const std::string bt_access_count_summary = test_path_ + "/" + pre + | |
531 | "_" + access_type + | |
532 | "_access_count_summary"; | |
533 | std::ifstream infile(bt_access_count_summary); | |
534 | std::string line; | |
535 | ASSERT_TRUE(getline(infile, line)); | |
536 | double sum_percent = 0; | |
537 | while (getline(infile, line)) { | |
538 | std::stringstream bt_percent(line); | |
539 | std::string bt; | |
540 | ASSERT_TRUE(getline(bt_percent, bt, ',')); | |
541 | std::string percent; | |
542 | ASSERT_TRUE(getline(bt_percent, percent, ',')); | |
543 | sum_percent += ParseDouble(percent); | |
544 | } | |
545 | ASSERT_EQ(100.0, sum_percent); | |
546 | ASSERT_OK(env_->DeleteFile(bt_access_count_summary)); | |
547 | } | |
548 | } | |
549 | } | |
550 | for (auto const& access_type : access_types) { | |
551 | std::vector<std::string> block_types{"Index", "Data", "Filter"}; | |
552 | for (auto block_type : block_types) { | |
553 | // Validate reuse block timeline. | |
554 | const std::string reuse_blocks_timeline = test_path_ + "/" + block_type + | |
555 | "_" + access_type + | |
556 | "_5_reuse_blocks_timeline"; | |
557 | std::ifstream infile(reuse_blocks_timeline); | |
558 | std::string line; | |
559 | ASSERT_TRUE(getline(infile, line)) << reuse_blocks_timeline; | |
560 | uint32_t index = 0; | |
561 | while (getline(infile, line)) { | |
562 | std::stringstream timeline(line); | |
563 | bool start_time = false; | |
564 | double sum = 0; | |
565 | while (timeline.good()) { | |
566 | std::string value; | |
567 | ASSERT_TRUE(getline(timeline, value, ',')); | |
568 | if (!start_time) { | |
569 | start_time = true; | |
570 | continue; | |
571 | } | |
572 | sum += ParseDouble(value); | |
573 | } | |
574 | index++; | |
575 | ASSERT_LT(sum, 100.0 * index + 1) << reuse_blocks_timeline; | |
576 | } | |
577 | ASSERT_OK(env_->DeleteFile(reuse_blocks_timeline)); | |
578 | } | |
579 | } | |
580 | ||
581 | std::stringstream ss(analyze_get_spatial_locality_labels_); | |
582 | while (ss.good()) { | |
583 | std::string l; | |
584 | ASSERT_TRUE(getline(ss, l, ',')); | |
585 | const std::vector<std::string> spatial_locality_files{ | |
586 | "_percent_ref_keys", "_percent_accesses_on_ref_keys", | |
587 | "_percent_data_size_on_ref_keys"}; | |
588 | for (auto const& spatial_locality_file : spatial_locality_files) { | |
589 | const std::string filename = test_path_ + "/" + l + spatial_locality_file; | |
590 | std::ifstream infile(filename); | |
591 | std::string line; | |
592 | ASSERT_TRUE(getline(infile, line)); | |
593 | double sum_percent = 0; | |
594 | uint32_t nrows = 0; | |
595 | while (getline(infile, line)) { | |
596 | std::stringstream bt_percent(line); | |
597 | std::string bt; | |
598 | ASSERT_TRUE(getline(bt_percent, bt, ',')); | |
599 | std::string percent; | |
600 | ASSERT_TRUE(getline(bt_percent, percent, ',')); | |
601 | sum_percent += ParseDouble(percent); | |
602 | nrows++; | |
603 | } | |
20effc67 | 604 | ASSERT_EQ(11u, nrows); |
f67539c2 TL |
605 | ASSERT_EQ(100.0, sum_percent); |
606 | ASSERT_OK(env_->DeleteFile(filename)); | |
607 | } | |
608 | } | |
609 | ASSERT_OK(env_->DeleteFile(block_cache_sim_config_path_)); | |
610 | } | |
611 | ||
612 | TEST_F(BlockCacheTracerTest, MixedBlocks) { | |
613 | { | |
614 | // Generate a trace file containing a mix of blocks. | |
615 | // It contains two SST files with 25 blocks of odd numbered block_key in | |
616 | // kSSTStoringOddKeys and 25 blocks of even numbered blocks_key in | |
617 | // kSSTStoringEvenKeys. | |
1e59de90 | 618 | BlockCacheTraceWriterOptions trace_writer_opt; |
f67539c2 | 619 | std::unique_ptr<TraceWriter> trace_writer; |
1e59de90 | 620 | const auto& clock = env_->GetSystemClock(); |
f67539c2 TL |
621 | ASSERT_OK(NewFileTraceWriter(env_, env_options_, trace_file_path_, |
622 | &trace_writer)); | |
1e59de90 TL |
623 | std::unique_ptr<BlockCacheTraceWriter> block_cache_trace_writer = |
624 | NewBlockCacheTraceWriter(clock.get(), trace_writer_opt, | |
625 | std::move(trace_writer)); | |
626 | ASSERT_NE(block_cache_trace_writer, nullptr); | |
627 | ASSERT_OK(block_cache_trace_writer->WriteHeader()); | |
f67539c2 | 628 | // Write blocks of different types. |
1e59de90 TL |
629 | WriteBlockAccess(block_cache_trace_writer.get(), 0, |
630 | TraceType::kBlockTraceUncompressionDictBlock, 10); | |
631 | WriteBlockAccess(block_cache_trace_writer.get(), 10, | |
632 | TraceType::kBlockTraceDataBlock, 10); | |
633 | WriteBlockAccess(block_cache_trace_writer.get(), 20, | |
634 | TraceType::kBlockTraceFilterBlock, 10); | |
635 | WriteBlockAccess(block_cache_trace_writer.get(), 30, | |
636 | TraceType::kBlockTraceIndexBlock, 10); | |
637 | WriteBlockAccess(block_cache_trace_writer.get(), 40, | |
638 | TraceType::kBlockTraceRangeDeletionBlock, 10); | |
f67539c2 TL |
639 | ASSERT_OK(env_->FileExists(trace_file_path_)); |
640 | } | |
641 | ||
642 | { | |
643 | // Verify trace file is generated correctly. | |
644 | std::unique_ptr<TraceReader> trace_reader; | |
645 | ASSERT_OK(NewFileTraceReader(env_, env_options_, trace_file_path_, | |
646 | &trace_reader)); | |
647 | BlockCacheTraceReader reader(std::move(trace_reader)); | |
648 | BlockCacheTraceHeader header; | |
649 | ASSERT_OK(reader.ReadHeader(&header)); | |
20effc67 TL |
650 | ASSERT_EQ(static_cast<uint32_t>(kMajorVersion), |
651 | header.rocksdb_major_version); | |
652 | ASSERT_EQ(static_cast<uint32_t>(kMinorVersion), | |
653 | header.rocksdb_minor_version); | |
f67539c2 TL |
654 | // Read blocks. |
655 | BlockCacheTraceAnalyzer analyzer( | |
656 | trace_file_path_, | |
657 | /*output_miss_ratio_curve_path=*/"", | |
658 | /*human_readable_trace_file_path=*/"", | |
659 | /*compute_reuse_distance=*/true, | |
660 | /*mrc_only=*/false, | |
661 | /*is_block_cache_human_readable_trace=*/false, | |
662 | /*simulator=*/nullptr); | |
663 | // The analyzer ends when it detects an incomplete access record. | |
664 | ASSERT_EQ(Status::Incomplete(""), analyzer.Analyze()); | |
665 | const uint64_t expected_num_cfs = 1; | |
666 | std::vector<uint64_t> expected_fds{kSSTStoringOddKeys, kSSTStoringEvenKeys}; | |
667 | const std::vector<TraceType> expected_types{ | |
668 | TraceType::kBlockTraceUncompressionDictBlock, | |
669 | TraceType::kBlockTraceDataBlock, TraceType::kBlockTraceFilterBlock, | |
670 | TraceType::kBlockTraceIndexBlock, | |
671 | TraceType::kBlockTraceRangeDeletionBlock}; | |
672 | const uint64_t expected_num_keys_per_type = 5; | |
673 | ||
674 | auto& stats = analyzer.TEST_cf_aggregates_map(); | |
675 | ASSERT_EQ(expected_num_cfs, stats.size()); | |
676 | ASSERT_TRUE(stats.find(kDefaultColumnFamilyName) != stats.end()); | |
677 | auto& cf_stats = stats.find(kDefaultColumnFamilyName)->second; | |
678 | ASSERT_EQ(expected_fds.size(), cf_stats.fd_aggregates_map.size()); | |
679 | for (auto fd_id : expected_fds) { | |
680 | ASSERT_TRUE(cf_stats.fd_aggregates_map.find(fd_id) != | |
681 | cf_stats.fd_aggregates_map.end()); | |
682 | ASSERT_EQ(kLevel, cf_stats.fd_aggregates_map.find(fd_id)->second.level); | |
683 | auto& block_type_aggregates_map = cf_stats.fd_aggregates_map.find(fd_id) | |
684 | ->second.block_type_aggregates_map; | |
685 | ASSERT_EQ(expected_types.size(), block_type_aggregates_map.size()); | |
686 | uint32_t key_id = 0; | |
687 | for (auto type : expected_types) { | |
688 | ASSERT_TRUE(block_type_aggregates_map.find(type) != | |
689 | block_type_aggregates_map.end()); | |
690 | auto& block_access_info_map = | |
691 | block_type_aggregates_map.find(type)->second.block_access_info_map; | |
692 | // Each block type has 5 blocks. | |
693 | ASSERT_EQ(expected_num_keys_per_type, block_access_info_map.size()); | |
694 | for (uint32_t i = 0; i < 10; i++) { | |
695 | // Verify that odd numbered blocks are stored in kSSTStoringOddKeys | |
696 | // and even numbered blocks are stored in kSSTStoringEvenKeys. | |
697 | auto key_id_str = kBlockKeyPrefix + std::to_string(key_id); | |
698 | if (fd_id == kSSTStoringOddKeys) { | |
699 | if (key_id % 2 == 1) { | |
700 | AssertBlockAccessInfo(key_id, type, block_access_info_map); | |
701 | } else { | |
702 | ASSERT_TRUE(block_access_info_map.find(key_id_str) == | |
703 | block_access_info_map.end()); | |
704 | } | |
705 | } else { | |
706 | if (key_id % 2 == 1) { | |
707 | ASSERT_TRUE(block_access_info_map.find(key_id_str) == | |
708 | block_access_info_map.end()); | |
709 | } else { | |
710 | AssertBlockAccessInfo(key_id, type, block_access_info_map); | |
711 | } | |
712 | } | |
713 | key_id++; | |
714 | } | |
715 | } | |
716 | } | |
717 | } | |
718 | } | |
719 | ||
720 | } // namespace ROCKSDB_NAMESPACE | |
721 | ||
722 | int main(int argc, char** argv) { | |
1e59de90 | 723 | ROCKSDB_NAMESPACE::port::InstallStackTraceHandler(); |
f67539c2 TL |
724 | ::testing::InitGoogleTest(&argc, argv); |
725 | return RUN_ALL_TESTS(); | |
726 | } | |
727 | #endif // GFLAG | |
728 | #else | |
729 | #include <stdio.h> | |
730 | int main(int /*argc*/, char** /*argv*/) { | |
731 | fprintf(stderr, | |
732 | "block_cache_trace_analyzer_test is not supported in ROCKSDB_LITE\n"); | |
733 | return 0; | |
734 | } | |
735 | #endif // ROCKSDB_LITE |