1 // Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
2 // This source code is licensed under both the GPLv2 (found in the
3 // COPYING file in the root directory) and Apache 2.0 License
4 // (found in the LICENSE.Apache file in the root directory).
6 // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
7 // Use of this source code is governed by a BSD-style license that can be
8 // found in the LICENSE file. See the AUTHORS file for names of contributors.
18 #include "cache/cache_entry_roles.h"
19 #include "db/version_set.h"
20 #include "rocksdb/system_clock.h"
21 #include "util/hash_containers.h"
23 namespace ROCKSDB_NAMESPACE
{
25 template <class Stats
>
26 class CacheEntryStatsCollector
;
30 // Config for retrieving a property's value.
31 struct DBPropertyInfo
{
32 bool need_out_of_mutex
;
34 // gcc had an internal error for initializing union of pointer-to-member-
35 // functions. Workaround is to populate exactly one of the following function
36 // pointers with a non-nullptr value.
38 // @param value Value-result argument for storing the property's string value
39 // @param suffix Argument portion of the property. For example, suffix would
40 // be "5" for the property "rocksdb.num-files-at-level5". So far, only
41 // certain string properties take an argument.
42 bool (InternalStats::*handle_string
)(std::string
* value
, Slice suffix
);
44 // @param value Value-result argument for storing the property's uint64 value
45 // @param db Many of the int properties rely on DBImpl methods.
46 // @param version Version is needed in case the property is retrieved without
47 // holding db mutex, which is only supported for int properties.
48 bool (InternalStats::*handle_int
)(uint64_t* value
, DBImpl
* db
,
51 // @param props Map of general properties to populate
52 // @param suffix Argument portion of the property. (see handle_string)
53 bool (InternalStats::*handle_map
)(std::map
<std::string
, std::string
>* props
,
56 // handle the string type properties rely on DBImpl methods
57 // @param value Value-result argument for storing the property's string value
58 bool (DBImpl::*handle_string_dbimpl
)(std::string
* value
);
61 extern const DBPropertyInfo
* GetPropertyInfo(const Slice
& property
);
65 enum class LevelStatType
{
88 TOTAL
// total number of types
92 // This what will be L?.property_name in the flat map returned to the user
93 std::string property_name
;
94 // This will be what we will print in the header in the cli
95 std::string header_name
;
99 // This what will be property_name in the flat map returned to the user
100 std::string property_name
;
103 class InternalStats
{
105 static const std::map
<LevelStatType
, LevelStat
> compaction_level_stats
;
107 enum InternalCFStatsType
{
108 L0_FILE_COUNT_LIMIT_SLOWDOWNS
,
109 LOCKED_L0_FILE_COUNT_LIMIT_SLOWDOWNS
,
110 MEMTABLE_LIMIT_STOPS
,
111 MEMTABLE_LIMIT_SLOWDOWNS
,
112 L0_FILE_COUNT_LIMIT_STOPS
,
113 LOCKED_L0_FILE_COUNT_LIMIT_STOPS
,
114 PENDING_COMPACTION_BYTES_LIMIT_SLOWDOWNS
,
115 PENDING_COMPACTION_BYTES_LIMIT_STOPS
,
116 WRITE_STALLS_ENUM_MAX
,
118 BYTES_INGESTED_ADD_FILE
,
119 INGESTED_NUM_FILES_TOTAL
,
120 INGESTED_LEVEL0_NUM_FILES_TOTAL
,
121 INGESTED_NUM_KEYS_TOTAL
,
122 INTERNAL_CF_STATS_ENUM_MAX
,
125 enum InternalDBStatsType
{
126 kIntStatsWalFileBytes
,
127 kIntStatsWalFileSynced
,
128 kIntStatsBytesWritten
,
129 kIntStatsNumKeysWritten
,
130 kIntStatsWriteDoneByOther
,
131 kIntStatsWriteDoneBySelf
,
132 kIntStatsWriteWithWal
,
133 kIntStatsWriteStallMicros
,
137 static const std::map
<InternalDBStatsType
, DBStatInfo
> db_stats_type_to_info
;
139 InternalStats(int num_levels
, SystemClock
* clock
, ColumnFamilyData
* cfd
);
141 // Per level compaction stats
142 struct CompactionOutputsStats
{
143 uint64_t num_output_records
= 0;
144 uint64_t bytes_written
= 0;
145 uint64_t bytes_written_blob
= 0;
146 uint64_t num_output_files
= 0;
147 uint64_t num_output_files_blob
= 0;
149 void Add(const CompactionOutputsStats
& stats
) {
150 this->num_output_records
+= stats
.num_output_records
;
151 this->bytes_written
+= stats
.bytes_written
;
152 this->bytes_written_blob
+= stats
.bytes_written_blob
;
153 this->num_output_files
+= stats
.num_output_files
;
154 this->num_output_files_blob
+= stats
.num_output_files_blob
;
158 // Per level compaction stats. comp_stats_[level] stores the stats for
159 // compactions that produced data for the specified "level".
160 struct CompactionStats
{
164 // The number of bytes read from all non-output levels (table files)
165 uint64_t bytes_read_non_output_levels
;
167 // The number of bytes read from the compaction output level (table files)
168 uint64_t bytes_read_output_level
;
170 // The number of bytes read from blob files
171 uint64_t bytes_read_blob
;
173 // Total number of bytes written to table files during compaction
174 uint64_t bytes_written
;
176 // Total number of bytes written to blob files during compaction
177 uint64_t bytes_written_blob
;
179 // Total number of bytes moved to the output level (table files)
180 uint64_t bytes_moved
;
182 // The number of compaction input files in all non-output levels (table
184 int num_input_files_in_non_output_levels
;
186 // The number of compaction input files in the output level (table files)
187 int num_input_files_in_output_level
;
189 // The number of compaction output files (table files)
190 int num_output_files
;
192 // The number of compaction output files (blob files)
193 int num_output_files_blob
;
195 // Total incoming entries during compaction between levels N and N+1
196 uint64_t num_input_records
;
198 // Accumulated diff number of entries
199 // (num input entries - num output entries) for compaction levels N and N+1
200 uint64_t num_dropped_records
;
202 // Total output entries from compaction
203 uint64_t num_output_records
;
205 // Number of compactions done
208 // Number of compactions done per CompactionReason
209 int counts
[static_cast<int>(CompactionReason::kNumOfReasons
)]{};
211 explicit CompactionStats()
214 bytes_read_non_output_levels(0),
215 bytes_read_output_level(0),
218 bytes_written_blob(0),
220 num_input_files_in_non_output_levels(0),
221 num_input_files_in_output_level(0),
223 num_output_files_blob(0),
224 num_input_records(0),
225 num_dropped_records(0),
226 num_output_records(0),
228 int num_of_reasons
= static_cast<int>(CompactionReason::kNumOfReasons
);
229 for (int i
= 0; i
< num_of_reasons
; i
++) {
234 explicit CompactionStats(CompactionReason reason
, int c
)
237 bytes_read_non_output_levels(0),
238 bytes_read_output_level(0),
241 bytes_written_blob(0),
243 num_input_files_in_non_output_levels(0),
244 num_input_files_in_output_level(0),
246 num_output_files_blob(0),
247 num_input_records(0),
248 num_dropped_records(0),
249 num_output_records(0),
251 int num_of_reasons
= static_cast<int>(CompactionReason::kNumOfReasons
);
252 for (int i
= 0; i
< num_of_reasons
; i
++) {
255 int r
= static_cast<int>(reason
);
256 if (r
>= 0 && r
< num_of_reasons
) {
263 CompactionStats(const CompactionStats
& c
)
265 cpu_micros(c
.cpu_micros
),
266 bytes_read_non_output_levels(c
.bytes_read_non_output_levels
),
267 bytes_read_output_level(c
.bytes_read_output_level
),
268 bytes_read_blob(c
.bytes_read_blob
),
269 bytes_written(c
.bytes_written
),
270 bytes_written_blob(c
.bytes_written_blob
),
271 bytes_moved(c
.bytes_moved
),
272 num_input_files_in_non_output_levels(
273 c
.num_input_files_in_non_output_levels
),
274 num_input_files_in_output_level(c
.num_input_files_in_output_level
),
275 num_output_files(c
.num_output_files
),
276 num_output_files_blob(c
.num_output_files_blob
),
277 num_input_records(c
.num_input_records
),
278 num_dropped_records(c
.num_dropped_records
),
279 num_output_records(c
.num_output_records
),
281 int num_of_reasons
= static_cast<int>(CompactionReason::kNumOfReasons
);
282 for (int i
= 0; i
< num_of_reasons
; i
++) {
283 counts
[i
] = c
.counts
[i
];
287 CompactionStats
& operator=(const CompactionStats
& c
) {
289 cpu_micros
= c
.cpu_micros
;
290 bytes_read_non_output_levels
= c
.bytes_read_non_output_levels
;
291 bytes_read_output_level
= c
.bytes_read_output_level
;
292 bytes_read_blob
= c
.bytes_read_blob
;
293 bytes_written
= c
.bytes_written
;
294 bytes_written_blob
= c
.bytes_written_blob
;
295 bytes_moved
= c
.bytes_moved
;
296 num_input_files_in_non_output_levels
=
297 c
.num_input_files_in_non_output_levels
;
298 num_input_files_in_output_level
= c
.num_input_files_in_output_level
;
299 num_output_files
= c
.num_output_files
;
300 num_output_files_blob
= c
.num_output_files_blob
;
301 num_input_records
= c
.num_input_records
;
302 num_dropped_records
= c
.num_dropped_records
;
303 num_output_records
= c
.num_output_records
;
306 int num_of_reasons
= static_cast<int>(CompactionReason::kNumOfReasons
);
307 for (int i
= 0; i
< num_of_reasons
; i
++) {
308 counts
[i
] = c
.counts
[i
];
315 this->cpu_micros
= 0;
316 this->bytes_read_non_output_levels
= 0;
317 this->bytes_read_output_level
= 0;
318 this->bytes_read_blob
= 0;
319 this->bytes_written
= 0;
320 this->bytes_written_blob
= 0;
321 this->bytes_moved
= 0;
322 this->num_input_files_in_non_output_levels
= 0;
323 this->num_input_files_in_output_level
= 0;
324 this->num_output_files
= 0;
325 this->num_output_files_blob
= 0;
326 this->num_input_records
= 0;
327 this->num_dropped_records
= 0;
328 this->num_output_records
= 0;
330 int num_of_reasons
= static_cast<int>(CompactionReason::kNumOfReasons
);
331 for (int i
= 0; i
< num_of_reasons
; i
++) {
336 void Add(const CompactionStats
& c
) {
337 this->micros
+= c
.micros
;
338 this->cpu_micros
+= c
.cpu_micros
;
339 this->bytes_read_non_output_levels
+= c
.bytes_read_non_output_levels
;
340 this->bytes_read_output_level
+= c
.bytes_read_output_level
;
341 this->bytes_read_blob
+= c
.bytes_read_blob
;
342 this->bytes_written
+= c
.bytes_written
;
343 this->bytes_written_blob
+= c
.bytes_written_blob
;
344 this->bytes_moved
+= c
.bytes_moved
;
345 this->num_input_files_in_non_output_levels
+=
346 c
.num_input_files_in_non_output_levels
;
347 this->num_input_files_in_output_level
+=
348 c
.num_input_files_in_output_level
;
349 this->num_output_files
+= c
.num_output_files
;
350 this->num_output_files_blob
+= c
.num_output_files_blob
;
351 this->num_input_records
+= c
.num_input_records
;
352 this->num_dropped_records
+= c
.num_dropped_records
;
353 this->num_output_records
+= c
.num_output_records
;
354 this->count
+= c
.count
;
355 int num_of_reasons
= static_cast<int>(CompactionReason::kNumOfReasons
);
356 for (int i
= 0; i
< num_of_reasons
; i
++) {
357 counts
[i
] += c
.counts
[i
];
361 void Add(const CompactionOutputsStats
& stats
) {
362 this->num_output_files
+= static_cast<int>(stats
.num_output_files
);
363 this->num_output_records
+= stats
.num_output_records
;
364 this->bytes_written
+= stats
.bytes_written
;
365 this->bytes_written_blob
+= stats
.bytes_written_blob
;
366 this->num_output_files_blob
+=
367 static_cast<int>(stats
.num_output_files_blob
);
370 void Subtract(const CompactionStats
& c
) {
371 this->micros
-= c
.micros
;
372 this->cpu_micros
-= c
.cpu_micros
;
373 this->bytes_read_non_output_levels
-= c
.bytes_read_non_output_levels
;
374 this->bytes_read_output_level
-= c
.bytes_read_output_level
;
375 this->bytes_read_blob
-= c
.bytes_read_blob
;
376 this->bytes_written
-= c
.bytes_written
;
377 this->bytes_written_blob
-= c
.bytes_written_blob
;
378 this->bytes_moved
-= c
.bytes_moved
;
379 this->num_input_files_in_non_output_levels
-=
380 c
.num_input_files_in_non_output_levels
;
381 this->num_input_files_in_output_level
-=
382 c
.num_input_files_in_output_level
;
383 this->num_output_files
-= c
.num_output_files
;
384 this->num_output_files_blob
-= c
.num_output_files_blob
;
385 this->num_input_records
-= c
.num_input_records
;
386 this->num_dropped_records
-= c
.num_dropped_records
;
387 this->num_output_records
-= c
.num_output_records
;
388 this->count
-= c
.count
;
389 int num_of_reasons
= static_cast<int>(CompactionReason::kNumOfReasons
);
390 for (int i
= 0; i
< num_of_reasons
; i
++) {
391 counts
[i
] -= c
.counts
[i
];
395 void ResetCompactionReason(CompactionReason reason
) {
396 int num_of_reasons
= static_cast<int>(CompactionReason::kNumOfReasons
);
397 assert(count
== 1); // only support update one compaction reason
398 for (int i
= 0; i
< num_of_reasons
; i
++) {
401 int r
= static_cast<int>(reason
);
402 assert(r
>= 0 && r
< num_of_reasons
);
407 // Compaction stats, for per_key_placement compaction, it includes 2 levels
408 // stats: the last level and the penultimate level.
409 struct CompactionStatsFull
{
410 // the stats for the target primary output level
411 CompactionStats stats
;
413 // stats for penultimate level output if exist
414 bool has_penultimate_level_output
= false;
415 CompactionStats penultimate_level_stats
;
417 explicit CompactionStatsFull() : stats(), penultimate_level_stats() {}
419 explicit CompactionStatsFull(CompactionReason reason
, int c
)
420 : stats(reason
, c
), penultimate_level_stats(reason
, c
){};
422 uint64_t TotalBytesWritten() const {
423 uint64_t bytes_written
= stats
.bytes_written
+ stats
.bytes_written_blob
;
424 if (has_penultimate_level_output
) {
425 bytes_written
+= penultimate_level_stats
.bytes_written
+
426 penultimate_level_stats
.bytes_written_blob
;
428 return bytes_written
;
431 uint64_t DroppedRecords() {
432 uint64_t output_records
= stats
.num_output_records
;
433 if (has_penultimate_level_output
) {
434 output_records
+= penultimate_level_stats
.num_output_records
;
436 if (stats
.num_input_records
> output_records
) {
437 return stats
.num_input_records
- output_records
;
442 void SetMicros(uint64_t val
) {
444 penultimate_level_stats
.micros
= val
;
447 void AddCpuMicros(uint64_t val
) {
448 stats
.cpu_micros
+= val
;
449 penultimate_level_stats
.cpu_micros
+= val
;
453 // For use with CacheEntryStatsCollector
454 struct CacheEntryRoleStats
{
455 uint64_t cache_capacity
= 0;
456 uint64_t cache_usage
= 0;
457 size_t table_size
= 0;
458 size_t occupancy
= 0;
459 std::string cache_id
;
460 std::array
<uint64_t, kNumCacheEntryRoles
> total_charges
;
461 std::array
<size_t, kNumCacheEntryRoles
> entry_counts
;
462 uint32_t collection_count
= 0;
463 uint32_t copies_of_last_collection
= 0;
464 uint64_t last_start_time_micros_
= 0;
465 uint64_t last_end_time_micros_
= 0;
468 // Wipe everything except collection_count
469 uint32_t saved_collection_count
= collection_count
;
470 *this = CacheEntryRoleStats();
471 collection_count
= saved_collection_count
;
474 void BeginCollection(Cache
*, SystemClock
*, uint64_t start_time_micros
);
475 std::function
<void(const Slice
&, void*, size_t, Cache::DeleterFn
)>
477 void EndCollection(Cache
*, SystemClock
*, uint64_t end_time_micros
);
478 void SkippedCollection();
480 std::string
ToString(SystemClock
* clock
) const;
481 void ToMap(std::map
<std::string
, std::string
>* values
,
482 SystemClock
* clock
) const;
485 UnorderedMap
<Cache::DeleterFn
, CacheEntryRole
> role_map_
;
486 uint64_t GetLastDurationMicros() const;
490 for (int i
= 0; i
< kIntStatsNumMax
; i
++) {
491 db_stats_
[i
].store(0);
493 for (int i
= 0; i
< INTERNAL_CF_STATS_ENUM_MAX
; i
++) {
494 cf_stats_count_
[i
] = 0;
495 cf_stats_value_
[i
] = 0;
497 for (auto& comp_stat
: comp_stats_
) {
500 per_key_placement_comp_stats_
.Clear();
501 for (auto& h
: file_read_latency_
) {
504 blob_file_read_latency_
.Clear();
505 cf_stats_snapshot_
.Clear();
506 db_stats_snapshot_
.Clear();
508 started_at_
= clock_
->NowMicros();
509 has_cf_change_since_dump_
= true;
512 void AddCompactionStats(int level
, Env::Priority thread_pri
,
513 const CompactionStats
& stats
) {
514 comp_stats_
[level
].Add(stats
);
515 comp_stats_by_pri_
[thread_pri
].Add(stats
);
518 void AddCompactionStats(int level
, Env::Priority thread_pri
,
519 const CompactionStatsFull
& comp_stats_full
) {
520 AddCompactionStats(level
, thread_pri
, comp_stats_full
.stats
);
521 if (comp_stats_full
.has_penultimate_level_output
) {
522 per_key_placement_comp_stats_
.Add(
523 comp_stats_full
.penultimate_level_stats
);
527 void IncBytesMoved(int level
, uint64_t amount
) {
528 comp_stats_
[level
].bytes_moved
+= amount
;
531 void AddCFStats(InternalCFStatsType type
, uint64_t value
) {
532 has_cf_change_since_dump_
= true;
533 cf_stats_value_
[type
] += value
;
534 ++cf_stats_count_
[type
];
537 void AddDBStats(InternalDBStatsType type
, uint64_t value
,
538 bool concurrent
= false) {
539 auto& v
= db_stats_
[type
];
541 v
.fetch_add(value
, std::memory_order_relaxed
);
543 v
.store(v
.load(std::memory_order_relaxed
) + value
,
544 std::memory_order_relaxed
);
548 uint64_t GetDBStats(InternalDBStatsType type
) {
549 return db_stats_
[type
].load(std::memory_order_relaxed
);
552 HistogramImpl
* GetFileReadHist(int level
) {
553 return &file_read_latency_
[level
];
556 HistogramImpl
* GetBlobFileReadHist() { return &blob_file_read_latency_
; }
558 uint64_t GetBackgroundErrorCount() const { return bg_error_count_
; }
560 uint64_t BumpAndGetBackgroundErrorCount() { return ++bg_error_count_
; }
562 bool GetStringProperty(const DBPropertyInfo
& property_info
,
563 const Slice
& property
, std::string
* value
);
565 bool GetMapProperty(const DBPropertyInfo
& property_info
,
566 const Slice
& property
,
567 std::map
<std::string
, std::string
>* value
);
569 bool GetIntProperty(const DBPropertyInfo
& property_info
, uint64_t* value
,
572 bool GetIntPropertyOutOfMutex(const DBPropertyInfo
& property_info
,
573 Version
* version
, uint64_t* value
);
575 // Unless there is a recent enough collection of the stats, collect and
576 // saved new cache entry stats. If `foreground`, require data to be more
577 // recent to skip re-collection.
579 // This should only be called while NOT holding the DB mutex.
580 void CollectCacheEntryStats(bool foreground
);
582 const uint64_t* TEST_GetCFStatsValue() const { return cf_stats_value_
; }
584 const std::vector
<CompactionStats
>& TEST_GetCompactionStats() const {
588 const CompactionStats
& TEST_GetPerKeyPlacementCompactionStats() const {
589 return per_key_placement_comp_stats_
;
592 void TEST_GetCacheEntryRoleStats(CacheEntryRoleStats
* stats
, bool foreground
);
594 // Store a mapping from the user-facing DB::Properties string to our
595 // DBPropertyInfo struct used internally for retrieving properties.
596 static const UnorderedMap
<std::string
, DBPropertyInfo
> ppt_name_to_info
;
598 static const std::string kPeriodicCFStats
;
601 void DumpDBMapStats(std::map
<std::string
, std::string
>* db_stats
);
602 void DumpDBStats(std::string
* value
);
603 void DumpCFMapStats(std::map
<std::string
, std::string
>* cf_stats
);
605 const VersionStorageInfo
* vstorage
,
606 std::map
<int, std::map
<LevelStatType
, double>>* level_stats
,
607 CompactionStats
* compaction_stats_sum
);
608 void DumpCFMapStatsByPriority(
609 std::map
<int, std::map
<LevelStatType
, double>>* priorities_stats
);
610 void DumpCFMapStatsIOStalls(std::map
<std::string
, std::string
>* cf_stats
);
611 void DumpCFStats(std::string
* value
);
612 // if is_periodic = true, it is an internal call by RocksDB periodically to
614 void DumpCFStatsNoFileHistogram(bool is_periodic
, std::string
* value
);
615 // if is_periodic = true, it is an internal call by RocksDB periodically to
617 void DumpCFFileHistogram(std::string
* value
);
619 Cache
* GetBlockCacheForStats();
620 Cache
* GetBlobCacheForStats();
623 std::atomic
<uint64_t> db_stats_
[kIntStatsNumMax
];
624 // Per-ColumnFamily stats
625 uint64_t cf_stats_value_
[INTERNAL_CF_STATS_ENUM_MAX
];
626 uint64_t cf_stats_count_
[INTERNAL_CF_STATS_ENUM_MAX
];
627 // Initialize/reference the collector in constructor so that we don't need
628 // additional synchronization in InternalStats, relying on synchronization
629 // in CacheEntryStatsCollector::GetStats. This collector is pinned in cache
630 // (through a shared_ptr) so that it does not get immediately ejected from
631 // a full cache, which would force a re-scan on the next GetStats.
632 std::shared_ptr
<CacheEntryStatsCollector
<CacheEntryRoleStats
>>
633 cache_entry_stats_collector_
;
634 // Per-ColumnFamily/level compaction stats
635 std::vector
<CompactionStats
> comp_stats_
;
636 std::vector
<CompactionStats
> comp_stats_by_pri_
;
637 CompactionStats per_key_placement_comp_stats_
;
638 std::vector
<HistogramImpl
> file_read_latency_
;
639 HistogramImpl blob_file_read_latency_
;
640 bool has_cf_change_since_dump_
;
641 // How many periods of no change since the last time stats are dumped for
643 int no_cf_change_period_since_dump_
= 0;
644 uint64_t last_histogram_num
= std::numeric_limits
<uint64_t>::max();
645 static const int kMaxNoChangePeriodSinceDump
;
647 // Used to compute per-interval statistics
648 struct CFStatsSnapshot
{
649 // ColumnFamily-level stats
650 CompactionStats comp_stats
;
651 uint64_t ingest_bytes_flush
; // Bytes written to L0 (Flush)
652 uint64_t stall_count
; // Stall count
653 // Stats from compaction jobs - bytes written, bytes read, duration.
654 uint64_t compact_bytes_write
;
655 uint64_t compact_bytes_read
;
656 uint64_t compact_micros
;
659 // AddFile specific stats
660 uint64_t ingest_bytes_addfile
; // Total Bytes ingested
661 uint64_t ingest_files_addfile
; // Total number of files ingested
662 uint64_t ingest_l0_files_addfile
; // Total number of files ingested to L0
663 uint64_t ingest_keys_addfile
; // Total number of keys ingested
666 : ingest_bytes_flush(0),
668 compact_bytes_write(0),
669 compact_bytes_read(0),
672 ingest_bytes_addfile(0),
673 ingest_files_addfile(0),
674 ingest_l0_files_addfile(0),
675 ingest_keys_addfile(0) {}
679 ingest_bytes_flush
= 0;
681 compact_bytes_write
= 0;
682 compact_bytes_read
= 0;
685 ingest_bytes_addfile
= 0;
686 ingest_files_addfile
= 0;
687 ingest_l0_files_addfile
= 0;
688 ingest_keys_addfile
= 0;
690 } cf_stats_snapshot_
;
692 struct DBStatsSnapshot
{
694 uint64_t ingest_bytes
; // Bytes written by user
695 uint64_t wal_bytes
; // Bytes written to WAL
696 uint64_t wal_synced
; // Number of times WAL is synced
697 uint64_t write_with_wal
; // Number of writes that request WAL
698 // These count the number of writes processed by the calling thread or
700 uint64_t write_other
;
702 // Total number of keys written. write_self and write_other measure number
703 // of write requests written, Each of the write request can contain updates
704 // to multiple keys. num_keys_written is total number of keys updated by all
706 uint64_t num_keys_written
;
707 // Total time writes delayed by stalls.
708 uint64_t write_stall_micros
;
719 write_stall_micros(0),
729 num_keys_written
= 0;
730 write_stall_micros
= 0;
733 } db_stats_snapshot_
;
735 // Handler functions for getting property values. They use "value" as a value-
736 // result argument, and return true upon successfully setting "value".
737 bool HandleNumFilesAtLevel(std::string
* value
, Slice suffix
);
738 bool HandleCompressionRatioAtLevelPrefix(std::string
* value
, Slice suffix
);
739 bool HandleLevelStats(std::string
* value
, Slice suffix
);
740 bool HandleStats(std::string
* value
, Slice suffix
);
741 bool HandleCFMapStats(std::map
<std::string
, std::string
>* compaction_stats
,
743 bool HandleCFStats(std::string
* value
, Slice suffix
);
744 bool HandleCFStatsNoFileHistogram(std::string
* value
, Slice suffix
);
745 bool HandleCFFileHistogram(std::string
* value
, Slice suffix
);
746 bool HandleCFStatsPeriodic(std::string
* value
, Slice suffix
);
747 bool HandleDBMapStats(std::map
<std::string
, std::string
>* compaction_stats
,
749 bool HandleDBStats(std::string
* value
, Slice suffix
);
750 bool HandleSsTables(std::string
* value
, Slice suffix
);
751 bool HandleAggregatedTableProperties(std::string
* value
, Slice suffix
);
752 bool HandleAggregatedTablePropertiesAtLevel(std::string
* value
, Slice suffix
);
753 bool HandleAggregatedTablePropertiesMap(
754 std::map
<std::string
, std::string
>* values
, Slice suffix
);
755 bool HandleAggregatedTablePropertiesAtLevelMap(
756 std::map
<std::string
, std::string
>* values
, Slice suffix
);
757 bool HandleNumImmutableMemTable(uint64_t* value
, DBImpl
* db
,
759 bool HandleNumImmutableMemTableFlushed(uint64_t* value
, DBImpl
* db
,
761 bool HandleMemTableFlushPending(uint64_t* value
, DBImpl
* db
,
763 bool HandleNumRunningFlushes(uint64_t* value
, DBImpl
* db
, Version
* version
);
764 bool HandleCompactionPending(uint64_t* value
, DBImpl
* db
, Version
* version
);
765 bool HandleNumRunningCompactions(uint64_t* value
, DBImpl
* db
,
767 bool HandleBackgroundErrors(uint64_t* value
, DBImpl
* db
, Version
* version
);
768 bool HandleCurSizeActiveMemTable(uint64_t* value
, DBImpl
* db
,
770 bool HandleCurSizeAllMemTables(uint64_t* value
, DBImpl
* db
, Version
* version
);
771 bool HandleSizeAllMemTables(uint64_t* value
, DBImpl
* db
, Version
* version
);
772 bool HandleNumEntriesActiveMemTable(uint64_t* value
, DBImpl
* db
,
774 bool HandleNumEntriesImmMemTables(uint64_t* value
, DBImpl
* db
,
776 bool HandleNumDeletesActiveMemTable(uint64_t* value
, DBImpl
* db
,
778 bool HandleNumDeletesImmMemTables(uint64_t* value
, DBImpl
* db
,
780 bool HandleEstimateNumKeys(uint64_t* value
, DBImpl
* db
, Version
* version
);
781 bool HandleNumSnapshots(uint64_t* value
, DBImpl
* db
, Version
* version
);
782 bool HandleOldestSnapshotTime(uint64_t* value
, DBImpl
* db
, Version
* version
);
783 bool HandleOldestSnapshotSequence(uint64_t* value
, DBImpl
* db
,
785 bool HandleNumLiveVersions(uint64_t* value
, DBImpl
* db
, Version
* version
);
786 bool HandleCurrentSuperVersionNumber(uint64_t* value
, DBImpl
* db
,
788 bool HandleIsFileDeletionsEnabled(uint64_t* value
, DBImpl
* db
,
790 bool HandleBaseLevel(uint64_t* value
, DBImpl
* db
, Version
* version
);
791 bool HandleTotalSstFilesSize(uint64_t* value
, DBImpl
* db
, Version
* version
);
792 bool HandleLiveSstFilesSize(uint64_t* value
, DBImpl
* db
, Version
* version
);
793 bool HandleEstimatePendingCompactionBytes(uint64_t* value
, DBImpl
* db
,
795 bool HandleEstimateTableReadersMem(uint64_t* value
, DBImpl
* db
,
797 bool HandleEstimateLiveDataSize(uint64_t* value
, DBImpl
* db
,
799 bool HandleMinLogNumberToKeep(uint64_t* value
, DBImpl
* db
, Version
* version
);
800 bool HandleMinObsoleteSstNumberToKeep(uint64_t* value
, DBImpl
* db
,
802 bool HandleActualDelayedWriteRate(uint64_t* value
, DBImpl
* db
,
804 bool HandleIsWriteStopped(uint64_t* value
, DBImpl
* db
, Version
* version
);
805 bool HandleEstimateOldestKeyTime(uint64_t* value
, DBImpl
* db
,
807 bool HandleBlockCacheCapacity(uint64_t* value
, DBImpl
* db
, Version
* version
);
808 bool HandleBlockCacheUsage(uint64_t* value
, DBImpl
* db
, Version
* version
);
809 bool HandleBlockCachePinnedUsage(uint64_t* value
, DBImpl
* db
,
811 bool HandleBlockCacheEntryStatsInternal(std::string
* value
, bool fast
);
812 bool HandleBlockCacheEntryStatsMapInternal(
813 std::map
<std::string
, std::string
>* values
, bool fast
);
814 bool HandleBlockCacheEntryStats(std::string
* value
, Slice suffix
);
815 bool HandleBlockCacheEntryStatsMap(std::map
<std::string
, std::string
>* values
,
817 bool HandleFastBlockCacheEntryStats(std::string
* value
, Slice suffix
);
818 bool HandleFastBlockCacheEntryStatsMap(
819 std::map
<std::string
, std::string
>* values
, Slice suffix
);
820 bool HandleLiveSstFilesSizeAtTemperature(std::string
* value
, Slice suffix
);
821 bool HandleNumBlobFiles(uint64_t* value
, DBImpl
* db
, Version
* version
);
822 bool HandleBlobStats(std::string
* value
, Slice suffix
);
823 bool HandleTotalBlobFileSize(uint64_t* value
, DBImpl
* db
, Version
* version
);
824 bool HandleLiveBlobFileSize(uint64_t* value
, DBImpl
* db
, Version
* version
);
825 bool HandleLiveBlobFileGarbageSize(uint64_t* value
, DBImpl
* db
,
827 bool HandleBlobCacheCapacity(uint64_t* value
, DBImpl
* db
, Version
* version
);
828 bool HandleBlobCacheUsage(uint64_t* value
, DBImpl
* db
, Version
* version
);
829 bool HandleBlobCachePinnedUsage(uint64_t* value
, DBImpl
* db
,
832 // Total number of background errors encountered. Every time a flush task
833 // or compaction task fails, this counter is incremented. The failure can
834 // be caused by any possible reason, including file system errors, out of
835 // resources, or input file corruption. Failing when retrying the same flush
836 // or compaction will cause the counter to increase too.
837 uint64_t bg_error_count_
;
839 const int number_levels_
;
841 ColumnFamilyData
* cfd_
;
842 uint64_t started_at_
;
847 class InternalStats
{
849 enum InternalCFStatsType
{
850 L0_FILE_COUNT_LIMIT_SLOWDOWNS
,
851 LOCKED_L0_FILE_COUNT_LIMIT_SLOWDOWNS
,
852 MEMTABLE_LIMIT_STOPS
,
853 MEMTABLE_LIMIT_SLOWDOWNS
,
854 L0_FILE_COUNT_LIMIT_STOPS
,
855 LOCKED_L0_FILE_COUNT_LIMIT_STOPS
,
856 PENDING_COMPACTION_BYTES_LIMIT_SLOWDOWNS
,
857 PENDING_COMPACTION_BYTES_LIMIT_STOPS
,
858 WRITE_STALLS_ENUM_MAX
,
860 BYTES_INGESTED_ADD_FILE
,
861 INGESTED_NUM_FILES_TOTAL
,
862 INGESTED_LEVEL0_NUM_FILES_TOTAL
,
863 INGESTED_NUM_KEYS_TOTAL
,
864 INTERNAL_CF_STATS_ENUM_MAX
,
867 enum InternalDBStatsType
{
868 kIntStatsWalFileBytes
,
869 kIntStatsWalFileSynced
,
870 kIntStatsBytesWritten
,
871 kIntStatsNumKeysWritten
,
872 kIntStatsWriteDoneByOther
,
873 kIntStatsWriteDoneBySelf
,
874 kIntStatsWriteWithWal
,
875 kIntStatsWriteStallMicros
,
879 InternalStats(int /*num_levels*/, SystemClock
* /*clock*/,
880 ColumnFamilyData
* /*cfd*/) {}
882 // Per level compaction stats
883 struct CompactionOutputsStats
{
884 uint64_t num_output_records
= 0;
885 uint64_t bytes_written
= 0;
886 uint64_t bytes_written_blob
= 0;
887 uint64_t num_output_files
= 0;
888 uint64_t num_output_files_blob
= 0;
890 void Add(const CompactionOutputsStats
& stats
) {
891 this->num_output_records
+= stats
.num_output_records
;
892 this->bytes_written
+= stats
.bytes_written
;
893 this->bytes_written_blob
+= stats
.bytes_written_blob
;
894 this->num_output_files
+= stats
.num_output_files
;
895 this->num_output_files_blob
+= stats
.num_output_files_blob
;
899 struct CompactionStats
{
902 uint64_t bytes_read_non_output_levels
;
903 uint64_t bytes_read_output_level
;
904 uint64_t bytes_read_blob
;
905 uint64_t bytes_written
;
906 uint64_t bytes_written_blob
;
907 uint64_t bytes_moved
;
908 int num_input_files_in_non_output_levels
;
909 int num_input_files_in_output_level
;
910 int num_output_files
;
911 int num_output_files_blob
;
912 uint64_t num_input_records
;
913 uint64_t num_dropped_records
;
914 uint64_t num_output_records
;
917 explicit CompactionStats() {}
919 explicit CompactionStats(CompactionReason
/*reason*/, int /*c*/) {}
921 explicit CompactionStats(const CompactionStats
& /*c*/) {}
923 void Add(const CompactionStats
& /*c*/) {}
925 void Add(const CompactionOutputsStats
& /*c*/) {}
927 void Subtract(const CompactionStats
& /*c*/) {}
930 struct CompactionStatsFull
{
931 // the stats for the target primary output level (per level stats)
932 CompactionStats stats
;
934 // stats for output_to_penultimate_level level (per level stats)
935 bool has_penultimate_level_output
= false;
936 CompactionStats penultimate_level_stats
;
938 explicit CompactionStatsFull(){};
940 explicit CompactionStatsFull(CompactionReason
/*reason*/, int /*c*/){};
942 uint64_t TotalBytesWritten() const { return 0; }
944 uint64_t DroppedRecords() { return 0; }
946 void SetMicros(uint64_t /*val*/){};
948 void AddCpuMicros(uint64_t /*val*/){};
951 void AddCompactionStats(int /*level*/, Env::Priority
/*thread_pri*/,
952 const CompactionStats
& /*stats*/) {}
954 void AddCompactionStats(int /*level*/, Env::Priority
/*thread_pri*/,
955 const CompactionStatsFull
& /*unmerged_stats*/) {}
957 void IncBytesMoved(int /*level*/, uint64_t /*amount*/) {}
959 void AddCFStats(InternalCFStatsType
/*type*/, uint64_t /*value*/) {}
961 void AddDBStats(InternalDBStatsType
/*type*/, uint64_t /*value*/,
962 bool /*concurrent */ = false) {}
964 HistogramImpl
* GetFileReadHist(int /*level*/) { return nullptr; }
966 HistogramImpl
* GetBlobFileReadHist() { return nullptr; }
968 uint64_t GetBackgroundErrorCount() const { return 0; }
970 uint64_t BumpAndGetBackgroundErrorCount() { return 0; }
972 bool GetStringProperty(const DBPropertyInfo
& /*property_info*/,
973 const Slice
& /*property*/, std::string
* /*value*/) {
977 bool GetMapProperty(const DBPropertyInfo
& /*property_info*/,
978 const Slice
& /*property*/,
979 std::map
<std::string
, std::string
>* /*value*/) {
983 bool GetIntProperty(const DBPropertyInfo
& /*property_info*/,
984 uint64_t* /*value*/, DBImpl
* /*db*/) const {
988 bool GetIntPropertyOutOfMutex(const DBPropertyInfo
& /*property_info*/,
989 Version
* /*version*/,
990 uint64_t* /*value*/) const {
994 #endif // !ROCKSDB_LITE
996 } // namespace ROCKSDB_NAMESPACE