ceph/src/rocksdb/tools/advisor/test/test_db_stats_fetcher.py

   1 # Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
   2 #  This source code is licensed under both the GPLv2 (found in the
   3 #  COPYING file in the root directory) and Apache 2.0 License
   4 #  (found in the LICENSE.Apache file in the root directory).
   5
   6 from advisor.db_stats_fetcher import LogStatsParser, DatabasePerfContext
   7 from advisor.db_timeseries_parser import NO_ENTITY
   8 from advisor.rule_parser import Condition, TimeSeriesCondition
   9 import os
  10 import time
  11 import unittest
  12 from unittest.mock import MagicMock
  13
  14
  15 class TestLogStatsParser(unittest.TestCase):
  16     def setUp(self):
  17         this_path = os.path.abspath(os.path.dirname(__file__))
  18         stats_file = os.path.join(
  19             this_path, 'input_files/log_stats_parser_keys_ts'
  20         )
  21         # populate the keys_ts dictionary of LogStatsParser
  22         self.stats_dict = {NO_ENTITY: {}}
  23         with open(stats_file, 'r') as fp:
  24             for line in fp:
  25                 stat_name = line.split(':')[0].strip()
  26                 self.stats_dict[NO_ENTITY][stat_name] = {}
  27                 token_list = line.split(':')[1].strip().split(',')
  28                 for token in token_list:
  29                     timestamp = int(token.split()[0])
  30                     value = float(token.split()[1])
  31                     self.stats_dict[NO_ENTITY][stat_name][timestamp] = value
  32         self.log_stats_parser = LogStatsParser('dummy_log_file', 20)
  33         self.log_stats_parser.keys_ts = self.stats_dict
  34
  35     def test_check_and_trigger_conditions_bursty(self):
  36         # mock fetch_timeseries() because 'keys_ts' has been pre-populated
  37         self.log_stats_parser.fetch_timeseries = MagicMock()
  38         # condition: bursty
  39         cond1 = Condition('cond-1')
  40         cond1 = TimeSeriesCondition.create(cond1)
  41         cond1.set_parameter('keys', 'rocksdb.db.get.micros.p50')
  42         cond1.set_parameter('behavior', 'bursty')
  43         cond1.set_parameter('window_sec', 40)
  44         cond1.set_parameter('rate_threshold', 0)
  45         self.log_stats_parser.check_and_trigger_conditions([cond1])
  46         expected_cond_trigger = {
  47             NO_ENTITY: {1530896440: 0.9767546362322214}
  48         }
  49         self.assertDictEqual(expected_cond_trigger, cond1.get_trigger())
  50         # ensure that fetch_timeseries() was called once
  51         self.log_stats_parser.fetch_timeseries.assert_called_once()
  52
  53     def test_check_and_trigger_conditions_eval_agg(self):
  54         # mock fetch_timeseries() because 'keys_ts' has been pre-populated
  55         self.log_stats_parser.fetch_timeseries = MagicMock()
  56         # condition: evaluate_expression
  57         cond1 = Condition('cond-1')
  58         cond1 = TimeSeriesCondition.create(cond1)
  59         cond1.set_parameter('keys', 'rocksdb.db.get.micros.p50')
  60         cond1.set_parameter('behavior', 'evaluate_expression')
  61         keys = [
  62             'rocksdb.manifest.file.sync.micros.p99',
  63             'rocksdb.db.get.micros.p50'
  64         ]
  65         cond1.set_parameter('keys', keys)
  66         cond1.set_parameter('aggregation_op', 'latest')
  67         # condition evaluates to FALSE
  68         cond1.set_parameter('evaluate', 'keys[0]-(keys[1]*100)>200')
  69         self.log_stats_parser.check_and_trigger_conditions([cond1])
  70         expected_cond_trigger = {NO_ENTITY: [1792.0, 15.9638]}
  71         self.assertIsNone(cond1.get_trigger())
  72         # condition evaluates to TRUE
  73         cond1.set_parameter('evaluate', 'keys[0]-(keys[1]*100)<200')
  74         self.log_stats_parser.check_and_trigger_conditions([cond1])
  75         expected_cond_trigger = {NO_ENTITY: [1792.0, 15.9638]}
  76         self.assertDictEqual(expected_cond_trigger, cond1.get_trigger())
  77         # ensure that fetch_timeseries() was called
  78         self.log_stats_parser.fetch_timeseries.assert_called()
  79
  80     def test_check_and_trigger_conditions_eval(self):
  81         # mock fetch_timeseries() because 'keys_ts' has been pre-populated
  82         self.log_stats_parser.fetch_timeseries = MagicMock()
  83         # condition: evaluate_expression
  84         cond1 = Condition('cond-1')
  85         cond1 = TimeSeriesCondition.create(cond1)
  86         cond1.set_parameter('keys', 'rocksdb.db.get.micros.p50')
  87         cond1.set_parameter('behavior', 'evaluate_expression')
  88         keys = [
  89             'rocksdb.manifest.file.sync.micros.p99',
  90             'rocksdb.db.get.micros.p50'
  91         ]
  92         cond1.set_parameter('keys', keys)
  93         cond1.set_parameter('evaluate', 'keys[0]-(keys[1]*100)>500')
  94         self.log_stats_parser.check_and_trigger_conditions([cond1])
  95         expected_trigger = {NO_ENTITY: {
  96             1530896414: [9938.0, 16.31508],
  97             1530896440: [9938.0, 16.346602],
  98             1530896466: [9938.0, 16.284669],
  99             1530896492: [9938.0, 16.16005]
 100         }}
 101         self.assertDictEqual(expected_trigger, cond1.get_trigger())
 102         self.log_stats_parser.fetch_timeseries.assert_called_once()
 103
 104
 105 class TestDatabasePerfContext(unittest.TestCase):
 106     def test_unaccumulate_metrics(self):
 107         perf_dict = {
 108             "user_key_comparison_count": 675903942,
 109             "block_cache_hit_count": 830086,
 110         }
 111         timestamp = int(time.time())
 112         perf_ts = {}
 113         for key in perf_dict:
 114             perf_ts[key] = {}
 115             start_val = perf_dict[key]
 116             for ix in range(5):
 117                 perf_ts[key][timestamp+(ix*10)] = start_val + (2 * ix * ix)
 118         db_perf_context = DatabasePerfContext(perf_ts, 10, True)
 119         timestamps = [timestamp+(ix*10) for ix in range(1, 5, 1)]
 120         values = [val for val in range(2, 15, 4)]
 121         inner_dict = {timestamps[ix]: values[ix] for ix in range(4)}
 122         expected_keys_ts = {NO_ENTITY: {
 123             'user_key_comparison_count': inner_dict,
 124             'block_cache_hit_count': inner_dict
 125         }}
 126         self.assertDictEqual(expected_keys_ts, db_perf_context.keys_ts)