ceph/qa/tasks/mgr/test_module_selftest.py

   1
   2 import time
   3 import requests
   4 import errno
   5 import logging
   6
   7 from teuthology.exceptions import CommandFailedError
   8
   9 from .mgr_test_case import MgrTestCase
  10
  11
  12 log = logging.getLogger(__name__)
  13
  14
  15 class TestModuleSelftest(MgrTestCase):
  16     """
  17     That modules with a self-test command can be loaded and execute it
  18     without errors.
  19
  20     This is not a substitute for really testing the modules, but it
  21     is quick and is designed to catch regressions that could occur
  22     if data structures change in a way that breaks how the modules
  23     touch them.
  24     """
  25     MGRS_REQUIRED = 1
  26
  27     def setUp(self):
  28         super(TestModuleSelftest, self).setUp()
  29         self.setup_mgrs()
  30
  31     def _selftest_plugin(self, module_name):
  32         self._load_module("selftest")
  33         self._load_module(module_name)
  34
  35         # Execute the module's self_test() method
  36         self.mgr_cluster.mon_manager.raw_cluster_cmd(
  37                 "mgr", "self-test", "module", module_name)
  38
  39     def test_zabbix(self):
  40         # Set these mandatory config fields so that the zabbix module
  41         # won't trigger health/log errors on load/serve.
  42         self.mgr_cluster.set_module_conf("zabbix", "zabbix_host", "localhost")
  43         self.mgr_cluster.set_module_conf("zabbix", "identifier", "foo")
  44         self._selftest_plugin("zabbix")
  45
  46     def test_prometheus(self):
  47         self._assign_ports("prometheus", "server_port", min_port=8100)
  48         self._selftest_plugin("prometheus")
  49
  50     def test_influx(self):
  51         self._selftest_plugin("influx")
  52
  53     def test_diskprediction_local(self):
  54         self._load_module("selftest")
  55         python_version = self.mgr_cluster.mon_manager.raw_cluster_cmd(
  56             "mgr", "self-test", "python-version")
  57         if tuple(int(v) for v in python_version.split('.')) == (3, 8):
  58             # https://tracker.ceph.com/issues/45147
  59             self.skipTest(f'python {python_version} not compatible with '
  60                           'diskprediction_local')
  61         self._selftest_plugin("diskprediction_local")
  62
  63     def test_telegraf(self):
  64         self._selftest_plugin("telegraf")
  65
  66     def test_iostat(self):
  67         self._selftest_plugin("iostat")
  68
  69     def test_devicehealth(self):
  70         self._selftest_plugin("devicehealth")
  71
  72     def test_selftest_run(self):
  73         self._load_module("selftest")
  74         self.mgr_cluster.mon_manager.raw_cluster_cmd("mgr", "self-test", "run")
  75
  76     def test_telemetry(self):
  77         self._selftest_plugin("telemetry")
  78
  79     def test_crash(self):
  80         self._selftest_plugin("crash")
  81
  82     def test_orchestrator(self):
  83         self._selftest_plugin("orchestrator")
  84
  85
  86     def test_selftest_config_update(self):
  87         """
  88         That configuration updates are seen by running mgr modules
  89         """
  90         self._load_module("selftest")
  91
  92         def get_value():
  93             return self.mgr_cluster.mon_manager.raw_cluster_cmd(
  94                 "mgr", "self-test", "config", "get", "testkey").strip()
  95
  96         self.assertEqual(get_value(), "None")
  97         self.mgr_cluster.mon_manager.raw_cluster_cmd(
  98             "config", "set", "mgr", "mgr/selftest/testkey", "foo")
  99         self.wait_until_equal(get_value, "foo", timeout=10)
 100
 101         def get_localized_value():
 102             return self.mgr_cluster.mon_manager.raw_cluster_cmd(
 103                 "mgr", "self-test", "config", "get_localized", "testkey").strip()
 104
 105         self.assertEqual(get_localized_value(), "foo")
 106         self.mgr_cluster.mon_manager.raw_cluster_cmd(
 107             "config", "set", "mgr", "mgr/selftest/{}/testkey".format(
 108                 self.mgr_cluster.get_active_id()),
 109             "bar")
 110         self.wait_until_equal(get_localized_value, "bar", timeout=10)
 111
 112
 113     def test_selftest_command_spam(self):
 114         # Use the selftest module to stress the mgr daemon
 115         self._load_module("selftest")
 116
 117         # Use the dashboard to test that the mgr is still able to do its job
 118         self._assign_ports("dashboard", "ssl_server_port")
 119         self._load_module("dashboard")
 120         self.mgr_cluster.mon_manager.raw_cluster_cmd("dashboard",
 121                                                      "create-self-signed-cert")
 122
 123         original_active = self.mgr_cluster.get_active_id()
 124         original_standbys = self.mgr_cluster.get_standby_ids()
 125
 126         self.mgr_cluster.mon_manager.raw_cluster_cmd("mgr", "self-test",
 127                                                      "background", "start",
 128                                                      "command_spam")
 129
 130         dashboard_uri = self._get_uri("dashboard")
 131
 132         delay = 10
 133         periods = 10
 134         for i in range(0, periods):
 135             t1 = time.time()
 136             # Check that an HTTP module remains responsive
 137             r = requests.get(dashboard_uri, verify=False)
 138             self.assertEqual(r.status_code, 200)
 139
 140             # Check that a native non-module command remains responsive
 141             self.mgr_cluster.mon_manager.raw_cluster_cmd("osd", "df")
 142
 143             time.sleep(delay - (time.time() - t1))
 144
 145         self.mgr_cluster.mon_manager.raw_cluster_cmd("mgr", "self-test",
 146                                                      "background", "stop")
 147
 148         # Check that all mgr daemons are still running
 149         self.assertEqual(original_active, self.mgr_cluster.get_active_id())
 150         self.assertEqual(original_standbys, self.mgr_cluster.get_standby_ids())
 151
 152     def test_module_commands(self):
 153         """
 154         That module-handled commands have appropriate  behavior on
 155         disabled/failed/recently-enabled modules.
 156         """
 157
 158         # Calling a command on a disabled module should return the proper
 159         # error code.
 160         self._load_module("selftest")
 161         self.mgr_cluster.mon_manager.raw_cluster_cmd(
 162             "mgr", "module", "disable", "selftest")
 163         with self.assertRaises(CommandFailedError) as exc_raised:
 164             self.mgr_cluster.mon_manager.raw_cluster_cmd(
 165                 "mgr", "self-test", "run")
 166
 167         self.assertEqual(exc_raised.exception.exitstatus, errno.EOPNOTSUPP)
 168
 169         # Calling a command that really doesn't exist should give me EINVAL.
 170         with self.assertRaises(CommandFailedError) as exc_raised:
 171             self.mgr_cluster.mon_manager.raw_cluster_cmd(
 172                 "osd", "albatross")
 173
 174         self.assertEqual(exc_raised.exception.exitstatus, errno.EINVAL)
 175
 176         # Enabling a module and then immediately using ones of its commands
 177         # should work (#21683)
 178         self._load_module("selftest")
 179         self.mgr_cluster.mon_manager.raw_cluster_cmd(
 180             "mgr", "self-test", "config", "get", "testkey")
 181
 182         # Calling a command for a failed module should return the proper
 183         # error code.
 184         self.mgr_cluster.mon_manager.raw_cluster_cmd(
 185             "mgr", "self-test", "background", "start", "throw_exception")
 186         with self.assertRaises(CommandFailedError) as exc_raised:
 187             self.mgr_cluster.mon_manager.raw_cluster_cmd(
 188                 "mgr", "self-test", "run"
 189             )
 190         self.assertEqual(exc_raised.exception.exitstatus, errno.EIO)
 191
 192         # A health alert should be raised for a module that has thrown
 193         # an exception from its serve() method
 194         self.wait_for_health(
 195             "Module 'selftest' has failed: Synthetic exception in serve",
 196             timeout=30)
 197         # prune the crash reports, so that the health report is back to
 198         # clean
 199         self.mgr_cluster.mon_manager.raw_cluster_cmd(
 200             "crash", "prune", "0")
 201         self.mgr_cluster.mon_manager.raw_cluster_cmd(
 202             "mgr", "module", "disable", "selftest")
 203
 204         self.wait_for_health_clear(timeout=30)
 205
 206     def test_module_remote(self):
 207         """
 208         Use the selftest module to exercise inter-module communication
 209         """
 210         self._load_module("selftest")
 211         # The "self-test remote" operation just happens to call into
 212         # influx.
 213         self._load_module("influx")
 214
 215         self.mgr_cluster.mon_manager.raw_cluster_cmd(
 216             "mgr", "self-test", "remote")
 217
 218     def test_selftest_cluster_log(self):
 219         """
 220         Use the selftest module to test the cluster/audit log interface.
 221         """
 222         priority_map = {
 223             "info": "INF",
 224             "security": "SEC",
 225             "warning": "WRN",
 226             "error": "ERR"
 227         }
 228         self._load_module("selftest")
 229         for priority in priority_map.keys():
 230             message = "foo bar {}".format(priority)
 231             log_message = "[{}] {}".format(priority_map[priority], message)
 232             # Check for cluster/audit logs:
 233             # 2018-09-24 09:37:10.977858 mgr.x [INF] foo bar info
 234             # 2018-09-24 09:37:10.977860 mgr.x [SEC] foo bar security
 235             # 2018-09-24 09:37:10.977863 mgr.x [WRN] foo bar warning
 236             # 2018-09-24 09:37:10.977866 mgr.x [ERR] foo bar error
 237             with self.assert_cluster_log(log_message):
 238                 self.mgr_cluster.mon_manager.raw_cluster_cmd(
 239                     "mgr", "self-test", "cluster-log", "cluster",
 240                     priority, message)
 241             with self.assert_cluster_log(log_message, watch_channel="audit"):
 242                 self.mgr_cluster.mon_manager.raw_cluster_cmd(
 243                     "mgr", "self-test", "cluster-log", "audit",
 244                     priority, message)
 245
 246     def test_selftest_cluster_log_unknown_channel(self):
 247         """
 248         Use the selftest module to test the cluster/audit log interface.
 249         """
 250         with self.assertRaises(CommandFailedError) as exc_raised:
 251             self.mgr_cluster.mon_manager.raw_cluster_cmd(
 252                 "mgr", "self-test", "cluster-log", "xyz",
 253                 "ERR", "The channel does not exist")
 254         self.assertEqual(exc_raised.exception.exitstatus, errno.EOPNOTSUPP)