]>
git.proxmox.com Git - ceph.git/blob - ceph/qa/tasks/mgr/test_module_selftest.py
6 from teuthology
.exceptions
import CommandFailedError
8 from tasks
.mgr
.mgr_test_case
import MgrTestCase
10 log
= logging
.getLogger(__name__
)
13 class TestModuleSelftest(MgrTestCase
):
15 That modules with a self-test command can be loaded and execute it
18 This is not a substitute for really testing the modules, but it
19 is quick and is designed to catch regressions that could occur
20 if data structures change in a way that breaks how the modules
26 super(TestModuleSelftest
, self
).setUp()
29 def _selftest_plugin(self
, module_name
):
30 self
._load
_module
("selftest")
31 self
._load
_module
(module_name
)
33 # Execute the module's self_test() method
34 self
.mgr_cluster
.mon_manager
.raw_cluster_cmd(
35 "mgr", "self-test", "module", module_name
)
37 def test_zabbix(self
):
38 # Set these mandatory config fields so that the zabbix module
39 # won't trigger health/log errors on load/serve.
40 self
.mgr_cluster
.set_module_conf("zabbix", "zabbix_host", "localhost")
41 self
.mgr_cluster
.set_module_conf("zabbix", "identifier", "foo")
42 self
._selftest
_plugin
("zabbix")
44 def test_prometheus(self
):
45 self
._assign
_ports
("prometheus", "server_port", min_port
=8100)
46 self
._selftest
_plugin
("prometheus")
48 def test_influx(self
):
49 self
._selftest
_plugin
("influx")
51 def test_diskprediction_local(self
):
52 self
._selftest
_plugin
("diskprediction_local")
54 # Not included in qa/packages/packages.yaml
55 #def test_diskprediction_cloud(self):
56 # self._selftest_plugin("diskprediction_cloud")
58 def test_telegraf(self
):
59 self
._selftest
_plugin
("telegraf")
61 def test_iostat(self
):
62 self
._selftest
_plugin
("iostat")
64 def test_devicehealth(self
):
65 self
._selftest
_plugin
("devicehealth")
66 # Clean up the pool that the module creates, because otherwise
67 # it's low PG count causes test failures.
68 pool_name
= "device_health_metrics"
69 self
.mgr_cluster
.mon_manager
.raw_cluster_cmd(
70 "osd", "pool", "delete", pool_name
, pool_name
,
71 "--yes-i-really-really-mean-it")
73 def test_selftest_run(self
):
74 self
._load
_module
("selftest")
75 self
.mgr_cluster
.mon_manager
.raw_cluster_cmd("mgr", "self-test", "run")
77 def test_telemetry(self
):
78 self
._selftest
_plugin
("telemetry")
81 self
._selftest
_plugin
("crash")
83 def test_orchestrator(self
):
84 self
._selftest
_plugin
("orchestrator")
87 def test_selftest_config_update(self
):
89 That configuration updates are seen by running mgr modules
91 self
._load
_module
("selftest")
94 return self
.mgr_cluster
.mon_manager
.raw_cluster_cmd(
95 "mgr", "self-test", "config", "get", "testkey").strip()
97 self
.assertEqual(get_value(), "None")
98 self
.mgr_cluster
.mon_manager
.raw_cluster_cmd(
99 "config", "set", "mgr", "mgr/selftest/testkey", "foo")
100 self
.wait_until_equal(get_value
, "foo", timeout
=10)
102 def get_localized_value():
103 return self
.mgr_cluster
.mon_manager
.raw_cluster_cmd(
104 "mgr", "self-test", "config", "get_localized", "testkey").strip()
106 self
.assertEqual(get_localized_value(), "foo")
107 self
.mgr_cluster
.mon_manager
.raw_cluster_cmd(
108 "config", "set", "mgr", "mgr/selftest/{}/testkey".format(
109 self
.mgr_cluster
.get_active_id()),
111 self
.wait_until_equal(get_localized_value
, "bar", timeout
=10)
113 def test_selftest_config_upgrade(self
):
115 That pre-mimic config-key config settings are migrated into
116 mimic-style config settings and visible from mgr modules.
118 self
._load
_module
("selftest")
121 return self
.mgr_cluster
.mon_manager
.raw_cluster_cmd(
122 "mgr", "self-test", "config", "get", "testkey").strip()
125 lines
= self
.mgr_cluster
.mon_manager
.raw_cluster_cmd(
129 for line
in lines
[1:]:
130 tokens
= line
.strip().split()
131 log
.info("tokens: {0}".format(tokens
))
132 subsys
, key
, value
= tokens
[0], tokens
[2], tokens
[3]
133 result
.append((subsys
, key
, value
))
137 # Stop ceph-mgr while we synthetically create a pre-mimic
138 # configuration scenario
139 for mgr_id
in self
.mgr_cluster
.mgr_daemons
.keys():
140 self
.mgr_cluster
.mgr_stop(mgr_id
)
141 self
.mgr_cluster
.mgr_fail(mgr_id
)
143 # Blow away any modern-style mgr module config options
144 # (the ceph-mgr implementation may only do the upgrade if
145 # it doesn't see new style options)
147 for subsys
, key
, value
in get_config():
148 if subsys
== "mgr" and key
.startswith("mgr/"):
149 log
.info("Removing config key {0} ahead of upgrade".format(
151 self
.mgr_cluster
.mon_manager
.raw_cluster_cmd(
152 "config", "rm", subsys
, key
)
153 stash
.append((subsys
, key
, value
))
155 # Inject an old-style configuration setting in config-key
156 self
.mgr_cluster
.mon_manager
.raw_cluster_cmd(
157 "config-key", "set", "mgr/selftest/testkey", "testvalue")
159 # Inject configuration settings that looks data-ish and should
160 # not be migrated to a config key
161 self
.mgr_cluster
.mon_manager
.raw_cluster_cmd(
162 "config-key", "set", "mgr/selftest/testnewline", "foo\nbar")
164 # Inject configuration setting that does not appear in the
165 # module's config schema
166 self
.mgr_cluster
.mon_manager
.raw_cluster_cmd(
167 "config-key", "set", "mgr/selftest/kvitem", "foo\nbar")
169 # Bring mgr daemons back online, the one that goes active
170 # should be doing the upgrade.
171 for mgr_id
in self
.mgr_cluster
.mgr_daemons
.keys():
172 self
.mgr_cluster
.mgr_restart(mgr_id
)
174 # Wait for a new active
175 self
.wait_until_true(
176 lambda: self
.mgr_cluster
.get_active_id() != "", timeout
=30)
178 # Check that the selftest module sees the upgraded value
179 self
.assertEqual(get_value(), "testvalue")
181 # Check that the upgraded value is visible in the configuration
182 seen_keys
= [k
for s
,k
,v
in get_config()]
183 self
.assertIn("mgr/selftest/testkey", seen_keys
)
185 # ...and that the non-config-looking one isn't
186 self
.assertNotIn("mgr/selftest/testnewline", seen_keys
)
188 # ...and that the not-in-schema one isn't
189 self
.assertNotIn("mgr/selftest/kvitem", seen_keys
)
191 # Restore previous configuration
192 for subsys
, key
, value
in stash
:
193 self
.mgr_cluster
.mon_manager
.raw_cluster_cmd(
194 "config", "set", subsys
, key
, value
)
196 def test_selftest_command_spam(self
):
197 # Use the selftest module to stress the mgr daemon
198 self
._load
_module
("selftest")
200 # Use the dashboard to test that the mgr is still able to do its job
201 self
._assign
_ports
("dashboard", "ssl_server_port")
202 self
._load
_module
("dashboard")
203 self
.mgr_cluster
.mon_manager
.raw_cluster_cmd("dashboard",
204 "create-self-signed-cert")
206 original_active
= self
.mgr_cluster
.get_active_id()
207 original_standbys
= self
.mgr_cluster
.get_standby_ids()
209 self
.mgr_cluster
.mon_manager
.raw_cluster_cmd("mgr", "self-test",
210 "background", "start",
213 dashboard_uri
= self
._get
_uri
("dashboard")
217 for i
in range(0, periods
):
219 # Check that an HTTP module remains responsive
220 r
= requests
.get(dashboard_uri
, verify
=False)
221 self
.assertEqual(r
.status_code
, 200)
223 # Check that a native non-module command remains responsive
224 self
.mgr_cluster
.mon_manager
.raw_cluster_cmd("osd", "df")
226 time
.sleep(delay
- (time
.time() - t1
))
228 self
.mgr_cluster
.mon_manager
.raw_cluster_cmd("mgr", "self-test",
229 "background", "stop")
231 # Check that all mgr daemons are still running
232 self
.assertEqual(original_active
, self
.mgr_cluster
.get_active_id())
233 self
.assertEqual(original_standbys
, self
.mgr_cluster
.get_standby_ids())
235 def test_module_commands(self
):
237 That module-handled commands have appropriate behavior on
238 disabled/failed/recently-enabled modules.
241 # Calling a command on a disabled module should return the proper
243 self
._load
_module
("selftest")
244 self
.mgr_cluster
.mon_manager
.raw_cluster_cmd(
245 "mgr", "module", "disable", "selftest")
246 with self
.assertRaises(CommandFailedError
) as exc_raised
:
247 self
.mgr_cluster
.mon_manager
.raw_cluster_cmd(
248 "mgr", "self-test", "run")
250 self
.assertEqual(exc_raised
.exception
.exitstatus
, errno
.EOPNOTSUPP
)
252 # Calling a command that really doesn't exist should give me EINVAL.
253 with self
.assertRaises(CommandFailedError
) as exc_raised
:
254 self
.mgr_cluster
.mon_manager
.raw_cluster_cmd(
257 self
.assertEqual(exc_raised
.exception
.exitstatus
, errno
.EINVAL
)
259 # Enabling a module and then immediately using ones of its commands
260 # should work (#21683)
261 self
._load
_module
("selftest")
262 self
.mgr_cluster
.mon_manager
.raw_cluster_cmd(
263 "mgr", "self-test", "config", "get", "testkey")
265 # Calling a command for a failed module should return the proper
267 self
.mgr_cluster
.mon_manager
.raw_cluster_cmd(
268 "mgr", "self-test", "background", "start", "throw_exception")
269 with self
.assertRaises(CommandFailedError
) as exc_raised
:
270 self
.mgr_cluster
.mon_manager
.raw_cluster_cmd(
271 "mgr", "self-test", "run"
273 self
.assertEqual(exc_raised
.exception
.exitstatus
, errno
.EIO
)
275 # A health alert should be raised for a module that has thrown
276 # an exception from its serve() method
277 self
.wait_for_health(
278 "Module 'selftest' has failed: Synthetic exception in serve",
281 self
.mgr_cluster
.mon_manager
.raw_cluster_cmd(
282 "mgr", "module", "disable", "selftest")
284 self
.wait_for_health_clear(timeout
=30)
286 def test_module_remote(self
):
288 Use the selftest module to exercise inter-module communication
290 self
._load
_module
("selftest")
291 # The "self-test remote" operation just happens to call into
293 self
._load
_module
("influx")
295 self
.mgr_cluster
.mon_manager
.raw_cluster_cmd(
296 "mgr", "self-test", "remote")
298 def test_selftest_cluster_log(self
):
300 Use the selftest module to test the cluster/audit log interface.
308 self
._load
_module
("selftest")
309 for priority
in priority_map
.keys():
310 message
= "foo bar {}".format(priority
)
311 log_message
= "[{}] {}".format(priority_map
[priority
], message
)
312 # Check for cluster/audit logs:
313 # 2018-09-24 09:37:10.977858 mgr.x [INF] foo bar info
314 # 2018-09-24 09:37:10.977860 mgr.x [SEC] foo bar security
315 # 2018-09-24 09:37:10.977863 mgr.x [WRN] foo bar warning
316 # 2018-09-24 09:37:10.977866 mgr.x [ERR] foo bar error
317 with self
.assert_cluster_log(log_message
):
318 self
.mgr_cluster
.mon_manager
.raw_cluster_cmd(
319 "mgr", "self-test", "cluster-log", "cluster",
321 with self
.assert_cluster_log(log_message
, watch_channel
="audit"):
322 self
.mgr_cluster
.mon_manager
.raw_cluster_cmd(
323 "mgr", "self-test", "cluster-log", "audit",
326 def test_selftest_cluster_log_unknown_channel(self
):
328 Use the selftest module to test the cluster/audit log interface.
330 with self
.assertRaises(CommandFailedError
) as exc_raised
:
331 self
.mgr_cluster
.mon_manager
.raw_cluster_cmd(
332 "mgr", "self-test", "cluster-log", "xyz",
333 "ERR", "The channel does not exist")
334 self
.assertEqual(exc_raised
.exception
.exitstatus
, errno
.EOPNOTSUPP
)