]>
Commit | Line | Data |
---|---|---|
7c673cae | 1 | import json |
3efd9988 | 2 | import logging |
7c673cae | 3 | |
adb31ebb TL |
4 | from unittest import SkipTest |
5 | ||
7c673cae FG |
6 | from teuthology import misc |
7 | from tasks.ceph_test_case import CephTestCase | |
8 | ||
3efd9988 | 9 | # TODO move definition of CephCluster away from the CephFS stuff |
f78120f9 | 10 | from tasks.cephfs.filesystem import CephClusterBase |
7c673cae FG |
11 | |
12 | ||
3efd9988 FG |
13 | log = logging.getLogger(__name__) |
14 | ||
15 | ||
f78120f9 | 16 | class MgrClusterBase(CephClusterBase): |
7c673cae | 17 | def __init__(self, ctx): |
f78120f9 | 18 | super(MgrClusterBase, self).__init__(ctx) |
7c673cae FG |
19 | self.mgr_ids = list(misc.all_roles_of_type(ctx.cluster, 'mgr')) |
20 | ||
21 | if len(self.mgr_ids) == 0: | |
22 | raise RuntimeError( | |
23 | "This task requires at least one manager daemon") | |
24 | ||
25 | self.mgr_daemons = dict( | |
26 | [(mgr_id, self._ctx.daemons.get_daemon('mgr', mgr_id)) for mgr_id | |
27 | in self.mgr_ids]) | |
28 | ||
29 | def mgr_stop(self, mgr_id): | |
30 | self.mgr_daemons[mgr_id].stop() | |
31 | ||
f38dd50b TL |
32 | def mgr_fail(self, mgr_id=None): |
33 | if mgr_id is None: | |
34 | self.mon_manager.raw_cluster_cmd("mgr", "fail") | |
35 | else: | |
36 | self.mon_manager.raw_cluster_cmd("mgr", "fail", mgr_id) | |
7c673cae FG |
37 | |
38 | def mgr_restart(self, mgr_id): | |
39 | self.mgr_daemons[mgr_id].restart() | |
40 | ||
41 | def get_mgr_map(self): | |
9f95a23c TL |
42 | return json.loads( |
43 | self.mon_manager.raw_cluster_cmd("mgr", "dump", "--format=json-pretty")) | |
7c673cae | 44 | |
aee94f69 TL |
45 | def get_registered_clients(self, name, mgr_map = None): |
46 | if mgr_map is None: | |
47 | mgr_map = self.get_mgr_map() | |
48 | for c in mgr_map['active_clients']: | |
49 | if c['name'] == name: | |
50 | return c['addrvec'] | |
51 | return None | |
52 | ||
7c673cae FG |
53 | def get_active_id(self): |
54 | return self.get_mgr_map()["active_name"] | |
55 | ||
56 | def get_standby_ids(self): | |
57 | return [s['name'] for s in self.get_mgr_map()["standbys"]] | |
58 | ||
b32b8144 | 59 | def set_module_conf(self, module, key, val): |
11fdf7f2 | 60 | self.mon_manager.raw_cluster_cmd("config", "set", "mgr", |
b32b8144 FG |
61 | "mgr/{0}/{1}".format( |
62 | module, key | |
63 | ), val) | |
64 | ||
9f95a23c TL |
65 | def set_module_localized_conf(self, module, mgr_id, key, val, force): |
66 | cmd = ["config", "set", "mgr", | |
67 | "/".join(["mgr", module, mgr_id, key]), | |
68 | val] | |
69 | if force: | |
70 | cmd.append("--force") | |
71 | self.mon_manager.raw_cluster_cmd(*cmd) | |
f78120f9 | 72 | MgrCluster = MgrClusterBase |
7c673cae FG |
73 | |
74 | class MgrTestCase(CephTestCase): | |
75 | MGRS_REQUIRED = 1 | |
76 | ||
11fdf7f2 TL |
77 | @classmethod |
78 | def setup_mgrs(cls): | |
79 | # Stop all the daemons | |
80 | for daemon in cls.mgr_cluster.mgr_daemons.values(): | |
81 | daemon.stop() | |
7c673cae | 82 | |
f38dd50b TL |
83 | cls.mgr_cluster.mon_manager.raw_cluster_cmd("mgr", "set", "down", "false") |
84 | ||
11fdf7f2 TL |
85 | for mgr_id in cls.mgr_cluster.mgr_ids: |
86 | cls.mgr_cluster.mgr_fail(mgr_id) | |
7c673cae | 87 | |
11fdf7f2 TL |
88 | # Unload all non-default plugins |
89 | loaded = json.loads(cls.mgr_cluster.mon_manager.raw_cluster_cmd( | |
20effc67 | 90 | "mgr", "module", "ls", "--format=json-pretty"))['enabled_modules'] |
e306af50 | 91 | unload_modules = set(loaded) - {"cephadm", "restful"} |
7c673cae | 92 | |
11fdf7f2 TL |
93 | for m in unload_modules: |
94 | cls.mgr_cluster.mon_manager.raw_cluster_cmd( | |
95 | "mgr", "module", "disable", m) | |
7c673cae | 96 | |
11fdf7f2 TL |
97 | # Start all the daemons |
98 | for daemon in cls.mgr_cluster.mgr_daemons.values(): | |
7c673cae FG |
99 | daemon.restart() |
100 | ||
101 | # Wait for an active to come up | |
11fdf7f2 | 102 | cls.wait_until_true(lambda: cls.mgr_cluster.get_active_id() != "", |
7c673cae FG |
103 | timeout=20) |
104 | ||
11fdf7f2 TL |
105 | expect_standbys = set(cls.mgr_cluster.mgr_ids) \ |
106 | - {cls.mgr_cluster.get_active_id()} | |
107 | cls.wait_until_true( | |
108 | lambda: set(cls.mgr_cluster.get_standby_ids()) == expect_standbys, | |
7c673cae | 109 | timeout=20) |
3efd9988 | 110 | |
11fdf7f2 TL |
111 | @classmethod |
112 | def setUpClass(cls): | |
113 | # The test runner should have populated this | |
114 | assert cls.mgr_cluster is not None | |
115 | ||
116 | if len(cls.mgr_cluster.mgr_ids) < cls.MGRS_REQUIRED: | |
adb31ebb | 117 | raise SkipTest( |
9f95a23c TL |
118 | "Only have {0} manager daemons, {1} are required".format( |
119 | len(cls.mgr_cluster.mgr_ids), cls.MGRS_REQUIRED)) | |
f38dd50b TL |
120 | |
121 | # We expect laggy OSDs in this testing environment so turn off this warning. | |
122 | # See https://tracker.ceph.com/issues/61907 | |
123 | cls.mgr_cluster.mon_manager.raw_cluster_cmd('config', 'set', 'mds', | |
124 | 'defer_client_eviction_on_laggy_osds', 'false') | |
11fdf7f2 TL |
125 | cls.setup_mgrs() |
126 | ||
9f95a23c TL |
127 | @classmethod |
128 | def _unload_module(cls, module_name): | |
129 | def is_disabled(): | |
130 | enabled_modules = json.loads(cls.mgr_cluster.mon_manager.raw_cluster_cmd( | |
20effc67 | 131 | 'mgr', 'module', 'ls', "--format=json-pretty"))['enabled_modules'] |
9f95a23c TL |
132 | return module_name not in enabled_modules |
133 | ||
134 | if is_disabled(): | |
135 | return | |
136 | ||
cd265ab1 | 137 | log.debug("Unloading Mgr module %s ...", module_name) |
9f95a23c TL |
138 | cls.mgr_cluster.mon_manager.raw_cluster_cmd('mgr', 'module', 'disable', module_name) |
139 | cls.wait_until_true(is_disabled, timeout=30) | |
140 | ||
11fdf7f2 TL |
141 | @classmethod |
142 | def _load_module(cls, module_name): | |
143 | loaded = json.loads(cls.mgr_cluster.mon_manager.raw_cluster_cmd( | |
20effc67 | 144 | "mgr", "module", "ls", "--format=json-pretty"))['enabled_modules'] |
3efd9988 FG |
145 | if module_name in loaded: |
146 | # The enable command is idempotent, but our wait for a restart | |
147 | # isn't, so let's return now if it's already loaded | |
148 | return | |
149 | ||
11fdf7f2 TL |
150 | initial_mgr_map = cls.mgr_cluster.get_mgr_map() |
151 | ||
152 | # check if the the module is configured as an always on module | |
153 | mgr_daemons = json.loads(cls.mgr_cluster.mon_manager.raw_cluster_cmd( | |
9f95a23c | 154 | "mgr", "metadata")) |
11fdf7f2 TL |
155 | |
156 | for daemon in mgr_daemons: | |
157 | if daemon["name"] == initial_mgr_map["active_name"]: | |
158 | ceph_version = daemon["ceph_release"] | |
159 | always_on = initial_mgr_map["always_on_modules"].get(ceph_version, []) | |
160 | if module_name in always_on: | |
161 | return | |
162 | ||
cd265ab1 | 163 | log.debug("Loading Mgr module %s ...", module_name) |
11fdf7f2 | 164 | initial_gid = initial_mgr_map['active_gid'] |
9f95a23c TL |
165 | cls.mgr_cluster.mon_manager.raw_cluster_cmd( |
166 | "mgr", "module", "enable", module_name, "--force") | |
3efd9988 FG |
167 | |
168 | # Wait for the module to load | |
169 | def has_restarted(): | |
11fdf7f2 | 170 | mgr_map = cls.mgr_cluster.get_mgr_map() |
3efd9988 FG |
171 | done = mgr_map['active_gid'] != initial_gid and mgr_map['available'] |
172 | if done: | |
cd265ab1 | 173 | log.debug("Restarted after module load (new active {0}/{1})".format( |
9f95a23c | 174 | mgr_map['active_name'], mgr_map['active_gid'])) |
3efd9988 | 175 | return done |
11fdf7f2 | 176 | cls.wait_until_true(has_restarted, timeout=30) |
3efd9988 FG |
177 | |
178 | ||
11fdf7f2 TL |
179 | @classmethod |
180 | def _get_uri(cls, service_name): | |
3efd9988 FG |
181 | # Little dict hack so that I can assign into this from |
182 | # the get_or_none function | |
183 | mgr_map = {'x': None} | |
184 | ||
185 | def _get_or_none(): | |
11fdf7f2 | 186 | mgr_map['x'] = cls.mgr_cluster.get_mgr_map() |
3efd9988 FG |
187 | result = mgr_map['x']['services'].get(service_name, None) |
188 | return result | |
189 | ||
11fdf7f2 | 190 | cls.wait_until_true(lambda: _get_or_none() is not None, 30) |
3efd9988 FG |
191 | |
192 | uri = mgr_map['x']['services'][service_name] | |
193 | ||
cd265ab1 | 194 | log.debug("Found {0} at {1} (daemon {2}/{3})".format( |
3efd9988 FG |
195 | service_name, uri, mgr_map['x']['active_name'], |
196 | mgr_map['x']['active_gid'])) | |
197 | ||
198 | return uri | |
199 | ||
11fdf7f2 TL |
200 | @classmethod |
201 | def _assign_ports(cls, module_name, config_name, min_port=7789): | |
3efd9988 FG |
202 | """ |
203 | To avoid the need to run lots of hosts in teuthology tests to | |
204 | get different URLs per mgr, we will hand out different ports | |
205 | to each mgr here. | |
206 | ||
207 | This is already taken care of for us when running in a vstart | |
208 | environment. | |
209 | """ | |
210 | # Start handing out ports well above Ceph's range. | |
211 | assign_port = min_port | |
212 | ||
11fdf7f2 TL |
213 | for mgr_id in cls.mgr_cluster.mgr_ids: |
214 | cls.mgr_cluster.mgr_stop(mgr_id) | |
215 | cls.mgr_cluster.mgr_fail(mgr_id) | |
3efd9988 | 216 | |
11fdf7f2 | 217 | for mgr_id in cls.mgr_cluster.mgr_ids: |
cd265ab1 | 218 | log.debug("Using port {0} for {1} on mgr.{2}".format( |
3efd9988 FG |
219 | assign_port, module_name, mgr_id |
220 | )) | |
11fdf7f2 TL |
221 | cls.mgr_cluster.set_module_localized_conf(module_name, mgr_id, |
222 | config_name, | |
9f95a23c TL |
223 | str(assign_port), |
224 | force=True) | |
3efd9988 FG |
225 | assign_port += 1 |
226 | ||
11fdf7f2 TL |
227 | for mgr_id in cls.mgr_cluster.mgr_ids: |
228 | cls.mgr_cluster.mgr_restart(mgr_id) | |
3efd9988 FG |
229 | |
230 | def is_available(): | |
11fdf7f2 | 231 | mgr_map = cls.mgr_cluster.get_mgr_map() |
3efd9988 FG |
232 | done = mgr_map['available'] |
233 | if done: | |
cd265ab1 | 234 | log.debug("Available after assign ports (new active {0}/{1})".format( |
11fdf7f2 | 235 | mgr_map['active_name'], mgr_map['active_gid'])) |
3efd9988 | 236 | return done |
11fdf7f2 | 237 | cls.wait_until_true(is_available, timeout=30) |