]>
Commit | Line | Data |
---|---|---|
7c673cae | 1 | import json |
3efd9988 | 2 | import logging |
7c673cae | 3 | |
adb31ebb TL |
4 | from unittest import SkipTest |
5 | ||
7c673cae FG |
6 | from teuthology import misc |
7 | from tasks.ceph_test_case import CephTestCase | |
8 | ||
3efd9988 | 9 | # TODO move definition of CephCluster away from the CephFS stuff |
7c673cae FG |
10 | from tasks.cephfs.filesystem import CephCluster |
11 | ||
12 | ||
3efd9988 FG |
13 | log = logging.getLogger(__name__) |
14 | ||
15 | ||
7c673cae FG |
16 | class MgrCluster(CephCluster): |
17 | def __init__(self, ctx): | |
18 | super(MgrCluster, self).__init__(ctx) | |
19 | self.mgr_ids = list(misc.all_roles_of_type(ctx.cluster, 'mgr')) | |
20 | ||
21 | if len(self.mgr_ids) == 0: | |
22 | raise RuntimeError( | |
23 | "This task requires at least one manager daemon") | |
24 | ||
25 | self.mgr_daemons = dict( | |
26 | [(mgr_id, self._ctx.daemons.get_daemon('mgr', mgr_id)) for mgr_id | |
27 | in self.mgr_ids]) | |
28 | ||
29 | def mgr_stop(self, mgr_id): | |
30 | self.mgr_daemons[mgr_id].stop() | |
31 | ||
32 | def mgr_fail(self, mgr_id): | |
33 | self.mon_manager.raw_cluster_cmd("mgr", "fail", mgr_id) | |
34 | ||
35 | def mgr_restart(self, mgr_id): | |
36 | self.mgr_daemons[mgr_id].restart() | |
37 | ||
38 | def get_mgr_map(self): | |
9f95a23c TL |
39 | return json.loads( |
40 | self.mon_manager.raw_cluster_cmd("mgr", "dump", "--format=json-pretty")) | |
7c673cae FG |
41 | |
42 | def get_active_id(self): | |
43 | return self.get_mgr_map()["active_name"] | |
44 | ||
45 | def get_standby_ids(self): | |
46 | return [s['name'] for s in self.get_mgr_map()["standbys"]] | |
47 | ||
b32b8144 | 48 | def set_module_conf(self, module, key, val): |
11fdf7f2 | 49 | self.mon_manager.raw_cluster_cmd("config", "set", "mgr", |
b32b8144 FG |
50 | "mgr/{0}/{1}".format( |
51 | module, key | |
52 | ), val) | |
53 | ||
9f95a23c TL |
54 | def set_module_localized_conf(self, module, mgr_id, key, val, force): |
55 | cmd = ["config", "set", "mgr", | |
56 | "/".join(["mgr", module, mgr_id, key]), | |
57 | val] | |
58 | if force: | |
59 | cmd.append("--force") | |
60 | self.mon_manager.raw_cluster_cmd(*cmd) | |
3efd9988 | 61 | |
7c673cae FG |
62 | |
63 | class MgrTestCase(CephTestCase): | |
64 | MGRS_REQUIRED = 1 | |
65 | ||
11fdf7f2 TL |
66 | @classmethod |
67 | def setup_mgrs(cls): | |
68 | # Stop all the daemons | |
69 | for daemon in cls.mgr_cluster.mgr_daemons.values(): | |
70 | daemon.stop() | |
7c673cae | 71 | |
11fdf7f2 TL |
72 | for mgr_id in cls.mgr_cluster.mgr_ids: |
73 | cls.mgr_cluster.mgr_fail(mgr_id) | |
7c673cae | 74 | |
11fdf7f2 TL |
75 | # Unload all non-default plugins |
76 | loaded = json.loads(cls.mgr_cluster.mon_manager.raw_cluster_cmd( | |
77 | "mgr", "module", "ls"))['enabled_modules'] | |
e306af50 | 78 | unload_modules = set(loaded) - {"cephadm", "restful"} |
7c673cae | 79 | |
11fdf7f2 TL |
80 | for m in unload_modules: |
81 | cls.mgr_cluster.mon_manager.raw_cluster_cmd( | |
82 | "mgr", "module", "disable", m) | |
7c673cae | 83 | |
11fdf7f2 TL |
84 | # Start all the daemons |
85 | for daemon in cls.mgr_cluster.mgr_daemons.values(): | |
7c673cae FG |
86 | daemon.restart() |
87 | ||
88 | # Wait for an active to come up | |
11fdf7f2 | 89 | cls.wait_until_true(lambda: cls.mgr_cluster.get_active_id() != "", |
7c673cae FG |
90 | timeout=20) |
91 | ||
11fdf7f2 TL |
92 | expect_standbys = set(cls.mgr_cluster.mgr_ids) \ |
93 | - {cls.mgr_cluster.get_active_id()} | |
94 | cls.wait_until_true( | |
95 | lambda: set(cls.mgr_cluster.get_standby_ids()) == expect_standbys, | |
7c673cae | 96 | timeout=20) |
3efd9988 | 97 | |
11fdf7f2 TL |
98 | @classmethod |
99 | def setUpClass(cls): | |
100 | # The test runner should have populated this | |
101 | assert cls.mgr_cluster is not None | |
102 | ||
103 | if len(cls.mgr_cluster.mgr_ids) < cls.MGRS_REQUIRED: | |
adb31ebb | 104 | raise SkipTest( |
9f95a23c TL |
105 | "Only have {0} manager daemons, {1} are required".format( |
106 | len(cls.mgr_cluster.mgr_ids), cls.MGRS_REQUIRED)) | |
11fdf7f2 TL |
107 | |
108 | cls.setup_mgrs() | |
109 | ||
9f95a23c TL |
110 | @classmethod |
111 | def _unload_module(cls, module_name): | |
112 | def is_disabled(): | |
113 | enabled_modules = json.loads(cls.mgr_cluster.mon_manager.raw_cluster_cmd( | |
114 | 'mgr', 'module', 'ls'))['enabled_modules'] | |
115 | return module_name not in enabled_modules | |
116 | ||
117 | if is_disabled(): | |
118 | return | |
119 | ||
cd265ab1 | 120 | log.debug("Unloading Mgr module %s ...", module_name) |
9f95a23c TL |
121 | cls.mgr_cluster.mon_manager.raw_cluster_cmd('mgr', 'module', 'disable', module_name) |
122 | cls.wait_until_true(is_disabled, timeout=30) | |
123 | ||
11fdf7f2 TL |
124 | @classmethod |
125 | def _load_module(cls, module_name): | |
126 | loaded = json.loads(cls.mgr_cluster.mon_manager.raw_cluster_cmd( | |
9f95a23c | 127 | "mgr", "module", "ls"))['enabled_modules'] |
3efd9988 FG |
128 | if module_name in loaded: |
129 | # The enable command is idempotent, but our wait for a restart | |
130 | # isn't, so let's return now if it's already loaded | |
131 | return | |
132 | ||
11fdf7f2 TL |
133 | initial_mgr_map = cls.mgr_cluster.get_mgr_map() |
134 | ||
135 | # check if the the module is configured as an always on module | |
136 | mgr_daemons = json.loads(cls.mgr_cluster.mon_manager.raw_cluster_cmd( | |
9f95a23c | 137 | "mgr", "metadata")) |
11fdf7f2 TL |
138 | |
139 | for daemon in mgr_daemons: | |
140 | if daemon["name"] == initial_mgr_map["active_name"]: | |
141 | ceph_version = daemon["ceph_release"] | |
142 | always_on = initial_mgr_map["always_on_modules"].get(ceph_version, []) | |
143 | if module_name in always_on: | |
144 | return | |
145 | ||
cd265ab1 | 146 | log.debug("Loading Mgr module %s ...", module_name) |
11fdf7f2 | 147 | initial_gid = initial_mgr_map['active_gid'] |
9f95a23c TL |
148 | cls.mgr_cluster.mon_manager.raw_cluster_cmd( |
149 | "mgr", "module", "enable", module_name, "--force") | |
3efd9988 FG |
150 | |
151 | # Wait for the module to load | |
152 | def has_restarted(): | |
11fdf7f2 | 153 | mgr_map = cls.mgr_cluster.get_mgr_map() |
3efd9988 FG |
154 | done = mgr_map['active_gid'] != initial_gid and mgr_map['available'] |
155 | if done: | |
cd265ab1 | 156 | log.debug("Restarted after module load (new active {0}/{1})".format( |
9f95a23c | 157 | mgr_map['active_name'], mgr_map['active_gid'])) |
3efd9988 | 158 | return done |
11fdf7f2 | 159 | cls.wait_until_true(has_restarted, timeout=30) |
3efd9988 FG |
160 | |
161 | ||
11fdf7f2 TL |
162 | @classmethod |
163 | def _get_uri(cls, service_name): | |
3efd9988 FG |
164 | # Little dict hack so that I can assign into this from |
165 | # the get_or_none function | |
166 | mgr_map = {'x': None} | |
167 | ||
168 | def _get_or_none(): | |
11fdf7f2 | 169 | mgr_map['x'] = cls.mgr_cluster.get_mgr_map() |
3efd9988 FG |
170 | result = mgr_map['x']['services'].get(service_name, None) |
171 | return result | |
172 | ||
11fdf7f2 | 173 | cls.wait_until_true(lambda: _get_or_none() is not None, 30) |
3efd9988 FG |
174 | |
175 | uri = mgr_map['x']['services'][service_name] | |
176 | ||
cd265ab1 | 177 | log.debug("Found {0} at {1} (daemon {2}/{3})".format( |
3efd9988 FG |
178 | service_name, uri, mgr_map['x']['active_name'], |
179 | mgr_map['x']['active_gid'])) | |
180 | ||
181 | return uri | |
182 | ||
11fdf7f2 TL |
183 | @classmethod |
184 | def _assign_ports(cls, module_name, config_name, min_port=7789): | |
3efd9988 FG |
185 | """ |
186 | To avoid the need to run lots of hosts in teuthology tests to | |
187 | get different URLs per mgr, we will hand out different ports | |
188 | to each mgr here. | |
189 | ||
190 | This is already taken care of for us when running in a vstart | |
191 | environment. | |
192 | """ | |
193 | # Start handing out ports well above Ceph's range. | |
194 | assign_port = min_port | |
195 | ||
11fdf7f2 TL |
196 | for mgr_id in cls.mgr_cluster.mgr_ids: |
197 | cls.mgr_cluster.mgr_stop(mgr_id) | |
198 | cls.mgr_cluster.mgr_fail(mgr_id) | |
3efd9988 | 199 | |
11fdf7f2 | 200 | for mgr_id in cls.mgr_cluster.mgr_ids: |
cd265ab1 | 201 | log.debug("Using port {0} for {1} on mgr.{2}".format( |
3efd9988 FG |
202 | assign_port, module_name, mgr_id |
203 | )) | |
11fdf7f2 TL |
204 | cls.mgr_cluster.set_module_localized_conf(module_name, mgr_id, |
205 | config_name, | |
9f95a23c TL |
206 | str(assign_port), |
207 | force=True) | |
3efd9988 FG |
208 | assign_port += 1 |
209 | ||
11fdf7f2 TL |
210 | for mgr_id in cls.mgr_cluster.mgr_ids: |
211 | cls.mgr_cluster.mgr_restart(mgr_id) | |
3efd9988 FG |
212 | |
213 | def is_available(): | |
11fdf7f2 | 214 | mgr_map = cls.mgr_cluster.get_mgr_map() |
3efd9988 FG |
215 | done = mgr_map['available'] |
216 | if done: | |
cd265ab1 | 217 | log.debug("Available after assign ports (new active {0}/{1})".format( |
11fdf7f2 | 218 | mgr_map['active_name'], mgr_map['active_gid'])) |
3efd9988 | 219 | return done |
11fdf7f2 | 220 | cls.wait_until_true(is_available, timeout=30) |