]>
Commit | Line | Data |
---|---|---|
31f18b77 FG |
1 | """ |
2 | rgw multisite configuration routines | |
3 | """ | |
4 | import argparse | |
31f18b77 FG |
5 | import logging |
6 | import random | |
7 | import string | |
8 | from copy import deepcopy | |
e306af50 TL |
9 | from tasks.util.rgw import rgwadmin, wait_for_radosgw |
10 | from tasks.util.rados import create_ec_pool, create_replicated_pool | |
11 | from tasks.rgw_multi import multisite | |
12 | from tasks.rgw_multi.zone_rados import RadosZone as RadosZone | |
13 | from tasks.rgw_multi.zone_ps import PSZone as PSZone | |
31f18b77 FG |
14 | |
15 | from teuthology.orchestra import run | |
16 | from teuthology import misc | |
17 | from teuthology.exceptions import ConfigError | |
18 | from teuthology.task import Task | |
19 | ||
20 | log = logging.getLogger(__name__) | |
21 | ||
22 | class RGWMultisite(Task): | |
23 | """ | |
24 | Performs rgw multisite configuration to match the given realm definition. | |
25 | ||
26 | - rgw-multisite: | |
27 | realm: | |
28 | name: test-realm | |
29 | is_default: true | |
30 | ||
31 | List one or more zonegroup definitions. These are provided as json | |
32 | input to `radosgw-admin zonegroup set`, with the exception of these keys: | |
33 | ||
34 | * 'is_master' is passed on the command line as --master | |
35 | * 'is_default' is passed on the command line as --default | |
eafe8130 | 36 | * 'is_pubsub' is used to create a zone with tier-type=pubsub |
31f18b77 FG |
37 | * 'endpoints' given as client names are replaced with actual endpoints |
38 | ||
39 | zonegroups: | |
40 | - name: test-zonegroup | |
41 | api_name: test-api | |
42 | is_master: true | |
43 | is_default: true | |
44 | endpoints: [c1.client.0] | |
45 | ||
46 | List each of the zones to be created in this zonegroup. | |
47 | ||
48 | zones: | |
49 | - name: test-zone1 | |
50 | is_master: true | |
51 | is_default: true | |
52 | endpoints: [c1.client.0] | |
53 | - name: test-zone2 | |
54 | is_default: true | |
55 | endpoints: [c2.client.0] | |
56 | ||
57 | A complete example: | |
58 | ||
59 | tasks: | |
60 | - install: | |
61 | - ceph: {cluster: c1} | |
62 | - ceph: {cluster: c2} | |
63 | - rgw: | |
64 | c1.client.0: | |
65 | c2.client.0: | |
66 | - rgw-multisite: | |
67 | realm: | |
68 | name: test-realm | |
69 | is_default: true | |
70 | zonegroups: | |
71 | - name: test-zonegroup | |
72 | is_master: true | |
73 | is_default: true | |
74 | zones: | |
75 | - name: test-zone1 | |
76 | is_master: true | |
77 | is_default: true | |
78 | endpoints: [c1.client.0] | |
79 | - name: test-zone2 | |
80 | is_default: true | |
81 | endpoints: [c2.client.0] | |
eafe8130 TL |
82 | - name: test-zone3 |
83 | is_pubsub: true | |
84 | endpoints: [c1.client.1] | |
31f18b77 FG |
85 | |
86 | """ | |
87 | def __init__(self, ctx, config): | |
88 | super(RGWMultisite, self).__init__(ctx, config) | |
89 | ||
90 | def setup(self): | |
91 | super(RGWMultisite, self).setup() | |
92 | ||
93 | overrides = self.ctx.config.get('overrides', {}) | |
94 | misc.deep_merge(self.config, overrides.get('rgw-multisite', {})) | |
95 | ||
96 | if not self.ctx.rgw: | |
97 | raise ConfigError('rgw-multisite must run after the rgw task') | |
98 | role_endpoints = self.ctx.rgw.role_endpoints | |
99 | ||
100 | # construct Clusters and Gateways for each client in the rgw task | |
101 | clusters, gateways = extract_clusters_and_gateways(self.ctx, | |
102 | role_endpoints) | |
103 | ||
104 | # get the master zone and zonegroup configuration | |
105 | mz, mzg = extract_master_zone_zonegroup(self.config['zonegroups']) | |
106 | cluster1 = cluster_for_zone(clusters, mz) | |
107 | ||
108 | # create the realm and period on the master zone's cluster | |
109 | log.info('creating realm..') | |
110 | realm = create_realm(cluster1, self.config['realm']) | |
111 | period = realm.current_period | |
112 | ||
113 | creds = gen_credentials() | |
114 | ||
115 | # create the master zonegroup and its master zone | |
116 | log.info('creating master zonegroup..') | |
117 | master_zonegroup = create_zonegroup(cluster1, gateways, period, | |
118 | deepcopy(mzg)) | |
119 | period.master_zonegroup = master_zonegroup | |
120 | ||
121 | log.info('creating master zone..') | |
122 | master_zone = create_zone(self.ctx, cluster1, gateways, creds, | |
123 | master_zonegroup, deepcopy(mz)) | |
124 | master_zonegroup.master_zone = master_zone | |
125 | ||
126 | period.update(master_zone, commit=True) | |
127 | restart_zone_gateways(master_zone) # restart with --rgw-zone | |
128 | ||
129 | # create the admin user on the master zone | |
130 | log.info('creating admin user..') | |
131 | user_args = ['--display-name', 'Realm Admin', '--system'] | |
132 | user_args += creds.credential_args() | |
133 | admin_user = multisite.User('realm-admin') | |
134 | admin_user.create(master_zone, user_args) | |
135 | ||
136 | # process 'zonegroups' | |
137 | for zg_config in self.config['zonegroups']: | |
138 | zones_config = zg_config.pop('zones') | |
139 | ||
140 | zonegroup = None | |
141 | for zone_config in zones_config: | |
142 | # get the cluster for this zone | |
143 | cluster = cluster_for_zone(clusters, zone_config) | |
144 | ||
145 | if cluster != cluster1: # already created on master cluster | |
146 | log.info('pulling realm configuration to %s', cluster.name) | |
147 | realm.pull(cluster, master_zone.gateways[0], creds) | |
148 | ||
149 | # use the first zone's cluster to create the zonegroup | |
150 | if not zonegroup: | |
151 | if zg_config['name'] == master_zonegroup.name: | |
152 | zonegroup = master_zonegroup | |
153 | else: | |
154 | log.info('creating zonegroup..') | |
155 | zonegroup = create_zonegroup(cluster, gateways, | |
156 | period, zg_config) | |
157 | ||
158 | if zone_config['name'] == master_zone.name: | |
159 | # master zone was already created | |
160 | zone = master_zone | |
161 | else: | |
162 | # create the zone and commit the period | |
163 | log.info('creating zone..') | |
164 | zone = create_zone(self.ctx, cluster, gateways, creds, | |
165 | zonegroup, zone_config) | |
166 | period.update(zone, commit=True) | |
167 | ||
168 | restart_zone_gateways(zone) # restart with --rgw-zone | |
169 | ||
170 | # attach configuration to the ctx for other tasks | |
171 | self.ctx.rgw_multisite = argparse.Namespace() | |
172 | self.ctx.rgw_multisite.clusters = clusters | |
173 | self.ctx.rgw_multisite.gateways = gateways | |
174 | self.ctx.rgw_multisite.realm = realm | |
175 | self.ctx.rgw_multisite.admin_user = admin_user | |
176 | ||
177 | log.info('rgw multisite configuration completed') | |
178 | ||
179 | def end(self): | |
180 | del self.ctx.rgw_multisite | |
181 | ||
182 | class Cluster(multisite.Cluster): | |
183 | """ Issues 'radosgw-admin' commands with the rgwadmin() helper """ | |
184 | def __init__(self, ctx, name, client): | |
185 | super(Cluster, self).__init__() | |
186 | self.ctx = ctx | |
187 | self.name = name | |
188 | self.client = client | |
189 | ||
190 | def admin(self, args = None, **kwargs): | |
191 | """ radosgw-admin command """ | |
192 | args = args or [] | |
193 | args += ['--cluster', self.name] | |
b32b8144 FG |
194 | args += ['--debug-rgw', str(kwargs.pop('debug_rgw', 0))] |
195 | args += ['--debug-ms', str(kwargs.pop('debug_ms', 0))] | |
31f18b77 FG |
196 | if kwargs.pop('read_only', False): |
197 | args += ['--rgw-cache-enabled', 'false'] | |
198 | kwargs['decode'] = False | |
199 | check_retcode = kwargs.pop('check_retcode', True) | |
200 | r, s = rgwadmin(self.ctx, self.client, args, **kwargs) | |
201 | if check_retcode: | |
202 | assert r == 0 | |
203 | return s, r | |
204 | ||
205 | class Gateway(multisite.Gateway): | |
206 | """ Controls a radosgw instance using its daemon """ | |
207 | def __init__(self, role, remote, daemon, *args, **kwargs): | |
208 | super(Gateway, self).__init__(*args, **kwargs) | |
209 | self.role = role | |
210 | self.remote = remote | |
211 | self.daemon = daemon | |
212 | ||
213 | def set_zone(self, zone): | |
214 | """ set the zone and add its args to the daemon's command line """ | |
215 | assert self.zone is None, 'zone can only be set once' | |
216 | self.zone = zone | |
217 | # daemon.restart_with_args() would be perfect for this, except that | |
218 | # radosgw args likely include a pipe and redirect. zone arguments at | |
219 | # the end won't actually apply to radosgw | |
220 | args = self.daemon.command_kwargs.get('args', []) | |
221 | try: | |
222 | # insert zone args before the first | | |
223 | pipe = args.index(run.Raw('|')) | |
224 | args = args[0:pipe] + zone.zone_args() + args[pipe:] | |
9f95a23c | 225 | except ValueError: |
31f18b77 FG |
226 | args += zone.zone_args() |
227 | self.daemon.command_kwargs['args'] = args | |
228 | ||
229 | def start(self, args = None): | |
230 | """ (re)start the daemon """ | |
231 | self.daemon.restart() | |
232 | # wait until startup completes | |
81eedcae | 233 | wait_for_radosgw(self.endpoint(), self.remote) |
31f18b77 FG |
234 | |
235 | def stop(self): | |
236 | """ stop the daemon """ | |
237 | self.daemon.stop() | |
238 | ||
239 | def extract_clusters_and_gateways(ctx, role_endpoints): | |
240 | """ create cluster and gateway instances for all of the radosgw roles """ | |
241 | clusters = {} | |
242 | gateways = {} | |
9f95a23c | 243 | for role, endpoint in role_endpoints.items(): |
31f18b77 FG |
244 | cluster_name, daemon_type, client_id = misc.split_role(role) |
245 | # find or create the cluster by name | |
246 | cluster = clusters.get(cluster_name) | |
247 | if not cluster: | |
248 | clusters[cluster_name] = cluster = Cluster(ctx, cluster_name, role) | |
249 | # create a gateway for this daemon | |
250 | client_with_id = daemon_type + '.' + client_id # match format from rgw.py | |
251 | daemon = ctx.daemons.get_daemon('rgw', client_with_id, cluster_name) | |
252 | if not daemon: | |
253 | raise ConfigError('no daemon for role=%s cluster=%s type=rgw id=%s' % \ | |
254 | (role, cluster_name, client_id)) | |
255 | (remote,) = ctx.cluster.only(role).remotes.keys() | |
11fdf7f2 TL |
256 | gateways[role] = Gateway(role, remote, daemon, endpoint.hostname, |
257 | endpoint.port, cluster) | |
31f18b77 FG |
258 | return clusters, gateways |
259 | ||
260 | def create_realm(cluster, config): | |
261 | """ create a realm from configuration and initialize its first period """ | |
262 | realm = multisite.Realm(config['name']) | |
263 | args = [] | |
264 | if config.get('is_default', False): | |
265 | args += ['--default'] | |
266 | realm.create(cluster, args) | |
267 | realm.current_period = multisite.Period(realm) | |
268 | return realm | |
269 | ||
270 | def extract_user_credentials(config): | |
271 | """ extract keys from configuration """ | |
272 | return multisite.Credentials(config['access_key'], config['secret_key']) | |
273 | ||
274 | def extract_master_zone(zonegroup_config): | |
275 | """ find and return the master zone definition """ | |
276 | master = None | |
277 | for zone in zonegroup_config['zones']: | |
278 | if not zone.get('is_master', False): | |
279 | continue | |
280 | if master: | |
281 | raise ConfigError('zones %s and %s cannot both set \'is_master\'' % \ | |
282 | (master['name'], zone['name'])) | |
283 | master = zone | |
284 | # continue the loop so we can detect duplicates | |
285 | if not master: | |
286 | raise ConfigError('one zone must set \'is_master\' in zonegroup %s' % \ | |
287 | zonegroup_config['name']) | |
288 | return master | |
289 | ||
290 | def extract_master_zone_zonegroup(zonegroups_config): | |
291 | """ find and return the master zone and zonegroup definitions """ | |
292 | master_zone, master_zonegroup = (None, None) | |
293 | for zonegroup in zonegroups_config: | |
294 | # verify that all zonegroups have a master zone set, even if they | |
295 | # aren't in the master zonegroup | |
296 | zone = extract_master_zone(zonegroup) | |
297 | if not zonegroup.get('is_master', False): | |
298 | continue | |
299 | if master_zonegroup: | |
300 | raise ConfigError('zonegroups %s and %s cannot both set \'is_master\'' % \ | |
301 | (master_zonegroup['name'], zonegroup['name'])) | |
302 | master_zonegroup = zonegroup | |
303 | master_zone = zone | |
304 | # continue the loop so we can detect duplicates | |
305 | if not master_zonegroup: | |
306 | raise ConfigError('one zonegroup must set \'is_master\'') | |
307 | return master_zone, master_zonegroup | |
308 | ||
309 | def extract_zone_cluster_name(zone_config): | |
310 | """ return the cluster (must be common to all zone endpoints) """ | |
311 | cluster_name = None | |
312 | endpoints = zone_config.get('endpoints') | |
313 | if not endpoints: | |
314 | raise ConfigError('zone %s missing \'endpoints\' list' % \ | |
315 | zone_config['name']) | |
316 | for role in endpoints: | |
317 | name, _, _ = misc.split_role(role) | |
318 | if not cluster_name: | |
319 | cluster_name = name | |
320 | elif cluster_name != name: | |
321 | raise ConfigError('all zone %s endpoints must be in the same cluster' % \ | |
322 | zone_config['name']) | |
323 | return cluster_name | |
324 | ||
325 | def cluster_for_zone(clusters, zone_config): | |
326 | """ return the cluster entry for the given zone """ | |
327 | name = extract_zone_cluster_name(zone_config) | |
328 | try: | |
329 | return clusters[name] | |
330 | except KeyError: | |
331 | raise ConfigError('no cluster %s found' % name) | |
332 | ||
333 | def gen_access_key(): | |
334 | return ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(16)) | |
335 | ||
336 | def gen_secret(): | |
337 | return ''.join(random.choice(string.ascii_uppercase + string.ascii_lowercase + string.digits) for _ in range(32)) | |
338 | ||
339 | def gen_credentials(): | |
340 | return multisite.Credentials(gen_access_key(), gen_secret()) | |
341 | ||
342 | def extract_gateway_endpoints(gateways, endpoints_config): | |
343 | """ return a list of gateway endpoints associated with the given roles """ | |
344 | endpoints = [] | |
345 | for role in endpoints_config: | |
346 | try: | |
347 | # replace role names with their gateway's endpoint | |
348 | endpoints.append(gateways[role].endpoint()) | |
349 | except KeyError: | |
350 | raise ConfigError('no radosgw endpoint found for role %s' % role) | |
351 | return endpoints | |
352 | ||
353 | def is_default_arg(config): | |
354 | return ['--default'] if config.pop('is_default', False) else [] | |
355 | ||
356 | def is_master_arg(config): | |
357 | return ['--master'] if config.pop('is_master', False) else [] | |
358 | ||
359 | def create_zonegroup(cluster, gateways, period, config): | |
360 | """ pass the zonegroup configuration to `zonegroup set` """ | |
361 | config.pop('zones', None) # remove 'zones' from input to `zonegroup set` | |
362 | endpoints = config.get('endpoints') | |
363 | if endpoints: | |
364 | # replace client names with their gateway endpoints | |
365 | config['endpoints'] = extract_gateway_endpoints(gateways, endpoints) | |
366 | zonegroup = multisite.ZoneGroup(config['name'], period) | |
367 | # `zonegroup set` needs --default on command line, and 'is_master' in json | |
368 | args = is_default_arg(config) | |
369 | zonegroup.set(cluster, config, args) | |
370 | period.zonegroups.append(zonegroup) | |
371 | return zonegroup | |
372 | ||
373 | def create_zone(ctx, cluster, gateways, creds, zonegroup, config): | |
374 | """ create a zone with the given configuration """ | |
375 | zone = multisite.Zone(config['name'], zonegroup, cluster) | |
eafe8130 TL |
376 | if config.pop('is_pubsub', False): |
377 | zone = PSZone(config['name'], zonegroup, cluster) | |
378 | else: | |
379 | zone = RadosZone(config['name'], zonegroup, cluster) | |
31f18b77 FG |
380 | |
381 | # collect Gateways for the zone's endpoints | |
382 | endpoints = config.get('endpoints') | |
383 | if not endpoints: | |
384 | raise ConfigError('no \'endpoints\' for zone %s' % config['name']) | |
385 | zone.gateways = [gateways[role] for role in endpoints] | |
386 | for gateway in zone.gateways: | |
387 | gateway.set_zone(zone) | |
388 | ||
389 | # format the gateway endpoints | |
390 | endpoints = [g.endpoint() for g in zone.gateways] | |
391 | ||
392 | args = is_default_arg(config) | |
393 | args += is_master_arg(config) | |
394 | args += creds.credential_args() | |
395 | if len(endpoints): | |
396 | args += ['--endpoints', ','.join(endpoints)] | |
397 | zone.create(cluster, args) | |
398 | zonegroup.zones.append(zone) | |
399 | ||
400 | create_zone_pools(ctx, zone) | |
401 | if ctx.rgw.compression_type: | |
402 | configure_zone_compression(zone, ctx.rgw.compression_type) | |
403 | ||
404 | zonegroup.zones_by_type.setdefault(zone.tier_type(), []).append(zone) | |
405 | ||
406 | if zone.is_read_only(): | |
407 | zonegroup.ro_zones.append(zone) | |
408 | else: | |
409 | zonegroup.rw_zones.append(zone) | |
410 | ||
411 | return zone | |
412 | ||
413 | def create_zone_pools(ctx, zone): | |
414 | """ Create the data_pool for each placement type """ | |
415 | gateway = zone.gateways[0] | |
416 | cluster = zone.cluster | |
417 | for pool_config in zone.data.get('placement_pools', []): | |
11fdf7f2 | 418 | pool_name = pool_config['val']['storage_classes']['STANDARD']['data_pool'] |
31f18b77 FG |
419 | if ctx.rgw.ec_data_pool: |
420 | create_ec_pool(gateway.remote, pool_name, zone.name, 64, | |
b5b8bbf5 | 421 | ctx.rgw.erasure_code_profile, cluster.name, 'rgw') |
31f18b77 | 422 | else: |
b5b8bbf5 | 423 | create_replicated_pool(gateway.remote, pool_name, 64, cluster.name, 'rgw') |
31f18b77 FG |
424 | |
425 | def configure_zone_compression(zone, compression): | |
426 | """ Set compression type in the zone's default-placement """ | |
427 | zone.json_command(zone.cluster, 'placement', ['modify', | |
428 | '--placement-id', 'default-placement', | |
429 | '--compression', compression | |
430 | ]) | |
431 | ||
432 | def restart_zone_gateways(zone): | |
433 | zone.stop() | |
434 | zone.start() | |
435 | ||
436 | task = RGWMultisite |