]>
Commit | Line | Data |
---|---|---|
31f18b77 FG |
1 | """ |
2 | rgw multisite configuration routines | |
3 | """ | |
4 | import argparse | |
5 | import contextlib | |
6 | import logging | |
7 | import random | |
8 | import string | |
9 | from copy import deepcopy | |
10 | from util.rgw import rgwadmin, wait_for_radosgw | |
11 | from util.rados import create_ec_pool, create_replicated_pool | |
12 | from rgw_multi import multisite | |
13 | from rgw_multi.zone_rados import RadosZone as RadosZone | |
14 | ||
15 | from teuthology.orchestra import run | |
16 | from teuthology import misc | |
17 | from teuthology.exceptions import ConfigError | |
18 | from teuthology.task import Task | |
19 | ||
20 | log = logging.getLogger(__name__) | |
21 | ||
22 | class RGWMultisite(Task): | |
23 | """ | |
24 | Performs rgw multisite configuration to match the given realm definition. | |
25 | ||
26 | - rgw-multisite: | |
27 | realm: | |
28 | name: test-realm | |
29 | is_default: true | |
30 | ||
31 | List one or more zonegroup definitions. These are provided as json | |
32 | input to `radosgw-admin zonegroup set`, with the exception of these keys: | |
33 | ||
34 | * 'is_master' is passed on the command line as --master | |
35 | * 'is_default' is passed on the command line as --default | |
36 | * 'endpoints' given as client names are replaced with actual endpoints | |
37 | ||
38 | zonegroups: | |
39 | - name: test-zonegroup | |
40 | api_name: test-api | |
41 | is_master: true | |
42 | is_default: true | |
43 | endpoints: [c1.client.0] | |
44 | ||
45 | List each of the zones to be created in this zonegroup. | |
46 | ||
47 | zones: | |
48 | - name: test-zone1 | |
49 | is_master: true | |
50 | is_default: true | |
51 | endpoints: [c1.client.0] | |
52 | - name: test-zone2 | |
53 | is_default: true | |
54 | endpoints: [c2.client.0] | |
55 | ||
56 | A complete example: | |
57 | ||
58 | tasks: | |
59 | - install: | |
60 | - ceph: {cluster: c1} | |
61 | - ceph: {cluster: c2} | |
62 | - rgw: | |
63 | c1.client.0: | |
64 | c2.client.0: | |
65 | - rgw-multisite: | |
66 | realm: | |
67 | name: test-realm | |
68 | is_default: true | |
69 | zonegroups: | |
70 | - name: test-zonegroup | |
71 | is_master: true | |
72 | is_default: true | |
73 | zones: | |
74 | - name: test-zone1 | |
75 | is_master: true | |
76 | is_default: true | |
77 | endpoints: [c1.client.0] | |
78 | - name: test-zone2 | |
79 | is_default: true | |
80 | endpoints: [c2.client.0] | |
81 | ||
82 | """ | |
83 | def __init__(self, ctx, config): | |
84 | super(RGWMultisite, self).__init__(ctx, config) | |
85 | ||
86 | def setup(self): | |
87 | super(RGWMultisite, self).setup() | |
88 | ||
89 | overrides = self.ctx.config.get('overrides', {}) | |
90 | misc.deep_merge(self.config, overrides.get('rgw-multisite', {})) | |
91 | ||
92 | if not self.ctx.rgw: | |
93 | raise ConfigError('rgw-multisite must run after the rgw task') | |
94 | role_endpoints = self.ctx.rgw.role_endpoints | |
95 | ||
96 | # construct Clusters and Gateways for each client in the rgw task | |
97 | clusters, gateways = extract_clusters_and_gateways(self.ctx, | |
98 | role_endpoints) | |
99 | ||
100 | # get the master zone and zonegroup configuration | |
101 | mz, mzg = extract_master_zone_zonegroup(self.config['zonegroups']) | |
102 | cluster1 = cluster_for_zone(clusters, mz) | |
103 | ||
104 | # create the realm and period on the master zone's cluster | |
105 | log.info('creating realm..') | |
106 | realm = create_realm(cluster1, self.config['realm']) | |
107 | period = realm.current_period | |
108 | ||
109 | creds = gen_credentials() | |
110 | ||
111 | # create the master zonegroup and its master zone | |
112 | log.info('creating master zonegroup..') | |
113 | master_zonegroup = create_zonegroup(cluster1, gateways, period, | |
114 | deepcopy(mzg)) | |
115 | period.master_zonegroup = master_zonegroup | |
116 | ||
117 | log.info('creating master zone..') | |
118 | master_zone = create_zone(self.ctx, cluster1, gateways, creds, | |
119 | master_zonegroup, deepcopy(mz)) | |
120 | master_zonegroup.master_zone = master_zone | |
121 | ||
122 | period.update(master_zone, commit=True) | |
123 | restart_zone_gateways(master_zone) # restart with --rgw-zone | |
124 | ||
125 | # create the admin user on the master zone | |
126 | log.info('creating admin user..') | |
127 | user_args = ['--display-name', 'Realm Admin', '--system'] | |
128 | user_args += creds.credential_args() | |
129 | admin_user = multisite.User('realm-admin') | |
130 | admin_user.create(master_zone, user_args) | |
131 | ||
132 | # process 'zonegroups' | |
133 | for zg_config in self.config['zonegroups']: | |
134 | zones_config = zg_config.pop('zones') | |
135 | ||
136 | zonegroup = None | |
137 | for zone_config in zones_config: | |
138 | # get the cluster for this zone | |
139 | cluster = cluster_for_zone(clusters, zone_config) | |
140 | ||
141 | if cluster != cluster1: # already created on master cluster | |
142 | log.info('pulling realm configuration to %s', cluster.name) | |
143 | realm.pull(cluster, master_zone.gateways[0], creds) | |
144 | ||
145 | # use the first zone's cluster to create the zonegroup | |
146 | if not zonegroup: | |
147 | if zg_config['name'] == master_zonegroup.name: | |
148 | zonegroup = master_zonegroup | |
149 | else: | |
150 | log.info('creating zonegroup..') | |
151 | zonegroup = create_zonegroup(cluster, gateways, | |
152 | period, zg_config) | |
153 | ||
154 | if zone_config['name'] == master_zone.name: | |
155 | # master zone was already created | |
156 | zone = master_zone | |
157 | else: | |
158 | # create the zone and commit the period | |
159 | log.info('creating zone..') | |
160 | zone = create_zone(self.ctx, cluster, gateways, creds, | |
161 | zonegroup, zone_config) | |
162 | period.update(zone, commit=True) | |
163 | ||
164 | restart_zone_gateways(zone) # restart with --rgw-zone | |
165 | ||
166 | # attach configuration to the ctx for other tasks | |
167 | self.ctx.rgw_multisite = argparse.Namespace() | |
168 | self.ctx.rgw_multisite.clusters = clusters | |
169 | self.ctx.rgw_multisite.gateways = gateways | |
170 | self.ctx.rgw_multisite.realm = realm | |
171 | self.ctx.rgw_multisite.admin_user = admin_user | |
172 | ||
173 | log.info('rgw multisite configuration completed') | |
174 | ||
175 | def end(self): | |
176 | del self.ctx.rgw_multisite | |
177 | ||
178 | class Cluster(multisite.Cluster): | |
179 | """ Issues 'radosgw-admin' commands with the rgwadmin() helper """ | |
180 | def __init__(self, ctx, name, client): | |
181 | super(Cluster, self).__init__() | |
182 | self.ctx = ctx | |
183 | self.name = name | |
184 | self.client = client | |
185 | ||
186 | def admin(self, args = None, **kwargs): | |
187 | """ radosgw-admin command """ | |
188 | args = args or [] | |
189 | args += ['--cluster', self.name] | |
190 | args += ['--debug-rgw', '0'] | |
191 | if kwargs.pop('read_only', False): | |
192 | args += ['--rgw-cache-enabled', 'false'] | |
193 | kwargs['decode'] = False | |
194 | check_retcode = kwargs.pop('check_retcode', True) | |
195 | r, s = rgwadmin(self.ctx, self.client, args, **kwargs) | |
196 | if check_retcode: | |
197 | assert r == 0 | |
198 | return s, r | |
199 | ||
200 | class Gateway(multisite.Gateway): | |
201 | """ Controls a radosgw instance using its daemon """ | |
202 | def __init__(self, role, remote, daemon, *args, **kwargs): | |
203 | super(Gateway, self).__init__(*args, **kwargs) | |
204 | self.role = role | |
205 | self.remote = remote | |
206 | self.daemon = daemon | |
207 | ||
208 | def set_zone(self, zone): | |
209 | """ set the zone and add its args to the daemon's command line """ | |
210 | assert self.zone is None, 'zone can only be set once' | |
211 | self.zone = zone | |
212 | # daemon.restart_with_args() would be perfect for this, except that | |
213 | # radosgw args likely include a pipe and redirect. zone arguments at | |
214 | # the end won't actually apply to radosgw | |
215 | args = self.daemon.command_kwargs.get('args', []) | |
216 | try: | |
217 | # insert zone args before the first | | |
218 | pipe = args.index(run.Raw('|')) | |
219 | args = args[0:pipe] + zone.zone_args() + args[pipe:] | |
220 | except ValueError, e: | |
221 | args += zone.zone_args() | |
222 | self.daemon.command_kwargs['args'] = args | |
223 | ||
224 | def start(self, args = None): | |
225 | """ (re)start the daemon """ | |
226 | self.daemon.restart() | |
227 | # wait until startup completes | |
228 | wait_for_radosgw(self.endpoint()) | |
229 | ||
230 | def stop(self): | |
231 | """ stop the daemon """ | |
232 | self.daemon.stop() | |
233 | ||
234 | def extract_clusters_and_gateways(ctx, role_endpoints): | |
235 | """ create cluster and gateway instances for all of the radosgw roles """ | |
236 | clusters = {} | |
237 | gateways = {} | |
238 | for role, (host, port) in role_endpoints.iteritems(): | |
239 | cluster_name, daemon_type, client_id = misc.split_role(role) | |
240 | # find or create the cluster by name | |
241 | cluster = clusters.get(cluster_name) | |
242 | if not cluster: | |
243 | clusters[cluster_name] = cluster = Cluster(ctx, cluster_name, role) | |
244 | # create a gateway for this daemon | |
245 | client_with_id = daemon_type + '.' + client_id # match format from rgw.py | |
246 | daemon = ctx.daemons.get_daemon('rgw', client_with_id, cluster_name) | |
247 | if not daemon: | |
248 | raise ConfigError('no daemon for role=%s cluster=%s type=rgw id=%s' % \ | |
249 | (role, cluster_name, client_id)) | |
250 | (remote,) = ctx.cluster.only(role).remotes.keys() | |
251 | gateways[role] = Gateway(role, remote, daemon, host, port, cluster) | |
252 | return clusters, gateways | |
253 | ||
254 | def create_realm(cluster, config): | |
255 | """ create a realm from configuration and initialize its first period """ | |
256 | realm = multisite.Realm(config['name']) | |
257 | args = [] | |
258 | if config.get('is_default', False): | |
259 | args += ['--default'] | |
260 | realm.create(cluster, args) | |
261 | realm.current_period = multisite.Period(realm) | |
262 | return realm | |
263 | ||
264 | def extract_user_credentials(config): | |
265 | """ extract keys from configuration """ | |
266 | return multisite.Credentials(config['access_key'], config['secret_key']) | |
267 | ||
268 | def extract_master_zone(zonegroup_config): | |
269 | """ find and return the master zone definition """ | |
270 | master = None | |
271 | for zone in zonegroup_config['zones']: | |
272 | if not zone.get('is_master', False): | |
273 | continue | |
274 | if master: | |
275 | raise ConfigError('zones %s and %s cannot both set \'is_master\'' % \ | |
276 | (master['name'], zone['name'])) | |
277 | master = zone | |
278 | # continue the loop so we can detect duplicates | |
279 | if not master: | |
280 | raise ConfigError('one zone must set \'is_master\' in zonegroup %s' % \ | |
281 | zonegroup_config['name']) | |
282 | return master | |
283 | ||
284 | def extract_master_zone_zonegroup(zonegroups_config): | |
285 | """ find and return the master zone and zonegroup definitions """ | |
286 | master_zone, master_zonegroup = (None, None) | |
287 | for zonegroup in zonegroups_config: | |
288 | # verify that all zonegroups have a master zone set, even if they | |
289 | # aren't in the master zonegroup | |
290 | zone = extract_master_zone(zonegroup) | |
291 | if not zonegroup.get('is_master', False): | |
292 | continue | |
293 | if master_zonegroup: | |
294 | raise ConfigError('zonegroups %s and %s cannot both set \'is_master\'' % \ | |
295 | (master_zonegroup['name'], zonegroup['name'])) | |
296 | master_zonegroup = zonegroup | |
297 | master_zone = zone | |
298 | # continue the loop so we can detect duplicates | |
299 | if not master_zonegroup: | |
300 | raise ConfigError('one zonegroup must set \'is_master\'') | |
301 | return master_zone, master_zonegroup | |
302 | ||
303 | def extract_zone_cluster_name(zone_config): | |
304 | """ return the cluster (must be common to all zone endpoints) """ | |
305 | cluster_name = None | |
306 | endpoints = zone_config.get('endpoints') | |
307 | if not endpoints: | |
308 | raise ConfigError('zone %s missing \'endpoints\' list' % \ | |
309 | zone_config['name']) | |
310 | for role in endpoints: | |
311 | name, _, _ = misc.split_role(role) | |
312 | if not cluster_name: | |
313 | cluster_name = name | |
314 | elif cluster_name != name: | |
315 | raise ConfigError('all zone %s endpoints must be in the same cluster' % \ | |
316 | zone_config['name']) | |
317 | return cluster_name | |
318 | ||
319 | def cluster_for_zone(clusters, zone_config): | |
320 | """ return the cluster entry for the given zone """ | |
321 | name = extract_zone_cluster_name(zone_config) | |
322 | try: | |
323 | return clusters[name] | |
324 | except KeyError: | |
325 | raise ConfigError('no cluster %s found' % name) | |
326 | ||
327 | def gen_access_key(): | |
328 | return ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(16)) | |
329 | ||
330 | def gen_secret(): | |
331 | return ''.join(random.choice(string.ascii_uppercase + string.ascii_lowercase + string.digits) for _ in range(32)) | |
332 | ||
333 | def gen_credentials(): | |
334 | return multisite.Credentials(gen_access_key(), gen_secret()) | |
335 | ||
336 | def extract_gateway_endpoints(gateways, endpoints_config): | |
337 | """ return a list of gateway endpoints associated with the given roles """ | |
338 | endpoints = [] | |
339 | for role in endpoints_config: | |
340 | try: | |
341 | # replace role names with their gateway's endpoint | |
342 | endpoints.append(gateways[role].endpoint()) | |
343 | except KeyError: | |
344 | raise ConfigError('no radosgw endpoint found for role %s' % role) | |
345 | return endpoints | |
346 | ||
347 | def is_default_arg(config): | |
348 | return ['--default'] if config.pop('is_default', False) else [] | |
349 | ||
350 | def is_master_arg(config): | |
351 | return ['--master'] if config.pop('is_master', False) else [] | |
352 | ||
353 | def create_zonegroup(cluster, gateways, period, config): | |
354 | """ pass the zonegroup configuration to `zonegroup set` """ | |
355 | config.pop('zones', None) # remove 'zones' from input to `zonegroup set` | |
356 | endpoints = config.get('endpoints') | |
357 | if endpoints: | |
358 | # replace client names with their gateway endpoints | |
359 | config['endpoints'] = extract_gateway_endpoints(gateways, endpoints) | |
360 | zonegroup = multisite.ZoneGroup(config['name'], period) | |
361 | # `zonegroup set` needs --default on command line, and 'is_master' in json | |
362 | args = is_default_arg(config) | |
363 | zonegroup.set(cluster, config, args) | |
364 | period.zonegroups.append(zonegroup) | |
365 | return zonegroup | |
366 | ||
367 | def create_zone(ctx, cluster, gateways, creds, zonegroup, config): | |
368 | """ create a zone with the given configuration """ | |
369 | zone = multisite.Zone(config['name'], zonegroup, cluster) | |
370 | zone = RadosZone(config['name'], zonegroup, cluster) | |
371 | ||
372 | # collect Gateways for the zone's endpoints | |
373 | endpoints = config.get('endpoints') | |
374 | if not endpoints: | |
375 | raise ConfigError('no \'endpoints\' for zone %s' % config['name']) | |
376 | zone.gateways = [gateways[role] for role in endpoints] | |
377 | for gateway in zone.gateways: | |
378 | gateway.set_zone(zone) | |
379 | ||
380 | # format the gateway endpoints | |
381 | endpoints = [g.endpoint() for g in zone.gateways] | |
382 | ||
383 | args = is_default_arg(config) | |
384 | args += is_master_arg(config) | |
385 | args += creds.credential_args() | |
386 | if len(endpoints): | |
387 | args += ['--endpoints', ','.join(endpoints)] | |
388 | zone.create(cluster, args) | |
389 | zonegroup.zones.append(zone) | |
390 | ||
391 | create_zone_pools(ctx, zone) | |
392 | if ctx.rgw.compression_type: | |
393 | configure_zone_compression(zone, ctx.rgw.compression_type) | |
394 | ||
395 | zonegroup.zones_by_type.setdefault(zone.tier_type(), []).append(zone) | |
396 | ||
397 | if zone.is_read_only(): | |
398 | zonegroup.ro_zones.append(zone) | |
399 | else: | |
400 | zonegroup.rw_zones.append(zone) | |
401 | ||
402 | return zone | |
403 | ||
404 | def create_zone_pools(ctx, zone): | |
405 | """ Create the data_pool for each placement type """ | |
406 | gateway = zone.gateways[0] | |
407 | cluster = zone.cluster | |
408 | for pool_config in zone.data.get('placement_pools', []): | |
409 | pool_name = pool_config['val']['data_pool'] | |
410 | if ctx.rgw.ec_data_pool: | |
411 | create_ec_pool(gateway.remote, pool_name, zone.name, 64, | |
b5b8bbf5 | 412 | ctx.rgw.erasure_code_profile, cluster.name, 'rgw') |
31f18b77 | 413 | else: |
b5b8bbf5 | 414 | create_replicated_pool(gateway.remote, pool_name, 64, cluster.name, 'rgw') |
31f18b77 FG |
415 | |
416 | def configure_zone_compression(zone, compression): | |
417 | """ Set compression type in the zone's default-placement """ | |
418 | zone.json_command(zone.cluster, 'placement', ['modify', | |
419 | '--placement-id', 'default-placement', | |
420 | '--compression', compression | |
421 | ]) | |
422 | ||
423 | def restart_zone_gateways(zone): | |
424 | zone.stop() | |
425 | zone.start() | |
426 | ||
427 | task = RGWMultisite |