]>
git.proxmox.com Git - ceph.git/blob - ceph/qa/tasks/rgw_multisite.py
2 rgw multisite configuration routines
8 from copy
import deepcopy
9 from tasks
.util
.rgw
import rgwadmin
, wait_for_radosgw
10 from tasks
.util
.rados
import create_ec_pool
, create_replicated_pool
11 from tasks
.rgw_multi
import multisite
12 from tasks
.rgw_multi
.zone_rados
import RadosZone
as RadosZone
13 from tasks
.rgw_multi
.zone_ps
import PSZone
as PSZone
15 from teuthology
.orchestra
import run
16 from teuthology
import misc
17 from teuthology
.exceptions
import ConfigError
18 from teuthology
.task
import Task
20 log
= logging
.getLogger(__name__
)
22 class RGWMultisite(Task
):
24 Performs rgw multisite configuration to match the given realm definition.
31 List one or more zonegroup definitions. These are provided as json
32 input to `radosgw-admin zonegroup set`, with the exception of these keys:
34 * 'is_master' is passed on the command line as --master
35 * 'is_default' is passed on the command line as --default
36 * 'is_pubsub' is used to create a zone with tier-type=pubsub
37 * 'endpoints' given as client names are replaced with actual endpoints
40 - name: test-zonegroup
44 endpoints: [c1.client.0]
46 List each of the zones to be created in this zonegroup.
52 endpoints: [c1.client.0]
55 endpoints: [c2.client.0]
71 - name: test-zonegroup
78 endpoints: [c1.client.0]
81 endpoints: [c2.client.0]
84 endpoints: [c1.client.1]
87 def __init__(self
, ctx
, config
):
88 super(RGWMultisite
, self
).__init
__(ctx
, config
)
91 super(RGWMultisite
, self
).setup()
93 overrides
= self
.ctx
.config
.get('overrides', {})
94 misc
.deep_merge(self
.config
, overrides
.get('rgw-multisite', {}))
97 raise ConfigError('rgw-multisite must run after the rgw task')
98 role_endpoints
= self
.ctx
.rgw
.role_endpoints
100 # construct Clusters and Gateways for each client in the rgw task
101 clusters
, gateways
= extract_clusters_and_gateways(self
.ctx
,
104 # get the master zone and zonegroup configuration
105 mz
, mzg
= extract_master_zone_zonegroup(self
.config
['zonegroups'])
106 cluster1
= cluster_for_zone(clusters
, mz
)
108 # create the realm and period on the master zone's cluster
109 log
.info('creating realm..')
110 realm
= create_realm(cluster1
, self
.config
['realm'])
111 period
= realm
.current_period
113 creds
= gen_credentials()
115 # create the master zonegroup and its master zone
116 log
.info('creating master zonegroup..')
117 master_zonegroup
= create_zonegroup(cluster1
, gateways
, period
,
119 period
.master_zonegroup
= master_zonegroup
121 log
.info('creating master zone..')
122 master_zone
= create_zone(self
.ctx
, cluster1
, gateways
, creds
,
123 master_zonegroup
, deepcopy(mz
))
124 master_zonegroup
.master_zone
= master_zone
126 period
.update(master_zone
, commit
=True)
127 restart_zone_gateways(master_zone
) # restart with --rgw-zone
129 # create the admin user on the master zone
130 log
.info('creating admin user..')
131 user_args
= ['--display-name', 'Realm Admin', '--system']
132 user_args
+= creds
.credential_args()
133 admin_user
= multisite
.User('realm-admin')
134 admin_user
.create(master_zone
, user_args
)
136 # process 'zonegroups'
137 for zg_config
in self
.config
['zonegroups']:
138 zones_config
= zg_config
.pop('zones')
141 for zone_config
in zones_config
:
142 # get the cluster for this zone
143 cluster
= cluster_for_zone(clusters
, zone_config
)
145 if cluster
!= cluster1
: # already created on master cluster
146 log
.info('pulling realm configuration to %s', cluster
.name
)
147 realm
.pull(cluster
, master_zone
.gateways
[0], creds
)
149 # use the first zone's cluster to create the zonegroup
151 if zg_config
['name'] == master_zonegroup
.name
:
152 zonegroup
= master_zonegroup
154 log
.info('creating zonegroup..')
155 zonegroup
= create_zonegroup(cluster
, gateways
,
158 if zone_config
['name'] == master_zone
.name
:
159 # master zone was already created
162 # create the zone and commit the period
163 log
.info('creating zone..')
164 zone
= create_zone(self
.ctx
, cluster
, gateways
, creds
,
165 zonegroup
, zone_config
)
166 period
.update(zone
, commit
=True)
168 restart_zone_gateways(zone
) # restart with --rgw-zone
170 # attach configuration to the ctx for other tasks
171 self
.ctx
.rgw_multisite
= argparse
.Namespace()
172 self
.ctx
.rgw_multisite
.clusters
= clusters
173 self
.ctx
.rgw_multisite
.gateways
= gateways
174 self
.ctx
.rgw_multisite
.realm
= realm
175 self
.ctx
.rgw_multisite
.admin_user
= admin_user
177 log
.info('rgw multisite configuration completed')
180 del self
.ctx
.rgw_multisite
182 class Cluster(multisite
.Cluster
):
183 """ Issues 'radosgw-admin' commands with the rgwadmin() helper """
184 def __init__(self
, ctx
, name
, client
):
185 super(Cluster
, self
).__init
__()
190 def admin(self
, args
= None, **kwargs
):
191 """ radosgw-admin command """
193 args
+= ['--cluster', self
.name
]
194 args
+= ['--debug-rgw', str(kwargs
.pop('debug_rgw', 0))]
195 args
+= ['--debug-ms', str(kwargs
.pop('debug_ms', 0))]
196 if kwargs
.pop('read_only', False):
197 args
+= ['--rgw-cache-enabled', 'false']
198 kwargs
['decode'] = False
199 check_retcode
= kwargs
.pop('check_retcode', True)
200 r
, s
= rgwadmin(self
.ctx
, self
.client
, args
, **kwargs
)
205 class Gateway(multisite
.Gateway
):
206 """ Controls a radosgw instance using its daemon """
207 def __init__(self
, role
, remote
, daemon
, *args
, **kwargs
):
208 super(Gateway
, self
).__init
__(*args
, **kwargs
)
213 def set_zone(self
, zone
):
214 """ set the zone and add its args to the daemon's command line """
215 assert self
.zone
is None, 'zone can only be set once'
217 # daemon.restart_with_args() would be perfect for this, except that
218 # radosgw args likely include a pipe and redirect. zone arguments at
219 # the end won't actually apply to radosgw
220 args
= self
.daemon
.command_kwargs
.get('args', [])
222 # insert zone args before the first |
223 pipe
= args
.index(run
.Raw('|'))
224 args
= args
[0:pipe
] + zone
.zone_args() + args
[pipe
:]
226 args
+= zone
.zone_args()
227 self
.daemon
.command_kwargs
['args'] = args
229 def start(self
, args
= None):
230 """ (re)start the daemon """
231 self
.daemon
.restart()
232 # wait until startup completes
233 wait_for_radosgw(self
.endpoint(), self
.remote
)
236 """ stop the daemon """
239 def extract_clusters_and_gateways(ctx
, role_endpoints
):
240 """ create cluster and gateway instances for all of the radosgw roles """
243 for role
, endpoint
in role_endpoints
.items():
244 cluster_name
, daemon_type
, client_id
= misc
.split_role(role
)
245 # find or create the cluster by name
246 cluster
= clusters
.get(cluster_name
)
248 clusters
[cluster_name
] = cluster
= Cluster(ctx
, cluster_name
, role
)
249 # create a gateway for this daemon
250 client_with_id
= daemon_type
+ '.' + client_id
# match format from rgw.py
251 daemon
= ctx
.daemons
.get_daemon('rgw', client_with_id
, cluster_name
)
253 raise ConfigError('no daemon for role=%s cluster=%s type=rgw id=%s' % \
254 (role
, cluster_name
, client_id
))
255 (remote
,) = ctx
.cluster
.only(role
).remotes
.keys()
256 gateways
[role
] = Gateway(role
, remote
, daemon
, endpoint
.hostname
,
257 endpoint
.port
, cluster
)
258 return clusters
, gateways
260 def create_realm(cluster
, config
):
261 """ create a realm from configuration and initialize its first period """
262 realm
= multisite
.Realm(config
['name'])
264 if config
.get('is_default', False):
265 args
+= ['--default']
266 realm
.create(cluster
, args
)
267 realm
.current_period
= multisite
.Period(realm
)
270 def extract_user_credentials(config
):
271 """ extract keys from configuration """
272 return multisite
.Credentials(config
['access_key'], config
['secret_key'])
274 def extract_master_zone(zonegroup_config
):
275 """ find and return the master zone definition """
277 for zone
in zonegroup_config
['zones']:
278 if not zone
.get('is_master', False):
281 raise ConfigError('zones %s and %s cannot both set \'is_master\'' % \
282 (master
['name'], zone
['name']))
284 # continue the loop so we can detect duplicates
286 raise ConfigError('one zone must set \'is_master\' in zonegroup %s' % \
287 zonegroup_config
['name'])
290 def extract_master_zone_zonegroup(zonegroups_config
):
291 """ find and return the master zone and zonegroup definitions """
292 master_zone
, master_zonegroup
= (None, None)
293 for zonegroup
in zonegroups_config
:
294 # verify that all zonegroups have a master zone set, even if they
295 # aren't in the master zonegroup
296 zone
= extract_master_zone(zonegroup
)
297 if not zonegroup
.get('is_master', False):
300 raise ConfigError('zonegroups %s and %s cannot both set \'is_master\'' % \
301 (master_zonegroup
['name'], zonegroup
['name']))
302 master_zonegroup
= zonegroup
304 # continue the loop so we can detect duplicates
305 if not master_zonegroup
:
306 raise ConfigError('one zonegroup must set \'is_master\'')
307 return master_zone
, master_zonegroup
309 def extract_zone_cluster_name(zone_config
):
310 """ return the cluster (must be common to all zone endpoints) """
312 endpoints
= zone_config
.get('endpoints')
314 raise ConfigError('zone %s missing \'endpoints\' list' % \
316 for role
in endpoints
:
317 name
, _
, _
= misc
.split_role(role
)
320 elif cluster_name
!= name
:
321 raise ConfigError('all zone %s endpoints must be in the same cluster' % \
325 def cluster_for_zone(clusters
, zone_config
):
326 """ return the cluster entry for the given zone """
327 name
= extract_zone_cluster_name(zone_config
)
329 return clusters
[name
]
331 raise ConfigError('no cluster %s found' % name
)
333 def gen_access_key():
334 return ''.join(random
.choice(string
.ascii_uppercase
+ string
.digits
) for _
in range(16))
337 return ''.join(random
.choice(string
.ascii_uppercase
+ string
.ascii_lowercase
+ string
.digits
) for _
in range(32))
339 def gen_credentials():
340 return multisite
.Credentials(gen_access_key(), gen_secret())
342 def extract_gateway_endpoints(gateways
, endpoints_config
):
343 """ return a list of gateway endpoints associated with the given roles """
345 for role
in endpoints_config
:
347 # replace role names with their gateway's endpoint
348 endpoints
.append(gateways
[role
].endpoint())
350 raise ConfigError('no radosgw endpoint found for role %s' % role
)
353 def is_default_arg(config
):
354 return ['--default'] if config
.pop('is_default', False) else []
356 def is_master_arg(config
):
357 return ['--master'] if config
.pop('is_master', False) else []
359 def create_zonegroup(cluster
, gateways
, period
, config
):
360 """ pass the zonegroup configuration to `zonegroup set` """
361 config
.pop('zones', None) # remove 'zones' from input to `zonegroup set`
362 endpoints
= config
.get('endpoints')
364 # replace client names with their gateway endpoints
365 config
['endpoints'] = extract_gateway_endpoints(gateways
, endpoints
)
366 zonegroup
= multisite
.ZoneGroup(config
['name'], period
)
367 # `zonegroup set` needs --default on command line, and 'is_master' in json
368 args
= is_default_arg(config
)
369 zonegroup
.set(cluster
, config
, args
)
370 period
.zonegroups
.append(zonegroup
)
373 def create_zone(ctx
, cluster
, gateways
, creds
, zonegroup
, config
):
374 """ create a zone with the given configuration """
375 zone
= multisite
.Zone(config
['name'], zonegroup
, cluster
)
376 if config
.pop('is_pubsub', False):
377 zone
= PSZone(config
['name'], zonegroup
, cluster
)
379 zone
= RadosZone(config
['name'], zonegroup
, cluster
)
381 # collect Gateways for the zone's endpoints
382 endpoints
= config
.get('endpoints')
384 raise ConfigError('no \'endpoints\' for zone %s' % config
['name'])
385 zone
.gateways
= [gateways
[role
] for role
in endpoints
]
386 for gateway
in zone
.gateways
:
387 gateway
.set_zone(zone
)
389 # format the gateway endpoints
390 endpoints
= [g
.endpoint() for g
in zone
.gateways
]
392 args
= is_default_arg(config
)
393 args
+= is_master_arg(config
)
394 args
+= creds
.credential_args()
396 args
+= ['--endpoints', ','.join(endpoints
)]
397 zone
.create(cluster
, args
)
398 zonegroup
.zones
.append(zone
)
400 create_zone_pools(ctx
, zone
)
401 if ctx
.rgw
.compression_type
:
402 configure_zone_compression(zone
, ctx
.rgw
.compression_type
)
404 zonegroup
.zones_by_type
.setdefault(zone
.tier_type(), []).append(zone
)
406 if zone
.is_read_only():
407 zonegroup
.ro_zones
.append(zone
)
409 zonegroup
.rw_zones
.append(zone
)
413 def create_zone_pools(ctx
, zone
):
414 """ Create the data_pool for each placement type """
415 gateway
= zone
.gateways
[0]
416 cluster
= zone
.cluster
417 for pool_config
in zone
.data
.get('placement_pools', []):
418 pool_name
= pool_config
['val']['storage_classes']['STANDARD']['data_pool']
419 if ctx
.rgw
.ec_data_pool
:
420 create_ec_pool(gateway
.remote
, pool_name
, zone
.name
, 64,
421 ctx
.rgw
.erasure_code_profile
, cluster
.name
, 'rgw')
423 create_replicated_pool(gateway
.remote
, pool_name
, 64, cluster
.name
, 'rgw')
425 def configure_zone_compression(zone
, compression
):
426 """ Set compression type in the zone's default-placement """
427 zone
.json_command(zone
.cluster
, 'placement', ['modify',
428 '--placement-id', 'default-placement',
429 '--compression', compression
432 def restart_zone_gateways(zone
):