]>
git.proxmox.com Git - ceph.git/blob - ceph/qa/tasks/rgw_multisite.py
2 rgw multisite configuration routines
9 from copy
import deepcopy
10 from util
.rgw
import rgwadmin
, wait_for_radosgw
11 from util
.rados
import create_ec_pool
, create_replicated_pool
12 from rgw_multi
import multisite
13 from rgw_multi
.zone_rados
import RadosZone
as RadosZone
15 from teuthology
.orchestra
import run
16 from teuthology
import misc
17 from teuthology
.exceptions
import ConfigError
18 from teuthology
.task
import Task
20 log
= logging
.getLogger(__name__
)
22 class RGWMultisite(Task
):
24 Performs rgw multisite configuration to match the given realm definition.
31 List one or more zonegroup definitions. These are provided as json
32 input to `radosgw-admin zonegroup set`, with the exception of these keys:
34 * 'is_master' is passed on the command line as --master
35 * 'is_default' is passed on the command line as --default
36 * 'endpoints' given as client names are replaced with actual endpoints
39 - name: test-zonegroup
43 endpoints: [c1.client.0]
45 List each of the zones to be created in this zonegroup.
51 endpoints: [c1.client.0]
54 endpoints: [c2.client.0]
70 - name: test-zonegroup
77 endpoints: [c1.client.0]
80 endpoints: [c2.client.0]
83 def __init__(self
, ctx
, config
):
84 super(RGWMultisite
, self
).__init
__(ctx
, config
)
87 super(RGWMultisite
, self
).setup()
89 overrides
= self
.ctx
.config
.get('overrides', {})
90 misc
.deep_merge(self
.config
, overrides
.get('rgw-multisite', {}))
93 raise ConfigError('rgw-multisite must run after the rgw task')
94 role_endpoints
= self
.ctx
.rgw
.role_endpoints
96 # construct Clusters and Gateways for each client in the rgw task
97 clusters
, gateways
= extract_clusters_and_gateways(self
.ctx
,
100 # get the master zone and zonegroup configuration
101 mz
, mzg
= extract_master_zone_zonegroup(self
.config
['zonegroups'])
102 cluster1
= cluster_for_zone(clusters
, mz
)
104 # create the realm and period on the master zone's cluster
105 log
.info('creating realm..')
106 realm
= create_realm(cluster1
, self
.config
['realm'])
107 period
= realm
.current_period
109 creds
= gen_credentials()
111 # create the master zonegroup and its master zone
112 log
.info('creating master zonegroup..')
113 master_zonegroup
= create_zonegroup(cluster1
, gateways
, period
,
115 period
.master_zonegroup
= master_zonegroup
117 log
.info('creating master zone..')
118 master_zone
= create_zone(self
.ctx
, cluster1
, gateways
, creds
,
119 master_zonegroup
, deepcopy(mz
))
120 master_zonegroup
.master_zone
= master_zone
122 period
.update(master_zone
, commit
=True)
123 restart_zone_gateways(master_zone
) # restart with --rgw-zone
125 # create the admin user on the master zone
126 log
.info('creating admin user..')
127 user_args
= ['--display-name', 'Realm Admin', '--system']
128 user_args
+= creds
.credential_args()
129 admin_user
= multisite
.User('realm-admin')
130 admin_user
.create(master_zone
, user_args
)
132 # process 'zonegroups'
133 for zg_config
in self
.config
['zonegroups']:
134 zones_config
= zg_config
.pop('zones')
137 for zone_config
in zones_config
:
138 # get the cluster for this zone
139 cluster
= cluster_for_zone(clusters
, zone_config
)
141 if cluster
!= cluster1
: # already created on master cluster
142 log
.info('pulling realm configuration to %s', cluster
.name
)
143 realm
.pull(cluster
, master_zone
.gateways
[0], creds
)
145 # use the first zone's cluster to create the zonegroup
147 if zg_config
['name'] == master_zonegroup
.name
:
148 zonegroup
= master_zonegroup
150 log
.info('creating zonegroup..')
151 zonegroup
= create_zonegroup(cluster
, gateways
,
154 if zone_config
['name'] == master_zone
.name
:
155 # master zone was already created
158 # create the zone and commit the period
159 log
.info('creating zone..')
160 zone
= create_zone(self
.ctx
, cluster
, gateways
, creds
,
161 zonegroup
, zone_config
)
162 period
.update(zone
, commit
=True)
164 restart_zone_gateways(zone
) # restart with --rgw-zone
166 # attach configuration to the ctx for other tasks
167 self
.ctx
.rgw_multisite
= argparse
.Namespace()
168 self
.ctx
.rgw_multisite
.clusters
= clusters
169 self
.ctx
.rgw_multisite
.gateways
= gateways
170 self
.ctx
.rgw_multisite
.realm
= realm
171 self
.ctx
.rgw_multisite
.admin_user
= admin_user
173 log
.info('rgw multisite configuration completed')
176 del self
.ctx
.rgw_multisite
178 class Cluster(multisite
.Cluster
):
179 """ Issues 'radosgw-admin' commands with the rgwadmin() helper """
180 def __init__(self
, ctx
, name
, client
):
181 super(Cluster
, self
).__init
__()
186 def admin(self
, args
= None, **kwargs
):
187 """ radosgw-admin command """
189 args
+= ['--cluster', self
.name
]
190 args
+= ['--debug-rgw', '0']
191 if kwargs
.pop('read_only', False):
192 args
+= ['--rgw-cache-enabled', 'false']
193 kwargs
['decode'] = False
194 check_retcode
= kwargs
.pop('check_retcode', True)
195 r
, s
= rgwadmin(self
.ctx
, self
.client
, args
, **kwargs
)
200 class Gateway(multisite
.Gateway
):
201 """ Controls a radosgw instance using its daemon """
202 def __init__(self
, role
, remote
, daemon
, *args
, **kwargs
):
203 super(Gateway
, self
).__init
__(*args
, **kwargs
)
208 def set_zone(self
, zone
):
209 """ set the zone and add its args to the daemon's command line """
210 assert self
.zone
is None, 'zone can only be set once'
212 # daemon.restart_with_args() would be perfect for this, except that
213 # radosgw args likely include a pipe and redirect. zone arguments at
214 # the end won't actually apply to radosgw
215 args
= self
.daemon
.command_kwargs
.get('args', [])
217 # insert zone args before the first |
218 pipe
= args
.index(run
.Raw('|'))
219 args
= args
[0:pipe
] + zone
.zone_args() + args
[pipe
:]
220 except ValueError, e
:
221 args
+= zone
.zone_args()
222 self
.daemon
.command_kwargs
['args'] = args
224 def start(self
, args
= None):
225 """ (re)start the daemon """
226 self
.daemon
.restart()
227 # wait until startup completes
228 wait_for_radosgw(self
.endpoint())
231 """ stop the daemon """
234 def extract_clusters_and_gateways(ctx
, role_endpoints
):
235 """ create cluster and gateway instances for all of the radosgw roles """
238 for role
, (host
, port
) in role_endpoints
.iteritems():
239 cluster_name
, daemon_type
, client_id
= misc
.split_role(role
)
240 # find or create the cluster by name
241 cluster
= clusters
.get(cluster_name
)
243 clusters
[cluster_name
] = cluster
= Cluster(ctx
, cluster_name
, role
)
244 # create a gateway for this daemon
245 client_with_id
= daemon_type
+ '.' + client_id
# match format from rgw.py
246 daemon
= ctx
.daemons
.get_daemon('rgw', client_with_id
, cluster_name
)
248 raise ConfigError('no daemon for role=%s cluster=%s type=rgw id=%s' % \
249 (role
, cluster_name
, client_id
))
250 (remote
,) = ctx
.cluster
.only(role
).remotes
.keys()
251 gateways
[role
] = Gateway(role
, remote
, daemon
, host
, port
, cluster
)
252 return clusters
, gateways
254 def create_realm(cluster
, config
):
255 """ create a realm from configuration and initialize its first period """
256 realm
= multisite
.Realm(config
['name'])
258 if config
.get('is_default', False):
259 args
+= ['--default']
260 realm
.create(cluster
, args
)
261 realm
.current_period
= multisite
.Period(realm
)
264 def extract_user_credentials(config
):
265 """ extract keys from configuration """
266 return multisite
.Credentials(config
['access_key'], config
['secret_key'])
268 def extract_master_zone(zonegroup_config
):
269 """ find and return the master zone definition """
271 for zone
in zonegroup_config
['zones']:
272 if not zone
.get('is_master', False):
275 raise ConfigError('zones %s and %s cannot both set \'is_master\'' % \
276 (master
['name'], zone
['name']))
278 # continue the loop so we can detect duplicates
280 raise ConfigError('one zone must set \'is_master\' in zonegroup %s' % \
281 zonegroup_config
['name'])
284 def extract_master_zone_zonegroup(zonegroups_config
):
285 """ find and return the master zone and zonegroup definitions """
286 master_zone
, master_zonegroup
= (None, None)
287 for zonegroup
in zonegroups_config
:
288 # verify that all zonegroups have a master zone set, even if they
289 # aren't in the master zonegroup
290 zone
= extract_master_zone(zonegroup
)
291 if not zonegroup
.get('is_master', False):
294 raise ConfigError('zonegroups %s and %s cannot both set \'is_master\'' % \
295 (master_zonegroup
['name'], zonegroup
['name']))
296 master_zonegroup
= zonegroup
298 # continue the loop so we can detect duplicates
299 if not master_zonegroup
:
300 raise ConfigError('one zonegroup must set \'is_master\'')
301 return master_zone
, master_zonegroup
303 def extract_zone_cluster_name(zone_config
):
304 """ return the cluster (must be common to all zone endpoints) """
306 endpoints
= zone_config
.get('endpoints')
308 raise ConfigError('zone %s missing \'endpoints\' list' % \
310 for role
in endpoints
:
311 name
, _
, _
= misc
.split_role(role
)
314 elif cluster_name
!= name
:
315 raise ConfigError('all zone %s endpoints must be in the same cluster' % \
319 def cluster_for_zone(clusters
, zone_config
):
320 """ return the cluster entry for the given zone """
321 name
= extract_zone_cluster_name(zone_config
)
323 return clusters
[name
]
325 raise ConfigError('no cluster %s found' % name
)
327 def gen_access_key():
328 return ''.join(random
.choice(string
.ascii_uppercase
+ string
.digits
) for _
in range(16))
331 return ''.join(random
.choice(string
.ascii_uppercase
+ string
.ascii_lowercase
+ string
.digits
) for _
in range(32))
333 def gen_credentials():
334 return multisite
.Credentials(gen_access_key(), gen_secret())
336 def extract_gateway_endpoints(gateways
, endpoints_config
):
337 """ return a list of gateway endpoints associated with the given roles """
339 for role
in endpoints_config
:
341 # replace role names with their gateway's endpoint
342 endpoints
.append(gateways
[role
].endpoint())
344 raise ConfigError('no radosgw endpoint found for role %s' % role
)
347 def is_default_arg(config
):
348 return ['--default'] if config
.pop('is_default', False) else []
350 def is_master_arg(config
):
351 return ['--master'] if config
.pop('is_master', False) else []
353 def create_zonegroup(cluster
, gateways
, period
, config
):
354 """ pass the zonegroup configuration to `zonegroup set` """
355 config
.pop('zones', None) # remove 'zones' from input to `zonegroup set`
356 endpoints
= config
.get('endpoints')
358 # replace client names with their gateway endpoints
359 config
['endpoints'] = extract_gateway_endpoints(gateways
, endpoints
)
360 zonegroup
= multisite
.ZoneGroup(config
['name'], period
)
361 # `zonegroup set` needs --default on command line, and 'is_master' in json
362 args
= is_default_arg(config
)
363 zonegroup
.set(cluster
, config
, args
)
364 period
.zonegroups
.append(zonegroup
)
367 def create_zone(ctx
, cluster
, gateways
, creds
, zonegroup
, config
):
368 """ create a zone with the given configuration """
369 zone
= multisite
.Zone(config
['name'], zonegroup
, cluster
)
370 zone
= RadosZone(config
['name'], zonegroup
, cluster
)
372 # collect Gateways for the zone's endpoints
373 endpoints
= config
.get('endpoints')
375 raise ConfigError('no \'endpoints\' for zone %s' % config
['name'])
376 zone
.gateways
= [gateways
[role
] for role
in endpoints
]
377 for gateway
in zone
.gateways
:
378 gateway
.set_zone(zone
)
380 # format the gateway endpoints
381 endpoints
= [g
.endpoint() for g
in zone
.gateways
]
383 args
= is_default_arg(config
)
384 args
+= is_master_arg(config
)
385 args
+= creds
.credential_args()
387 args
+= ['--endpoints', ','.join(endpoints
)]
388 zone
.create(cluster
, args
)
389 zonegroup
.zones
.append(zone
)
391 create_zone_pools(ctx
, zone
)
392 if ctx
.rgw
.compression_type
:
393 configure_zone_compression(zone
, ctx
.rgw
.compression_type
)
395 zonegroup
.zones_by_type
.setdefault(zone
.tier_type(), []).append(zone
)
397 if zone
.is_read_only():
398 zonegroup
.ro_zones
.append(zone
)
400 zonegroup
.rw_zones
.append(zone
)
404 def create_zone_pools(ctx
, zone
):
405 """ Create the data_pool for each placement type """
406 gateway
= zone
.gateways
[0]
407 cluster
= zone
.cluster
408 for pool_config
in zone
.data
.get('placement_pools', []):
409 pool_name
= pool_config
['val']['data_pool']
410 if ctx
.rgw
.ec_data_pool
:
411 create_ec_pool(gateway
.remote
, pool_name
, zone
.name
, 64,
412 ctx
.rgw
.erasure_code_profile
, cluster
.name
)
414 create_replicated_pool(gateway
.remote
, pool_name
, 64, cluster
.name
)
416 def configure_zone_compression(zone
, compression
):
417 """ Set compression type in the zone's default-placement """
418 zone
.json_command(zone
.cluster
, 'placement', ['modify',
419 '--placement-id', 'default-placement',
420 '--compression', compression
423 def restart_zone_gateways(zone
):