]>
git.proxmox.com Git - ceph.git/blob - ceph/qa/tasks/rgw_multisite.py
668696b63b47114ee5d063838b12a4d61070be31
2 rgw multisite configuration routines
9 from copy
import deepcopy
10 from util
.rgw
import rgwadmin
, wait_for_radosgw
11 from util
.rados
import create_ec_pool
, create_replicated_pool
12 from rgw_multi
import multisite
13 from rgw_multi
.zone_rados
import RadosZone
as RadosZone
15 from teuthology
.orchestra
import run
16 from teuthology
import misc
17 from teuthology
.exceptions
import ConfigError
18 from teuthology
.task
import Task
20 log
= logging
.getLogger(__name__
)
22 class RGWMultisite(Task
):
24 Performs rgw multisite configuration to match the given realm definition.
31 List one or more zonegroup definitions. These are provided as json
32 input to `radosgw-admin zonegroup set`, with the exception of these keys:
34 * 'is_master' is passed on the command line as --master
35 * 'is_default' is passed on the command line as --default
36 * 'endpoints' given as client names are replaced with actual endpoints
39 - name: test-zonegroup
43 endpoints: [c1.client.0]
45 List each of the zones to be created in this zonegroup.
51 endpoints: [c1.client.0]
54 endpoints: [c2.client.0]
70 - name: test-zonegroup
77 endpoints: [c1.client.0]
80 endpoints: [c2.client.0]
83 def __init__(self
, ctx
, config
):
84 super(RGWMultisite
, self
).__init
__(ctx
, config
)
87 super(RGWMultisite
, self
).setup()
89 overrides
= self
.ctx
.config
.get('overrides', {})
90 misc
.deep_merge(self
.config
, overrides
.get('rgw-multisite', {}))
93 raise ConfigError('rgw-multisite must run after the rgw task')
94 role_endpoints
= self
.ctx
.rgw
.role_endpoints
96 # construct Clusters and Gateways for each client in the rgw task
97 clusters
, gateways
= extract_clusters_and_gateways(self
.ctx
,
100 # get the master zone and zonegroup configuration
101 mz
, mzg
= extract_master_zone_zonegroup(self
.config
['zonegroups'])
102 cluster1
= cluster_for_zone(clusters
, mz
)
104 # create the realm and period on the master zone's cluster
105 log
.info('creating realm..')
106 realm
= create_realm(cluster1
, self
.config
['realm'])
107 period
= realm
.current_period
109 creds
= gen_credentials()
111 # create the master zonegroup and its master zone
112 log
.info('creating master zonegroup..')
113 master_zonegroup
= create_zonegroup(cluster1
, gateways
, period
,
115 period
.master_zonegroup
= master_zonegroup
117 log
.info('creating master zone..')
118 master_zone
= create_zone(self
.ctx
, cluster1
, gateways
, creds
,
119 master_zonegroup
, deepcopy(mz
))
120 master_zonegroup
.master_zone
= master_zone
122 period
.update(master_zone
, commit
=True)
123 restart_zone_gateways(master_zone
) # restart with --rgw-zone
125 # create the admin user on the master zone
126 log
.info('creating admin user..')
127 user_args
= ['--display-name', 'Realm Admin', '--system']
128 user_args
+= creds
.credential_args()
129 admin_user
= multisite
.User('realm-admin')
130 admin_user
.create(master_zone
, user_args
)
132 # process 'zonegroups'
133 for zg_config
in self
.config
['zonegroups']:
134 zones_config
= zg_config
.pop('zones')
137 for zone_config
in zones_config
:
138 # get the cluster for this zone
139 cluster
= cluster_for_zone(clusters
, zone_config
)
141 if cluster
!= cluster1
: # already created on master cluster
142 log
.info('pulling realm configuration to %s', cluster
.name
)
143 realm
.pull(cluster
, master_zone
.gateways
[0], creds
)
145 # use the first zone's cluster to create the zonegroup
147 if zg_config
['name'] == master_zonegroup
.name
:
148 zonegroup
= master_zonegroup
150 log
.info('creating zonegroup..')
151 zonegroup
= create_zonegroup(cluster
, gateways
,
154 if zone_config
['name'] == master_zone
.name
:
155 # master zone was already created
158 # create the zone and commit the period
159 log
.info('creating zone..')
160 zone
= create_zone(self
.ctx
, cluster
, gateways
, creds
,
161 zonegroup
, zone_config
)
162 period
.update(zone
, commit
=True)
164 restart_zone_gateways(zone
) # restart with --rgw-zone
166 # attach configuration to the ctx for other tasks
167 self
.ctx
.rgw_multisite
= argparse
.Namespace()
168 self
.ctx
.rgw_multisite
.clusters
= clusters
169 self
.ctx
.rgw_multisite
.gateways
= gateways
170 self
.ctx
.rgw_multisite
.realm
= realm
171 self
.ctx
.rgw_multisite
.admin_user
= admin_user
173 log
.info('rgw multisite configuration completed')
176 del self
.ctx
.rgw_multisite
178 class Cluster(multisite
.Cluster
):
179 """ Issues 'radosgw-admin' commands with the rgwadmin() helper """
180 def __init__(self
, ctx
, name
, client
):
181 super(Cluster
, self
).__init
__()
186 def admin(self
, args
= None, **kwargs
):
187 """ radosgw-admin command """
189 args
+= ['--cluster', self
.name
]
190 args
+= ['--debug-rgw', str(kwargs
.pop('debug_rgw', 0))]
191 args
+= ['--debug-ms', str(kwargs
.pop('debug_ms', 0))]
192 if kwargs
.pop('read_only', False):
193 args
+= ['--rgw-cache-enabled', 'false']
194 kwargs
['decode'] = False
195 check_retcode
= kwargs
.pop('check_retcode', True)
196 r
, s
= rgwadmin(self
.ctx
, self
.client
, args
, **kwargs
)
201 class Gateway(multisite
.Gateway
):
202 """ Controls a radosgw instance using its daemon """
203 def __init__(self
, role
, remote
, daemon
, *args
, **kwargs
):
204 super(Gateway
, self
).__init
__(*args
, **kwargs
)
209 def set_zone(self
, zone
):
210 """ set the zone and add its args to the daemon's command line """
211 assert self
.zone
is None, 'zone can only be set once'
213 # daemon.restart_with_args() would be perfect for this, except that
214 # radosgw args likely include a pipe and redirect. zone arguments at
215 # the end won't actually apply to radosgw
216 args
= self
.daemon
.command_kwargs
.get('args', [])
218 # insert zone args before the first |
219 pipe
= args
.index(run
.Raw('|'))
220 args
= args
[0:pipe
] + zone
.zone_args() + args
[pipe
:]
221 except ValueError, e
:
222 args
+= zone
.zone_args()
223 self
.daemon
.command_kwargs
['args'] = args
225 def start(self
, args
= None):
226 """ (re)start the daemon """
227 self
.daemon
.restart()
228 # wait until startup completes
229 wait_for_radosgw(self
.endpoint())
232 """ stop the daemon """
235 def extract_clusters_and_gateways(ctx
, role_endpoints
):
236 """ create cluster and gateway instances for all of the radosgw roles """
239 for role
, (host
, port
) in role_endpoints
.iteritems():
240 cluster_name
, daemon_type
, client_id
= misc
.split_role(role
)
241 # find or create the cluster by name
242 cluster
= clusters
.get(cluster_name
)
244 clusters
[cluster_name
] = cluster
= Cluster(ctx
, cluster_name
, role
)
245 # create a gateway for this daemon
246 client_with_id
= daemon_type
+ '.' + client_id
# match format from rgw.py
247 daemon
= ctx
.daemons
.get_daemon('rgw', client_with_id
, cluster_name
)
249 raise ConfigError('no daemon for role=%s cluster=%s type=rgw id=%s' % \
250 (role
, cluster_name
, client_id
))
251 (remote
,) = ctx
.cluster
.only(role
).remotes
.keys()
252 gateways
[role
] = Gateway(role
, remote
, daemon
, host
, port
, cluster
)
253 return clusters
, gateways
255 def create_realm(cluster
, config
):
256 """ create a realm from configuration and initialize its first period """
257 realm
= multisite
.Realm(config
['name'])
259 if config
.get('is_default', False):
260 args
+= ['--default']
261 realm
.create(cluster
, args
)
262 realm
.current_period
= multisite
.Period(realm
)
265 def extract_user_credentials(config
):
266 """ extract keys from configuration """
267 return multisite
.Credentials(config
['access_key'], config
['secret_key'])
269 def extract_master_zone(zonegroup_config
):
270 """ find and return the master zone definition """
272 for zone
in zonegroup_config
['zones']:
273 if not zone
.get('is_master', False):
276 raise ConfigError('zones %s and %s cannot both set \'is_master\'' % \
277 (master
['name'], zone
['name']))
279 # continue the loop so we can detect duplicates
281 raise ConfigError('one zone must set \'is_master\' in zonegroup %s' % \
282 zonegroup_config
['name'])
285 def extract_master_zone_zonegroup(zonegroups_config
):
286 """ find and return the master zone and zonegroup definitions """
287 master_zone
, master_zonegroup
= (None, None)
288 for zonegroup
in zonegroups_config
:
289 # verify that all zonegroups have a master zone set, even if they
290 # aren't in the master zonegroup
291 zone
= extract_master_zone(zonegroup
)
292 if not zonegroup
.get('is_master', False):
295 raise ConfigError('zonegroups %s and %s cannot both set \'is_master\'' % \
296 (master_zonegroup
['name'], zonegroup
['name']))
297 master_zonegroup
= zonegroup
299 # continue the loop so we can detect duplicates
300 if not master_zonegroup
:
301 raise ConfigError('one zonegroup must set \'is_master\'')
302 return master_zone
, master_zonegroup
304 def extract_zone_cluster_name(zone_config
):
305 """ return the cluster (must be common to all zone endpoints) """
307 endpoints
= zone_config
.get('endpoints')
309 raise ConfigError('zone %s missing \'endpoints\' list' % \
311 for role
in endpoints
:
312 name
, _
, _
= misc
.split_role(role
)
315 elif cluster_name
!= name
:
316 raise ConfigError('all zone %s endpoints must be in the same cluster' % \
320 def cluster_for_zone(clusters
, zone_config
):
321 """ return the cluster entry for the given zone """
322 name
= extract_zone_cluster_name(zone_config
)
324 return clusters
[name
]
326 raise ConfigError('no cluster %s found' % name
)
328 def gen_access_key():
329 return ''.join(random
.choice(string
.ascii_uppercase
+ string
.digits
) for _
in range(16))
332 return ''.join(random
.choice(string
.ascii_uppercase
+ string
.ascii_lowercase
+ string
.digits
) for _
in range(32))
334 def gen_credentials():
335 return multisite
.Credentials(gen_access_key(), gen_secret())
337 def extract_gateway_endpoints(gateways
, endpoints_config
):
338 """ return a list of gateway endpoints associated with the given roles """
340 for role
in endpoints_config
:
342 # replace role names with their gateway's endpoint
343 endpoints
.append(gateways
[role
].endpoint())
345 raise ConfigError('no radosgw endpoint found for role %s' % role
)
348 def is_default_arg(config
):
349 return ['--default'] if config
.pop('is_default', False) else []
351 def is_master_arg(config
):
352 return ['--master'] if config
.pop('is_master', False) else []
354 def create_zonegroup(cluster
, gateways
, period
, config
):
355 """ pass the zonegroup configuration to `zonegroup set` """
356 config
.pop('zones', None) # remove 'zones' from input to `zonegroup set`
357 endpoints
= config
.get('endpoints')
359 # replace client names with their gateway endpoints
360 config
['endpoints'] = extract_gateway_endpoints(gateways
, endpoints
)
361 zonegroup
= multisite
.ZoneGroup(config
['name'], period
)
362 # `zonegroup set` needs --default on command line, and 'is_master' in json
363 args
= is_default_arg(config
)
364 zonegroup
.set(cluster
, config
, args
)
365 period
.zonegroups
.append(zonegroup
)
368 def create_zone(ctx
, cluster
, gateways
, creds
, zonegroup
, config
):
369 """ create a zone with the given configuration """
370 zone
= multisite
.Zone(config
['name'], zonegroup
, cluster
)
371 zone
= RadosZone(config
['name'], zonegroup
, cluster
)
373 # collect Gateways for the zone's endpoints
374 endpoints
= config
.get('endpoints')
376 raise ConfigError('no \'endpoints\' for zone %s' % config
['name'])
377 zone
.gateways
= [gateways
[role
] for role
in endpoints
]
378 for gateway
in zone
.gateways
:
379 gateway
.set_zone(zone
)
381 # format the gateway endpoints
382 endpoints
= [g
.endpoint() for g
in zone
.gateways
]
384 args
= is_default_arg(config
)
385 args
+= is_master_arg(config
)
386 args
+= creds
.credential_args()
388 args
+= ['--endpoints', ','.join(endpoints
)]
389 zone
.create(cluster
, args
)
390 zonegroup
.zones
.append(zone
)
392 create_zone_pools(ctx
, zone
)
393 if ctx
.rgw
.compression_type
:
394 configure_zone_compression(zone
, ctx
.rgw
.compression_type
)
396 zonegroup
.zones_by_type
.setdefault(zone
.tier_type(), []).append(zone
)
398 if zone
.is_read_only():
399 zonegroup
.ro_zones
.append(zone
)
401 zonegroup
.rw_zones
.append(zone
)
405 def create_zone_pools(ctx
, zone
):
406 """ Create the data_pool for each placement type """
407 gateway
= zone
.gateways
[0]
408 cluster
= zone
.cluster
409 for pool_config
in zone
.data
.get('placement_pools', []):
410 pool_name
= pool_config
['val']['data_pool']
411 if ctx
.rgw
.ec_data_pool
:
412 create_ec_pool(gateway
.remote
, pool_name
, zone
.name
, 64,
413 ctx
.rgw
.erasure_code_profile
, cluster
.name
, 'rgw')
415 create_replicated_pool(gateway
.remote
, pool_name
, 64, cluster
.name
, 'rgw')
417 def configure_zone_compression(zone
, compression
):
418 """ Set compression type in the zone's default-placement """
419 zone
.json_command(zone
.cluster
, 'placement', ['modify',
420 '--placement-id', 'default-placement',
421 '--compression', compression
424 def restart_zone_gateways(zone
):