ceph/qa/tasks/rgw_multisite.py

   1 """
   2 rgw multisite configuration routines
   3 """
   4 import argparse
   5 import logging
   6 import random
   7 import string
   8 from copy import deepcopy
   9 from tasks.util.rgw import rgwadmin, wait_for_radosgw
  10 from tasks.util.rados import create_ec_pool, create_replicated_pool
  11 from tasks.rgw_multi import multisite
  12 from tasks.rgw_multi.zone_rados import RadosZone as RadosZone
  13 from tasks.rgw_multi.zone_ps import PSZone as PSZone
  14
  15 from teuthology.orchestra import run
  16 from teuthology import misc
  17 from teuthology.exceptions import ConfigError
  18 from teuthology.task import Task
  19
  20 log = logging.getLogger(__name__)
  21
  22 class RGWMultisite(Task):
  23     """
  24     Performs rgw multisite configuration to match the given realm definition.
  25
  26         - rgw-multisite:
  27             realm:
  28               name: test-realm
  29               is_default: true
  30
  31     List one or more zonegroup definitions. These are provided as json
  32     input to `radosgw-admin zonegroup set`, with the exception of these keys:
  33
  34     * 'is_master' is passed on the command line as --master
  35     * 'is_default' is passed on the command line as --default
  36     * 'is_pubsub' is used to create a zone with tier-type=pubsub
  37     * 'endpoints' given as client names are replaced with actual endpoints
  38
  39             zonegroups:
  40               - name: test-zonegroup
  41                 api_name: test-api
  42                 is_master: true
  43                 is_default: true
  44                 endpoints: [c1.client.0]
  45
  46     List each of the zones to be created in this zonegroup.
  47
  48                 zones:
  49                   - name: test-zone1
  50                     is_master: true
  51                     is_default: true
  52                     endpoints: [c1.client.0]
  53                   - name: test-zone2
  54                     is_default: true
  55                     endpoints: [c2.client.0]
  56
  57     A complete example:
  58
  59         tasks:
  60         - install:
  61         - ceph: {cluster: c1}
  62         - ceph: {cluster: c2}
  63         - rgw:
  64             c1.client.0:
  65             c2.client.0:
  66         - rgw-multisite:
  67             realm:
  68               name: test-realm
  69               is_default: true
  70             zonegroups:
  71               - name: test-zonegroup
  72                 is_master: true
  73                 is_default: true
  74                 zones:
  75                   - name: test-zone1
  76                     is_master: true
  77                     is_default: true
  78                     endpoints: [c1.client.0]
  79                   - name: test-zone2
  80                     is_default: true
  81                     endpoints: [c2.client.0]
  82                   - name: test-zone3
  83                     is_pubsub: true
  84                     endpoints: [c1.client.1]
  85
  86     """
  87     def __init__(self, ctx, config):
  88         super(RGWMultisite, self).__init__(ctx, config)
  89
  90     def setup(self):
  91         super(RGWMultisite, self).setup()
  92
  93         overrides = self.ctx.config.get('overrides', {})
  94         misc.deep_merge(self.config, overrides.get('rgw-multisite', {}))
  95
  96         if not self.ctx.rgw:
  97             raise ConfigError('rgw-multisite must run after the rgw task')
  98         role_endpoints = self.ctx.rgw.role_endpoints
  99
 100         # construct Clusters and Gateways for each client in the rgw task
 101         clusters, gateways = extract_clusters_and_gateways(self.ctx,
 102                                                            role_endpoints)
 103
 104         # get the master zone and zonegroup configuration
 105         mz, mzg = extract_master_zone_zonegroup(self.config['zonegroups'])
 106         cluster1 = cluster_for_zone(clusters, mz)
 107
 108         # create the realm and period on the master zone's cluster
 109         log.info('creating realm..')
 110         realm = create_realm(cluster1, self.config['realm'])
 111         period = realm.current_period
 112
 113         creds = gen_credentials()
 114
 115         # create the master zonegroup and its master zone
 116         log.info('creating master zonegroup..')
 117         master_zonegroup = create_zonegroup(cluster1, gateways, period,
 118                                             deepcopy(mzg))
 119         period.master_zonegroup = master_zonegroup
 120
 121         log.info('creating master zone..')
 122         master_zone = create_zone(self.ctx, cluster1, gateways, creds,
 123                                   master_zonegroup, deepcopy(mz))
 124         master_zonegroup.master_zone = master_zone
 125
 126         period.update(master_zone, commit=True)
 127         restart_zone_gateways(master_zone) # restart with --rgw-zone
 128
 129         # create the admin user on the master zone
 130         log.info('creating admin user..')
 131         user_args = ['--display-name', 'Realm Admin', '--system']
 132         user_args += creds.credential_args()
 133         admin_user = multisite.User('realm-admin')
 134         admin_user.create(master_zone, user_args)
 135
 136         # process 'zonegroups'
 137         for zg_config in self.config['zonegroups']:
 138             zones_config = zg_config.pop('zones')
 139
 140             zonegroup = None
 141             for zone_config in zones_config:
 142                 # get the cluster for this zone
 143                 cluster = cluster_for_zone(clusters, zone_config)
 144
 145                 if cluster != cluster1: # already created on master cluster
 146                     log.info('pulling realm configuration to %s', cluster.name)
 147                     realm.pull(cluster, master_zone.gateways[0], creds)
 148
 149                 # use the first zone's cluster to create the zonegroup
 150                 if not zonegroup:
 151                     if zg_config['name'] == master_zonegroup.name:
 152                         zonegroup = master_zonegroup
 153                     else:
 154                         log.info('creating zonegroup..')
 155                         zonegroup = create_zonegroup(cluster, gateways,
 156                                                      period, zg_config)
 157
 158                 if zone_config['name'] == master_zone.name:
 159                     # master zone was already created
 160                     zone = master_zone
 161                 else:
 162                     # create the zone and commit the period
 163                     log.info('creating zone..')
 164                     zone = create_zone(self.ctx, cluster, gateways, creds,
 165                                        zonegroup, zone_config)
 166                     period.update(zone, commit=True)
 167
 168                     restart_zone_gateways(zone) # restart with --rgw-zone
 169
 170         # attach configuration to the ctx for other tasks
 171         self.ctx.rgw_multisite = argparse.Namespace()
 172         self.ctx.rgw_multisite.clusters = clusters
 173         self.ctx.rgw_multisite.gateways = gateways
 174         self.ctx.rgw_multisite.realm = realm
 175         self.ctx.rgw_multisite.admin_user = admin_user
 176
 177         log.info('rgw multisite configuration completed')
 178
 179     def end(self):
 180         del self.ctx.rgw_multisite
 181
 182 class Cluster(multisite.Cluster):
 183     """ Issues 'radosgw-admin' commands with the rgwadmin() helper """
 184     def __init__(self, ctx, name, client):
 185         super(Cluster, self).__init__()
 186         self.ctx = ctx
 187         self.name = name
 188         self.client = client
 189
 190     def admin(self, args = None, **kwargs):
 191         """ radosgw-admin command """
 192         args = args or []
 193         args += ['--cluster', self.name]
 194         args += ['--debug-rgw', str(kwargs.pop('debug_rgw', 0))]
 195         args += ['--debug-ms', str(kwargs.pop('debug_ms', 0))]
 196         if kwargs.pop('read_only', False):
 197             args += ['--rgw-cache-enabled', 'false']
 198         kwargs['decode'] = False
 199         check_retcode = kwargs.pop('check_retcode', True)
 200         r, s = rgwadmin(self.ctx, self.client, args, **kwargs)
 201         if check_retcode:
 202             assert r == 0
 203         return s, r
 204
 205 class Gateway(multisite.Gateway):
 206     """ Controls a radosgw instance using its daemon """
 207     def __init__(self, role, remote, daemon, *args, **kwargs):
 208         super(Gateway, self).__init__(*args, **kwargs)
 209         self.role = role
 210         self.remote = remote
 211         self.daemon = daemon
 212
 213     def set_zone(self, zone):
 214         """ set the zone and add its args to the daemon's command line """
 215         assert self.zone is None, 'zone can only be set once'
 216         self.zone = zone
 217         # daemon.restart_with_args() would be perfect for this, except that
 218         # radosgw args likely include a pipe and redirect. zone arguments at
 219         # the end won't actually apply to radosgw
 220         args = self.daemon.command_kwargs.get('args', [])
 221         try:
 222             # insert zone args before the first |
 223             pipe = args.index(run.Raw('|'))
 224             args = args[0:pipe] + zone.zone_args() + args[pipe:]
 225         except ValueError:
 226             args += zone.zone_args()
 227         self.daemon.command_kwargs['args'] = args
 228
 229     def start(self, args = None):
 230         """ (re)start the daemon """
 231         self.daemon.restart()
 232         # wait until startup completes
 233         wait_for_radosgw(self.endpoint(), self.remote)
 234
 235     def stop(self):
 236         """ stop the daemon """
 237         self.daemon.stop()
 238
 239 def extract_clusters_and_gateways(ctx, role_endpoints):
 240     """ create cluster and gateway instances for all of the radosgw roles """
 241     clusters = {}
 242     gateways = {}
 243     for role, endpoint in role_endpoints.items():
 244         cluster_name, daemon_type, client_id = misc.split_role(role)
 245         # find or create the cluster by name
 246         cluster = clusters.get(cluster_name)
 247         if not cluster:
 248             clusters[cluster_name] = cluster = Cluster(ctx, cluster_name, role)
 249         # create a gateway for this daemon
 250         client_with_id = daemon_type + '.' + client_id # match format from rgw.py
 251         daemon = ctx.daemons.get_daemon('rgw', client_with_id, cluster_name)
 252         if not daemon:
 253             raise ConfigError('no daemon for role=%s cluster=%s type=rgw id=%s' % \
 254                               (role, cluster_name, client_id))
 255         (remote,) = ctx.cluster.only(role).remotes.keys()
 256         gateways[role] = Gateway(role, remote, daemon, endpoint.hostname,
 257                 endpoint.port, cluster)
 258     return clusters, gateways
 259
 260 def create_realm(cluster, config):
 261     """ create a realm from configuration and initialize its first period """
 262     realm = multisite.Realm(config['name'])
 263     args = []
 264     if config.get('is_default', False):
 265         args += ['--default']
 266     realm.create(cluster, args)
 267     realm.current_period = multisite.Period(realm)
 268     return realm
 269
 270 def extract_user_credentials(config):
 271     """ extract keys from configuration """
 272     return multisite.Credentials(config['access_key'], config['secret_key'])
 273
 274 def extract_master_zone(zonegroup_config):
 275     """ find and return the master zone definition """
 276     master = None
 277     for zone in zonegroup_config['zones']:
 278         if not zone.get('is_master', False):
 279             continue
 280         if master:
 281             raise ConfigError('zones %s and %s cannot both set \'is_master\'' % \
 282                               (master['name'], zone['name']))
 283         master = zone
 284         # continue the loop so we can detect duplicates
 285     if not master:
 286         raise ConfigError('one zone must set \'is_master\' in zonegroup %s' % \
 287                           zonegroup_config['name'])
 288     return master
 289
 290 def extract_master_zone_zonegroup(zonegroups_config):
 291     """ find and return the master zone and zonegroup definitions """
 292     master_zone, master_zonegroup = (None, None)
 293     for zonegroup in zonegroups_config:
 294         # verify that all zonegroups have a master zone set, even if they
 295         # aren't in the master zonegroup
 296         zone = extract_master_zone(zonegroup)
 297         if not zonegroup.get('is_master', False):
 298             continue
 299         if master_zonegroup:
 300             raise ConfigError('zonegroups %s and %s cannot both set \'is_master\'' % \
 301                               (master_zonegroup['name'], zonegroup['name']))
 302         master_zonegroup = zonegroup
 303         master_zone = zone
 304         # continue the loop so we can detect duplicates
 305     if not master_zonegroup:
 306         raise ConfigError('one zonegroup must set \'is_master\'')
 307     return master_zone, master_zonegroup
 308
 309 def extract_zone_cluster_name(zone_config):
 310     """ return the cluster (must be common to all zone endpoints) """
 311     cluster_name = None
 312     endpoints = zone_config.get('endpoints')
 313     if not endpoints:
 314         raise ConfigError('zone %s missing \'endpoints\' list' % \
 315                           zone_config['name'])
 316     for role in endpoints:
 317         name, _, _ = misc.split_role(role)
 318         if not cluster_name:
 319             cluster_name = name
 320         elif cluster_name != name:
 321             raise ConfigError('all zone %s endpoints must be in the same cluster' % \
 322                               zone_config['name'])
 323     return cluster_name
 324
 325 def cluster_for_zone(clusters, zone_config):
 326     """ return the cluster entry for the given zone """
 327     name = extract_zone_cluster_name(zone_config)
 328     try:
 329         return clusters[name]
 330     except KeyError:
 331         raise ConfigError('no cluster %s found' % name)
 332
 333 def gen_access_key():
 334     return ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(16))
 335
 336 def gen_secret():
 337     return ''.join(random.choice(string.ascii_uppercase + string.ascii_lowercase + string.digits) for _ in range(32))
 338
 339 def gen_credentials():
 340     return multisite.Credentials(gen_access_key(), gen_secret())
 341
 342 def extract_gateway_endpoints(gateways, endpoints_config):
 343     """ return a list of gateway endpoints associated with the given roles """
 344     endpoints = []
 345     for role in endpoints_config:
 346         try:
 347             # replace role names with their gateway's endpoint
 348             endpoints.append(gateways[role].endpoint())
 349         except KeyError:
 350             raise ConfigError('no radosgw endpoint found for role %s' % role)
 351     return endpoints
 352
 353 def is_default_arg(config):
 354     return ['--default'] if config.pop('is_default', False) else []
 355
 356 def is_master_arg(config):
 357     return ['--master'] if config.pop('is_master', False) else []
 358
 359 def create_zonegroup(cluster, gateways, period, config):
 360     """ pass the zonegroup configuration to `zonegroup set` """
 361     config.pop('zones', None) # remove 'zones' from input to `zonegroup set`
 362     endpoints = config.get('endpoints')
 363     if endpoints:
 364         # replace client names with their gateway endpoints
 365         config['endpoints'] = extract_gateway_endpoints(gateways, endpoints)
 366     zonegroup = multisite.ZoneGroup(config['name'], period)
 367     # `zonegroup set` needs --default on command line, and 'is_master' in json
 368     args = is_default_arg(config)
 369     zonegroup.set(cluster, config, args)
 370     period.zonegroups.append(zonegroup)
 371     return zonegroup
 372
 373 def create_zone(ctx, cluster, gateways, creds, zonegroup, config):
 374     """ create a zone with the given configuration """
 375     zone = multisite.Zone(config['name'], zonegroup, cluster)
 376     if config.pop('is_pubsub', False):
 377         zone = PSZone(config['name'], zonegroup, cluster)
 378     else:
 379         zone = RadosZone(config['name'], zonegroup, cluster)
 380
 381     # collect Gateways for the zone's endpoints
 382     endpoints = config.get('endpoints')
 383     if not endpoints:
 384         raise ConfigError('no \'endpoints\' for zone %s' % config['name'])
 385     zone.gateways = [gateways[role] for role in endpoints]
 386     for gateway in zone.gateways:
 387         gateway.set_zone(zone)
 388
 389     # format the gateway endpoints
 390     endpoints = [g.endpoint() for g in zone.gateways]
 391
 392     args = is_default_arg(config)
 393     args += is_master_arg(config)
 394     args += creds.credential_args()
 395     if len(endpoints):
 396         args += ['--endpoints', ','.join(endpoints)]
 397     zone.create(cluster, args)
 398     zonegroup.zones.append(zone)
 399
 400     create_zone_pools(ctx, zone)
 401     if ctx.rgw.compression_type:
 402         configure_zone_compression(zone, ctx.rgw.compression_type)
 403
 404     zonegroup.zones_by_type.setdefault(zone.tier_type(), []).append(zone)
 405
 406     if zone.is_read_only():
 407         zonegroup.ro_zones.append(zone)
 408     else:
 409         zonegroup.rw_zones.append(zone)
 410
 411     return zone
 412
 413 def create_zone_pools(ctx, zone):
 414     """ Create the data_pool for each placement type """
 415     gateway = zone.gateways[0]
 416     cluster = zone.cluster
 417     for pool_config in zone.data.get('placement_pools', []):
 418         pool_name = pool_config['val']['storage_classes']['STANDARD']['data_pool']
 419         if ctx.rgw.ec_data_pool:
 420             create_ec_pool(gateway.remote, pool_name, zone.name, 64,
 421                            ctx.rgw.erasure_code_profile, cluster.name, 'rgw')
 422         else:
 423             create_replicated_pool(gateway.remote, pool_name, 64, cluster.name, 'rgw')
 424
 425 def configure_zone_compression(zone, compression):
 426     """ Set compression type in the zone's default-placement """
 427     zone.json_command(zone.cluster, 'placement', ['modify',
 428                           '--placement-id', 'default-placement',
 429                           '--compression', compression
 430                       ])
 431
 432 def restart_zone_gateways(zone):
 433     zone.stop()
 434     zone.start()
 435
 436 task = RGWMultisite