ceph/qa/tasks/rgw.py

   1 """
   2 rgw routines
   3 """
   4 import argparse
   5 import contextlib
   6 import logging
   7
   8 from teuthology.orchestra import run
   9 from teuthology import misc as teuthology
  10 from teuthology import contextutil
  11 from teuthology.exceptions import ConfigError
  12 from tasks.ceph_manager import get_valgrind_args
  13 from tasks.util import get_remote_for_role
  14 from tasks.util.rgw import rgwadmin, wait_for_radosgw
  15 from tasks.util.rados import (create_ec_pool,
  16                               create_replicated_pool,
  17                               create_cache_pool)
  18
  19 log = logging.getLogger(__name__)
  20
  21 class RGWEndpoint:
  22     def __init__(self, hostname=None, port=None, cert=None, dns_name=None, website_dns_name=None):
  23         self.hostname = hostname
  24         self.port = port
  25         self.cert = cert
  26         self.dns_name = dns_name
  27         self.website_dns_name = website_dns_name
  28
  29     def url(self):
  30         proto = 'https' if self.cert else 'http'
  31         return '{proto}://{hostname}:{port}/'.format(proto=proto, hostname=self.hostname, port=self.port)
  32
  33 @contextlib.contextmanager
  34 def start_rgw(ctx, config, clients):
  35     """
  36     Start rgw on remote sites.
  37     """
  38     log.info('Starting rgw...')
  39     testdir = teuthology.get_testdir(ctx)
  40     for client in clients:
  41         (remote,) = ctx.cluster.only(client).remotes.keys()
  42         cluster_name, daemon_type, client_id = teuthology.split_role(client)
  43         client_with_id = daemon_type + '.' + client_id
  44         client_with_cluster = cluster_name + '.' + client_with_id
  45
  46         client_config = config.get(client)
  47         if client_config is None:
  48             client_config = {}
  49         log.info("rgw %s config is %s", client, client_config)
  50         cmd_prefix = [
  51             'sudo',
  52             'adjust-ulimits',
  53             'ceph-coverage',
  54             '{tdir}/archive/coverage'.format(tdir=testdir),
  55             'daemon-helper',
  56             'term',
  57             ]
  58
  59         rgw_cmd = ['radosgw']
  60
  61         log.info("Using %s as radosgw frontend", ctx.rgw.frontend)
  62
  63         endpoint = ctx.rgw.role_endpoints[client]
  64         frontends = ctx.rgw.frontend
  65         frontend_prefix = client_config.get('frontend_prefix', None)
  66         if frontend_prefix:
  67             frontends += ' prefix={pfx}'.format(pfx=frontend_prefix)
  68
  69         if endpoint.cert:
  70             # add the ssl certificate path
  71             frontends += ' ssl_certificate={}'.format(endpoint.cert.certificate)
  72             frontends += ' ssl_port={}'.format(endpoint.port)
  73         else:
  74             frontends += ' port={}'.format(endpoint.port)
  75
  76         rgw_cmd.extend([
  77             '--rgw-frontends', frontends,
  78             '-n', client_with_id,
  79             '--cluster', cluster_name,
  80             '-k', '/etc/ceph/{client_with_cluster}.keyring'.format(client_with_cluster=client_with_cluster),
  81             '--log-file',
  82             '/var/log/ceph/rgw.{client_with_cluster}.log'.format(client_with_cluster=client_with_cluster),
  83             '--rgw_ops_log_socket_path',
  84             '{tdir}/rgw.opslog.{client_with_cluster}.sock'.format(tdir=testdir,
  85                                                      client_with_cluster=client_with_cluster),
  86             ])
  87
  88         keystone_role = client_config.get('use-keystone-role', None)
  89         if keystone_role is not None:
  90             if not ctx.keystone:
  91                 raise ConfigError('rgw must run after the keystone task')
  92             url = 'http://{host}:{port}/v1/KEY_$(tenant_id)s'.format(host=endpoint.hostname,
  93                                                                      port=endpoint.port)
  94             ctx.keystone.create_endpoint(ctx, keystone_role, 'swift', url)
  95
  96             keystone_host, keystone_port = \
  97                 ctx.keystone.public_endpoints[keystone_role]
  98             rgw_cmd.extend([
  99                 '--rgw_keystone_url',
 100                 'http://{khost}:{kport}'.format(khost=keystone_host,
 101                                                 kport=keystone_port),
 102                 ])
 103
 104
 105         if client_config.get('dns-name') is not None:
 106             rgw_cmd.extend(['--rgw-dns-name', endpoint.dns_name])
 107         if client_config.get('dns-s3website-name') is not None:
 108             rgw_cmd.extend(['--rgw-dns-s3website-name', endpoint.website_dns_name])
 109
 110
 111         vault_role = client_config.get('use-vault-role', None)
 112         barbican_role = client_config.get('use-barbican-role', None)
 113         pykmip_role = client_config.get('use-pykmip-role', None)
 114
 115         token_path = '/etc/ceph/vault-root-token'
 116         if barbican_role is not None:
 117             if not hasattr(ctx, 'barbican'):
 118                 raise ConfigError('rgw must run after the barbican task')
 119
 120             barbican_host, barbican_port = \
 121                 ctx.barbican.endpoints[barbican_role]
 122             log.info("Use barbican url=%s:%s", barbican_host, barbican_port)
 123
 124             rgw_cmd.extend([
 125                 '--rgw_barbican_url',
 126                 'http://{bhost}:{bport}'.format(bhost=barbican_host,
 127                                                 bport=barbican_port),
 128                 ])
 129         elif vault_role is not None:
 130             if not ctx.vault.root_token:
 131                 raise ConfigError('vault: no "root_token" specified')
 132             # create token on file
 133             ctx.rgw.vault_role = vault_role
 134             ctx.cluster.only(client).run(args=['sudo', 'echo', '-n', ctx.vault.root_token, run.Raw('|'), 'sudo', 'tee', token_path])
 135             log.info("Token file content")
 136             ctx.cluster.only(client).run(args=['cat', token_path])
 137             log.info("Restrict access to token file")
 138             ctx.cluster.only(client).run(args=['sudo', 'chmod', '600', token_path])
 139             ctx.cluster.only(client).run(args=['sudo', 'chown', 'ceph', token_path])
 140
 141             vault_addr = "{}:{}".format(*ctx.vault.endpoints[vault_role])
 142             rgw_cmd.extend([
 143                 '--rgw_crypt_vault_addr', vault_addr,
 144                 '--rgw_crypt_vault_token_file', token_path,
 145                 '--rgw_crypt_sse_s3_vault_addr', vault_addr,
 146                 '--rgw_crypt_sse_s3_vault_token_file', token_path,
 147             ])
 148         elif pykmip_role is not None:
 149             if not hasattr(ctx, 'pykmip'):
 150                 raise ConfigError('rgw must run after the pykmip task')
 151             ctx.rgw.pykmip_role = pykmip_role
 152             rgw_cmd.extend([
 153                 '--rgw_crypt_kmip_addr', "{}:{}".format(*ctx.pykmip.endpoints[pykmip_role]),
 154             ])
 155
 156             clientcert = ctx.ssl_certificates.get('kmip-client')
 157             servercert = ctx.ssl_certificates.get('kmip-server')
 158             clientca = ctx.ssl_certificates.get('kmiproot')
 159
 160             clientkey = clientcert.key
 161             clientcert = clientcert.certificate
 162             serverkey = servercert.key
 163             servercert = servercert.certificate
 164             rootkey = clientca.key
 165             rootcert = clientca.certificate
 166
 167             cert_path = '/etc/ceph/'
 168             ctx.cluster.only(client).run(args=['sudo', 'cp', clientcert, cert_path])
 169             ctx.cluster.only(client).run(args=['sudo', 'cp', clientkey, cert_path])
 170             ctx.cluster.only(client).run(args=['sudo', 'cp', servercert, cert_path])
 171             ctx.cluster.only(client).run(args=['sudo', 'cp', serverkey, cert_path])
 172             ctx.cluster.only(client).run(args=['sudo', 'cp', rootkey, cert_path])
 173             ctx.cluster.only(client).run(args=['sudo', 'cp', rootcert, cert_path])
 174
 175             clientcert = cert_path + 'kmip-client.crt'
 176             clientkey = cert_path + 'kmip-client.key'
 177             servercert = cert_path + 'kmip-server.crt'
 178             serverkey = cert_path + 'kmip-server.key'
 179             rootkey = cert_path + 'kmiproot.key'
 180             rootcert = cert_path + 'kmiproot.crt'
 181
 182             ctx.cluster.only(client).run(args=['sudo', 'chmod', '600', clientcert, clientkey, servercert, serverkey, rootkey, rootcert])
 183             ctx.cluster.only(client).run(args=['sudo', 'chown', 'ceph', clientcert, clientkey, servercert, serverkey, rootkey, rootcert])
 184
 185         rgw_cmd.extend([
 186             '--foreground',
 187             run.Raw('|'),
 188             'sudo',
 189             'tee',
 190             '/var/log/ceph/rgw.{client_with_cluster}.stdout'.format(client_with_cluster=client_with_cluster),
 191             run.Raw('2>&1'),
 192             ])
 193
 194         if client_config.get('valgrind'):
 195             cmd_prefix = get_valgrind_args(
 196                 testdir,
 197                 client_with_cluster,
 198                 cmd_prefix,
 199                 client_config.get('valgrind'),
 200                 # see https://github.com/ceph/teuthology/pull/1600
 201                 exit_on_first_error=False
 202                 )
 203
 204         run_cmd = list(cmd_prefix)
 205         run_cmd.extend(rgw_cmd)
 206
 207         ctx.daemons.add_daemon(
 208             remote, 'rgw', client_with_id,
 209             cluster=cluster_name,
 210             fsid=ctx.ceph[cluster_name].fsid,
 211             args=run_cmd,
 212             logger=log.getChild(client),
 213             stdin=run.PIPE,
 214             wait=False,
 215             )
 216
 217     # XXX: add_daemon() doesn't let us wait until radosgw finishes startup
 218     for client in clients:
 219         endpoint = ctx.rgw.role_endpoints[client]
 220         url = endpoint.url()
 221         log.info('Polling {client} until it starts accepting connections on {url}'.format(client=client, url=url))
 222         (remote,) = ctx.cluster.only(client).remotes.keys()
 223         wait_for_radosgw(url, remote)
 224
 225     try:
 226         yield
 227     finally:
 228         for client in clients:
 229             cluster_name, daemon_type, client_id = teuthology.split_role(client)
 230             client_with_id = daemon_type + '.' + client_id
 231             client_with_cluster = cluster_name + '.' + client_with_id
 232             ctx.daemons.get_daemon('rgw', client_with_id, cluster_name).stop()
 233             ctx.cluster.only(client).run(
 234                 args=[
 235                     'rm',
 236                     '-f',
 237                     '{tdir}/rgw.opslog.{client}.sock'.format(tdir=testdir,
 238                                                              client=client_with_cluster),
 239                     ],
 240                 )
 241             ctx.cluster.only(client).run(args=['sudo', 'rm', '-f', token_path])
 242
 243 def assign_endpoints(ctx, config, default_cert):
 244     role_endpoints = {}
 245     for role, client_config in config.items():
 246         client_config = client_config or {}
 247         remote = get_remote_for_role(ctx, role)
 248
 249         cert = client_config.get('ssl certificate', default_cert)
 250         if cert:
 251             # find the certificate created by the ssl task
 252             if not hasattr(ctx, 'ssl_certificates'):
 253                 raise ConfigError('rgw: no ssl task found for option "ssl certificate"')
 254             ssl_certificate = ctx.ssl_certificates.get(cert, None)
 255             if not ssl_certificate:
 256                 raise ConfigError('rgw: missing ssl certificate "{}"'.format(cert))
 257         else:
 258             ssl_certificate = None
 259
 260         port = client_config.get('port', 443 if ssl_certificate else 80)
 261
 262         # if dns-name is given, use it as the hostname (or as a prefix)
 263         dns_name = client_config.get('dns-name', '')
 264         if len(dns_name) == 0 or dns_name.endswith('.'):
 265             dns_name += remote.hostname
 266
 267         website_dns_name = client_config.get('dns-s3website-name')
 268         if website_dns_name is not None and (len(website_dns_name) == 0 or website_dns_name.endswith('.')):
 269             website_dns_name += remote.hostname
 270
 271         role_endpoints[role] = RGWEndpoint(remote.hostname, port, ssl_certificate, dns_name, website_dns_name)
 272
 273     return role_endpoints
 274
 275 @contextlib.contextmanager
 276 def create_pools(ctx, clients):
 277     """Create replicated or erasure coded data pools for rgw."""
 278
 279     log.info('Creating data pools')
 280     for client in clients:
 281         log.debug("Obtaining remote for client {}".format(client))
 282         (remote,) = ctx.cluster.only(client).remotes.keys()
 283         data_pool = 'default.rgw.buckets.data'
 284         cluster_name, daemon_type, client_id = teuthology.split_role(client)
 285
 286         if ctx.rgw.ec_data_pool:
 287             create_ec_pool(remote, data_pool, client, ctx.rgw.data_pool_pg_size,
 288                            ctx.rgw.erasure_code_profile, cluster_name, 'rgw')
 289         else:
 290             create_replicated_pool(remote, data_pool, ctx.rgw.data_pool_pg_size, cluster_name, 'rgw')
 291
 292         index_pool = 'default.rgw.buckets.index'
 293         create_replicated_pool(remote, index_pool, ctx.rgw.index_pool_pg_size, cluster_name, 'rgw')
 294
 295         if ctx.rgw.cache_pools:
 296             create_cache_pool(remote, data_pool, data_pool + '.cache', 64,
 297                               64*1024*1024, cluster_name)
 298     log.debug('Pools created')
 299     yield
 300
 301 @contextlib.contextmanager
 302 def configure_compression(ctx, clients, compression):
 303     """ set a compression type in the default zone placement """
 304     log.info('Configuring compression type = %s', compression)
 305     for client in clients:
 306         # XXX: the 'default' zone and zonegroup aren't created until we run RGWRados::init_complete().
 307         # issue a 'radosgw-admin user list' command to trigger this
 308         rgwadmin(ctx, client, cmd=['user', 'list'], check_status=True)
 309
 310         rgwadmin(ctx, client,
 311                 cmd=['zone', 'placement', 'modify', '--rgw-zone', 'default',
 312                      '--placement-id', 'default-placement',
 313                      '--compression', compression],
 314                 check_status=True)
 315     yield
 316
 317 @contextlib.contextmanager
 318 def configure_datacache(ctx, clients, datacache_path):
 319     """ create directory for rgw datacache """
 320     log.info('Preparing directory for rgw datacache at %s', datacache_path)
 321     for client in clients:
 322         if(datacache_path != None):
 323             ctx.cluster.only(client).run(args=['mkdir', '-p', datacache_path])
 324             ctx.cluster.only(client).run(args=['sudo', 'chmod', 'a+rwx', datacache_path])
 325         else:
 326             log.info('path for datacache was not provided')
 327     yield
 328
 329 @contextlib.contextmanager
 330 def configure_storage_classes(ctx, clients, storage_classes):
 331     """ set a compression type in the default zone placement """
 332
 333     sc = [s.strip() for s in storage_classes.split(',')]
 334
 335     for client in clients:
 336         # XXX: the 'default' zone and zonegroup aren't created until we run RGWRados::init_complete().
 337         # issue a 'radosgw-admin user list' command to trigger this
 338         rgwadmin(ctx, client, cmd=['user', 'list'], check_status=True)
 339
 340         for storage_class in sc:
 341             log.info('Configuring storage class type = %s', storage_class)
 342             rgwadmin(ctx, client,
 343                     cmd=['zonegroup', 'placement', 'add',
 344                         '--rgw-zone', 'default',
 345                         '--placement-id', 'default-placement',
 346                         '--storage-class', storage_class],
 347                     check_status=True)
 348             rgwadmin(ctx, client,
 349                     cmd=['zone', 'placement', 'add',
 350                         '--rgw-zone', 'default',
 351                         '--placement-id', 'default-placement',
 352                         '--storage-class', storage_class,
 353                         '--data-pool', 'default.rgw.buckets.data.' + storage_class.lower()],
 354                     check_status=True)
 355     yield
 356
 357 @contextlib.contextmanager
 358 def task(ctx, config):
 359     """
 360     For example, to run rgw on all clients::
 361
 362         tasks:
 363         - ceph:
 364         - rgw:
 365
 366     To only run on certain clients::
 367
 368         tasks:
 369         - ceph:
 370         - rgw: [client.0, client.3]
 371
 372     or
 373
 374         tasks:
 375         - ceph:
 376         - rgw:
 377             client.0:
 378             client.3:
 379
 380     To run radosgw through valgrind:
 381
 382         tasks:
 383         - ceph:
 384         - rgw:
 385             client.0:
 386               valgrind: [--tool=memcheck]
 387             client.3:
 388               valgrind: [--tool=memcheck]
 389
 390     To configure data or index pool pg_size:
 391
 392         overrides:
 393           rgw:
 394             data_pool_pg_size: 256
 395             index_pool_pg_size: 128
 396     """
 397     if config is None:
 398         config = dict(('client.{id}'.format(id=id_), None)
 399                       for id_ in teuthology.all_roles_of_type(
 400                           ctx.cluster, 'client'))
 401     elif isinstance(config, list):
 402         config = dict((name, None) for name in config)
 403
 404     clients = config.keys() # http://tracker.ceph.com/issues/20417
 405
 406     overrides = ctx.config.get('overrides', {})
 407     teuthology.deep_merge(config, overrides.get('rgw', {}))
 408
 409     ctx.rgw = argparse.Namespace()
 410
 411     ctx.rgw.ec_data_pool = bool(config.pop('ec-data-pool', False))
 412     ctx.rgw.erasure_code_profile = config.pop('erasure_code_profile', {})
 413     ctx.rgw.cache_pools = bool(config.pop('cache-pools', False))
 414     ctx.rgw.frontend = config.pop('frontend', 'beast')
 415     ctx.rgw.compression_type = config.pop('compression type', None)
 416     ctx.rgw.storage_classes = config.pop('storage classes', None)
 417     default_cert = config.pop('ssl certificate', None)
 418     ctx.rgw.data_pool_pg_size = config.pop('data_pool_pg_size', 64)
 419     ctx.rgw.index_pool_pg_size = config.pop('index_pool_pg_size', 64)
 420     ctx.rgw.datacache = bool(config.pop('datacache', False))
 421     ctx.rgw.datacache_path = config.pop('datacache_path', None)
 422     ctx.rgw.config = config
 423
 424     log.debug("config is {}".format(config))
 425     log.debug("client list is {}".format(clients))
 426
 427     ctx.rgw.role_endpoints = assign_endpoints(ctx, config, default_cert)
 428
 429     subtasks = [
 430         lambda: create_pools(ctx=ctx, clients=clients),
 431     ]
 432     if ctx.rgw.compression_type:
 433         subtasks.extend([
 434             lambda: configure_compression(ctx=ctx, clients=clients,
 435                                           compression=ctx.rgw.compression_type),
 436         ])
 437     if ctx.rgw.datacache:
 438         subtasks.extend([
 439             lambda: configure_datacache(ctx=ctx, clients=clients,
 440                                         datacache_path=ctx.rgw.datacache_path),
 441         ])
 442     if ctx.rgw.storage_classes:
 443         subtasks.extend([
 444             lambda: configure_storage_classes(ctx=ctx, clients=clients,
 445                                               storage_classes=ctx.rgw.storage_classes),
 446         ])
 447     subtasks.extend([
 448         lambda: start_rgw(ctx=ctx, config=config, clients=clients),
 449     ])
 450
 451     with contextutil.nested(*subtasks):
 452         yield