ceph/qa/tasks/s3tests.py

   1 """
   2 Run a set of s3 tests on rgw.
   3 """
   4 from io import BytesIO
   5 from configobj import ConfigObj
   6 import base64
   7 import contextlib
   8 import logging
   9 import os
  10 import random
  11 import string
  12
  13 from teuthology import misc as teuthology
  14 from teuthology import contextutil
  15 from teuthology.config import config as teuth_config
  16 from teuthology.orchestra import run
  17 from teuthology.exceptions import ConfigError
  18
  19 log = logging.getLogger(__name__)
  20
  21 @contextlib.contextmanager
  22 def download(ctx, config):
  23     """
  24     Download the s3 tests from the git builder.
  25     Remove downloaded s3 file upon exit.
  26
  27     The context passed in should be identical to the context
  28     passed in to the main task.
  29     """
  30     assert isinstance(config, dict)
  31     log.info('Downloading s3-tests...')
  32     testdir = teuthology.get_testdir(ctx)
  33     for (client, client_config) in config.items():
  34         s3tests_branch = client_config.get('force-branch', None)
  35         if not s3tests_branch:
  36             raise ValueError(
  37                 "Could not determine what branch to use for s3-tests. Please add 'force-branch: {s3-tests branch name}' to the .yaml config for this s3tests task.")
  38
  39         log.info("Using branch '%s' for s3tests", s3tests_branch)
  40         sha1 = client_config.get('sha1')
  41         git_remote = client_config.get('git_remote', teuth_config.ceph_git_base_url)
  42         ctx.cluster.only(client).run(
  43             args=[
  44                 'git', 'clone',
  45                 '-b', s3tests_branch,
  46                 git_remote + 's3-tests.git',
  47                 '{tdir}/s3-tests'.format(tdir=testdir),
  48                 ],
  49             )
  50         if sha1 is not None:
  51             ctx.cluster.only(client).run(
  52                 args=[
  53                     'cd', '{tdir}/s3-tests'.format(tdir=testdir),
  54                     run.Raw('&&'),
  55                     'git', 'reset', '--hard', sha1,
  56                     ],
  57                 )
  58     try:
  59         yield
  60     finally:
  61         log.info('Removing s3-tests...')
  62         testdir = teuthology.get_testdir(ctx)
  63         for client in config:
  64             ctx.cluster.only(client).run(
  65                 args=[
  66                     'rm',
  67                     '-rf',
  68                     '{tdir}/s3-tests'.format(tdir=testdir),
  69                     ],
  70                 )
  71
  72
  73 def _config_user(s3tests_conf, section, user):
  74     """
  75     Configure users for this section by stashing away keys, ids, and
  76     email addresses.
  77     """
  78     s3tests_conf[section].setdefault('user_id', user)
  79     s3tests_conf[section].setdefault('email', '{user}+test@test.test'.format(user=user))
  80     s3tests_conf[section].setdefault('display_name', 'Mr. {user}'.format(user=user))
  81     s3tests_conf[section].setdefault('access_key',
  82         ''.join(random.choice(string.ascii_uppercase) for i in range(20)))
  83     s3tests_conf[section].setdefault('secret_key',
  84         base64.b64encode(os.urandom(40)).decode())
  85     s3tests_conf[section].setdefault('totp_serial',
  86         ''.join(random.choice(string.digits) for i in range(10)))
  87     s3tests_conf[section].setdefault('totp_seed',
  88         base64.b32encode(os.urandom(40)).decode())
  89     s3tests_conf[section].setdefault('totp_seconds', '5')
  90
  91
  92 @contextlib.contextmanager
  93 def create_users(ctx, config):
  94     """
  95     Create a main and an alternate s3 user.
  96     """
  97     assert isinstance(config, dict)
  98     log.info('Creating rgw users...')
  99     testdir = teuthology.get_testdir(ctx)
 100
 101     if ctx.sts_variable:
 102         users = {'s3 main': 'foo', 's3 alt': 'bar', 's3 tenant': 'testx$tenanteduser', 'iam': 'foobar'}
 103         for client in config['clients']:
 104             s3tests_conf = config['s3tests_conf'][client]
 105             s3tests_conf.setdefault('fixtures', {})
 106             s3tests_conf['fixtures'].setdefault('bucket prefix', 'test-' + client + '-{random}-')
 107             for section, user in users.items():
 108                 _config_user(s3tests_conf, section, '{user}.{client}'.format(user=user, client=client))
 109                 log.debug('Creating user {user} on {host}'.format(user=s3tests_conf[section]['user_id'], host=client))
 110                 cluster_name, daemon_type, client_id = teuthology.split_role(client)
 111                 client_with_id = daemon_type + '.' + client_id
 112                 if section=='iam':
 113                     ctx.cluster.only(client).run(
 114                         args=[
 115                             'adjust-ulimits',
 116                             'ceph-coverage',
 117                             '{tdir}/archive/coverage'.format(tdir=testdir),
 118                             'radosgw-admin',
 119                             '-n', client_with_id,
 120                             'user', 'create',
 121                             '--uid', s3tests_conf[section]['user_id'],
 122                             '--display-name', s3tests_conf[section]['display_name'],
 123                             '--access-key', s3tests_conf[section]['access_key'],
 124                             '--secret', s3tests_conf[section]['secret_key'],
 125                             '--cluster', cluster_name,
 126                         ],
 127                     )
 128                     ctx.cluster.only(client).run(
 129                         args=[
 130                             'adjust-ulimits',
 131                             'ceph-coverage',
 132                             '{tdir}/archive/coverage'.format(tdir=testdir),
 133                             'radosgw-admin',
 134                             '-n', client_with_id,
 135                             'caps', 'add',
 136                             '--uid', s3tests_conf[section]['user_id'],
 137                             '--caps', 'user-policy=*',
 138                             '--cluster', cluster_name,
 139                         ],
 140                     )
 141                     ctx.cluster.only(client).run(
 142                         args=[
 143                             'adjust-ulimits',
 144                             'ceph-coverage',
 145                             '{tdir}/archive/coverage'.format(tdir=testdir),
 146                             'radosgw-admin',
 147                             '-n', client_with_id,
 148                             'caps', 'add',
 149                             '--uid', s3tests_conf[section]['user_id'],
 150                             '--caps', 'roles=*',
 151                             '--cluster', cluster_name,
 152                         ],
 153                     )
 154                     ctx.cluster.only(client).run(
 155                         args=[
 156                             'adjust-ulimits',
 157                             'ceph-coverage',
 158                             '{tdir}/archive/coverage'.format(tdir=testdir),
 159                             'radosgw-admin',
 160                             '-n', client_with_id,
 161                             'caps', 'add',
 162                             '--uid', s3tests_conf[section]['user_id'],
 163                             '--caps', 'oidc-provider=*',
 164                             '--cluster', cluster_name,
 165                         ],
 166                     )
 167
 168                 else:
 169                     ctx.cluster.only(client).run(
 170                         args=[
 171                             'adjust-ulimits',
 172                             'ceph-coverage',
 173                             '{tdir}/archive/coverage'.format(tdir=testdir),
 174                             'radosgw-admin',
 175                             '-n', client_with_id,
 176                             'user', 'create',
 177                             '--uid', s3tests_conf[section]['user_id'],
 178                             '--display-name', s3tests_conf[section]['display_name'],
 179                             '--access-key', s3tests_conf[section]['access_key'],
 180                             '--secret', s3tests_conf[section]['secret_key'],
 181                             '--email', s3tests_conf[section]['email'],
 182                             '--caps', 'user-policy=*',
 183                             '--cluster', cluster_name,
 184                         ],
 185                     )
 186                     ctx.cluster.only(client).run(
 187                         args=[
 188                             'adjust-ulimits',
 189                             'ceph-coverage',
 190                             '{tdir}/archive/coverage'.format(tdir=testdir),
 191                             'radosgw-admin',
 192                             '-n', client_with_id,
 193                             'mfa', 'create',
 194                             '--uid', s3tests_conf[section]['user_id'],
 195                             '--totp-serial', s3tests_conf[section]['totp_serial'],
 196                             '--totp-seed', s3tests_conf[section]['totp_seed'],
 197                             '--totp-seconds', s3tests_conf[section]['totp_seconds'],
 198                             '--totp-window', '8',
 199                             '--totp-seed-type', 'base32',
 200                             '--cluster', cluster_name,
 201                         ],
 202                     )
 203
 204     else:
 205         users = {'s3 main': 'foo', 's3 alt': 'bar', 's3 tenant': 'testx$tenanteduser'}
 206         for client in config['clients']:
 207             s3tests_conf = config['s3tests_conf'][client]
 208             s3tests_conf.setdefault('fixtures', {})
 209             s3tests_conf['fixtures'].setdefault('bucket prefix', 'test-' + client + '-{random}-')
 210             for section, user in users.items():
 211                 _config_user(s3tests_conf, section, '{user}.{client}'.format(user=user, client=client))
 212                 log.debug('Creating user {user} on {host}'.format(user=s3tests_conf[section]['user_id'], host=client))
 213                 cluster_name, daemon_type, client_id = teuthology.split_role(client)
 214                 client_with_id = daemon_type + '.' + client_id
 215                 ctx.cluster.only(client).run(
 216                         args=[
 217                             'adjust-ulimits',
 218                             'ceph-coverage',
 219                             '{tdir}/archive/coverage'.format(tdir=testdir),
 220                             'radosgw-admin',
 221                             '-n', client_with_id,
 222                             'user', 'create',
 223                             '--uid', s3tests_conf[section]['user_id'],
 224                             '--display-name', s3tests_conf[section]['display_name'],
 225                             '--access-key', s3tests_conf[section]['access_key'],
 226                             '--secret', s3tests_conf[section]['secret_key'],
 227                             '--email', s3tests_conf[section]['email'],
 228                             '--caps', 'user-policy=*',
 229                             '--cluster', cluster_name,
 230                         ],
 231                     )
 232                 ctx.cluster.only(client).run(
 233                         args=[
 234                             'adjust-ulimits',
 235                             'ceph-coverage',
 236                             '{tdir}/archive/coverage'.format(tdir=testdir),
 237                             'radosgw-admin',
 238                             '-n', client_with_id,
 239                             'mfa', 'create',
 240                             '--uid', s3tests_conf[section]['user_id'],
 241                             '--totp-serial', s3tests_conf[section]['totp_serial'],
 242                             '--totp-seed', s3tests_conf[section]['totp_seed'],
 243                             '--totp-seconds', s3tests_conf[section]['totp_seconds'],
 244                             '--totp-window', '8',
 245                             '--totp-seed-type', 'base32',
 246                             '--cluster', cluster_name,
 247                         ],
 248                     )
 249
 250     if "TOKEN" in os.environ:
 251         s3tests_conf.setdefault('webidentity', {})
 252         s3tests_conf['webidentity'].setdefault('token',os.environ['TOKEN'])
 253         s3tests_conf['webidentity'].setdefault('aud',os.environ['AUD'])
 254         s3tests_conf['webidentity'].setdefault('sub',os.environ['SUB'])
 255         s3tests_conf['webidentity'].setdefault('azp',os.environ['AZP'])
 256         s3tests_conf['webidentity'].setdefault('user_token',os.environ['USER_TOKEN'])
 257         s3tests_conf['webidentity'].setdefault('thumbprint',os.environ['THUMBPRINT'])
 258         s3tests_conf['webidentity'].setdefault('KC_REALM',os.environ['KC_REALM'])
 259
 260     try:
 261         yield
 262     finally:
 263         for client in config['clients']:
 264             for user in users.values():
 265                 uid = '{user}.{client}'.format(user=user, client=client)
 266                 cluster_name, daemon_type, client_id = teuthology.split_role(client)
 267                 client_with_id = daemon_type + '.' + client_id
 268                 ctx.cluster.only(client).run(
 269                     args=[
 270                         'adjust-ulimits',
 271                         'ceph-coverage',
 272                         '{tdir}/archive/coverage'.format(tdir=testdir),
 273                         'radosgw-admin',
 274                         '-n', client_with_id,
 275                         'user', 'rm',
 276                         '--uid', uid,
 277                         '--purge-data',
 278                         '--cluster', cluster_name,
 279                         ],
 280                     )
 281
 282
 283 @contextlib.contextmanager
 284 def configure(ctx, config):
 285     """
 286     Configure the s3-tests.  This includes the running of the
 287     bootstrap code and the updating of local conf files.
 288     """
 289     assert isinstance(config, dict)
 290     log.info('Configuring s3-tests...')
 291     testdir = teuthology.get_testdir(ctx)
 292     for client, properties in config['clients'].items():
 293         properties = properties or {}
 294         s3tests_conf = config['s3tests_conf'][client]
 295         s3tests_conf['DEFAULT']['calling_format'] = properties.get('calling-format', 'ordinary')
 296
 297         # use rgw_server if given, or default to local client
 298         role = properties.get('rgw_server', client)
 299
 300         endpoint = ctx.rgw.role_endpoints.get(role)
 301         assert endpoint, 's3tests: no rgw endpoint for {}'.format(role)
 302
 303         s3tests_conf['DEFAULT']['host'] = endpoint.dns_name
 304
 305         website_role = properties.get('rgw_website_server')
 306         if website_role:
 307             website_endpoint = ctx.rgw.role_endpoints.get(website_role)
 308             assert website_endpoint, \
 309                     's3tests: no rgw endpoint for rgw_website_server {}'.format(website_role)
 310             assert website_endpoint.website_dns_name, \
 311                     's3tests: no dns-s3website-name for rgw_website_server {}'.format(website_role)
 312             s3tests_conf['DEFAULT']['s3website_domain'] = website_endpoint.website_dns_name
 313
 314         if hasattr(ctx, 'barbican'):
 315             properties = properties['barbican']
 316             if properties is not None and 'kms_key' in properties:
 317                 if not (properties['kms_key'] in ctx.barbican.keys):
 318                     raise ConfigError('Key '+properties['kms_key']+' not defined')
 319
 320                 if not (properties['kms_key2'] in ctx.barbican.keys):
 321                     raise ConfigError('Key '+properties['kms_key2']+' not defined')
 322
 323                 key = ctx.barbican.keys[properties['kms_key']]
 324                 s3tests_conf['DEFAULT']['kms_keyid'] = key['id']
 325
 326                 key = ctx.barbican.keys[properties['kms_key2']]
 327                 s3tests_conf['DEFAULT']['kms_keyid2'] = key['id']
 328
 329         elif hasattr(ctx, 'vault'):
 330             engine_or_flavor = vars(ctx.vault).get('flavor',ctx.vault.engine)
 331             keys=[]
 332             for name in (x['Path'] for x in vars(ctx.vault).get('keys', {}).get(ctx.rgw.vault_role)):
 333                 keys.append(name)
 334
 335             keys.extend(['testkey-1','testkey-2'])
 336             if engine_or_flavor == "old":
 337                 keys=[keys[i] + "/1" for i in range(len(keys))]
 338
 339             properties = properties.get('vault_%s' % engine_or_flavor, {})
 340             s3tests_conf['DEFAULT']['kms_keyid'] = properties.get('key_path', keys[0])
 341             s3tests_conf['DEFAULT']['kms_keyid2'] = properties.get('key_path2', keys[1])
 342         elif hasattr(ctx.rgw, 'pykmip_role'):
 343             keys=[]
 344             for name in (x['Name'] for x in ctx.pykmip.keys[ctx.rgw.pykmip_role]):
 345                 p=name.partition('-')
 346                 keys.append(p[2] if p[2] else p[0])
 347             keys.extend(['testkey-1', 'testkey-2'])
 348             s3tests_conf['DEFAULT']['kms_keyid'] = properties.get('kms_key', keys[0])
 349             s3tests_conf['DEFAULT']['kms_keyid2'] = properties.get('kms_key2', keys[1])
 350         else:
 351             # Fallback scenario where it's the local (ceph.conf) kms being tested
 352             s3tests_conf['DEFAULT']['kms_keyid'] = 'testkey-1'
 353             s3tests_conf['DEFAULT']['kms_keyid2'] = 'testkey-2'
 354
 355         slow_backend = properties.get('slow_backend')
 356         if slow_backend:
 357             s3tests_conf['fixtures']['slow backend'] = slow_backend
 358
 359         (remote,) = ctx.cluster.only(client).remotes.keys()
 360         remote.run(
 361             args=[
 362                 'cd',
 363                 '{tdir}/s3-tests'.format(tdir=testdir),
 364                 run.Raw('&&'),
 365                 './bootstrap',
 366                 ],
 367             )
 368         conf_fp = BytesIO()
 369         s3tests_conf.write(conf_fp)
 370         remote.write_file(
 371             path='{tdir}/archive/s3-tests.{client}.conf'.format(tdir=testdir, client=client),
 372             data=conf_fp.getvalue(),
 373             )
 374
 375     log.info('Configuring boto...')
 376     boto_src = os.path.join(os.path.dirname(__file__), 'boto.cfg.template')
 377     for client, properties in config['clients'].items():
 378         with open(boto_src) as f:
 379             (remote,) = ctx.cluster.only(client).remotes.keys()
 380             conf = f.read().format(
 381                 idle_timeout=config.get('idle_timeout', 30)
 382                 )
 383             remote.write_file('{tdir}/boto.cfg'.format(tdir=testdir), conf)
 384
 385     try:
 386         yield
 387
 388     finally:
 389         log.info('Cleaning up boto...')
 390         for client, properties in config['clients'].items():
 391             (remote,) = ctx.cluster.only(client).remotes.keys()
 392             remote.run(
 393                 args=[
 394                     'rm',
 395                     '{tdir}/boto.cfg'.format(tdir=testdir),
 396                     ],
 397                 )
 398
 399 @contextlib.contextmanager
 400 def run_tests(ctx, config):
 401     """
 402     Run the s3tests after everything is set up.
 403
 404     :param ctx: Context passed to task
 405     :param config: specific configuration information
 406     """
 407     assert isinstance(config, dict)
 408     testdir = teuthology.get_testdir(ctx)
 409     for client, client_config in config.items():
 410         client_config = client_config or {}
 411         (remote,) = ctx.cluster.only(client).remotes.keys()
 412         args = [
 413             'S3TEST_CONF={tdir}/archive/s3-tests.{client}.conf'.format(tdir=testdir, client=client),
 414             'BOTO_CONFIG={tdir}/boto.cfg'.format(tdir=testdir)
 415             ]
 416         # the 'requests' library comes with its own ca bundle to verify ssl
 417         # certificates - override that to use the system's ca bundle, which
 418         # is where the ssl task installed this certificate
 419         if remote.os.package_type == 'deb':
 420             args += ['REQUESTS_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt']
 421         else:
 422             args += ['REQUESTS_CA_BUNDLE=/etc/pki/tls/certs/ca-bundle.crt']
 423         # civetweb > 1.8 && beast parsers are strict on rfc2616
 424         attrs = ["!fails_on_rgw", "!lifecycle_expiration", "!fails_strict_rfc2616","!test_of_sts","!webidentity_test"]
 425         if client_config.get('calling-format') != 'ordinary':
 426             attrs += ['!fails_with_subdomain']
 427
 428         if 'extra_attrs' in client_config:
 429             attrs = client_config.get('extra_attrs')
 430         args += [
 431             '{tdir}/s3-tests/virtualenv/bin/python'.format(tdir=testdir),
 432             '-m', 'nose',
 433             '-w',
 434             '{tdir}/s3-tests'.format(tdir=testdir),
 435             '-v',
 436             '-a', ','.join(attrs),
 437             ]
 438         if 'extra_args' in client_config:
 439             args.append(client_config['extra_args'])
 440
 441         remote.run(
 442             args=args,
 443             label="s3 tests against rgw"
 444             )
 445     yield
 446
 447 @contextlib.contextmanager
 448 def scan_for_leaked_encryption_keys(ctx, config):
 449     """
 450     Scan radosgw logs for the encryption keys used by s3tests to
 451     verify that we're not leaking secrets.
 452
 453     :param ctx: Context passed to task
 454     :param config: specific configuration information
 455     """
 456     assert isinstance(config, dict)
 457
 458     try:
 459         yield
 460     finally:
 461         # x-amz-server-side-encryption-customer-key
 462         s3test_customer_key = 'pO3upElrwuEXSoFwCfnZPdSsmt/xWeFa0N9KgDijwVs='
 463
 464         log.debug('Scanning radosgw logs for leaked encryption keys...')
 465         procs = list()
 466         for client, client_config in config.items():
 467             if not client_config.get('scan_for_encryption_keys', True):
 468                 continue
 469             cluster_name, daemon_type, client_id = teuthology.split_role(client)
 470             client_with_cluster = '.'.join((cluster_name, daemon_type, client_id))
 471             (remote,) = ctx.cluster.only(client).remotes.keys()
 472             proc = remote.run(
 473                 args=[
 474                     'grep',
 475                     '--binary-files=text',
 476                     s3test_customer_key,
 477                     '/var/log/ceph/rgw.{client}.log'.format(client=client_with_cluster),
 478                 ],
 479                 wait=False,
 480                 check_status=False,
 481             )
 482             procs.append(proc)
 483
 484         for proc in procs:
 485             proc.wait()
 486             if proc.returncode == 1: # 1 means no matches
 487                 continue
 488             log.error('radosgw log is leaking encryption keys!')
 489             raise Exception('radosgw log is leaking encryption keys')
 490
 491 @contextlib.contextmanager
 492 def task(ctx, config):
 493     """
 494     Run the s3-tests suite against rgw.
 495
 496     To run all tests on all clients::
 497
 498         tasks:
 499         - ceph:
 500         - rgw:
 501         - s3tests:
 502
 503     To restrict testing to particular clients::
 504
 505         tasks:
 506         - ceph:
 507         - rgw: [client.0]
 508         - s3tests: [client.0]
 509
 510     To run against a server on client.1 and increase the boto timeout to 10m::
 511
 512         tasks:
 513         - ceph:
 514         - rgw: [client.1]
 515         - s3tests:
 516             client.0:
 517               rgw_server: client.1
 518               idle_timeout: 600
 519
 520     To pass extra arguments to nose (e.g. to run a certain test)::
 521
 522         tasks:
 523         - ceph:
 524         - rgw: [client.0]
 525         - s3tests:
 526             client.0:
 527               extra_args: ['test_s3:test_object_acl_grand_public_read']
 528             client.1:
 529               extra_args: ['--exclude', 'test_100_continue']
 530
 531     To run any sts-tests don't forget to set a config variable named 'sts_tests' to 'True' as follows::
 532
 533         tasks:
 534         - ceph:
 535         - rgw: [client.0]
 536         - s3tests:
 537             client.0:
 538               sts_tests: True
 539               rgw_server: client.0
 540
 541     """
 542     assert hasattr(ctx, 'rgw'), 's3tests must run after the rgw task'
 543     assert config is None or isinstance(config, list) \
 544         or isinstance(config, dict), \
 545         "task s3tests only supports a list or dictionary for configuration"
 546     all_clients = ['client.{id}'.format(id=id_)
 547                    for id_ in teuthology.all_roles_of_type(ctx.cluster, 'client')]
 548     if config is None:
 549         config = all_clients
 550     if isinstance(config, list):
 551         config = dict.fromkeys(config)
 552     clients = config.keys()
 553
 554     overrides = ctx.config.get('overrides', {})
 555     # merge each client section, not the top level.
 556     for client in config.keys():
 557         if not config[client]:
 558             config[client] = {}
 559         teuthology.deep_merge(config[client], overrides.get('s3tests', {}))
 560
 561     log.debug('s3tests config is %s', config)
 562
 563     s3tests_conf = {}
 564
 565     for client, client_config in config.items():
 566         if 'sts_tests' in client_config:
 567             ctx.sts_variable = True
 568         else:
 569             ctx.sts_variable = False
 570         #This will be the structure of config file when you want to run webidentity_test (sts-test)
 571         if ctx.sts_variable and "TOKEN" in os.environ:
 572             for client in clients:
 573                 endpoint = ctx.rgw.role_endpoints.get(client)
 574                 assert endpoint, 's3tests: no rgw endpoint for {}'.format(client)
 575
 576                 s3tests_conf[client] = ConfigObj(
 577                     indent_type='',
 578                     infile={
 579                         'DEFAULT':
 580                             {
 581                             'port'      : endpoint.port,
 582                             'is_secure' : endpoint.cert is not None,
 583                             'api_name'  : 'default',
 584                             },
 585                         'fixtures'   : {},
 586                         's3 main'    : {},
 587                         's3 alt'     : {},
 588                         's3 tenant'  : {},
 589                         'iam'        : {},
 590                         'webidentity': {},
 591                     }
 592                 )
 593
 594         elif ctx.sts_variable:
 595             #This will be the structure of config file when you want to run assume_role_test and get_session_token_test (sts-test)
 596             for client in clients:
 597                 endpoint = ctx.rgw.role_endpoints.get(client)
 598                 assert endpoint, 's3tests: no rgw endpoint for {}'.format(client)
 599
 600                 s3tests_conf[client] = ConfigObj(
 601                     indent_type='',
 602                     infile={
 603                         'DEFAULT':
 604                             {
 605                             'port'      : endpoint.port,
 606                             'is_secure' : endpoint.cert is not None,
 607                             'api_name'  : 'default',
 608                             },
 609                         'fixtures'   : {},
 610                         's3 main'    : {},
 611                         's3 alt'     : {},
 612                         's3 tenant'  : {},
 613                         'iam'        : {},
 614                         }
 615                     )
 616
 617         else:
 618             #This will be the structure of config file when you want to run normal s3-tests
 619             for client in clients:
 620                 endpoint = ctx.rgw.role_endpoints.get(client)
 621                 assert endpoint, 's3tests: no rgw endpoint for {}'.format(client)
 622
 623                 s3tests_conf[client] = ConfigObj(
 624                     indent_type='',
 625                     infile={
 626                         'DEFAULT':
 627                             {
 628                             'port'      : endpoint.port,
 629                             'is_secure' : endpoint.cert is not None,
 630                             'api_name'  : 'default',
 631                             },
 632                         'fixtures'   : {},
 633                         's3 main'    : {},
 634                         's3 alt'     : {},
 635                         's3 tenant'  : {},
 636                         }
 637                     )
 638
 639     with contextutil.nested(
 640         lambda: download(ctx=ctx, config=config),
 641         lambda: create_users(ctx=ctx, config=dict(
 642                 clients=clients,
 643                 s3tests_conf=s3tests_conf,
 644                 )),
 645         lambda: configure(ctx=ctx, config=dict(
 646                 clients=config,
 647                 s3tests_conf=s3tests_conf,
 648                 )),
 649         lambda: run_tests(ctx=ctx, config=config),
 650         lambda: scan_for_leaked_encryption_keys(ctx=ctx, config=config),
 651         ):
 652         pass
 653     yield