ceph/qa/tasks/s3tests.py

   1 """
   2 Run a set of s3 tests on rgw.
   3 """
   4 from io import BytesIO
   5 from configobj import ConfigObj
   6 import base64
   7 import contextlib
   8 import logging
   9 import os
  10 import random
  11 import string
  12
  13 from teuthology import misc as teuthology
  14 from teuthology import contextutil
  15 from teuthology.config import config as teuth_config
  16 from teuthology.orchestra import run
  17 from teuthology.exceptions import ConfigError
  18
  19 log = logging.getLogger(__name__)
  20
  21 @contextlib.contextmanager
  22 def download(ctx, config):
  23     """
  24     Download the s3 tests from the git builder.
  25     Remove downloaded s3 file upon exit.
  26
  27     The context passed in should be identical to the context
  28     passed in to the main task.
  29     """
  30     assert isinstance(config, dict)
  31     log.info('Downloading s3-tests...')
  32     testdir = teuthology.get_testdir(ctx)
  33     for (client, client_config) in config.items():
  34         s3tests_branch = client_config.get('force-branch', None)
  35         if not s3tests_branch:
  36             raise ValueError(
  37                 "Could not determine what branch to use for s3-tests. Please add 'force-branch: {s3-tests branch name}' to the .yaml config for this s3tests task.")
  38
  39         log.info("Using branch '%s' for s3tests", s3tests_branch)
  40         sha1 = client_config.get('sha1')
  41         git_remote = client_config.get('git_remote', teuth_config.ceph_git_base_url)
  42         ctx.cluster.only(client).run(
  43             args=[
  44                 'git', 'clone',
  45                 '-b', s3tests_branch,
  46                 git_remote + 's3-tests.git',
  47                 '{tdir}/s3-tests'.format(tdir=testdir),
  48                 ],
  49             )
  50         if sha1 is not None:
  51             ctx.cluster.only(client).run(
  52                 args=[
  53                     'cd', '{tdir}/s3-tests'.format(tdir=testdir),
  54                     run.Raw('&&'),
  55                     'git', 'reset', '--hard', sha1,
  56                     ],
  57                 )
  58     try:
  59         yield
  60     finally:
  61         log.info('Removing s3-tests...')
  62         testdir = teuthology.get_testdir(ctx)
  63         for client in config:
  64             ctx.cluster.only(client).run(
  65                 args=[
  66                     'rm',
  67                     '-rf',
  68                     '{tdir}/s3-tests'.format(tdir=testdir),
  69                     ],
  70                 )
  71
  72
  73 def _config_user(s3tests_conf, section, user):
  74     """
  75     Configure users for this section by stashing away keys, ids, and
  76     email addresses.
  77     """
  78     s3tests_conf[section].setdefault('user_id', user)
  79     s3tests_conf[section].setdefault('email', '{user}+test@test.test'.format(user=user))
  80     s3tests_conf[section].setdefault('display_name', 'Mr. {user}'.format(user=user))
  81     s3tests_conf[section].setdefault('access_key',
  82         ''.join(random.choice(string.ascii_uppercase) for i in range(20)))
  83     s3tests_conf[section].setdefault('secret_key',
  84         base64.b64encode(os.urandom(40)).decode())
  85     s3tests_conf[section].setdefault('totp_serial',
  86         ''.join(random.choice(string.digits) for i in range(10)))
  87     s3tests_conf[section].setdefault('totp_seed',
  88         base64.b32encode(os.urandom(40)).decode())
  89     s3tests_conf[section].setdefault('totp_seconds', '5')
  90
  91
  92 @contextlib.contextmanager
  93 def create_users(ctx, config):
  94     """
  95     Create a main and an alternate s3 user.
  96     """
  97     assert isinstance(config, dict)
  98     log.info('Creating rgw users...')
  99     testdir = teuthology.get_testdir(ctx)
 100
 101     if ctx.sts_variable:
 102         users = {'s3 main': 'foo', 's3 alt': 'bar', 's3 tenant': 'testx$tenanteduser', 'iam': 'foobar'}
 103         for client in config['clients']:
 104             s3tests_conf = config['s3tests_conf'][client]
 105             s3tests_conf.setdefault('fixtures', {})
 106             s3tests_conf['fixtures'].setdefault('bucket prefix', 'test-' + client + '-{random}-')
 107             for section, user in users.items():
 108                 _config_user(s3tests_conf, section, '{user}.{client}'.format(user=user, client=client))
 109                 log.debug('Creating user {user} on {host}'.format(user=s3tests_conf[section]['user_id'], host=client))
 110                 cluster_name, daemon_type, client_id = teuthology.split_role(client)
 111                 client_with_id = daemon_type + '.' + client_id
 112                 if section=='iam':
 113                     ctx.cluster.only(client).run(
 114                         args=[
 115                             'adjust-ulimits',
 116                             'ceph-coverage',
 117                             '{tdir}/archive/coverage'.format(tdir=testdir),
 118                             'radosgw-admin',
 119                             '-n', client_with_id,
 120                             'user', 'create',
 121                             '--uid', s3tests_conf[section]['user_id'],
 122                             '--display-name', s3tests_conf[section]['display_name'],
 123                             '--access-key', s3tests_conf[section]['access_key'],
 124                             '--secret', s3tests_conf[section]['secret_key'],
 125                             '--cluster', cluster_name,
 126                         ],
 127                     )
 128                     ctx.cluster.only(client).run(
 129                         args=[
 130                             'adjust-ulimits',
 131                             'ceph-coverage',
 132                             '{tdir}/archive/coverage'.format(tdir=testdir),
 133                             'radosgw-admin',
 134                             '-n', client_with_id,
 135                             'caps', 'add',
 136                             '--uid', s3tests_conf[section]['user_id'],
 137                             '--caps', 'user-policy=*',
 138                             '--cluster', cluster_name,
 139                         ],
 140                     )
 141                     ctx.cluster.only(client).run(
 142                         args=[
 143                             'adjust-ulimits',
 144                             'ceph-coverage',
 145                             '{tdir}/archive/coverage'.format(tdir=testdir),
 146                             'radosgw-admin',
 147                             '-n', client_with_id,
 148                             'caps', 'add',
 149                             '--uid', s3tests_conf[section]['user_id'],
 150                             '--caps', 'roles=*',
 151                             '--cluster', cluster_name,
 152                         ],
 153                     )
 154                     ctx.cluster.only(client).run(
 155                         args=[
 156                             'adjust-ulimits',
 157                             'ceph-coverage',
 158                             '{tdir}/archive/coverage'.format(tdir=testdir),
 159                             'radosgw-admin',
 160                             '-n', client_with_id,
 161                             'caps', 'add',
 162                             '--uid', s3tests_conf[section]['user_id'],
 163                             '--caps', 'oidc-provider=*',
 164                             '--cluster', cluster_name,
 165                         ],
 166                     )
 167
 168                 else:
 169                     ctx.cluster.only(client).run(
 170                         args=[
 171                             'adjust-ulimits',
 172                             'ceph-coverage',
 173                             '{tdir}/archive/coverage'.format(tdir=testdir),
 174                             'radosgw-admin',
 175                             '-n', client_with_id,
 176                             'user', 'create',
 177                             '--uid', s3tests_conf[section]['user_id'],
 178                             '--display-name', s3tests_conf[section]['display_name'],
 179                             '--access-key', s3tests_conf[section]['access_key'],
 180                             '--secret', s3tests_conf[section]['secret_key'],
 181                             '--email', s3tests_conf[section]['email'],
 182                             '--caps', 'user-policy=*',
 183                             '--cluster', cluster_name,
 184                         ],
 185                     )
 186                     ctx.cluster.only(client).run(
 187                         args=[
 188                             'adjust-ulimits',
 189                             'ceph-coverage',
 190                             '{tdir}/archive/coverage'.format(tdir=testdir),
 191                             'radosgw-admin',
 192                             '-n', client_with_id,
 193                             'mfa', 'create',
 194                             '--uid', s3tests_conf[section]['user_id'],
 195                             '--totp-serial', s3tests_conf[section]['totp_serial'],
 196                             '--totp-seed', s3tests_conf[section]['totp_seed'],
 197                             '--totp-seconds', s3tests_conf[section]['totp_seconds'],
 198                             '--totp-window', '8',
 199                             '--totp-seed-type', 'base32',
 200                             '--cluster', cluster_name,
 201                         ],
 202                     )
 203
 204     else:
 205         users = {'s3 main': 'foo', 's3 alt': 'bar', 's3 tenant': 'testx$tenanteduser'}
 206         for client in config['clients']:
 207             s3tests_conf = config['s3tests_conf'][client]
 208             s3tests_conf.setdefault('fixtures', {})
 209             s3tests_conf['fixtures'].setdefault('bucket prefix', 'test-' + client + '-{random}-')
 210             for section, user in users.items():
 211                 _config_user(s3tests_conf, section, '{user}.{client}'.format(user=user, client=client))
 212                 log.debug('Creating user {user} on {host}'.format(user=s3tests_conf[section]['user_id'], host=client))
 213                 cluster_name, daemon_type, client_id = teuthology.split_role(client)
 214                 client_with_id = daemon_type + '.' + client_id
 215                 ctx.cluster.only(client).run(
 216                         args=[
 217                             'adjust-ulimits',
 218                             'ceph-coverage',
 219                             '{tdir}/archive/coverage'.format(tdir=testdir),
 220                             'radosgw-admin',
 221                             '-n', client_with_id,
 222                             'user', 'create',
 223                             '--uid', s3tests_conf[section]['user_id'],
 224                             '--display-name', s3tests_conf[section]['display_name'],
 225                             '--access-key', s3tests_conf[section]['access_key'],
 226                             '--secret', s3tests_conf[section]['secret_key'],
 227                             '--email', s3tests_conf[section]['email'],
 228                             '--caps', 'user-policy=*',
 229                             '--cluster', cluster_name,
 230                         ],
 231                     )
 232                 ctx.cluster.only(client).run(
 233                         args=[
 234                             'adjust-ulimits',
 235                             'ceph-coverage',
 236                             '{tdir}/archive/coverage'.format(tdir=testdir),
 237                             'radosgw-admin',
 238                             '-n', client_with_id,
 239                             'mfa', 'create',
 240                             '--uid', s3tests_conf[section]['user_id'],
 241                             '--totp-serial', s3tests_conf[section]['totp_serial'],
 242                             '--totp-seed', s3tests_conf[section]['totp_seed'],
 243                             '--totp-seconds', s3tests_conf[section]['totp_seconds'],
 244                             '--totp-window', '8',
 245                             '--totp-seed-type', 'base32',
 246                             '--cluster', cluster_name,
 247                         ],
 248                     )
 249
 250     if "TOKEN" in os.environ:
 251         s3tests_conf.setdefault('webidentity', {})
 252         s3tests_conf['webidentity'].setdefault('token',os.environ['TOKEN'])
 253         s3tests_conf['webidentity'].setdefault('aud',os.environ['AUD'])
 254         s3tests_conf['webidentity'].setdefault('sub',os.environ['SUB'])
 255         s3tests_conf['webidentity'].setdefault('azp',os.environ['AZP'])
 256         s3tests_conf['webidentity'].setdefault('user_token',os.environ['USER_TOKEN'])
 257         s3tests_conf['webidentity'].setdefault('thumbprint',os.environ['THUMBPRINT'])
 258         s3tests_conf['webidentity'].setdefault('KC_REALM',os.environ['KC_REALM'])
 259
 260     try:
 261         yield
 262     finally:
 263         for client in config['clients']:
 264             for user in users.values():
 265                 uid = '{user}.{client}'.format(user=user, client=client)
 266                 cluster_name, daemon_type, client_id = teuthology.split_role(client)
 267                 client_with_id = daemon_type + '.' + client_id
 268                 ctx.cluster.only(client).run(
 269                     args=[
 270                         'adjust-ulimits',
 271                         'ceph-coverage',
 272                         '{tdir}/archive/coverage'.format(tdir=testdir),
 273                         'radosgw-admin',
 274                         '-n', client_with_id,
 275                         'user', 'rm',
 276                         '--uid', uid,
 277                         '--purge-data',
 278                         '--cluster', cluster_name,
 279                         ],
 280                     )
 281
 282
 283 @contextlib.contextmanager
 284 def configure(ctx, config):
 285     """
 286     Configure the s3-tests.  This includes the running of the
 287     bootstrap code and the updating of local conf files.
 288     """
 289     assert isinstance(config, dict)
 290     log.info('Configuring s3-tests...')
 291     testdir = teuthology.get_testdir(ctx)
 292     for client, properties in config['clients'].items():
 293         properties = properties or {}
 294         s3tests_conf = config['s3tests_conf'][client]
 295         s3tests_conf['DEFAULT']['calling_format'] = properties.get('calling-format', 'ordinary')
 296
 297         # use rgw_server if given, or default to local client
 298         role = properties.get('rgw_server', client)
 299
 300         endpoint = ctx.rgw.role_endpoints.get(role)
 301         assert endpoint, 's3tests: no rgw endpoint for {}'.format(role)
 302
 303         s3tests_conf['DEFAULT']['host'] = endpoint.dns_name
 304
 305         website_role = properties.get('rgw_website_server')
 306         if website_role:
 307             website_endpoint = ctx.rgw.role_endpoints.get(website_role)
 308             assert website_endpoint, \
 309                     's3tests: no rgw endpoint for rgw_website_server {}'.format(website_role)
 310             assert website_endpoint.website_dns_name, \
 311                     's3tests: no dns-s3website-name for rgw_website_server {}'.format(website_role)
 312             s3tests_conf['DEFAULT']['s3website_domain'] = website_endpoint.website_dns_name
 313
 314         if hasattr(ctx, 'barbican'):
 315             properties = properties['barbican']
 316             if properties is not None and 'kms_key' in properties:
 317                 if not (properties['kms_key'] in ctx.barbican.keys):
 318                     raise ConfigError('Key '+properties['kms_key']+' not defined')
 319
 320                 if not (properties['kms_key2'] in ctx.barbican.keys):
 321                     raise ConfigError('Key '+properties['kms_key2']+' not defined')
 322
 323                 key = ctx.barbican.keys[properties['kms_key']]
 324                 s3tests_conf['DEFAULT']['kms_keyid'] = key['id']
 325
 326                 key = ctx.barbican.keys[properties['kms_key2']]
 327                 s3tests_conf['DEFAULT']['kms_keyid2'] = key['id']
 328
 329         elif hasattr(ctx, 'vault'):
 330             engine_or_flavor = vars(ctx.vault).get('flavor',ctx.vault.engine)
 331             keys=[]
 332             for name in (x['Path'] for x in vars(ctx.vault).get('keys', {}).get(ctx.rgw.vault_role)):
 333                 keys.append(name)
 334
 335             keys.extend(['testkey-1','testkey-2'])
 336             if engine_or_flavor == "old":
 337                 keys=[keys[i] + "/1" for i in range(len(keys))]
 338
 339             properties = properties.get('vault_%s' % engine_or_flavor, {})
 340             s3tests_conf['DEFAULT']['kms_keyid'] = properties.get('key_path', keys[0])
 341             s3tests_conf['DEFAULT']['kms_keyid2'] = properties.get('key_path2', keys[1])
 342         elif hasattr(ctx.rgw, 'pykmip_role'):
 343             keys=[]
 344             for name in (x['Name'] for x in ctx.pykmip.keys[ctx.rgw.pykmip_role]):
 345                 p=name.partition('-')
 346                 keys.append(p[2] if p[2] else p[0])
 347             keys.extend(['testkey-1', 'testkey-2'])
 348             s3tests_conf['DEFAULT']['kms_keyid'] = properties.get('kms_key', keys[0])
 349             s3tests_conf['DEFAULT']['kms_keyid2'] = properties.get('kms_key2', keys[1])
 350         else:
 351             # Fallback scenario where it's the local (ceph.conf) kms being tested
 352             s3tests_conf['DEFAULT']['kms_keyid'] = 'testkey-1'
 353             s3tests_conf['DEFAULT']['kms_keyid2'] = 'testkey-2'
 354
 355         slow_backend = properties.get('slow_backend')
 356         if slow_backend:
 357             s3tests_conf['fixtures']['slow backend'] = slow_backend
 358
 359         storage_classes = properties.get('storage classes')
 360         if storage_classes:
 361             s3tests_conf['s3 main']['storage_classes'] = storage_classes
 362
 363         lc_debug_interval = properties.get('lc_debug_interval')
 364         if lc_debug_interval:
 365             s3tests_conf['s3 main']['lc_debug_interval'] = lc_debug_interval
 366
 367         (remote,) = ctx.cluster.only(client).remotes.keys()
 368         remote.run(
 369             args=[
 370                 'cd',
 371                 '{tdir}/s3-tests'.format(tdir=testdir),
 372                 run.Raw('&&'),
 373                 './bootstrap',
 374                 ],
 375             )
 376         conf_fp = BytesIO()
 377         s3tests_conf.write(conf_fp)
 378         remote.write_file(
 379             path='{tdir}/archive/s3-tests.{client}.conf'.format(tdir=testdir, client=client),
 380             data=conf_fp.getvalue(),
 381             )
 382
 383     log.info('Configuring boto...')
 384     boto_src = os.path.join(os.path.dirname(__file__), 'boto.cfg.template')
 385     for client, properties in config['clients'].items():
 386         with open(boto_src) as f:
 387             (remote,) = ctx.cluster.only(client).remotes.keys()
 388             conf = f.read().format(
 389                 idle_timeout=config.get('idle_timeout', 30)
 390                 )
 391             remote.write_file('{tdir}/boto.cfg'.format(tdir=testdir), conf)
 392
 393     try:
 394         yield
 395
 396     finally:
 397         log.info('Cleaning up boto...')
 398         for client, properties in config['clients'].items():
 399             (remote,) = ctx.cluster.only(client).remotes.keys()
 400             remote.run(
 401                 args=[
 402                     'rm',
 403                     '{tdir}/boto.cfg'.format(tdir=testdir),
 404                     ],
 405                 )
 406
 407 @contextlib.contextmanager
 408 def run_tests(ctx, config):
 409     """
 410     Run the s3tests after everything is set up.
 411
 412     :param ctx: Context passed to task
 413     :param config: specific configuration information
 414     """
 415     assert isinstance(config, dict)
 416     testdir = teuthology.get_testdir(ctx)
 417     for client, client_config in config.items():
 418         client_config = client_config or {}
 419         (remote,) = ctx.cluster.only(client).remotes.keys()
 420         args = [
 421             'S3TEST_CONF={tdir}/archive/s3-tests.{client}.conf'.format(tdir=testdir, client=client),
 422             'BOTO_CONFIG={tdir}/boto.cfg'.format(tdir=testdir)
 423             ]
 424         # the 'requests' library comes with its own ca bundle to verify ssl
 425         # certificates - override that to use the system's ca bundle, which
 426         # is where the ssl task installed this certificate
 427         if remote.os.package_type == 'deb':
 428             args += ['REQUESTS_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt']
 429         else:
 430             args += ['REQUESTS_CA_BUNDLE=/etc/pki/tls/certs/ca-bundle.crt']
 431         # civetweb > 1.8 && beast parsers are strict on rfc2616
 432         attrs = ["!fails_on_rgw", "!lifecycle_expiration", "!fails_strict_rfc2616","!test_of_sts","!webidentity_test"]
 433         if client_config.get('calling-format') != 'ordinary':
 434             attrs += ['!fails_with_subdomain']
 435
 436         if 'extra_attrs' in client_config:
 437             attrs = client_config.get('extra_attrs')
 438         args += [
 439             '{tdir}/s3-tests/virtualenv/bin/python'.format(tdir=testdir),
 440             '-m', 'nose',
 441             '-w',
 442             '{tdir}/s3-tests'.format(tdir=testdir),
 443             '-v',
 444             '-a', ','.join(attrs),
 445             ]
 446         if 'extra_args' in client_config:
 447             args.append(client_config['extra_args'])
 448
 449         remote.run(
 450             args=args,
 451             label="s3 tests against rgw"
 452             )
 453     yield
 454
 455 @contextlib.contextmanager
 456 def scan_for_leaked_encryption_keys(ctx, config):
 457     """
 458     Scan radosgw logs for the encryption keys used by s3tests to
 459     verify that we're not leaking secrets.
 460
 461     :param ctx: Context passed to task
 462     :param config: specific configuration information
 463     """
 464     assert isinstance(config, dict)
 465
 466     try:
 467         yield
 468     finally:
 469         # x-amz-server-side-encryption-customer-key
 470         s3test_customer_key = 'pO3upElrwuEXSoFwCfnZPdSsmt/xWeFa0N9KgDijwVs='
 471
 472         log.debug('Scanning radosgw logs for leaked encryption keys...')
 473         procs = list()
 474         for client, client_config in config.items():
 475             if not client_config.get('scan_for_encryption_keys', True):
 476                 continue
 477             cluster_name, daemon_type, client_id = teuthology.split_role(client)
 478             client_with_cluster = '.'.join((cluster_name, daemon_type, client_id))
 479             (remote,) = ctx.cluster.only(client).remotes.keys()
 480             proc = remote.run(
 481                 args=[
 482                     'grep',
 483                     '--binary-files=text',
 484                     s3test_customer_key,
 485                     '/var/log/ceph/rgw.{client}.log'.format(client=client_with_cluster),
 486                 ],
 487                 wait=False,
 488                 check_status=False,
 489             )
 490             procs.append(proc)
 491
 492         for proc in procs:
 493             proc.wait()
 494             if proc.returncode == 1: # 1 means no matches
 495                 continue
 496             log.error('radosgw log is leaking encryption keys!')
 497             raise Exception('radosgw log is leaking encryption keys')
 498
 499 @contextlib.contextmanager
 500 def task(ctx, config):
 501     """
 502     Run the s3-tests suite against rgw.
 503
 504     To run all tests on all clients::
 505
 506         tasks:
 507         - ceph:
 508         - rgw:
 509         - s3tests:
 510
 511     To restrict testing to particular clients::
 512
 513         tasks:
 514         - ceph:
 515         - rgw: [client.0]
 516         - s3tests: [client.0]
 517
 518     To run against a server on client.1 and increase the boto timeout to 10m::
 519
 520         tasks:
 521         - ceph:
 522         - rgw: [client.1]
 523         - s3tests:
 524             client.0:
 525               rgw_server: client.1
 526               idle_timeout: 600
 527
 528     To pass extra arguments to nose (e.g. to run a certain test)::
 529
 530         tasks:
 531         - ceph:
 532         - rgw: [client.0]
 533         - s3tests:
 534             client.0:
 535               extra_args: ['test_s3:test_object_acl_grand_public_read']
 536             client.1:
 537               extra_args: ['--exclude', 'test_100_continue']
 538
 539     To run any sts-tests don't forget to set a config variable named 'sts_tests' to 'True' as follows::
 540
 541         tasks:
 542         - ceph:
 543         - rgw: [client.0]
 544         - s3tests:
 545             client.0:
 546               sts_tests: True
 547               rgw_server: client.0
 548
 549     """
 550     assert hasattr(ctx, 'rgw'), 's3tests must run after the rgw task'
 551     assert config is None or isinstance(config, list) \
 552         or isinstance(config, dict), \
 553         "task s3tests only supports a list or dictionary for configuration"
 554     all_clients = ['client.{id}'.format(id=id_)
 555                    for id_ in teuthology.all_roles_of_type(ctx.cluster, 'client')]
 556     if config is None:
 557         config = all_clients
 558     if isinstance(config, list):
 559         config = dict.fromkeys(config)
 560     clients = config.keys()
 561
 562     overrides = ctx.config.get('overrides', {})
 563     # merge each client section, not the top level.
 564     for client in config.keys():
 565         if not config[client]:
 566             config[client] = {}
 567         teuthology.deep_merge(config[client], overrides.get('s3tests', {}))
 568
 569     log.debug('s3tests config is %s', config)
 570
 571     s3tests_conf = {}
 572
 573     for client, client_config in config.items():
 574         if 'sts_tests' in client_config:
 575             ctx.sts_variable = True
 576         else:
 577             ctx.sts_variable = False
 578         #This will be the structure of config file when you want to run webidentity_test (sts-test)
 579         if ctx.sts_variable and "TOKEN" in os.environ:
 580             for client in clients:
 581                 endpoint = ctx.rgw.role_endpoints.get(client)
 582                 assert endpoint, 's3tests: no rgw endpoint for {}'.format(client)
 583
 584                 s3tests_conf[client] = ConfigObj(
 585                     indent_type='',
 586                     infile={
 587                         'DEFAULT':
 588                             {
 589                             'port'      : endpoint.port,
 590                             'is_secure' : endpoint.cert is not None,
 591                             'api_name'  : 'default',
 592                             },
 593                         'fixtures'   : {},
 594                         's3 main'    : {},
 595                         's3 alt'     : {},
 596                         's3 tenant'  : {},
 597                         'iam'        : {},
 598                         'webidentity': {},
 599                     }
 600                 )
 601
 602         elif ctx.sts_variable:
 603             #This will be the structure of config file when you want to run assume_role_test and get_session_token_test (sts-test)
 604             for client in clients:
 605                 endpoint = ctx.rgw.role_endpoints.get(client)
 606                 assert endpoint, 's3tests: no rgw endpoint for {}'.format(client)
 607
 608                 s3tests_conf[client] = ConfigObj(
 609                     indent_type='',
 610                     infile={
 611                         'DEFAULT':
 612                             {
 613                             'port'      : endpoint.port,
 614                             'is_secure' : endpoint.cert is not None,
 615                             'api_name'  : 'default',
 616                             },
 617                         'fixtures'   : {},
 618                         's3 main'    : {},
 619                         's3 alt'     : {},
 620                         's3 tenant'  : {},
 621                         'iam'        : {},
 622                         }
 623                     )
 624
 625         else:
 626             #This will be the structure of config file when you want to run normal s3-tests
 627             for client in clients:
 628                 endpoint = ctx.rgw.role_endpoints.get(client)
 629                 assert endpoint, 's3tests: no rgw endpoint for {}'.format(client)
 630
 631                 s3tests_conf[client] = ConfigObj(
 632                     indent_type='',
 633                     infile={
 634                         'DEFAULT':
 635                             {
 636                             'port'      : endpoint.port,
 637                             'is_secure' : endpoint.cert is not None,
 638                             'api_name'  : 'default',
 639                             },
 640                         'fixtures'   : {},
 641                         's3 main'    : {},
 642                         's3 alt'     : {},
 643                         's3 tenant'  : {},
 644                         }
 645                     )
 646
 647     with contextutil.nested(
 648         lambda: download(ctx=ctx, config=config),
 649         lambda: create_users(ctx=ctx, config=dict(
 650                 clients=clients,
 651                 s3tests_conf=s3tests_conf,
 652                 )),
 653         lambda: configure(ctx=ctx, config=dict(
 654                 clients=config,
 655                 s3tests_conf=s3tests_conf,
 656                 )),
 657         lambda: run_tests(ctx=ctx, config=config),
 658         lambda: scan_for_leaked_encryption_keys(ctx=ctx, config=config),
 659         ):
 660         pass
 661     yield