ceph/qa/tasks/s3tests.py

   1 """
   2 Run a set of s3 tests on rgw.
   3 """
   4 from io import BytesIO
   5 from configobj import ConfigObj
   6 import base64
   7 import contextlib
   8 import logging
   9 import os
  10 import random
  11 import string
  12
  13 from teuthology import misc as teuthology
  14 from teuthology import contextutil
  15 from teuthology.config import config as teuth_config
  16 from teuthology.orchestra import run
  17 from teuthology.exceptions import ConfigError
  18
  19 log = logging.getLogger(__name__)
  20
  21 @contextlib.contextmanager
  22 def download(ctx, config):
  23     """
  24     Download the s3 tests from the git builder.
  25     Remove downloaded s3 file upon exit.
  26
  27     The context passed in should be identical to the context
  28     passed in to the main task.
  29     """
  30     assert isinstance(config, dict)
  31     log.info('Downloading s3-tests...')
  32     testdir = teuthology.get_testdir(ctx)
  33     for (client, client_config) in config.items():
  34         s3tests_branch = client_config.get('force-branch', None)
  35         if not s3tests_branch:
  36             raise ValueError(
  37                 "Could not determine what branch to use for s3-tests. Please add 'force-branch: {s3-tests branch name}' to the .yaml config for this s3tests task.")
  38
  39         log.info("Using branch '%s' for s3tests", s3tests_branch)
  40         sha1 = client_config.get('sha1')
  41         git_remote = client_config.get('git_remote', teuth_config.ceph_git_base_url)
  42         ctx.cluster.only(client).run(
  43             args=[
  44                 'git', 'clone',
  45                 '-b', s3tests_branch,
  46                 git_remote + 's3-tests.git',
  47                 '{tdir}/s3-tests'.format(tdir=testdir),
  48                 ],
  49             )
  50         if sha1 is not None:
  51             ctx.cluster.only(client).run(
  52                 args=[
  53                     'cd', '{tdir}/s3-tests'.format(tdir=testdir),
  54                     run.Raw('&&'),
  55                     'git', 'reset', '--hard', sha1,
  56                     ],
  57                 )
  58     try:
  59         yield
  60     finally:
  61         log.info('Removing s3-tests...')
  62         testdir = teuthology.get_testdir(ctx)
  63         for client in config:
  64             ctx.cluster.only(client).run(
  65                 args=[
  66                     'rm',
  67                     '-rf',
  68                     '{tdir}/s3-tests'.format(tdir=testdir),
  69                     ],
  70                 )
  71
  72
  73 def _config_user(s3tests_conf, section, user):
  74     """
  75     Configure users for this section by stashing away keys, ids, and
  76     email addresses.
  77     """
  78     s3tests_conf[section].setdefault('user_id', user)
  79     s3tests_conf[section].setdefault('email', '{user}+test@test.test'.format(user=user))
  80     s3tests_conf[section].setdefault('display_name', 'Mr. {user}'.format(user=user))
  81     s3tests_conf[section].setdefault('access_key',
  82         ''.join(random.choice(string.ascii_uppercase) for i in range(20)))
  83     s3tests_conf[section].setdefault('secret_key',
  84         base64.b64encode(os.urandom(40)).decode())
  85     s3tests_conf[section].setdefault('totp_serial',
  86         ''.join(random.choice(string.digits) for i in range(10)))
  87     s3tests_conf[section].setdefault('totp_seed',
  88         base64.b32encode(os.urandom(40)).decode())
  89     s3tests_conf[section].setdefault('totp_seconds', '5')
  90
  91
  92 @contextlib.contextmanager
  93 def create_users(ctx, config):
  94     """
  95     Create a main and an alternate s3 user.
  96     """
  97     assert isinstance(config, dict)
  98     log.info('Creating rgw users...')
  99     testdir = teuthology.get_testdir(ctx)
 100
 101     if ctx.sts_variable:
 102         users = {'s3 main': 'foo', 's3 alt': 'bar', 's3 tenant': 'testx$tenanteduser', 'iam': 'foobar'}
 103         for client in config['clients']:
 104             s3tests_conf = config['s3tests_conf'][client]
 105             s3tests_conf.setdefault('fixtures', {})
 106             s3tests_conf['fixtures'].setdefault('bucket prefix', 'test-' + client + '-{random}-')
 107             for section, user in users.items():
 108                 _config_user(s3tests_conf, section, '{user}.{client}'.format(user=user, client=client))
 109                 log.debug('Creating user {user} on {host}'.format(user=s3tests_conf[section]['user_id'], host=client))
 110                 cluster_name, daemon_type, client_id = teuthology.split_role(client)
 111                 client_with_id = daemon_type + '.' + client_id
 112                 if section=='iam':
 113                     ctx.cluster.only(client).run(
 114                         args=[
 115                             'adjust-ulimits',
 116                             'ceph-coverage',
 117                             '{tdir}/archive/coverage'.format(tdir=testdir),
 118                             'radosgw-admin',
 119                             '-n', client_with_id,
 120                             'user', 'create',
 121                             '--uid', s3tests_conf[section]['user_id'],
 122                             '--display-name', s3tests_conf[section]['display_name'],
 123                             '--access-key', s3tests_conf[section]['access_key'],
 124                             '--secret', s3tests_conf[section]['secret_key'],
 125                             '--cluster', cluster_name,
 126                         ],
 127                     )
 128                     ctx.cluster.only(client).run(
 129                         args=[
 130                             'adjust-ulimits',
 131                             'ceph-coverage',
 132                             '{tdir}/archive/coverage'.format(tdir=testdir),
 133                             'radosgw-admin',
 134                             '-n', client_with_id,
 135                             'caps', 'add',
 136                             '--uid', s3tests_conf[section]['user_id'],
 137                             '--caps', 'user-policy=*',
 138                             '--cluster', cluster_name,
 139                         ],
 140                     )
 141                     ctx.cluster.only(client).run(
 142                         args=[
 143                             'adjust-ulimits',
 144                             'ceph-coverage',
 145                             '{tdir}/archive/coverage'.format(tdir=testdir),
 146                             'radosgw-admin',
 147                             '-n', client_with_id,
 148                             'caps', 'add',
 149                             '--uid', s3tests_conf[section]['user_id'],
 150                             '--caps', 'roles=*',
 151                             '--cluster', cluster_name,
 152                         ],
 153                     )
 154                     ctx.cluster.only(client).run(
 155                         args=[
 156                             'adjust-ulimits',
 157                             'ceph-coverage',
 158                             '{tdir}/archive/coverage'.format(tdir=testdir),
 159                             'radosgw-admin',
 160                             '-n', client_with_id,
 161                             'caps', 'add',
 162                             '--uid', s3tests_conf[section]['user_id'],
 163                             '--caps', 'oidc-provider=*',
 164                             '--cluster', cluster_name,
 165                         ],
 166                     )
 167
 168                 else:
 169                     ctx.cluster.only(client).run(
 170                         args=[
 171                             'adjust-ulimits',
 172                             'ceph-coverage',
 173                             '{tdir}/archive/coverage'.format(tdir=testdir),
 174                             'radosgw-admin',
 175                             '-n', client_with_id,
 176                             'user', 'create',
 177                             '--uid', s3tests_conf[section]['user_id'],
 178                             '--display-name', s3tests_conf[section]['display_name'],
 179                             '--access-key', s3tests_conf[section]['access_key'],
 180                             '--secret', s3tests_conf[section]['secret_key'],
 181                             '--email', s3tests_conf[section]['email'],
 182                             '--caps', 'user-policy=*',
 183                             '--cluster', cluster_name,
 184                         ],
 185                     )
 186                     ctx.cluster.only(client).run(
 187                         args=[
 188                             'adjust-ulimits',
 189                             'ceph-coverage',
 190                             '{tdir}/archive/coverage'.format(tdir=testdir),
 191                             'radosgw-admin',
 192                             '-n', client_with_id,
 193                             'mfa', 'create',
 194                             '--uid', s3tests_conf[section]['user_id'],
 195                             '--totp-serial', s3tests_conf[section]['totp_serial'],
 196                             '--totp-seed', s3tests_conf[section]['totp_seed'],
 197                             '--totp-seconds', s3tests_conf[section]['totp_seconds'],
 198                             '--totp-window', '8',
 199                             '--totp-seed-type', 'base32',
 200                             '--cluster', cluster_name,
 201                         ],
 202                     )
 203
 204     else:
 205         users = {'s3 main': 'foo', 's3 alt': 'bar', 's3 tenant': 'testx$tenanteduser'}
 206         for client in config['clients']:
 207             s3tests_conf = config['s3tests_conf'][client]
 208             s3tests_conf.setdefault('fixtures', {})
 209             s3tests_conf['fixtures'].setdefault('bucket prefix', 'test-' + client + '-{random}-')
 210             for section, user in users.items():
 211                 _config_user(s3tests_conf, section, '{user}.{client}'.format(user=user, client=client))
 212                 log.debug('Creating user {user} on {host}'.format(user=s3tests_conf[section]['user_id'], host=client))
 213                 cluster_name, daemon_type, client_id = teuthology.split_role(client)
 214                 client_with_id = daemon_type + '.' + client_id
 215                 ctx.cluster.only(client).run(
 216                         args=[
 217                             'adjust-ulimits',
 218                             'ceph-coverage',
 219                             '{tdir}/archive/coverage'.format(tdir=testdir),
 220                             'radosgw-admin',
 221                             '-n', client_with_id,
 222                             'user', 'create',
 223                             '--uid', s3tests_conf[section]['user_id'],
 224                             '--display-name', s3tests_conf[section]['display_name'],
 225                             '--access-key', s3tests_conf[section]['access_key'],
 226                             '--secret', s3tests_conf[section]['secret_key'],
 227                             '--email', s3tests_conf[section]['email'],
 228                             '--caps', 'user-policy=*',
 229                             '--cluster', cluster_name,
 230                         ],
 231                     )
 232                 ctx.cluster.only(client).run(
 233                         args=[
 234                             'adjust-ulimits',
 235                             'ceph-coverage',
 236                             '{tdir}/archive/coverage'.format(tdir=testdir),
 237                             'radosgw-admin',
 238                             '-n', client_with_id,
 239                             'mfa', 'create',
 240                             '--uid', s3tests_conf[section]['user_id'],
 241                             '--totp-serial', s3tests_conf[section]['totp_serial'],
 242                             '--totp-seed', s3tests_conf[section]['totp_seed'],
 243                             '--totp-seconds', s3tests_conf[section]['totp_seconds'],
 244                             '--totp-window', '8',
 245                             '--totp-seed-type', 'base32',
 246                             '--cluster', cluster_name,
 247                         ],
 248                     )
 249
 250     if "TOKEN" in os.environ:
 251         s3tests_conf.setdefault('webidentity', {})
 252         s3tests_conf['webidentity'].setdefault('token',os.environ['TOKEN'])
 253         s3tests_conf['webidentity'].setdefault('aud',os.environ['AUD'])
 254         s3tests_conf['webidentity'].setdefault('sub',os.environ['SUB'])
 255         s3tests_conf['webidentity'].setdefault('azp',os.environ['AZP'])
 256         s3tests_conf['webidentity'].setdefault('user_token',os.environ['USER_TOKEN'])
 257         s3tests_conf['webidentity'].setdefault('thumbprint',os.environ['THUMBPRINT'])
 258         s3tests_conf['webidentity'].setdefault('KC_REALM',os.environ['KC_REALM'])
 259
 260     try:
 261         yield
 262     finally:
 263         for client in config['clients']:
 264             for user in users.values():
 265                 uid = '{user}.{client}'.format(user=user, client=client)
 266                 cluster_name, daemon_type, client_id = teuthology.split_role(client)
 267                 client_with_id = daemon_type + '.' + client_id
 268                 ctx.cluster.only(client).run(
 269                     args=[
 270                         'adjust-ulimits',
 271                         'ceph-coverage',
 272                         '{tdir}/archive/coverage'.format(tdir=testdir),
 273                         'radosgw-admin',
 274                         '-n', client_with_id,
 275                         'user', 'rm',
 276                         '--uid', uid,
 277                         '--purge-data',
 278                         '--cluster', cluster_name,
 279                         ],
 280                     )
 281
 282
 283 @contextlib.contextmanager
 284 def configure(ctx, config):
 285     """
 286     Configure the s3-tests.  This includes the running of the
 287     bootstrap code and the updating of local conf files.
 288     """
 289     assert isinstance(config, dict)
 290     log.info('Configuring s3-tests...')
 291     testdir = teuthology.get_testdir(ctx)
 292     for client, properties in config['clients'].items():
 293         properties = properties or {}
 294         s3tests_conf = config['s3tests_conf'][client]
 295         s3tests_conf['DEFAULT']['calling_format'] = properties.get('calling-format', 'ordinary')
 296
 297         # use rgw_server if given, or default to local client
 298         role = properties.get('rgw_server', client)
 299
 300         endpoint = ctx.rgw.role_endpoints.get(role)
 301         assert endpoint, 's3tests: no rgw endpoint for {}'.format(role)
 302
 303         s3tests_conf['DEFAULT']['host'] = endpoint.dns_name
 304
 305         website_role = properties.get('rgw_website_server')
 306         if website_role:
 307             website_endpoint = ctx.rgw.role_endpoints.get(website_role)
 308             assert website_endpoint, \
 309                     's3tests: no rgw endpoint for rgw_website_server {}'.format(website_role)
 310             assert website_endpoint.website_dns_name, \
 311                     's3tests: no dns-s3website-name for rgw_website_server {}'.format(website_role)
 312             s3tests_conf['DEFAULT']['s3website_domain'] = website_endpoint.website_dns_name
 313
 314         if hasattr(ctx, 'barbican'):
 315             properties = properties['barbican']
 316             if properties is not None and 'kms_key' in properties:
 317                 if not (properties['kms_key'] in ctx.barbican.keys):
 318                     raise ConfigError('Key '+properties['kms_key']+' not defined')
 319
 320                 if not (properties['kms_key2'] in ctx.barbican.keys):
 321                     raise ConfigError('Key '+properties['kms_key2']+' not defined')
 322
 323                 key = ctx.barbican.keys[properties['kms_key']]
 324                 s3tests_conf['DEFAULT']['kms_keyid'] = key['id']
 325
 326                 key = ctx.barbican.keys[properties['kms_key2']]
 327                 s3tests_conf['DEFAULT']['kms_keyid2'] = key['id']
 328
 329         elif hasattr(ctx, 'vault'):
 330             engine_or_flavor = vars(ctx.vault).get('flavor',ctx.vault.engine)
 331             keys=[]
 332             for name in (x['Path'] for x in vars(ctx.vault).get('keys', {}).get(ctx.rgw.vault_role)):
 333                 keys.append(name)
 334
 335             keys.extend(['testkey-1','testkey-2'])
 336             if engine_or_flavor == "old":
 337                 keys=[keys[i] + "/1" for i in range(len(keys))]
 338
 339             properties = properties.get('vault_%s' % engine_or_flavor, {})
 340             s3tests_conf['DEFAULT']['kms_keyid'] = properties.get('key_path', keys[0])
 341             s3tests_conf['DEFAULT']['kms_keyid2'] = properties.get('key_path2', keys[1])
 342         elif hasattr(ctx.rgw, 'pykmip_role'):
 343             keys=[]
 344             for name in (x['Name'] for x in ctx.pykmip.keys[ctx.rgw.pykmip_role]):
 345                 p=name.partition('-')
 346                 keys.append(p[2] if p[2] else p[0])
 347             keys.extend(['testkey-1', 'testkey-2'])
 348             s3tests_conf['DEFAULT']['kms_keyid'] = properties.get('kms_key', keys[0])
 349             s3tests_conf['DEFAULT']['kms_keyid2'] = properties.get('kms_key2', keys[1])
 350         else:
 351             # Fallback scenario where it's the local (ceph.conf) kms being tested
 352             s3tests_conf['DEFAULT']['kms_keyid'] = 'testkey-1'
 353             s3tests_conf['DEFAULT']['kms_keyid2'] = 'testkey-2'
 354
 355         slow_backend = properties.get('slow_backend')
 356         if slow_backend:
 357             s3tests_conf['fixtures']['slow backend'] = slow_backend
 358
 359         storage_classes = properties.get('storage classes')
 360         if storage_classes:
 361             s3tests_conf['s3 main']['storage_classes'] = storage_classes
 362
 363         lc_debug_interval = properties.get('lc_debug_interval')
 364         if lc_debug_interval:
 365             s3tests_conf['s3 main']['lc_debug_interval'] = lc_debug_interval
 366
 367         (remote,) = ctx.cluster.only(client).remotes.keys()
 368         remote.run(
 369             args=[
 370                 'cd',
 371                 '{tdir}/s3-tests'.format(tdir=testdir),
 372                 run.Raw('&&'),
 373                 './bootstrap',
 374                 ],
 375             )
 376         conf_fp = BytesIO()
 377         s3tests_conf.write(conf_fp)
 378         remote.write_file(
 379             path='{tdir}/archive/s3-tests.{client}.conf'.format(tdir=testdir, client=client),
 380             data=conf_fp.getvalue(),
 381             )
 382
 383     log.info('Configuring boto...')
 384     boto_src = os.path.join(os.path.dirname(__file__), 'boto.cfg.template')
 385     for client, properties in config['clients'].items():
 386         with open(boto_src) as f:
 387             (remote,) = ctx.cluster.only(client).remotes.keys()
 388             conf = f.read().format(
 389                 idle_timeout=config.get('idle_timeout', 30)
 390                 )
 391             remote.write_file('{tdir}/boto.cfg'.format(tdir=testdir), conf)
 392
 393     try:
 394         yield
 395
 396     finally:
 397         log.info('Cleaning up boto...')
 398         for client, properties in config['clients'].items():
 399             (remote,) = ctx.cluster.only(client).remotes.keys()
 400             remote.run(
 401                 args=[
 402                     'rm',
 403                     '{tdir}/boto.cfg'.format(tdir=testdir),
 404                     ],
 405                 )
 406
 407 @contextlib.contextmanager
 408 def run_tests(ctx, config):
 409     """
 410     Run the s3tests after everything is set up.
 411
 412     :param ctx: Context passed to task
 413     :param config: specific configuration information
 414     """
 415     assert isinstance(config, dict)
 416     testdir = teuthology.get_testdir(ctx)
 417     for client, client_config in config.items():
 418         client_config = client_config or {}
 419         (cluster_name,_,_) = teuthology.split_role(client)
 420         (remote,) = ctx.cluster.only(client).remotes.keys()
 421         args = [
 422             'S3TEST_CONF={tdir}/archive/s3-tests.{client}.conf'.format(tdir=testdir, client=client),
 423             'BOTO_CONFIG={tdir}/boto.cfg'.format(tdir=testdir)
 424             ]
 425         # the 'requests' library comes with its own ca bundle to verify ssl
 426         # certificates - override that to use the system's ca bundle, which
 427         # is where the ssl task installed this certificate
 428         if remote.os.package_type == 'deb':
 429             args += ['REQUESTS_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt']
 430         else:
 431             args += ['REQUESTS_CA_BUNDLE=/etc/pki/tls/certs/ca-bundle.crt']
 432         # civetweb > 1.8 && beast parsers are strict on rfc2616
 433         attrs = ["!fails_on_rgw", "!lifecycle_expiration", "!fails_strict_rfc2616","!test_of_sts","!webidentity_test"]
 434         if client_config.get('calling-format') != 'ordinary':
 435             attrs += ['!fails_with_subdomain']
 436         if not client_config.get('with-sse-s3'):
 437             attrs += ['!sse-s3']
 438         elif client_config.get('with-sse-s3'):
 439             pass
 440         elif ctx.ceph[cluster_name].rgw_crypt_sse_s3_backend is None:
 441             attrs += ['!sse-s3']
 442
 443         if 'extra_attrs' in client_config:
 444             attrs = client_config.get('extra_attrs')
 445         args += [
 446             '{tdir}/s3-tests/virtualenv/bin/python'.format(tdir=testdir),
 447             '-m', 'nose',
 448             '-w',
 449             '{tdir}/s3-tests'.format(tdir=testdir),
 450             '-v',
 451             '-a', ','.join(attrs),
 452             ]
 453         if 'extra_args' in client_config:
 454             args.append(client_config['extra_args'])
 455
 456         remote.run(
 457             args=args,
 458             label="s3 tests against rgw"
 459             )
 460     yield
 461
 462 @contextlib.contextmanager
 463 def scan_for_leaked_encryption_keys(ctx, config):
 464     """
 465     Scan radosgw logs for the encryption keys used by s3tests to
 466     verify that we're not leaking secrets.
 467
 468     :param ctx: Context passed to task
 469     :param config: specific configuration information
 470     """
 471     assert isinstance(config, dict)
 472
 473     try:
 474         yield
 475     finally:
 476         # x-amz-server-side-encryption-customer-key
 477         s3test_customer_key = 'pO3upElrwuEXSoFwCfnZPdSsmt/xWeFa0N9KgDijwVs='
 478
 479         log.debug('Scanning radosgw logs for leaked encryption keys...')
 480         procs = list()
 481         for client, client_config in config.items():
 482             if not client_config.get('scan_for_encryption_keys', True):
 483                 continue
 484             cluster_name, daemon_type, client_id = teuthology.split_role(client)
 485             client_with_cluster = '.'.join((cluster_name, daemon_type, client_id))
 486             (remote,) = ctx.cluster.only(client).remotes.keys()
 487             proc = remote.run(
 488                 args=[
 489                     'grep',
 490                     '--binary-files=text',
 491                     s3test_customer_key,
 492                     '/var/log/ceph/rgw.{client}.log'.format(client=client_with_cluster),
 493                 ],
 494                 wait=False,
 495                 check_status=False,
 496             )
 497             procs.append(proc)
 498
 499         for proc in procs:
 500             proc.wait()
 501             if proc.returncode == 1: # 1 means no matches
 502                 continue
 503             log.error('radosgw log is leaking encryption keys!')
 504             raise Exception('radosgw log is leaking encryption keys')
 505
 506 @contextlib.contextmanager
 507 def task(ctx, config):
 508     """
 509     Run the s3-tests suite against rgw.
 510
 511     To run all tests on all clients::
 512
 513         tasks:
 514         - ceph:
 515         - rgw:
 516         - s3tests:
 517
 518     To restrict testing to particular clients::
 519
 520         tasks:
 521         - ceph:
 522         - rgw: [client.0]
 523         - s3tests: [client.0]
 524
 525     To run against a server on client.1 and increase the boto timeout to 10m::
 526
 527         tasks:
 528         - ceph:
 529         - rgw: [client.1]
 530         - s3tests:
 531             client.0:
 532               rgw_server: client.1
 533               idle_timeout: 600
 534
 535     To pass extra arguments to nose (e.g. to run a certain test)::
 536
 537         tasks:
 538         - ceph:
 539         - rgw: [client.0]
 540         - s3tests:
 541             client.0:
 542               extra_args: ['test_s3:test_object_acl_grand_public_read']
 543             client.1:
 544               extra_args: ['--exclude', 'test_100_continue']
 545
 546     To run any sts-tests don't forget to set a config variable named 'sts_tests' to 'True' as follows::
 547
 548         tasks:
 549         - ceph:
 550         - rgw: [client.0]
 551         - s3tests:
 552             client.0:
 553               sts_tests: True
 554               rgw_server: client.0
 555
 556     """
 557     assert hasattr(ctx, 'rgw'), 's3tests must run after the rgw task'
 558     assert config is None or isinstance(config, list) \
 559         or isinstance(config, dict), \
 560         "task s3tests only supports a list or dictionary for configuration"
 561     all_clients = ['client.{id}'.format(id=id_)
 562                    for id_ in teuthology.all_roles_of_type(ctx.cluster, 'client')]
 563     if config is None:
 564         config = all_clients
 565     if isinstance(config, list):
 566         config = dict.fromkeys(config)
 567     clients = config.keys()
 568
 569     overrides = ctx.config.get('overrides', {})
 570     # merge each client section, not the top level.
 571     for client in config.keys():
 572         if not config[client]:
 573             config[client] = {}
 574         teuthology.deep_merge(config[client], overrides.get('s3tests', {}))
 575
 576     log.debug('s3tests config is %s', config)
 577
 578     s3tests_conf = {}
 579
 580     for client, client_config in config.items():
 581         if 'sts_tests' in client_config:
 582             ctx.sts_variable = True
 583         else:
 584             ctx.sts_variable = False
 585         #This will be the structure of config file when you want to run webidentity_test (sts-test)
 586         if ctx.sts_variable and "TOKEN" in os.environ:
 587             for client in clients:
 588                 endpoint = ctx.rgw.role_endpoints.get(client)
 589                 assert endpoint, 's3tests: no rgw endpoint for {}'.format(client)
 590
 591                 s3tests_conf[client] = ConfigObj(
 592                     indent_type='',
 593                     infile={
 594                         'DEFAULT':
 595                             {
 596                             'port'      : endpoint.port,
 597                             'is_secure' : endpoint.cert is not None,
 598                             'api_name'  : 'default',
 599                             },
 600                         'fixtures'   : {},
 601                         's3 main'    : {},
 602                         's3 alt'     : {},
 603                         's3 tenant'  : {},
 604                         'iam'        : {},
 605                         'webidentity': {},
 606                     }
 607                 )
 608
 609         elif ctx.sts_variable:
 610             #This will be the structure of config file when you want to run assume_role_test and get_session_token_test (sts-test)
 611             for client in clients:
 612                 endpoint = ctx.rgw.role_endpoints.get(client)
 613                 assert endpoint, 's3tests: no rgw endpoint for {}'.format(client)
 614
 615                 s3tests_conf[client] = ConfigObj(
 616                     indent_type='',
 617                     infile={
 618                         'DEFAULT':
 619                             {
 620                             'port'      : endpoint.port,
 621                             'is_secure' : endpoint.cert is not None,
 622                             'api_name'  : 'default',
 623                             },
 624                         'fixtures'   : {},
 625                         's3 main'    : {},
 626                         's3 alt'     : {},
 627                         's3 tenant'  : {},
 628                         'iam'        : {},
 629                         }
 630                     )
 631
 632         else:
 633             #This will be the structure of config file when you want to run normal s3-tests
 634             for client in clients:
 635                 endpoint = ctx.rgw.role_endpoints.get(client)
 636                 assert endpoint, 's3tests: no rgw endpoint for {}'.format(client)
 637
 638                 s3tests_conf[client] = ConfigObj(
 639                     indent_type='',
 640                     infile={
 641                         'DEFAULT':
 642                             {
 643                             'port'      : endpoint.port,
 644                             'is_secure' : endpoint.cert is not None,
 645                             'api_name'  : 'default',
 646                             },
 647                         'fixtures'   : {},
 648                         's3 main'    : {},
 649                         's3 alt'     : {},
 650                         's3 tenant'  : {},
 651                         }
 652                     )
 653
 654     with contextutil.nested(
 655         lambda: download(ctx=ctx, config=config),
 656         lambda: create_users(ctx=ctx, config=dict(
 657                 clients=clients,
 658                 s3tests_conf=s3tests_conf,
 659                 )),
 660         lambda: configure(ctx=ctx, config=dict(
 661                 clients=config,
 662                 s3tests_conf=s3tests_conf,
 663                 )),
 664         lambda: run_tests(ctx=ctx, config=config),
 665         lambda: scan_for_leaked_encryption_keys(ctx=ctx, config=config),
 666         ):
 667         pass
 668     yield