ceph/qa/tasks/swift.py

   1 """
   2 Test Swift API
   3 """
   4 from cStringIO import StringIO
   5 from configobj import ConfigObj
   6 import base64
   7 import contextlib
   8 import logging
   9 import os
  10
  11 from distutils.version import LooseVersion
  12 from teuthology import misc as teuthology
  13 from teuthology import contextutil
  14 from teuthology.config import config as teuth_config
  15 from teuthology.orchestra import run
  16 from teuthology.orchestra.connection import split_user
  17
  18 log = logging.getLogger(__name__)
  19
  20
  21 @contextlib.contextmanager
  22 def download(ctx, config):
  23     """
  24     Download the Swift API.
  25     """
  26     testdir = teuthology.get_testdir(ctx)
  27     assert isinstance(config, dict)
  28     log.info('Downloading swift...')
  29     for (client, cconf) in config.items():
  30         ctx.cluster.only(client).run(
  31             args=[
  32                 'git', 'clone',
  33                 '-b', cconf.get('force-branch', 'ceph-nautilus'),
  34                 teuth_config.ceph_git_base_url + 'swift.git',
  35                 '{tdir}/swift'.format(tdir=testdir),
  36                 ],
  37             )
  38     try:
  39         yield
  40     finally:
  41         log.info('Removing swift...')
  42         testdir = teuthology.get_testdir(ctx)
  43         for (client, _) in config.items():
  44             ctx.cluster.only(client).run(
  45                 args=[
  46                     'rm',
  47                     '-rf',
  48                     '{tdir}/swift'.format(tdir=testdir),
  49                     ],
  50                 )
  51
  52 def _config_user(testswift_conf, account, user, suffix):
  53     """
  54     Configure a swift user
  55
  56     :param account: Swift account
  57     :param user: User name
  58     :param suffix: user name and email suffixes.
  59     """
  60     testswift_conf['func_test'].setdefault('account{s}'.format(s=suffix), account)
  61     testswift_conf['func_test'].setdefault('username{s}'.format(s=suffix), user)
  62     testswift_conf['func_test'].setdefault('email{s}'.format(s=suffix), '{account}+test@test.test'.format(account=account))
  63     testswift_conf['func_test'].setdefault('display_name{s}'.format(s=suffix), 'Mr. {account} {user}'.format(account=account, user=user))
  64     testswift_conf['func_test'].setdefault('password{s}'.format(s=suffix), base64.b64encode(os.urandom(40)))
  65
  66 @contextlib.contextmanager
  67 def create_users(ctx, config):
  68     """
  69     Create rgw users to interact with the swift interface.
  70     """
  71     assert isinstance(config, dict)
  72     log.info('Creating rgw users...')
  73     testdir = teuthology.get_testdir(ctx)
  74     users = {'': 'foo', '2': 'bar'}
  75     for client, testswift_conf in config.iteritems():
  76         cluster_name, daemon_type, client_id = teuthology.split_role(client)
  77         for suffix, user in users.iteritems():
  78             _config_user(testswift_conf, '{user}.{client}'.format(user=user, client=client), user, suffix)
  79             ctx.cluster.only(client).run(
  80                 args=[
  81                     'adjust-ulimits',
  82                     'ceph-coverage',
  83                     '{tdir}/archive/coverage'.format(tdir=testdir),
  84                     'radosgw-admin',
  85                     '-n', client,
  86                     '--cluster', cluster_name,
  87                     'user', 'create',
  88                     '--subuser', '{account}:{user}'.format(account=testswift_conf['func_test']['account{s}'.format(s=suffix)],user=user),
  89                     '--display-name', testswift_conf['func_test']['display_name{s}'.format(s=suffix)],
  90                     '--secret', testswift_conf['func_test']['password{s}'.format(s=suffix)],
  91                     '--email', testswift_conf['func_test']['email{s}'.format(s=suffix)],
  92                     '--key-type', 'swift',
  93                     '--access', 'full',
  94                 ],
  95             )
  96     try:
  97         yield
  98     finally:
  99         for client in config.iterkeys():
 100             for user in users.itervalues():
 101                 uid = '{user}.{client}'.format(user=user, client=client)
 102                 cluster_name, daemon_type, client_id = teuthology.split_role(client)
 103                 ctx.cluster.only(client).run(
 104                     args=[
 105                         'adjust-ulimits',
 106                         'ceph-coverage',
 107                         '{tdir}/archive/coverage'.format(tdir=testdir),
 108                         'radosgw-admin',
 109                         '-n', client,
 110                         '--cluster', cluster_name,
 111                         'user', 'rm',
 112                         '--uid', uid,
 113                         '--purge-data',
 114                         ],
 115                     )
 116
 117 @contextlib.contextmanager
 118 def configure(ctx, config):
 119     """
 120     Configure rgw and Swift
 121     """
 122     assert isinstance(config, dict)
 123     log.info('Configuring testswift...')
 124     testdir = teuthology.get_testdir(ctx)
 125     for client, testswift_conf in config.iteritems():
 126         (remote,) = ctx.cluster.only(client).remotes.keys()
 127         remote.run(
 128             args=[
 129                 'cd',
 130                 '{tdir}/swift'.format(tdir=testdir),
 131                 run.Raw('&&'),
 132                 './bootstrap',
 133                 ],
 134             )
 135         conf_fp = StringIO()
 136         testswift_conf.write(conf_fp)
 137         teuthology.write_file(
 138             remote=remote,
 139             path='{tdir}/archive/testswift.{client}.conf'.format(tdir=testdir, client=client),
 140             data=conf_fp.getvalue(),
 141             )
 142     yield
 143
 144
 145 @contextlib.contextmanager
 146 def run_tests(ctx, config):
 147     """
 148     Run an individual Swift test.
 149     """
 150     assert isinstance(config, dict)
 151     testdir = teuthology.get_testdir(ctx)
 152     for client, client_config in config.iteritems():
 153         args = [
 154                 'SWIFT_TEST_CONFIG_FILE={tdir}/archive/testswift.{client}.conf'.format(tdir=testdir, client=client),
 155                 '{tdir}/swift/virtualenv/bin/nosetests'.format(tdir=testdir),
 156                 '-w',
 157                 '{tdir}/swift/test/functional'.format(tdir=testdir),
 158                 '-v',
 159                 '-a', '!fails_on_rgw',
 160                 ]
 161         if client_config is not None and 'extra_args' in client_config:
 162             args.extend(client_config['extra_args'])
 163
 164         ctx.cluster.only(client).run(
 165             args=args,
 166             )
 167     yield
 168
 169 @contextlib.contextmanager
 170 def task(ctx, config):
 171     """
 172     Run the testswift suite against rgw.
 173
 174     To run all tests on all clients::
 175
 176         tasks:
 177         - ceph:
 178         - rgw:
 179         - testswift:
 180
 181     To restrict testing to particular clients::
 182
 183         tasks:
 184         - ceph:
 185         - rgw: [client.0]
 186         - testswift: [client.0]
 187
 188     To run against a server on client.1::
 189
 190         tasks:
 191         - ceph:
 192         - rgw: [client.1]
 193         - testswift:
 194             client.0:
 195               rgw_server: client.1
 196
 197     To pass extra arguments to nose (e.g. to run a certain test)::
 198
 199         tasks:
 200         - ceph:
 201         - rgw: [client.0]
 202         - testswift:
 203             client.0:
 204               extra_args: ['test.functional.tests:TestFileUTF8', '-m', 'testCopy']
 205             client.1:
 206               extra_args: ['--exclude', 'TestFile']
 207     """
 208     assert hasattr(ctx, 'rgw'), 'swift must run after the rgw task'
 209     assert config is None or isinstance(config, list) \
 210         or isinstance(config, dict), \
 211         "task testswift only supports a list or dictionary for configuration"
 212     all_clients = ['client.{id}'.format(id=id_)
 213                    for id_ in teuthology.all_roles_of_type(ctx.cluster, 'client')]
 214     if config is None:
 215         config = all_clients
 216     if isinstance(config, list):
 217         config = dict.fromkeys(config)
 218
 219     testswift_conf = {}
 220     clients = []
 221     for client, client_config in config.iteritems():
 222         # http://tracker.ceph.com/issues/40304 can't bootstrap on rhel 7.6+
 223         (remote,) = ctx.cluster.only(client).remotes.keys()
 224         if remote.os.name == 'rhel' and LooseVersion(remote.os.version) >= LooseVersion('7.6'):
 225             log.warning('Swift tests cannot run on rhel 7.6+, skipping client {}'.format(client))
 226             continue
 227
 228         clients.append(client)
 229
 230         server = client_config.get('rgw_server', client)
 231         endpoint = ctx.rgw.role_endpoints.get(server)
 232         assert endpoint, 'swift: no rgw endpoint for {}'.format(server)
 233
 234         testswift_conf[client] = ConfigObj(
 235                 indent_type='',
 236                 infile={
 237                     'func_test':
 238                         {
 239                         'auth_host' : endpoint.hostname,
 240                         'auth_port' : endpoint.port,
 241                         'auth_ssl' : 'yes' if endpoint.cert else 'no',
 242                         'auth_prefix' : '/auth/',
 243                         },
 244                     }
 245                 )
 246     # only take config for valid clients
 247     config = {c: config[c] for c in clients}
 248
 249     log.info('clients={c}'.format(c=config.keys()))
 250     with contextutil.nested(
 251         lambda: download(ctx=ctx, config=config),
 252         lambda: create_users(ctx=ctx, config=testswift_conf),
 253         lambda: configure(ctx=ctx, config=testswift_conf),
 254         lambda: run_tests(ctx=ctx, config=config),
 255         ):
 256         pass
 257     yield