ceph/qa/tasks/swift.py

   1 """
   2 Test Swift API
   3 """
   4 from cStringIO import StringIO
   5 from configobj import ConfigObj
   6 import base64
   7 import contextlib
   8 import logging
   9 import os
  10
  11 from teuthology import misc as teuthology
  12 from teuthology import contextutil
  13 from teuthology.config import config as teuth_config
  14 from teuthology.orchestra import run
  15 from teuthology.orchestra.connection import split_user
  16
  17 log = logging.getLogger(__name__)
  18
  19
  20 @contextlib.contextmanager
  21 def download(ctx, config):
  22     """
  23     Download the Swift API.
  24     """
  25     testdir = teuthology.get_testdir(ctx)
  26     assert isinstance(config, dict)
  27     log.info('Downloading swift...')
  28     for (client, cconf) in config.items():
  29         ctx.cluster.only(client).run(
  30             args=[
  31                 'git', 'clone',
  32                 '-b', cconf.get('force-branch', 'ceph-nautilus'),
  33                 teuth_config.ceph_git_base_url + 'swift.git',
  34                 '{tdir}/swift'.format(tdir=testdir),
  35                 ],
  36             )
  37     try:
  38         yield
  39     finally:
  40         log.info('Removing swift...')
  41         testdir = teuthology.get_testdir(ctx)
  42         for (client, _) in config.items():
  43             ctx.cluster.only(client).run(
  44                 args=[
  45                     'rm',
  46                     '-rf',
  47                     '{tdir}/swift'.format(tdir=testdir),
  48                     ],
  49                 )
  50
  51 def _config_user(testswift_conf, account, user, suffix):
  52     """
  53     Configure a swift user
  54
  55     :param account: Swift account
  56     :param user: User name
  57     :param suffix: user name and email suffixes.
  58     """
  59     testswift_conf['func_test'].setdefault('account{s}'.format(s=suffix), account)
  60     testswift_conf['func_test'].setdefault('username{s}'.format(s=suffix), user)
  61     testswift_conf['func_test'].setdefault('email{s}'.format(s=suffix), '{account}+test@test.test'.format(account=account))
  62     testswift_conf['func_test'].setdefault('display_name{s}'.format(s=suffix), 'Mr. {account} {user}'.format(account=account, user=user))
  63     testswift_conf['func_test'].setdefault('password{s}'.format(s=suffix), base64.b64encode(os.urandom(40)))
  64
  65 @contextlib.contextmanager
  66 def create_users(ctx, config):
  67     """
  68     Create rgw users to interact with the swift interface.
  69     """
  70     assert isinstance(config, dict)
  71     log.info('Creating rgw users...')
  72     testdir = teuthology.get_testdir(ctx)
  73     users = {'': 'foo', '2': 'bar'}
  74     for client in config['clients']:
  75         cluster_name, daemon_type, client_id = teuthology.split_role(client)
  76         testswift_conf = config['testswift_conf'][client]
  77         for suffix, user in users.iteritems():
  78             _config_user(testswift_conf, '{user}.{client}'.format(user=user, client=client), user, suffix)
  79             ctx.cluster.only(client).run(
  80                 args=[
  81                     'adjust-ulimits',
  82                     'ceph-coverage',
  83                     '{tdir}/archive/coverage'.format(tdir=testdir),
  84                     'radosgw-admin',
  85                     '-n', client,
  86                     '--cluster', cluster_name,
  87                     'user', 'create',
  88                     '--subuser', '{account}:{user}'.format(account=testswift_conf['func_test']['account{s}'.format(s=suffix)],user=user),
  89                     '--display-name', testswift_conf['func_test']['display_name{s}'.format(s=suffix)],
  90                     '--secret', testswift_conf['func_test']['password{s}'.format(s=suffix)],
  91                     '--email', testswift_conf['func_test']['email{s}'.format(s=suffix)],
  92                     '--key-type', 'swift',
  93                     '--access', 'full',
  94                 ],
  95             )
  96     try:
  97         yield
  98     finally:
  99         for client in config['clients']:
 100             for user in users.itervalues():
 101                 uid = '{user}.{client}'.format(user=user, client=client)
 102                 cluster_name, daemon_type, client_id = teuthology.split_role(client)
 103                 ctx.cluster.only(client).run(
 104                     args=[
 105                         'adjust-ulimits',
 106                         'ceph-coverage',
 107                         '{tdir}/archive/coverage'.format(tdir=testdir),
 108                         'radosgw-admin',
 109                         '-n', client,
 110                         '--cluster', cluster_name,
 111                         'user', 'rm',
 112                         '--uid', uid,
 113                         '--purge-data',
 114                         ],
 115                     )
 116
 117 @contextlib.contextmanager
 118 def configure(ctx, config):
 119     """
 120     Configure rgw and Swift
 121     """
 122     assert isinstance(config, dict)
 123     log.info('Configuring testswift...')
 124     testdir = teuthology.get_testdir(ctx)
 125     for client, properties in config['clients'].iteritems():
 126         log.info('client={c}'.format(c=client))
 127         log.info('config={c}'.format(c=config))
 128         testswift_conf = config['testswift_conf'][client]
 129         if properties is not None and 'rgw_server' in properties:
 130             host = None
 131             for target, roles in zip(ctx.config['targets'].iterkeys(), ctx.config['roles']):
 132                 log.info('roles: ' + str(roles))
 133                 log.info('target: ' + str(target))
 134                 if properties['rgw_server'] in roles:
 135                     _, host = split_user(target)
 136             assert host is not None, "Invalid client specified as the rgw_server"
 137             testswift_conf['func_test']['auth_host'] = host
 138         else:
 139             testswift_conf['func_test']['auth_host'] = 'localhost'
 140
 141         log.info(client)
 142         (remote,) = ctx.cluster.only(client).remotes.keys()
 143         remote.run(
 144             args=[
 145                 'cd',
 146                 '{tdir}/swift'.format(tdir=testdir),
 147                 run.Raw('&&'),
 148                 './bootstrap',
 149                 ],
 150             )
 151         conf_fp = StringIO()
 152         testswift_conf.write(conf_fp)
 153         teuthology.write_file(
 154             remote=remote,
 155             path='{tdir}/archive/testswift.{client}.conf'.format(tdir=testdir, client=client),
 156             data=conf_fp.getvalue(),
 157             )
 158     yield
 159
 160
 161 @contextlib.contextmanager
 162 def run_tests(ctx, config):
 163     """
 164     Run an individual Swift test.
 165     """
 166     assert isinstance(config, dict)
 167     testdir = teuthology.get_testdir(ctx)
 168     for client, client_config in config.iteritems():
 169         args = [
 170                 'SWIFT_TEST_CONFIG_FILE={tdir}/archive/testswift.{client}.conf'.format(tdir=testdir, client=client),
 171                 '{tdir}/swift/virtualenv/bin/nosetests'.format(tdir=testdir),
 172                 '-w',
 173                 '{tdir}/swift/test/functional'.format(tdir=testdir),
 174                 '-v',
 175                 '-a', '!fails_on_rgw',
 176                 ]
 177         if client_config is not None and 'extra_args' in client_config:
 178             args.extend(client_config['extra_args'])
 179
 180         ctx.cluster.only(client).run(
 181             args=args,
 182             )
 183     yield
 184
 185 @contextlib.contextmanager
 186 def task(ctx, config):
 187     """
 188     Run the testswift suite against rgw.
 189
 190     To run all tests on all clients::
 191
 192         tasks:
 193         - ceph:
 194         - rgw:
 195         - testswift:
 196
 197     To restrict testing to particular clients::
 198
 199         tasks:
 200         - ceph:
 201         - rgw: [client.0]
 202         - testswift: [client.0]
 203
 204     To run against a server on client.1::
 205
 206         tasks:
 207         - ceph:
 208         - rgw: [client.1]
 209         - testswift:
 210             client.0:
 211               rgw_server: client.1
 212
 213     To pass extra arguments to nose (e.g. to run a certain test)::
 214
 215         tasks:
 216         - ceph:
 217         - rgw: [client.0]
 218         - testswift:
 219             client.0:
 220               extra_args: ['test.functional.tests:TestFileUTF8', '-m', 'testCopy']
 221             client.1:
 222               extra_args: ['--exclude', 'TestFile']
 223     """
 224     assert hasattr(ctx, 'rgw'), 'swift must run after the rgw task'
 225     assert config is None or isinstance(config, list) \
 226         or isinstance(config, dict), \
 227         "task testswift only supports a list or dictionary for configuration"
 228     all_clients = ['client.{id}'.format(id=id_)
 229                    for id_ in teuthology.all_roles_of_type(ctx.cluster, 'client')]
 230     if config is None:
 231         config = all_clients
 232     if isinstance(config, list):
 233         config = dict.fromkeys(config)
 234     clients = config.keys()
 235
 236     log.info('clients={c}'.format(c=clients))
 237
 238     testswift_conf = {}
 239     for client in clients:
 240         endpoint = ctx.rgw.role_endpoints.get(client)
 241         assert endpoint, 'swift: no rgw endpoint for {}'.format(client)
 242
 243         testswift_conf[client] = ConfigObj(
 244                 indent_type='',
 245                 infile={
 246                     'func_test':
 247                         {
 248                         'auth_port'      : endpoint.port,
 249                         'auth_ssl' : 'yes' if endpoint.cert else 'no',
 250                         'auth_prefix' : '/auth/',
 251                         },
 252                     }
 253                 )
 254
 255     with contextutil.nested(
 256         lambda: download(ctx=ctx, config=config),
 257         lambda: create_users(ctx=ctx, config=dict(
 258                 clients=clients,
 259                 testswift_conf=testswift_conf,
 260                 )),
 261         lambda: configure(ctx=ctx, config=dict(
 262                 clients=config,
 263                 testswift_conf=testswift_conf,
 264                 )),
 265         lambda: run_tests(ctx=ctx, config=config),
 266         ):
 267         pass
 268     yield