ceph/qa/tasks/swift.py

   1 """
   2 Test Swift API
   3 """
   4 from cStringIO import StringIO
   5 from configobj import ConfigObj
   6 import base64
   7 import contextlib
   8 import logging
   9 import os
  10
  11 from teuthology import misc as teuthology
  12 from teuthology import contextutil
  13 from teuthology.config import config as teuth_config
  14 from teuthology.orchestra import run
  15 from teuthology.orchestra.connection import split_user
  16
  17 log = logging.getLogger(__name__)
  18
  19
  20 @contextlib.contextmanager
  21 def download(ctx, config):
  22     """
  23     Download the Swift API.
  24     """
  25     testdir = teuthology.get_testdir(ctx)
  26     assert isinstance(config, list)
  27     log.info('Downloading swift...')
  28     for client in config:
  29         ctx.cluster.only(client).run(
  30             args=[
  31                 'git',
  32                 'clone',
  33                 '--branch',
  34                 'ceph-luminous',
  35                 teuth_config.ceph_git_base_url + 'swift.git',
  36                 '{tdir}/swift'.format(tdir=testdir),
  37                 ],
  38             )
  39     try:
  40         yield
  41     finally:
  42         log.info('Removing swift...')
  43         testdir = teuthology.get_testdir(ctx)
  44         for client in config:
  45             ctx.cluster.only(client).run(
  46                 args=[
  47                     'rm',
  48                     '-rf',
  49                     '{tdir}/swift'.format(tdir=testdir),
  50                     ],
  51                 )
  52
  53 def _config_user(testswift_conf, account, user, suffix):
  54     """
  55     Configure a swift user
  56
  57     :param account: Swift account
  58     :param user: User name
  59     :param suffix: user name and email suffixes.
  60     """
  61     testswift_conf['func_test'].setdefault('account{s}'.format(s=suffix), account)
  62     testswift_conf['func_test'].setdefault('username{s}'.format(s=suffix), user)
  63     testswift_conf['func_test'].setdefault('email{s}'.format(s=suffix), '{account}+test@test.test'.format(account=account))
  64     testswift_conf['func_test'].setdefault('display_name{s}'.format(s=suffix), 'Mr. {account} {user}'.format(account=account, user=user))
  65     testswift_conf['func_test'].setdefault('password{s}'.format(s=suffix), base64.b64encode(os.urandom(40)))
  66
  67 @contextlib.contextmanager
  68 def create_users(ctx, config):
  69     """
  70     Create rgw users to interact with the swift interface.
  71     """
  72     assert isinstance(config, dict)
  73     log.info('Creating rgw users...')
  74     testdir = teuthology.get_testdir(ctx)
  75     users = {'': 'foo', '2': 'bar'}
  76     for client in config['clients']:
  77         cluster_name, daemon_type, client_id = teuthology.split_role(client)
  78         testswift_conf = config['testswift_conf'][client]
  79         for suffix, user in users.iteritems():
  80             _config_user(testswift_conf, '{user}.{client}'.format(user=user, client=client), user, suffix)
  81             ctx.cluster.only(client).run(
  82                 args=[
  83                     'adjust-ulimits',
  84                     'ceph-coverage',
  85                     '{tdir}/archive/coverage'.format(tdir=testdir),
  86                     'radosgw-admin',
  87                     '-n', client,
  88                     '--cluster', cluster_name,
  89                     'user', 'create',
  90                     '--subuser', '{account}:{user}'.format(account=testswift_conf['func_test']['account{s}'.format(s=suffix)],user=user),
  91                     '--display-name', testswift_conf['func_test']['display_name{s}'.format(s=suffix)],
  92                     '--secret', testswift_conf['func_test']['password{s}'.format(s=suffix)],
  93                     '--email', testswift_conf['func_test']['email{s}'.format(s=suffix)],
  94                     '--key-type', 'swift',
  95                     '--access', 'full',
  96                 ],
  97             )
  98     try:
  99         yield
 100     finally:
 101         for client in config['clients']:
 102             for user in users.itervalues():
 103                 uid = '{user}.{client}'.format(user=user, client=client)
 104                 cluster_name, daemon_type, client_id = teuthology.split_role(client)
 105                 ctx.cluster.only(client).run(
 106                     args=[
 107                         'adjust-ulimits',
 108                         'ceph-coverage',
 109                         '{tdir}/archive/coverage'.format(tdir=testdir),
 110                         'radosgw-admin',
 111                         '-n', client,
 112                         '--cluster', cluster_name,
 113                         'user', 'rm',
 114                         '--uid', uid,
 115                         '--purge-data',
 116                         ],
 117                     )
 118
 119 @contextlib.contextmanager
 120 def configure(ctx, config):
 121     """
 122     Configure rgw and Swift
 123     """
 124     assert isinstance(config, dict)
 125     log.info('Configuring testswift...')
 126     testdir = teuthology.get_testdir(ctx)
 127     for client, properties in config['clients'].iteritems():
 128         log.info('client={c}'.format(c=client))
 129         log.info('config={c}'.format(c=config))
 130         testswift_conf = config['testswift_conf'][client]
 131         if properties is not None and 'rgw_server' in properties:
 132             host = None
 133             for target, roles in zip(ctx.config['targets'].iterkeys(), ctx.config['roles']):
 134                 log.info('roles: ' + str(roles))
 135                 log.info('target: ' + str(target))
 136                 if properties['rgw_server'] in roles:
 137                     _, host = split_user(target)
 138             assert host is not None, "Invalid client specified as the rgw_server"
 139             testswift_conf['func_test']['auth_host'] = host
 140         else:
 141             testswift_conf['func_test']['auth_host'] = 'localhost'
 142
 143         log.info(client)
 144         (remote,) = ctx.cluster.only(client).remotes.keys()
 145         remote.run(
 146             args=[
 147                 'cd',
 148                 '{tdir}/swift'.format(tdir=testdir),
 149                 run.Raw('&&'),
 150                 './bootstrap',
 151                 ],
 152             )
 153         conf_fp = StringIO()
 154         testswift_conf.write(conf_fp)
 155         teuthology.write_file(
 156             remote=remote,
 157             path='{tdir}/archive/testswift.{client}.conf'.format(tdir=testdir, client=client),
 158             data=conf_fp.getvalue(),
 159             )
 160     yield
 161
 162
 163 @contextlib.contextmanager
 164 def run_tests(ctx, config):
 165     """
 166     Run an individual Swift test.
 167     """
 168     assert isinstance(config, dict)
 169     testdir = teuthology.get_testdir(ctx)
 170     for client, client_config in config.iteritems():
 171         args = [
 172                 'SWIFT_TEST_CONFIG_FILE={tdir}/archive/testswift.{client}.conf'.format(tdir=testdir, client=client),
 173                 '{tdir}/swift/virtualenv/bin/nosetests'.format(tdir=testdir),
 174                 '-w',
 175                 '{tdir}/swift/test/functional'.format(tdir=testdir),
 176                 '-v',
 177                 '-a', '!fails_on_rgw',
 178                 ]
 179         if client_config is not None and 'extra_args' in client_config:
 180             args.extend(client_config['extra_args'])
 181
 182         ctx.cluster.only(client).run(
 183             args=args,
 184             )
 185     yield
 186
 187 @contextlib.contextmanager
 188 def task(ctx, config):
 189     """
 190     Run the testswift suite against rgw.
 191
 192     To run all tests on all clients::
 193
 194         tasks:
 195         - ceph:
 196         - rgw:
 197         - testswift:
 198
 199     To restrict testing to particular clients::
 200
 201         tasks:
 202         - ceph:
 203         - rgw: [client.0]
 204         - testswift: [client.0]
 205
 206     To run against a server on client.1::
 207
 208         tasks:
 209         - ceph:
 210         - rgw: [client.1]
 211         - testswift:
 212             client.0:
 213               rgw_server: client.1
 214
 215     To pass extra arguments to nose (e.g. to run a certain test)::
 216
 217         tasks:
 218         - ceph:
 219         - rgw: [client.0]
 220         - testswift:
 221             client.0:
 222               extra_args: ['test.functional.tests:TestFileUTF8', '-m', 'testCopy']
 223             client.1:
 224               extra_args: ['--exclude', 'TestFile']
 225     """
 226     assert config is None or isinstance(config, list) \
 227         or isinstance(config, dict), \
 228         "task testswift only supports a list or dictionary for configuration"
 229     all_clients = ['client.{id}'.format(id=id_)
 230                    for id_ in teuthology.all_roles_of_type(ctx.cluster, 'client')]
 231     if config is None:
 232         config = all_clients
 233     if isinstance(config, list):
 234         config = dict.fromkeys(config)
 235     clients = config.keys()
 236
 237     log.info('clients={c}'.format(c=clients))
 238
 239     testswift_conf = {}
 240     for client in clients:
 241         testswift_conf[client] = ConfigObj(
 242                 indent_type='',
 243                 infile={
 244                     'func_test':
 245                         {
 246                         'auth_port'      : 7280,
 247                         'auth_ssl' : 'no',
 248                         'auth_prefix' : '/auth/',
 249                         },
 250                     }
 251                 )
 252
 253     with contextutil.nested(
 254         lambda: download(ctx=ctx, config=clients),
 255         lambda: create_users(ctx=ctx, config=dict(
 256                 clients=clients,
 257                 testswift_conf=testswift_conf,
 258                 )),
 259         lambda: configure(ctx=ctx, config=dict(
 260                 clients=config,
 261                 testswift_conf=testswift_conf,
 262                 )),
 263         lambda: run_tests(ctx=ctx, config=config),
 264         ):
 265         pass
 266     yield