ceph/qa/tasks/s3tests.py

   1 """
   2 Run a set of s3 tests on rgw.
   3 """
   4 from cStringIO import StringIO
   5 from configobj import ConfigObj
   6 import base64
   7 import contextlib
   8 import logging
   9 import os
  10 import random
  11 import string
  12
  13 from teuthology import misc as teuthology
  14 from teuthology import contextutil
  15 from teuthology.config import config as teuth_config
  16 from teuthology.orchestra import run
  17 from teuthology.orchestra.connection import split_user
  18
  19 log = logging.getLogger(__name__)
  20
  21 @contextlib.contextmanager
  22 def download(ctx, config):
  23     """
  24     Download the s3 tests from the git builder.
  25     Remove downloaded s3 file upon exit.
  26
  27     The context passed in should be identical to the context
  28     passed in to the main task.
  29     """
  30     assert isinstance(config, dict)
  31     log.info('Downloading s3-tests...')
  32     testdir = teuthology.get_testdir(ctx)
  33     s3_branches = [ 'giant', 'firefly', 'firefly-original', 'hammer' ]
  34     for (client, cconf) in config.items():
  35         branch = cconf.get('force-branch', None)
  36         if not branch:
  37             ceph_branch = ctx.config.get('branch')
  38             suite_branch = ctx.config.get('suite_branch', ceph_branch)
  39             if suite_branch in s3_branches:
  40                 branch = cconf.get('branch', suite_branch)
  41             else:
  42                 branch = cconf.get('branch', 'ceph-' + suite_branch)
  43         if not branch:
  44             raise ValueError(
  45                 "Could not determine what branch to use for s3tests!")
  46         else:
  47             log.info("Using branch '%s' for s3tests", branch)
  48         sha1 = cconf.get('sha1')
  49         git_remote = cconf.get('git_remote', None) or teuth_config.ceph_git_base_url
  50         ctx.cluster.only(client).run(
  51             args=[
  52                 'git', 'clone',
  53                 '-b', branch,
  54                 git_remote + 's3-tests.git',
  55                 '{tdir}/s3-tests'.format(tdir=testdir),
  56                 ],
  57             )
  58         if sha1 is not None:
  59             ctx.cluster.only(client).run(
  60                 args=[
  61                     'cd', '{tdir}/s3-tests'.format(tdir=testdir),
  62                     run.Raw('&&'),
  63                     'git', 'reset', '--hard', sha1,
  64                     ],
  65                 )
  66     try:
  67         yield
  68     finally:
  69         log.info('Removing s3-tests...')
  70         testdir = teuthology.get_testdir(ctx)
  71         for client in config:
  72             ctx.cluster.only(client).run(
  73                 args=[
  74                     'rm',
  75                     '-rf',
  76                     '{tdir}/s3-tests'.format(tdir=testdir),
  77                     ],
  78                 )
  79
  80
  81 def _config_user(s3tests_conf, section, user):
  82     """
  83     Configure users for this section by stashing away keys, ids, and
  84     email addresses.
  85     """
  86     s3tests_conf[section].setdefault('user_id', user)
  87     s3tests_conf[section].setdefault('email', '{user}+test@test.test'.format(user=user))
  88     s3tests_conf[section].setdefault('display_name', 'Mr. {user}'.format(user=user))
  89     s3tests_conf[section].setdefault('access_key', ''.join(random.choice(string.uppercase) for i in xrange(20)))
  90     s3tests_conf[section].setdefault('secret_key', base64.b64encode(os.urandom(40)))
  91
  92
  93 @contextlib.contextmanager
  94 def create_users(ctx, config):
  95     """
  96     Create a main and an alternate s3 user.
  97     """
  98     assert isinstance(config, dict)
  99     log.info('Creating rgw users...')
 100     testdir = teuthology.get_testdir(ctx)
 101     users = {'s3 main': 'foo', 's3 alt': 'bar', 's3 tenant': 'testx$tenanteduser'}
 102     for client in config['clients']:
 103         s3tests_conf = config['s3tests_conf'][client]
 104         s3tests_conf.setdefault('fixtures', {})
 105         s3tests_conf['fixtures'].setdefault('bucket prefix', 'test-' + client + '-{random}-')
 106         for section, user in users.iteritems():
 107             _config_user(s3tests_conf, section, '{user}.{client}'.format(user=user, client=client))
 108             log.debug('Creating user {user} on {host}'.format(user=s3tests_conf[section]['user_id'], host=client))
 109             cluster_name, daemon_type, client_id = teuthology.split_role(client)
 110             client_with_id = daemon_type + '.' + client_id
 111             ctx.cluster.only(client).run(
 112                 args=[
 113                     'adjust-ulimits',
 114                     'ceph-coverage',
 115                     '{tdir}/archive/coverage'.format(tdir=testdir),
 116                     'radosgw-admin',
 117                     '-n', client_with_id,
 118                     'user', 'create',
 119                     '--uid', s3tests_conf[section]['user_id'],
 120                     '--display-name', s3tests_conf[section]['display_name'],
 121                     '--access-key', s3tests_conf[section]['access_key'],
 122                     '--secret', s3tests_conf[section]['secret_key'],
 123                     '--email', s3tests_conf[section]['email'],
 124                     '--cluster', cluster_name,
 125                 ],
 126             )
 127     try:
 128         yield
 129     finally:
 130         for client in config['clients']:
 131             for user in users.itervalues():
 132                 uid = '{user}.{client}'.format(user=user, client=client)
 133                 cluster_name, daemon_type, client_id = teuthology.split_role(client)
 134                 client_with_id = daemon_type + '.' + client_id
 135                 ctx.cluster.only(client).run(
 136                     args=[
 137                         'adjust-ulimits',
 138                         'ceph-coverage',
 139                         '{tdir}/archive/coverage'.format(tdir=testdir),
 140                         'radosgw-admin',
 141                         '-n', client_with_id,
 142                         'user', 'rm',
 143                         '--uid', uid,
 144                         '--purge-data',
 145                         '--cluster', cluster_name,
 146                         ],
 147                     )
 148
 149
 150 @contextlib.contextmanager
 151 def configure(ctx, config):
 152     """
 153     Configure the s3-tests.  This includes the running of the
 154     bootstrap code and the updating of local conf files.
 155     """
 156     assert isinstance(config, dict)
 157     log.info('Configuring s3-tests...')
 158     testdir = teuthology.get_testdir(ctx)
 159     for client, properties in config['clients'].iteritems():
 160         s3tests_conf = config['s3tests_conf'][client]
 161         if properties is not None and 'rgw_server' in properties:
 162             host = None
 163             for target, roles in zip(ctx.config['targets'].iterkeys(), ctx.config['roles']):
 164                 log.info('roles: ' + str(roles))
 165                 log.info('target: ' + str(target))
 166                 if properties['rgw_server'] in roles:
 167                     _, host = split_user(target)
 168             assert host is not None, "Invalid client specified as the rgw_server"
 169             s3tests_conf['DEFAULT']['host'] = host
 170         else:
 171             s3tests_conf['DEFAULT']['host'] = 'localhost'
 172
 173         if properties is not None and 'slow_backend' in properties:
 174             s3tests_conf['fixtures']['slow backend'] = properties['slow_backend']
 175
 176         (remote,) = ctx.cluster.only(client).remotes.keys()
 177         remote.run(
 178             args=[
 179                 'cd',
 180                 '{tdir}/s3-tests'.format(tdir=testdir),
 181                 run.Raw('&&'),
 182                 './bootstrap',
 183                 ],
 184             )
 185         conf_fp = StringIO()
 186         s3tests_conf.write(conf_fp)
 187         teuthology.write_file(
 188             remote=remote,
 189             path='{tdir}/archive/s3-tests.{client}.conf'.format(tdir=testdir, client=client),
 190             data=conf_fp.getvalue(),
 191             )
 192
 193     log.info('Configuring boto...')
 194     boto_src = os.path.join(os.path.dirname(__file__), 'boto.cfg.template')
 195     for client, properties in config['clients'].iteritems():
 196         with file(boto_src, 'rb') as f:
 197             (remote,) = ctx.cluster.only(client).remotes.keys()
 198             conf = f.read().format(
 199                 idle_timeout=config.get('idle_timeout', 30)
 200                 )
 201             teuthology.write_file(
 202                 remote=remote,
 203                 path='{tdir}/boto.cfg'.format(tdir=testdir),
 204                 data=conf,
 205                 )
 206
 207     try:
 208         yield
 209
 210     finally:
 211         log.info('Cleaning up boto...')
 212         for client, properties in config['clients'].iteritems():
 213             (remote,) = ctx.cluster.only(client).remotes.keys()
 214             remote.run(
 215                 args=[
 216                     'rm',
 217                     '{tdir}/boto.cfg'.format(tdir=testdir),
 218                     ],
 219                 )
 220
 221 @contextlib.contextmanager
 222 def run_tests(ctx, config):
 223     """
 224     Run the s3tests after everything is set up.
 225
 226     :param ctx: Context passed to task
 227     :param config: specific configuration information
 228     """
 229     assert isinstance(config, dict)
 230     testdir = teuthology.get_testdir(ctx)
 231     attrs = ["!fails_on_rgw", "!lifecycle"]
 232     # beast parser is strict about unreadable headers
 233     if ctx.rgw.frontend == 'beast':
 234         attrs.append("!fails_strict_rfc2616")
 235     for client, client_config in config.iteritems():
 236         args = [
 237             'S3TEST_CONF={tdir}/archive/s3-tests.{client}.conf'.format(tdir=testdir, client=client),
 238             'BOTO_CONFIG={tdir}/boto.cfg'.format(tdir=testdir),
 239             '{tdir}/s3-tests/virtualenv/bin/nosetests'.format(tdir=testdir),
 240             '-w',
 241             '{tdir}/s3-tests'.format(tdir=testdir),
 242             '-v',
 243             '-a', ','.join(attrs),
 244             ]
 245         if client_config is not None and 'extra_args' in client_config:
 246             args.extend(client_config['extra_args'])
 247
 248         ctx.cluster.only(client).run(
 249             args=args,
 250             label="s3 tests against rgw"
 251             )
 252     yield
 253
 254 @contextlib.contextmanager
 255 def scan_for_leaked_encryption_keys(ctx, config):
 256     """
 257     Scan radosgw logs for the encryption keys used by s3tests to
 258     verify that we're not leaking secrets.
 259
 260     :param ctx: Context passed to task
 261     :param config: specific configuration information
 262     """
 263     assert isinstance(config, dict)
 264
 265     try:
 266         yield
 267     finally:
 268         # x-amz-server-side-encryption-customer-key
 269         s3test_customer_key = 'pO3upElrwuEXSoFwCfnZPdSsmt/xWeFa0N9KgDijwVs='
 270
 271         log.debug('Scanning radosgw logs for leaked encryption keys...')
 272         procs = list()
 273         for client, client_config in config.iteritems():
 274             if not client_config.get('scan_for_encryption_keys', True):
 275                 continue
 276             cluster_name, daemon_type, client_id = teuthology.split_role(client)
 277             client_with_cluster = '.'.join((cluster_name, daemon_type, client_id))
 278             (remote,) = ctx.cluster.only(client).remotes.keys()
 279             proc = remote.run(
 280                 args=[
 281                     'grep',
 282                     '--binary-files=text',
 283                     s3test_customer_key,
 284                     '/var/log/ceph/rgw.{client}.log'.format(client=client_with_cluster),
 285                 ],
 286                 wait=False,
 287                 check_status=False,
 288             )
 289             procs.append(proc)
 290
 291         for proc in procs:
 292             proc.wait()
 293             if proc.returncode == 1: # 1 means no matches
 294                 continue
 295             log.error('radosgw log is leaking encryption keys!')
 296             raise Exception('radosgw log is leaking encryption keys')
 297
 298 @contextlib.contextmanager
 299 def task(ctx, config):
 300     """
 301     Run the s3-tests suite against rgw.
 302
 303     To run all tests on all clients::
 304
 305         tasks:
 306         - ceph:
 307         - rgw:
 308         - s3tests:
 309
 310     To restrict testing to particular clients::
 311
 312         tasks:
 313         - ceph:
 314         - rgw: [client.0]
 315         - s3tests: [client.0]
 316
 317     To run against a server on client.1 and increase the boto timeout to 10m::
 318
 319         tasks:
 320         - ceph:
 321         - rgw: [client.1]
 322         - s3tests:
 323             client.0:
 324               rgw_server: client.1
 325               idle_timeout: 600
 326
 327     To pass extra arguments to nose (e.g. to run a certain test)::
 328
 329         tasks:
 330         - ceph:
 331         - rgw: [client.0]
 332         - s3tests:
 333             client.0:
 334               extra_args: ['test_s3:test_object_acl_grand_public_read']
 335             client.1:
 336               extra_args: ['--exclude', 'test_100_continue']
 337     """
 338     assert config is None or isinstance(config, list) \
 339         or isinstance(config, dict), \
 340         "task s3tests only supports a list or dictionary for configuration"
 341     all_clients = ['client.{id}'.format(id=id_)
 342                    for id_ in teuthology.all_roles_of_type(ctx.cluster, 'client')]
 343     if config is None:
 344         config = all_clients
 345     if isinstance(config, list):
 346         config = dict.fromkeys(config)
 347     clients = config.keys()
 348
 349     overrides = ctx.config.get('overrides', {})
 350     # merge each client section, not the top level.
 351     for client in config.iterkeys():
 352         if not config[client]:
 353             config[client] = {}
 354         teuthology.deep_merge(config[client], overrides.get('s3tests', {}))
 355
 356     log.debug('s3tests config is %s', config)
 357
 358     s3tests_conf = {}
 359     for client in clients:
 360         s3tests_conf[client] = ConfigObj(
 361             indent_type='',
 362             infile={
 363                 'DEFAULT':
 364                     {
 365                     'port'      : 7280,
 366                     'is_secure' : 'no',
 367                     },
 368                 'fixtures' : {},
 369                 's3 main'  : {},
 370                 's3 alt'   : {},
 371                 's3 tenant': {},
 372                 }
 373             )
 374
 375     with contextutil.nested(
 376         lambda: download(ctx=ctx, config=config),
 377         lambda: create_users(ctx=ctx, config=dict(
 378                 clients=clients,
 379                 s3tests_conf=s3tests_conf,
 380                 )),
 381         lambda: configure(ctx=ctx, config=dict(
 382                 clients=config,
 383                 s3tests_conf=s3tests_conf,
 384                 )),
 385         lambda: run_tests(ctx=ctx, config=config),
 386         lambda: scan_for_leaked_encryption_keys(ctx=ctx, config=config),
 387         ):
 388         pass
 389     yield