ceph/qa/tasks/s3tests.py

   1 """
   2 Run a set of s3 tests on rgw.
   3 """
   4 from cStringIO import StringIO
   5 from configobj import ConfigObj
   6 import base64
   7 import contextlib
   8 import logging
   9 import os
  10 import random
  11 import string
  12
  13 from teuthology import misc as teuthology
  14 from teuthology import contextutil
  15 from teuthology.config import config as teuth_config
  16 from teuthology.orchestra import run
  17 from teuthology.orchestra.connection import split_user
  18
  19 log = logging.getLogger(__name__)
  20
  21 @contextlib.contextmanager
  22 def download(ctx, config):
  23     """
  24     Download the s3 tests from the git builder.
  25     Remove downloaded s3 file upon exit.
  26
  27     The context passed in should be identical to the context
  28     passed in to the main task.
  29     """
  30     assert isinstance(config, dict)
  31     log.info('Downloading s3-tests...')
  32     testdir = teuthology.get_testdir(ctx)
  33     for (client, client_config) in config.items():
  34         s3tests_branch = client_config.get('force-branch', None)
  35         if not s3tests_branch:
  36             raise ValueError(
  37                 "Could not determine what branch to use for s3-tests. Please add 'force-branch: {s3-tests branch name}' to the .yaml config for this s3tests task.")
  38
  39         log.info("Using branch '%s' for s3tests", s3tests_branch)
  40         sha1 = client_config.get('sha1')
  41         git_remote = client_config.get('git_remote', teuth_config.ceph_git_base_url)
  42         ctx.cluster.only(client).run(
  43             args=[
  44                 'git', 'clone',
  45                 '-b', s3tests_branch,
  46                 git_remote + 's3-tests.git',
  47                 '{tdir}/s3-tests'.format(tdir=testdir),
  48                 ],
  49             )
  50         if sha1 is not None:
  51             ctx.cluster.only(client).run(
  52                 args=[
  53                     'cd', '{tdir}/s3-tests'.format(tdir=testdir),
  54                     run.Raw('&&'),
  55                     'git', 'reset', '--hard', sha1,
  56                     ],
  57                 )
  58     try:
  59         yield
  60     finally:
  61         log.info('Removing s3-tests...')
  62         testdir = teuthology.get_testdir(ctx)
  63         for client in config:
  64             ctx.cluster.only(client).run(
  65                 args=[
  66                     'rm',
  67                     '-rf',
  68                     '{tdir}/s3-tests'.format(tdir=testdir),
  69                     ],
  70                 )
  71
  72
  73 def _config_user(s3tests_conf, section, user):
  74     """
  75     Configure users for this section by stashing away keys, ids, and
  76     email addresses.
  77     """
  78     s3tests_conf[section].setdefault('user_id', user)
  79     s3tests_conf[section].setdefault('email', '{user}+test@test.test'.format(user=user))
  80     s3tests_conf[section].setdefault('display_name', 'Mr. {user}'.format(user=user))
  81     s3tests_conf[section].setdefault('access_key', ''.join(random.choice(string.uppercase) for i in xrange(20)))
  82     s3tests_conf[section].setdefault('secret_key', base64.b64encode(os.urandom(40)))
  83     s3tests_conf[section].setdefault('totp_serial', ''.join(random.choice(string.digits) for i in xrange(10)))
  84     s3tests_conf[section].setdefault('totp_seed', base64.b32encode(os.urandom(40)))
  85     s3tests_conf[section].setdefault('totp_seconds', '5')
  86
  87
  88 @contextlib.contextmanager
  89 def create_users(ctx, config):
  90     """
  91     Create a main and an alternate s3 user.
  92     """
  93     assert isinstance(config, dict)
  94     log.info('Creating rgw users...')
  95     testdir = teuthology.get_testdir(ctx)
  96     users = {'s3 main': 'foo', 's3 alt': 'bar', 's3 tenant': 'testx$tenanteduser'}
  97     for client in config['clients']:
  98         s3tests_conf = config['s3tests_conf'][client]
  99         s3tests_conf.setdefault('fixtures', {})
 100         s3tests_conf['fixtures'].setdefault('bucket prefix', 'test-' + client + '-{random}-')
 101         for section, user in users.iteritems():
 102             _config_user(s3tests_conf, section, '{user}.{client}'.format(user=user, client=client))
 103             log.debug('Creating user {user} on {host}'.format(user=s3tests_conf[section]['user_id'], host=client))
 104             cluster_name, daemon_type, client_id = teuthology.split_role(client)
 105             client_with_id = daemon_type + '.' + client_id
 106             ctx.cluster.only(client).run(
 107                 args=[
 108                     'adjust-ulimits',
 109                     'ceph-coverage',
 110                     '{tdir}/archive/coverage'.format(tdir=testdir),
 111                     'radosgw-admin',
 112                     '-n', client_with_id,
 113                     'user', 'create',
 114                     '--uid', s3tests_conf[section]['user_id'],
 115                     '--display-name', s3tests_conf[section]['display_name'],
 116                     '--access-key', s3tests_conf[section]['access_key'],
 117                     '--secret', s3tests_conf[section]['secret_key'],
 118                     '--email', s3tests_conf[section]['email'],
 119                     '--cluster', cluster_name,
 120                 ],
 121             )
 122             ctx.cluster.only(client).run(
 123                 args=[
 124                     'adjust-ulimits',
 125                     'ceph-coverage',
 126                     '{tdir}/archive/coverage'.format(tdir=testdir),
 127                     'radosgw-admin',
 128                     '-n', client_with_id,
 129                     'mfa', 'create',
 130                     '--uid', s3tests_conf[section]['user_id'],
 131                     '--totp-serial', s3tests_conf[section]['totp_serial'],
 132                     '--totp-seed', s3tests_conf[section]['totp_seed'],
 133                     '--totp-seconds', s3tests_conf[section]['totp_seconds'],
 134                     '--totp-window', '8',
 135                     '--totp-seed-type', 'base32',
 136                     '--cluster', cluster_name,
 137                 ],
 138             )
 139     try:
 140         yield
 141     finally:
 142         for client in config['clients']:
 143             for user in users.itervalues():
 144                 uid = '{user}.{client}'.format(user=user, client=client)
 145                 cluster_name, daemon_type, client_id = teuthology.split_role(client)
 146                 client_with_id = daemon_type + '.' + client_id
 147                 ctx.cluster.only(client).run(
 148                     args=[
 149                         'adjust-ulimits',
 150                         'ceph-coverage',
 151                         '{tdir}/archive/coverage'.format(tdir=testdir),
 152                         'radosgw-admin',
 153                         '-n', client_with_id,
 154                         'user', 'rm',
 155                         '--uid', uid,
 156                         '--purge-data',
 157                         '--cluster', cluster_name,
 158                         ],
 159                     )
 160
 161
 162 @contextlib.contextmanager
 163 def configure(ctx, config):
 164     """
 165     Configure the s3-tests.  This includes the running of the
 166     bootstrap code and the updating of local conf files.
 167     """
 168     assert isinstance(config, dict)
 169     log.info('Configuring s3-tests...')
 170     testdir = teuthology.get_testdir(ctx)
 171     for client, properties in config['clients'].iteritems():
 172         s3tests_conf = config['s3tests_conf'][client]
 173         if properties is not None and 'rgw_server' in properties:
 174             host = None
 175             for target, roles in zip(ctx.config['targets'].iterkeys(), ctx.config['roles']):
 176                 log.info('roles: ' + str(roles))
 177                 log.info('target: ' + str(target))
 178                 if properties['rgw_server'] in roles:
 179                     _, host = split_user(target)
 180             assert host is not None, "Invalid client specified as the rgw_server"
 181             s3tests_conf['DEFAULT']['host'] = host
 182         else:
 183             s3tests_conf['DEFAULT']['host'] = 'localhost'
 184
 185         if properties is not None and 'slow_backend' in properties:
 186             s3tests_conf['fixtures']['slow backend'] = properties['slow_backend']
 187
 188         (remote,) = ctx.cluster.only(client).remotes.keys()
 189         remote.run(
 190             args=[
 191                 'cd',
 192                 '{tdir}/s3-tests'.format(tdir=testdir),
 193                 run.Raw('&&'),
 194                 './bootstrap',
 195                 ],
 196             )
 197         conf_fp = StringIO()
 198         s3tests_conf.write(conf_fp)
 199         teuthology.write_file(
 200             remote=remote,
 201             path='{tdir}/archive/s3-tests.{client}.conf'.format(tdir=testdir, client=client),
 202             data=conf_fp.getvalue(),
 203             )
 204
 205     log.info('Configuring boto...')
 206     boto_src = os.path.join(os.path.dirname(__file__), 'boto.cfg.template')
 207     for client, properties in config['clients'].iteritems():
 208         with file(boto_src, 'rb') as f:
 209             (remote,) = ctx.cluster.only(client).remotes.keys()
 210             conf = f.read().format(
 211                 idle_timeout=config.get('idle_timeout', 30)
 212                 )
 213             teuthology.write_file(
 214                 remote=remote,
 215                 path='{tdir}/boto.cfg'.format(tdir=testdir),
 216                 data=conf,
 217                 )
 218
 219     try:
 220         yield
 221
 222     finally:
 223         log.info('Cleaning up boto...')
 224         for client, properties in config['clients'].iteritems():
 225             (remote,) = ctx.cluster.only(client).remotes.keys()
 226             remote.run(
 227                 args=[
 228                     'rm',
 229                     '{tdir}/boto.cfg'.format(tdir=testdir),
 230                     ],
 231                 )
 232
 233 @contextlib.contextmanager
 234 def run_tests(ctx, config):
 235     """
 236     Run the s3tests after everything is set up.
 237
 238     :param ctx: Context passed to task
 239     :param config: specific configuration information
 240     """
 241     assert isinstance(config, dict)
 242     testdir = teuthology.get_testdir(ctx)
 243     # civetweb > 1.8 && beast parsers are strict on rfc2616
 244     attrs = ["!fails_on_rgw", "!lifecycle_expiration", "!fails_strict_rfc2616"]
 245     for client, client_config in config.iteritems():
 246         (remote,) = ctx.cluster.only(client).remotes.keys()
 247         args = [
 248             'S3TEST_CONF={tdir}/archive/s3-tests.{client}.conf'.format(tdir=testdir, client=client),
 249             'BOTO_CONFIG={tdir}/boto.cfg'.format(tdir=testdir)
 250             ]
 251         # the 'requests' library comes with its own ca bundle to verify ssl
 252         # certificates - override that to use the system's ca bundle, which
 253         # is where the ssl task installed this certificate
 254         if remote.os.package_type == 'deb':
 255             args += ['REQUESTS_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt']
 256         else:
 257             args += ['REQUESTS_CA_BUNDLE=/etc/pki/tls/certs/ca-bundle.crt']
 258         args += [
 259             '{tdir}/s3-tests/virtualenv/bin/nosetests'.format(tdir=testdir),
 260             '-w',
 261             '{tdir}/s3-tests'.format(tdir=testdir),
 262             '-v',
 263             '-a', ','.join(attrs),
 264             ]
 265         if client_config is not None and 'extra_args' in client_config:
 266             args.extend(client_config['extra_args'])
 267
 268         remote.run(
 269             args=args,
 270             label="s3 tests against rgw"
 271             )
 272     yield
 273
 274 @contextlib.contextmanager
 275 def scan_for_leaked_encryption_keys(ctx, config):
 276     """
 277     Scan radosgw logs for the encryption keys used by s3tests to
 278     verify that we're not leaking secrets.
 279
 280     :param ctx: Context passed to task
 281     :param config: specific configuration information
 282     """
 283     assert isinstance(config, dict)
 284
 285     try:
 286         yield
 287     finally:
 288         # x-amz-server-side-encryption-customer-key
 289         s3test_customer_key = 'pO3upElrwuEXSoFwCfnZPdSsmt/xWeFa0N9KgDijwVs='
 290
 291         log.debug('Scanning radosgw logs for leaked encryption keys...')
 292         procs = list()
 293         for client, client_config in config.iteritems():
 294             if not client_config.get('scan_for_encryption_keys', True):
 295                 continue
 296             cluster_name, daemon_type, client_id = teuthology.split_role(client)
 297             client_with_cluster = '.'.join((cluster_name, daemon_type, client_id))
 298             (remote,) = ctx.cluster.only(client).remotes.keys()
 299             proc = remote.run(
 300                 args=[
 301                     'grep',
 302                     '--binary-files=text',
 303                     s3test_customer_key,
 304                     '/var/log/ceph/rgw.{client}.log'.format(client=client_with_cluster),
 305                 ],
 306                 wait=False,
 307                 check_status=False,
 308             )
 309             procs.append(proc)
 310
 311         for proc in procs:
 312             proc.wait()
 313             if proc.returncode == 1: # 1 means no matches
 314                 continue
 315             log.error('radosgw log is leaking encryption keys!')
 316             raise Exception('radosgw log is leaking encryption keys')
 317
 318 @contextlib.contextmanager
 319 def task(ctx, config):
 320     """
 321     Run the s3-tests suite against rgw.
 322
 323     To run all tests on all clients::
 324
 325         tasks:
 326         - ceph:
 327         - rgw:
 328         - s3tests:
 329
 330     To restrict testing to particular clients::
 331
 332         tasks:
 333         - ceph:
 334         - rgw: [client.0]
 335         - s3tests:
 336             client.0:
 337               force-branch: ceph-nautilus
 338
 339     To run against a server on client.1 and increase the boto timeout to 10m::
 340
 341         tasks:
 342         - ceph:
 343         - rgw: [client.1]
 344         - s3tests:
 345             client.0:
 346               force-branch: ceph-nautilus
 347               rgw_server: client.1
 348               idle_timeout: 600
 349
 350     To pass extra arguments to nose (e.g. to run a certain test)::
 351
 352         tasks:
 353         - ceph:
 354         - rgw: [client.0]
 355         - s3tests:
 356             client.0:
 357               force-branch: ceph-nautilus
 358               extra_args: ['test_s3:test_object_acl_grand_public_read']
 359             client.1:
 360               force-branch: ceph-nautilus
 361               extra_args: ['--exclude', 'test_100_continue']
 362     """
 363     assert hasattr(ctx, 'rgw'), 's3tests must run after the rgw task'
 364     assert config is None or isinstance(config, list) \
 365         or isinstance(config, dict), \
 366         "task s3tests only supports a list or dictionary for configuration"
 367     all_clients = ['client.{id}'.format(id=id_)
 368                    for id_ in teuthology.all_roles_of_type(ctx.cluster, 'client')]
 369     if config is None:
 370         config = all_clients
 371     if isinstance(config, list):
 372         config = dict.fromkeys(config)
 373     clients = config.keys()
 374
 375     overrides = ctx.config.get('overrides', {})
 376     # merge each client section, not the top level.
 377     for client in config.iterkeys():
 378         if not config[client]:
 379             config[client] = {}
 380         teuthology.deep_merge(config[client], overrides.get('s3tests', {}))
 381
 382     log.debug('s3tests config is %s', config)
 383
 384     s3tests_conf = {}
 385     for client in clients:
 386         endpoint = ctx.rgw.role_endpoints.get(client)
 387         assert endpoint, 's3tests: no rgw endpoint for {}'.format(client)
 388
 389         s3tests_conf[client] = ConfigObj(
 390             indent_type='',
 391             infile={
 392                 'DEFAULT':
 393                     {
 394                     'port'      : endpoint.port,
 395                     'is_secure' : endpoint.cert is not None,
 396                     'api_name'  : 'default',
 397                     },
 398                 'fixtures' : {},
 399                 's3 main'  : {},
 400                 's3 alt'   : {},
 401                 's3 tenant': {},
 402                 }
 403             )
 404
 405     with contextutil.nested(
 406         lambda: download(ctx=ctx, config=config),
 407         lambda: create_users(ctx=ctx, config=dict(
 408                 clients=clients,
 409                 s3tests_conf=s3tests_conf,
 410                 )),
 411         lambda: configure(ctx=ctx, config=dict(
 412                 clients=config,
 413                 s3tests_conf=s3tests_conf,
 414                 )),
 415         lambda: run_tests(ctx=ctx, config=config),
 416         lambda: scan_for_leaked_encryption_keys(ctx=ctx, config=config),
 417         ):
 418         pass
 419     yield