ceph/qa/tasks/s3tests_java.py

   1 """
   2 Task for running RGW S3 tests with the AWS Java SDK
   3 """
   4 from io import BytesIO
   5 import logging
   6
   7 import base64
   8 import os
   9 import random
  10 import string
  11 import yaml
  12 import getpass
  13
  14 from teuthology import misc as teuthology
  15 from teuthology.task import Task
  16 from teuthology.orchestra import run
  17
  18 log = logging.getLogger(__name__)
  19
  20 """
  21     Task for running RGW S3 tests with the AWS Java SDK
  22
  23     Tests run only on clients specified in the s3tests-java config section.
  24     If no client is given a default 'client.0' is chosen.
  25     If such does not match the rgw client the task will fail.
  26
  27         tasks:
  28         - ceph:
  29         - rgw: [client.0]
  30         - s3tests-java:
  31             client.0:
  32
  33     Extra arguments can be passed by adding options to the corresponding client
  34     section under the s3tests-java task (e.g. to run a certain test,
  35     specify a different repository and branch for the test suite,
  36     run in info/debug mode (for the java suite) or forward the gradle output to a log file):
  37
  38         tasks:
  39         - ceph:
  40         - rgw: [client.0]
  41         - s3tests-java:
  42             client.0:
  43                 force-branch: wip
  44                 force-repo: 'https://github.com/adamyanova/java_s3tests.git'
  45                 log-fwd: '../s3tests-java.log'
  46                 log-level: info
  47                 extra-args: ['--tests', 'ObjectTest.testEncryptionKeySSECInvalidMd5']
  48
  49     To run a specific test, provide its name to the extra-args section e.g.:
  50         - s3tests-java:
  51             client.0:
  52                 extra-args: ['--tests', 'ObjectTest.testEncryptionKeySSECInvalidMd5']
  53
  54 """
  55
  56
  57 class S3tests_java(Task):
  58     """
  59     Download and install S3 tests in Java
  60     This will require openjdk and gradle
  61     """
  62
  63     def __init__(self, ctx, config):
  64         super(S3tests_java, self).__init__(ctx, config)
  65         self.log = log
  66         log.debug('S3 Tests Java: __INIT__ ')
  67         assert hasattr(ctx, 'rgw'), 'S3tests_java must run after the rgw task'
  68         clients = ['client.{id}'.format(id=id_)
  69                    for id_ in teuthology.all_roles_of_type(self.ctx.cluster, 'client')]
  70         self.all_clients = []
  71         for client in clients:
  72             if client in self.config:
  73                 self.all_clients.extend([client])
  74         if self.all_clients is None:
  75             self.all_clients = 'client.0'
  76         self.users = {'s3main': 'tester',
  77                       's3alt': 'johndoe', 'tenanted': 'testx$tenanteduser'}
  78
  79     def setup(self):
  80         super(S3tests_java, self).setup()
  81         log.debug('S3 Tests Java: SETUP')
  82         for client in self.all_clients:
  83             self.download_test_suite(client)
  84             self.install_required_packages(client)
  85
  86     def begin(self):
  87         super(S3tests_java, self).begin()
  88         log.debug('S3 Tests Java: BEGIN')
  89         for (host, roles) in self.ctx.cluster.remotes.items():
  90             log.debug(
  91                 'S3 Tests Java: Cluster config is: {cfg}'.format(cfg=roles))
  92             log.debug('S3 Tests Java: Host is: {host}'.format(host=host))
  93         self.create_users()
  94         self.run_tests()
  95
  96     def end(self):
  97         super(S3tests_java, self).end()
  98         log.debug('S3 Tests Java: END')
  99         for client in self.all_clients:
 100             self.remove_tests(client)
 101             self.delete_users(client)
 102
 103     def download_test_suite(self, client):
 104         log.info("S3 Tests Java: Downloading test suite...")
 105         testdir = teuthology.get_testdir(self.ctx)
 106         branch = 'master'
 107         repo = 'https://github.com/ceph/java_s3tests.git'
 108         if client in self.config and self.config[client] is not None:
 109             if 'force-branch' in self.config[client] and self.config[client]['force-branch'] is not None:
 110                 branch = self.config[client]['force-branch']
 111             if 'force-repo' in self.config[client] and self.config[client]['force-repo'] is not None:
 112                 repo = self.config[client]['force-repo']
 113         self.ctx.cluster.only(client).run(
 114             args=[
 115                 'git', 'clone',
 116                 '-b', branch,
 117                 repo,
 118                 '{tdir}/s3-tests-java'.format(tdir=testdir),
 119             ],
 120             stdout=BytesIO()
 121         )
 122         if client in self.config and self.config[client] is not None:
 123             if 'sha1' in self.config[client] and self.config[client]['sha1'] is not None:
 124                 self.ctx.cluster.only(client).run(
 125                     args=[
 126                         'cd', '{tdir}/s3-tests-java'.format(tdir=testdir),
 127                         run.Raw('&&'),
 128                         'git', 'reset', '--hard', self.config[client]['sha1'],
 129                     ],
 130                 )
 131
 132             if 'log-level' in self.config[client]:
 133                 if self.config[client]['log-level'] == 'info':
 134                     self.ctx.cluster.only(client).run(
 135                         args=[
 136                             'sed', '-i', '\'s/log4j.rootLogger=WARN/log4j.rootLogger=INFO/g\'',
 137                             '{tdir}/s3-tests-java/src/main/resources/log4j.properties'.format(
 138                                 tdir=testdir)
 139                         ]
 140                     )
 141                 if self.config[client]['log-level'] == 'debug':
 142                     self.ctx.cluster.only(client).run(
 143                         args=[
 144                             'sed', '-i', '\'s/log4j.rootLogger=WARN/log4j.rootLogger=DEBUG/g\'',
 145                             '{tdir}/s3-tests-java/src/main/resources/log4j.properties'.format(
 146                                 tdir=testdir)
 147                         ]
 148                     )
 149
 150     def install_required_packages(self, client):
 151         """
 152         Run bootstrap script to install openjdk and gradle.
 153         Add certificates to java keystore
 154         """
 155         log.info("S3 Tests Java: Installing required packages...")
 156         testdir = teuthology.get_testdir(self.ctx)
 157         self.ctx.cluster.only(client).run(
 158             args=['{tdir}/s3-tests-java/bootstrap.sh'.format(tdir=testdir)],
 159             stdout=BytesIO()
 160         )
 161
 162         endpoint = self.ctx.rgw.role_endpoints[client]
 163         if endpoint.cert:
 164             path = 'lib/security/cacerts'
 165             self.ctx.cluster.only(client).run(
 166                 args=['sudo',
 167                       'keytool',
 168                       '-import', '-alias', '{alias}'.format(
 169                           alias=endpoint.hostname),
 170                       '-keystore',
 171                       run.Raw(
 172                           '$(readlink -e $(dirname $(readlink -e $(which keytool)))/../{path})'.format(path=path)),
 173                       '-file', endpoint.cert.certificate,
 174                       '-storepass', 'changeit',
 175                       ],
 176                 stdout=BytesIO()
 177             )
 178
 179     def create_users(self):
 180         """
 181         Create a main and an alternative s3 user.
 182         Configuration is read from a skelethon config file
 183         s3tests.teuth.config.yaml in the java-s3tests repository
 184         and missing information is added from the task.
 185         Existing values are NOT overriden unless they are empty!
 186         """
 187         log.info("S3 Tests Java: Creating S3 users...")
 188         testdir = teuthology.get_testdir(self.ctx)
 189         for client in self.all_clients:
 190             endpoint = self.ctx.rgw.role_endpoints.get(client)
 191             local_user = getpass.getuser()
 192             remote_user = teuthology.get_test_user()
 193             os.system("scp {remote}@{host}:{tdir}/s3-tests-java/s3tests.teuth.config.yaml /home/{local}/".format(
 194                 host=endpoint.hostname, tdir=testdir, remote=remote_user, local=local_user))
 195             s3tests_conf = teuthology.config_file(
 196                 '/home/{local}/s3tests.teuth.config.yaml'.format(local=local_user))
 197             log.debug("S3 Tests Java: s3tests_conf is {s3cfg}".format(
 198                 s3cfg=s3tests_conf))
 199             for section, user in list(self.users.items()):
 200                 if section in s3tests_conf:
 201                     s3_user_id = '{user}.{client}'.format(
 202                         user=user, client=client)
 203                     log.debug(
 204                         'S3 Tests Java: Creating user {s3_user_id}'.format(s3_user_id=s3_user_id))
 205                     self._config_user(s3tests_conf=s3tests_conf,
 206                                       section=section, user=s3_user_id, client=client)
 207                     cluster_name, daemon_type, client_id = teuthology.split_role(
 208                         client)
 209                     client_with_id = daemon_type + '.' + client_id
 210                     args = [
 211                         'adjust-ulimits',
 212                         'ceph-coverage',
 213                         '{tdir}/archive/coverage'.format(tdir=testdir),
 214                         'radosgw-admin',
 215                         '-n', client_with_id,
 216                         'user', 'create',
 217                         '--uid', s3tests_conf[section]['user_id'],
 218                         '--display-name', s3tests_conf[section]['display_name'],
 219                         '--access-key', s3tests_conf[section]['access_key'],
 220                         '--secret', s3tests_conf[section]['access_secret'],
 221                         '--email', s3tests_conf[section]['email'],
 222                         '--cluster', cluster_name,
 223                     ]
 224                     log.info('{args}'.format(args=args))
 225                     self.ctx.cluster.only(client).run(
 226                         args=args,
 227                         stdout=BytesIO()
 228                     )
 229                 else:
 230                     self.users.pop(section)
 231             self._write_cfg_file(s3tests_conf, client)
 232             os.system(
 233                 "rm -rf /home/{local}/s3tests.teuth.config.yaml".format(local=local_user))
 234
 235     def _config_user(self, s3tests_conf, section, user, client):
 236         """
 237         Generate missing users data for this section by stashing away keys, ids, and
 238         email addresses.
 239         """
 240         access_key = ''.join(random.choice(string.ascii_uppercase)
 241                              for i in range(20))
 242         access_secret = base64.b64encode(os.urandom(40)).decode('ascii')
 243         endpoint = self.ctx.rgw.role_endpoints.get(client)
 244
 245         self._set_cfg_entry(
 246             s3tests_conf[section], 'user_id', '{user}'.format(user=user))
 247         self._set_cfg_entry(
 248             s3tests_conf[section], 'email', '{user}_test@test.test'.format(user=user))
 249         self._set_cfg_entry(
 250             s3tests_conf[section], 'display_name', 'Ms. {user}'.format(user=user))
 251         self._set_cfg_entry(
 252             s3tests_conf[section], 'access_key', '{ak}'.format(ak=access_key))
 253         self._set_cfg_entry(
 254             s3tests_conf[section], 'access_secret', '{asc}'.format(asc=access_secret))
 255         self._set_cfg_entry(
 256             s3tests_conf[section], 'region', 'us-east-1')
 257         self._set_cfg_entry(
 258             s3tests_conf[section], 'endpoint', '{ip}:{port}'.format(
 259                 ip=endpoint.hostname, port=endpoint.port))
 260         self._set_cfg_entry(
 261             s3tests_conf[section], 'host', endpoint.hostname)
 262         self._set_cfg_entry(
 263             s3tests_conf[section], 'port', endpoint.port)
 264         self._set_cfg_entry(
 265             s3tests_conf[section], 'is_secure', True if endpoint.cert else False)
 266
 267         log.debug("S3 Tests Java: s3tests_conf[{sect}] is {s3cfg}".format(
 268             sect=section, s3cfg=s3tests_conf[section]))
 269         log.debug('S3 Tests Java: Setion, User = {sect}, {user}'.format(
 270             sect=section, user=user))
 271
 272     def _write_cfg_file(self, cfg_dict, client):
 273         """
 274         To write the final s3 tests config file on the remote
 275         a temporary one is created on the local machine
 276         """
 277         testdir = teuthology.get_testdir(self.ctx)
 278         (remote,) = self.ctx.cluster.only(client).remotes.keys()
 279         with open('s3_tests_tmp.yaml', 'w') as outfile:
 280             yaml.dump(cfg_dict, outfile, default_flow_style=False)
 281
 282         conf_fp = BytesIO()
 283         with open('s3_tests_tmp.yaml', 'rb') as infile:
 284             for line in infile:
 285                 conf_fp.write(line)
 286
 287         teuthology.write_file(
 288             remote=remote,
 289             path='{tdir}/archive/s3-tests-java.{client}.conf'.format(
 290                 tdir=testdir, client=client),
 291             data=conf_fp.getvalue(),
 292         )
 293         os.remove('s3_tests_tmp.yaml')
 294
 295     def _set_cfg_entry(self, cfg_dict, key, value):
 296         if not (key in cfg_dict):
 297             cfg_dict.setdefault(key, value)
 298         elif cfg_dict[key] is None:
 299             cfg_dict[key] = value
 300
 301     def run_tests(self):
 302         log.info("S3 Tests Java: Running tests...")
 303         testdir = teuthology.get_testdir(self.ctx)
 304         for client in self.all_clients:
 305             self.ctx.cluster.only(client).run(
 306                 args=['cp',
 307                       '{tdir}/archive/s3-tests-java.{client}.conf'.format(
 308                           tdir=testdir, client=client),
 309                       '{tdir}/s3-tests-java/config.properties'.format(
 310                           tdir=testdir)
 311                       ],
 312                 stdout=BytesIO()
 313             )
 314             args = ['cd',
 315                     '{tdir}/s3-tests-java'.format(tdir=testdir),
 316                     run.Raw('&&'),
 317                     '/opt/gradle/gradle/bin/gradle', 'clean', 'test',
 318                     '--rerun-tasks', '--no-build-cache',
 319                     ]
 320             extra_args = []
 321             suppress_groups = False
 322             self.log_fwd = False
 323             self.log_name = ''
 324             if client in self.config and self.config[client] is not None:
 325                 if 'extra-args' in self.config[client]:
 326                     extra_args.extend(self.config[client]['extra-args'])
 327                     suppress_groups = True
 328                 if 'log-level' in self.config[client] and self.config[client]['log-level'] == 'debug':
 329                     extra_args += ['--debug']
 330                 if 'log-fwd' in self.config[client]:
 331                     self.log_fwd = True
 332                     self.log_name = '{tdir}/s3tests_log.txt'.format(
 333                         tdir=testdir)
 334                     if self.config[client]['log-fwd'] is not None:
 335                         self.log_name = self.config[client]['log-fwd']
 336                     extra_args += [run.Raw('>>'),
 337                                    self.log_name]
 338
 339             if not suppress_groups:
 340                 test_groups = ['AWS4Test', 'BucketTest', 'ObjectTest']
 341             else:
 342                 test_groups = ['All']
 343
 344             for gr in test_groups:
 345                 for i in range(2):
 346                     self.ctx.cluster.only(client).run(
 347                         args=['radosgw-admin', 'gc',
 348                               'process', '--include-all'],
 349                         stdout=BytesIO()
 350                     )
 351
 352                 if gr != 'All':
 353                     self.ctx.cluster.only(client).run(
 354                         args=args + ['--tests'] + [gr] + extra_args,
 355                         stdout=BytesIO()
 356                     )
 357                 else:
 358                     self.ctx.cluster.only(client).run(
 359                         args=args + extra_args,
 360                         stdout=BytesIO()
 361                     )
 362
 363                 for i in range(2):
 364                     self.ctx.cluster.only(client).run(
 365                         args=['radosgw-admin', 'gc',
 366                               'process', '--include-all'],
 367                         stdout=BytesIO()
 368                     )
 369
 370     def remove_tests(self, client):
 371         log.info('S3 Tests Java: Cleaning up s3-tests-java...')
 372         testdir = teuthology.get_testdir(self.ctx)
 373
 374         if self.log_fwd:
 375             self.ctx.cluster.only(client).run(
 376                 args=['cd',
 377                       '{tdir}/s3-tests-java'.format(tdir=testdir),
 378                       run.Raw('&&'),
 379                       'cat', self.log_name,
 380                       run.Raw('&&'),
 381                       'rm', self.log_name],
 382                 stdout=BytesIO()
 383             )
 384
 385         self.ctx.cluster.only(client).run(
 386             args=[
 387                 'rm',
 388                 '-rf',
 389                 '{tdir}/s3-tests-java'.format(tdir=testdir),
 390             ],
 391             stdout=BytesIO()
 392         )
 393
 394     def delete_users(self, client):
 395         log.info("S3 Tests Java: Deleting S3 users...")
 396         testdir = teuthology.get_testdir(self.ctx)
 397         for section, user in self.users.items():
 398             s3_user_id = '{user}.{client}'.format(user=user, client=client)
 399             self.ctx.cluster.only(client).run(
 400                 args=[
 401                     'adjust-ulimits',
 402                     'ceph-coverage',
 403                     '{tdir}/archive/coverage'.format(tdir=testdir),
 404                     'radosgw-admin',
 405                     '-n', client,
 406                     'user', 'rm',
 407                     '--uid', s3_user_id,
 408                     '--purge-data',
 409                     '--cluster', 'ceph',
 410                 ],
 411                 stdout=BytesIO()
 412             )
 413
 414
 415 task = S3tests_java