ceph/qa/tasks/s3a_hadoop.py

   1 import contextlib
   2 import logging
   3 import time
   4 from teuthology import misc
   5 from teuthology.orchestra import run
   6
   7 log = logging.getLogger(__name__)
   8
   9
  10 @contextlib.contextmanager
  11 def task(ctx, config):
  12     """
  13      Run Hadoop S3A tests using Ceph
  14      usage:
  15       -tasks:
  16          ceph-ansible:
  17          s3a-hadoop:
  18            maven-version: '3.3.9' (default)
  19            hadoop-version: '2.7.3'
  20            bucket-name: 's3atest' (default)
  21            access-key: 'anykey' (uses a default value)
  22            secret-key: 'secretkey' ( uses a default value)
  23     """
  24     if config is None:
  25         config = {}
  26
  27     assert isinstance(config, dict), \
  28         "task only supports a dictionary for configuration"
  29
  30     overrides = ctx.config.get('overrides', {})
  31     misc.deep_merge(config, overrides.get('s3a-hadoop', {}))
  32     testdir = misc.get_testdir(ctx)
  33     rgws = ctx.cluster.only(misc.is_type('rgw'))
  34     # use the first rgw node to test s3a
  35     rgw_node = rgws.remotes.keys()[0]
  36     # get versions
  37     maven_major = config.get('maven-major', 'maven-3')
  38     maven_version = config.get('maven-version', '3.3.9')
  39     hadoop_ver = config.get('hadoop-version', '2.7.3')
  40     bucket_name = config.get('bucket-name', 's3atest')
  41     access_key = config.get('access-key', 'EGAQRD2ULOIFKFSKCT4F')
  42     secret_key = config.get(
  43         'secret-key',
  44         'zi816w1vZKfaSM85Cl0BxXTwSLyN7zB4RbTswrGb')
  45
  46     # set versions for cloning the repo
  47     apache_maven = 'apache-maven-{maven_version}-bin.tar.gz'.format(
  48         maven_version=maven_version)
  49     maven_link = 'http://mirror.jax.hugeserver.com/apache/maven/' + \
  50         '{maven_major}/{maven_version}/binaries/'.format(maven_major=maven_major, maven_version=maven_version) + apache_maven
  51     hadoop_git = 'https://github.com/apache/hadoop'
  52     hadoop_rel = 'hadoop-{ver} rel/release-{ver}'.format(ver=hadoop_ver)
  53     install_prereq(rgw_node)
  54     rgw_node.run(
  55         args=[
  56             'cd',
  57             testdir,
  58             run.Raw('&&'),
  59             'wget',
  60             maven_link,
  61             run.Raw('&&'),
  62             'tar',
  63             '-xvf',
  64             apache_maven,
  65             run.Raw('&&'),
  66             'git',
  67             'clone',
  68             run.Raw(hadoop_git),
  69             run.Raw('&&'),
  70             'cd',
  71             'hadoop',
  72             run.Raw('&&'),
  73             'git',
  74             'checkout',
  75             '-b',
  76             run.Raw(hadoop_rel)
  77         ]
  78     )
  79     dnsmasq_name = 's3.ceph.com'
  80     configure_s3a(rgw_node, dnsmasq_name, access_key, secret_key, bucket_name, testdir)
  81     setup_dnsmasq(rgw_node, dnsmasq_name)
  82     fix_rgw_config(rgw_node, dnsmasq_name)
  83     setup_user_bucket(rgw_node, dnsmasq_name, access_key, secret_key, bucket_name, testdir)
  84     if hadoop_ver.startswith('2.8'):
  85         # test all ITtests but skip AWS test using public bucket landsat-pds
  86         # which is not available from within this test
  87         test_options = '-Dit.test=ITestS3A* -Dit.test=\!ITestS3AAWSCredentialsProvider* -Dparallel-tests -Dscale -Dfs.s3a.scale.test.huge.filesize=128M verify'
  88     else:
  89         test_options = 'test -Dtest=S3a*,TestS3A*'
  90     try:
  91         run_s3atest(rgw_node, maven_version, testdir, test_options)
  92         yield
  93     finally:
  94         log.info("Done s3a testing, Cleaning up")
  95         for fil in ['apache*', 'hadoop*', 'venv*', 'create*']:
  96             rgw_node.run(args=['rm', run.Raw('-rf'), run.Raw('{tdir}/{file}'.format(tdir=testdir, file=fil))])
  97         # restart and let NM restore original config
  98         rgw_node.run(args=['sudo', 'systemctl', 'stop', 'dnsmasq'])
  99         rgw_node.run(args=['sudo', 'systemctl', 'restart', 'network.service'], check_status=False)
 100         rgw_node.run(args=['sudo', 'systemctl', 'status', 'network.service'], check_status=False)
 101
 102
 103 def install_prereq(client):
 104     """
 105     Install pre requisites for RHEL and CentOS
 106     TBD: Ubuntu
 107     """
 108     if client.os.name == 'rhel' or client.os.name == 'centos':
 109         client.run(
 110                args=[
 111                     'sudo',
 112                     'yum',
 113                     'install',
 114                     '-y',
 115                     'protobuf-c.x86_64',
 116                     'java',
 117                     'java-1.8.0-openjdk-devel',
 118                     'dnsmasq'
 119                     ]
 120                 )
 121
 122
 123 def setup_dnsmasq(client, name):
 124     """
 125     Setup simple dnsmasq name eg: s3.ceph.com
 126     Local RGW host can then be used with whatever name has been setup with.
 127     """
 128     resolv_conf = "nameserver 127.0.0.1\n"
 129     dnsmasq_template = """address=/{name}/{ip_address}
 130 server=8.8.8.8
 131 server=8.8.4.4
 132 """.format(name=name, ip_address=client.ip_address)
 133     dnsmasq_config_path = '/etc/dnsmasq.d/ceph'
 134     # point resolv.conf to local dnsmasq
 135     misc.sudo_write_file(
 136         remote=client,
 137         path='/etc/resolv.conf',
 138         data=resolv_conf,
 139     )
 140     misc.sudo_write_file(
 141         remote=client,
 142         path=dnsmasq_config_path,
 143         data=dnsmasq_template,
 144     )
 145     client.run(args=['cat', dnsmasq_config_path])
 146     # restart dnsmasq
 147     client.run(args=['sudo', 'systemctl', 'restart', 'dnsmasq'])
 148     client.run(args=['sudo', 'systemctl', 'status', 'dnsmasq'])
 149     time.sleep(5)
 150     # verify dns name is set
 151     client.run(args=['ping', '-c', '4', name])
 152
 153
 154 def fix_rgw_config(client, name):
 155     """
 156     Fix RGW config in ceph.conf, we need rgw dns name entry
 157     and also modify the port to use :80 for s3a tests to work
 158     """
 159     rgw_dns_name = 'rgw dns name = {name}'.format(name=name)
 160     ceph_conf_path = '/etc/ceph/ceph.conf'
 161     # append rgw_dns_name
 162     client.run(
 163         args=[
 164             'sudo',
 165             'sed',
 166             run.Raw('-i'),
 167             run.Raw("'/client.rgw*/a {rgw_name}'".format(rgw_name=rgw_dns_name)),
 168             ceph_conf_path
 169
 170         ]
 171     )
 172     # listen on port 80
 173     client.run(
 174         args=[
 175             'sudo',
 176             'sed',
 177             run.Raw('-i'),
 178             run.Raw('s/:8080/:80/'),
 179             ceph_conf_path
 180         ]
 181     )
 182     client.run(args=['cat', ceph_conf_path])
 183     client.run(args=['sudo', 'systemctl', 'restart', 'ceph-radosgw.target'])
 184     client.run(args=['sudo', 'systemctl', 'status', 'ceph-radosgw.target'])
 185
 186
 187 def setup_user_bucket(client, dns_name, access_key, secret_key, bucket_name, testdir):
 188     """
 189     Create user with access_key and secret_key that will be
 190     used for the s3a testdir
 191     """
 192     client.run(
 193         args=[
 194             'sudo',
 195             'radosgw-admin',
 196             'user',
 197             'create',
 198             run.Raw('--uid'),
 199             's3a',
 200             run.Raw('--display-name=s3a cephtests'),
 201             run.Raw('--access-key={access_key}'.format(access_key=access_key)),
 202             run.Raw('--secret-key={secret_key}'.format(secret_key=secret_key)),
 203             run.Raw('--email=s3a@ceph.com'),
 204         ]
 205     )
 206     client.run(
 207         args=[
 208             'virtualenv',
 209             '{testdir}/venv'.format(testdir=testdir),
 210             run.Raw('&&'),
 211             run.Raw('{testdir}/venv/bin/pip'.format(testdir=testdir)),
 212             'install',
 213             'boto'
 214         ]
 215     )
 216     create_bucket = """
 217 #!/usr/bin/env python
 218 import boto
 219 import boto.s3.connection
 220 access_key = '{access_key}'
 221 secret_key = '{secret_key}'
 222
 223 conn = boto.connect_s3(
 224         aws_access_key_id = access_key,
 225         aws_secret_access_key = secret_key,
 226         host = '{dns_name}',
 227         is_secure=False,
 228         calling_format = boto.s3.connection.OrdinaryCallingFormat(),
 229         )
 230 bucket = conn.create_bucket('{bucket_name}')
 231 for bucket in conn.get_all_buckets():
 232         print bucket.name + "\t" + bucket.creation_date
 233 """.format(access_key=access_key, secret_key=secret_key, dns_name=dns_name, bucket_name=bucket_name)
 234     py_bucket_file = '{testdir}/create_bucket.py'.format(testdir=testdir)
 235     misc.sudo_write_file(
 236         remote=client,
 237         path=py_bucket_file,
 238         data=create_bucket,
 239         perms='0744',
 240         )
 241     client.run(
 242         args=[
 243             'cat',
 244             '{testdir}/create_bucket.py'.format(testdir=testdir),
 245         ]
 246     )
 247     client.run(
 248         args=[
 249             '{testdir}/venv/bin/python'.format(testdir=testdir),
 250             '{testdir}/create_bucket.py'.format(testdir=testdir),
 251         ]
 252     )
 253
 254
 255 def run_s3atest(client, maven_version, testdir, test_options):
 256     """
 257     Finally run the s3a test
 258     """
 259     aws_testdir = '{testdir}/hadoop/hadoop-tools/hadoop-aws/'.format(testdir=testdir)
 260     run_test = '{testdir}/apache-maven-{maven_version}/bin/mvn'.format(testdir=testdir, maven_version=maven_version)
 261     client.run(
 262         args=[
 263             'cd',
 264             run.Raw(aws_testdir),
 265             run.Raw('&&'),
 266             run.Raw(run_test),
 267             run.Raw(test_options)
 268         ]
 269     )
 270
 271
 272 def configure_s3a(client, dns_name, access_key, secret_key, bucket_name, testdir):
 273     """
 274     Use the template to configure s3a test, Fill in access_key, secret_key
 275     and other details required for test.
 276     """
 277     config_template = """<configuration>
 278 <property>
 279 <name>fs.s3a.endpoint</name>
 280 <value>{name}</value>
 281 </property>
 282
 283 <property>
 284 <name>fs.s3a.connection.ssl.enabled</name>
 285 <value>false</value>
 286 </property>
 287
 288 <property>
 289 <name>test.fs.s3n.name</name>
 290 <value>s3n://{bucket_name}/</value>
 291 </property>
 292
 293 <property>
 294 <name>test.fs.s3a.name</name>
 295 <value>s3a://{bucket_name}/</value>
 296 </property>
 297
 298 <property>
 299 <name>test.fs.s3.name</name>
 300 <value>s3://{bucket_name}/</value>
 301 </property>
 302
 303 <property>
 304 <name>fs.s3.awsAccessKeyId</name>
 305 <value>{access_key}</value>
 306 </property>
 307
 308 <property>
 309 <name>fs.s3.awsSecretAccessKey</name>
 310 <value>{secret_key}</value>
 311 </property>
 312
 313 <property>
 314 <name>fs.s3n.awsAccessKeyId</name>
 315 <value>{access_key}</value>
 316 </property>
 317
 318 <property>
 319 <name>fs.s3n.awsSecretAccessKey</name>
 320 <value>{secret_key}</value>
 321 </property>
 322
 323 <property>
 324 <name>fs.s3a.access.key</name>
 325 <description>AWS access key ID. Omit for Role-based authentication.</description>
 326 <value>{access_key}</value>
 327 </property>
 328
 329 <property>
 330 <name>fs.s3a.secret.key</name>
 331 <description>AWS secret key. Omit for Role-based authentication.</description>
 332 <value>{secret_key}</value>
 333 </property>
 334 </configuration>
 335 """.format(name=dns_name, bucket_name=bucket_name, access_key=access_key, secret_key=secret_key)
 336     config_path = testdir + '/hadoop/hadoop-tools/hadoop-aws/src/test/resources/auth-keys.xml'
 337     misc.write_file(
 338         remote=client,
 339         path=config_path,
 340         data=config_template,
 341     )
 342     # output for debug
 343     client.run(args=['cat', config_path])