ceph/qa/tasks/s3a_hadoop.py

   1 import contextlib
   2 import logging
   3 import time
   4 from teuthology import misc
   5 from teuthology.orchestra import run
   6
   7 log = logging.getLogger(__name__)
   8
   9
  10 @contextlib.contextmanager
  11 def task(ctx, config):
  12     """
  13      Run Hadoop S3A tests using Ceph
  14      usage:
  15       -tasks:
  16          ceph-ansible:
  17          s3a-hadoop:
  18            maven-version: '3.3.9' (default)
  19            hadoop-version: '2.7.3'
  20            bucket-name: 's3atest' (default)
  21            access-key: 'anykey' (uses a default value)
  22            secret-key: 'secretkey' ( uses a default value)
  23     """
  24     if config is None:
  25         config = {}
  26
  27     assert isinstance(config, dict), \
  28         "task only supports a dictionary for configuration"
  29
  30     overrides = ctx.config.get('overrides', {})
  31     misc.deep_merge(config, overrides.get('s3a-hadoop', {}))
  32     testdir = misc.get_testdir(ctx)
  33     rgws = ctx.cluster.only(misc.is_type('rgw'))
  34     # use the first rgw node to test s3a
  35     rgw_node = rgws.remotes.keys()[0]
  36     # get versions
  37     maven_major = config.get('maven-major', 'maven-3')
  38     maven_version = config.get('maven-version', '3.3.9')
  39     hadoop_ver = config.get('hadoop-version', '2.7.3')
  40     bucket_name = config.get('bucket-name', 's3atest')
  41     access_key = config.get('access-key', 'EGAQRD2ULOIFKFSKCT4F')
  42     secret_key = config.get(
  43         'secret-key',
  44         'zi816w1vZKfaSM85Cl0BxXTwSLyN7zB4RbTswrGb')
  45
  46     # set versions for cloning the repo
  47     apache_maven = 'apache-maven-{maven_version}-bin.tar.gz'.format(
  48         maven_version=maven_version)
  49     maven_link = 'http://mirror.jax.hugeserver.com/apache/maven/' + \
  50         '{maven_major}/{maven_version}/binaries/'.format(maven_major=maven_major, maven_version=maven_version) + apache_maven
  51     hadoop_git = 'https://github.com/apache/hadoop'
  52     hadoop_rel = 'hadoop-{ver} rel/release-{ver}'.format(ver=hadoop_ver)
  53     install_prereq(rgw_node)
  54     rgw_node.run(
  55         args=[
  56             'cd',
  57             testdir,
  58             run.Raw('&&'),
  59             'wget',
  60             maven_link,
  61             run.Raw('&&'),
  62             'tar',
  63             '-xvf',
  64             apache_maven,
  65             run.Raw('&&'),
  66             'git',
  67             'clone',
  68             run.Raw(hadoop_git),
  69             run.Raw('&&'),
  70             'cd',
  71             'hadoop',
  72             run.Raw('&&'),
  73             'git',
  74             'checkout',
  75             '-b',
  76             run.Raw(hadoop_rel)
  77         ]
  78     )
  79     dnsmasq_name = 's3.ceph.com'
  80     configure_s3a(rgw_node, dnsmasq_name, access_key, secret_key, bucket_name, testdir)
  81     setup_dnsmasq(rgw_node, dnsmasq_name)
  82     fix_rgw_config(rgw_node, dnsmasq_name)
  83     setup_user_bucket(rgw_node, dnsmasq_name, access_key, secret_key, bucket_name, testdir)
  84     if hadoop_ver.startswith('2.8'):
  85         test_options = '-Dit.test=ITestS3A* -Dparallel-tests -Dscale -Dfs.s3a.scale.test.huge.filesize=128M verify'
  86     else:
  87         test_options = 'test -Dtest=S3a*,TestS3A*'
  88     try:
  89         run_s3atest(rgw_node, maven_version, testdir, test_options)
  90         yield
  91     finally:
  92         log.info("Done s3a testing, Cleaning up")
  93         for fil in ['apache*', 'hadoop*', 'venv*', 'create*']:
  94             rgw_node.run(args=['rm', run.Raw('-rf'), run.Raw('{tdir}/{file}'.format(tdir=testdir, file=fil))])
  95         # restart and let NM restore original config
  96         rgw_node.run(args=['sudo', 'systemctl', 'stop', 'dnsmasq'])
  97         rgw_node.run(args=['sudo', 'systemctl', 'restart', 'network.service'], check_status=False)
  98         rgw_node.run(args=['sudo', 'systemctl', 'status', 'network.service'], check_status=False)
  99
 100
 101 def install_prereq(client):
 102     """
 103     Install pre requisites for RHEL and CentOS
 104     TBD: Ubuntu
 105     """
 106     if client.os.name == 'rhel' or client.os.name == 'centos':
 107         client.run(
 108                args=[
 109                     'sudo',
 110                     'yum',
 111                     'install',
 112                     '-y',
 113                     'protobuf-c.x86_64',
 114                     'java',
 115                     'java-1.8.0-openjdk-devel',
 116                     'dnsmasq'
 117                     ]
 118                 )
 119
 120
 121 def setup_dnsmasq(client, name):
 122     """
 123     Setup simple dnsmasq name eg: s3.ceph.com
 124     Local RGW host can then be used with whatever name has been setup with.
 125     """
 126     resolv_conf = "nameserver 127.0.0.1\n"
 127     dnsmasq_template = """address=/{name}/{ip_address}
 128 server=8.8.8.8
 129 server=8.8.4.4
 130 """.format(name=name, ip_address=client.ip_address)
 131     dnsmasq_config_path = '/etc/dnsmasq.d/ceph'
 132     # point resolv.conf to local dnsmasq
 133     misc.sudo_write_file(
 134         remote=client,
 135         path='/etc/resolv.conf',
 136         data=resolv_conf,
 137     )
 138     misc.sudo_write_file(
 139         remote=client,
 140         path=dnsmasq_config_path,
 141         data=dnsmasq_template,
 142     )
 143     client.run(args=['cat', dnsmasq_config_path])
 144     # restart dnsmasq
 145     client.run(args=['sudo', 'systemctl', 'restart', 'dnsmasq'])
 146     client.run(args=['sudo', 'systemctl', 'status', 'dnsmasq'])
 147     time.sleep(5)
 148     # verify dns name is set
 149     client.run(args=['ping', '-c', '4', name])
 150
 151
 152 def fix_rgw_config(client, name):
 153     """
 154     Fix RGW config in ceph.conf, we need rgw dns name entry
 155     and also modify the port to use :80 for s3a tests to work
 156     """
 157     rgw_dns_name = 'rgw dns name = {name}'.format(name=name)
 158     ceph_conf_path = '/etc/ceph/ceph.conf'
 159     # append rgw_dns_name
 160     client.run(
 161         args=[
 162             'sudo',
 163             'sed',
 164             run.Raw('-i'),
 165             run.Raw("'/client.rgw*/a {rgw_name}'".format(rgw_name=rgw_dns_name)),
 166             ceph_conf_path
 167
 168         ]
 169     )
 170     # listen on port 80
 171     client.run(
 172         args=[
 173             'sudo',
 174             'sed',
 175             run.Raw('-i'),
 176             run.Raw('s/:8080/:80/'),
 177             ceph_conf_path
 178         ]
 179     )
 180     client.run(args=['cat', ceph_conf_path])
 181     client.run(args=['sudo', 'systemctl', 'restart', 'ceph-radosgw.target'])
 182     client.run(args=['sudo', 'systemctl', 'status', 'ceph-radosgw.target'])
 183
 184
 185 def setup_user_bucket(client, dns_name, access_key, secret_key, bucket_name, testdir):
 186     """
 187     Create user with access_key and secret_key that will be
 188     used for the s3a testdir
 189     """
 190     client.run(
 191         args=[
 192             'sudo',
 193             'radosgw-admin',
 194             'user',
 195             'create',
 196             run.Raw('--uid'),
 197             's3a',
 198             run.Raw('--display-name=s3a cephtests'),
 199             run.Raw('--access-key={access_key}'.format(access_key=access_key)),
 200             run.Raw('--secret-key={secret_key}'.format(secret_key=secret_key)),
 201             run.Raw('--email=s3a@ceph.com'),
 202         ]
 203     )
 204     client.run(
 205         args=[
 206             'virtualenv',
 207             '{testdir}/venv'.format(testdir=testdir),
 208             run.Raw('&&'),
 209             run.Raw('{testdir}/venv/bin/pip'.format(testdir=testdir)),
 210             'install',
 211             'boto'
 212         ]
 213     )
 214     create_bucket = """
 215 #!/usr/bin/env python
 216 import boto
 217 import boto.s3.connection
 218 access_key = '{access_key}'
 219 secret_key = '{secret_key}'
 220
 221 conn = boto.connect_s3(
 222         aws_access_key_id = access_key,
 223         aws_secret_access_key = secret_key,
 224         host = '{dns_name}',
 225         is_secure=False,
 226         calling_format = boto.s3.connection.OrdinaryCallingFormat(),
 227         )
 228 bucket = conn.create_bucket('{bucket_name}')
 229 for bucket in conn.get_all_buckets():
 230         print bucket.name + "\t" + bucket.creation_date
 231 """.format(access_key=access_key, secret_key=secret_key, dns_name=dns_name, bucket_name=bucket_name)
 232     py_bucket_file = '{testdir}/create_bucket.py'.format(testdir=testdir)
 233     misc.sudo_write_file(
 234         remote=client,
 235         path=py_bucket_file,
 236         data=create_bucket,
 237         perms='0744',
 238         )
 239     client.run(
 240         args=[
 241             'cat',
 242             '{testdir}/create_bucket.py'.format(testdir=testdir),
 243         ]
 244     )
 245     client.run(
 246         args=[
 247             '{testdir}/venv/bin/python'.format(testdir=testdir),
 248             '{testdir}/create_bucket.py'.format(testdir=testdir),
 249         ]
 250     )
 251
 252
 253 def run_s3atest(client, maven_version, testdir, test_options):
 254     """
 255     Finally run the s3a test
 256     """
 257     aws_testdir = '{testdir}/hadoop/hadoop-tools/hadoop-aws/'.format(testdir=testdir)
 258     run_test = '{testdir}/apache-maven-{maven_version}/bin/mvn'.format(testdir=testdir, maven_version=maven_version)
 259     client.run(
 260         args=[
 261             'cd',
 262             run.Raw(aws_testdir),
 263             run.Raw('&&'),
 264             run.Raw(run_test),
 265             run.Raw(test_options)
 266         ]
 267     )
 268
 269
 270 def configure_s3a(client, dns_name, access_key, secret_key, bucket_name, testdir):
 271     """
 272     Use the template to configure s3a test, Fill in access_key, secret_key
 273     and other details required for test.
 274     """
 275     config_template = """<configuration>
 276 <property>
 277 <name>fs.s3a.endpoint</name>
 278 <value>{name}</value>
 279 </property>
 280
 281 <property>
 282 <name>fs.s3a.connection.ssl.enabled</name>
 283 <value>false</value>
 284 </property>
 285
 286 <property>
 287 <name>test.fs.s3n.name</name>
 288 <value>s3n://{bucket_name}/</value>
 289 </property>
 290
 291 <property>
 292 <name>test.fs.s3a.name</name>
 293 <value>s3a://{bucket_name}/</value>
 294 </property>
 295
 296 <property>
 297 <name>test.fs.s3.name</name>
 298 <value>s3://{bucket_name}/</value>
 299 </property>
 300
 301 <property>
 302 <name>fs.s3.awsAccessKeyId</name>
 303 <value>{access_key}</value>
 304 </property>
 305
 306 <property>
 307 <name>fs.s3.awsSecretAccessKey</name>
 308 <value>{secret_key}</value>
 309 </property>
 310
 311 <property>
 312 <name>fs.s3n.awsAccessKeyId</name>
 313 <value>{access_key}</value>
 314 </property>
 315
 316 <property>
 317 <name>fs.s3n.awsSecretAccessKey</name>
 318 <value>{secret_key}</value>
 319 </property>
 320
 321 <property>
 322 <name>fs.s3a.access.key</name>
 323 <description>AWS access key ID. Omit for Role-based authentication.</description>
 324 <value>{access_key}</value>
 325 </property>
 326
 327 <property>
 328 <name>fs.s3a.secret.key</name>
 329 <description>AWS secret key. Omit for Role-based authentication.</description>
 330 <value>{secret_key}</value>
 331 </property>
 332 </configuration>
 333 """.format(name=dns_name, bucket_name=bucket_name, access_key=access_key, secret_key=secret_key)
 334     config_path = testdir + '/hadoop/hadoop-tools/hadoop-aws/src/test/resources/auth-keys.xml'
 335     misc.write_file(
 336         remote=client,
 337         path=config_path,
 338         data=config_template,
 339     )
 340     # output for debug
 341     client.run(args=['cat', config_path])