]>
git.proxmox.com Git - ceph.git/blob - ceph/qa/tasks/s3a_hadoop.py
4 from teuthology
import misc
5 from teuthology
.orchestra
import run
7 log
= logging
.getLogger(__name__
)
10 @contextlib.contextmanager
11 def task(ctx
, config
):
13 Run Hadoop S3A tests using Ceph
18 maven-version: '3.3.9' (default)
19 hadoop-version: '2.7.3'
20 bucket-name: 's3atest' (default)
21 access-key: 'anykey' (uses a default value)
22 secret-key: 'secretkey' ( uses a default value)
27 assert isinstance(config
, dict), \
28 "task only supports a dictionary for configuration"
30 overrides
= ctx
.config
.get('overrides', {})
31 misc
.deep_merge(config
, overrides
.get('s3a-hadoop', {}))
32 testdir
= misc
.get_testdir(ctx
)
33 rgws
= ctx
.cluster
.only(misc
.is_type('rgw'))
34 # use the first rgw node to test s3a
35 rgw_node
= rgws
.remotes
.keys()[0]
37 maven_major
= config
.get('maven-major', 'maven-3')
38 maven_version
= config
.get('maven-version', '3.3.9')
39 hadoop_ver
= config
.get('hadoop-version', '2.7.3')
40 bucket_name
= config
.get('bucket-name', 's3atest')
41 access_key
= config
.get('access-key', 'EGAQRD2ULOIFKFSKCT4F')
42 secret_key
= config
.get(
44 'zi816w1vZKfaSM85Cl0BxXTwSLyN7zB4RbTswrGb')
46 # set versions for cloning the repo
47 apache_maven
= 'apache-maven-{maven_version}-bin.tar.gz'.format(
48 maven_version
=maven_version
)
49 maven_link
= 'http://apache.mirrors.lucidnetworks.net/maven/' + \
50 '{maven_major}/{maven_version}/binaries/'.format(maven_major
=maven_major
, maven_version
=maven_version
) + apache_maven
51 hadoop_git
= 'https://github.com/apache/hadoop'
52 hadoop_rel
= 'hadoop-{ver} rel/release-{ver}'.format(ver
=hadoop_ver
)
53 install_prereq(rgw_node
)
79 dnsmasq_name
= 's3.ceph.com'
80 configure_s3a(rgw_node
, dnsmasq_name
, access_key
, secret_key
, bucket_name
, testdir
)
81 setup_dnsmasq(rgw_node
, dnsmasq_name
)
82 fix_rgw_config(rgw_node
, dnsmasq_name
)
83 setup_user_bucket(rgw_node
, dnsmasq_name
, access_key
, secret_key
, bucket_name
, testdir
)
84 if hadoop_ver
.startswith('2.8'):
85 # test all ITtests but skip AWS test using public bucket landsat-pds
86 # which is not available from within this test
87 test_options
= '-Dit.test=ITestS3A* -Dit.test=\!ITestS3AAWSCredentialsProvider* -Dparallel-tests -Dscale -Dfs.s3a.scale.test.huge.filesize=128M verify'
89 test_options
= 'test -Dtest=S3a*,TestS3A*'
91 run_s3atest(rgw_node
, maven_version
, testdir
, test_options
)
94 log
.info("Done s3a testing, Cleaning up")
95 for fil
in ['apache*', 'hadoop*', 'venv*', 'create*']:
96 rgw_node
.run(args
=['rm', run
.Raw('-rf'), run
.Raw('{tdir}/{file}'.format(tdir
=testdir
, file=fil
))])
97 # restart and let NM restore original config
98 rgw_node
.run(args
=['sudo', 'systemctl', 'stop', 'dnsmasq'])
99 rgw_node
.run(args
=['sudo', 'systemctl', 'restart', 'network.service'], check_status
=False)
100 rgw_node
.run(args
=['sudo', 'systemctl', 'status', 'network.service'], check_status
=False)
103 def install_prereq(client
):
105 Install pre requisites for RHEL and CentOS
108 if client
.os
.name
== 'rhel' or client
.os
.name
== 'centos':
117 'java-1.8.0-openjdk-devel',
123 def setup_dnsmasq(client
, name
):
125 Setup simple dnsmasq name eg: s3.ceph.com
126 Local RGW host can then be used with whatever name has been setup with.
128 resolv_conf
= "nameserver 127.0.0.1\n"
129 dnsmasq_template
= """address=/{name}/{ip_address}
132 """.format(name
=name
, ip_address
=client
.ip_address
)
133 dnsmasq_config_path
= '/etc/dnsmasq.d/ceph'
134 # point resolv.conf to local dnsmasq
135 misc
.sudo_write_file(
137 path
='/etc/resolv.conf',
140 misc
.sudo_write_file(
142 path
=dnsmasq_config_path
,
143 data
=dnsmasq_template
,
145 client
.run(args
=['cat', dnsmasq_config_path
])
147 client
.run(args
=['sudo', 'systemctl', 'restart', 'dnsmasq'])
148 client
.run(args
=['sudo', 'systemctl', 'status', 'dnsmasq'])
150 # verify dns name is set
151 client
.run(args
=['ping', '-c', '4', name
])
154 def fix_rgw_config(client
, name
):
156 Fix RGW config in ceph.conf, we need rgw dns name entry
157 and also modify the port to use :80 for s3a tests to work
159 rgw_dns_name
= 'rgw dns name = {name}'.format(name
=name
)
160 ceph_conf_path
= '/etc/ceph/ceph.conf'
161 # append rgw_dns_name
167 run
.Raw("'/client.rgw*/a {rgw_name}'".format(rgw_name
=rgw_dns_name
)),
178 run
.Raw('s/:8080/:80/'),
182 client
.run(args
=['cat', ceph_conf_path
])
183 client
.run(args
=['sudo', 'systemctl', 'restart', 'ceph-radosgw.target'])
184 client
.run(args
=['sudo', 'systemctl', 'status', 'ceph-radosgw.target'])
187 def setup_user_bucket(client
, dns_name
, access_key
, secret_key
, bucket_name
, testdir
):
189 Create user with access_key and secret_key that will be
190 used for the s3a testdir
200 run
.Raw('--display-name=s3a cephtests'),
201 run
.Raw('--access-key={access_key}'.format(access_key
=access_key
)),
202 run
.Raw('--secret-key={secret_key}'.format(secret_key
=secret_key
)),
203 run
.Raw('--email=s3a@ceph.com'),
209 '{testdir}/venv'.format(testdir
=testdir
),
211 run
.Raw('{testdir}/venv/bin/pip'.format(testdir
=testdir
)),
217 #!/usr/bin/env python
219 import boto.s3.connection
220 access_key = '{access_key}'
221 secret_key = '{secret_key}'
223 conn = boto.connect_s3(
224 aws_access_key_id = access_key,
225 aws_secret_access_key = secret_key,
228 calling_format = boto.s3.connection.OrdinaryCallingFormat(),
230 bucket = conn.create_bucket('{bucket_name}')
231 for bucket in conn.get_all_buckets():
232 print bucket.name + "\t" + bucket.creation_date
233 """.format(access_key
=access_key
, secret_key
=secret_key
, dns_name
=dns_name
, bucket_name
=bucket_name
)
234 py_bucket_file
= '{testdir}/create_bucket.py'.format(testdir
=testdir
)
235 misc
.sudo_write_file(
244 '{testdir}/create_bucket.py'.format(testdir
=testdir
),
249 '{testdir}/venv/bin/python'.format(testdir
=testdir
),
250 '{testdir}/create_bucket.py'.format(testdir
=testdir
),
255 def run_s3atest(client
, maven_version
, testdir
, test_options
):
257 Finally run the s3a test
259 aws_testdir
= '{testdir}/hadoop/hadoop-tools/hadoop-aws/'.format(testdir
=testdir
)
260 run_test
= '{testdir}/apache-maven-{maven_version}/bin/mvn'.format(testdir
=testdir
, maven_version
=maven_version
)
264 run
.Raw(aws_testdir
),
267 run
.Raw(test_options
)
272 def configure_s3a(client
, dns_name
, access_key
, secret_key
, bucket_name
, testdir
):
274 Use the template to configure s3a test, Fill in access_key, secret_key
275 and other details required for test.
277 config_template
= """<configuration>
279 <name>fs.s3a.endpoint</name>
280 <value>{name}</value>
284 <name>fs.s3a.connection.ssl.enabled</name>
289 <name>test.fs.s3n.name</name>
290 <value>s3n://{bucket_name}/</value>
294 <name>test.fs.s3a.name</name>
295 <value>s3a://{bucket_name}/</value>
299 <name>test.fs.s3.name</name>
300 <value>s3://{bucket_name}/</value>
304 <name>fs.s3.awsAccessKeyId</name>
305 <value>{access_key}</value>
309 <name>fs.s3.awsSecretAccessKey</name>
310 <value>{secret_key}</value>
314 <name>fs.s3n.awsAccessKeyId</name>
315 <value>{access_key}</value>
319 <name>fs.s3n.awsSecretAccessKey</name>
320 <value>{secret_key}</value>
324 <name>fs.s3a.access.key</name>
325 <description>AWS access key ID. Omit for Role-based authentication.</description>
326 <value>{access_key}</value>
330 <name>fs.s3a.secret.key</name>
331 <description>AWS secret key. Omit for Role-based authentication.</description>
332 <value>{secret_key}</value>
335 """.format(name
=dns_name
, bucket_name
=bucket_name
, access_key
=access_key
, secret_key
=secret_key
)
336 config_path
= testdir
+ '/hadoop/hadoop-tools/hadoop-aws/src/test/resources/auth-keys.xml'
340 data
=config_template
,
343 client
.run(args
=['cat', config_path
])