]>
git.proxmox.com Git - ceph.git/blob - ceph/qa/tasks/s3a_hadoop.py
3 from teuthology
import misc
4 from teuthology
.orchestra
import run
6 log
= logging
.getLogger(__name__
)
9 @contextlib.contextmanager
10 def task(ctx
, config
):
12 Run Hadoop S3A tests using Ceph
17 maven-version: '3.6.3' (default)
18 hadoop-version: '2.9.2'
19 bucket-name: 's3atest' (default)
20 access-key: 'anykey' (uses a default value)
21 secret-key: 'secretkey' ( uses a default value)
27 assert isinstance(config
, dict), \
28 "task only supports a dictionary for configuration"
30 assert hasattr(ctx
, 'rgw'), 's3a-hadoop must run after the rgw task'
32 overrides
= ctx
.config
.get('overrides', {})
33 misc
.deep_merge(config
, overrides
.get('s3a-hadoop', {}))
34 testdir
= misc
.get_testdir(ctx
)
36 role
= config
.get('role')
37 (remote
,) = ctx
.cluster
.only(role
).remotes
.keys()
38 endpoint
= ctx
.rgw
.role_endpoints
.get(role
)
39 assert endpoint
, 's3tests: no rgw endpoint for {}'.format(role
)
42 maven_major
= config
.get('maven-major', 'maven-3')
43 maven_version
= config
.get('maven-version', '3.6.3')
44 hadoop_ver
= config
.get('hadoop-version', '2.9.2')
45 bucket_name
= config
.get('bucket-name', 's3atest')
46 access_key
= config
.get('access-key', 'EGAQRD2ULOIFKFSKCT4F')
47 secret_key
= config
.get(
49 'zi816w1vZKfaSM85Cl0BxXTwSLyN7zB4RbTswrGb')
51 # set versions for cloning the repo
52 apache_maven
= 'apache-maven-{maven_version}-bin.tar.gz'.format(
53 maven_version
=maven_version
)
54 maven_link
= 'http://archive.apache.org/dist/maven/' + \
55 '{maven_major}/{maven_version}/binaries/'.format(maven_major
=maven_major
, maven_version
=maven_version
) + apache_maven
56 hadoop_git
= 'https://github.com/apache/hadoop'
57 hadoop_rel
= 'hadoop-{ver} rel/release-{ver}'.format(ver
=hadoop_ver
)
58 if hadoop_ver
== 'trunk':
59 # just checkout a new branch out of trunk
60 hadoop_rel
= 'hadoop-ceph-trunk'
61 install_prereq(remote
)
87 configure_s3a(remote
, endpoint
.dns_name
, access_key
, secret_key
, bucket_name
, testdir
)
88 setup_user_bucket(remote
, endpoint
.dns_name
, access_key
, secret_key
, bucket_name
, testdir
)
89 if hadoop_ver
.startswith('2.8'):
90 # test all ITtests but skip AWS test using public bucket landsat-pds
91 # which is not available from within this test
92 test_options
= '-Dit.test=ITestS3A* -Dparallel-tests -Dscale \
93 -Dfs.s3a.scale.test.timeout=1200 \
94 -Dfs.s3a.scale.test.huge.filesize=256M verify'
96 test_options
= 'test -Dtest=S3a*,TestS3A*'
98 run_s3atest(remote
, maven_version
, testdir
, test_options
)
101 log
.info("Done s3a testing, Cleaning up")
102 for fil
in ['apache*', 'hadoop*', 'venv*', 'create*']:
103 remote
.run(args
=['rm', run
.Raw('-rf'), run
.Raw('{tdir}/{file}'.format(tdir
=testdir
, file=fil
))])
106 def install_prereq(client
):
108 Install pre requisites for RHEL and CentOS
111 if client
.os
.name
== 'rhel' or client
.os
.name
== 'centos':
120 'java-1.8.0-openjdk-devel',
126 def setup_user_bucket(client
, dns_name
, access_key
, secret_key
, bucket_name
, testdir
):
128 Create user with access_key and secret_key that will be
129 used for the s3a testdir
139 run
.Raw('--display-name="s3a cephtests"'),
140 run
.Raw('--access-key={access_key}'.format(access_key
=access_key
)),
141 run
.Raw('--secret-key={secret_key}'.format(secret_key
=secret_key
)),
142 run
.Raw('--email=s3a@ceph.com'),
150 '{testdir}/venv'.format(testdir
=testdir
),
152 run
.Raw('{testdir}/venv/bin/pip'.format(testdir
=testdir
)),
158 #!/usr/bin/env python
160 import boto.s3.connection
161 access_key = '{access_key}'
162 secret_key = '{secret_key}'
164 conn = boto.connect_s3(
165 aws_access_key_id = access_key,
166 aws_secret_access_key = secret_key,
169 calling_format = boto.s3.connection.OrdinaryCallingFormat(),
171 bucket = conn.create_bucket('{bucket_name}')
172 for bucket in conn.get_all_buckets():
173 print(bucket.name + "\t" + bucket.creation_date)
174 """.format(access_key
=access_key
, secret_key
=secret_key
, dns_name
=dns_name
, bucket_name
=bucket_name
)
175 py_bucket_file
= '{testdir}/create_bucket.py'.format(testdir
=testdir
)
176 client
.sudo_write_file(py_bucket_file
, create_bucket
, mode
='0744')
180 '{testdir}/create_bucket.py'.format(testdir
=testdir
),
185 '{testdir}/venv/bin/python'.format(testdir
=testdir
),
186 '{testdir}/create_bucket.py'.format(testdir
=testdir
),
191 def run_s3atest(client
, maven_version
, testdir
, test_options
):
193 Finally run the s3a test
195 aws_testdir
= '{testdir}/hadoop/hadoop-tools/hadoop-aws/'.format(testdir
=testdir
)
196 run_test
= '{testdir}/apache-maven-{maven_version}/bin/mvn'.format(testdir
=testdir
, maven_version
=maven_version
)
197 # Remove AWS CredentialsProvider tests as it hits public bucket from AWS
198 # better solution is to create the public bucket on local server and test
199 rm_test
= 'rm src/test/java/org/apache/hadoop/fs/s3a/ITestS3AAWSCredentialsProvider.java'
203 run
.Raw(aws_testdir
),
208 run
.Raw(test_options
)
213 def configure_s3a(client
, dns_name
, access_key
, secret_key
, bucket_name
, testdir
):
215 Use the template to configure s3a test, Fill in access_key, secret_key
216 and other details required for test.
218 config_template
= """<configuration>
220 <name>fs.s3a.endpoint</name>
221 <value>{name}</value>
225 <name>fs.contract.test.fs.s3a</name>
226 <value>s3a://{bucket_name}/</value>
230 <name>fs.s3a.connection.ssl.enabled</name>
235 <name>test.fs.s3n.name</name>
236 <value>s3n://{bucket_name}/</value>
240 <name>test.fs.s3a.name</name>
241 <value>s3a://{bucket_name}/</value>
245 <name>test.fs.s3.name</name>
246 <value>s3://{bucket_name}/</value>
250 <name>fs.s3.awsAccessKeyId</name>
251 <value>{access_key}</value>
255 <name>fs.s3.awsSecretAccessKey</name>
256 <value>{secret_key}</value>
260 <name>fs.s3n.awsAccessKeyId</name>
261 <value>{access_key}</value>
265 <name>fs.s3n.awsSecretAccessKey</name>
266 <value>{secret_key}</value>
270 <name>fs.s3a.access.key</name>
271 <description>AWS access key ID. Omit for Role-based authentication.</description>
272 <value>{access_key}</value>
276 <name>fs.s3a.secret.key</name>
277 <description>AWS secret key. Omit for Role-based authentication.</description>
278 <value>{secret_key}</value>
281 """.format(name
=dns_name
, bucket_name
=bucket_name
, access_key
=access_key
, secret_key
=secret_key
)
282 config_path
= testdir
+ '/hadoop/hadoop-tools/hadoop-aws/src/test/resources/auth-keys.xml'
283 client
.write_file(config_path
, config_template
)
285 client
.run(args
=['cat', config_path
])