]>
Commit | Line | Data |
---|---|---|
224ce89b WB |
1 | import contextlib |
2 | import logging | |
224ce89b WB |
3 | from teuthology import misc |
4 | from teuthology.orchestra import run | |
5 | ||
6 | log = logging.getLogger(__name__) | |
7 | ||
8 | ||
9 | @contextlib.contextmanager | |
10 | def task(ctx, config): | |
11 | """ | |
12 | Run Hadoop S3A tests using Ceph | |
13 | usage: | |
14 | -tasks: | |
15 | ceph-ansible: | |
16 | s3a-hadoop: | |
9f95a23c TL |
17 | maven-version: '3.6.3' (default) |
18 | hadoop-version: '2.9.2' | |
224ce89b WB |
19 | bucket-name: 's3atest' (default) |
20 | access-key: 'anykey' (uses a default value) | |
21 | secret-key: 'secretkey' ( uses a default value) | |
494da23a | 22 | role: client.0 |
224ce89b WB |
23 | """ |
24 | if config is None: | |
25 | config = {} | |
26 | ||
27 | assert isinstance(config, dict), \ | |
28 | "task only supports a dictionary for configuration" | |
29 | ||
494da23a TL |
30 | assert hasattr(ctx, 'rgw'), 's3a-hadoop must run after the rgw task' |
31 | ||
224ce89b WB |
32 | overrides = ctx.config.get('overrides', {}) |
33 | misc.deep_merge(config, overrides.get('s3a-hadoop', {})) | |
34 | testdir = misc.get_testdir(ctx) | |
494da23a TL |
35 | |
36 | role = config.get('role') | |
37 | (remote,) = ctx.cluster.only(role).remotes.keys() | |
38 | endpoint = ctx.rgw.role_endpoints.get(role) | |
39 | assert endpoint, 's3tests: no rgw endpoint for {}'.format(role) | |
40 | ||
224ce89b WB |
41 | # get versions |
42 | maven_major = config.get('maven-major', 'maven-3') | |
9f95a23c TL |
43 | maven_version = config.get('maven-version', '3.6.3') |
44 | hadoop_ver = config.get('hadoop-version', '2.9.2') | |
224ce89b WB |
45 | bucket_name = config.get('bucket-name', 's3atest') |
46 | access_key = config.get('access-key', 'EGAQRD2ULOIFKFSKCT4F') | |
47 | secret_key = config.get( | |
48 | 'secret-key', | |
49 | 'zi816w1vZKfaSM85Cl0BxXTwSLyN7zB4RbTswrGb') | |
50 | ||
51 | # set versions for cloning the repo | |
52 | apache_maven = 'apache-maven-{maven_version}-bin.tar.gz'.format( | |
53 | maven_version=maven_version) | |
9f95a23c | 54 | maven_link = 'http://www-us.apache.org/dist/maven/' + \ |
224ce89b WB |
55 | '{maven_major}/{maven_version}/binaries/'.format(maven_major=maven_major, maven_version=maven_version) + apache_maven |
56 | hadoop_git = 'https://github.com/apache/hadoop' | |
57 | hadoop_rel = 'hadoop-{ver} rel/release-{ver}'.format(ver=hadoop_ver) | |
9f95a23c TL |
58 | if hadoop_ver == 'trunk': |
59 | # just checkout a new branch out of trunk | |
60 | hadoop_rel = 'hadoop-ceph-trunk' | |
494da23a TL |
61 | install_prereq(remote) |
62 | remote.run( | |
224ce89b WB |
63 | args=[ |
64 | 'cd', | |
65 | testdir, | |
66 | run.Raw('&&'), | |
67 | 'wget', | |
68 | maven_link, | |
69 | run.Raw('&&'), | |
70 | 'tar', | |
71 | '-xvf', | |
72 | apache_maven, | |
73 | run.Raw('&&'), | |
74 | 'git', | |
75 | 'clone', | |
76 | run.Raw(hadoop_git), | |
77 | run.Raw('&&'), | |
78 | 'cd', | |
79 | 'hadoop', | |
80 | run.Raw('&&'), | |
81 | 'git', | |
82 | 'checkout', | |
83 | '-b', | |
84 | run.Raw(hadoop_rel) | |
85 | ] | |
86 | ) | |
494da23a TL |
87 | configure_s3a(remote, endpoint.dns_name, access_key, secret_key, bucket_name, testdir) |
88 | setup_user_bucket(remote, endpoint.dns_name, access_key, secret_key, bucket_name, testdir) | |
224ce89b | 89 | if hadoop_ver.startswith('2.8'): |
3efd9988 FG |
90 | # test all ITtests but skip AWS test using public bucket landsat-pds |
91 | # which is not available from within this test | |
11fdf7f2 TL |
92 | test_options = '-Dit.test=ITestS3A* -Dparallel-tests -Dscale \ |
93 | -Dfs.s3a.scale.test.timeout=1200 \ | |
94 | -Dfs.s3a.scale.test.huge.filesize=256M verify' | |
224ce89b WB |
95 | else: |
96 | test_options = 'test -Dtest=S3a*,TestS3A*' | |
97 | try: | |
494da23a | 98 | run_s3atest(remote, maven_version, testdir, test_options) |
224ce89b WB |
99 | yield |
100 | finally: | |
101 | log.info("Done s3a testing, Cleaning up") | |
102 | for fil in ['apache*', 'hadoop*', 'venv*', 'create*']: | |
494da23a | 103 | remote.run(args=['rm', run.Raw('-rf'), run.Raw('{tdir}/{file}'.format(tdir=testdir, file=fil))]) |
224ce89b WB |
104 | |
105 | ||
106 | def install_prereq(client): | |
107 | """ | |
108 | Install pre requisites for RHEL and CentOS | |
109 | TBD: Ubuntu | |
110 | """ | |
111 | if client.os.name == 'rhel' or client.os.name == 'centos': | |
112 | client.run( | |
113 | args=[ | |
114 | 'sudo', | |
115 | 'yum', | |
116 | 'install', | |
117 | '-y', | |
118 | 'protobuf-c.x86_64', | |
119 | 'java', | |
120 | 'java-1.8.0-openjdk-devel', | |
121 | 'dnsmasq' | |
122 | ] | |
123 | ) | |
124 | ||
125 | ||
224ce89b WB |
126 | def setup_user_bucket(client, dns_name, access_key, secret_key, bucket_name, testdir): |
127 | """ | |
128 | Create user with access_key and secret_key that will be | |
129 | used for the s3a testdir | |
130 | """ | |
131 | client.run( | |
132 | args=[ | |
133 | 'sudo', | |
134 | 'radosgw-admin', | |
135 | 'user', | |
136 | 'create', | |
137 | run.Raw('--uid'), | |
138 | 's3a', | |
9f95a23c | 139 | run.Raw('--display-name="s3a cephtests"'), |
224ce89b WB |
140 | run.Raw('--access-key={access_key}'.format(access_key=access_key)), |
141 | run.Raw('--secret-key={secret_key}'.format(secret_key=secret_key)), | |
142 | run.Raw('--email=s3a@ceph.com'), | |
143 | ] | |
144 | ) | |
145 | client.run( | |
146 | args=[ | |
147 | 'virtualenv', | |
148 | '{testdir}/venv'.format(testdir=testdir), | |
149 | run.Raw('&&'), | |
150 | run.Raw('{testdir}/venv/bin/pip'.format(testdir=testdir)), | |
151 | 'install', | |
152 | 'boto' | |
153 | ] | |
154 | ) | |
155 | create_bucket = """ | |
156 | #!/usr/bin/env python | |
157 | import boto | |
158 | import boto.s3.connection | |
159 | access_key = '{access_key}' | |
160 | secret_key = '{secret_key}' | |
161 | ||
162 | conn = boto.connect_s3( | |
163 | aws_access_key_id = access_key, | |
164 | aws_secret_access_key = secret_key, | |
165 | host = '{dns_name}', | |
166 | is_secure=False, | |
167 | calling_format = boto.s3.connection.OrdinaryCallingFormat(), | |
168 | ) | |
169 | bucket = conn.create_bucket('{bucket_name}') | |
170 | for bucket in conn.get_all_buckets(): | |
9f95a23c | 171 | print(bucket.name + "\t" + bucket.creation_date) |
224ce89b WB |
172 | """.format(access_key=access_key, secret_key=secret_key, dns_name=dns_name, bucket_name=bucket_name) |
173 | py_bucket_file = '{testdir}/create_bucket.py'.format(testdir=testdir) | |
174 | misc.sudo_write_file( | |
175 | remote=client, | |
176 | path=py_bucket_file, | |
177 | data=create_bucket, | |
178 | perms='0744', | |
179 | ) | |
180 | client.run( | |
181 | args=[ | |
182 | 'cat', | |
183 | '{testdir}/create_bucket.py'.format(testdir=testdir), | |
184 | ] | |
185 | ) | |
186 | client.run( | |
187 | args=[ | |
188 | '{testdir}/venv/bin/python'.format(testdir=testdir), | |
189 | '{testdir}/create_bucket.py'.format(testdir=testdir), | |
190 | ] | |
191 | ) | |
192 | ||
193 | ||
194 | def run_s3atest(client, maven_version, testdir, test_options): | |
195 | """ | |
196 | Finally run the s3a test | |
197 | """ | |
198 | aws_testdir = '{testdir}/hadoop/hadoop-tools/hadoop-aws/'.format(testdir=testdir) | |
199 | run_test = '{testdir}/apache-maven-{maven_version}/bin/mvn'.format(testdir=testdir, maven_version=maven_version) | |
11fdf7f2 TL |
200 | # Remove AWS CredentialsProvider tests as it hits public bucket from AWS |
201 | # better solution is to create the public bucket on local server and test | |
202 | rm_test = 'rm src/test/java/org/apache/hadoop/fs/s3a/ITestS3AAWSCredentialsProvider.java' | |
224ce89b WB |
203 | client.run( |
204 | args=[ | |
205 | 'cd', | |
206 | run.Raw(aws_testdir), | |
207 | run.Raw('&&'), | |
11fdf7f2 TL |
208 | run.Raw(rm_test), |
209 | run.Raw('&&'), | |
224ce89b WB |
210 | run.Raw(run_test), |
211 | run.Raw(test_options) | |
212 | ] | |
213 | ) | |
214 | ||
215 | ||
216 | def configure_s3a(client, dns_name, access_key, secret_key, bucket_name, testdir): | |
217 | """ | |
218 | Use the template to configure s3a test, Fill in access_key, secret_key | |
219 | and other details required for test. | |
220 | """ | |
221 | config_template = """<configuration> | |
222 | <property> | |
223 | <name>fs.s3a.endpoint</name> | |
224 | <value>{name}</value> | |
225 | </property> | |
226 | ||
11fdf7f2 TL |
227 | <property> |
228 | <name>fs.contract.test.fs.s3a</name> | |
229 | <value>s3a://{bucket_name}/</value> | |
230 | </property> | |
231 | ||
224ce89b WB |
232 | <property> |
233 | <name>fs.s3a.connection.ssl.enabled</name> | |
234 | <value>false</value> | |
235 | </property> | |
236 | ||
237 | <property> | |
238 | <name>test.fs.s3n.name</name> | |
239 | <value>s3n://{bucket_name}/</value> | |
240 | </property> | |
241 | ||
242 | <property> | |
243 | <name>test.fs.s3a.name</name> | |
244 | <value>s3a://{bucket_name}/</value> | |
245 | </property> | |
246 | ||
247 | <property> | |
248 | <name>test.fs.s3.name</name> | |
249 | <value>s3://{bucket_name}/</value> | |
250 | </property> | |
251 | ||
252 | <property> | |
253 | <name>fs.s3.awsAccessKeyId</name> | |
254 | <value>{access_key}</value> | |
255 | </property> | |
256 | ||
257 | <property> | |
258 | <name>fs.s3.awsSecretAccessKey</name> | |
259 | <value>{secret_key}</value> | |
260 | </property> | |
261 | ||
262 | <property> | |
263 | <name>fs.s3n.awsAccessKeyId</name> | |
264 | <value>{access_key}</value> | |
265 | </property> | |
266 | ||
267 | <property> | |
268 | <name>fs.s3n.awsSecretAccessKey</name> | |
269 | <value>{secret_key}</value> | |
270 | </property> | |
271 | ||
272 | <property> | |
273 | <name>fs.s3a.access.key</name> | |
274 | <description>AWS access key ID. Omit for Role-based authentication.</description> | |
275 | <value>{access_key}</value> | |
276 | </property> | |
277 | ||
278 | <property> | |
279 | <name>fs.s3a.secret.key</name> | |
280 | <description>AWS secret key. Omit for Role-based authentication.</description> | |
281 | <value>{secret_key}</value> | |
282 | </property> | |
283 | </configuration> | |
284 | """.format(name=dns_name, bucket_name=bucket_name, access_key=access_key, secret_key=secret_key) | |
285 | config_path = testdir + '/hadoop/hadoop-tools/hadoop-aws/src/test/resources/auth-keys.xml' | |
286 | misc.write_file( | |
287 | remote=client, | |
288 | path=config_path, | |
289 | data=config_template, | |
290 | ) | |
291 | # output for debug | |
292 | client.run(args=['cat', config_path]) |