]>
Commit | Line | Data |
---|---|---|
224ce89b WB |
1 | import contextlib |
2 | import logging | |
224ce89b WB |
3 | from teuthology import misc |
4 | from teuthology.orchestra import run | |
5 | ||
6 | log = logging.getLogger(__name__) | |
7 | ||
8 | ||
9 | @contextlib.contextmanager | |
10 | def task(ctx, config): | |
11 | """ | |
12 | Run Hadoop S3A tests using Ceph | |
13 | usage: | |
14 | -tasks: | |
15 | ceph-ansible: | |
16 | s3a-hadoop: | |
9f95a23c TL |
17 | maven-version: '3.6.3' (default) |
18 | hadoop-version: '2.9.2' | |
224ce89b WB |
19 | bucket-name: 's3atest' (default) |
20 | access-key: 'anykey' (uses a default value) | |
21 | secret-key: 'secretkey' ( uses a default value) | |
494da23a | 22 | role: client.0 |
224ce89b WB |
23 | """ |
24 | if config is None: | |
25 | config = {} | |
26 | ||
27 | assert isinstance(config, dict), \ | |
28 | "task only supports a dictionary for configuration" | |
29 | ||
494da23a TL |
30 | assert hasattr(ctx, 'rgw'), 's3a-hadoop must run after the rgw task' |
31 | ||
224ce89b WB |
32 | overrides = ctx.config.get('overrides', {}) |
33 | misc.deep_merge(config, overrides.get('s3a-hadoop', {})) | |
34 | testdir = misc.get_testdir(ctx) | |
494da23a TL |
35 | |
36 | role = config.get('role') | |
37 | (remote,) = ctx.cluster.only(role).remotes.keys() | |
38 | endpoint = ctx.rgw.role_endpoints.get(role) | |
39 | assert endpoint, 's3tests: no rgw endpoint for {}'.format(role) | |
40 | ||
224ce89b WB |
41 | # get versions |
42 | maven_major = config.get('maven-major', 'maven-3') | |
9f95a23c TL |
43 | maven_version = config.get('maven-version', '3.6.3') |
44 | hadoop_ver = config.get('hadoop-version', '2.9.2') | |
224ce89b WB |
45 | bucket_name = config.get('bucket-name', 's3atest') |
46 | access_key = config.get('access-key', 'EGAQRD2ULOIFKFSKCT4F') | |
47 | secret_key = config.get( | |
48 | 'secret-key', | |
49 | 'zi816w1vZKfaSM85Cl0BxXTwSLyN7zB4RbTswrGb') | |
50 | ||
51 | # set versions for cloning the repo | |
52 | apache_maven = 'apache-maven-{maven_version}-bin.tar.gz'.format( | |
53 | maven_version=maven_version) | |
20effc67 | 54 | maven_link = 'http://archive.apache.org/dist/maven/' + \ |
224ce89b WB |
55 | '{maven_major}/{maven_version}/binaries/'.format(maven_major=maven_major, maven_version=maven_version) + apache_maven |
56 | hadoop_git = 'https://github.com/apache/hadoop' | |
57 | hadoop_rel = 'hadoop-{ver} rel/release-{ver}'.format(ver=hadoop_ver) | |
9f95a23c TL |
58 | if hadoop_ver == 'trunk': |
59 | # just checkout a new branch out of trunk | |
60 | hadoop_rel = 'hadoop-ceph-trunk' | |
494da23a TL |
61 | install_prereq(remote) |
62 | remote.run( | |
224ce89b WB |
63 | args=[ |
64 | 'cd', | |
65 | testdir, | |
66 | run.Raw('&&'), | |
67 | 'wget', | |
68 | maven_link, | |
69 | run.Raw('&&'), | |
70 | 'tar', | |
71 | '-xvf', | |
72 | apache_maven, | |
73 | run.Raw('&&'), | |
74 | 'git', | |
75 | 'clone', | |
76 | run.Raw(hadoop_git), | |
77 | run.Raw('&&'), | |
78 | 'cd', | |
79 | 'hadoop', | |
80 | run.Raw('&&'), | |
81 | 'git', | |
82 | 'checkout', | |
83 | '-b', | |
84 | run.Raw(hadoop_rel) | |
85 | ] | |
86 | ) | |
494da23a TL |
87 | configure_s3a(remote, endpoint.dns_name, access_key, secret_key, bucket_name, testdir) |
88 | setup_user_bucket(remote, endpoint.dns_name, access_key, secret_key, bucket_name, testdir) | |
224ce89b | 89 | if hadoop_ver.startswith('2.8'): |
3efd9988 FG |
90 | # test all ITtests but skip AWS test using public bucket landsat-pds |
91 | # which is not available from within this test | |
11fdf7f2 TL |
92 | test_options = '-Dit.test=ITestS3A* -Dparallel-tests -Dscale \ |
93 | -Dfs.s3a.scale.test.timeout=1200 \ | |
94 | -Dfs.s3a.scale.test.huge.filesize=256M verify' | |
224ce89b WB |
95 | else: |
96 | test_options = 'test -Dtest=S3a*,TestS3A*' | |
97 | try: | |
494da23a | 98 | run_s3atest(remote, maven_version, testdir, test_options) |
224ce89b WB |
99 | yield |
100 | finally: | |
101 | log.info("Done s3a testing, Cleaning up") | |
102 | for fil in ['apache*', 'hadoop*', 'venv*', 'create*']: | |
494da23a | 103 | remote.run(args=['rm', run.Raw('-rf'), run.Raw('{tdir}/{file}'.format(tdir=testdir, file=fil))]) |
224ce89b WB |
104 | |
105 | ||
106 | def install_prereq(client): | |
107 | """ | |
108 | Install pre requisites for RHEL and CentOS | |
109 | TBD: Ubuntu | |
110 | """ | |
111 | if client.os.name == 'rhel' or client.os.name == 'centos': | |
112 | client.run( | |
113 | args=[ | |
114 | 'sudo', | |
115 | 'yum', | |
116 | 'install', | |
117 | '-y', | |
118 | 'protobuf-c.x86_64', | |
119 | 'java', | |
120 | 'java-1.8.0-openjdk-devel', | |
121 | 'dnsmasq' | |
122 | ] | |
123 | ) | |
124 | ||
125 | ||
224ce89b WB |
126 | def setup_user_bucket(client, dns_name, access_key, secret_key, bucket_name, testdir): |
127 | """ | |
128 | Create user with access_key and secret_key that will be | |
129 | used for the s3a testdir | |
130 | """ | |
131 | client.run( | |
132 | args=[ | |
133 | 'sudo', | |
134 | 'radosgw-admin', | |
135 | 'user', | |
136 | 'create', | |
137 | run.Raw('--uid'), | |
138 | 's3a', | |
9f95a23c | 139 | run.Raw('--display-name="s3a cephtests"'), |
224ce89b WB |
140 | run.Raw('--access-key={access_key}'.format(access_key=access_key)), |
141 | run.Raw('--secret-key={secret_key}'.format(secret_key=secret_key)), | |
142 | run.Raw('--email=s3a@ceph.com'), | |
143 | ] | |
144 | ) | |
145 | client.run( | |
146 | args=[ | |
a4b75251 TL |
147 | 'python3', |
148 | '-m', | |
149 | 'venv', | |
224ce89b WB |
150 | '{testdir}/venv'.format(testdir=testdir), |
151 | run.Raw('&&'), | |
152 | run.Raw('{testdir}/venv/bin/pip'.format(testdir=testdir)), | |
153 | 'install', | |
154 | 'boto' | |
155 | ] | |
156 | ) | |
157 | create_bucket = """ | |
158 | #!/usr/bin/env python | |
159 | import boto | |
160 | import boto.s3.connection | |
161 | access_key = '{access_key}' | |
162 | secret_key = '{secret_key}' | |
163 | ||
164 | conn = boto.connect_s3( | |
165 | aws_access_key_id = access_key, | |
166 | aws_secret_access_key = secret_key, | |
167 | host = '{dns_name}', | |
168 | is_secure=False, | |
169 | calling_format = boto.s3.connection.OrdinaryCallingFormat(), | |
170 | ) | |
171 | bucket = conn.create_bucket('{bucket_name}') | |
172 | for bucket in conn.get_all_buckets(): | |
9f95a23c | 173 | print(bucket.name + "\t" + bucket.creation_date) |
224ce89b WB |
174 | """.format(access_key=access_key, secret_key=secret_key, dns_name=dns_name, bucket_name=bucket_name) |
175 | py_bucket_file = '{testdir}/create_bucket.py'.format(testdir=testdir) | |
f67539c2 | 176 | client.sudo_write_file(py_bucket_file, create_bucket, mode='0744') |
224ce89b WB |
177 | client.run( |
178 | args=[ | |
179 | 'cat', | |
180 | '{testdir}/create_bucket.py'.format(testdir=testdir), | |
181 | ] | |
182 | ) | |
183 | client.run( | |
184 | args=[ | |
185 | '{testdir}/venv/bin/python'.format(testdir=testdir), | |
186 | '{testdir}/create_bucket.py'.format(testdir=testdir), | |
187 | ] | |
188 | ) | |
189 | ||
190 | ||
191 | def run_s3atest(client, maven_version, testdir, test_options): | |
192 | """ | |
193 | Finally run the s3a test | |
194 | """ | |
195 | aws_testdir = '{testdir}/hadoop/hadoop-tools/hadoop-aws/'.format(testdir=testdir) | |
196 | run_test = '{testdir}/apache-maven-{maven_version}/bin/mvn'.format(testdir=testdir, maven_version=maven_version) | |
11fdf7f2 TL |
197 | # Remove AWS CredentialsProvider tests as it hits public bucket from AWS |
198 | # better solution is to create the public bucket on local server and test | |
199 | rm_test = 'rm src/test/java/org/apache/hadoop/fs/s3a/ITestS3AAWSCredentialsProvider.java' | |
224ce89b WB |
200 | client.run( |
201 | args=[ | |
202 | 'cd', | |
203 | run.Raw(aws_testdir), | |
204 | run.Raw('&&'), | |
11fdf7f2 TL |
205 | run.Raw(rm_test), |
206 | run.Raw('&&'), | |
224ce89b WB |
207 | run.Raw(run_test), |
208 | run.Raw(test_options) | |
209 | ] | |
210 | ) | |
211 | ||
212 | ||
213 | def configure_s3a(client, dns_name, access_key, secret_key, bucket_name, testdir): | |
214 | """ | |
215 | Use the template to configure s3a test, Fill in access_key, secret_key | |
216 | and other details required for test. | |
217 | """ | |
218 | config_template = """<configuration> | |
219 | <property> | |
220 | <name>fs.s3a.endpoint</name> | |
221 | <value>{name}</value> | |
222 | </property> | |
223 | ||
11fdf7f2 TL |
224 | <property> |
225 | <name>fs.contract.test.fs.s3a</name> | |
226 | <value>s3a://{bucket_name}/</value> | |
227 | </property> | |
228 | ||
224ce89b WB |
229 | <property> |
230 | <name>fs.s3a.connection.ssl.enabled</name> | |
231 | <value>false</value> | |
232 | </property> | |
233 | ||
234 | <property> | |
235 | <name>test.fs.s3n.name</name> | |
236 | <value>s3n://{bucket_name}/</value> | |
237 | </property> | |
238 | ||
239 | <property> | |
240 | <name>test.fs.s3a.name</name> | |
241 | <value>s3a://{bucket_name}/</value> | |
242 | </property> | |
243 | ||
244 | <property> | |
245 | <name>test.fs.s3.name</name> | |
246 | <value>s3://{bucket_name}/</value> | |
247 | </property> | |
248 | ||
249 | <property> | |
250 | <name>fs.s3.awsAccessKeyId</name> | |
251 | <value>{access_key}</value> | |
252 | </property> | |
253 | ||
254 | <property> | |
255 | <name>fs.s3.awsSecretAccessKey</name> | |
256 | <value>{secret_key}</value> | |
257 | </property> | |
258 | ||
259 | <property> | |
260 | <name>fs.s3n.awsAccessKeyId</name> | |
261 | <value>{access_key}</value> | |
262 | </property> | |
263 | ||
264 | <property> | |
265 | <name>fs.s3n.awsSecretAccessKey</name> | |
266 | <value>{secret_key}</value> | |
267 | </property> | |
268 | ||
269 | <property> | |
270 | <name>fs.s3a.access.key</name> | |
271 | <description>AWS access key ID. Omit for Role-based authentication.</description> | |
272 | <value>{access_key}</value> | |
273 | </property> | |
274 | ||
275 | <property> | |
276 | <name>fs.s3a.secret.key</name> | |
277 | <description>AWS secret key. Omit for Role-based authentication.</description> | |
278 | <value>{secret_key}</value> | |
279 | </property> | |
280 | </configuration> | |
281 | """.format(name=dns_name, bucket_name=bucket_name, access_key=access_key, secret_key=secret_key) | |
282 | config_path = testdir + '/hadoop/hadoop-tools/hadoop-aws/src/test/resources/auth-keys.xml' | |
f67539c2 | 283 | client.write_file(config_path, config_template) |
224ce89b WB |
284 | # output for debug |
285 | client.run(args=['cat', config_path]) |