]>
Commit | Line | Data |
---|---|---|
9f95a23c TL |
1 | """ |
2 | Task for running RGW S3 tests with the AWS Java SDK | |
3 | """ | |
e306af50 | 4 | from io import BytesIO |
9f95a23c TL |
5 | import logging |
6 | ||
7 | import base64 | |
8 | import os | |
9 | import random | |
10 | import string | |
11 | import yaml | |
12 | import getpass | |
13 | ||
14 | from teuthology import misc as teuthology | |
15 | from teuthology.task import Task | |
16 | from teuthology.orchestra import run | |
17 | ||
18 | log = logging.getLogger(__name__) | |
19 | ||
20 | """ | |
21 | Task for running RGW S3 tests with the AWS Java SDK | |
22 | ||
23 | Tests run only on clients specified in the s3tests-java config section. | |
24 | If no client is given a default 'client.0' is chosen. | |
25 | If such does not match the rgw client the task will fail. | |
26 | ||
27 | tasks: | |
28 | - ceph: | |
29 | - rgw: [client.0] | |
30 | - s3tests-java: | |
31 | client.0: | |
32 | ||
33 | Extra arguments can be passed by adding options to the corresponding client | |
34 | section under the s3tests-java task (e.g. to run a certain test, | |
35 | specify a different repository and branch for the test suite, | |
36 | run in info/debug mode (for the java suite) or forward the gradle output to a log file): | |
37 | ||
38 | tasks: | |
39 | - ceph: | |
40 | - rgw: [client.0] | |
41 | - s3tests-java: | |
42 | client.0: | |
43 | force-branch: wip | |
44 | force-repo: 'https://github.com/adamyanova/java_s3tests.git' | |
45 | log-fwd: '../s3tests-java.log' | |
46 | log-level: info | |
47 | extra-args: ['--tests', 'ObjectTest.testEncryptionKeySSECInvalidMd5'] | |
48 | ||
49 | To run a specific test, provide its name to the extra-args section e.g.: | |
50 | - s3tests-java: | |
51 | client.0: | |
52 | extra-args: ['--tests', 'ObjectTest.testEncryptionKeySSECInvalidMd5'] | |
53 | ||
54 | """ | |
55 | ||
56 | ||
57 | class S3tests_java(Task): | |
58 | """ | |
59 | Download and install S3 tests in Java | |
60 | This will require openjdk and gradle | |
61 | """ | |
62 | ||
63 | def __init__(self, ctx, config): | |
64 | super(S3tests_java, self).__init__(ctx, config) | |
65 | self.log = log | |
66 | log.debug('S3 Tests Java: __INIT__ ') | |
67 | assert hasattr(ctx, 'rgw'), 'S3tests_java must run after the rgw task' | |
68 | clients = ['client.{id}'.format(id=id_) | |
69 | for id_ in teuthology.all_roles_of_type(self.ctx.cluster, 'client')] | |
70 | self.all_clients = [] | |
71 | for client in clients: | |
72 | if client in self.config: | |
73 | self.all_clients.extend([client]) | |
74 | if self.all_clients is None: | |
75 | self.all_clients = 'client.0' | |
76 | self.users = {'s3main': 'tester', | |
77 | 's3alt': 'johndoe', 'tenanted': 'testx$tenanteduser'} | |
78 | ||
79 | def setup(self): | |
80 | super(S3tests_java, self).setup() | |
81 | log.debug('S3 Tests Java: SETUP') | |
82 | for client in self.all_clients: | |
83 | self.download_test_suite(client) | |
84 | self.install_required_packages(client) | |
85 | ||
86 | def begin(self): | |
87 | super(S3tests_java, self).begin() | |
88 | log.debug('S3 Tests Java: BEGIN') | |
89 | for (host, roles) in self.ctx.cluster.remotes.items(): | |
90 | log.debug( | |
91 | 'S3 Tests Java: Cluster config is: {cfg}'.format(cfg=roles)) | |
92 | log.debug('S3 Tests Java: Host is: {host}'.format(host=host)) | |
93 | self.create_users() | |
94 | self.run_tests() | |
95 | ||
96 | def end(self): | |
97 | super(S3tests_java, self).end() | |
98 | log.debug('S3 Tests Java: END') | |
99 | for client in self.all_clients: | |
100 | self.remove_tests(client) | |
101 | self.delete_users(client) | |
102 | ||
103 | def download_test_suite(self, client): | |
104 | log.info("S3 Tests Java: Downloading test suite...") | |
105 | testdir = teuthology.get_testdir(self.ctx) | |
106 | branch = 'master' | |
107 | repo = 'https://github.com/ceph/java_s3tests.git' | |
108 | if client in self.config and self.config[client] is not None: | |
109 | if 'force-branch' in self.config[client] and self.config[client]['force-branch'] is not None: | |
110 | branch = self.config[client]['force-branch'] | |
111 | if 'force-repo' in self.config[client] and self.config[client]['force-repo'] is not None: | |
112 | repo = self.config[client]['force-repo'] | |
113 | self.ctx.cluster.only(client).run( | |
114 | args=[ | |
115 | 'git', 'clone', | |
116 | '-b', branch, | |
117 | repo, | |
118 | '{tdir}/s3-tests-java'.format(tdir=testdir), | |
119 | ], | |
e306af50 | 120 | stdout=BytesIO() |
9f95a23c TL |
121 | ) |
122 | if client in self.config and self.config[client] is not None: | |
123 | if 'sha1' in self.config[client] and self.config[client]['sha1'] is not None: | |
124 | self.ctx.cluster.only(client).run( | |
125 | args=[ | |
126 | 'cd', '{tdir}/s3-tests-java'.format(tdir=testdir), | |
127 | run.Raw('&&'), | |
128 | 'git', 'reset', '--hard', self.config[client]['sha1'], | |
129 | ], | |
130 | ) | |
131 | ||
132 | if 'log-level' in self.config[client]: | |
133 | if self.config[client]['log-level'] == 'info': | |
134 | self.ctx.cluster.only(client).run( | |
135 | args=[ | |
136 | 'sed', '-i', '\'s/log4j.rootLogger=WARN/log4j.rootLogger=INFO/g\'', | |
137 | '{tdir}/s3-tests-java/src/main/resources/log4j.properties'.format( | |
138 | tdir=testdir) | |
139 | ] | |
140 | ) | |
141 | if self.config[client]['log-level'] == 'debug': | |
142 | self.ctx.cluster.only(client).run( | |
143 | args=[ | |
144 | 'sed', '-i', '\'s/log4j.rootLogger=WARN/log4j.rootLogger=DEBUG/g\'', | |
145 | '{tdir}/s3-tests-java/src/main/resources/log4j.properties'.format( | |
146 | tdir=testdir) | |
147 | ] | |
148 | ) | |
149 | ||
150 | def install_required_packages(self, client): | |
151 | """ | |
152 | Run bootstrap script to install openjdk and gradle. | |
153 | Add certificates to java keystore | |
154 | """ | |
155 | log.info("S3 Tests Java: Installing required packages...") | |
156 | testdir = teuthology.get_testdir(self.ctx) | |
157 | self.ctx.cluster.only(client).run( | |
158 | args=['{tdir}/s3-tests-java/bootstrap.sh'.format(tdir=testdir)], | |
e306af50 | 159 | stdout=BytesIO() |
9f95a23c TL |
160 | ) |
161 | ||
162 | endpoint = self.ctx.rgw.role_endpoints[client] | |
163 | if endpoint.cert: | |
164 | path = 'lib/security/cacerts' | |
165 | self.ctx.cluster.only(client).run( | |
166 | args=['sudo', | |
167 | 'keytool', | |
168 | '-import', '-alias', '{alias}'.format( | |
169 | alias=endpoint.hostname), | |
170 | '-keystore', | |
171 | run.Raw( | |
172 | '$(readlink -e $(dirname $(readlink -e $(which keytool)))/../{path})'.format(path=path)), | |
173 | '-file', endpoint.cert.certificate, | |
174 | '-storepass', 'changeit', | |
175 | ], | |
e306af50 | 176 | stdout=BytesIO() |
9f95a23c TL |
177 | ) |
178 | ||
179 | def create_users(self): | |
180 | """ | |
181 | Create a main and an alternative s3 user. | |
182 | Configuration is read from a skelethon config file | |
183 | s3tests.teuth.config.yaml in the java-s3tests repository | |
184 | and missing information is added from the task. | |
185 | Existing values are NOT overriden unless they are empty! | |
186 | """ | |
187 | log.info("S3 Tests Java: Creating S3 users...") | |
188 | testdir = teuthology.get_testdir(self.ctx) | |
189 | for client in self.all_clients: | |
190 | endpoint = self.ctx.rgw.role_endpoints.get(client) | |
191 | local_user = getpass.getuser() | |
192 | remote_user = teuthology.get_test_user() | |
193 | os.system("scp {remote}@{host}:{tdir}/s3-tests-java/s3tests.teuth.config.yaml /home/{local}/".format( | |
194 | host=endpoint.hostname, tdir=testdir, remote=remote_user, local=local_user)) | |
195 | s3tests_conf = teuthology.config_file( | |
196 | '/home/{local}/s3tests.teuth.config.yaml'.format(local=local_user)) | |
197 | log.debug("S3 Tests Java: s3tests_conf is {s3cfg}".format( | |
198 | s3cfg=s3tests_conf)) | |
e306af50 | 199 | for section, user in list(self.users.items()): |
9f95a23c TL |
200 | if section in s3tests_conf: |
201 | s3_user_id = '{user}.{client}'.format( | |
202 | user=user, client=client) | |
203 | log.debug( | |
204 | 'S3 Tests Java: Creating user {s3_user_id}'.format(s3_user_id=s3_user_id)) | |
205 | self._config_user(s3tests_conf=s3tests_conf, | |
206 | section=section, user=s3_user_id, client=client) | |
207 | cluster_name, daemon_type, client_id = teuthology.split_role( | |
208 | client) | |
209 | client_with_id = daemon_type + '.' + client_id | |
210 | args = [ | |
211 | 'adjust-ulimits', | |
212 | 'ceph-coverage', | |
213 | '{tdir}/archive/coverage'.format(tdir=testdir), | |
214 | 'radosgw-admin', | |
215 | '-n', client_with_id, | |
216 | 'user', 'create', | |
217 | '--uid', s3tests_conf[section]['user_id'], | |
218 | '--display-name', s3tests_conf[section]['display_name'], | |
219 | '--access-key', s3tests_conf[section]['access_key'], | |
220 | '--secret', s3tests_conf[section]['access_secret'], | |
221 | '--email', s3tests_conf[section]['email'], | |
222 | '--cluster', cluster_name, | |
223 | ] | |
224 | log.info('{args}'.format(args=args)) | |
225 | self.ctx.cluster.only(client).run( | |
226 | args=args, | |
e306af50 | 227 | stdout=BytesIO() |
9f95a23c TL |
228 | ) |
229 | else: | |
230 | self.users.pop(section) | |
231 | self._write_cfg_file(s3tests_conf, client) | |
232 | os.system( | |
233 | "rm -rf /home/{local}/s3tests.teuth.config.yaml".format(local=local_user)) | |
234 | ||
235 | def _config_user(self, s3tests_conf, section, user, client): | |
236 | """ | |
237 | Generate missing users data for this section by stashing away keys, ids, and | |
238 | email addresses. | |
239 | """ | |
240 | access_key = ''.join(random.choice(string.ascii_uppercase) | |
241 | for i in range(20)) | |
e306af50 | 242 | access_secret = base64.b64encode(os.urandom(40)).decode('ascii') |
9f95a23c TL |
243 | endpoint = self.ctx.rgw.role_endpoints.get(client) |
244 | ||
245 | self._set_cfg_entry( | |
246 | s3tests_conf[section], 'user_id', '{user}'.format(user=user)) | |
247 | self._set_cfg_entry( | |
248 | s3tests_conf[section], 'email', '{user}_test@test.test'.format(user=user)) | |
249 | self._set_cfg_entry( | |
250 | s3tests_conf[section], 'display_name', 'Ms. {user}'.format(user=user)) | |
251 | self._set_cfg_entry( | |
252 | s3tests_conf[section], 'access_key', '{ak}'.format(ak=access_key)) | |
253 | self._set_cfg_entry( | |
254 | s3tests_conf[section], 'access_secret', '{asc}'.format(asc=access_secret)) | |
255 | self._set_cfg_entry( | |
256 | s3tests_conf[section], 'region', 'us-east-1') | |
257 | self._set_cfg_entry( | |
258 | s3tests_conf[section], 'endpoint', '{ip}:{port}'.format( | |
259 | ip=endpoint.hostname, port=endpoint.port)) | |
260 | self._set_cfg_entry( | |
261 | s3tests_conf[section], 'host', endpoint.hostname) | |
262 | self._set_cfg_entry( | |
263 | s3tests_conf[section], 'port', endpoint.port) | |
264 | self._set_cfg_entry( | |
265 | s3tests_conf[section], 'is_secure', True if endpoint.cert else False) | |
266 | ||
267 | log.debug("S3 Tests Java: s3tests_conf[{sect}] is {s3cfg}".format( | |
268 | sect=section, s3cfg=s3tests_conf[section])) | |
269 | log.debug('S3 Tests Java: Setion, User = {sect}, {user}'.format( | |
270 | sect=section, user=user)) | |
271 | ||
272 | def _write_cfg_file(self, cfg_dict, client): | |
273 | """ | |
274 | To write the final s3 tests config file on the remote | |
275 | a temporary one is created on the local machine | |
276 | """ | |
277 | testdir = teuthology.get_testdir(self.ctx) | |
278 | (remote,) = self.ctx.cluster.only(client).remotes.keys() | |
279 | with open('s3_tests_tmp.yaml', 'w') as outfile: | |
280 | yaml.dump(cfg_dict, outfile, default_flow_style=False) | |
281 | ||
e306af50 TL |
282 | conf_fp = BytesIO() |
283 | with open('s3_tests_tmp.yaml', 'rb') as infile: | |
9f95a23c TL |
284 | for line in infile: |
285 | conf_fp.write(line) | |
286 | ||
287 | teuthology.write_file( | |
288 | remote=remote, | |
289 | path='{tdir}/archive/s3-tests-java.{client}.conf'.format( | |
290 | tdir=testdir, client=client), | |
291 | data=conf_fp.getvalue(), | |
292 | ) | |
293 | os.remove('s3_tests_tmp.yaml') | |
294 | ||
295 | def _set_cfg_entry(self, cfg_dict, key, value): | |
296 | if not (key in cfg_dict): | |
297 | cfg_dict.setdefault(key, value) | |
298 | elif cfg_dict[key] is None: | |
299 | cfg_dict[key] = value | |
300 | ||
301 | def run_tests(self): | |
302 | log.info("S3 Tests Java: Running tests...") | |
303 | testdir = teuthology.get_testdir(self.ctx) | |
304 | for client in self.all_clients: | |
305 | self.ctx.cluster.only(client).run( | |
306 | args=['cp', | |
307 | '{tdir}/archive/s3-tests-java.{client}.conf'.format( | |
308 | tdir=testdir, client=client), | |
309 | '{tdir}/s3-tests-java/config.properties'.format( | |
310 | tdir=testdir) | |
311 | ], | |
e306af50 | 312 | stdout=BytesIO() |
9f95a23c TL |
313 | ) |
314 | args = ['cd', | |
315 | '{tdir}/s3-tests-java'.format(tdir=testdir), | |
316 | run.Raw('&&'), | |
317 | '/opt/gradle/gradle/bin/gradle', 'clean', 'test', | |
318 | '--rerun-tasks', '--no-build-cache', | |
319 | ] | |
320 | extra_args = [] | |
321 | suppress_groups = False | |
322 | self.log_fwd = False | |
323 | self.log_name = '' | |
324 | if client in self.config and self.config[client] is not None: | |
325 | if 'extra-args' in self.config[client]: | |
326 | extra_args.extend(self.config[client]['extra-args']) | |
327 | suppress_groups = True | |
328 | if 'log-level' in self.config[client] and self.config[client]['log-level'] == 'debug': | |
329 | extra_args += ['--debug'] | |
330 | if 'log-fwd' in self.config[client]: | |
331 | self.log_fwd = True | |
332 | self.log_name = '{tdir}/s3tests_log.txt'.format( | |
333 | tdir=testdir) | |
334 | if self.config[client]['log-fwd'] is not None: | |
335 | self.log_name = self.config[client]['log-fwd'] | |
336 | extra_args += [run.Raw('>>'), | |
337 | self.log_name] | |
338 | ||
339 | if not suppress_groups: | |
340 | test_groups = ['AWS4Test', 'BucketTest', 'ObjectTest'] | |
341 | else: | |
342 | test_groups = ['All'] | |
343 | ||
344 | for gr in test_groups: | |
345 | for i in range(2): | |
346 | self.ctx.cluster.only(client).run( | |
347 | args=['radosgw-admin', 'gc', | |
348 | 'process', '--include-all'], | |
e306af50 | 349 | stdout=BytesIO() |
9f95a23c TL |
350 | ) |
351 | ||
352 | if gr != 'All': | |
353 | self.ctx.cluster.only(client).run( | |
354 | args=args + ['--tests'] + [gr] + extra_args, | |
e306af50 | 355 | stdout=BytesIO() |
9f95a23c TL |
356 | ) |
357 | else: | |
358 | self.ctx.cluster.only(client).run( | |
359 | args=args + extra_args, | |
e306af50 | 360 | stdout=BytesIO() |
9f95a23c TL |
361 | ) |
362 | ||
363 | for i in range(2): | |
364 | self.ctx.cluster.only(client).run( | |
365 | args=['radosgw-admin', 'gc', | |
366 | 'process', '--include-all'], | |
e306af50 | 367 | stdout=BytesIO() |
9f95a23c TL |
368 | ) |
369 | ||
370 | def remove_tests(self, client): | |
371 | log.info('S3 Tests Java: Cleaning up s3-tests-java...') | |
372 | testdir = teuthology.get_testdir(self.ctx) | |
373 | ||
374 | if self.log_fwd: | |
375 | self.ctx.cluster.only(client).run( | |
376 | args=['cd', | |
377 | '{tdir}/s3-tests-java'.format(tdir=testdir), | |
378 | run.Raw('&&'), | |
379 | 'cat', self.log_name, | |
380 | run.Raw('&&'), | |
381 | 'rm', self.log_name], | |
e306af50 | 382 | stdout=BytesIO() |
9f95a23c TL |
383 | ) |
384 | ||
385 | self.ctx.cluster.only(client).run( | |
386 | args=[ | |
387 | 'rm', | |
388 | '-rf', | |
389 | '{tdir}/s3-tests-java'.format(tdir=testdir), | |
390 | ], | |
e306af50 | 391 | stdout=BytesIO() |
9f95a23c TL |
392 | ) |
393 | ||
394 | def delete_users(self, client): | |
395 | log.info("S3 Tests Java: Deleting S3 users...") | |
396 | testdir = teuthology.get_testdir(self.ctx) | |
397 | for section, user in self.users.items(): | |
398 | s3_user_id = '{user}.{client}'.format(user=user, client=client) | |
399 | self.ctx.cluster.only(client).run( | |
400 | args=[ | |
401 | 'adjust-ulimits', | |
402 | 'ceph-coverage', | |
403 | '{tdir}/archive/coverage'.format(tdir=testdir), | |
404 | 'radosgw-admin', | |
405 | '-n', client, | |
406 | 'user', 'rm', | |
407 | '--uid', s3_user_id, | |
408 | '--purge-data', | |
409 | '--cluster', 'ceph', | |
410 | ], | |
e306af50 | 411 | stdout=BytesIO() |
9f95a23c TL |
412 | ) |
413 | ||
414 | ||
415 | task = S3tests_java |