]>
git.proxmox.com Git - ceph.git/blob - ceph/qa/tasks/s3tests_java.py
2 Task for running RGW S3 tests with the AWS Java SDK
14 from teuthology
import misc
as teuthology
15 from teuthology
.task
import Task
16 from teuthology
.orchestra
import run
18 log
= logging
.getLogger(__name__
)
21 Task for running RGW S3 tests with the AWS Java SDK
23 Tests run only on clients specified in the s3tests-java config section.
24 If no client is given a default 'client.0' is chosen.
25 If such does not match the rgw client the task will fail.
33 Extra arguments can be passed by adding options to the corresponding client
34 section under the s3tests-java task (e.g. to run a certain test,
35 specify a different repository and branch for the test suite,
36 run in info/debug mode (for the java suite) or forward the gradle output to a log file):
44 force-repo: 'https://github.com/adamyanova/java_s3tests.git'
45 log-fwd: '../s3tests-java.log'
47 extra-args: ['--tests', 'ObjectTest.testEncryptionKeySSECInvalidMd5']
49 To run a specific test, provide its name to the extra-args section e.g.:
52 extra-args: ['--tests', 'ObjectTest.testEncryptionKeySSECInvalidMd5']
57 class S3tests_java(Task
):
59 Download and install S3 tests in Java
60 This will require openjdk and gradle
63 def __init__(self
, ctx
, config
):
64 super(S3tests_java
, self
).__init
__(ctx
, config
)
66 log
.debug('S3 Tests Java: __INIT__ ')
67 assert hasattr(ctx
, 'rgw'), 'S3tests_java must run after the rgw task'
68 clients
= ['client.{id}'.format(id=id_
)
69 for id_
in teuthology
.all_roles_of_type(self
.ctx
.cluster
, 'client')]
71 for client
in clients
:
72 if client
in self
.config
:
73 self
.all_clients
.extend([client
])
74 if self
.all_clients
is None:
75 self
.all_clients
= 'client.0'
76 self
.users
= {'s3main': 'tester',
77 's3alt': 'johndoe', 'tenanted': 'testx$tenanteduser'}
80 super(S3tests_java
, self
).setup()
81 log
.debug('S3 Tests Java: SETUP')
82 for client
in self
.all_clients
:
83 self
.download_test_suite(client
)
84 self
.install_required_packages(client
)
87 super(S3tests_java
, self
).begin()
88 log
.debug('S3 Tests Java: BEGIN')
89 for (host
, roles
) in self
.ctx
.cluster
.remotes
.items():
91 'S3 Tests Java: Cluster config is: {cfg}'.format(cfg
=roles
))
92 log
.debug('S3 Tests Java: Host is: {host}'.format(host
=host
))
97 super(S3tests_java
, self
).end()
98 log
.debug('S3 Tests Java: END')
99 for client
in self
.all_clients
:
100 self
.remove_tests(client
)
101 self
.delete_users(client
)
103 def download_test_suite(self
, client
):
104 log
.info("S3 Tests Java: Downloading test suite...")
105 testdir
= teuthology
.get_testdir(self
.ctx
)
107 repo
= 'https://github.com/ceph/java_s3tests.git'
108 if client
in self
.config
and self
.config
[client
] is not None:
109 if 'force-branch' in self
.config
[client
] and self
.config
[client
]['force-branch'] is not None:
110 branch
= self
.config
[client
]['force-branch']
111 if 'force-repo' in self
.config
[client
] and self
.config
[client
]['force-repo'] is not None:
112 repo
= self
.config
[client
]['force-repo']
113 self
.ctx
.cluster
.only(client
).run(
118 '{tdir}/s3-tests-java'.format(tdir
=testdir
),
122 if client
in self
.config
and self
.config
[client
] is not None:
123 if 'sha1' in self
.config
[client
] and self
.config
[client
]['sha1'] is not None:
124 self
.ctx
.cluster
.only(client
).run(
126 'cd', '{tdir}/s3-tests-java'.format(tdir
=testdir
),
128 'git', 'reset', '--hard', self
.config
[client
]['sha1'],
132 if 'log-level' in self
.config
[client
]:
133 if self
.config
[client
]['log-level'] == 'info':
134 self
.ctx
.cluster
.only(client
).run(
136 'sed', '-i', '\'s/log4j.rootLogger=WARN/log4j.rootLogger=INFO/g\'',
137 '{tdir}/s3-tests-java/src/main/resources/log4j.properties'.format(
141 if self
.config
[client
]['log-level'] == 'debug':
142 self
.ctx
.cluster
.only(client
).run(
144 'sed', '-i', '\'s/log4j.rootLogger=WARN/log4j.rootLogger=DEBUG/g\'',
145 '{tdir}/s3-tests-java/src/main/resources/log4j.properties'.format(
150 def install_required_packages(self
, client
):
152 Run bootstrap script to install openjdk and gradle.
153 Add certificates to java keystore
155 log
.info("S3 Tests Java: Installing required packages...")
156 testdir
= teuthology
.get_testdir(self
.ctx
)
157 self
.ctx
.cluster
.only(client
).run(
158 args
=['{tdir}/s3-tests-java/bootstrap.sh'.format(tdir
=testdir
)],
162 endpoint
= self
.ctx
.rgw
.role_endpoints
[client
]
164 path
= 'lib/security/cacerts'
165 self
.ctx
.cluster
.only(client
).run(
168 '-import', '-alias', '{alias}'.format(
169 alias
=endpoint
.hostname
),
172 '$(readlink -e $(dirname $(readlink -e $(which keytool)))/../{path})'.format(path
=path
)),
173 '-file', endpoint
.cert
.certificate
,
174 '-storepass', 'changeit',
179 def create_users(self
):
181 Create a main and an alternative s3 user.
182 Configuration is read from a skelethon config file
183 s3tests.teuth.config.yaml in the java-s3tests repository
184 and missing information is added from the task.
185 Existing values are NOT overriden unless they are empty!
187 log
.info("S3 Tests Java: Creating S3 users...")
188 testdir
= teuthology
.get_testdir(self
.ctx
)
189 for client
in self
.all_clients
:
190 endpoint
= self
.ctx
.rgw
.role_endpoints
.get(client
)
191 local_user
= getpass
.getuser()
192 remote_user
= teuthology
.get_test_user()
193 os
.system("scp {remote}@{host}:{tdir}/s3-tests-java/s3tests.teuth.config.yaml /home/{local}/".format(
194 host
=endpoint
.hostname
, tdir
=testdir
, remote
=remote_user
, local
=local_user
))
195 s3tests_conf
= teuthology
.config_file(
196 '/home/{local}/s3tests.teuth.config.yaml'.format(local
=local_user
))
197 log
.debug("S3 Tests Java: s3tests_conf is {s3cfg}".format(
199 for section
, user
in list(self
.users
.items()):
200 if section
in s3tests_conf
:
201 s3_user_id
= '{user}.{client}'.format(
202 user
=user
, client
=client
)
204 'S3 Tests Java: Creating user {s3_user_id}'.format(s3_user_id
=s3_user_id
))
205 self
._config
_user
(s3tests_conf
=s3tests_conf
,
206 section
=section
, user
=s3_user_id
, client
=client
)
207 cluster_name
, daemon_type
, client_id
= teuthology
.split_role(
209 client_with_id
= daemon_type
+ '.' + client_id
213 '{tdir}/archive/coverage'.format(tdir
=testdir
),
215 '-n', client_with_id
,
217 '--uid', s3tests_conf
[section
]['user_id'],
218 '--display-name', s3tests_conf
[section
]['display_name'],
219 '--access-key', s3tests_conf
[section
]['access_key'],
220 '--secret', s3tests_conf
[section
]['access_secret'],
221 '--email', s3tests_conf
[section
]['email'],
222 '--cluster', cluster_name
,
224 log
.info('{args}'.format(args
=args
))
225 self
.ctx
.cluster
.only(client
).run(
230 self
.users
.pop(section
)
231 self
._write
_cfg
_file
(s3tests_conf
, client
)
233 "rm -rf /home/{local}/s3tests.teuth.config.yaml".format(local
=local_user
))
235 def _config_user(self
, s3tests_conf
, section
, user
, client
):
237 Generate missing users data for this section by stashing away keys, ids, and
240 access_key
= ''.join(random
.choice(string
.ascii_uppercase
)
242 access_secret
= base64
.b64encode(os
.urandom(40)).decode('ascii')
243 endpoint
= self
.ctx
.rgw
.role_endpoints
.get(client
)
246 s3tests_conf
[section
], 'user_id', '{user}'.format(user
=user
))
248 s3tests_conf
[section
], 'email', '{user}_test@test.test'.format(user
=user
))
250 s3tests_conf
[section
], 'display_name', 'Ms. {user}'.format(user
=user
))
252 s3tests_conf
[section
], 'access_key', '{ak}'.format(ak
=access_key
))
254 s3tests_conf
[section
], 'access_secret', '{asc}'.format(asc
=access_secret
))
256 s3tests_conf
[section
], 'region', 'us-east-1')
258 s3tests_conf
[section
], 'endpoint', '{ip}:{port}'.format(
259 ip
=endpoint
.hostname
, port
=endpoint
.port
))
261 s3tests_conf
[section
], 'host', endpoint
.hostname
)
263 s3tests_conf
[section
], 'port', endpoint
.port
)
265 s3tests_conf
[section
], 'is_secure', True if endpoint
.cert
else False)
267 log
.debug("S3 Tests Java: s3tests_conf[{sect}] is {s3cfg}".format(
268 sect
=section
, s3cfg
=s3tests_conf
[section
]))
269 log
.debug('S3 Tests Java: Setion, User = {sect}, {user}'.format(
270 sect
=section
, user
=user
))
272 def _write_cfg_file(self
, cfg_dict
, client
):
274 Write s3 tests java config file on the remote node.
276 testdir
= teuthology
.get_testdir(self
.ctx
)
277 (remote
,) = self
.ctx
.cluster
.only(client
).remotes
.keys()
278 data
= yaml
.safe_dump(cfg_dict
, default_flow_style
=False)
279 path
= testdir
+ '/archive/s3-tests-java.' + client
+ '.conf'
280 remote
.write_file(path
, data
)
282 def _set_cfg_entry(self
, cfg_dict
, key
, value
):
283 if not (key
in cfg_dict
):
284 cfg_dict
.setdefault(key
, value
)
285 elif cfg_dict
[key
] is None:
286 cfg_dict
[key
] = value
289 log
.info("S3 Tests Java: Running tests...")
290 testdir
= teuthology
.get_testdir(self
.ctx
)
291 for client
in self
.all_clients
:
292 self
.ctx
.cluster
.only(client
).run(
294 '{tdir}/archive/s3-tests-java.{client}.conf'.format(
295 tdir
=testdir
, client
=client
),
296 '{tdir}/s3-tests-java/config.properties'.format(
302 '{tdir}/s3-tests-java'.format(tdir
=testdir
),
304 '/opt/gradle/gradle/bin/gradle', 'clean', 'test',
305 '--rerun-tasks', '--no-build-cache',
308 suppress_groups
= False
311 if client
in self
.config
and self
.config
[client
] is not None:
312 if 'extra-args' in self
.config
[client
]:
313 extra_args
.extend(self
.config
[client
]['extra-args'])
314 suppress_groups
= True
315 if 'log-level' in self
.config
[client
] and self
.config
[client
]['log-level'] == 'debug':
316 extra_args
+= ['--debug']
317 if 'log-fwd' in self
.config
[client
]:
319 self
.log_name
= '{tdir}/s3tests_log.txt'.format(
321 if self
.config
[client
]['log-fwd'] is not None:
322 self
.log_name
= self
.config
[client
]['log-fwd']
323 extra_args
+= [run
.Raw('>>'),
326 if not suppress_groups
:
327 test_groups
= ['AWS4Test', 'BucketTest', 'ObjectTest']
329 test_groups
= ['All']
331 for gr
in test_groups
:
333 self
.ctx
.cluster
.only(client
).run(
334 args
=['radosgw-admin', 'gc',
335 'process', '--include-all'],
340 self
.ctx
.cluster
.only(client
).run(
341 args
=args
+ ['--tests'] + [gr
] + extra_args
,
345 self
.ctx
.cluster
.only(client
).run(
346 args
=args
+ extra_args
,
351 self
.ctx
.cluster
.only(client
).run(
352 args
=['radosgw-admin', 'gc',
353 'process', '--include-all'],
357 def remove_tests(self
, client
):
358 log
.info('S3 Tests Java: Cleaning up s3-tests-java...')
359 testdir
= teuthology
.get_testdir(self
.ctx
)
362 self
.ctx
.cluster
.only(client
).run(
364 '{tdir}/s3-tests-java'.format(tdir
=testdir
),
366 'cat', self
.log_name
,
368 'rm', self
.log_name
],
372 self
.ctx
.cluster
.only(client
).run(
376 '{tdir}/s3-tests-java'.format(tdir
=testdir
),
381 def delete_users(self
, client
):
382 log
.info("S3 Tests Java: Deleting S3 users...")
383 testdir
= teuthology
.get_testdir(self
.ctx
)
384 for section
, user
in self
.users
.items():
385 s3_user_id
= '{user}.{client}'.format(user
=user
, client
=client
)
386 self
.ctx
.cluster
.only(client
).run(
390 '{tdir}/archive/coverage'.format(tdir
=testdir
),