]>
Commit | Line | Data |
---|---|---|
7c673cae FG |
1 | """ |
2 | Run a set of s3 tests on rgw. | |
3 | """ | |
e306af50 | 4 | from io import BytesIO |
7c673cae FG |
5 | from configobj import ConfigObj |
6 | import base64 | |
7 | import contextlib | |
8 | import logging | |
9 | import os | |
10 | import random | |
e306af50 | 11 | import six |
7c673cae FG |
12 | import string |
13 | ||
7c673cae FG |
14 | from teuthology import misc as teuthology |
15 | from teuthology import contextutil | |
16 | from teuthology.config import config as teuth_config | |
17 | from teuthology.orchestra import run | |
9f95a23c | 18 | from teuthology.exceptions import ConfigError |
7c673cae FG |
19 | |
20 | log = logging.getLogger(__name__) | |
21 | ||
7c673cae FG |
22 | @contextlib.contextmanager |
23 | def download(ctx, config): | |
24 | """ | |
25 | Download the s3 tests from the git builder. | |
26 | Remove downloaded s3 file upon exit. | |
27 | ||
28 | The context passed in should be identical to the context | |
29 | passed in to the main task. | |
30 | """ | |
31 | assert isinstance(config, dict) | |
32 | log.info('Downloading s3-tests...') | |
33 | testdir = teuthology.get_testdir(ctx) | |
92f5a8d4 TL |
34 | for (client, client_config) in config.items(): |
35 | s3tests_branch = client_config.get('force-branch', None) | |
36 | if not s3tests_branch: | |
7c673cae | 37 | raise ValueError( |
92f5a8d4 TL |
38 | "Could not determine what branch to use for s3-tests. Please add 'force-branch: {s3-tests branch name}' to the .yaml config for this s3tests task.") |
39 | ||
40 | log.info("Using branch '%s' for s3tests", s3tests_branch) | |
41 | sha1 = client_config.get('sha1') | |
42 | git_remote = client_config.get('git_remote', teuth_config.ceph_git_base_url) | |
7c673cae FG |
43 | ctx.cluster.only(client).run( |
44 | args=[ | |
45 | 'git', 'clone', | |
92f5a8d4 | 46 | '-b', s3tests_branch, |
31f18b77 | 47 | git_remote + 's3-tests.git', |
7c673cae FG |
48 | '{tdir}/s3-tests'.format(tdir=testdir), |
49 | ], | |
50 | ) | |
51 | if sha1 is not None: | |
52 | ctx.cluster.only(client).run( | |
53 | args=[ | |
54 | 'cd', '{tdir}/s3-tests'.format(tdir=testdir), | |
55 | run.Raw('&&'), | |
56 | 'git', 'reset', '--hard', sha1, | |
57 | ], | |
58 | ) | |
59 | try: | |
60 | yield | |
61 | finally: | |
62 | log.info('Removing s3-tests...') | |
63 | testdir = teuthology.get_testdir(ctx) | |
64 | for client in config: | |
65 | ctx.cluster.only(client).run( | |
66 | args=[ | |
67 | 'rm', | |
68 | '-rf', | |
69 | '{tdir}/s3-tests'.format(tdir=testdir), | |
70 | ], | |
71 | ) | |
72 | ||
73 | ||
74 | def _config_user(s3tests_conf, section, user): | |
75 | """ | |
76 | Configure users for this section by stashing away keys, ids, and | |
77 | email addresses. | |
78 | """ | |
79 | s3tests_conf[section].setdefault('user_id', user) | |
80 | s3tests_conf[section].setdefault('email', '{user}+test@test.test'.format(user=user)) | |
81 | s3tests_conf[section].setdefault('display_name', 'Mr. {user}'.format(user=user)) | |
e306af50 TL |
82 | s3tests_conf[section].setdefault('access_key', |
83 | ''.join(random.choice(string.ascii_uppercase) for i in range(20))) | |
84 | s3tests_conf[section].setdefault('secret_key', | |
85 | six.ensure_str(base64.b64encode(os.urandom(40)))) | |
86 | s3tests_conf[section].setdefault('totp_serial', | |
87 | ''.join(random.choice(string.digits) for i in range(10))) | |
88 | s3tests_conf[section].setdefault('totp_seed', | |
89 | six.ensure_str(base64.b32encode(os.urandom(40)))) | |
11fdf7f2 | 90 | s3tests_conf[section].setdefault('totp_seconds', '5') |
7c673cae FG |
91 | |
92 | ||
93 | @contextlib.contextmanager | |
94 | def create_users(ctx, config): | |
95 | """ | |
96 | Create a main and an alternate s3 user. | |
97 | """ | |
98 | assert isinstance(config, dict) | |
99 | log.info('Creating rgw users...') | |
100 | testdir = teuthology.get_testdir(ctx) | |
31f18b77 | 101 | users = {'s3 main': 'foo', 's3 alt': 'bar', 's3 tenant': 'testx$tenanteduser'} |
7c673cae FG |
102 | for client in config['clients']: |
103 | s3tests_conf = config['s3tests_conf'][client] | |
104 | s3tests_conf.setdefault('fixtures', {}) | |
105 | s3tests_conf['fixtures'].setdefault('bucket prefix', 'test-' + client + '-{random}-') | |
9f95a23c | 106 | for section, user in users.items(): |
7c673cae FG |
107 | _config_user(s3tests_conf, section, '{user}.{client}'.format(user=user, client=client)) |
108 | log.debug('Creating user {user} on {host}'.format(user=s3tests_conf[section]['user_id'], host=client)) | |
109 | cluster_name, daemon_type, client_id = teuthology.split_role(client) | |
110 | client_with_id = daemon_type + '.' + client_id | |
111 | ctx.cluster.only(client).run( | |
112 | args=[ | |
113 | 'adjust-ulimits', | |
114 | 'ceph-coverage', | |
115 | '{tdir}/archive/coverage'.format(tdir=testdir), | |
116 | 'radosgw-admin', | |
117 | '-n', client_with_id, | |
118 | 'user', 'create', | |
119 | '--uid', s3tests_conf[section]['user_id'], | |
120 | '--display-name', s3tests_conf[section]['display_name'], | |
121 | '--access-key', s3tests_conf[section]['access_key'], | |
122 | '--secret', s3tests_conf[section]['secret_key'], | |
123 | '--email', s3tests_conf[section]['email'], | |
9f95a23c | 124 | '--caps', 'user-policy=*', |
7c673cae FG |
125 | '--cluster', cluster_name, |
126 | ], | |
127 | ) | |
11fdf7f2 TL |
128 | ctx.cluster.only(client).run( |
129 | args=[ | |
130 | 'adjust-ulimits', | |
131 | 'ceph-coverage', | |
132 | '{tdir}/archive/coverage'.format(tdir=testdir), | |
133 | 'radosgw-admin', | |
134 | '-n', client_with_id, | |
135 | 'mfa', 'create', | |
136 | '--uid', s3tests_conf[section]['user_id'], | |
137 | '--totp-serial', s3tests_conf[section]['totp_serial'], | |
138 | '--totp-seed', s3tests_conf[section]['totp_seed'], | |
139 | '--totp-seconds', s3tests_conf[section]['totp_seconds'], | |
140 | '--totp-window', '8', | |
141 | '--totp-seed-type', 'base32', | |
142 | '--cluster', cluster_name, | |
143 | ], | |
144 | ) | |
7c673cae FG |
145 | try: |
146 | yield | |
147 | finally: | |
148 | for client in config['clients']: | |
e306af50 | 149 | for user in users.values(): |
7c673cae FG |
150 | uid = '{user}.{client}'.format(user=user, client=client) |
151 | cluster_name, daemon_type, client_id = teuthology.split_role(client) | |
152 | client_with_id = daemon_type + '.' + client_id | |
153 | ctx.cluster.only(client).run( | |
154 | args=[ | |
155 | 'adjust-ulimits', | |
156 | 'ceph-coverage', | |
157 | '{tdir}/archive/coverage'.format(tdir=testdir), | |
158 | 'radosgw-admin', | |
159 | '-n', client_with_id, | |
160 | 'user', 'rm', | |
161 | '--uid', uid, | |
162 | '--purge-data', | |
163 | '--cluster', cluster_name, | |
164 | ], | |
165 | ) | |
166 | ||
167 | ||
168 | @contextlib.contextmanager | |
169 | def configure(ctx, config): | |
170 | """ | |
171 | Configure the s3-tests. This includes the running of the | |
172 | bootstrap code and the updating of local conf files. | |
173 | """ | |
174 | assert isinstance(config, dict) | |
175 | log.info('Configuring s3-tests...') | |
176 | testdir = teuthology.get_testdir(ctx) | |
9f95a23c TL |
177 | for client, properties in config['clients'].items(): |
178 | properties = properties or {} | |
7c673cae | 179 | s3tests_conf = config['s3tests_conf'][client] |
9f95a23c TL |
180 | s3tests_conf['DEFAULT']['calling_format'] = properties.get('calling-format', 'ordinary') |
181 | ||
182 | # use rgw_server if given, or default to local client | |
183 | role = properties.get('rgw_server', client) | |
184 | ||
185 | endpoint = ctx.rgw.role_endpoints.get(role) | |
186 | assert endpoint, 's3tests: no rgw endpoint for {}'.format(role) | |
187 | ||
188 | s3tests_conf['DEFAULT']['host'] = endpoint.dns_name | |
189 | ||
190 | website_role = properties.get('rgw_website_server') | |
191 | if website_role: | |
192 | website_endpoint = ctx.rgw.role_endpoints.get(website_role) | |
193 | assert website_endpoint, \ | |
194 | 's3tests: no rgw endpoint for rgw_website_server {}'.format(website_role) | |
195 | assert website_endpoint.website_dns_name, \ | |
196 | 's3tests: no dns-s3website-name for rgw_website_server {}'.format(website_role) | |
197 | s3tests_conf['DEFAULT']['s3website_domain'] = website_endpoint.website_dns_name | |
198 | ||
199 | if hasattr(ctx, 'barbican'): | |
200 | properties = properties['barbican'] | |
201 | if properties is not None and 'kms_key' in properties: | |
202 | if not (properties['kms_key'] in ctx.barbican.keys): | |
203 | raise ConfigError('Key '+properties['kms_key']+' not defined') | |
204 | ||
205 | if not (properties['kms_key2'] in ctx.barbican.keys): | |
206 | raise ConfigError('Key '+properties['kms_key2']+' not defined') | |
207 | ||
208 | key = ctx.barbican.keys[properties['kms_key']] | |
209 | s3tests_conf['DEFAULT']['kms_keyid'] = key['id'] | |
210 | ||
211 | key = ctx.barbican.keys[properties['kms_key2']] | |
212 | s3tests_conf['DEFAULT']['kms_keyid2'] = key['id'] | |
213 | ||
214 | elif hasattr(ctx, 'vault'): | |
215 | properties = properties['vault_%s' % ctx.vault.engine] | |
216 | s3tests_conf['DEFAULT']['kms_keyid'] = properties['key_path'] | |
217 | s3tests_conf['DEFAULT']['kms_keyid2'] = properties['key_path2'] | |
218 | ||
7c673cae | 219 | else: |
9f95a23c TL |
220 | # Fallback scenario where it's the local (ceph.conf) kms being tested |
221 | s3tests_conf['DEFAULT']['kms_keyid'] = 'testkey-1' | |
222 | s3tests_conf['DEFAULT']['kms_keyid2'] = 'testkey-2' | |
7c673cae | 223 | |
9f95a23c TL |
224 | slow_backend = properties.get('slow_backend') |
225 | if slow_backend: | |
226 | s3tests_conf['fixtures']['slow backend'] = slow_backend | |
7c673cae FG |
227 | |
228 | (remote,) = ctx.cluster.only(client).remotes.keys() | |
229 | remote.run( | |
230 | args=[ | |
231 | 'cd', | |
232 | '{tdir}/s3-tests'.format(tdir=testdir), | |
233 | run.Raw('&&'), | |
234 | './bootstrap', | |
235 | ], | |
236 | ) | |
e306af50 | 237 | conf_fp = BytesIO() |
7c673cae FG |
238 | s3tests_conf.write(conf_fp) |
239 | teuthology.write_file( | |
240 | remote=remote, | |
241 | path='{tdir}/archive/s3-tests.{client}.conf'.format(tdir=testdir, client=client), | |
242 | data=conf_fp.getvalue(), | |
243 | ) | |
244 | ||
245 | log.info('Configuring boto...') | |
246 | boto_src = os.path.join(os.path.dirname(__file__), 'boto.cfg.template') | |
9f95a23c TL |
247 | for client, properties in config['clients'].items(): |
248 | with open(boto_src, 'rb') as f: | |
7c673cae | 249 | (remote,) = ctx.cluster.only(client).remotes.keys() |
e306af50 | 250 | conf = six.ensure_str(f.read()).format( |
7c673cae FG |
251 | idle_timeout=config.get('idle_timeout', 30) |
252 | ) | |
253 | teuthology.write_file( | |
254 | remote=remote, | |
255 | path='{tdir}/boto.cfg'.format(tdir=testdir), | |
e306af50 | 256 | data=six.ensure_binary(conf), |
7c673cae FG |
257 | ) |
258 | ||
259 | try: | |
260 | yield | |
261 | ||
262 | finally: | |
263 | log.info('Cleaning up boto...') | |
9f95a23c | 264 | for client, properties in config['clients'].items(): |
7c673cae FG |
265 | (remote,) = ctx.cluster.only(client).remotes.keys() |
266 | remote.run( | |
267 | args=[ | |
268 | 'rm', | |
269 | '{tdir}/boto.cfg'.format(tdir=testdir), | |
270 | ], | |
271 | ) | |
272 | ||
7c673cae FG |
273 | @contextlib.contextmanager |
274 | def run_tests(ctx, config): | |
275 | """ | |
276 | Run the s3tests after everything is set up. | |
277 | ||
278 | :param ctx: Context passed to task | |
279 | :param config: specific configuration information | |
280 | """ | |
281 | assert isinstance(config, dict) | |
282 | testdir = teuthology.get_testdir(ctx) | |
9f95a23c TL |
283 | for client, client_config in config.items(): |
284 | client_config = client_config or {} | |
11fdf7f2 | 285 | (remote,) = ctx.cluster.only(client).remotes.keys() |
7c673cae FG |
286 | args = [ |
287 | 'S3TEST_CONF={tdir}/archive/s3-tests.{client}.conf'.format(tdir=testdir, client=client), | |
11fdf7f2 TL |
288 | 'BOTO_CONFIG={tdir}/boto.cfg'.format(tdir=testdir) |
289 | ] | |
290 | # the 'requests' library comes with its own ca bundle to verify ssl | |
291 | # certificates - override that to use the system's ca bundle, which | |
292 | # is where the ssl task installed this certificate | |
293 | if remote.os.package_type == 'deb': | |
294 | args += ['REQUESTS_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt'] | |
295 | else: | |
296 | args += ['REQUESTS_CA_BUNDLE=/etc/pki/tls/certs/ca-bundle.crt'] | |
9f95a23c TL |
297 | # civetweb > 1.8 && beast parsers are strict on rfc2616 |
298 | attrs = ["!fails_on_rgw", "!lifecycle_expiration", "!fails_strict_rfc2616"] | |
299 | if client_config.get('calling-format') != 'ordinary': | |
300 | attrs += ['!fails_with_subdomain'] | |
11fdf7f2 | 301 | args += [ |
9f95a23c TL |
302 | '{tdir}/s3-tests/virtualenv/bin/python'.format(tdir=testdir), |
303 | '-m', 'nose', | |
7c673cae FG |
304 | '-w', |
305 | '{tdir}/s3-tests'.format(tdir=testdir), | |
306 | '-v', | |
307 | '-a', ','.join(attrs), | |
308 | ] | |
9f95a23c TL |
309 | if 'extra_args' in client_config: |
310 | args.append(client_config['extra_args']) | |
7c673cae | 311 | |
11fdf7f2 | 312 | remote.run( |
7c673cae FG |
313 | args=args, |
314 | label="s3 tests against rgw" | |
315 | ) | |
316 | yield | |
317 | ||
318 | @contextlib.contextmanager | |
319 | def scan_for_leaked_encryption_keys(ctx, config): | |
320 | """ | |
321 | Scan radosgw logs for the encryption keys used by s3tests to | |
322 | verify that we're not leaking secrets. | |
323 | ||
324 | :param ctx: Context passed to task | |
325 | :param config: specific configuration information | |
326 | """ | |
327 | assert isinstance(config, dict) | |
328 | ||
329 | try: | |
330 | yield | |
331 | finally: | |
332 | # x-amz-server-side-encryption-customer-key | |
333 | s3test_customer_key = 'pO3upElrwuEXSoFwCfnZPdSsmt/xWeFa0N9KgDijwVs=' | |
334 | ||
335 | log.debug('Scanning radosgw logs for leaked encryption keys...') | |
336 | procs = list() | |
9f95a23c | 337 | for client, client_config in config.items(): |
7c673cae FG |
338 | if not client_config.get('scan_for_encryption_keys', True): |
339 | continue | |
340 | cluster_name, daemon_type, client_id = teuthology.split_role(client) | |
341 | client_with_cluster = '.'.join((cluster_name, daemon_type, client_id)) | |
342 | (remote,) = ctx.cluster.only(client).remotes.keys() | |
343 | proc = remote.run( | |
344 | args=[ | |
345 | 'grep', | |
346 | '--binary-files=text', | |
347 | s3test_customer_key, | |
348 | '/var/log/ceph/rgw.{client}.log'.format(client=client_with_cluster), | |
349 | ], | |
350 | wait=False, | |
351 | check_status=False, | |
352 | ) | |
353 | procs.append(proc) | |
354 | ||
355 | for proc in procs: | |
356 | proc.wait() | |
357 | if proc.returncode == 1: # 1 means no matches | |
358 | continue | |
359 | log.error('radosgw log is leaking encryption keys!') | |
360 | raise Exception('radosgw log is leaking encryption keys') | |
361 | ||
362 | @contextlib.contextmanager | |
363 | def task(ctx, config): | |
364 | """ | |
365 | Run the s3-tests suite against rgw. | |
366 | ||
367 | To run all tests on all clients:: | |
368 | ||
369 | tasks: | |
370 | - ceph: | |
371 | - rgw: | |
372 | - s3tests: | |
373 | ||
374 | To restrict testing to particular clients:: | |
375 | ||
376 | tasks: | |
377 | - ceph: | |
378 | - rgw: [client.0] | |
9f95a23c | 379 | - s3tests: [client.0] |
7c673cae FG |
380 | |
381 | To run against a server on client.1 and increase the boto timeout to 10m:: | |
382 | ||
383 | tasks: | |
384 | - ceph: | |
385 | - rgw: [client.1] | |
386 | - s3tests: | |
387 | client.0: | |
388 | rgw_server: client.1 | |
389 | idle_timeout: 600 | |
390 | ||
391 | To pass extra arguments to nose (e.g. to run a certain test):: | |
392 | ||
393 | tasks: | |
394 | - ceph: | |
395 | - rgw: [client.0] | |
396 | - s3tests: | |
397 | client.0: | |
398 | extra_args: ['test_s3:test_object_acl_grand_public_read'] | |
399 | client.1: | |
400 | extra_args: ['--exclude', 'test_100_continue'] | |
401 | """ | |
11fdf7f2 | 402 | assert hasattr(ctx, 'rgw'), 's3tests must run after the rgw task' |
7c673cae FG |
403 | assert config is None or isinstance(config, list) \ |
404 | or isinstance(config, dict), \ | |
405 | "task s3tests only supports a list or dictionary for configuration" | |
406 | all_clients = ['client.{id}'.format(id=id_) | |
407 | for id_ in teuthology.all_roles_of_type(ctx.cluster, 'client')] | |
408 | if config is None: | |
409 | config = all_clients | |
410 | if isinstance(config, list): | |
411 | config = dict.fromkeys(config) | |
412 | clients = config.keys() | |
413 | ||
414 | overrides = ctx.config.get('overrides', {}) | |
415 | # merge each client section, not the top level. | |
9f95a23c | 416 | for client in config.keys(): |
7c673cae FG |
417 | if not config[client]: |
418 | config[client] = {} | |
419 | teuthology.deep_merge(config[client], overrides.get('s3tests', {})) | |
420 | ||
421 | log.debug('s3tests config is %s', config) | |
422 | ||
423 | s3tests_conf = {} | |
424 | for client in clients: | |
11fdf7f2 TL |
425 | endpoint = ctx.rgw.role_endpoints.get(client) |
426 | assert endpoint, 's3tests: no rgw endpoint for {}'.format(client) | |
427 | ||
7c673cae FG |
428 | s3tests_conf[client] = ConfigObj( |
429 | indent_type='', | |
430 | infile={ | |
431 | 'DEFAULT': | |
432 | { | |
11fdf7f2 | 433 | 'port' : endpoint.port, |
494da23a | 434 | 'is_secure' : endpoint.cert is not None, |
11fdf7f2 | 435 | 'api_name' : 'default', |
7c673cae FG |
436 | }, |
437 | 'fixtures' : {}, | |
438 | 's3 main' : {}, | |
439 | 's3 alt' : {}, | |
31f18b77 | 440 | 's3 tenant': {}, |
7c673cae FG |
441 | } |
442 | ) | |
443 | ||
7c673cae FG |
444 | with contextutil.nested( |
445 | lambda: download(ctx=ctx, config=config), | |
446 | lambda: create_users(ctx=ctx, config=dict( | |
447 | clients=clients, | |
448 | s3tests_conf=s3tests_conf, | |
449 | )), | |
7c673cae FG |
450 | lambda: configure(ctx=ctx, config=dict( |
451 | clients=config, | |
452 | s3tests_conf=s3tests_conf, | |
453 | )), | |
454 | lambda: run_tests(ctx=ctx, config=config), | |
455 | lambda: scan_for_leaked_encryption_keys(ctx=ctx, config=config), | |
456 | ): | |
457 | pass | |
458 | yield |