]>
git.proxmox.com Git - ceph.git/blob - ceph/qa/tasks/radosgw_admin.py
2 Rgw admin testing against a running instance
4 # The test cases in this file have been annotated for inventory.
5 # To extract the inventory (in csv format) use the command:
7 # grep '^ *# TESTCASE' | sed 's/^ *# TESTCASE //'
9 # to run this standalone:
10 # python qa/tasks/radosgw_admin.py [--user=uid] --host=host --port=port
19 from io
import StringIO
20 from queue
import Queue
23 import boto
.s3
.connection
30 import tasks
.vstart_runner
31 from tasks
.rgw
import RGWEndpoint
32 from tasks
.util
.rgw
import rgwadmin
as tasks_util_rgw_rgwadmin
33 from tasks
.util
.rgw
import get_user_summary
, get_user_successful_ops
35 log
= logging
.getLogger(__name__
)
37 def rgwadmin(*args
, **kwargs
):
39 # Is this a local runner?
40 omit_sudo
= hasattr(ctx
.rgw
, 'omit_sudo') and ctx
.rgw
.omit_sudo
== True
41 omit_tdir
= hasattr(ctx
.rgw
, 'omit_tdir') and ctx
.rgw
.omit_tdir
== True
42 return tasks_util_rgw_rgwadmin(*args
, **kwargs
, omit_sudo
=omit_sudo
, omit_tdir
=omit_tdir
)
44 def usage_acc_findentry2(entries
, user
, add
=True):
50 e
= {'user': user
, 'buckets': []}
53 def usage_acc_findsum2(summaries
, user
, add
=True):
59 e
= {'user': user
, 'categories': [],
60 'total': {'bytes_received': 0,
61 'bytes_sent': 0, 'ops': 0, 'successful_ops': 0 }}
64 def usage_acc_update2(x
, out
, b_in
, err
):
65 x
['bytes_sent'] += b_in
66 x
['bytes_received'] += out
69 x
['successful_ops'] += 1
70 def usage_acc_validate_fields(r
, x
, x2
, what
):
72 for field
in ['bytes_sent', 'bytes_received', 'ops', 'successful_ops']:
74 if x2
[field
] < x
[field
]:
75 q
.append("field %s: %d < %d" % (field
, x2
[field
], x
[field
]))
76 except Exception as ex
:
77 r
.append( "missing/bad field " + field
+ " in " + what
+ " " + str(ex
))
80 r
.append("incomplete counts in " + what
+ ": " + ", ".join(q
))
83 self
.results
= {'entries': [], 'summary': []}
84 def findentry(self
, user
):
85 return usage_acc_findentry2(self
.results
['entries'], user
)
86 def findsum(self
, user
):
87 return usage_acc_findsum2(self
.results
['summary'], user
)
88 def e2b(self
, e
, bucket
, add
=True):
89 for b
in e
['buckets']:
90 if b
['bucket'] == bucket
:
94 b
= {'bucket': bucket
, 'categories': []}
95 e
['buckets'].append(b
)
97 def c2x(self
, c
, cat
, add
=True):
99 if x
['category'] == cat
:
103 x
= {'bytes_received': 0, 'category': cat
,
104 'bytes_sent': 0, 'ops': 0, 'successful_ops': 0 }
107 def update(self
, c
, cat
, user
, out
, b_in
, err
):
109 usage_acc_update2(x
, out
, b_in
, err
)
110 if not err
and cat
== 'create_bucket' and 'owner' not in x
:
112 def make_entry(self
, cat
, bucket
, user
, out
, b_in
, err
):
113 if cat
== 'create_bucket' and err
:
115 e
= self
.findentry(user
)
116 b
= self
.e2b(e
, bucket
)
117 self
.update(b
['categories'], cat
, user
, out
, b_in
, err
)
118 s
= self
.findsum(user
)
119 x
= self
.c2x(s
['categories'], cat
)
120 usage_acc_update2(x
, out
, b_in
, err
)
122 usage_acc_update2(x
, out
, b_in
, err
)
123 def generate_make_entry(self
):
124 return lambda cat
,bucket
,user
,out
,b_in
,err
: self
.make_entry(cat
, bucket
, user
, out
, b_in
, err
)
127 def compare_results(self
, results
):
128 if 'entries' not in results
or 'summary' not in results
:
129 return ['Missing entries or summary']
131 for e
in self
.results
['entries']:
133 e2
= usage_acc_findentry2(results
['entries'], e
['user'], False)
134 except Exception as ex
:
135 r
.append("malformed entry looking for user "
136 + e
['user'] + " " + str(ex
))
139 r
.append("missing entry for user " + e
['user'])
141 for b
in e
['buckets']:
143 if b
['bucket'] == 'nosuchbucket':
146 b2
= self
.e2b(e2
, b
['bucket'], False)
148 c2
= b2
['categories']
149 except Exception as ex
:
150 r
.append("malformed entry looking for bucket "
151 + b
['bucket'] + " in user " + e
['user'] + " " + str(ex
))
154 r
.append("can't find bucket " + b
['bucket']
155 + " in user " + e
['user'])
159 x2
= self
.c2x(c2
, x
['category'], False)
160 except Exception as ex
:
161 r
.append("malformed entry looking for "
162 + x
['category'] + " in bucket " + b
['bucket']
163 + " user " + e
['user'] + " " + str(ex
))
165 usage_acc_validate_fields(r
, x
, x2
, "entry: category "
166 + x
['category'] + " bucket " + b
['bucket']
167 + " in user " + e
['user'])
168 for s
in self
.results
['summary']:
171 s2
= usage_acc_findsum2(results
['summary'], s
['user'], False)
172 except Exception as ex
:
173 r
.append("malformed summary looking for user " + e
['user']
177 r
.append("missing summary for user " + e
['user'] + " " + str(ex
))
180 c2
= s2
['categories']
181 except Exception as ex
:
182 r
.append("malformed summary missing categories for user "
183 + e
['user'] + " " + str(ex
))
187 x2
= self
.c2x(c2
, x
['category'], False)
188 except Exception as ex
:
189 r
.append("malformed summary looking for "
190 + x
['category'] + " user " + e
['user'] + " " + str(ex
))
192 usage_acc_validate_fields(r
, x
, x2
, "summary: category "
193 + x
['category'] + " in user " + e
['user'])
197 except Exception as ex
:
198 r
.append("malformed summary looking for totals for user "
199 + e
['user'] + " " + str(ex
))
201 usage_acc_validate_fields(r
, x
, x2
, "summary: totals for user" + e
['user'])
204 def ignore_this_entry(cat
, bucket
, user
, out
, b_in
, err
):
206 class requestlog_queue():
207 def __init__(self
, add
):
210 def handle_request_data(self
, request
, response
, error
=False):
211 now
= datetime
.datetime
.now()
214 elif response
.status
< 200 or response
.status
>= 400:
216 self
.q
.put({'t': now
, 'o': request
, 'i': response
, 'e': error
})
220 def log_and_clear(self
, cat
, bucket
, user
, add_entry
= None):
221 while not self
.q
.empty():
224 if 'Content-Length' in j
['o'].headers
:
225 bytes_out
= int(j
['o'].headers
['Content-Length'])
228 if 'content-length'in msg
:
229 bytes_in
= int(msg
['content-length'])
230 log
.info('RL: %s %s %s bytes_out=%d bytes_in=%d failed=%r'
231 % (cat
, bucket
, user
, bytes_out
, bytes_in
, j
['e']))
232 if add_entry
== None:
233 add_entry
= self
.adder
234 add_entry(cat
, bucket
, user
, bytes_out
, bytes_in
, j
['e'])
236 def create_presigned_url(conn
, method
, bucket_name
, key_name
, expiration
):
237 return conn
.generate_url(expires_in
=expiration
,
244 def send_raw_http_request(conn
, method
, bucket_name
, key_name
, follow_redirects
= False):
245 url
= create_presigned_url(conn
, method
, bucket_name
, key_name
, 3600)
248 h
.follow_redirects
= follow_redirects
249 return h
.request(url
, method
)
254 Helper function to get the xml acl from a key, ensuring that the xml
255 version tag is removed from the acl response
257 raw_acl
= key
.get_xml_acl().decode()
259 def remove_version(string
):
261 '<?xml version="1.0" encoding="UTF-8"?>'
264 def remove_newlines(string
):
265 return string
.strip('\n')
267 return remove_version(
268 remove_newlines(raw_acl
)
271 def cleanup(ctx
, client
):
272 # remove objects and buckets
273 (err
, out
) = rgwadmin(ctx
, client
, ['bucket', 'list'], check_status
=True)
276 (err
, out
) = rgwadmin(ctx
, client
, [
277 'bucket', 'rm', '--bucket', bucket
, '--purge-objects'],
282 # remove test user(s)
283 users
= ['foo', 'fud', 'bar', 'bud']
287 (err
, out
) = rgwadmin(ctx
, client
, [
288 'user', 'rm', '--uid', user
],
293 # remove custom placement
295 zonecmd
= ['zone', 'placement', 'rm', '--rgw-zone', 'default',
296 '--placement-id', 'new-placement']
297 (err
, out
) = rgwadmin(ctx
, client
, zonecmd
, check_status
=True)
301 def task(ctx
, config
):
303 Test radosgw-admin functionality against a running rgw instance.
307 assert ctx
.rgw
.config
, \
308 "radosgw_admin task needs a config passed from the rgw task"
309 config
= ctx
.rgw
.config
310 log
.debug('config is: %r', config
)
312 clients_from_config
= config
.keys()
314 # choose first client as default
315 client
= next(iter(clients_from_config
))
317 # once the client is chosen, pull the host name and assigned port out of
318 # the role_endpoints that were assigned by the rgw task
319 endpoint
= ctx
.rgw
.role_endpoints
[client
]
334 access_key
='9te6NH5mcdcq0Tc5i8i1'
335 secret_key
='Ny4IOauQoL18Gp2zM7lC1vLmoawgqcYP/YGcWfXu'
336 access_key2
='p5YnriCv1nAtykxBrupQ'
337 secret_key2
='Q8Tk6Q/27hfbFSYdSkPtUqhqx1GgzvpXa4WARozh'
338 access_key3
='NX5QOQKC6BH2IDN8HC7A'
339 secret_key3
='LnEsqNNqZIpkzauboDcLXLcYaWwLQ3Kop0zAnKIn'
340 swift_secret1
='gpS2G9RREMrnbqlp29PP2D36kgPR1tm72n5fPYfL'
341 swift_secret2
='ri2VJQcKSYATOY6uaDUX7pxgkW+W1YmC6OCxPHwy'
347 connection
= boto
.s3
.connection
.S3Connection(
348 aws_access_key_id
=access_key
,
349 aws_secret_access_key
=secret_key
,
352 host
=endpoint
.hostname
,
353 calling_format
=boto
.s3
.connection
.OrdinaryCallingFormat(),
355 connection
.auth_region_name
='us-east-1'
357 connection2
= boto
.s3
.connection
.S3Connection(
358 aws_access_key_id
=access_key2
,
359 aws_secret_access_key
=secret_key2
,
362 host
=endpoint
.hostname
,
363 calling_format
=boto
.s3
.connection
.OrdinaryCallingFormat(),
365 connection2
.auth_region_name
='us-east-1'
367 connection3
= boto
.s3
.connection
.S3Connection(
368 aws_access_key_id
=access_key3
,
369 aws_secret_access_key
=secret_key3
,
372 host
=endpoint
.hostname
,
373 calling_format
=boto
.s3
.connection
.OrdinaryCallingFormat(),
375 connection3
.auth_region_name
='us-east-1'
378 rl
= requestlog_queue(acc
.generate_make_entry())
379 connection
.set_request_hook(rl
)
380 connection2
.set_request_hook(rl
)
381 connection3
.set_request_hook(rl
)
383 # legend (test cases can be easily grep-ed out)
384 # TESTCASE 'testname','object','method','operation','assertion'
386 # TESTCASE 'usage-show0' 'usage' 'show' 'all usage' 'succeeds'
387 (err
, summary0
) = rgwadmin(ctx
, client
, ['usage', 'show'], check_status
=True)
389 # TESTCASE 'info-nosuch','user','info','non-existent user','fails'
390 (err
, out
) = rgwadmin(ctx
, client
, ['user', 'info', '--uid', user1
])
393 # TESTCASE 'create-ok','user','create','w/all valid info','succeeds'
394 (err
, out
) = rgwadmin(ctx
, client
, [
397 '--display-name', display_name1
,
399 '--access-key', access_key
,
400 '--secret', secret_key
,
405 # TESTCASE 'duplicate email','user','create','existing user email','fails'
406 (err
, out
) = rgwadmin(ctx
, client
, [
409 '--display-name', display_name2
,
414 # TESTCASE 'info-existing','user','info','existing user','returns correct info'
415 (err
, out
) = rgwadmin(ctx
, client
, ['user', 'info', '--uid', user1
], check_status
=True)
416 assert out
['user_id'] == user1
417 assert out
['email'] == email
418 assert out
['display_name'] == display_name1
419 assert len(out
['keys']) == 1
420 assert out
['keys'][0]['access_key'] == access_key
421 assert out
['keys'][0]['secret_key'] == secret_key
422 assert not out
['suspended']
424 # TESTCASE 'suspend-ok','user','suspend','active user','succeeds'
425 (err
, out
) = rgwadmin(ctx
, client
, ['user', 'suspend', '--uid', user1
],
428 # TESTCASE 'suspend-suspended','user','suspend','suspended user','succeeds w/advisory'
429 (err
, out
) = rgwadmin(ctx
, client
, ['user', 'info', '--uid', user1
], check_status
=True)
430 assert out
['suspended']
432 # TESTCASE 're-enable','user','enable','suspended user','succeeds'
433 (err
, out
) = rgwadmin(ctx
, client
, ['user', 'enable', '--uid', user1
], check_status
=True)
435 # TESTCASE 'info-re-enabled','user','info','re-enabled user','no longer suspended'
436 (err
, out
) = rgwadmin(ctx
, client
, ['user', 'info', '--uid', user1
], check_status
=True)
437 assert not out
['suspended']
439 # TESTCASE 'add-keys','key','create','w/valid info','succeeds'
440 (err
, out
) = rgwadmin(ctx
, client
, [
441 'key', 'create', '--uid', user1
,
442 '--access-key', access_key2
, '--secret', secret_key2
,
443 ], check_status
=True)
445 # TESTCASE 'info-new-key','user','info','after key addition','returns all keys'
446 (err
, out
) = rgwadmin(ctx
, client
, ['user', 'info', '--uid', user1
],
448 assert len(out
['keys']) == 2
449 assert out
['keys'][0]['access_key'] == access_key2
or out
['keys'][1]['access_key'] == access_key2
450 assert out
['keys'][0]['secret_key'] == secret_key2
or out
['keys'][1]['secret_key'] == secret_key2
452 # TESTCASE 'rm-key','key','rm','newly added key','succeeds, key is removed'
453 (err
, out
) = rgwadmin(ctx
, client
, [
454 'key', 'rm', '--uid', user1
,
455 '--access-key', access_key2
,
456 ], check_status
=True)
457 assert len(out
['keys']) == 1
458 assert out
['keys'][0]['access_key'] == access_key
459 assert out
['keys'][0]['secret_key'] == secret_key
461 # TESTCASE 'add-swift-key','key','create','swift key','succeeds'
462 subuser_access
= 'full'
463 subuser_perm
= 'full-control'
465 (err
, out
) = rgwadmin(ctx
, client
, [
466 'subuser', 'create', '--subuser', subuser1
,
467 '--access', subuser_access
468 ], check_status
=True)
470 # TESTCASE 'add-swift-key','key','create','swift key','succeeds'
471 (err
, out
) = rgwadmin(ctx
, client
, [
472 'subuser', 'modify', '--subuser', subuser1
,
473 '--secret', swift_secret1
,
474 '--key-type', 'swift',
475 ], check_status
=True)
477 # TESTCASE 'subuser-perm-mask', 'subuser', 'info', 'test subuser perm mask durability', 'succeeds'
478 (err
, out
) = rgwadmin(ctx
, client
, ['user', 'info', '--uid', user1
])
480 assert out
['subusers'][0]['permissions'] == subuser_perm
482 # TESTCASE 'info-swift-key','user','info','after key addition','returns all keys'
483 (err
, out
) = rgwadmin(ctx
, client
, ['user', 'info', '--uid', user1
], check_status
=True)
484 assert len(out
['swift_keys']) == 1
485 assert out
['swift_keys'][0]['user'] == subuser1
486 assert out
['swift_keys'][0]['secret_key'] == swift_secret1
488 # TESTCASE 'add-swift-subuser','key','create','swift sub-user key','succeeds'
489 (err
, out
) = rgwadmin(ctx
, client
, [
490 'subuser', 'create', '--subuser', subuser2
,
491 '--secret', swift_secret2
,
492 '--key-type', 'swift',
493 ], check_status
=True)
495 # TESTCASE 'info-swift-subuser','user','info','after key addition','returns all sub-users/keys'
496 (err
, out
) = rgwadmin(ctx
, client
, ['user', 'info', '--uid', user1
], check_status
=True)
497 assert len(out
['swift_keys']) == 2
498 assert out
['swift_keys'][0]['user'] == subuser2
or out
['swift_keys'][1]['user'] == subuser2
499 assert out
['swift_keys'][0]['secret_key'] == swift_secret2
or out
['swift_keys'][1]['secret_key'] == swift_secret2
501 # TESTCASE 'rm-swift-key1','key','rm','subuser','succeeds, one key is removed'
502 (err
, out
) = rgwadmin(ctx
, client
, [
503 'key', 'rm', '--subuser', subuser1
,
504 '--key-type', 'swift',
505 ], check_status
=True)
506 assert len(out
['swift_keys']) == 1
508 # TESTCASE 'rm-subuser','subuser','rm','subuser','success, subuser is removed'
509 (err
, out
) = rgwadmin(ctx
, client
, [
510 'subuser', 'rm', '--subuser', subuser1
,
511 ], check_status
=True)
512 assert len(out
['subusers']) == 1
514 # TESTCASE 'rm-subuser-with-keys','subuser','rm','subuser','succeeds, second subser and key is removed'
515 (err
, out
) = rgwadmin(ctx
, client
, [
516 'subuser', 'rm', '--subuser', subuser2
,
517 '--key-type', 'swift', '--purge-keys',
518 ], check_status
=True)
519 assert len(out
['swift_keys']) == 0
520 assert len(out
['subusers']) == 0
522 # TESTCASE 'bucket-stats','bucket','stats','no session/buckets','succeeds, empty list'
523 (err
, out
) = rgwadmin(ctx
, client
, ['bucket', 'stats', '--uid', user1
],
527 # TESTCASE 'bucket-stats2','bucket','stats','no buckets','succeeds, empty list'
528 (err
, out
) = rgwadmin(ctx
, client
, ['bucket', 'list', '--uid', user1
], check_status
=True)
531 # create a first bucket
532 bucket
= connection
.create_bucket(bucket_name
)
534 rl
.log_and_clear("create_bucket", bucket_name
, user1
)
536 # TESTCASE 'bucket-list','bucket','list','one bucket','succeeds, expected list'
537 (err
, out
) = rgwadmin(ctx
, client
, ['bucket', 'list', '--uid', user1
], check_status
=True)
539 assert out
[0] == bucket_name
541 bucket_list
= connection
.get_all_buckets()
542 assert len(bucket_list
) == 1
543 assert bucket_list
[0].name
== bucket_name
545 rl
.log_and_clear("list_buckets", '', user1
)
547 # TESTCASE 'bucket-list-all','bucket','list','all buckets','succeeds, expected list'
548 (err
, out
) = rgwadmin(ctx
, client
, ['bucket', 'list'], check_status
=True)
550 assert bucket_name
in out
;
552 # TESTCASE 'max-bucket-limit,'bucket','create','4 buckets','5th bucket fails due to max buckets == 4'
553 bucket2
= connection
.create_bucket(bucket_name
+ '2')
554 rl
.log_and_clear("create_bucket", bucket_name
+ '2', user1
)
555 bucket3
= connection
.create_bucket(bucket_name
+ '3')
556 rl
.log_and_clear("create_bucket", bucket_name
+ '3', user1
)
557 bucket4
= connection
.create_bucket(bucket_name
+ '4')
558 rl
.log_and_clear("create_bucket", bucket_name
+ '4', user1
)
559 # the 5th should fail.
562 connection
.create_bucket(bucket_name
+ '5')
566 rl
.log_and_clear("create_bucket", bucket_name
+ '5', user1
)
570 rl
.log_and_clear("delete_bucket", bucket_name
+ '2', user1
)
572 rl
.log_and_clear("delete_bucket", bucket_name
+ '3', user1
)
574 rl
.log_and_clear("delete_bucket", bucket_name
+ '4', user1
)
576 # TESTCASE 'bucket-stats3','bucket','stats','new empty bucket','succeeds, empty list'
577 (err
, out
) = rgwadmin(ctx
, client
, [
578 'bucket', 'stats', '--bucket', bucket_name
], check_status
=True)
579 assert out
['owner'] == user1
580 bucket_id
= out
['id']
582 # TESTCASE 'bucket-stats4','bucket','stats','new empty bucket','succeeds, expected bucket ID'
583 (err
, out
) = rgwadmin(ctx
, client
, ['bucket', 'stats', '--uid', user1
], check_status
=True)
585 assert out
[0]['id'] == bucket_id
# does it return the same ID twice in a row?
588 key
= boto
.s3
.key
.Key(bucket
)
589 key
.set_contents_from_string('one')
590 rl
.log_and_clear("put_obj", bucket_name
, user1
)
592 # TESTCASE 'bucket-stats5','bucket','stats','after creating key','succeeds, lists one non-empty object'
593 (err
, out
) = rgwadmin(ctx
, client
, [
594 'bucket', 'stats', '--bucket', bucket_name
], check_status
=True)
595 assert out
['id'] == bucket_id
596 assert out
['usage']['rgw.main']['num_objects'] == 1
597 assert out
['usage']['rgw.main']['size_kb'] > 0
599 #validate we have a positive user stats now
600 (err
, out
) = rgwadmin(ctx
, client
,
601 ['user', 'stats','--uid', user1
, '--sync-stats'],
603 assert out
['stats']['size'] > 0
607 rl
.log_and_clear("delete_obj", bucket_name
, user1
)
609 # TESTCASE 'bucket unlink', 'bucket', 'unlink', 'unlink bucket from user', 'fails', 'access denied error'
610 (err
, out
) = rgwadmin(ctx
, client
,
611 ['bucket', 'unlink', '--uid', user1
, '--bucket', bucket_name
],
614 # create a second user to link the bucket to
615 (err
, out
) = rgwadmin(ctx
, client
, [
618 '--display-name', display_name2
,
619 '--access-key', access_key2
,
620 '--secret', secret_key2
,
621 '--max-buckets', '1',
625 # try creating an object with the first user before the bucket is relinked
627 key
= boto
.s3
.key
.Key(bucket
)
630 key
.set_contents_from_string('two')
631 except boto
.exception
.S3ResponseError
:
635 rl
.log_and_clear("put_obj", bucket_name
, user1
)
639 rl
.log_and_clear("delete_obj", bucket_name
, user1
)
641 # link the bucket to another user
642 (err
, out
) = rgwadmin(ctx
, client
, ['metadata', 'get', 'bucket:{n}'.format(n
=bucket_name
)],
645 bucket_data
= out
['data']
646 assert bucket_data
['bucket']['name'] == bucket_name
648 bucket_id
= bucket_data
['bucket']['bucket_id']
650 # link the bucket to another user
651 (err
, out
) = rgwadmin(ctx
, client
, ['bucket', 'link', '--uid', user2
, '--bucket', bucket_name
, '--bucket-id', bucket_id
],
654 # try to remove user, should fail (has a linked bucket)
655 (err
, out
) = rgwadmin(ctx
, client
, ['user', 'rm', '--uid', user2
])
658 # TESTCASE 'bucket unlink', 'bucket', 'unlink', 'unlink bucket from user', 'succeeds, bucket unlinked'
659 (err
, out
) = rgwadmin(ctx
, client
, ['bucket', 'unlink', '--uid', user2
, '--bucket', bucket_name
],
662 # relink the bucket to the first user and delete the second user
663 (err
, out
) = rgwadmin(ctx
, client
,
664 ['bucket', 'link', '--uid', user1
, '--bucket', bucket_name
, '--bucket-id', bucket_id
],
667 (err
, out
) = rgwadmin(ctx
, client
, ['user', 'rm', '--uid', user2
],
670 #TESTCASE 'bucket link', 'bucket', 'tenanted user', 'succeeds'
671 tenant_name
= "testx"
672 # create a tenanted user to link the bucket to
673 (err
, out
) = rgwadmin(ctx
, client
, [
675 '--tenant', tenant_name
,
676 '--uid', 'tenanteduser',
677 '--display-name', 'tenanted-user',
678 '--access-key', access_key2
,
679 '--secret', secret_key2
,
680 '--max-buckets', '1',
684 # link the bucket to a tenanted user
685 (err
, out
) = rgwadmin(ctx
, client
, ['bucket', 'link', '--bucket', '/' + bucket_name
, '--tenant', tenant_name
, '--uid', 'tenanteduser'],
688 # check if the bucket name has tenant/ prefix
689 (err
, out
) = rgwadmin(ctx
, client
, ['metadata', 'get', 'bucket:{n}'.format(n
= tenant_name
+ '/' + bucket_name
)],
692 bucket_data
= out
['data']
693 assert bucket_data
['bucket']['name'] == bucket_name
694 assert bucket_data
['bucket']['tenant'] == tenant_name
696 # relink the bucket to the first user and delete the tenanted user
697 (err
, out
) = rgwadmin(ctx
, client
,
698 ['bucket', 'link', '--bucket', tenant_name
+ '/' + bucket_name
, '--uid', user1
],
701 (err
, out
) = rgwadmin(ctx
, client
, ['user', 'rm', '--tenant', tenant_name
, '--uid', 'tenanteduser'],
704 # TESTCASE 'object-rm', 'object', 'rm', 'remove object', 'succeeds, object is removed'
708 key
= boto
.s3
.key
.Key(bucket
, object_name
)
709 key
.set_contents_from_string(object_name
)
710 rl
.log_and_clear("put_obj", bucket_name
, user1
)
712 # fetch it too (for usage stats presently)
713 s
= key
.get_contents_as_string(encoding
='ascii')
714 rl
.log_and_clear("get_obj", bucket_name
, user1
)
715 assert s
== object_name
716 # list bucket too (for usage stats presently)
717 keys
= list(bucket
.list())
718 rl
.log_and_clear("list_bucket", bucket_name
, user1
)
719 assert len(keys
) == 1
720 assert keys
[0].name
== object_name
723 (err
, out
) = rgwadmin(ctx
, client
,
724 ['object', 'rm', '--bucket', bucket_name
, '--object', object_name
],
727 # TESTCASE 'bucket-stats6','bucket','stats','after deleting key','succeeds, lists one no objects'
728 (err
, out
) = rgwadmin(ctx
, client
, [
729 'bucket', 'stats', '--bucket', bucket_name
],
731 assert out
['id'] == bucket_id
732 assert out
['usage']['rgw.main']['num_objects'] == 0
735 # TESTCASE 'log-list','log','list','after activity','succeeds, lists one no objects'
736 (err
, out
) = rgwadmin(ctx
, client
, ['log', 'list'], check_status
=True)
740 # TESTCASE 'log-show','log','show','after activity','returns expected info'
741 if obj
[:4] == 'meta' or obj
[:4] == 'data' or obj
[:18] == 'obj_delete_at_hint':
744 (err
, rgwlog
) = rgwadmin(ctx
, client
, ['log', 'show', '--object', obj
],
746 assert len(rgwlog
) > 0
748 # skip any entry for which there is no bucket name--e.g., list_buckets,
749 # since that is valid but cannot pass the following checks
750 entry_bucket_name
= rgwlog
['bucket']
751 if entry_bucket_name
.strip() != "":
752 # exempt bucket_name2 from checking as it was only used for multi-region tests
753 assert rgwlog
['bucket'].find(bucket_name
) == 0 or rgwlog
['bucket'].find(bucket_name2
) == 0
754 assert rgwlog
['bucket'] != bucket_name
or rgwlog
['bucket_id'] == bucket_id
755 assert rgwlog
['bucket_owner'] == user1
or rgwlog
['bucket'] == bucket_name
+ '5' or rgwlog
['bucket'] == bucket_name2
756 for entry
in rgwlog
['log_entries']:
757 log
.debug('checking log entry: ', entry
)
758 assert entry
['bucket'] == rgwlog
['bucket']
759 possible_buckets
= [bucket_name
+ '5', bucket_name2
]
761 assert user
== user1
or user
.endswith('system-user') or \
762 rgwlog
['bucket'] in possible_buckets
764 # TESTCASE 'log-rm','log','rm','delete log objects','succeeds'
765 (err
, out
) = rgwadmin(ctx
, client
, ['log', 'rm', '--object', obj
],
768 # TODO: show log by bucket+date
770 # TESTCASE 'user-suspend2','user','suspend','existing user','succeeds'
771 (err
, out
) = rgwadmin(ctx
, client
, ['user', 'suspend', '--uid', user1
],
774 # TESTCASE 'user-suspend3','user','suspend','suspended user','cannot write objects'
777 key
= boto
.s3
.key
.Key(bucket
)
778 key
.set_contents_from_string('five')
779 except boto
.exception
.S3ResponseError
as e
:
781 assert e
.status
== 403
784 rl
.log_and_clear("put_obj", bucket_name
, user1
)
786 # TESTCASE 'user-renable2','user','enable','suspended user','succeeds'
787 (err
, out
) = rgwadmin(ctx
, client
, ['user', 'enable', '--uid', user1
],
790 # TESTCASE 'user-renable3','user','enable','reenabled user','can write objects'
791 key
= boto
.s3
.key
.Key(bucket
)
792 key
.set_contents_from_string('six')
793 rl
.log_and_clear("put_obj", bucket_name
, user1
)
795 # TESTCASE 'gc-list', 'gc', 'list', 'get list of objects ready for garbage collection'
797 # create an object large enough to be split into multiple parts
798 test_string
= 'foo'*10000000
800 big_key
= boto
.s3
.key
.Key(bucket
)
801 big_key
.set_contents_from_string(test_string
)
802 rl
.log_and_clear("put_obj", bucket_name
, user1
)
804 # now delete the head
806 rl
.log_and_clear("delete_obj", bucket_name
, user1
)
808 # wait a bit to give the garbage collector time to cycle
811 (err
, out
) = rgwadmin(ctx
, client
, ['gc', 'list', '--include-all'])
814 # TESTCASE 'gc-process', 'gc', 'process', 'manually collect garbage'
815 (err
, out
) = rgwadmin(ctx
, client
, ['gc', 'process'], check_status
=True)
818 (err
, out
) = rgwadmin(ctx
, client
, ['gc', 'list', '--include-all'])
820 # don't assume rgw_gc_obj_min_wait has been overridden
821 omit_tdir
= hasattr(ctx
.rgw
, 'omit_tdir') and ctx
.rgw
.omit_tdir
== True
825 # TESTCASE 'rm-user-buckets','user','rm','existing user','fails, still has buckets'
826 (err
, out
) = rgwadmin(ctx
, client
, ['user', 'rm', '--uid', user1
])
829 # delete should fail because ``key`` still exists
832 except boto
.exception
.S3ResponseError
as e
:
833 assert e
.status
== 409
834 rl
.log_and_clear("delete_bucket", bucket_name
, user1
)
837 rl
.log_and_clear("delete_obj", bucket_name
, user1
)
839 rl
.log_and_clear("delete_bucket", bucket_name
, user1
)
841 # TESTCASE 'policy', 'bucket', 'policy', 'get bucket policy', 'returns S3 policy'
842 bucket
= connection
.create_bucket(bucket_name
)
843 rl
.log_and_clear("create_bucket", bucket_name
, user1
)
846 key
= boto
.s3
.key
.Key(bucket
)
847 key
.set_contents_from_string('seven')
848 rl
.log_and_clear("put_obj", bucket_name
, user1
)
850 # should be private already but guarantee it
851 key
.set_acl('private')
852 rl
.log_and_clear("put_acls", bucket_name
, user1
)
854 (err
, out
) = rgwadmin(ctx
, client
,
855 ['policy', '--bucket', bucket
.name
, '--object', key
.key
.decode()],
856 check_status
=True, format
='xml')
859 rl
.log_and_clear("get_acls", bucket_name
, user1
)
861 assert acl
== out
.strip('\n')
863 # add another grantee by making the object public read
864 key
.set_acl('public-read')
865 rl
.log_and_clear("put_acls", bucket_name
, user1
)
867 (err
, out
) = rgwadmin(ctx
, client
,
868 ['policy', '--bucket', bucket
.name
, '--object', key
.key
.decode()],
869 check_status
=True, format
='xml')
872 rl
.log_and_clear("get_acls", bucket_name
, user1
)
874 assert acl
== out
.strip('\n')
876 # TESTCASE 'rm-bucket', 'bucket', 'rm', 'bucket with objects', 'succeeds'
877 bucket
= connection
.create_bucket(bucket_name
)
878 rl
.log_and_clear("create_bucket", bucket_name
, user1
)
879 key_name
= ['eight', 'nine', 'ten', 'eleven']
881 key
= boto
.s3
.key
.Key(bucket
)
882 key
.set_contents_from_string(key_name
[i
])
883 rl
.log_and_clear("put_obj", bucket_name
, user1
)
885 (err
, out
) = rgwadmin(ctx
, client
,
886 ['bucket', 'rm', '--bucket', bucket_name
, '--purge-objects'],
889 # TESTCASE 'caps-add', 'caps', 'add', 'add user cap', 'succeeds'
891 (err
, out
) = rgwadmin(ctx
, client
, ['caps', 'add', '--uid', user1
, '--caps', caps
])
893 assert out
['caps'][0]['perm'] == 'read'
895 # TESTCASE 'caps-rm', 'caps', 'rm', 'remove existing cap from user', 'succeeds'
896 (err
, out
) = rgwadmin(ctx
, client
, ['caps', 'rm', '--uid', user1
, '--caps', caps
])
898 assert not out
['caps']
900 # TESTCASE 'rm-user','user','rm','existing user','fails, still has buckets'
901 bucket
= connection
.create_bucket(bucket_name
)
902 rl
.log_and_clear("create_bucket", bucket_name
, user1
)
903 key
= boto
.s3
.key
.Key(bucket
)
905 (err
, out
) = rgwadmin(ctx
, client
, ['user', 'rm', '--uid', user1
])
908 # TESTCASE 'rm-user2', 'user', 'rm', 'user with data', 'succeeds'
909 bucket
= connection
.create_bucket(bucket_name
)
910 rl
.log_and_clear("create_bucket", bucket_name
, user1
)
911 key
= boto
.s3
.key
.Key(bucket
)
912 key
.set_contents_from_string('twelve')
913 rl
.log_and_clear("put_obj", bucket_name
, user1
)
917 # need to wait for all usage data to get flushed, should take up to 30 seconds
918 timestamp
= time
.time()
919 while time
.time() - timestamp
<= (2 * 60): # wait up to 20 minutes
920 (err
, out
) = rgwadmin(ctx
, client
, ['usage', 'show', '--categories', 'delete_obj']) # one of the operations we did is delete_obj, should be present.
921 if get_user_successful_ops(out
, user1
) > 0:
925 assert time
.time() - timestamp
<= (20 * 60)
927 # TESTCASE 'usage-show' 'usage' 'show' 'all usage' 'succeeds'
928 (err
, out
) = rgwadmin(ctx
, client
, ['usage', 'show'], check_status
=True)
929 assert len(out
['entries']) > 0
930 assert len(out
['summary']) > 0
932 r
= acc
.compare_results(out
)
934 sys
.stderr
.write(("\n".join(r
))+"\n")
937 user_summary
= get_user_summary(out
, user1
)
939 total
= user_summary
['total']
940 assert total
['successful_ops'] > 0
942 # TESTCASE 'usage-show2' 'usage' 'show' 'user usage' 'succeeds'
943 (err
, out
) = rgwadmin(ctx
, client
, ['usage', 'show', '--uid', user1
],
945 assert len(out
['entries']) > 0
946 assert len(out
['summary']) > 0
947 user_summary
= out
['summary'][0]
948 for entry
in user_summary
['categories']:
949 assert entry
['successful_ops'] > 0
950 assert user_summary
['user'] == user1
952 # TESTCASE 'usage-show3' 'usage' 'show' 'user usage categories' 'succeeds'
953 test_categories
= ['create_bucket', 'put_obj', 'delete_obj', 'delete_bucket']
954 for cat
in test_categories
:
955 (err
, out
) = rgwadmin(ctx
, client
, ['usage', 'show', '--uid', user1
, '--categories', cat
],
957 assert len(out
['summary']) > 0
958 user_summary
= out
['summary'][0]
959 assert user_summary
['user'] == user1
960 assert len(user_summary
['categories']) == 1
961 entry
= user_summary
['categories'][0]
962 assert entry
['category'] == cat
963 assert entry
['successful_ops'] > 0
965 # TESTCASE 'user-rename', 'user', 'rename', 'existing user', 'new user', 'succeeds'
966 # create a new user user3
967 (err
, out
) = rgwadmin(ctx
, client
, [
970 '--display-name', display_name3
,
971 '--access-key', access_key3
,
972 '--secret', secret_key3
,
978 bucket
= connection3
.create_bucket(bucket_name
+ '6')
980 rl
.log_and_clear("create_bucket", bucket_name
+ '6', user3
)
983 object_name1
= 'thirteen'
984 key1
= boto
.s3
.key
.Key(bucket
, object_name1
)
985 key1
.set_contents_from_string(object_name1
)
986 rl
.log_and_clear("put_obj", bucket_name
+ '6', user3
)
989 (err
, out
) = rgwadmin(ctx
, client
, ['user', 'rename', '--uid', user3
, '--new-uid', user4
], check_status
=True)
990 assert out
['user_id'] == user4
991 assert out
['keys'][0]['access_key'] == access_key3
992 assert out
['keys'][0]['secret_key'] == secret_key3
996 # get bucket and object to test if user keys are preserved
997 bucket
= connection3
.get_bucket(bucket_name
+ '6')
998 s
= key1
.get_contents_as_string(encoding
='ascii')
999 rl
.log_and_clear("get_obj", bucket_name
+ '6', user4
)
1000 assert s
== object_name1
1002 # TESTCASE 'user-rename', 'user', 'rename', 'existing user', 'another existing user', 'fails'
1003 # create a new user user2
1004 (err
, out
) = rgwadmin(ctx
, client
, [
1007 '--display-name', display_name2
,
1008 '--access-key', access_key2
,
1009 '--secret', secret_key2
,
1010 '--max-buckets', '4'
1015 bucket
= connection2
.create_bucket(bucket_name
+ '7')
1017 rl
.log_and_clear("create_bucket", bucket_name
+ '7', user2
)
1020 object_name2
= 'fourteen'
1021 key2
= boto
.s3
.key
.Key(bucket
, object_name2
)
1022 key2
.set_contents_from_string(object_name2
)
1023 rl
.log_and_clear("put_obj", bucket_name
+ '7', user2
)
1025 (err
, out
) = rgwadmin(ctx
, client
, ['user', 'rename', '--uid', user4
, '--new-uid', user2
])
1028 # test if user 2 and user4 can still access their bucket and objects after rename fails
1029 bucket
= connection3
.get_bucket(bucket_name
+ '6')
1030 s
= key1
.get_contents_as_string(encoding
='ascii')
1031 rl
.log_and_clear("get_obj", bucket_name
+ '6', user4
)
1032 assert s
== object_name1
1034 bucket
= connection2
.get_bucket(bucket_name
+ '7')
1035 s
= key2
.get_contents_as_string(encoding
='ascii')
1036 rl
.log_and_clear("get_obj", bucket_name
+ '7', user2
)
1037 assert s
== object_name2
1039 (err
, out
) = rgwadmin(ctx
, client
,
1040 ['user', 'rm', '--uid', user4
, '--purge-data' ],
1043 (err
, out
) = rgwadmin(ctx
, client
,
1044 ['user', 'rm', '--uid', user2
, '--purge-data' ],
1049 # should be all through with connection. (anything using connection
1050 # should be BEFORE the usage stuff above.)
1051 rl
.log_and_clear("(before-close)", '-', '-', ignore_this_entry
)
1055 # the usage flush interval is 30 seconds, wait that much an then some
1056 # to make sure everything has been flushed
1059 # TESTCASE 'usage-trim' 'usage' 'trim' 'user usage' 'succeeds, usage removed'
1060 (err
, out
) = rgwadmin(ctx
, client
, ['usage', 'trim', '--uid', user1
],
1062 (err
, out
) = rgwadmin(ctx
, client
, ['usage', 'show', '--uid', user1
],
1064 assert len(out
['entries']) == 0
1065 assert len(out
['summary']) == 0
1067 (err
, out
) = rgwadmin(ctx
, client
,
1068 ['user', 'rm', '--uid', user1
, '--purge-data' ],
1071 # TESTCASE 'rm-user3','user','rm','deleted user','fails'
1072 (err
, out
) = rgwadmin(ctx
, client
, ['user', 'info', '--uid', user1
])
1075 # TESTCASE 'zone-info', 'zone', 'get', 'get zone info', 'succeeds, has default placement rule'
1076 (err
, out
) = rgwadmin(ctx
, client
, ['zone', 'get','--rgw-zone','default'])
1077 orig_placement_pools
= len(out
['placement_pools'])
1079 # removed this test, it is not correct to assume that zone has default placement, it really
1080 # depends on how we set it up before
1082 # assert len(out) > 0
1083 # assert len(out['placement_pools']) == 1
1085 # default_rule = out['placement_pools'][0]
1086 # assert default_rule['key'] == 'default-placement'
1088 rule
={'key': 'new-placement', 'val': {'data_pool': '.rgw.buckets.2', 'index_pool': '.rgw.buckets.index.2'}}
1090 out
['placement_pools'].append(rule
)
1092 (err
, out
) = rgwadmin(ctx
, client
, ['zone', 'set'],
1093 stdin
=StringIO(json
.dumps(out
)),
1096 (err
, out
) = rgwadmin(ctx
, client
, ['zone', 'get'])
1098 assert len(out
['placement_pools']) == orig_placement_pools
+ 1
1100 zonecmd
= ['zone', 'placement', 'rm',
1101 '--rgw-zone', 'default',
1102 '--placement-id', 'new-placement']
1104 (err
, out
) = rgwadmin(ctx
, client
, zonecmd
, check_status
=True)
1106 # TESTCASE 'zonegroup-info', 'zonegroup', 'get', 'get zonegroup info', 'succeeds'
1107 (err
, out
) = rgwadmin(ctx
, client
, ['zonegroup', 'get'], check_status
=True)
1109 from teuthology
.config
import config
1110 from teuthology
.orchestra
import cluster
1115 parser
= argparse
.ArgumentParser()
1116 parser
.add_argument('--uid')
1117 parser
.add_argument('--host', required
=True)
1118 parser
.add_argument('--port', type=int)
1120 args
= parser
.parse_args()
1127 client0
= tasks
.vstart_runner
.LocalRemote()
1129 ctx
.cluster
=cluster
.Cluster(remotes
=[(client0
,
1130 [ 'ceph.client.rgw.%s' % (port
), ]),])
1131 ctx
.rgw
= argparse
.Namespace()
1133 endpoints
['ceph.client.rgw.%s' % port
] = RGWEndpoint(
1136 ctx
.rgw
.role_endpoints
= endpoints
1137 ctx
.rgw
.realm
= None
1138 ctx
.rgw
.regions
= {'region0': { 'api name': 'api1',
1139 'is master': True, 'master zone': 'r0z0',
1140 'zones': ['r0z0', 'r0z1'] }}
1141 ctx
.rgw
.omit_sudo
= True
1142 ctx
.rgw
.omit_tdir
= True
1143 ctx
.rgw
.config
= {'ceph.client.rgw.%s' % port
: {'system user': {'name': '%s-system-user' % port
}}}
1147 if __name__
== '__main__':