]>
git.proxmox.com Git - ceph.git/blob - ceph/qa/tasks/radosgw_admin_rest.py
2 Run a series of rgw admin commands through the rest interface.
4 The test cases in this file have been annotated for inventory.
5 To extract the inventory (in csv format) use the command:
7 grep '^ *# TESTCASE' | sed 's/^ *# TESTCASE //'
10 from cStringIO
import StringIO
15 import boto
.s3
.connection
21 from boto
.connection
import AWSAuthConnection
22 from teuthology
import misc
as teuthology
23 from util
.rgw
import get_user_summary
, get_user_successful_ops
25 log
= logging
.getLogger(__name__
)
27 def rgwadmin(ctx
, client
, cmd
):
29 Perform rgw admin command
32 :param cmd: command to execute.
33 :return: command exit status, json result.
35 log
.info('radosgw-admin: %s' % cmd
)
36 testdir
= teuthology
.get_testdir(ctx
)
40 '{tdir}/archive/coverage'.format(tdir
=testdir
),
46 (remote
,) = ctx
.cluster
.only(client
).remotes
.iterkeys()
54 out
= proc
.stdout
.getvalue()
56 if not r
and out
!= '':
59 log
.info(' json result: %s' % j
)
62 log
.info(' raw result: %s' % j
)
66 def rgwadmin_rest(connection
, cmd
, params
=None, headers
=None, raw
=False):
68 perform a rest command
70 log
.info('radosgw-admin-rest: %s %s' % (cmd
, params
))
71 put_cmds
= ['create', 'link', 'add']
72 post_cmds
= ['unlink', 'modify']
73 delete_cmds
= ['trim', 'rm', 'process']
74 get_cmds
= ['check', 'info', 'show', 'list']
76 bucket_sub_resources
= ['object', 'policy', 'index']
77 user_sub_resources
= ['subuser', 'key', 'caps']
78 zone_sub_resources
= ['pool', 'log', 'garbage']
80 def get_cmd_method_and_handler(cmd
):
82 Get the rest command and handler from information in cmd and
83 from the imported requests object.
85 if cmd
[1] in put_cmds
:
86 return 'PUT', requests
.put
87 elif cmd
[1] in delete_cmds
:
88 return 'DELETE', requests
.delete
89 elif cmd
[1] in post_cmds
:
90 return 'POST', requests
.post
91 elif cmd
[1] in get_cmds
:
92 return 'GET', requests
.get
94 def get_resource(cmd
):
96 Get the name of the resource from information in cmd.
98 if cmd
[0] == 'bucket' or cmd
[0] in bucket_sub_resources
:
99 if cmd
[0] == 'bucket':
102 return 'bucket', cmd
[0]
103 elif cmd
[0] == 'user' or cmd
[0] in user_sub_resources
:
107 return 'user', cmd
[0]
108 elif cmd
[0] == 'usage':
110 elif cmd
[0] == 'zone' or cmd
[0] in zone_sub_resources
:
114 return 'zone', cmd
[0]
116 def build_admin_request(conn
, method
, resource
= '', headers
=None, data
='',
117 query_args
=None, params
=None):
119 Build an administative request adapted from the build_request()
120 method of boto.connection
123 path
= conn
.calling_format
.build_path_base('admin', resource
)
124 auth_path
= conn
.calling_format
.build_auth_path('admin', resource
)
125 host
= conn
.calling_format
.build_host(conn
.server_name(), 'admin')
127 path
+= '?' + query_args
128 boto
.log
.debug('path=%s' % path
)
129 auth_path
+= '?' + query_args
130 boto
.log
.debug('auth_path=%s' % auth_path
)
131 return AWSAuthConnection
.build_base_http_request(conn
, method
, path
,
132 auth_path
, params
, headers
, data
, host
)
134 method
, handler
= get_cmd_method_and_handler(cmd
)
135 resource
, query_args
= get_resource(cmd
)
136 request
= build_admin_request(connection
, method
, resource
,
137 query_args
=query_args
, headers
=headers
)
139 url
= '{protocol}://{host}{path}'.format(protocol
=request
.protocol
,
140 host
=request
.host
, path
=request
.path
)
142 request
.authorize(connection
=connection
)
143 result
= handler(url
, params
=params
, headers
=request
.headers
)
146 log
.info(' text result: %s' % result
.txt
)
147 return result
.status_code
, result
.txt
149 log
.info(' json result: %s' % result
.json())
150 return result
.status_code
, result
.json()
153 def task(ctx
, config
):
155 Test radosgw-admin functionality through the RESTful interface
157 assert config
is None or isinstance(config
, list) \
158 or isinstance(config
, dict), \
159 "task s3tests only supports a list or dictionary for configuration"
160 all_clients
= ['client.{id}'.format(id=id_
)
161 for id_
in teuthology
.all_roles_of_type(ctx
.cluster
, 'client')]
164 if isinstance(config
, list):
165 config
= dict.fromkeys(config
)
166 clients
= config
.keys()
168 # just use the first client...
173 admin_display_name
= 'Ms. Admin User'
174 admin_access_key
= 'MH1WC2XQ1S8UISFDZC8W'
175 admin_secret_key
= 'dQyrTPA0s248YeN5bBv4ukvKU0kh54LWWywkrpoG'
176 admin_caps
= 'users=read, write; usage=read, write; buckets=read, write; zone=read, write'
180 subuser1
= 'foo:foo1'
181 subuser2
= 'foo:foo2'
182 display_name1
= 'Foo'
183 display_name2
= 'Fud'
184 email
= 'foo@foo.com'
185 access_key
= '9te6NH5mcdcq0Tc5i8i1'
186 secret_key
= 'Ny4IOauQoL18Gp2zM7lC1vLmoawgqcYP/YGcWfXu'
187 access_key2
= 'p5YnriCv1nAtykxBrupQ'
188 secret_key2
= 'Q8Tk6Q/27hfbFSYdSkPtUqhqx1GgzvpXa4WARozh'
189 swift_secret1
= 'gpS2G9RREMrnbqlp29PP2D36kgPR1tm72n5fPYfL'
190 swift_secret2
= 'ri2VJQcKSYATOY6uaDUX7pxgkW+W1YmC6OCxPHwy'
192 bucket_name
= 'myfoo'
194 # legend (test cases can be easily grep-ed out)
195 # TESTCASE 'testname','object','method','operation','assertion'
196 # TESTCASE 'create-admin-user','user','create','administrative user','succeeds'
197 (err
, out
) = rgwadmin(ctx
, client
, [
200 '--display-name', admin_display_name
,
201 '--access-key', admin_access_key
,
202 '--secret', admin_secret_key
,
203 '--max-buckets', '0',
210 (remote
,) = ctx
.cluster
.only(client
).remotes
.iterkeys()
211 remote_host
= remote
.name
.split('@')[1]
212 admin_conn
= boto
.s3
.connection
.S3Connection(
213 aws_access_key_id
=admin_access_key
,
214 aws_secret_access_key
=admin_secret_key
,
218 calling_format
=boto
.s3
.connection
.OrdinaryCallingFormat(),
221 # TESTCASE 'info-nosuch','user','info','non-existent user','fails'
222 (ret
, out
) = rgwadmin_rest(admin_conn
, ['user', 'info'], {"uid": user1
})
225 # TESTCASE 'create-ok','user','create','w/all valid info','succeeds'
226 (ret
, out
) = rgwadmin_rest(admin_conn
,
229 'display-name' : display_name1
,
231 'access-key' : access_key
,
232 'secret-key' : secret_key
,
238 # TESTCASE 'info-existing','user','info','existing user','returns correct info'
239 (ret
, out
) = rgwadmin_rest(admin_conn
, ['user', 'info'], {'uid' : user1
})
241 assert out
['user_id'] == user1
242 assert out
['email'] == email
243 assert out
['display_name'] == display_name1
244 assert len(out
['keys']) == 1
245 assert out
['keys'][0]['access_key'] == access_key
246 assert out
['keys'][0]['secret_key'] == secret_key
247 assert not out
['suspended']
249 # TESTCASE 'suspend-ok','user','suspend','active user','succeeds'
250 (ret
, out
) = rgwadmin_rest(admin_conn
, ['user', 'modify'], {'uid' : user1
, 'suspended' : True})
253 # TESTCASE 'suspend-suspended','user','suspend','suspended user','succeeds w/advisory'
254 (ret
, out
) = rgwadmin_rest(admin_conn
, ['user', 'info'], {'uid' : user1
})
256 assert out
['suspended']
257 assert out
['email'] == email
259 # TESTCASE 're-enable','user','enable','suspended user','succeeds'
260 (ret
, out
) = rgwadmin_rest(admin_conn
, ['user', 'modify'], {'uid' : user1
, 'suspended' : 'false'})
263 # TESTCASE 'info-re-enabled','user','info','re-enabled user','no longer suspended'
264 (ret
, out
) = rgwadmin_rest(admin_conn
, ['user', 'info'], {'uid' : user1
})
266 assert not out
['suspended']
268 # TESTCASE 'add-keys','key','create','w/valid info','succeeds'
269 (ret
, out
) = rgwadmin_rest(admin_conn
,
272 'access-key' : access_key2
,
273 'secret-key' : secret_key2
279 # TESTCASE 'info-new-key','user','info','after key addition','returns all keys'
280 (ret
, out
) = rgwadmin_rest(admin_conn
, ['user', 'info'], {'uid' : user1
})
282 assert len(out
['keys']) == 2
283 assert out
['keys'][0]['access_key'] == access_key2
or out
['keys'][1]['access_key'] == access_key2
284 assert out
['keys'][0]['secret_key'] == secret_key2
or out
['keys'][1]['secret_key'] == secret_key2
286 # TESTCASE 'rm-key','key','rm','newly added key','succeeds, key is removed'
287 (ret
, out
) = rgwadmin_rest(admin_conn
,
290 'access-key' : access_key2
295 (ret
, out
) = rgwadmin_rest(admin_conn
, ['user', 'info'], {'uid' : user1
})
297 assert len(out
['keys']) == 1
298 assert out
['keys'][0]['access_key'] == access_key
299 assert out
['keys'][0]['secret_key'] == secret_key
301 # TESTCASE 'add-swift-key','key','create','swift key','succeeds'
302 (ret
, out
) = rgwadmin_rest(admin_conn
,
303 ['subuser', 'create'],
304 {'subuser' : subuser1
,
305 'secret-key' : swift_secret1
,
311 # TESTCASE 'info-swift-key','user','info','after key addition','returns all keys'
312 (ret
, out
) = rgwadmin_rest(admin_conn
, ['user', 'info'], {'uid' : user1
})
314 assert len(out
['swift_keys']) == 1
315 assert out
['swift_keys'][0]['user'] == subuser1
316 assert out
['swift_keys'][0]['secret_key'] == swift_secret1
318 # TESTCASE 'add-swift-subuser','key','create','swift sub-user key','succeeds'
319 (ret
, out
) = rgwadmin_rest(admin_conn
,
320 ['subuser', 'create'],
321 {'subuser' : subuser2
,
322 'secret-key' : swift_secret2
,
328 # TESTCASE 'info-swift-subuser','user','info','after key addition','returns all sub-users/keys'
329 (ret
, out
) = rgwadmin_rest(admin_conn
, ['user', 'info'], {'uid' : user1
})
331 assert len(out
['swift_keys']) == 2
332 assert out
['swift_keys'][0]['user'] == subuser2
or out
['swift_keys'][1]['user'] == subuser2
333 assert out
['swift_keys'][0]['secret_key'] == swift_secret2
or out
['swift_keys'][1]['secret_key'] == swift_secret2
335 # TESTCASE 'rm-swift-key1','key','rm','subuser','succeeds, one key is removed'
336 (ret
, out
) = rgwadmin_rest(admin_conn
,
338 {'subuser' : subuser1
,
344 (ret
, out
) = rgwadmin_rest(admin_conn
, ['user', 'info'], {'uid' : user1
})
345 assert len(out
['swift_keys']) == 1
347 # TESTCASE 'rm-subuser','subuser','rm','subuser','success, subuser is removed'
348 (ret
, out
) = rgwadmin_rest(admin_conn
,
350 {'subuser' : subuser1
355 (ret
, out
) = rgwadmin_rest(admin_conn
, ['user', 'info'], {'uid' : user1
})
356 assert len(out
['subusers']) == 1
358 # TESTCASE 'rm-subuser-with-keys','subuser','rm','subuser','succeeds, second subser and key is removed'
359 (ret
, out
) = rgwadmin_rest(admin_conn
,
361 {'subuser' : subuser2
,
362 'key-type' : 'swift',
368 (ret
, out
) = rgwadmin_rest(admin_conn
, ['user', 'info'], {'uid' : user1
})
369 assert len(out
['swift_keys']) == 0
370 assert len(out
['subusers']) == 0
372 # TESTCASE 'bucket-stats','bucket','info','no session/buckets','succeeds, empty list'
373 (ret
, out
) = rgwadmin_rest(admin_conn
, ['bucket', 'info'], {'uid' : user1
})
378 connection
= boto
.s3
.connection
.S3Connection(
379 aws_access_key_id
=access_key
,
380 aws_secret_access_key
=secret_key
,
384 calling_format
=boto
.s3
.connection
.OrdinaryCallingFormat(),
387 # TESTCASE 'bucket-stats2','bucket','stats','no buckets','succeeds, empty list'
388 (ret
, out
) = rgwadmin_rest(admin_conn
, ['bucket', 'info'], {'uid' : user1
, 'stats' : True})
392 # create a first bucket
393 bucket
= connection
.create_bucket(bucket_name
)
395 # TESTCASE 'bucket-list','bucket','list','one bucket','succeeds, expected list'
396 (ret
, out
) = rgwadmin_rest(admin_conn
, ['bucket', 'info'], {'uid' : user1
})
399 assert out
[0] == bucket_name
401 # TESTCASE 'bucket-stats3','bucket','stats','new empty bucket','succeeds, empty list'
402 (ret
, out
) = rgwadmin_rest(admin_conn
,
403 ['bucket', 'info'], {'bucket' : bucket_name
, 'stats' : True})
406 assert out
['owner'] == user1
407 bucket_id
= out
['id']
409 # TESTCASE 'bucket-stats4','bucket','stats','new empty bucket','succeeds, expected bucket ID'
410 (ret
, out
) = rgwadmin_rest(admin_conn
, ['bucket', 'info'], {'uid' : user1
, 'stats' : True})
413 assert out
[0]['id'] == bucket_id
# does it return the same ID twice in a row?
416 key
= boto
.s3
.key
.Key(bucket
)
417 key
.set_contents_from_string('one')
419 # TESTCASE 'bucket-stats5','bucket','stats','after creating key','succeeds, lists one non-empty object'
420 (ret
, out
) = rgwadmin_rest(admin_conn
, ['bucket', 'info'], {'bucket' : bucket_name
, 'stats' : True})
422 assert out
['id'] == bucket_id
423 assert out
['usage']['rgw.main']['num_objects'] == 1
424 assert out
['usage']['rgw.main']['size_kb'] > 0
429 # TESTCASE 'bucket unlink', 'bucket', 'unlink', 'unlink bucket from user', 'fails', 'access denied error'
430 (ret
, out
) = rgwadmin_rest(admin_conn
, ['bucket', 'unlink'], {'uid' : user1
, 'bucket' : bucket_name
})
434 # create a second user to link the bucket to
435 (ret
, out
) = rgwadmin_rest(admin_conn
,
438 'display-name' : display_name2
,
439 'access-key' : access_key2
,
440 'secret-key' : secret_key2
,
446 # try creating an object with the first user before the bucket is relinked
448 key
= boto
.s3
.key
.Key(bucket
)
451 key
.set_contents_from_string('two')
452 except boto
.exception
.S3ResponseError
:
460 # link the bucket to another user
461 (ret
, out
) = rgwadmin_rest(admin_conn
, ['bucket', 'link'], {'uid' : user2
, 'bucket' : bucket_name
})
465 # try creating an object with the first user which should cause an error
466 key
= boto
.s3
.key
.Key(bucket
)
469 key
.set_contents_from_string('three')
470 except boto
.exception
.S3ResponseError
:
475 # relink the bucket to the first user and delete the second user
476 (ret
, out
) = rgwadmin_rest(admin_conn
, ['bucket', 'link'], {'uid' : user1
, 'bucket' : bucket_name
})
479 (ret
, out
) = rgwadmin_rest(admin_conn
, ['user', 'rm'], {'uid' : user2
})
482 # TESTCASE 'object-rm', 'object', 'rm', 'remove object', 'succeeds, object is removed'
486 key
= boto
.s3
.key
.Key(bucket
, object_name
)
487 key
.set_contents_from_string(object_name
)
490 (ret
, out
) = rgwadmin_rest(admin_conn
, ['object', 'rm'], {'bucket' : bucket_name
, 'object' : object_name
})
493 # TESTCASE 'bucket-stats6','bucket','stats','after deleting key','succeeds, lists one no objects'
494 (ret
, out
) = rgwadmin_rest(admin_conn
, ['bucket', 'info'], {'bucket' : bucket_name
, 'stats' : True})
496 assert out
['id'] == bucket_id
497 assert out
['usage']['rgw.main']['num_objects'] == 0
499 # create a bucket for deletion stats
500 useless_bucket
= connection
.create_bucket('useless_bucket')
501 useless_key
= useless_bucket
.new_key('useless_key')
502 useless_key
.set_contents_from_string('useless string')
506 useless_bucket
.delete()
508 # wait for the statistics to flush
511 # need to wait for all usage data to get flushed, should take up to 30 seconds
512 timestamp
= time
.time()
513 while time
.time() - timestamp
<= (20 * 60): # wait up to 20 minutes
514 (ret
, out
) = rgwadmin_rest(admin_conn
, ['usage', 'show'], {'categories' : 'delete_obj'}) # last operation we did is delete obj, wait for it to flush
516 if get_user_successful_ops(out
, user1
) > 0:
520 assert time
.time() - timestamp
<= (20 * 60)
522 # TESTCASE 'usage-show' 'usage' 'show' 'all usage' 'succeeds'
523 (ret
, out
) = rgwadmin_rest(admin_conn
, ['usage', 'show'])
525 assert len(out
['entries']) > 0
526 assert len(out
['summary']) > 0
527 user_summary
= get_user_summary(out
, user1
)
528 total
= user_summary
['total']
529 assert total
['successful_ops'] > 0
531 # TESTCASE 'usage-show2' 'usage' 'show' 'user usage' 'succeeds'
532 (ret
, out
) = rgwadmin_rest(admin_conn
, ['usage', 'show'], {'uid' : user1
})
534 assert len(out
['entries']) > 0
535 assert len(out
['summary']) > 0
536 user_summary
= out
['summary'][0]
537 for entry
in user_summary
['categories']:
538 assert entry
['successful_ops'] > 0
539 assert user_summary
['user'] == user1
541 # TESTCASE 'usage-show3' 'usage' 'show' 'user usage categories' 'succeeds'
542 test_categories
= ['create_bucket', 'put_obj', 'delete_obj', 'delete_bucket']
543 for cat
in test_categories
:
544 (ret
, out
) = rgwadmin_rest(admin_conn
, ['usage', 'show'], {'uid' : user1
, 'categories' : cat
})
546 assert len(out
['summary']) > 0
547 user_summary
= out
['summary'][0]
548 assert user_summary
['user'] == user1
549 assert len(user_summary
['categories']) == 1
550 entry
= user_summary
['categories'][0]
551 assert entry
['category'] == cat
552 assert entry
['successful_ops'] > 0
554 # TESTCASE 'usage-trim' 'usage' 'trim' 'user usage' 'succeeds, usage removed'
555 (ret
, out
) = rgwadmin_rest(admin_conn
, ['usage', 'trim'], {'uid' : user1
})
557 (ret
, out
) = rgwadmin_rest(admin_conn
, ['usage', 'show'], {'uid' : user1
})
559 assert len(out
['entries']) == 0
560 assert len(out
['summary']) == 0
562 # TESTCASE 'user-suspend2','user','suspend','existing user','succeeds'
563 (ret
, out
) = rgwadmin_rest(admin_conn
, ['user', 'modify'], {'uid' : user1
, 'suspended' : True})
566 # TESTCASE 'user-suspend3','user','suspend','suspended user','cannot write objects'
568 key
= boto
.s3
.key
.Key(bucket
)
569 key
.set_contents_from_string('five')
570 except boto
.exception
.S3ResponseError
as e
:
571 assert e
.status
== 403
573 # TESTCASE 'user-renable2','user','enable','suspended user','succeeds'
574 (ret
, out
) = rgwadmin_rest(admin_conn
, ['user', 'modify'], {'uid' : user1
, 'suspended' : 'false'})
577 # TESTCASE 'user-renable3','user','enable','reenabled user','can write objects'
578 key
= boto
.s3
.key
.Key(bucket
)
579 key
.set_contents_from_string('six')
581 # TESTCASE 'garbage-list', 'garbage', 'list', 'get list of objects ready for garbage collection'
583 # create an object large enough to be split into multiple parts
584 test_string
= 'foo'*10000000
586 big_key
= boto
.s3
.key
.Key(bucket
)
587 big_key
.set_contents_from_string(test_string
)
589 # now delete the head
592 # TESTCASE 'rm-user-buckets','user','rm','existing user','fails, still has buckets'
593 (ret
, out
) = rgwadmin_rest(admin_conn
, ['user', 'rm'], {'uid' : user1
})
596 # delete should fail because ``key`` still exists
599 except boto
.exception
.S3ResponseError
as e
:
600 assert e
.status
== 409
605 # TESTCASE 'policy', 'bucket', 'policy', 'get bucket policy', 'returns S3 policy'
606 bucket
= connection
.create_bucket(bucket_name
)
609 key
= boto
.s3
.key
.Key(bucket
)
610 key
.set_contents_from_string('seven')
612 # should be private already but guarantee it
613 key
.set_acl('private')
615 (ret
, out
) = rgwadmin_rest(admin_conn
, ['policy', 'show'], {'bucket' : bucket
.name
, 'object' : key
.key
})
618 acl
= key
.get_xml_acl()
619 assert acl
== out
.strip('\n')
621 # add another grantee by making the object public read
622 key
.set_acl('public-read')
624 (ret
, out
) = rgwadmin_rest(admin_conn
, ['policy', 'show'], {'bucket' : bucket
.name
, 'object' : key
.key
})
627 acl
= key
.get_xml_acl()
628 assert acl
== out
.strip('\n')
630 # TESTCASE 'rm-bucket', 'bucket', 'rm', 'bucket with objects', 'succeeds'
631 bucket
= connection
.create_bucket(bucket_name
)
632 key_name
= ['eight', 'nine', 'ten', 'eleven']
634 key
= boto
.s3
.key
.Key(bucket
)
635 key
.set_contents_from_string(key_name
[i
])
637 (ret
, out
) = rgwadmin_rest(admin_conn
, ['bucket', 'rm'], {'bucket' : bucket_name
, 'purge-objects' : True})
640 # TESTCASE 'caps-add', 'caps', 'add', 'add user cap', 'succeeds'
642 (ret
, out
) = rgwadmin_rest(admin_conn
, ['caps', 'add'], {'uid' : user1
, 'user-caps' : caps
})
644 assert out
[0]['perm'] == 'read'
646 # TESTCASE 'caps-rm', 'caps', 'rm', 'remove existing cap from user', 'succeeds'
647 (ret
, out
) = rgwadmin_rest(admin_conn
, ['caps', 'rm'], {'uid' : user1
, 'user-caps' : caps
})
651 # TESTCASE 'rm-user','user','rm','existing user','fails, still has buckets'
652 bucket
= connection
.create_bucket(bucket_name
)
653 key
= boto
.s3
.key
.Key(bucket
)
655 (ret
, out
) = rgwadmin_rest(admin_conn
, ['user', 'rm'], {'uid' : user1
})
658 # TESTCASE 'rm-user2', 'user', 'rm', user with data', 'succeeds'
659 bucket
= connection
.create_bucket(bucket_name
)
660 key
= boto
.s3
.key
.Key(bucket
)
661 key
.set_contents_from_string('twelve')
663 (ret
, out
) = rgwadmin_rest(admin_conn
, ['user', 'rm'], {'uid' : user1
, 'purge-data' : True})
666 # TESTCASE 'rm-user3','user','info','deleted user','fails'
667 (ret
, out
) = rgwadmin_rest(admin_conn
, ['user', 'info'], {'uid' : user1
})