ceph/qa/tasks/radosgw_admin_rest.py

   1 """
   2 Run a series of rgw admin commands through the rest interface.
   3
   4 The test cases in this file have been annotated for inventory.
   5 To extract the inventory (in csv format) use the command:
   6
   7    grep '^ *# TESTCASE' | sed 's/^ *# TESTCASE //'
   8
   9 """
  10 from cStringIO import StringIO
  11 import logging
  12 import json
  13
  14 import boto.exception
  15 import boto.s3.connection
  16 import boto.s3.acl
  17
  18 import requests
  19 import time
  20
  21 from boto.connection import AWSAuthConnection
  22 from teuthology import misc as teuthology
  23 from util.rgw import get_user_summary, get_user_successful_ops, rgwadmin
  24
  25 log = logging.getLogger(__name__)
  26
  27 def rgwadmin_rest(connection, cmd, params=None, headers=None, raw=False):
  28     """
  29     perform a rest command
  30     """
  31     log.info('radosgw-admin-rest: %s %s' % (cmd, params))
  32     put_cmds = ['create', 'link', 'add']
  33     post_cmds = ['unlink', 'modify']
  34     delete_cmds = ['trim', 'rm', 'process']
  35     get_cmds = ['check', 'info', 'show', 'list']
  36
  37     bucket_sub_resources = ['object', 'policy', 'index']
  38     user_sub_resources = ['subuser', 'key', 'caps']
  39     zone_sub_resources = ['pool', 'log', 'garbage']
  40
  41     def get_cmd_method_and_handler(cmd):
  42         """
  43         Get the rest command and handler from information in cmd and
  44         from the imported requests object.
  45         """
  46         if cmd[1] in put_cmds:
  47             return 'PUT', requests.put
  48         elif cmd[1] in delete_cmds:
  49             return 'DELETE', requests.delete
  50         elif cmd[1] in post_cmds:
  51             return 'POST', requests.post
  52         elif cmd[1] in get_cmds:
  53             return 'GET', requests.get
  54
  55     def get_resource(cmd):
  56         """
  57         Get the name of the resource from information in cmd.
  58         """
  59         if cmd[0] == 'bucket' or cmd[0] in bucket_sub_resources:
  60             if cmd[0] == 'bucket':
  61                 return 'bucket', ''
  62             else:
  63                 return 'bucket', cmd[0]
  64         elif cmd[0] == 'user' or cmd[0] in user_sub_resources:
  65             if cmd[0] == 'user':
  66                 return 'user', ''
  67             else:
  68                 return 'user', cmd[0]
  69         elif cmd[0] == 'usage':
  70             return 'usage', ''
  71         elif cmd[0] == 'zone' or cmd[0] in zone_sub_resources:
  72             if cmd[0] == 'zone':
  73                 return 'zone', ''
  74             else:
  75                 return 'zone', cmd[0]
  76
  77     def build_admin_request(conn, method, resource = '', headers=None, data='',
  78             query_args=None, params=None):
  79         """
  80         Build an administative request adapted from the build_request()
  81         method of boto.connection
  82         """
  83
  84         path = conn.calling_format.build_path_base('admin', resource)
  85         auth_path = conn.calling_format.build_auth_path('admin', resource)
  86         host = conn.calling_format.build_host(conn.server_name(), 'admin')
  87         if query_args:
  88             path += '?' + query_args
  89             boto.log.debug('path=%s' % path)
  90             auth_path += '?' + query_args
  91             boto.log.debug('auth_path=%s' % auth_path)
  92         return AWSAuthConnection.build_base_http_request(conn, method, path,
  93                 auth_path, params, headers, data, host)
  94
  95     method, handler = get_cmd_method_and_handler(cmd)
  96     resource, query_args = get_resource(cmd)
  97     request = build_admin_request(connection, method, resource,
  98             query_args=query_args, headers=headers)
  99
 100     url = '{protocol}://{host}{path}'.format(protocol=request.protocol,
 101             host=request.host, path=request.path)
 102
 103     request.authorize(connection=connection)
 104     result = handler(url, params=params, headers=request.headers)
 105
 106     if raw:
 107         log.info(' text result: %s' % result.text)
 108         return result.status_code, result.text
 109     elif len(result.content) == 0:
 110         # many admin requests return no body, so json() throws a JSONDecodeError
 111         log.info(' empty result')
 112         return result.status_code, None
 113     else:
 114         log.info(' json result: %s' % result.json())
 115         return result.status_code, result.json()
 116
 117
 118 def task(ctx, config):
 119     """
 120     Test radosgw-admin functionality through the RESTful interface
 121     """
 122     assert config is None or isinstance(config, list) \
 123         or isinstance(config, dict), \
 124         "task s3tests only supports a list or dictionary for configuration"
 125     all_clients = ['client.{id}'.format(id=id_)
 126                    for id_ in teuthology.all_roles_of_type(ctx.cluster, 'client')]
 127     if config is None:
 128         config = all_clients
 129     if isinstance(config, list):
 130         config = dict.fromkeys(config)
 131     clients = config.keys()
 132
 133     # just use the first client...
 134     client = clients[0]
 135
 136     ##
 137     admin_user = 'ada'
 138     admin_display_name = 'Ms. Admin User'
 139     admin_access_key = 'MH1WC2XQ1S8UISFDZC8W'
 140     admin_secret_key = 'dQyrTPA0s248YeN5bBv4ukvKU0kh54LWWywkrpoG'
 141     admin_caps = 'users=read, write; usage=read, write; buckets=read, write; zone=read, write'
 142
 143     user1 = 'foo'
 144     user2 = 'fud'
 145     subuser1 = 'foo:foo1'
 146     subuser2 = 'foo:foo2'
 147     display_name1 = 'Foo'
 148     display_name2 = 'Fud'
 149     email = 'foo@foo.com'
 150     access_key = '9te6NH5mcdcq0Tc5i8i1'
 151     secret_key = 'Ny4IOauQoL18Gp2zM7lC1vLmoawgqcYP/YGcWfXu'
 152     access_key2 = 'p5YnriCv1nAtykxBrupQ'
 153     secret_key2 = 'Q8Tk6Q/27hfbFSYdSkPtUqhqx1GgzvpXa4WARozh'
 154     swift_secret1 = 'gpS2G9RREMrnbqlp29PP2D36kgPR1tm72n5fPYfL'
 155     swift_secret2 = 'ri2VJQcKSYATOY6uaDUX7pxgkW+W1YmC6OCxPHwy'
 156
 157     bucket_name = 'myfoo'
 158
 159     # legend (test cases can be easily grep-ed out)
 160     # TESTCASE 'testname','object','method','operation','assertion'
 161     # TESTCASE 'create-admin-user','user','create','administrative user','succeeds'
 162     (err, out) = rgwadmin(ctx, client, [
 163             'user', 'create',
 164             '--uid', admin_user,
 165             '--display-name', admin_display_name,
 166             '--access-key', admin_access_key,
 167             '--secret', admin_secret_key,
 168             '--max-buckets', '0',
 169             '--caps', admin_caps
 170             ])
 171     logging.error(out)
 172     logging.error(err)
 173     assert not err
 174
 175     assert hasattr(ctx, 'rgw'), 'radosgw-admin-rest must run after the rgw task'
 176     endpoint = ctx.rgw.role_endpoints.get(client)
 177     assert endpoint, 'no rgw endpoint for {}'.format(client)
 178
 179     admin_conn = boto.s3.connection.S3Connection(
 180         aws_access_key_id=admin_access_key,
 181         aws_secret_access_key=admin_secret_key,
 182         is_secure=True if endpoint.cert else False,
 183         port=endpoint.port,
 184         host=endpoint.hostname,
 185         calling_format=boto.s3.connection.OrdinaryCallingFormat(),
 186         )
 187
 188     # TESTCASE 'info-nosuch','user','info','non-existent user','fails'
 189     (ret, out) = rgwadmin_rest(admin_conn, ['user', 'info'], {"uid": user1})
 190     assert ret == 404
 191
 192     # TESTCASE 'create-ok','user','create','w/all valid info','succeeds'
 193     (ret, out) = rgwadmin_rest(admin_conn,
 194             ['user', 'create'],
 195             {'uid' : user1,
 196              'display-name' :  display_name1,
 197              'email' : email,
 198              'access-key' : access_key,
 199              'secret-key' : secret_key,
 200              'max-buckets' : '4'
 201             })
 202
 203     assert ret == 200
 204
 205     # TESTCASE 'list-no-user','user','list','list user keys','user list object'
 206     (ret, out) = rgwadmin_rest(admin_conn, ['user', 'list'], {'list' : '', 'max-entries' : 0})
 207     assert ret == 200
 208     assert out['count'] == 0
 209     assert out['truncated'] == True
 210     assert len(out['keys']) == 0
 211     assert len(out['marker']) > 0
 212
 213     # TESTCASE 'list-user-without-marker','user','list','list user keys','user list object'
 214     (ret, out) = rgwadmin_rest(admin_conn, ['user', 'list'], {'list' : '', 'max-entries' : 1})
 215     assert ret == 200
 216     assert out['count'] == 1
 217     assert out['truncated'] == True
 218     assert len(out['keys']) == 1
 219     assert len(out['marker']) > 0
 220     marker = out['marker']
 221
 222     # TESTCASE 'list-user-with-marker','user','list','list user keys','user list object'
 223     (ret, out) = rgwadmin_rest(admin_conn, ['user', 'list'], {'list' : '', 'max-entries' : 1, 'marker': marker})
 224     assert ret == 200
 225     assert out['count'] == 1
 226     assert out['truncated'] == False
 227     assert len(out['keys']) == 1
 228
 229     # TESTCASE 'info-existing','user','info','existing user','returns correct info'
 230     (ret, out) = rgwadmin_rest(admin_conn, ['user', 'info'], {'uid' : user1})
 231
 232     assert out['user_id'] == user1
 233     assert out['email'] == email
 234     assert out['display_name'] == display_name1
 235     assert len(out['keys']) == 1
 236     assert out['keys'][0]['access_key'] == access_key
 237     assert out['keys'][0]['secret_key'] == secret_key
 238     assert not out['suspended']
 239     assert out['tenant'] == ''
 240     assert out['max_buckets'] == 4
 241     assert out['caps'] == []
 242     assert out['op_mask'] == 'read, write, delete'
 243     assert out['default_placement'] == ''
 244     assert out['default_storage_class'] == ''
 245     assert out['placement_tags'] == []
 246     assert not out['bucket_quota']['enabled']
 247     assert not out['bucket_quota']['check_on_raw']
 248     assert out['bucket_quota']['max_size'] == -1
 249     assert out['bucket_quota']['max_size_kb'] == 0
 250     assert out['bucket_quota']['max_objects'] == -1
 251     assert not out['user_quota']['enabled']
 252     assert not out['user_quota']['check_on_raw']
 253     assert out['user_quota']['max_size'] == -1
 254     assert out['user_quota']['max_size_kb'] == 0
 255     assert out['user_quota']['max_objects'] == -1
 256     assert out['temp_url_keys'] == []
 257     assert out['type'] == 'rgw'
 258     assert out['mfa_ids'] == []
 259     # TESTCASE 'info-existing','user','info','existing user query with wrong uid but correct access key','returns correct info'
 260     (ret, out) = rgwadmin_rest(admin_conn, ['user', 'info'], {'access-key' : access_key, 'uid': 'uid_not_exist'})
 261
 262     assert out['user_id'] == user1
 263     assert out['email'] == email
 264     assert out['display_name'] == display_name1
 265     assert len(out['keys']) == 1
 266     assert out['keys'][0]['access_key'] == access_key
 267     assert out['keys'][0]['secret_key'] == secret_key
 268     assert not out['suspended']
 269     assert out['tenant'] == ''
 270     assert out['max_buckets'] == 4
 271     assert out['caps'] == []
 272     assert out['op_mask'] == "read, write, delete"
 273     assert out['default_placement'] == ''
 274     assert out['default_storage_class'] == ''
 275     assert out['placement_tags'] == []
 276     assert not out['bucket_quota']['enabled']
 277     assert not out['bucket_quota']['check_on_raw']
 278     assert out ['bucket_quota']['max_size'] == -1
 279     assert out ['bucket_quota']['max_size_kb'] == 0
 280     assert out ['bucket_quota']['max_objects'] == -1
 281     assert not out['user_quota']['enabled']
 282     assert not out['user_quota']['check_on_raw']
 283     assert out['user_quota']['max_size'] == -1
 284     assert out['user_quota']['max_size_kb'] == 0
 285     assert out['user_quota']['max_objects'] == -1
 286     assert out['temp_url_keys'] == []
 287     assert out['type'] == 'rgw'
 288     assert out['mfa_ids'] == []
 289
 290     # TESTCASE 'suspend-ok','user','suspend','active user','succeeds'
 291     (ret, out) = rgwadmin_rest(admin_conn, ['user', 'modify'], {'uid' : user1, 'suspended' : True})
 292     assert ret == 200
 293
 294     # TESTCASE 'suspend-suspended','user','suspend','suspended user','succeeds w/advisory'
 295     (ret, out) = rgwadmin_rest(admin_conn, ['user', 'info'], {'uid' : user1})
 296     assert ret == 200
 297     assert out['suspended']
 298     assert out['email'] == email
 299
 300     # TESTCASE 're-enable','user','enable','suspended user','succeeds'
 301     (ret, out) = rgwadmin_rest(admin_conn, ['user', 'modify'], {'uid' : user1, 'suspended' : 'false'})
 302     assert not err
 303
 304     # TESTCASE 'info-re-enabled','user','info','re-enabled user','no longer suspended'
 305     (ret, out) = rgwadmin_rest(admin_conn, ['user', 'info'], {'uid' : user1})
 306     assert ret == 200
 307     assert not out['suspended']
 308
 309     # TESTCASE 'add-keys','key','create','w/valid info','succeeds'
 310     (ret, out) = rgwadmin_rest(admin_conn,
 311             ['key', 'create'],
 312             {'uid' : user1,
 313              'access-key' : access_key2,
 314              'secret-key' : secret_key2
 315             })
 316
 317
 318     assert ret == 200
 319
 320     # TESTCASE 'info-new-key','user','info','after key addition','returns all keys'
 321     (ret, out) = rgwadmin_rest(admin_conn, ['user', 'info'], {'uid' : user1})
 322     assert ret == 200
 323     assert len(out['keys']) == 2
 324     assert out['keys'][0]['access_key'] == access_key2 or out['keys'][1]['access_key'] == access_key2
 325     assert out['keys'][0]['secret_key'] == secret_key2 or out['keys'][1]['secret_key'] == secret_key2
 326
 327     # TESTCASE 'rm-key','key','rm','newly added key','succeeds, key is removed'
 328     (ret, out) = rgwadmin_rest(admin_conn,
 329             ['key', 'rm'],
 330             {'uid' : user1,
 331              'access-key' : access_key2
 332             })
 333
 334     assert ret == 200
 335
 336     (ret, out) = rgwadmin_rest(admin_conn, ['user', 'info'], {'uid' : user1})
 337
 338     assert len(out['keys']) == 1
 339     assert out['keys'][0]['access_key'] == access_key
 340     assert out['keys'][0]['secret_key'] == secret_key
 341
 342     # TESTCASE 'add-swift-key','key','create','swift key','succeeds'
 343     (ret, out) = rgwadmin_rest(admin_conn,
 344             ['subuser', 'create'],
 345             {'subuser' : subuser1,
 346              'secret-key' : swift_secret1,
 347              'key-type' : 'swift'
 348             })
 349
 350     assert ret == 200
 351
 352     # TESTCASE 'info-swift-key','user','info','after key addition','returns all keys'
 353     (ret, out) = rgwadmin_rest(admin_conn, ['user', 'info'], {'uid' : user1})
 354     assert ret == 200
 355     assert len(out['swift_keys']) == 1
 356     assert out['swift_keys'][0]['user'] == subuser1
 357     assert out['swift_keys'][0]['secret_key'] == swift_secret1
 358
 359     # TESTCASE 'add-swift-subuser','key','create','swift sub-user key','succeeds'
 360     (ret, out) = rgwadmin_rest(admin_conn,
 361             ['subuser', 'create'],
 362             {'subuser' : subuser2,
 363              'secret-key' : swift_secret2,
 364              'key-type' : 'swift'
 365             })
 366
 367     assert ret == 200
 368
 369     # TESTCASE 'info-swift-subuser','user','info','after key addition','returns all sub-users/keys'
 370     (ret, out) = rgwadmin_rest(admin_conn, ['user', 'info'], {'uid' :  user1})
 371     assert ret == 200
 372     assert len(out['swift_keys']) == 2
 373     assert out['swift_keys'][0]['user'] == subuser2 or out['swift_keys'][1]['user'] == subuser2
 374     assert out['swift_keys'][0]['secret_key'] == swift_secret2 or out['swift_keys'][1]['secret_key'] == swift_secret2
 375
 376     # TESTCASE 'rm-swift-key1','key','rm','subuser','succeeds, one key is removed'
 377     (ret, out) = rgwadmin_rest(admin_conn,
 378             ['key', 'rm'],
 379             {'subuser' : subuser1,
 380              'key-type' :'swift'
 381             })
 382
 383     assert ret == 200
 384
 385     (ret, out) = rgwadmin_rest(admin_conn, ['user', 'info'], {'uid' :  user1})
 386     assert len(out['swift_keys']) == 1
 387
 388     # TESTCASE 'rm-subuser','subuser','rm','subuser','success, subuser is removed'
 389     (ret, out) = rgwadmin_rest(admin_conn,
 390             ['subuser', 'rm'],
 391             {'subuser' : subuser1
 392             })
 393
 394     assert ret == 200
 395
 396     (ret, out) = rgwadmin_rest(admin_conn, ['user', 'info'], {'uid' :  user1})
 397     assert len(out['subusers']) == 1
 398
 399     # TESTCASE 'rm-subuser-with-keys','subuser','rm','subuser','succeeds, second subser and key is removed'
 400     (ret, out) = rgwadmin_rest(admin_conn,
 401             ['subuser', 'rm'],
 402             {'subuser' : subuser2,
 403              'key-type' : 'swift',
 404              '{purge-keys' :True
 405             })
 406
 407     assert ret == 200
 408
 409     (ret, out) = rgwadmin_rest(admin_conn, ['user', 'info'], {'uid' :  user1})
 410     assert len(out['swift_keys']) == 0
 411     assert len(out['subusers']) == 0
 412
 413     # TESTCASE 'bucket-stats','bucket','info','no session/buckets','succeeds, empty list'
 414     (ret, out) = rgwadmin_rest(admin_conn, ['bucket', 'info'], {'uid' :  user1})
 415     assert ret == 200
 416     assert len(out) == 0
 417
 418     # connect to rgw
 419     connection = boto.s3.connection.S3Connection(
 420         aws_access_key_id=access_key,
 421         aws_secret_access_key=secret_key,
 422         is_secure=True if endpoint.cert else False,
 423         port=endpoint.port,
 424         host=endpoint.hostname,
 425         calling_format=boto.s3.connection.OrdinaryCallingFormat(),
 426         )
 427
 428     # TESTCASE 'bucket-stats2','bucket','stats','no buckets','succeeds, empty list'
 429     (ret, out) = rgwadmin_rest(admin_conn, ['bucket', 'info'], {'uid' : user1, 'stats' : True})
 430     assert ret == 200
 431     assert len(out) == 0
 432
 433     # create a first bucket
 434     bucket = connection.create_bucket(bucket_name)
 435
 436     # TESTCASE 'bucket-list','bucket','list','one bucket','succeeds, expected list'
 437     (ret, out) = rgwadmin_rest(admin_conn, ['bucket', 'info'], {'uid' : user1})
 438     assert ret == 200
 439     assert len(out) == 1
 440     assert out[0] == bucket_name
 441
 442     # TESTCASE 'bucket-stats3','bucket','stats','new empty bucket','succeeds, empty list'
 443     (ret, out) = rgwadmin_rest(admin_conn,
 444             ['bucket', 'info'], {'bucket' : bucket_name, 'stats' : True})
 445
 446     assert ret == 200
 447     assert out['owner'] == user1
 448     assert out['tenant'] == ''
 449     bucket_id = out['id']
 450
 451     # TESTCASE 'bucket-stats4','bucket','stats','new empty bucket','succeeds, expected bucket ID'
 452     (ret, out) = rgwadmin_rest(admin_conn, ['bucket', 'info'], {'uid' : user1, 'stats' : True})
 453     assert ret == 200
 454     assert len(out) == 1
 455     assert out[0]['id'] == bucket_id    # does it return the same ID twice in a row?
 456
 457     # use some space
 458     key = boto.s3.key.Key(bucket)
 459     key.set_contents_from_string('one')
 460
 461     # TESTCASE 'bucket-stats5','bucket','stats','after creating key','succeeds, lists one non-empty object'
 462     (ret, out) = rgwadmin_rest(admin_conn, ['bucket', 'info'], {'bucket' : bucket_name, 'stats' : True})
 463     assert ret == 200
 464     assert out['id'] == bucket_id
 465     assert out['usage']['rgw.main']['num_objects'] == 1
 466     assert out['usage']['rgw.main']['size_kb'] > 0
 467
 468     # reclaim it
 469     key.delete()
 470
 471     # TESTCASE 'bucket unlink', 'bucket', 'unlink', 'unlink bucket from user', 'fails', 'access denied error'
 472     (ret, out) = rgwadmin_rest(admin_conn, ['bucket', 'unlink'], {'uid' : user1, 'bucket' : bucket_name})
 473
 474     assert ret == 200
 475
 476     # create a second user to link the bucket to
 477     (ret, out) = rgwadmin_rest(admin_conn,
 478             ['user', 'create'],
 479             {'uid' : user2,
 480             'display-name' :  display_name2,
 481             'access-key' : access_key2,
 482             'secret-key' : secret_key2,
 483             'max-buckets' : '1',
 484             })
 485
 486     assert ret == 200
 487
 488     # try creating an object with the first user before the bucket is relinked
 489     denied = False
 490     key = boto.s3.key.Key(bucket)
 491
 492     try:
 493         key.set_contents_from_string('two')
 494     except boto.exception.S3ResponseError:
 495         denied = True
 496
 497     assert not denied
 498
 499     # delete the object
 500     key.delete()
 501
 502     # link the bucket to another user
 503     (ret, out) = rgwadmin_rest(admin_conn,
 504             ['bucket', 'link'],
 505             {'uid' : user2,
 506              'bucket' : bucket_name,
 507              'bucket-id' : bucket_id,
 508             })
 509
 510     assert ret == 200
 511
 512     # try creating an object with the first user which should cause an error
 513     key = boto.s3.key.Key(bucket)
 514
 515     try:
 516         key.set_contents_from_string('three')
 517     except boto.exception.S3ResponseError:
 518         denied = True
 519
 520     assert denied
 521
 522     # relink the bucket to the first user and delete the second user
 523     (ret, out) = rgwadmin_rest(admin_conn,
 524             ['bucket', 'link'],
 525             {'uid' : user1,
 526              'bucket' : bucket_name,
 527              'bucket-id' : bucket_id,
 528             })
 529     assert ret == 200
 530
 531     (ret, out) = rgwadmin_rest(admin_conn, ['user', 'rm'], {'uid' : user2})
 532     assert ret == 200
 533
 534     # TESTCASE 'object-rm', 'object', 'rm', 'remove object', 'succeeds, object is removed'
 535
 536     # upload an object
 537     object_name = 'four'
 538     key = boto.s3.key.Key(bucket, object_name)
 539     key.set_contents_from_string(object_name)
 540
 541     # now delete it
 542     (ret, out) = rgwadmin_rest(admin_conn, ['object', 'rm'], {'bucket' : bucket_name, 'object' : object_name})
 543     assert ret == 200
 544
 545     # TESTCASE 'bucket-stats6','bucket','stats','after deleting key','succeeds, lists one no objects'
 546     (ret, out) = rgwadmin_rest(admin_conn, ['bucket', 'info'], {'bucket' : bucket_name, 'stats' : True})
 547     assert ret == 200
 548     assert out['id'] == bucket_id
 549     assert out['usage']['rgw.main']['num_objects'] == 0
 550
 551     # create a bucket for deletion stats
 552     useless_bucket = connection.create_bucket('useless_bucket')
 553     useless_key = useless_bucket.new_key('useless_key')
 554     useless_key.set_contents_from_string('useless string')
 555
 556     # delete it
 557     useless_key.delete()
 558     useless_bucket.delete()
 559
 560     # wait for the statistics to flush
 561     time.sleep(60)
 562
 563     # need to wait for all usage data to get flushed, should take up to 30 seconds
 564     timestamp = time.time()
 565     while time.time() - timestamp <= (20 * 60):      # wait up to 20 minutes
 566         (ret, out) = rgwadmin_rest(admin_conn, ['usage', 'show'], {'categories' : 'delete_obj'})  # last operation we did is delete obj, wait for it to flush
 567
 568         if get_user_successful_ops(out, user1) > 0:
 569             break
 570         time.sleep(1)
 571
 572     assert time.time() - timestamp <= (20 * 60)
 573
 574     # TESTCASE 'usage-show' 'usage' 'show' 'all usage' 'succeeds'
 575     (ret, out) = rgwadmin_rest(admin_conn, ['usage', 'show'])
 576     assert ret == 200
 577     assert len(out['entries']) > 0
 578     assert len(out['summary']) > 0
 579     user_summary = get_user_summary(out, user1)
 580     total = user_summary['total']
 581     assert total['successful_ops'] > 0
 582
 583     # TESTCASE 'usage-show2' 'usage' 'show' 'user usage' 'succeeds'
 584     (ret, out) = rgwadmin_rest(admin_conn, ['usage', 'show'], {'uid' : user1})
 585     assert ret == 200
 586     assert len(out['entries']) > 0
 587     assert len(out['summary']) > 0
 588     user_summary = out['summary'][0]
 589     for entry in user_summary['categories']:
 590         assert entry['successful_ops'] > 0
 591     assert user_summary['user'] == user1
 592
 593     # TESTCASE 'usage-show3' 'usage' 'show' 'user usage categories' 'succeeds'
 594     test_categories = ['create_bucket', 'put_obj', 'delete_obj', 'delete_bucket']
 595     for cat in test_categories:
 596         (ret, out) = rgwadmin_rest(admin_conn, ['usage', 'show'], {'uid' : user1, 'categories' : cat})
 597         assert ret == 200
 598         assert len(out['summary']) > 0
 599         user_summary = out['summary'][0]
 600         assert user_summary['user'] == user1
 601         assert len(user_summary['categories']) == 1
 602         entry = user_summary['categories'][0]
 603         assert entry['category'] == cat
 604         assert entry['successful_ops'] > 0
 605
 606     # TESTCASE 'usage-trim' 'usage' 'trim' 'user usage' 'succeeds, usage removed'
 607     (ret, out) = rgwadmin_rest(admin_conn, ['usage', 'trim'], {'uid' : user1})
 608     assert ret == 200
 609     (ret, out) = rgwadmin_rest(admin_conn, ['usage', 'show'], {'uid' : user1})
 610     assert ret == 200
 611     assert len(out['entries']) == 0
 612     assert len(out['summary']) == 0
 613
 614     # TESTCASE 'user-suspend2','user','suspend','existing user','succeeds'
 615     (ret, out) = rgwadmin_rest(admin_conn, ['user', 'modify'], {'uid' : user1, 'suspended' : True})
 616     assert ret == 200
 617
 618     # TESTCASE 'user-suspend3','user','suspend','suspended user','cannot write objects'
 619     try:
 620         key = boto.s3.key.Key(bucket)
 621         key.set_contents_from_string('five')
 622     except boto.exception.S3ResponseError as e:
 623         assert e.status == 403
 624
 625     # TESTCASE 'user-renable2','user','enable','suspended user','succeeds'
 626     (ret, out) = rgwadmin_rest(admin_conn, ['user', 'modify'], {'uid' :  user1, 'suspended' : 'false'})
 627     assert ret == 200
 628
 629     # TESTCASE 'user-renable3','user','enable','reenabled user','can write objects'
 630     key = boto.s3.key.Key(bucket)
 631     key.set_contents_from_string('six')
 632
 633     # TESTCASE 'garbage-list', 'garbage', 'list', 'get list of objects ready for garbage collection'
 634
 635     # create an object large enough to be split into multiple parts
 636     test_string = 'foo'*10000000
 637
 638     big_key = boto.s3.key.Key(bucket)
 639     big_key.set_contents_from_string(test_string)
 640
 641     # now delete the head
 642     big_key.delete()
 643
 644     # TESTCASE 'rm-user-buckets','user','rm','existing user','fails, still has buckets'
 645     (ret, out) = rgwadmin_rest(admin_conn, ['user', 'rm'], {'uid' : user1})
 646     assert ret == 409
 647
 648     # delete should fail because ``key`` still exists
 649     try:
 650         bucket.delete()
 651     except boto.exception.S3ResponseError as e:
 652         assert e.status == 409
 653
 654     key.delete()
 655     bucket.delete()
 656
 657     # TESTCASE 'policy', 'bucket', 'policy', 'get bucket policy', 'returns S3 policy'
 658     bucket = connection.create_bucket(bucket_name)
 659
 660     # create an object
 661     key = boto.s3.key.Key(bucket)
 662     key.set_contents_from_string('seven')
 663
 664     # should be private already but guarantee it
 665     key.set_acl('private')
 666
 667     (ret, out) = rgwadmin_rest(admin_conn, ['policy', 'show'], {'bucket' : bucket.name, 'object' : key.key})
 668     assert ret == 200
 669     assert len(out['acl']['grant_map']) == 1
 670
 671     # add another grantee by making the object public read
 672     key.set_acl('public-read')
 673
 674     (ret, out) = rgwadmin_rest(admin_conn, ['policy', 'show'], {'bucket' : bucket.name, 'object' : key.key})
 675     assert ret == 200
 676     assert len(out['acl']['grant_map']) == 2
 677
 678     # TESTCASE 'rm-bucket', 'bucket', 'rm', 'bucket with objects', 'succeeds'
 679     bucket = connection.create_bucket(bucket_name)
 680     key_name = ['eight', 'nine', 'ten', 'eleven']
 681     for i in range(4):
 682         key = boto.s3.key.Key(bucket)
 683         key.set_contents_from_string(key_name[i])
 684
 685     (ret, out) = rgwadmin_rest(admin_conn, ['bucket', 'rm'], {'bucket' : bucket_name, 'purge-objects' : True})
 686     assert ret == 200
 687
 688     # TESTCASE 'caps-add', 'caps', 'add', 'add user cap', 'succeeds'
 689     caps = 'usage=read'
 690     (ret, out) = rgwadmin_rest(admin_conn, ['caps', 'add'], {'uid' :  user1, 'user-caps' : caps})
 691     assert ret == 200
 692     assert out[0]['perm'] == 'read'
 693
 694     # TESTCASE 'caps-rm', 'caps', 'rm', 'remove existing cap from user', 'succeeds'
 695     (ret, out) = rgwadmin_rest(admin_conn, ['caps', 'rm'], {'uid' :  user1, 'user-caps' : caps})
 696     assert ret == 200
 697     assert not out
 698
 699     # TESTCASE 'rm-user','user','rm','existing user','fails, still has buckets'
 700     bucket = connection.create_bucket(bucket_name)
 701     key = boto.s3.key.Key(bucket)
 702
 703     (ret, out) = rgwadmin_rest(admin_conn, ['user', 'rm'], {'uid' : user1})
 704     assert ret == 409
 705
 706     # TESTCASE 'rm-user2', 'user', 'rm', user with data', 'succeeds'
 707     bucket = connection.create_bucket(bucket_name)
 708     key = boto.s3.key.Key(bucket)
 709     key.set_contents_from_string('twelve')
 710
 711     (ret, out) = rgwadmin_rest(admin_conn, ['user', 'rm'], {'uid' : user1, 'purge-data' : True})
 712     assert ret == 200
 713
 714     # TESTCASE 'rm-user3','user','info','deleted user','fails'
 715     (ret, out) = rgwadmin_rest(admin_conn, ['user', 'info'], {'uid' :  user1})
 716     assert ret == 404
 717