ceph/qa/tasks/radosgw_admin_rest.py

   1 """
   2 Run a series of rgw admin commands through the rest interface.
   3
   4 The test cases in this file have been annotated for inventory.
   5 To extract the inventory (in csv format) use the command:
   6
   7    grep '^ *# TESTCASE' | sed 's/^ *# TESTCASE //'
   8
   9 """
  10 import logging
  11
  12
  13 import boto.exception
  14 import boto.s3.connection
  15 import boto.s3.acl
  16
  17 import requests
  18 import time
  19
  20 from boto.connection import AWSAuthConnection
  21 from teuthology import misc as teuthology
  22 from tasks.util.rgw import get_user_summary, get_user_successful_ops, rgwadmin
  23
  24 log = logging.getLogger(__name__)
  25
  26 def rgwadmin_rest(connection, cmd, params=None, headers=None, raw=False):
  27     """
  28     perform a rest command
  29     """
  30     log.info('radosgw-admin-rest: %s %s' % (cmd, params))
  31     put_cmds = ['create', 'link', 'add']
  32     post_cmds = ['unlink', 'modify']
  33     delete_cmds = ['trim', 'rm', 'process']
  34     get_cmds = ['check', 'info', 'show', 'list']
  35
  36     bucket_sub_resources = ['object', 'policy', 'index']
  37     user_sub_resources = ['subuser', 'key', 'caps']
  38     zone_sub_resources = ['pool', 'log', 'garbage']
  39
  40     def get_cmd_method_and_handler(cmd):
  41         """
  42         Get the rest command and handler from information in cmd and
  43         from the imported requests object.
  44         """
  45         if cmd[1] in put_cmds:
  46             return 'PUT', requests.put
  47         elif cmd[1] in delete_cmds:
  48             return 'DELETE', requests.delete
  49         elif cmd[1] in post_cmds:
  50             return 'POST', requests.post
  51         elif cmd[1] in get_cmds:
  52             return 'GET', requests.get
  53
  54     def get_resource(cmd):
  55         """
  56         Get the name of the resource from information in cmd.
  57         """
  58         if cmd[0] == 'bucket' or cmd[0] in bucket_sub_resources:
  59             if cmd[0] == 'bucket':
  60                 return 'bucket', ''
  61             else:
  62                 return 'bucket', cmd[0]
  63         elif cmd[0] == 'user' or cmd[0] in user_sub_resources:
  64             if cmd[0] == 'user':
  65                 return 'user', ''
  66             else:
  67                 return 'user', cmd[0]
  68         elif cmd[0] == 'usage':
  69             return 'usage', ''
  70         elif cmd[0] == 'zone' or cmd[0] in zone_sub_resources:
  71             if cmd[0] == 'zone':
  72                 return 'zone', ''
  73             else:
  74                 return 'zone', cmd[0]
  75
  76     def build_admin_request(conn, method, resource = '', headers=None, data='',
  77             query_args=None, params=None):
  78         """
  79         Build an administative request adapted from the build_request()
  80         method of boto.connection
  81         """
  82
  83         path = conn.calling_format.build_path_base('admin', resource)
  84         auth_path = conn.calling_format.build_auth_path('admin', resource)
  85         host = conn.calling_format.build_host(conn.server_name(), 'admin')
  86         if query_args:
  87             path += '?' + query_args
  88             boto.log.debug('path=%s' % path)
  89             auth_path += '?' + query_args
  90             boto.log.debug('auth_path=%s' % auth_path)
  91         return AWSAuthConnection.build_base_http_request(conn, method, path,
  92                 auth_path, params, headers, data, host)
  93
  94     method, handler = get_cmd_method_and_handler(cmd)
  95     resource, query_args = get_resource(cmd)
  96     request = build_admin_request(connection, method, resource,
  97             query_args=query_args, headers=headers)
  98
  99     url = '{protocol}://{host}{path}'.format(protocol=request.protocol,
 100             host=request.host, path=request.path)
 101
 102     request.authorize(connection=connection)
 103     result = handler(url, params=params, headers=request.headers)
 104
 105     if raw:
 106         log.info(' text result: %s' % result.text)
 107         return result.status_code, result.text
 108     elif len(result.content) == 0:
 109         # many admin requests return no body, so json() throws a JSONDecodeError
 110         log.info(' empty result')
 111         return result.status_code, None
 112     else:
 113         log.info(' json result: %s' % result.json())
 114         return result.status_code, result.json()
 115
 116
 117 def task(ctx, config):
 118     """
 119     Test radosgw-admin functionality through the RESTful interface
 120     """
 121     assert config is None or isinstance(config, list) \
 122         or isinstance(config, dict), \
 123         "task s3tests only supports a list or dictionary for configuration"
 124     all_clients = ['client.{id}'.format(id=id_)
 125                    for id_ in teuthology.all_roles_of_type(ctx.cluster, 'client')]
 126     if config is None:
 127         config = all_clients
 128     if isinstance(config, list):
 129         config = dict.fromkeys(config)
 130     clients = config.keys()
 131
 132     # just use the first client...
 133     client = next(iter(clients))
 134
 135     ##
 136     admin_user = 'ada'
 137     admin_display_name = 'Ms. Admin User'
 138     admin_access_key = 'MH1WC2XQ1S8UISFDZC8W'
 139     admin_secret_key = 'dQyrTPA0s248YeN5bBv4ukvKU0kh54LWWywkrpoG'
 140     admin_caps = 'users=read, write; usage=read, write; buckets=read, write; zone=read, write'
 141
 142     user1 = 'foo'
 143     user2 = 'fud'
 144     subuser1 = 'foo:foo1'
 145     subuser2 = 'foo:foo2'
 146     display_name1 = 'Foo'
 147     display_name2 = 'Fud'
 148     email = 'foo@foo.com'
 149     access_key = '9te6NH5mcdcq0Tc5i8i1'
 150     secret_key = 'Ny4IOauQoL18Gp2zM7lC1vLmoawgqcYP/YGcWfXu'
 151     access_key2 = 'p5YnriCv1nAtykxBrupQ'
 152     secret_key2 = 'Q8Tk6Q/27hfbFSYdSkPtUqhqx1GgzvpXa4WARozh'
 153     swift_secret1 = 'gpS2G9RREMrnbqlp29PP2D36kgPR1tm72n5fPYfL'
 154     swift_secret2 = 'ri2VJQcKSYATOY6uaDUX7pxgkW+W1YmC6OCxPHwy'
 155
 156     bucket_name = 'myfoo'
 157
 158     # legend (test cases can be easily grep-ed out)
 159     # TESTCASE 'testname','object','method','operation','assertion'
 160     # TESTCASE 'create-admin-user','user','create','administrative user','succeeds'
 161     (err, out) = rgwadmin(ctx, client, [
 162             'user', 'create',
 163             '--uid', admin_user,
 164             '--display-name', admin_display_name,
 165             '--access-key', admin_access_key,
 166             '--secret', admin_secret_key,
 167             '--max-buckets', '0',
 168             '--caps', admin_caps
 169             ])
 170     logging.error(out)
 171     logging.error(err)
 172     assert not err
 173
 174     assert hasattr(ctx, 'rgw'), 'radosgw-admin-rest must run after the rgw task'
 175     endpoint = ctx.rgw.role_endpoints.get(client)
 176     assert endpoint, 'no rgw endpoint for {}'.format(client)
 177
 178     admin_conn = boto.s3.connection.S3Connection(
 179         aws_access_key_id=admin_access_key,
 180         aws_secret_access_key=admin_secret_key,
 181         is_secure=True if endpoint.cert else False,
 182         port=endpoint.port,
 183         host=endpoint.hostname,
 184         calling_format=boto.s3.connection.OrdinaryCallingFormat(),
 185         )
 186
 187     # TESTCASE 'info-nosuch','user','info','non-existent user','fails'
 188     (ret, out) = rgwadmin_rest(admin_conn, ['user', 'info'], {"uid": user1})
 189     assert ret == 404
 190
 191     # TESTCASE 'create-ok','user','create','w/all valid info','succeeds'
 192     (ret, out) = rgwadmin_rest(admin_conn,
 193             ['user', 'create'],
 194             {'uid' : user1,
 195              'display-name' :  display_name1,
 196              'email' : email,
 197              'access-key' : access_key,
 198              'secret-key' : secret_key,
 199              'max-buckets' : '4'
 200             })
 201
 202     assert ret == 200
 203
 204     # TESTCASE 'list-no-user','user','list','list user keys','user list object'
 205     (ret, out) = rgwadmin_rest(admin_conn, ['user', 'list'], {'list' : '', 'max-entries' : 0})
 206     assert ret == 200
 207     assert out['count'] == 0
 208     assert out['truncated'] == True
 209     assert len(out['keys']) == 0
 210     assert len(out['marker']) > 0
 211
 212     # TESTCASE 'list-user-without-marker','user','list','list user keys','user list object'
 213     (ret, out) = rgwadmin_rest(admin_conn, ['user', 'list'], {'list' : '', 'max-entries' : 1})
 214     assert ret == 200
 215     assert out['count'] == 1
 216     assert out['truncated'] == True
 217     assert len(out['keys']) == 1
 218     assert len(out['marker']) > 0
 219     marker = out['marker']
 220
 221     # TESTCASE 'list-user-with-marker','user','list','list user keys','user list object'
 222     (ret, out) = rgwadmin_rest(admin_conn, ['user', 'list'], {'list' : '', 'max-entries' : 1, 'marker': marker})
 223     assert ret == 200
 224     assert out['count'] == 1
 225     assert out['truncated'] == False
 226     assert len(out['keys']) == 1
 227
 228     # TESTCASE 'info-existing','user','info','existing user','returns correct info'
 229     (ret, out) = rgwadmin_rest(admin_conn, ['user', 'info'], {'uid' : user1})
 230
 231     assert out['user_id'] == user1
 232     assert out['email'] == email
 233     assert out['display_name'] == display_name1
 234     assert len(out['keys']) == 1
 235     assert out['keys'][0]['access_key'] == access_key
 236     assert out['keys'][0]['secret_key'] == secret_key
 237     assert not out['suspended']
 238     assert out['tenant'] == ''
 239     assert out['max_buckets'] == 4
 240     assert out['caps'] == []
 241     assert out['op_mask'] == 'read, write, delete'
 242     assert out['default_placement'] == ''
 243     assert out['default_storage_class'] == ''
 244     assert out['placement_tags'] == []
 245     assert not out['bucket_quota']['enabled']
 246     assert not out['bucket_quota']['check_on_raw']
 247     assert out['bucket_quota']['max_size'] == -1
 248     assert out['bucket_quota']['max_size_kb'] == 0
 249     assert out['bucket_quota']['max_objects'] == -1
 250     assert not out['user_quota']['enabled']
 251     assert not out['user_quota']['check_on_raw']
 252     assert out['user_quota']['max_size'] == -1
 253     assert out['user_quota']['max_size_kb'] == 0
 254     assert out['user_quota']['max_objects'] == -1
 255     assert out['temp_url_keys'] == []
 256     assert out['type'] == 'rgw'
 257     assert out['mfa_ids'] == []
 258     # TESTCASE 'info-existing','user','info','existing user query with wrong uid but correct access key','returns correct info'
 259     (ret, out) = rgwadmin_rest(admin_conn, ['user', 'info'], {'access-key' : access_key, 'uid': 'uid_not_exist'})
 260
 261     assert out['user_id'] == user1
 262     assert out['email'] == email
 263     assert out['display_name'] == display_name1
 264     assert len(out['keys']) == 1
 265     assert out['keys'][0]['access_key'] == access_key
 266     assert out['keys'][0]['secret_key'] == secret_key
 267     assert not out['suspended']
 268     assert out['tenant'] == ''
 269     assert out['max_buckets'] == 4
 270     assert out['caps'] == []
 271     assert out['op_mask'] == "read, write, delete"
 272     assert out['default_placement'] == ''
 273     assert out['default_storage_class'] == ''
 274     assert out['placement_tags'] == []
 275     assert not out['bucket_quota']['enabled']
 276     assert not out['bucket_quota']['check_on_raw']
 277     assert out ['bucket_quota']['max_size'] == -1
 278     assert out ['bucket_quota']['max_size_kb'] == 0
 279     assert out ['bucket_quota']['max_objects'] == -1
 280     assert not out['user_quota']['enabled']
 281     assert not out['user_quota']['check_on_raw']
 282     assert out['user_quota']['max_size'] == -1
 283     assert out['user_quota']['max_size_kb'] == 0
 284     assert out['user_quota']['max_objects'] == -1
 285     assert out['temp_url_keys'] == []
 286     assert out['type'] == 'rgw'
 287     assert out['mfa_ids'] == []
 288
 289     # TESTCASE 'suspend-ok','user','suspend','active user','succeeds'
 290     (ret, out) = rgwadmin_rest(admin_conn, ['user', 'modify'], {'uid' : user1, 'suspended' : True})
 291     assert ret == 200
 292
 293     # TESTCASE 'suspend-suspended','user','suspend','suspended user','succeeds w/advisory'
 294     (ret, out) = rgwadmin_rest(admin_conn, ['user', 'info'], {'uid' : user1})
 295     assert ret == 200
 296     assert out['suspended']
 297     assert out['email'] == email
 298
 299     # TESTCASE 're-enable','user','enable','suspended user','succeeds'
 300     (ret, out) = rgwadmin_rest(admin_conn, ['user', 'modify'], {'uid' : user1, 'suspended' : 'false'})
 301     assert not err
 302
 303     # TESTCASE 'info-re-enabled','user','info','re-enabled user','no longer suspended'
 304     (ret, out) = rgwadmin_rest(admin_conn, ['user', 'info'], {'uid' : user1})
 305     assert ret == 200
 306     assert not out['suspended']
 307
 308     # TESTCASE 'add-keys','key','create','w/valid info','succeeds'
 309     (ret, out) = rgwadmin_rest(admin_conn,
 310             ['key', 'create'],
 311             {'uid' : user1,
 312              'access-key' : access_key2,
 313              'secret-key' : secret_key2
 314             })
 315
 316
 317     assert ret == 200
 318
 319     # TESTCASE 'info-new-key','user','info','after key addition','returns all keys'
 320     (ret, out) = rgwadmin_rest(admin_conn, ['user', 'info'], {'uid' : user1})
 321     assert ret == 200
 322     assert len(out['keys']) == 2
 323     assert out['keys'][0]['access_key'] == access_key2 or out['keys'][1]['access_key'] == access_key2
 324     assert out['keys'][0]['secret_key'] == secret_key2 or out['keys'][1]['secret_key'] == secret_key2
 325
 326     # TESTCASE 'rm-key','key','rm','newly added key','succeeds, key is removed'
 327     (ret, out) = rgwadmin_rest(admin_conn,
 328             ['key', 'rm'],
 329             {'uid' : user1,
 330              'access-key' : access_key2
 331             })
 332
 333     assert ret == 200
 334
 335     (ret, out) = rgwadmin_rest(admin_conn, ['user', 'info'], {'uid' : user1})
 336
 337     assert len(out['keys']) == 1
 338     assert out['keys'][0]['access_key'] == access_key
 339     assert out['keys'][0]['secret_key'] == secret_key
 340
 341     # TESTCASE 'add-swift-key','key','create','swift key','succeeds'
 342     (ret, out) = rgwadmin_rest(admin_conn,
 343             ['subuser', 'create'],
 344             {'subuser' : subuser1,
 345              'secret-key' : swift_secret1,
 346              'key-type' : 'swift'
 347             })
 348
 349     assert ret == 200
 350
 351     # TESTCASE 'info-swift-key','user','info','after key addition','returns all keys'
 352     (ret, out) = rgwadmin_rest(admin_conn, ['user', 'info'], {'uid' : user1})
 353     assert ret == 200
 354     assert len(out['swift_keys']) == 1
 355     assert out['swift_keys'][0]['user'] == subuser1
 356     assert out['swift_keys'][0]['secret_key'] == swift_secret1
 357
 358     # TESTCASE 'add-swift-subuser','key','create','swift sub-user key','succeeds'
 359     (ret, out) = rgwadmin_rest(admin_conn,
 360             ['subuser', 'create'],
 361             {'subuser' : subuser2,
 362              'secret-key' : swift_secret2,
 363              'key-type' : 'swift'
 364             })
 365
 366     assert ret == 200
 367
 368     # TESTCASE 'info-swift-subuser','user','info','after key addition','returns all sub-users/keys'
 369     (ret, out) = rgwadmin_rest(admin_conn, ['user', 'info'], {'uid' :  user1})
 370     assert ret == 200
 371     assert len(out['swift_keys']) == 2
 372     assert out['swift_keys'][0]['user'] == subuser2 or out['swift_keys'][1]['user'] == subuser2
 373     assert out['swift_keys'][0]['secret_key'] == swift_secret2 or out['swift_keys'][1]['secret_key'] == swift_secret2
 374
 375     # TESTCASE 'rm-swift-key1','key','rm','subuser','succeeds, one key is removed'
 376     (ret, out) = rgwadmin_rest(admin_conn,
 377             ['key', 'rm'],
 378             {'subuser' : subuser1,
 379              'key-type' :'swift'
 380             })
 381
 382     assert ret == 200
 383
 384     (ret, out) = rgwadmin_rest(admin_conn, ['user', 'info'], {'uid' :  user1})
 385     assert len(out['swift_keys']) == 1
 386
 387     # TESTCASE 'rm-subuser','subuser','rm','subuser','success, subuser is removed'
 388     (ret, out) = rgwadmin_rest(admin_conn,
 389             ['subuser', 'rm'],
 390             {'subuser' : subuser1
 391             })
 392
 393     assert ret == 200
 394
 395     (ret, out) = rgwadmin_rest(admin_conn, ['user', 'info'], {'uid' :  user1})
 396     assert len(out['subusers']) == 1
 397
 398     # TESTCASE 'rm-subuser-with-keys','subuser','rm','subuser','succeeds, second subser and key is removed'
 399     (ret, out) = rgwadmin_rest(admin_conn,
 400             ['subuser', 'rm'],
 401             {'subuser' : subuser2,
 402              'key-type' : 'swift',
 403              '{purge-keys' :True
 404             })
 405
 406     assert ret == 200
 407
 408     (ret, out) = rgwadmin_rest(admin_conn, ['user', 'info'], {'uid' :  user1})
 409     assert len(out['swift_keys']) == 0
 410     assert len(out['subusers']) == 0
 411
 412     # TESTCASE 'bucket-stats','bucket','info','no session/buckets','succeeds, empty list'
 413     (ret, out) = rgwadmin_rest(admin_conn, ['bucket', 'info'], {'uid' :  user1})
 414     assert ret == 200
 415     assert len(out) == 0
 416
 417     # connect to rgw
 418     connection = boto.s3.connection.S3Connection(
 419         aws_access_key_id=access_key,
 420         aws_secret_access_key=secret_key,
 421         is_secure=True if endpoint.cert else False,
 422         port=endpoint.port,
 423         host=endpoint.hostname,
 424         calling_format=boto.s3.connection.OrdinaryCallingFormat(),
 425         )
 426
 427     # TESTCASE 'bucket-stats2','bucket','stats','no buckets','succeeds, empty list'
 428     (ret, out) = rgwadmin_rest(admin_conn, ['bucket', 'info'], {'uid' : user1, 'stats' : True})
 429     assert ret == 200
 430     assert len(out) == 0
 431
 432     # create a first bucket
 433     bucket = connection.create_bucket(bucket_name)
 434
 435     # TESTCASE 'bucket-list','bucket','list','one bucket','succeeds, expected list'
 436     (ret, out) = rgwadmin_rest(admin_conn, ['bucket', 'info'], {'uid' : user1})
 437     assert ret == 200
 438     assert len(out) == 1
 439     assert out[0] == bucket_name
 440
 441     # TESTCASE 'bucket-stats3','bucket','stats','new empty bucket','succeeds, empty list'
 442     (ret, out) = rgwadmin_rest(admin_conn,
 443             ['bucket', 'info'], {'bucket' : bucket_name, 'stats' : True})
 444
 445     assert ret == 200
 446     assert out['owner'] == user1
 447     assert out['tenant'] == ''
 448     bucket_id = out['id']
 449
 450     # TESTCASE 'bucket-stats4','bucket','stats','new empty bucket','succeeds, expected bucket ID'
 451     (ret, out) = rgwadmin_rest(admin_conn, ['bucket', 'info'], {'uid' : user1, 'stats' : True})
 452     assert ret == 200
 453     assert len(out) == 1
 454     assert out[0]['id'] == bucket_id    # does it return the same ID twice in a row?
 455
 456     # use some space
 457     key = boto.s3.key.Key(bucket)
 458     key.set_contents_from_string('one')
 459
 460     # TESTCASE 'bucket-stats5','bucket','stats','after creating key','succeeds, lists one non-empty object'
 461     (ret, out) = rgwadmin_rest(admin_conn, ['bucket', 'info'], {'bucket' : bucket_name, 'stats' : True})
 462     assert ret == 200
 463     assert out['id'] == bucket_id
 464     assert out['usage']['rgw.main']['num_objects'] == 1
 465     assert out['usage']['rgw.main']['size_kb'] > 0
 466
 467     # TESTCASE 'bucket-stats6', 'bucket', 'stats', 'non-existent bucket', 'fails, 'bucket not found error'
 468     (ret, out) = rgwadmin_rest(admin_conn, ['bucket', 'info'], {'bucket' : 'doesnotexist'})
 469     assert ret == 404
 470     assert out['Code'] == 'NoSuchBucket'
 471
 472     # reclaim it
 473     key.delete()
 474
 475     # TESTCASE 'bucket unlink', 'bucket', 'unlink', 'unlink bucket from user', 'fails', 'access denied error'
 476     (ret, out) = rgwadmin_rest(admin_conn, ['bucket', 'unlink'], {'uid' : user1, 'bucket' : bucket_name})
 477
 478     assert ret == 200
 479
 480     # create a second user to link the bucket to
 481     (ret, out) = rgwadmin_rest(admin_conn,
 482             ['user', 'create'],
 483             {'uid' : user2,
 484             'display-name' :  display_name2,
 485             'access-key' : access_key2,
 486             'secret-key' : secret_key2,
 487             'max-buckets' : '1',
 488             })
 489
 490     assert ret == 200
 491
 492     # try creating an object with the first user before the bucket is relinked
 493     denied = False
 494     key = boto.s3.key.Key(bucket)
 495
 496     try:
 497         key.set_contents_from_string('two')
 498     except boto.exception.S3ResponseError:
 499         denied = True
 500
 501     assert not denied
 502
 503     # delete the object
 504     key.delete()
 505
 506     # link the bucket to another user
 507     (ret, out) = rgwadmin_rest(admin_conn,
 508             ['bucket', 'link'],
 509             {'uid' : user2,
 510              'bucket' : bucket_name,
 511              'bucket-id' : bucket_id,
 512             })
 513
 514     assert ret == 200
 515
 516     # try creating an object with the first user which should cause an error
 517     key = boto.s3.key.Key(bucket)
 518
 519     try:
 520         key.set_contents_from_string('three')
 521     except boto.exception.S3ResponseError:
 522         denied = True
 523
 524     assert denied
 525
 526     # relink the bucket to the first user and delete the second user
 527     (ret, out) = rgwadmin_rest(admin_conn,
 528             ['bucket', 'link'],
 529             {'uid' : user1,
 530              'bucket' : bucket_name,
 531              'bucket-id' : bucket_id,
 532             })
 533     assert ret == 200
 534
 535     (ret, out) = rgwadmin_rest(admin_conn, ['user', 'rm'], {'uid' : user2})
 536     assert ret == 200
 537
 538     # TESTCASE 'object-rm', 'object', 'rm', 'remove object', 'succeeds, object is removed'
 539
 540     # upload an object
 541     object_name = 'four'
 542     key = boto.s3.key.Key(bucket, object_name)
 543     key.set_contents_from_string(object_name)
 544
 545     # now delete it
 546     (ret, out) = rgwadmin_rest(admin_conn, ['object', 'rm'], {'bucket' : bucket_name, 'object' : object_name})
 547     assert ret == 200
 548
 549     # TESTCASE 'bucket-stats6','bucket','stats','after deleting key','succeeds, lists one no objects'
 550     (ret, out) = rgwadmin_rest(admin_conn, ['bucket', 'info'], {'bucket' : bucket_name, 'stats' : True})
 551     assert ret == 200
 552     assert out['id'] == bucket_id
 553     assert out['usage']['rgw.main']['num_objects'] == 0
 554
 555     # create a bucket for deletion stats
 556     useless_bucket = connection.create_bucket('useless-bucket')
 557     useless_key = useless_bucket.new_key('useless_key')
 558     useless_key.set_contents_from_string('useless string')
 559
 560     # delete it
 561     useless_key.delete()
 562     useless_bucket.delete()
 563
 564     # wait for the statistics to flush
 565     time.sleep(60)
 566
 567     # need to wait for all usage data to get flushed, should take up to 30 seconds
 568     timestamp = time.time()
 569     while time.time() - timestamp <= (20 * 60):      # wait up to 20 minutes
 570         (ret, out) = rgwadmin_rest(admin_conn, ['usage', 'show'], {'categories' : 'delete_obj'})  # last operation we did is delete obj, wait for it to flush
 571
 572         if get_user_successful_ops(out, user1) > 0:
 573             break
 574         time.sleep(1)
 575
 576     assert time.time() - timestamp <= (20 * 60)
 577
 578     # TESTCASE 'usage-show' 'usage' 'show' 'all usage' 'succeeds'
 579     (ret, out) = rgwadmin_rest(admin_conn, ['usage', 'show'])
 580     assert ret == 200
 581     assert len(out['entries']) > 0
 582     assert len(out['summary']) > 0
 583     user_summary = get_user_summary(out, user1)
 584     total = user_summary['total']
 585     assert total['successful_ops'] > 0
 586
 587     # TESTCASE 'usage-show2' 'usage' 'show' 'user usage' 'succeeds'
 588     (ret, out) = rgwadmin_rest(admin_conn, ['usage', 'show'], {'uid' : user1})
 589     assert ret == 200
 590     assert len(out['entries']) > 0
 591     assert len(out['summary']) > 0
 592     user_summary = out['summary'][0]
 593     for entry in user_summary['categories']:
 594         assert entry['successful_ops'] > 0
 595     assert user_summary['user'] == user1
 596
 597     # TESTCASE 'usage-show3' 'usage' 'show' 'user usage categories' 'succeeds'
 598     test_categories = ['create_bucket', 'put_obj', 'delete_obj', 'delete_bucket']
 599     for cat in test_categories:
 600         (ret, out) = rgwadmin_rest(admin_conn, ['usage', 'show'], {'uid' : user1, 'categories' : cat})
 601         assert ret == 200
 602         assert len(out['summary']) > 0
 603         user_summary = out['summary'][0]
 604         assert user_summary['user'] == user1
 605         assert len(user_summary['categories']) == 1
 606         entry = user_summary['categories'][0]
 607         assert entry['category'] == cat
 608         assert entry['successful_ops'] > 0
 609
 610     # TESTCASE 'usage-trim' 'usage' 'trim' 'user usage' 'succeeds, usage removed'
 611     (ret, out) = rgwadmin_rest(admin_conn, ['usage', 'trim'], {'uid' : user1})
 612     assert ret == 200
 613     (ret, out) = rgwadmin_rest(admin_conn, ['usage', 'show'], {'uid' : user1})
 614     assert ret == 200
 615     assert len(out['entries']) == 0
 616     assert len(out['summary']) == 0
 617
 618     # TESTCASE 'user-suspend2','user','suspend','existing user','succeeds'
 619     (ret, out) = rgwadmin_rest(admin_conn, ['user', 'modify'], {'uid' : user1, 'suspended' : True})
 620     assert ret == 200
 621
 622     # TESTCASE 'user-suspend3','user','suspend','suspended user','cannot write objects'
 623     try:
 624         key = boto.s3.key.Key(bucket)
 625         key.set_contents_from_string('five')
 626     except boto.exception.S3ResponseError as e:
 627         assert e.status == 403
 628
 629     # TESTCASE 'user-renable2','user','enable','suspended user','succeeds'
 630     (ret, out) = rgwadmin_rest(admin_conn, ['user', 'modify'], {'uid' :  user1, 'suspended' : 'false'})
 631     assert ret == 200
 632
 633     # TESTCASE 'user-renable3','user','enable','reenabled user','can write objects'
 634     key = boto.s3.key.Key(bucket)
 635     key.set_contents_from_string('six')
 636
 637     # TESTCASE 'garbage-list', 'garbage', 'list', 'get list of objects ready for garbage collection'
 638
 639     # create an object large enough to be split into multiple parts
 640     test_string = 'foo'*10000000
 641
 642     big_key = boto.s3.key.Key(bucket)
 643     big_key.set_contents_from_string(test_string)
 644
 645     # now delete the head
 646     big_key.delete()
 647
 648     # TESTCASE 'rm-user-buckets','user','rm','existing user','fails, still has buckets'
 649     (ret, out) = rgwadmin_rest(admin_conn, ['user', 'rm'], {'uid' : user1})
 650     assert ret == 409
 651
 652     # delete should fail because ``key`` still exists
 653     try:
 654         bucket.delete()
 655     except boto.exception.S3ResponseError as e:
 656         assert e.status == 409
 657
 658     key.delete()
 659     bucket.delete()
 660
 661     # TESTCASE 'policy', 'bucket', 'policy', 'get bucket policy', 'returns S3 policy'
 662     bucket = connection.create_bucket(bucket_name)
 663
 664     # create an object
 665     key = boto.s3.key.Key(bucket)
 666     key.set_contents_from_string('seven')
 667
 668     # should be private already but guarantee it
 669     key.set_acl('private')
 670
 671     (ret, out) = rgwadmin_rest(admin_conn, ['policy', 'show'], {'bucket' : bucket.name, 'object' : key.key})
 672     assert ret == 200
 673     assert len(out['acl']['grant_map']) == 1
 674
 675     # add another grantee by making the object public read
 676     key.set_acl('public-read')
 677
 678     (ret, out) = rgwadmin_rest(admin_conn, ['policy', 'show'], {'bucket' : bucket.name, 'object' : key.key})
 679     assert ret == 200
 680     assert len(out['acl']['grant_map']) == 2
 681
 682     # TESTCASE 'rm-bucket', 'bucket', 'rm', 'bucket with objects', 'succeeds'
 683     bucket = connection.create_bucket(bucket_name)
 684     key_name = ['eight', 'nine', 'ten', 'eleven']
 685     for i in range(4):
 686         key = boto.s3.key.Key(bucket)
 687         key.set_contents_from_string(key_name[i])
 688
 689     (ret, out) = rgwadmin_rest(admin_conn, ['bucket', 'rm'], {'bucket' : bucket_name, 'purge-objects' : True})
 690     assert ret == 200
 691
 692     # TESTCASE 'caps-add', 'caps', 'add', 'add user cap', 'succeeds'
 693     caps = 'usage=read'
 694     (ret, out) = rgwadmin_rest(admin_conn, ['caps', 'add'], {'uid' :  user1, 'user-caps' : caps})
 695     assert ret == 200
 696     assert out[0]['perm'] == 'read'
 697
 698     # TESTCASE 'caps-rm', 'caps', 'rm', 'remove existing cap from user', 'succeeds'
 699     (ret, out) = rgwadmin_rest(admin_conn, ['caps', 'rm'], {'uid' :  user1, 'user-caps' : caps})
 700     assert ret == 200
 701     assert not out
 702
 703     # TESTCASE 'rm-user','user','rm','existing user','fails, still has buckets'
 704     bucket = connection.create_bucket(bucket_name)
 705     key = boto.s3.key.Key(bucket)
 706
 707     (ret, out) = rgwadmin_rest(admin_conn, ['user', 'rm'], {'uid' : user1})
 708     assert ret == 409
 709
 710     # TESTCASE 'rm-user2', 'user', 'rm', user with data', 'succeeds'
 711     bucket = connection.create_bucket(bucket_name)
 712     key = boto.s3.key.Key(bucket)
 713     key.set_contents_from_string('twelve')
 714
 715     (ret, out) = rgwadmin_rest(admin_conn, ['user', 'rm'], {'uid' : user1, 'purge-data' : True})
 716     assert ret == 200
 717
 718     # TESTCASE 'rm-user3','user','info','deleted user','fails'
 719     (ret, out) = rgwadmin_rest(admin_conn, ['user', 'info'], {'uid' :  user1})
 720     assert ret == 404
 721