ceph/qa/tasks/radosgw_admin.py

   1 """
   2 Rgw admin testing against a running instance
   3 """
   4 # The test cases in this file have been annotated for inventory.
   5 # To extract the inventory (in csv format) use the command:
   6 #
   7 #   grep '^ *# TESTCASE' | sed 's/^ *# TESTCASE //'
   8 #
   9
  10 import copy
  11 import json
  12 import logging
  13 import time
  14 import datetime
  15
  16 from cStringIO import StringIO
  17
  18 import boto.exception
  19 import boto.s3.connection
  20 import boto.s3.acl
  21
  22 import httplib2
  23
  24 import util.rgw as rgw_utils
  25
  26 from util.rgw import rgwadmin, get_user_summary, get_user_successful_ops
  27
  28 log = logging.getLogger(__name__)
  29
  30 def create_presigned_url(conn, method, bucket_name, key_name, expiration):
  31     return conn.generate_url(expires_in=expiration,
  32         method=method,
  33         bucket=bucket_name,
  34         key=key_name,
  35         query_auth=True,
  36     )
  37
  38 def send_raw_http_request(conn, method, bucket_name, key_name, follow_redirects = False):
  39     url = create_presigned_url(conn, method, bucket_name, key_name, 3600)
  40     print url
  41     h = httplib2.Http()
  42     h.follow_redirects = follow_redirects
  43     return h.request(url, method)
  44
  45
  46 def get_acl(key):
  47     """
  48     Helper function to get the xml acl from a key, ensuring that the xml
  49     version tag is removed from the acl response
  50     """
  51     raw_acl = key.get_xml_acl()
  52
  53     def remove_version(string):
  54         return string.split(
  55             '<?xml version="1.0" encoding="UTF-8"?>'
  56         )[-1]
  57
  58     def remove_newlines(string):
  59         return string.strip('\n')
  60
  61     return remove_version(
  62         remove_newlines(raw_acl)
  63     )
  64
  65 def task(ctx, config):
  66     """
  67     Test radosgw-admin functionality against a running rgw instance.
  68     """
  69     global log
  70
  71     # regions and config found from rgw task
  72     assert ctx.rgw.regions is not None, \
  73         "radosgw_admin task needs region(s) declared from the rgw task"
  74     regions = ctx.rgw.regions
  75     log.debug('regions are: %r', regions)
  76
  77     assert ctx.rgw.config, \
  78         "radosgw_admin task needs a config passed from the rgw task"
  79     config = ctx.rgw.config
  80     log.debug('config is: %r', config)
  81
  82     clients_from_config = config.keys()
  83
  84     # choose first client as default
  85     client = clients_from_config[0]
  86
  87     multi_region_run = rgw_utils.multi_region_enabled(ctx)
  88     if multi_region_run:
  89         client = rgw_utils.get_config_master_client(ctx, config, regions)
  90
  91     log.debug('multi_region_run is: %r', multi_region_run)
  92     log.debug('master_client is: %r', client)
  93
  94     # once the client is chosen, pull the host name and  assigned port out of
  95     # the role_endpoints that were assigned by the rgw task
  96     (remote_host, remote_port) = ctx.rgw.role_endpoints[client]
  97
  98     realm = ctx.rgw.realm
  99     log.debug('radosgw-admin: realm %r', realm)
 100
 101     ##
 102     user1='foo'
 103     user2='fud'
 104     subuser1='foo:foo1'
 105     subuser2='foo:foo2'
 106     display_name1='Foo'
 107     display_name2='Fud'
 108     email='foo@foo.com'
 109     email2='bar@bar.com'
 110     access_key='9te6NH5mcdcq0Tc5i8i1'
 111     secret_key='Ny4IOauQoL18Gp2zM7lC1vLmoawgqcYP/YGcWfXu'
 112     access_key2='p5YnriCv1nAtykxBrupQ'
 113     secret_key2='Q8Tk6Q/27hfbFSYdSkPtUqhqx1GgzvpXa4WARozh'
 114     swift_secret1='gpS2G9RREMrnbqlp29PP2D36kgPR1tm72n5fPYfL'
 115     swift_secret2='ri2VJQcKSYATOY6uaDUX7pxgkW+W1YmC6OCxPHwy'
 116
 117     bucket_name='myfoo'
 118     bucket_name2='mybar'
 119
 120     # connect to rgw
 121     connection = boto.s3.connection.S3Connection(
 122         aws_access_key_id=access_key,
 123         aws_secret_access_key=secret_key,
 124         is_secure=False,
 125         port=remote_port,
 126         host=remote_host,
 127         calling_format=boto.s3.connection.OrdinaryCallingFormat(),
 128         )
 129     connection2 = boto.s3.connection.S3Connection(
 130         aws_access_key_id=access_key2,
 131         aws_secret_access_key=secret_key2,
 132         is_secure=False,
 133         port=remote_port,
 134         host=remote_host,
 135         calling_format=boto.s3.connection.OrdinaryCallingFormat(),
 136         )
 137
 138     # legend (test cases can be easily grep-ed out)
 139     # TESTCASE 'testname','object','method','operation','assertion'
 140     # TESTCASE 'info-nosuch','user','info','non-existent user','fails'
 141     (err, out) = rgwadmin(ctx, client, ['user', 'info', '--uid', user1])
 142     assert err
 143
 144     # TESTCASE 'create-ok','user','create','w/all valid info','succeeds'
 145     (err, out) = rgwadmin(ctx, client, [
 146             'user', 'create',
 147             '--uid', user1,
 148             '--display-name', display_name1,
 149             '--email', email,
 150             '--access-key', access_key,
 151             '--secret', secret_key,
 152             '--max-buckets', '4'
 153             ],
 154             check_status=True)
 155
 156     # TESTCASE 'duplicate email','user','create','existing user email','fails'
 157     (err, out) = rgwadmin(ctx, client, [
 158             'user', 'create',
 159             '--uid', user2,
 160             '--display-name', display_name2,
 161             '--email', email,
 162             ])
 163     assert err
 164
 165     # TESTCASE 'info-existing','user','info','existing user','returns correct info'
 166     (err, out) = rgwadmin(ctx, client, ['user', 'info', '--uid', user1], check_status=True)
 167     assert out['user_id'] == user1
 168     assert out['email'] == email
 169     assert out['display_name'] == display_name1
 170     assert len(out['keys']) == 1
 171     assert out['keys'][0]['access_key'] == access_key
 172     assert out['keys'][0]['secret_key'] == secret_key
 173     assert not out['suspended']
 174
 175     # this whole block should only be run if regions have been configured
 176     if multi_region_run:
 177         rgw_utils.radosgw_agent_sync_all(ctx)
 178         # post-sync, validate that user1 exists on the sync destination host
 179         for agent_client, c_config in ctx.radosgw_agent.config.iteritems():
 180             dest_client = c_config['dest']
 181             (err, out) = rgwadmin(ctx, dest_client, ['metadata', 'list', 'user'])
 182             (err, out) = rgwadmin(ctx, dest_client, ['user', 'info', '--uid', user1], check_status=True)
 183             assert out['user_id'] == user1
 184             assert out['email'] == email
 185             assert out['display_name'] == display_name1
 186             assert len(out['keys']) == 1
 187             assert out['keys'][0]['access_key'] == access_key
 188             assert out['keys'][0]['secret_key'] == secret_key
 189             assert not out['suspended']
 190
 191         # compare the metadata between different regions, make sure it matches
 192         log.debug('compare the metadata between different regions, make sure it matches')
 193         for agent_client, c_config in ctx.radosgw_agent.config.iteritems():
 194             source_client = c_config['src']
 195             dest_client = c_config['dest']
 196             (err1, out1) = rgwadmin(ctx, source_client,
 197                 ['metadata', 'get', 'user:{uid}'.format(uid=user1)], check_status=True)
 198             (err2, out2) = rgwadmin(ctx, dest_client,
 199                 ['metadata', 'get', 'user:{uid}'.format(uid=user1)], check_status=True)
 200             assert out1 == out2
 201
 202         # suspend a user on the master, then check the status on the destination
 203         log.debug('suspend a user on the master, then check the status on the destination')
 204         for agent_client, c_config in ctx.radosgw_agent.config.iteritems():
 205             source_client = c_config['src']
 206             dest_client = c_config['dest']
 207             (err, out) = rgwadmin(ctx, source_client, ['user', 'suspend', '--uid', user1])
 208             rgw_utils.radosgw_agent_sync_all(ctx)
 209             (err, out) = rgwadmin(ctx, dest_client, ['user', 'info', '--uid', user1], check_status=True)
 210             assert out['suspended']
 211
 212         # delete a user on the master, then check that it's gone on the destination
 213         log.debug('delete a user on the master, then check that it\'s gone on the destination')
 214         for agent_client, c_config in ctx.radosgw_agent.config.iteritems():
 215             source_client = c_config['src']
 216             dest_client = c_config['dest']
 217             (err, out) = rgwadmin(ctx, source_client, ['user', 'rm', '--uid', user1], check_status=True)
 218             rgw_utils.radosgw_agent_sync_all(ctx)
 219             (err, out) = rgwadmin(ctx, source_client, ['user', 'info', '--uid', user1])
 220             assert out is None
 221             (err, out) = rgwadmin(ctx, dest_client, ['user', 'info', '--uid', user1])
 222             assert out is None
 223
 224             # then recreate it so later tests pass
 225             (err, out) = rgwadmin(ctx, client, [
 226                 'user', 'create',
 227                 '--uid', user1,
 228                 '--display-name', display_name1,
 229                 '--email', email,
 230                 '--access-key', access_key,
 231                 '--secret', secret_key,
 232                 '--max-buckets', '4'
 233                 ],
 234                 check_status=True)
 235
 236         # now do the multi-region bucket tests
 237         log.debug('now do the multi-region bucket tests')
 238
 239         # Create a second user for the following tests
 240         log.debug('Create a second user for the following tests')
 241         (err, out) = rgwadmin(ctx, client, [
 242             'user', 'create',
 243             '--uid', user2,
 244             '--display-name', display_name2,
 245             '--email', email2,
 246             '--access-key', access_key2,
 247             '--secret', secret_key2,
 248             '--max-buckets', '4'
 249             ],
 250             check_status=True)
 251         (err, out) = rgwadmin(ctx, client, ['user', 'info', '--uid', user2], check_status=True)
 252         assert out is not None
 253
 254         # create a bucket and do a sync
 255         log.debug('create a bucket and do a sync')
 256         bucket = connection.create_bucket(bucket_name2)
 257         rgw_utils.radosgw_agent_sync_all(ctx)
 258
 259         # compare the metadata for the bucket between different regions, make sure it matches
 260         log.debug('compare the metadata for the bucket between different regions, make sure it matches')
 261         for agent_client, c_config in ctx.radosgw_agent.config.iteritems():
 262             source_client = c_config['src']
 263             dest_client = c_config['dest']
 264             (err1, out1) = rgwadmin(ctx, source_client,
 265                 ['metadata', 'get', 'bucket:{bucket_name}'.format(bucket_name=bucket_name2)],
 266                 check_status=True)
 267             (err2, out2) = rgwadmin(ctx, dest_client,
 268                 ['metadata', 'get', 'bucket:{bucket_name}'.format(bucket_name=bucket_name2)],
 269                 check_status=True)
 270             log.debug('metadata 1 %r', out1)
 271             log.debug('metadata 2 %r', out2)
 272             assert out1 == out2
 273
 274             # get the bucket.instance info and compare that
 275             src_bucket_id = out1['data']['bucket']['bucket_id']
 276             dest_bucket_id = out2['data']['bucket']['bucket_id']
 277             (err1, out1) = rgwadmin(ctx, source_client, ['metadata', 'get',
 278                 'bucket.instance:{bucket_name}:{bucket_instance}'.format(
 279                 bucket_name=bucket_name2,bucket_instance=src_bucket_id)],
 280                 check_status=True)
 281             (err2, out2) = rgwadmin(ctx, dest_client, ['metadata', 'get',
 282                 'bucket.instance:{bucket_name}:{bucket_instance}'.format(
 283                 bucket_name=bucket_name2,bucket_instance=dest_bucket_id)],
 284                 check_status=True)
 285             assert out1 == out2
 286
 287         same_region = 0
 288         for agent_client, c_config in ctx.radosgw_agent.config.iteritems():
 289             source_client = c_config['src']
 290             dest_client = c_config['dest']
 291
 292             source_region = rgw_utils.region_for_client(ctx, source_client)
 293             dest_region = rgw_utils.region_for_client(ctx, dest_client)
 294
 295             # 301 is only returned for requests to something in a different region
 296             if source_region == dest_region:
 297                 log.debug('301 is only returned for requests to something in a different region')
 298                 same_region += 1
 299                 continue
 300
 301             # Attempt to create a new connection with user1 to the destination RGW
 302             log.debug('Attempt to create a new connection with user1 to the destination RGW')
 303             # and use that to attempt a delete (that should fail)
 304
 305             (dest_remote_host, dest_remote_port) = ctx.rgw.role_endpoints[dest_client]
 306             connection_dest = boto.s3.connection.S3Connection(
 307                 aws_access_key_id=access_key,
 308                 aws_secret_access_key=secret_key,
 309                 is_secure=False,
 310                 port=dest_remote_port,
 311                 host=dest_remote_host,
 312                 calling_format=boto.s3.connection.OrdinaryCallingFormat(),
 313                 )
 314
 315             # this should fail
 316             r, content = send_raw_http_request(connection_dest, 'DELETE', bucket_name2, '', follow_redirects = False)
 317             assert r.status == 301
 318
 319             # now delete the bucket on the source RGW and do another sync
 320             log.debug('now delete the bucket on the source RGW and do another sync')
 321             bucket.delete()
 322             rgw_utils.radosgw_agent_sync_all(ctx)
 323
 324         if same_region == len(ctx.radosgw_agent.config):
 325             bucket.delete()
 326             rgw_utils.radosgw_agent_sync_all(ctx)
 327
 328         # make sure that the bucket no longer exists in either region
 329         log.debug('make sure that the bucket no longer exists in either region')
 330         for agent_client, c_config in ctx.radosgw_agent.config.iteritems():
 331             source_client = c_config['src']
 332             dest_client = c_config['dest']
 333             (err1, out1) = rgwadmin(ctx, source_client, ['metadata', 'get',
 334                 'bucket:{bucket_name}'.format(bucket_name=bucket_name2)])
 335             (err2, out2) = rgwadmin(ctx, dest_client, ['metadata', 'get',
 336                 'bucket:{bucket_name}'.format(bucket_name=bucket_name2)])
 337             # Both of the previous calls should have errors due to requesting
 338             # metadata for non-existent buckets
 339             assert err1
 340             assert err2
 341
 342         # create a bucket and then sync it
 343         log.debug('create a bucket and then sync it')
 344         bucket = connection.create_bucket(bucket_name2)
 345         rgw_utils.radosgw_agent_sync_all(ctx)
 346
 347         # compare the metadata for the bucket between different regions, make sure it matches
 348         log.debug('compare the metadata for the bucket between different regions, make sure it matches')
 349         for agent_client, c_config in ctx.radosgw_agent.config.iteritems():
 350             source_client = c_config['src']
 351             dest_client = c_config['dest']
 352             (err1, out1) = rgwadmin(ctx, source_client,
 353                 ['metadata', 'get', 'bucket:{bucket_name}'.format(bucket_name=bucket_name2)],
 354                 check_status=True)
 355             (err2, out2) = rgwadmin(ctx, dest_client,
 356                 ['metadata', 'get', 'bucket:{bucket_name}'.format(bucket_name=bucket_name2)],
 357                 check_status=True)
 358             assert out1 == out2
 359
 360         # Now delete the bucket and recreate it with a different user
 361         log.debug('Now delete the bucket and recreate it with a different user')
 362         # within the same window of time and then sync.
 363         bucket.delete()
 364         bucket = connection2.create_bucket(bucket_name2)
 365         rgw_utils.radosgw_agent_sync_all(ctx)
 366
 367         # compare the metadata for the bucket between different regions, make sure it matches
 368         log.debug('compare the metadata for the bucket between different regions, make sure it matches')
 369         # user2 should own the bucket in both regions
 370         for agent_client, c_config in ctx.radosgw_agent.config.iteritems():
 371             source_client = c_config['src']
 372             dest_client = c_config['dest']
 373             (err1, out1) = rgwadmin(ctx, source_client,
 374                 ['metadata', 'get', 'bucket:{bucket_name}'.format(bucket_name=bucket_name2)],
 375                 check_status=True)
 376             (err2, out2) = rgwadmin(ctx, dest_client,
 377                 ['metadata', 'get', 'bucket:{bucket_name}'.format(bucket_name=bucket_name2)],
 378                 check_status=True)
 379             assert out1 == out2
 380             assert out1['data']['owner'] == user2
 381             assert out1['data']['owner'] != user1
 382
 383         # now we're going to use this bucket to test meta-data update propagation
 384         log.debug('now we\'re going to use this bucket to test meta-data update propagation')
 385         for agent_client, c_config in ctx.radosgw_agent.config.iteritems():
 386             source_client = c_config['src']
 387             dest_client = c_config['dest']
 388
 389             # get the metadata so we can tweak it
 390             log.debug('get the metadata so we can tweak it')
 391             (err, orig_data) = rgwadmin(ctx, source_client,
 392                 ['metadata', 'get', 'bucket:{bucket_name}'.format(bucket_name=bucket_name2)],
 393                 check_status=True)
 394
 395             # manually edit mtime for this bucket to be 300 seconds in the past
 396             log.debug('manually edit mtime for this bucket to be 300 seconds in the past')
 397             new_data = copy.deepcopy(orig_data)
 398             mtime = datetime.datetime.strptime(orig_data['mtime'], "%Y-%m-%d %H:%M:%S.%fZ") - datetime.timedelta(300)
 399             new_data['mtime'] =  unicode(mtime.strftime("%Y-%m-%d %H:%M:%S.%fZ"))
 400             log.debug("new mtime ", mtime)
 401             assert new_data != orig_data
 402             (err, out) = rgwadmin(ctx, source_client,
 403                 ['metadata', 'put', 'bucket:{bucket_name}'.format(bucket_name=bucket_name2)],
 404                 stdin=StringIO(json.dumps(new_data)),
 405                 check_status=True)
 406
 407             # get the metadata and make sure that the 'put' worked
 408             log.debug('get the metadata and make sure that the \'put\' worked')
 409             (err, out) = rgwadmin(ctx, source_client,
 410                 ['metadata', 'get', 'bucket:{bucket_name}'.format(bucket_name=bucket_name2)],
 411                 check_status=True)
 412             assert out == new_data
 413
 414             # sync to propagate the new metadata
 415             log.debug('sync to propagate the new metadata')
 416             rgw_utils.radosgw_agent_sync_all(ctx)
 417
 418             # get the metadata from the dest and compare it to what we just set
 419             log.debug('get the metadata from the dest and compare it to what we just set')
 420             # and what the source region has.
 421             (err1, out1) = rgwadmin(ctx, source_client,
 422                 ['metadata', 'get', 'bucket:{bucket_name}'.format(bucket_name=bucket_name2)],
 423                 check_status=True)
 424             (err2, out2) = rgwadmin(ctx, dest_client,
 425                 ['metadata', 'get', 'bucket:{bucket_name}'.format(bucket_name=bucket_name2)],
 426                 check_status=True)
 427             # yeah for the transitive property
 428             assert out1 == out2
 429             assert out1 == new_data
 430
 431         # now we delete the bucket
 432         log.debug('now we delete the bucket')
 433         bucket.delete()
 434
 435         log.debug('sync to propagate the deleted bucket')
 436         rgw_utils.radosgw_agent_sync_all(ctx)
 437
 438         # Delete user2 as later tests do not expect it to exist.
 439         # Verify that it is gone on both regions
 440         for agent_client, c_config in ctx.radosgw_agent.config.iteritems():
 441             source_client = c_config['src']
 442             dest_client = c_config['dest']
 443             (err, out) = rgwadmin(ctx, source_client,
 444                 ['user', 'rm', '--uid', user2], check_status=True)
 445             rgw_utils.radosgw_agent_sync_all(ctx)
 446             # The two 'user info' calls should fail and not return any data
 447             # since we just deleted this user.
 448             (err, out) = rgwadmin(ctx, source_client, ['user', 'info', '--uid', user2])
 449             assert out is None
 450             (err, out) = rgwadmin(ctx, dest_client, ['user', 'info', '--uid', user2])
 451             assert out is None
 452
 453         # Test data sync
 454
 455         # First create a bucket for data sync test purpose
 456         bucket = connection.create_bucket(bucket_name + 'data')
 457
 458         # Create a tiny file and check if in sync
 459         for agent_client, c_config in ctx.radosgw_agent.config.iteritems():
 460             if c_config.get('metadata-only'):
 461                 continue
 462
 463             for full in (True, False):
 464                 source_client = c_config['src']
 465                 dest_client = c_config['dest']
 466                 k = boto.s3.key.Key(bucket)
 467                 k.key = 'tiny_file'
 468                 k.set_contents_from_string("123456789")
 469                 safety_window = rgw_utils.radosgw_data_log_window(ctx, source_client)
 470                 time.sleep(safety_window)
 471                 rgw_utils.radosgw_agent_sync_all(ctx, data=True, full=full)
 472                 (dest_host, dest_port) = ctx.rgw.role_endpoints[dest_client]
 473                 dest_connection = boto.s3.connection.S3Connection(
 474                     aws_access_key_id=access_key,
 475                     aws_secret_access_key=secret_key,
 476                     is_secure=False,
 477                     port=dest_port,
 478                     host=dest_host,
 479                     calling_format=boto.s3.connection.OrdinaryCallingFormat(),
 480                 )
 481                 dest_k = dest_connection.get_bucket(bucket_name + 'data').get_key('tiny_file')
 482                 assert k.get_contents_as_string() == dest_k.get_contents_as_string()
 483
 484                 # check that deleting it removes it from the dest zone
 485                 k.delete()
 486                 time.sleep(safety_window)
 487                 # full sync doesn't handle deleted objects yet
 488                 rgw_utils.radosgw_agent_sync_all(ctx, data=True, full=False)
 489
 490                 dest_bucket = dest_connection.get_bucket(bucket_name + 'data')
 491                 dest_k = dest_bucket.get_key('tiny_file')
 492                 assert dest_k == None, 'object not deleted from destination zone'
 493
 494         # finally we delete the bucket
 495         bucket.delete()
 496
 497         bucket = connection.create_bucket(bucket_name + 'data2')
 498         for agent_client, c_config in ctx.radosgw_agent.config.iteritems():
 499             if c_config.get('metadata-only'):
 500                 continue
 501
 502             for full in (True, False):
 503                 source_client = c_config['src']
 504                 dest_client = c_config['dest']
 505                 (dest_host, dest_port) = ctx.rgw.role_endpoints[dest_client]
 506                 dest_connection = boto.s3.connection.S3Connection(
 507                     aws_access_key_id=access_key,
 508                     aws_secret_access_key=secret_key,
 509                     is_secure=False,
 510                     port=dest_port,
 511                     host=dest_host,
 512                     calling_format=boto.s3.connection.OrdinaryCallingFormat(),
 513                 )
 514                 for i in range(20):
 515                     k = boto.s3.key.Key(bucket)
 516                     k.key = 'tiny_file_' + str(i)
 517                     k.set_contents_from_string(str(i) * 100)
 518
 519                 safety_window = rgw_utils.radosgw_data_log_window(ctx, source_client)
 520                 time.sleep(safety_window)
 521                 rgw_utils.radosgw_agent_sync_all(ctx, data=True, full=full)
 522
 523                 for i in range(20):
 524                     dest_k = dest_connection.get_bucket(bucket_name + 'data2').get_key('tiny_file_' + str(i))
 525                     assert (str(i) * 100) == dest_k.get_contents_as_string()
 526                     k = boto.s3.key.Key(bucket)
 527                     k.key = 'tiny_file_' + str(i)
 528                     k.delete()
 529
 530                 # check that deleting removes the objects from the dest zone
 531                 time.sleep(safety_window)
 532                 # full sync doesn't delete deleted objects yet
 533                 rgw_utils.radosgw_agent_sync_all(ctx, data=True, full=False)
 534
 535                 for i in range(20):
 536                     dest_bucket = dest_connection.get_bucket(bucket_name + 'data2')
 537                     dest_k = dest_bucket.get_key('tiny_file_' + str(i))
 538                     assert dest_k == None, 'object %d not deleted from destination zone' % i
 539         bucket.delete()
 540
 541     # end of 'if multi_region_run:'
 542
 543     # TESTCASE 'suspend-ok','user','suspend','active user','succeeds'
 544     (err, out) = rgwadmin(ctx, client, ['user', 'suspend', '--uid', user1],
 545         check_status=True)
 546
 547     # TESTCASE 'suspend-suspended','user','suspend','suspended user','succeeds w/advisory'
 548     (err, out) = rgwadmin(ctx, client, ['user', 'info', '--uid', user1], check_status=True)
 549     assert out['suspended']
 550
 551     # TESTCASE 're-enable','user','enable','suspended user','succeeds'
 552     (err, out) = rgwadmin(ctx, client, ['user', 'enable', '--uid', user1], check_status=True)
 553
 554     # TESTCASE 'info-re-enabled','user','info','re-enabled user','no longer suspended'
 555     (err, out) = rgwadmin(ctx, client, ['user', 'info', '--uid', user1], check_status=True)
 556     assert not out['suspended']
 557
 558     # TESTCASE 'add-keys','key','create','w/valid info','succeeds'
 559     (err, out) = rgwadmin(ctx, client, [
 560             'key', 'create', '--uid', user1,
 561             '--access-key', access_key2, '--secret', secret_key2,
 562             ], check_status=True)
 563
 564     # TESTCASE 'info-new-key','user','info','after key addition','returns all keys'
 565     (err, out) = rgwadmin(ctx, client, ['user', 'info', '--uid', user1],
 566         check_status=True)
 567     assert len(out['keys']) == 2
 568     assert out['keys'][0]['access_key'] == access_key2 or out['keys'][1]['access_key'] == access_key2
 569     assert out['keys'][0]['secret_key'] == secret_key2 or out['keys'][1]['secret_key'] == secret_key2
 570
 571     # TESTCASE 'rm-key','key','rm','newly added key','succeeds, key is removed'
 572     (err, out) = rgwadmin(ctx, client, [
 573             'key', 'rm', '--uid', user1,
 574             '--access-key', access_key2,
 575             ], check_status=True)
 576     assert len(out['keys']) == 1
 577     assert out['keys'][0]['access_key'] == access_key
 578     assert out['keys'][0]['secret_key'] == secret_key
 579
 580     # TESTCASE 'add-swift-key','key','create','swift key','succeeds'
 581     subuser_access = 'full'
 582     subuser_perm = 'full-control'
 583
 584     (err, out) = rgwadmin(ctx, client, [
 585             'subuser', 'create', '--subuser', subuser1,
 586             '--access', subuser_access
 587             ], check_status=True)
 588
 589     # TESTCASE 'add-swift-key','key','create','swift key','succeeds'
 590     (err, out) = rgwadmin(ctx, client, [
 591             'subuser', 'modify', '--subuser', subuser1,
 592             '--secret', swift_secret1,
 593             '--key-type', 'swift',
 594             ], check_status=True)
 595
 596     # TESTCASE 'subuser-perm-mask', 'subuser', 'info', 'test subuser perm mask durability', 'succeeds'
 597     (err, out) = rgwadmin(ctx, client, ['user', 'info', '--uid', user1])
 598
 599     assert out['subusers'][0]['permissions'] == subuser_perm
 600
 601     # TESTCASE 'info-swift-key','user','info','after key addition','returns all keys'
 602     (err, out) = rgwadmin(ctx, client, ['user', 'info', '--uid', user1], check_status=True)
 603     assert len(out['swift_keys']) == 1
 604     assert out['swift_keys'][0]['user'] == subuser1
 605     assert out['swift_keys'][0]['secret_key'] == swift_secret1
 606
 607     # TESTCASE 'add-swift-subuser','key','create','swift sub-user key','succeeds'
 608     (err, out) = rgwadmin(ctx, client, [
 609             'subuser', 'create', '--subuser', subuser2,
 610             '--secret', swift_secret2,
 611             '--key-type', 'swift',
 612             ], check_status=True)
 613
 614     # TESTCASE 'info-swift-subuser','user','info','after key addition','returns all sub-users/keys'
 615     (err, out) = rgwadmin(ctx, client, ['user', 'info', '--uid', user1], check_status=True)
 616     assert len(out['swift_keys']) == 2
 617     assert out['swift_keys'][0]['user'] == subuser2 or out['swift_keys'][1]['user'] == subuser2
 618     assert out['swift_keys'][0]['secret_key'] == swift_secret2 or out['swift_keys'][1]['secret_key'] == swift_secret2
 619
 620     # TESTCASE 'rm-swift-key1','key','rm','subuser','succeeds, one key is removed'
 621     (err, out) = rgwadmin(ctx, client, [
 622             'key', 'rm', '--subuser', subuser1,
 623             '--key-type', 'swift',
 624             ], check_status=True)
 625     assert len(out['swift_keys']) == 1
 626
 627     # TESTCASE 'rm-subuser','subuser','rm','subuser','success, subuser is removed'
 628     (err, out) = rgwadmin(ctx, client, [
 629             'subuser', 'rm', '--subuser', subuser1,
 630             ], check_status=True)
 631     assert len(out['subusers']) == 1
 632
 633     # TESTCASE 'rm-subuser-with-keys','subuser','rm','subuser','succeeds, second subser and key is removed'
 634     (err, out) = rgwadmin(ctx, client, [
 635             'subuser', 'rm', '--subuser', subuser2,
 636             '--key-type', 'swift', '--purge-keys',
 637             ], check_status=True)
 638     assert len(out['swift_keys']) == 0
 639     assert len(out['subusers']) == 0
 640
 641     # TESTCASE 'bucket-stats','bucket','stats','no session/buckets','succeeds, empty list'
 642     (err, out) = rgwadmin(ctx, client, ['bucket', 'stats', '--uid', user1],
 643         check_status=True)
 644     assert len(out) == 0
 645
 646     if multi_region_run:
 647         rgw_utils.radosgw_agent_sync_all(ctx)
 648
 649     # TESTCASE 'bucket-stats2','bucket','stats','no buckets','succeeds, empty list'
 650     (err, out) = rgwadmin(ctx, client, ['bucket', 'list', '--uid', user1], check_status=True)
 651     assert len(out) == 0
 652
 653     # create a first bucket
 654     bucket = connection.create_bucket(bucket_name)
 655
 656     # TESTCASE 'bucket-list','bucket','list','one bucket','succeeds, expected list'
 657     (err, out) = rgwadmin(ctx, client, ['bucket', 'list', '--uid', user1], check_status=True)
 658     assert len(out) == 1
 659     assert out[0] == bucket_name
 660
 661     # TESTCASE 'bucket-list-all','bucket','list','all buckets','succeeds, expected list'
 662     (err, out) = rgwadmin(ctx, client, ['bucket', 'list'], check_status=True)
 663     assert len(out) >= 1
 664     assert bucket_name in out;
 665
 666     # TESTCASE 'max-bucket-limit,'bucket','create','4 buckets','5th bucket fails due to max buckets == 4'
 667     bucket2 = connection.create_bucket(bucket_name + '2')
 668     bucket3 = connection.create_bucket(bucket_name + '3')
 669     bucket4 = connection.create_bucket(bucket_name + '4')
 670     # the 5th should fail.
 671     failed = False
 672     try:
 673         connection.create_bucket(bucket_name + '5')
 674     except Exception:
 675         failed = True
 676     assert failed
 677
 678     # delete the buckets
 679     bucket2.delete()
 680     bucket3.delete()
 681     bucket4.delete()
 682
 683     # TESTCASE 'bucket-stats3','bucket','stats','new empty bucket','succeeds, empty list'
 684     (err, out) = rgwadmin(ctx, client, [
 685             'bucket', 'stats', '--bucket', bucket_name], check_status=True)
 686     assert out['owner'] == user1
 687     bucket_id = out['id']
 688
 689     # TESTCASE 'bucket-stats4','bucket','stats','new empty bucket','succeeds, expected bucket ID'
 690     (err, out) = rgwadmin(ctx, client, ['bucket', 'stats', '--uid', user1], check_status=True)
 691     assert len(out) == 1
 692     assert out[0]['id'] == bucket_id    # does it return the same ID twice in a row?
 693
 694     # use some space
 695     key = boto.s3.key.Key(bucket)
 696     key.set_contents_from_string('one')
 697
 698     # TESTCASE 'bucket-stats5','bucket','stats','after creating key','succeeds, lists one non-empty object'
 699     (err, out) = rgwadmin(ctx, client, [
 700             'bucket', 'stats', '--bucket', bucket_name], check_status=True)
 701     assert out['id'] == bucket_id
 702     assert out['usage']['rgw.main']['num_objects'] == 1
 703     assert out['usage']['rgw.main']['size_kb'] > 0
 704
 705     # reclaim it
 706     key.delete()
 707
 708     # TESTCASE 'bucket unlink', 'bucket', 'unlink', 'unlink bucket from user', 'fails', 'access denied error'
 709     (err, out) = rgwadmin(ctx, client,
 710         ['bucket', 'unlink', '--uid', user1, '--bucket', bucket_name],
 711         check_status=True)
 712
 713     # create a second user to link the bucket to
 714     (err, out) = rgwadmin(ctx, client, [
 715             'user', 'create',
 716             '--uid', user2,
 717             '--display-name', display_name2,
 718             '--access-key', access_key2,
 719             '--secret', secret_key2,
 720             '--max-buckets', '1',
 721             ],
 722             check_status=True)
 723
 724     # try creating an object with the first user before the bucket is relinked
 725     denied = False
 726     key = boto.s3.key.Key(bucket)
 727
 728     try:
 729         key.set_contents_from_string('two')
 730     except boto.exception.S3ResponseError:
 731         denied = True
 732
 733     assert not denied
 734
 735     # delete the object
 736     key.delete()
 737
 738     # link the bucket to another user
 739     (err, out) = rgwadmin(ctx, client, ['metadata', 'get', 'bucket:{n}'.format(n=bucket_name)],
 740         check_status=True)
 741
 742     bucket_data = out['data']
 743     assert bucket_data['bucket']['name'] == bucket_name
 744
 745     bucket_id = bucket_data['bucket']['bucket_id']
 746
 747     # link the bucket to another user
 748     (err, out) = rgwadmin(ctx, client, ['bucket', 'link', '--uid', user2, '--bucket', bucket_name, '--bucket-id', bucket_id],
 749         check_status=True)
 750
 751     # try to remove user, should fail (has a linked bucket)
 752     (err, out) = rgwadmin(ctx, client, ['user', 'rm', '--uid', user2])
 753     assert err
 754
 755     # TESTCASE 'bucket unlink', 'bucket', 'unlink', 'unlink bucket from user', 'succeeds, bucket unlinked'
 756     (err, out) = rgwadmin(ctx, client, ['bucket', 'unlink', '--uid', user2, '--bucket', bucket_name],
 757         check_status=True)
 758
 759     # relink the bucket to the first user and delete the second user
 760     (err, out) = rgwadmin(ctx, client,
 761         ['bucket', 'link', '--uid', user1, '--bucket', bucket_name, '--bucket-id', bucket_id],
 762         check_status=True)
 763
 764     (err, out) = rgwadmin(ctx, client, ['user', 'rm', '--uid', user2],
 765         check_status=True)
 766
 767     # TESTCASE 'object-rm', 'object', 'rm', 'remove object', 'succeeds, object is removed'
 768
 769     # upload an object
 770     object_name = 'four'
 771     key = boto.s3.key.Key(bucket, object_name)
 772     key.set_contents_from_string(object_name)
 773
 774     # now delete it
 775     (err, out) = rgwadmin(ctx, client,
 776         ['object', 'rm', '--bucket', bucket_name, '--object', object_name],
 777         check_status=True)
 778
 779     # TESTCASE 'bucket-stats6','bucket','stats','after deleting key','succeeds, lists one no objects'
 780     (err, out) = rgwadmin(ctx, client, [
 781             'bucket', 'stats', '--bucket', bucket_name],
 782             check_status=True)
 783     assert out['id'] == bucket_id
 784     assert out['usage']['rgw.main']['num_objects'] == 0
 785
 786     # list log objects
 787     # TESTCASE 'log-list','log','list','after activity','succeeds, lists one no objects'
 788     (err, out) = rgwadmin(ctx, client, ['log', 'list'], check_status=True)
 789     assert len(out) > 0
 790
 791     for obj in out:
 792         # TESTCASE 'log-show','log','show','after activity','returns expected info'
 793         if obj[:4] == 'meta' or obj[:4] == 'data' or obj[:18] == 'obj_delete_at_hint':
 794             continue
 795
 796         (err, rgwlog) = rgwadmin(ctx, client, ['log', 'show', '--object', obj],
 797             check_status=True)
 798         assert len(rgwlog) > 0
 799
 800         # exempt bucket_name2 from checking as it was only used for multi-region tests
 801         assert rgwlog['bucket'].find(bucket_name) == 0 or rgwlog['bucket'].find(bucket_name2) == 0
 802         assert rgwlog['bucket'] != bucket_name or rgwlog['bucket_id'] == bucket_id
 803         assert rgwlog['bucket_owner'] == user1 or rgwlog['bucket'] == bucket_name + '5' or rgwlog['bucket'] == bucket_name2
 804         for entry in rgwlog['log_entries']:
 805             log.debug('checking log entry: ', entry)
 806             assert entry['bucket'] == rgwlog['bucket']
 807             possible_buckets = [bucket_name + '5', bucket_name2]
 808             user = entry['user']
 809             assert user == user1 or user.endswith('system-user') or \
 810                 rgwlog['bucket'] in possible_buckets
 811
 812         # TESTCASE 'log-rm','log','rm','delete log objects','succeeds'
 813         (err, out) = rgwadmin(ctx, client, ['log', 'rm', '--object', obj],
 814             check_status=True)
 815
 816     # TODO: show log by bucket+date
 817
 818     # need to wait for all usage data to get flushed, should take up to 30 seconds
 819     timestamp = time.time()
 820     while time.time() - timestamp <= (20 * 60):      # wait up to 20 minutes
 821         (err, out) = rgwadmin(ctx, client, ['usage', 'show', '--categories', 'delete_obj'])  # last operation we did is delete obj, wait for it to flush
 822         if get_user_successful_ops(out, user1) > 0:
 823             break
 824         time.sleep(1)
 825
 826     assert time.time() - timestamp <= (20 * 60)
 827
 828     # TESTCASE 'usage-show' 'usage' 'show' 'all usage' 'succeeds'
 829     (err, out) = rgwadmin(ctx, client, ['usage', 'show'], check_status=True)
 830     assert len(out['entries']) > 0
 831     assert len(out['summary']) > 0
 832
 833     user_summary = get_user_summary(out, user1)
 834
 835     total = user_summary['total']
 836     assert total['successful_ops'] > 0
 837
 838     # TESTCASE 'usage-show2' 'usage' 'show' 'user usage' 'succeeds'
 839     (err, out) = rgwadmin(ctx, client, ['usage', 'show', '--uid', user1],
 840         check_status=True)
 841     assert len(out['entries']) > 0
 842     assert len(out['summary']) > 0
 843     user_summary = out['summary'][0]
 844     for entry in user_summary['categories']:
 845         assert entry['successful_ops'] > 0
 846     assert user_summary['user'] == user1
 847
 848     # TESTCASE 'usage-show3' 'usage' 'show' 'user usage categories' 'succeeds'
 849     test_categories = ['create_bucket', 'put_obj', 'delete_obj', 'delete_bucket']
 850     for cat in test_categories:
 851         (err, out) = rgwadmin(ctx, client, ['usage', 'show', '--uid', user1, '--categories', cat],
 852             check_status=True)
 853         assert len(out['summary']) > 0
 854         user_summary = out['summary'][0]
 855         assert user_summary['user'] == user1
 856         assert len(user_summary['categories']) == 1
 857         entry = user_summary['categories'][0]
 858         assert entry['category'] == cat
 859         assert entry['successful_ops'] > 0
 860
 861     # the usage flush interval is 30 seconds, wait that much an then some
 862     # to make sure everything has been flushed
 863     time.sleep(35)
 864
 865     # TESTCASE 'usage-trim' 'usage' 'trim' 'user usage' 'succeeds, usage removed'
 866     (err, out) = rgwadmin(ctx, client, ['usage', 'trim', '--uid', user1],
 867         check_status=True)
 868     (err, out) = rgwadmin(ctx, client, ['usage', 'show', '--uid', user1],
 869         check_status=True)
 870     assert len(out['entries']) == 0
 871     assert len(out['summary']) == 0
 872
 873     # TESTCASE 'user-suspend2','user','suspend','existing user','succeeds'
 874     (err, out) = rgwadmin(ctx, client, ['user', 'suspend', '--uid', user1],
 875         check_status=True)
 876
 877     # TESTCASE 'user-suspend3','user','suspend','suspended user','cannot write objects'
 878     try:
 879         key = boto.s3.key.Key(bucket)
 880         key.set_contents_from_string('five')
 881     except boto.exception.S3ResponseError as e:
 882         assert e.status == 403
 883
 884     # TESTCASE 'user-renable2','user','enable','suspended user','succeeds'
 885     (err, out) = rgwadmin(ctx, client, ['user', 'enable', '--uid', user1],
 886         check_status=True)
 887
 888     # TESTCASE 'user-renable3','user','enable','reenabled user','can write objects'
 889     key = boto.s3.key.Key(bucket)
 890     key.set_contents_from_string('six')
 891
 892     # TESTCASE 'gc-list', 'gc', 'list', 'get list of objects ready for garbage collection'
 893
 894     # create an object large enough to be split into multiple parts
 895     test_string = 'foo'*10000000
 896
 897     big_key = boto.s3.key.Key(bucket)
 898     big_key.set_contents_from_string(test_string)
 899
 900     # now delete the head
 901     big_key.delete()
 902
 903     # wait a bit to give the garbage collector time to cycle
 904     time.sleep(15)
 905
 906     (err, out) = rgwadmin(ctx, client, ['gc', 'list'])
 907
 908     assert len(out) > 0
 909
 910     # TESTCASE 'gc-process', 'gc', 'process', 'manually collect garbage'
 911     (err, out) = rgwadmin(ctx, client, ['gc', 'process'], check_status=True)
 912
 913     #confirm
 914     (err, out) = rgwadmin(ctx, client, ['gc', 'list'])
 915
 916     assert len(out) == 0
 917
 918     # TESTCASE 'rm-user-buckets','user','rm','existing user','fails, still has buckets'
 919     (err, out) = rgwadmin(ctx, client, ['user', 'rm', '--uid', user1])
 920     assert err
 921
 922     # delete should fail because ``key`` still exists
 923     try:
 924         bucket.delete()
 925     except boto.exception.S3ResponseError as e:
 926         assert e.status == 409
 927
 928     key.delete()
 929     bucket.delete()
 930
 931     # TESTCASE 'policy', 'bucket', 'policy', 'get bucket policy', 'returns S3 policy'
 932     bucket = connection.create_bucket(bucket_name)
 933
 934     # create an object
 935     key = boto.s3.key.Key(bucket)
 936     key.set_contents_from_string('seven')
 937
 938     # should be private already but guarantee it
 939     key.set_acl('private')
 940
 941     (err, out) = rgwadmin(ctx, client,
 942         ['policy', '--bucket', bucket.name, '--object', key.key],
 943         check_status=True, format='xml')
 944
 945     acl = get_acl(key)
 946
 947     assert acl == out.strip('\n')
 948
 949     # add another grantee by making the object public read
 950     key.set_acl('public-read')
 951
 952     (err, out) = rgwadmin(ctx, client,
 953         ['policy', '--bucket', bucket.name, '--object', key.key],
 954         check_status=True, format='xml')
 955
 956     acl = get_acl(key)
 957
 958     assert acl == out.strip('\n')
 959
 960     # TESTCASE 'rm-bucket', 'bucket', 'rm', 'bucket with objects', 'succeeds'
 961     bucket = connection.create_bucket(bucket_name)
 962     key_name = ['eight', 'nine', 'ten', 'eleven']
 963     for i in range(4):
 964         key = boto.s3.key.Key(bucket)
 965         key.set_contents_from_string(key_name[i])
 966
 967     (err, out) = rgwadmin(ctx, client,
 968         ['bucket', 'rm', '--bucket', bucket_name, '--purge-objects'],
 969         check_status=True)
 970
 971     # TESTCASE 'caps-add', 'caps', 'add', 'add user cap', 'succeeds'
 972     caps='user=read'
 973     (err, out) = rgwadmin(ctx, client, ['caps', 'add', '--uid', user1, '--caps', caps])
 974
 975     assert out['caps'][0]['perm'] == 'read'
 976
 977     # TESTCASE 'caps-rm', 'caps', 'rm', 'remove existing cap from user', 'succeeds'
 978     (err, out) = rgwadmin(ctx, client, ['caps', 'rm', '--uid', user1, '--caps', caps])
 979
 980     assert not out['caps']
 981
 982     # TESTCASE 'rm-user','user','rm','existing user','fails, still has buckets'
 983     bucket = connection.create_bucket(bucket_name)
 984     key = boto.s3.key.Key(bucket)
 985
 986     (err, out) = rgwadmin(ctx, client, ['user', 'rm', '--uid', user1])
 987     assert err
 988
 989     # TESTCASE 'rm-user2', 'user', 'rm', 'user with data', 'succeeds'
 990     bucket = connection.create_bucket(bucket_name)
 991     key = boto.s3.key.Key(bucket)
 992     key.set_contents_from_string('twelve')
 993
 994     (err, out) = rgwadmin(ctx, client,
 995         ['user', 'rm', '--uid', user1, '--purge-data' ],
 996         check_status=True)
 997
 998     # TESTCASE 'rm-user3','user','rm','deleted user','fails'
 999     (err, out) = rgwadmin(ctx, client, ['user', 'info', '--uid', user1])
1000     assert err
1001
1002     # TESTCASE 'zone-info', 'zone', 'get', 'get zone info', 'succeeds, has default placement rule'
1003     #
1004
1005     if realm is None:
1006         (err, out) = rgwadmin(ctx, client, ['zone', 'get','--rgw-zone','default'])
1007     else:
1008         (err, out) = rgwadmin(ctx, client, ['zone', 'get'])
1009     orig_placement_pools = len(out['placement_pools'])
1010
1011     # removed this test, it is not correct to assume that zone has default placement, it really
1012     # depends on how we set it up before
1013     #
1014     # assert len(out) > 0
1015     # assert len(out['placement_pools']) == 1
1016
1017     # default_rule = out['placement_pools'][0]
1018     # assert default_rule['key'] == 'default-placement'
1019
1020     rule={'key': 'new-placement', 'val': {'data_pool': '.rgw.buckets.2', 'index_pool': '.rgw.buckets.index.2'}}
1021
1022     out['placement_pools'].append(rule)
1023
1024     (err, out) = rgwadmin(ctx, client, ['zone', 'set'],
1025         stdin=StringIO(json.dumps(out)),
1026         check_status=True)
1027
1028     if realm is None:
1029         (err, out) = rgwadmin(ctx, client, ['zone', 'get','--rgw-zone','default'])
1030     else:
1031         (err, out) = rgwadmin(ctx, client, ['zone', 'get'])
1032     assert len(out) > 0
1033     assert len(out['placement_pools']) == orig_placement_pools + 1