ceph/qa/tasks/radosgw_admin.py

   1 """
   2 Rgw admin testing against a running instance
   3 """
   4 # The test cases in this file have been annotated for inventory.
   5 # To extract the inventory (in csv format) use the command:
   6 #
   7 #   grep '^ *# TESTCASE' | sed 's/^ *# TESTCASE //'
   8 #
   9 # to run this standalone:
  10 #       python qa/tasks/radosgw_admin.py [USER] HOSTNAME
  11 #
  12
  13 import json
  14 import logging
  15 import time
  16 import datetime
  17 from six.moves import queue
  18
  19 import sys
  20 import six
  21
  22 from io import BytesIO
  23
  24 import boto.exception
  25 import boto.s3.connection
  26 import boto.s3.acl
  27
  28 import httplib2
  29
  30
  31 from tasks.util.rgw import rgwadmin, get_user_summary, get_user_successful_ops
  32
  33 log = logging.getLogger(__name__)
  34
  35 def usage_acc_findentry2(entries, user, add=True):
  36     for e in entries:
  37         if e['user'] == user:
  38             return e
  39     if not add:
  40             return None
  41     e = {'user': user, 'buckets': []}
  42     entries.append(e)
  43     return e
  44 def usage_acc_findsum2(summaries, user, add=True):
  45     for e in summaries:
  46         if e['user'] == user:
  47             return e
  48     if not add:
  49         return None
  50     e = {'user': user, 'categories': [],
  51         'total': {'bytes_received': 0,
  52             'bytes_sent': 0, 'ops': 0, 'successful_ops': 0 }}
  53     summaries.append(e)
  54     return e
  55 def usage_acc_update2(x, out, b_in, err):
  56     x['bytes_sent'] += b_in
  57     x['bytes_received'] += out
  58     x['ops'] += 1
  59     if not err:
  60         x['successful_ops'] += 1
  61 def usage_acc_validate_fields(r, x, x2, what):
  62     q=[]
  63     for field in ['bytes_sent', 'bytes_received', 'ops', 'successful_ops']:
  64         try:
  65             if x2[field] < x[field]:
  66                 q.append("field %s: %d < %d" % (field, x2[field], x[field]))
  67         except Exception as ex:
  68             r.append( "missing/bad field " + field + " in " + what + " " + str(ex))
  69             return
  70     if len(q) > 0:
  71         r.append("incomplete counts in " + what + ": " + ", ".join(q))
  72 class usage_acc:
  73     def __init__(self):
  74         self.results = {'entries': [], 'summary': []}
  75     def findentry(self, user):
  76         return usage_acc_findentry2(self.results['entries'], user)
  77     def findsum(self, user):
  78         return usage_acc_findsum2(self.results['summary'], user)
  79     def e2b(self, e, bucket, add=True):
  80         for b in e['buckets']:
  81             if b['bucket'] == bucket:
  82                 return b
  83         if not add:
  84                 return None
  85         b = {'bucket': bucket, 'categories': []}
  86         e['buckets'].append(b)
  87         return b
  88     def c2x(self, c, cat, add=True):
  89         for x in c:
  90             if x['category'] == cat:
  91                 return x
  92         if not add:
  93                 return None
  94         x = {'bytes_received': 0, 'category': cat,
  95             'bytes_sent': 0, 'ops': 0, 'successful_ops': 0 }
  96         c.append(x)
  97         return x
  98     def update(self, c, cat, user, out, b_in, err):
  99         x = self.c2x(c, cat)
 100         usage_acc_update2(x, out, b_in, err)
 101         if not err and cat == 'create_bucket' and 'owner' not in x:
 102             x['owner'] = user
 103     def make_entry(self, cat, bucket, user, out, b_in, err):
 104         if cat == 'create_bucket' and err:
 105                 return
 106         e = self.findentry(user)
 107         b = self.e2b(e, bucket)
 108         self.update(b['categories'], cat, user, out, b_in, err)
 109         s = self.findsum(user)
 110         x = self.c2x(s['categories'], cat)
 111         usage_acc_update2(x, out, b_in, err)
 112         x = s['total']
 113         usage_acc_update2(x, out, b_in, err)
 114     def generate_make_entry(self):
 115         return lambda cat,bucket,user,out,b_in,err: self.make_entry(cat, bucket, user, out, b_in, err)
 116     def get_usage(self):
 117         return self.results
 118     def compare_results(self, results):
 119         if 'entries' not in results or 'summary' not in results:
 120             return ['Missing entries or summary']
 121         r = []
 122         for e in self.results['entries']:
 123             try:
 124                 e2 = usage_acc_findentry2(results['entries'], e['user'], False)
 125             except Exception as ex:
 126                 r.append("malformed entry looking for user "
 127                     + e['user'] + " " + str(ex))
 128                 break
 129             if e2 == None:
 130                 r.append("missing entry for user " + e['user'])
 131                 continue
 132             for b in e['buckets']:
 133                 c = b['categories']
 134                 if b['bucket'] == 'nosuchbucket':
 135                     print("got here")
 136                 try:
 137                     b2 = self.e2b(e2, b['bucket'], False)
 138                     if b2 != None:
 139                             c2 = b2['categories']
 140                 except Exception as ex:
 141                     r.append("malformed entry looking for bucket "
 142                         + b['bucket'] + " in user " + e['user'] + " " + str(ex))
 143                     break
 144                 if b2 == None:
 145                     r.append("can't find bucket " + b['bucket']
 146                         + " in user " + e['user'])
 147                     continue
 148                 for x in c:
 149                     try:
 150                         x2 = self.c2x(c2, x['category'], False)
 151                     except Exception as ex:
 152                         r.append("malformed entry looking for "
 153                             + x['category'] + " in bucket " + b['bucket']
 154                             + " user " + e['user'] + " " + str(ex))
 155                         break
 156                     usage_acc_validate_fields(r, x, x2, "entry: category "
 157                         + x['category'] + " bucket " + b['bucket']
 158                         + " in user " + e['user'])
 159         for s in self.results['summary']:
 160             c = s['categories']
 161             try:
 162                 s2 = usage_acc_findsum2(results['summary'], s['user'], False)
 163             except Exception as ex:
 164                 r.append("malformed summary looking for user " + e['user']
 165                     + " " + str(ex))
 166                 break
 167                 if s2 == None:
 168                     r.append("missing summary for user " + e['user'] + " " + str(ex))
 169                     continue
 170             try:
 171                 c2 = s2['categories']
 172             except Exception as ex:
 173                 r.append("malformed summary missing categories for user "
 174                     + e['user'] + " " + str(ex))
 175                 break
 176             for x in c:
 177                 try:
 178                     x2 = self.c2x(c2, x['category'], False)
 179                 except Exception as ex:
 180                     r.append("malformed summary looking for "
 181                         + x['category'] + " user " + e['user'] + " " + str(ex))
 182                     break
 183                 usage_acc_validate_fields(r, x, x2, "summary: category "
 184                     + x['category'] + " in user " + e['user'])
 185             x = s['total']
 186             try:
 187                 x2 = s2['total']
 188             except Exception as ex:
 189                 r.append("malformed summary looking for totals for user "
 190                          + e['user'] + " " + str(ex))
 191                 break
 192             usage_acc_validate_fields(r, x, x2, "summary: totals for user" + e['user'])
 193         return r
 194
 195 def ignore_this_entry(cat, bucket, user, out, b_in, err):
 196     pass
 197 class requestlog_queue():
 198     def __init__(self, add):
 199         self.q = queue.Queue(1000)
 200         self.adder = add
 201     def handle_request_data(self, request, response, error=False):
 202         now = datetime.datetime.now()
 203         if error:
 204             pass
 205         elif response.status < 200 or response.status >= 400:
 206             error = True
 207         self.q.put({'t': now, 'o': request, 'i': response, 'e': error})
 208     def clear(self):
 209         with self.q.mutex:
 210             self.q.queue.clear()
 211     def log_and_clear(self, cat, bucket, user, add_entry = None):
 212         while not self.q.empty():
 213             j = self.q.get()
 214             bytes_out = 0
 215             if 'Content-Length' in j['o'].headers:
 216                 bytes_out = int(j['o'].headers['Content-Length'])
 217             bytes_in = 0
 218             msg = j['i'].msg if six.PY3 else j['i'].msg.dict
 219             if 'content-length'in msg:
 220                 bytes_in = int(msg['content-length'])
 221             log.info('RL: %s %s %s bytes_out=%d bytes_in=%d failed=%r'
 222                      % (cat, bucket, user, bytes_out, bytes_in, j['e']))
 223             if add_entry == None:
 224                 add_entry = self.adder
 225             add_entry(cat, bucket, user, bytes_out, bytes_in, j['e'])
 226
 227 def create_presigned_url(conn, method, bucket_name, key_name, expiration):
 228     return conn.generate_url(expires_in=expiration,
 229         method=method,
 230         bucket=bucket_name,
 231         key=key_name,
 232         query_auth=True,
 233     )
 234
 235 def send_raw_http_request(conn, method, bucket_name, key_name, follow_redirects = False):
 236     url = create_presigned_url(conn, method, bucket_name, key_name, 3600)
 237     print(url)
 238     h = httplib2.Http()
 239     h.follow_redirects = follow_redirects
 240     return h.request(url, method)
 241
 242
 243 def get_acl(key):
 244     """
 245     Helper function to get the xml acl from a key, ensuring that the xml
 246     version tag is removed from the acl response
 247     """
 248     raw_acl = six.ensure_str(key.get_xml_acl())
 249
 250     def remove_version(string):
 251         return string.split(
 252             '<?xml version="1.0" encoding="UTF-8"?>'
 253         )[-1]
 254
 255     def remove_newlines(string):
 256         return string.strip('\n')
 257
 258     return remove_version(
 259         remove_newlines(raw_acl)
 260     )
 261
 262 def task(ctx, config):
 263     """
 264     Test radosgw-admin functionality against a running rgw instance.
 265     """
 266     global log
 267
 268     assert ctx.rgw.config, \
 269         "radosgw_admin task needs a config passed from the rgw task"
 270     config = ctx.rgw.config
 271     log.debug('config is: %r', config)
 272
 273     clients_from_config = config.keys()
 274
 275     # choose first client as default
 276     client = next(iter(clients_from_config))
 277
 278     # once the client is chosen, pull the host name and  assigned port out of
 279     # the role_endpoints that were assigned by the rgw task
 280     endpoint = ctx.rgw.role_endpoints[client]
 281
 282     ##
 283     user1='foo'
 284     user2='fud'
 285     user3='bar'
 286     user4='bud'
 287     subuser1='foo:foo1'
 288     subuser2='foo:foo2'
 289     display_name1='Foo'
 290     display_name2='Fud'
 291     display_name3='Bar'
 292     email='foo@foo.com'
 293     access_key='9te6NH5mcdcq0Tc5i8i1'
 294     secret_key='Ny4IOauQoL18Gp2zM7lC1vLmoawgqcYP/YGcWfXu'
 295     access_key2='p5YnriCv1nAtykxBrupQ'
 296     secret_key2='Q8Tk6Q/27hfbFSYdSkPtUqhqx1GgzvpXa4WARozh'
 297     access_key3='NX5QOQKC6BH2IDN8HC7A'
 298     secret_key3='LnEsqNNqZIpkzauboDcLXLcYaWwLQ3Kop0zAnKIn'
 299     swift_secret1='gpS2G9RREMrnbqlp29PP2D36kgPR1tm72n5fPYfL'
 300     swift_secret2='ri2VJQcKSYATOY6uaDUX7pxgkW+W1YmC6OCxPHwy'
 301
 302     bucket_name='myfoo'
 303     bucket_name2='mybar'
 304
 305     # connect to rgw
 306     connection = boto.s3.connection.S3Connection(
 307         aws_access_key_id=access_key,
 308         aws_secret_access_key=secret_key,
 309         is_secure=False,
 310         port=endpoint.port,
 311         host=endpoint.hostname,
 312         calling_format=boto.s3.connection.OrdinaryCallingFormat(),
 313         )
 314     connection2 = boto.s3.connection.S3Connection(
 315         aws_access_key_id=access_key2,
 316         aws_secret_access_key=secret_key2,
 317         is_secure=False,
 318         port=endpoint.port,
 319         host=endpoint.hostname,
 320         calling_format=boto.s3.connection.OrdinaryCallingFormat(),
 321         )
 322     connection3 = boto.s3.connection.S3Connection(
 323         aws_access_key_id=access_key3,
 324         aws_secret_access_key=secret_key3,
 325         is_secure=False,
 326         port=endpoint.port,
 327         host=endpoint.hostname,
 328         calling_format=boto.s3.connection.OrdinaryCallingFormat(),
 329         )
 330
 331     acc = usage_acc()
 332     rl = requestlog_queue(acc.generate_make_entry())
 333     connection.set_request_hook(rl)
 334     connection2.set_request_hook(rl)
 335     connection3.set_request_hook(rl)
 336
 337     # legend (test cases can be easily grep-ed out)
 338     # TESTCASE 'testname','object','method','operation','assertion'
 339
 340     # TESTCASE 'usage-show0' 'usage' 'show' 'all usage' 'succeeds'
 341     (err, summary0) = rgwadmin(ctx, client, ['usage', 'show'], check_status=True)
 342
 343     # TESTCASE 'info-nosuch','user','info','non-existent user','fails'
 344     (err, out) = rgwadmin(ctx, client, ['user', 'info', '--uid', user1])
 345     assert err
 346
 347     # TESTCASE 'create-ok','user','create','w/all valid info','succeeds'
 348     (err, out) = rgwadmin(ctx, client, [
 349             'user', 'create',
 350             '--uid', user1,
 351             '--display-name', display_name1,
 352             '--email', email,
 353             '--access-key', access_key,
 354             '--secret', secret_key,
 355             '--max-buckets', '4'
 356             ],
 357             check_status=True)
 358
 359     # TESTCASE 'duplicate email','user','create','existing user email','fails'
 360     (err, out) = rgwadmin(ctx, client, [
 361             'user', 'create',
 362             '--uid', user2,
 363             '--display-name', display_name2,
 364             '--email', email,
 365             ])
 366     assert err
 367
 368     # TESTCASE 'info-existing','user','info','existing user','returns correct info'
 369     (err, out) = rgwadmin(ctx, client, ['user', 'info', '--uid', user1], check_status=True)
 370     assert out['user_id'] == user1
 371     assert out['email'] == email
 372     assert out['display_name'] == display_name1
 373     assert len(out['keys']) == 1
 374     assert out['keys'][0]['access_key'] == access_key
 375     assert out['keys'][0]['secret_key'] == secret_key
 376     assert not out['suspended']
 377
 378     # TESTCASE 'suspend-ok','user','suspend','active user','succeeds'
 379     (err, out) = rgwadmin(ctx, client, ['user', 'suspend', '--uid', user1],
 380         check_status=True)
 381
 382     # TESTCASE 'suspend-suspended','user','suspend','suspended user','succeeds w/advisory'
 383     (err, out) = rgwadmin(ctx, client, ['user', 'info', '--uid', user1], check_status=True)
 384     assert out['suspended']
 385
 386     # TESTCASE 're-enable','user','enable','suspended user','succeeds'
 387     (err, out) = rgwadmin(ctx, client, ['user', 'enable', '--uid', user1], check_status=True)
 388
 389     # TESTCASE 'info-re-enabled','user','info','re-enabled user','no longer suspended'
 390     (err, out) = rgwadmin(ctx, client, ['user', 'info', '--uid', user1], check_status=True)
 391     assert not out['suspended']
 392
 393     # TESTCASE 'add-keys','key','create','w/valid info','succeeds'
 394     (err, out) = rgwadmin(ctx, client, [
 395             'key', 'create', '--uid', user1,
 396             '--access-key', access_key2, '--secret', secret_key2,
 397             ], check_status=True)
 398
 399     # TESTCASE 'info-new-key','user','info','after key addition','returns all keys'
 400     (err, out) = rgwadmin(ctx, client, ['user', 'info', '--uid', user1],
 401         check_status=True)
 402     assert len(out['keys']) == 2
 403     assert out['keys'][0]['access_key'] == access_key2 or out['keys'][1]['access_key'] == access_key2
 404     assert out['keys'][0]['secret_key'] == secret_key2 or out['keys'][1]['secret_key'] == secret_key2
 405
 406     # TESTCASE 'rm-key','key','rm','newly added key','succeeds, key is removed'
 407     (err, out) = rgwadmin(ctx, client, [
 408             'key', 'rm', '--uid', user1,
 409             '--access-key', access_key2,
 410             ], check_status=True)
 411     assert len(out['keys']) == 1
 412     assert out['keys'][0]['access_key'] == access_key
 413     assert out['keys'][0]['secret_key'] == secret_key
 414
 415     # TESTCASE 'add-swift-key','key','create','swift key','succeeds'
 416     subuser_access = 'full'
 417     subuser_perm = 'full-control'
 418
 419     (err, out) = rgwadmin(ctx, client, [
 420             'subuser', 'create', '--subuser', subuser1,
 421             '--access', subuser_access
 422             ], check_status=True)
 423
 424     # TESTCASE 'add-swift-key','key','create','swift key','succeeds'
 425     (err, out) = rgwadmin(ctx, client, [
 426             'subuser', 'modify', '--subuser', subuser1,
 427             '--secret', swift_secret1,
 428             '--key-type', 'swift',
 429             ], check_status=True)
 430
 431     # TESTCASE 'subuser-perm-mask', 'subuser', 'info', 'test subuser perm mask durability', 'succeeds'
 432     (err, out) = rgwadmin(ctx, client, ['user', 'info', '--uid', user1])
 433
 434     assert out['subusers'][0]['permissions'] == subuser_perm
 435
 436     # TESTCASE 'info-swift-key','user','info','after key addition','returns all keys'
 437     (err, out) = rgwadmin(ctx, client, ['user', 'info', '--uid', user1], check_status=True)
 438     assert len(out['swift_keys']) == 1
 439     assert out['swift_keys'][0]['user'] == subuser1
 440     assert out['swift_keys'][0]['secret_key'] == swift_secret1
 441
 442     # TESTCASE 'add-swift-subuser','key','create','swift sub-user key','succeeds'
 443     (err, out) = rgwadmin(ctx, client, [
 444             'subuser', 'create', '--subuser', subuser2,
 445             '--secret', swift_secret2,
 446             '--key-type', 'swift',
 447             ], check_status=True)
 448
 449     # TESTCASE 'info-swift-subuser','user','info','after key addition','returns all sub-users/keys'
 450     (err, out) = rgwadmin(ctx, client, ['user', 'info', '--uid', user1], check_status=True)
 451     assert len(out['swift_keys']) == 2
 452     assert out['swift_keys'][0]['user'] == subuser2 or out['swift_keys'][1]['user'] == subuser2
 453     assert out['swift_keys'][0]['secret_key'] == swift_secret2 or out['swift_keys'][1]['secret_key'] == swift_secret2
 454
 455     # TESTCASE 'rm-swift-key1','key','rm','subuser','succeeds, one key is removed'
 456     (err, out) = rgwadmin(ctx, client, [
 457             'key', 'rm', '--subuser', subuser1,
 458             '--key-type', 'swift',
 459             ], check_status=True)
 460     assert len(out['swift_keys']) == 1
 461
 462     # TESTCASE 'rm-subuser','subuser','rm','subuser','success, subuser is removed'
 463     (err, out) = rgwadmin(ctx, client, [
 464             'subuser', 'rm', '--subuser', subuser1,
 465             ], check_status=True)
 466     assert len(out['subusers']) == 1
 467
 468     # TESTCASE 'rm-subuser-with-keys','subuser','rm','subuser','succeeds, second subser and key is removed'
 469     (err, out) = rgwadmin(ctx, client, [
 470             'subuser', 'rm', '--subuser', subuser2,
 471             '--key-type', 'swift', '--purge-keys',
 472             ], check_status=True)
 473     assert len(out['swift_keys']) == 0
 474     assert len(out['subusers']) == 0
 475
 476     # TESTCASE 'bucket-stats','bucket','stats','no session/buckets','succeeds, empty list'
 477     (err, out) = rgwadmin(ctx, client, ['bucket', 'stats', '--uid', user1],
 478         check_status=True)
 479     assert len(out) == 0
 480
 481     # TESTCASE 'bucket-stats2','bucket','stats','no buckets','succeeds, empty list'
 482     (err, out) = rgwadmin(ctx, client, ['bucket', 'list', '--uid', user1], check_status=True)
 483     assert len(out) == 0
 484
 485     # create a first bucket
 486     bucket = connection.create_bucket(bucket_name)
 487
 488     rl.log_and_clear("create_bucket", bucket_name, user1)
 489
 490     # TESTCASE 'bucket-list','bucket','list','one bucket','succeeds, expected list'
 491     (err, out) = rgwadmin(ctx, client, ['bucket', 'list', '--uid', user1], check_status=True)
 492     assert len(out) == 1
 493     assert out[0] == bucket_name
 494
 495     bucket_list = connection.get_all_buckets()
 496     assert len(bucket_list) == 1
 497     assert bucket_list[0].name == bucket_name
 498
 499     rl.log_and_clear("list_buckets", '', user1)
 500
 501     # TESTCASE 'bucket-list-all','bucket','list','all buckets','succeeds, expected list'
 502     (err, out) = rgwadmin(ctx, client, ['bucket', 'list'], check_status=True)
 503     assert len(out) >= 1
 504     assert bucket_name in out;
 505
 506     # TESTCASE 'max-bucket-limit,'bucket','create','4 buckets','5th bucket fails due to max buckets == 4'
 507     bucket2 = connection.create_bucket(bucket_name + '2')
 508     rl.log_and_clear("create_bucket", bucket_name + '2', user1)
 509     bucket3 = connection.create_bucket(bucket_name + '3')
 510     rl.log_and_clear("create_bucket", bucket_name + '3', user1)
 511     bucket4 = connection.create_bucket(bucket_name + '4')
 512     rl.log_and_clear("create_bucket", bucket_name + '4', user1)
 513     # the 5th should fail.
 514     failed = False
 515     try:
 516         connection.create_bucket(bucket_name + '5')
 517     except Exception:
 518         failed = True
 519     assert failed
 520     rl.log_and_clear("create_bucket", bucket_name + '5', user1)
 521
 522     # delete the buckets
 523     bucket2.delete()
 524     rl.log_and_clear("delete_bucket", bucket_name + '2', user1)
 525     bucket3.delete()
 526     rl.log_and_clear("delete_bucket", bucket_name + '3', user1)
 527     bucket4.delete()
 528     rl.log_and_clear("delete_bucket", bucket_name + '4', user1)
 529
 530     # TESTCASE 'bucket-stats3','bucket','stats','new empty bucket','succeeds, empty list'
 531     (err, out) = rgwadmin(ctx, client, [
 532             'bucket', 'stats', '--bucket', bucket_name], check_status=True)
 533     assert out['owner'] == user1
 534     bucket_id = out['id']
 535
 536     # TESTCASE 'bucket-stats4','bucket','stats','new empty bucket','succeeds, expected bucket ID'
 537     (err, out) = rgwadmin(ctx, client, ['bucket', 'stats', '--uid', user1], check_status=True)
 538     assert len(out) == 1
 539     assert out[0]['id'] == bucket_id    # does it return the same ID twice in a row?
 540
 541     # use some space
 542     key = boto.s3.key.Key(bucket)
 543     key.set_contents_from_string('one')
 544     rl.log_and_clear("put_obj", bucket_name, user1)
 545
 546     # TESTCASE 'bucket-stats5','bucket','stats','after creating key','succeeds, lists one non-empty object'
 547     (err, out) = rgwadmin(ctx, client, [
 548             'bucket', 'stats', '--bucket', bucket_name], check_status=True)
 549     assert out['id'] == bucket_id
 550     assert out['usage']['rgw.main']['num_objects'] == 1
 551     assert out['usage']['rgw.main']['size_kb'] > 0
 552
 553     #validate we have a positive user stats now
 554     (err, out) = rgwadmin(ctx, client,
 555                           ['user', 'stats','--uid', user1, '--sync-stats'],
 556                           check_status=True)
 557     assert out['stats']['size'] > 0
 558
 559     # reclaim it
 560     key.delete()
 561     rl.log_and_clear("delete_obj", bucket_name, user1)
 562
 563     # TESTCASE 'bucket unlink', 'bucket', 'unlink', 'unlink bucket from user', 'fails', 'access denied error'
 564     (err, out) = rgwadmin(ctx, client,
 565         ['bucket', 'unlink', '--uid', user1, '--bucket', bucket_name],
 566         check_status=True)
 567
 568     # create a second user to link the bucket to
 569     (err, out) = rgwadmin(ctx, client, [
 570             'user', 'create',
 571             '--uid', user2,
 572             '--display-name', display_name2,
 573             '--access-key', access_key2,
 574             '--secret', secret_key2,
 575             '--max-buckets', '1',
 576             ],
 577             check_status=True)
 578
 579     # try creating an object with the first user before the bucket is relinked
 580     denied = False
 581     key = boto.s3.key.Key(bucket)
 582
 583     try:
 584         key.set_contents_from_string('two')
 585     except boto.exception.S3ResponseError:
 586         denied = True
 587
 588     assert not denied
 589     rl.log_and_clear("put_obj", bucket_name, user1)
 590
 591     # delete the object
 592     key.delete()
 593     rl.log_and_clear("delete_obj", bucket_name, user1)
 594
 595     # link the bucket to another user
 596     (err, out) = rgwadmin(ctx, client, ['metadata', 'get', 'bucket:{n}'.format(n=bucket_name)],
 597         check_status=True)
 598
 599     bucket_data = out['data']
 600     assert bucket_data['bucket']['name'] == bucket_name
 601
 602     bucket_id = bucket_data['bucket']['bucket_id']
 603
 604     # link the bucket to another user
 605     (err, out) = rgwadmin(ctx, client, ['bucket', 'link', '--uid', user2, '--bucket', bucket_name, '--bucket-id', bucket_id],
 606         check_status=True)
 607
 608     # try to remove user, should fail (has a linked bucket)
 609     (err, out) = rgwadmin(ctx, client, ['user', 'rm', '--uid', user2])
 610     assert err
 611
 612     # TESTCASE 'bucket unlink', 'bucket', 'unlink', 'unlink bucket from user', 'succeeds, bucket unlinked'
 613     (err, out) = rgwadmin(ctx, client, ['bucket', 'unlink', '--uid', user2, '--bucket', bucket_name],
 614         check_status=True)
 615
 616     # relink the bucket to the first user and delete the second user
 617     (err, out) = rgwadmin(ctx, client,
 618         ['bucket', 'link', '--uid', user1, '--bucket', bucket_name, '--bucket-id', bucket_id],
 619         check_status=True)
 620
 621     (err, out) = rgwadmin(ctx, client, ['user', 'rm', '--uid', user2],
 622         check_status=True)
 623
 624     #TESTCASE 'bucket link', 'bucket', 'tenanted user', 'succeeds'
 625     tenant_name = "testx"
 626     # create a tenanted user to link the bucket to
 627     (err, out) = rgwadmin(ctx, client, [
 628             'user', 'create',
 629             '--tenant', tenant_name,
 630             '--uid', 'tenanteduser',
 631             '--display-name', 'tenanted-user',
 632             '--access-key', access_key2,
 633             '--secret', secret_key2,
 634             '--max-buckets', '1',
 635             ],
 636             check_status=True)
 637
 638     # link the bucket to a tenanted user
 639     (err, out) = rgwadmin(ctx, client, ['bucket', 'link', '--bucket', '/' + bucket_name, '--tenant', tenant_name, '--uid', 'tenanteduser'],
 640         check_status=True)
 641
 642     # check if the bucket name has tenant/ prefix
 643     (err, out) = rgwadmin(ctx, client, ['metadata', 'get', 'bucket:{n}'.format(n= tenant_name + '/' + bucket_name)],
 644         check_status=True)
 645
 646     bucket_data = out['data']
 647     assert bucket_data['bucket']['name'] == bucket_name
 648     assert bucket_data['bucket']['tenant'] == tenant_name
 649
 650     # relink the bucket to the first user and delete the tenanted user
 651     (err, out) = rgwadmin(ctx, client,
 652         ['bucket', 'link', '--bucket', tenant_name + '/' + bucket_name, '--uid', user1],
 653         check_status=True)
 654
 655     (err, out) = rgwadmin(ctx, client, ['user', 'rm', '--tenant', tenant_name, '--uid', 'tenanteduser'],
 656         check_status=True)
 657
 658     # TESTCASE 'object-rm', 'object', 'rm', 'remove object', 'succeeds, object is removed'
 659
 660     # upload an object
 661     object_name = 'four'
 662     key = boto.s3.key.Key(bucket, object_name)
 663     key.set_contents_from_string(object_name)
 664     rl.log_and_clear("put_obj", bucket_name, user1)
 665
 666     # fetch it too (for usage stats presently)
 667     s = key.get_contents_as_string(encoding='ascii')
 668     rl.log_and_clear("get_obj", bucket_name, user1)
 669     assert s == object_name
 670     # list bucket too (for usage stats presently)
 671     keys = list(bucket.list())
 672     rl.log_and_clear("list_bucket", bucket_name, user1)
 673     assert len(keys) == 1
 674     assert keys[0].name == object_name
 675
 676     # now delete it
 677     (err, out) = rgwadmin(ctx, client,
 678         ['object', 'rm', '--bucket', bucket_name, '--object', object_name],
 679         check_status=True)
 680
 681     # TESTCASE 'bucket-stats6','bucket','stats','after deleting key','succeeds, lists one no objects'
 682     (err, out) = rgwadmin(ctx, client, [
 683             'bucket', 'stats', '--bucket', bucket_name],
 684             check_status=True)
 685     assert out['id'] == bucket_id
 686     assert out['usage']['rgw.main']['num_objects'] == 0
 687
 688     # list log objects
 689     # TESTCASE 'log-list','log','list','after activity','succeeds, lists one no objects'
 690     (err, out) = rgwadmin(ctx, client, ['log', 'list'], check_status=True)
 691     assert len(out) > 0
 692
 693     for obj in out:
 694         # TESTCASE 'log-show','log','show','after activity','returns expected info'
 695         if obj[:4] == 'meta' or obj[:4] == 'data' or obj[:18] == 'obj_delete_at_hint':
 696             continue
 697
 698         (err, rgwlog) = rgwadmin(ctx, client, ['log', 'show', '--object', obj],
 699             check_status=True)
 700         assert len(rgwlog) > 0
 701
 702         # exempt bucket_name2 from checking as it was only used for multi-region tests
 703         assert rgwlog['bucket'].find(bucket_name) == 0 or rgwlog['bucket'].find(bucket_name2) == 0
 704         assert rgwlog['bucket'] != bucket_name or rgwlog['bucket_id'] == bucket_id
 705         assert rgwlog['bucket_owner'] == user1 or rgwlog['bucket'] == bucket_name + '5' or rgwlog['bucket'] == bucket_name2
 706         for entry in rgwlog['log_entries']:
 707             log.debug('checking log entry: ', entry)
 708             assert entry['bucket'] == rgwlog['bucket']
 709             possible_buckets = [bucket_name + '5', bucket_name2]
 710             user = entry['user']
 711             assert user == user1 or user.endswith('system-user') or \
 712                 rgwlog['bucket'] in possible_buckets
 713
 714         # TESTCASE 'log-rm','log','rm','delete log objects','succeeds'
 715         (err, out) = rgwadmin(ctx, client, ['log', 'rm', '--object', obj],
 716             check_status=True)
 717
 718     # TODO: show log by bucket+date
 719
 720     # TESTCASE 'user-suspend2','user','suspend','existing user','succeeds'
 721     (err, out) = rgwadmin(ctx, client, ['user', 'suspend', '--uid', user1],
 722         check_status=True)
 723
 724     # TESTCASE 'user-suspend3','user','suspend','suspended user','cannot write objects'
 725     denied = False
 726     try:
 727         key = boto.s3.key.Key(bucket)
 728         key.set_contents_from_string('five')
 729     except boto.exception.S3ResponseError as e:
 730         denied = True
 731         assert e.status == 403
 732
 733     assert denied
 734     rl.log_and_clear("put_obj", bucket_name, user1)
 735
 736     # TESTCASE 'user-renable2','user','enable','suspended user','succeeds'
 737     (err, out) = rgwadmin(ctx, client, ['user', 'enable', '--uid', user1],
 738         check_status=True)
 739
 740     # TESTCASE 'user-renable3','user','enable','reenabled user','can write objects'
 741     key = boto.s3.key.Key(bucket)
 742     key.set_contents_from_string('six')
 743     rl.log_and_clear("put_obj", bucket_name, user1)
 744
 745     # TESTCASE 'gc-list', 'gc', 'list', 'get list of objects ready for garbage collection'
 746
 747     # create an object large enough to be split into multiple parts
 748     test_string = 'foo'*10000000
 749
 750     big_key = boto.s3.key.Key(bucket)
 751     big_key.set_contents_from_string(test_string)
 752     rl.log_and_clear("put_obj", bucket_name, user1)
 753
 754     # now delete the head
 755     big_key.delete()
 756     rl.log_and_clear("delete_obj", bucket_name, user1)
 757
 758     # wait a bit to give the garbage collector time to cycle
 759     time.sleep(15)
 760
 761     (err, out) = rgwadmin(ctx, client, ['gc', 'list'])
 762
 763     assert len(out) > 0
 764
 765     # TESTCASE 'gc-process', 'gc', 'process', 'manually collect garbage'
 766     (err, out) = rgwadmin(ctx, client, ['gc', 'process'], check_status=True)
 767
 768     #confirm
 769     (err, out) = rgwadmin(ctx, client, ['gc', 'list'])
 770
 771     assert len(out) == 0
 772
 773     # TESTCASE 'rm-user-buckets','user','rm','existing user','fails, still has buckets'
 774     (err, out) = rgwadmin(ctx, client, ['user', 'rm', '--uid', user1])
 775     assert err
 776
 777     # delete should fail because ``key`` still exists
 778     try:
 779         bucket.delete()
 780     except boto.exception.S3ResponseError as e:
 781         assert e.status == 409
 782     rl.log_and_clear("delete_bucket", bucket_name, user1)
 783
 784     key.delete()
 785     rl.log_and_clear("delete_obj", bucket_name, user1)
 786     bucket.delete()
 787     rl.log_and_clear("delete_bucket", bucket_name, user1)
 788
 789     # TESTCASE 'policy', 'bucket', 'policy', 'get bucket policy', 'returns S3 policy'
 790     bucket = connection.create_bucket(bucket_name)
 791     rl.log_and_clear("create_bucket", bucket_name, user1)
 792
 793     # create an object
 794     key = boto.s3.key.Key(bucket)
 795     key.set_contents_from_string('seven')
 796     rl.log_and_clear("put_obj", bucket_name, user1)
 797
 798     # should be private already but guarantee it
 799     key.set_acl('private')
 800     rl.log_and_clear("put_acls", bucket_name, user1)
 801
 802     (err, out) = rgwadmin(ctx, client,
 803         ['policy', '--bucket', bucket.name, '--object', six.ensure_str(key.key)],
 804         check_status=True, format='xml')
 805
 806     acl = get_acl(key)
 807     rl.log_and_clear("get_acls", bucket_name, user1)
 808
 809     assert acl == out.strip('\n')
 810
 811     # add another grantee by making the object public read
 812     key.set_acl('public-read')
 813     rl.log_and_clear("put_acls", bucket_name, user1)
 814
 815     (err, out) = rgwadmin(ctx, client,
 816         ['policy', '--bucket', bucket.name, '--object', six.ensure_str(key.key)],
 817         check_status=True, format='xml')
 818
 819     acl = get_acl(key)
 820     rl.log_and_clear("get_acls", bucket_name, user1)
 821
 822     assert acl == out.strip('\n')
 823
 824     # TESTCASE 'rm-bucket', 'bucket', 'rm', 'bucket with objects', 'succeeds'
 825     bucket = connection.create_bucket(bucket_name)
 826     rl.log_and_clear("create_bucket", bucket_name, user1)
 827     key_name = ['eight', 'nine', 'ten', 'eleven']
 828     for i in range(4):
 829         key = boto.s3.key.Key(bucket)
 830         key.set_contents_from_string(key_name[i])
 831     rl.log_and_clear("put_obj", bucket_name, user1)
 832
 833     (err, out) = rgwadmin(ctx, client,
 834         ['bucket', 'rm', '--bucket', bucket_name, '--purge-objects'],
 835         check_status=True)
 836
 837     # TESTCASE 'caps-add', 'caps', 'add', 'add user cap', 'succeeds'
 838     caps='user=read'
 839     (err, out) = rgwadmin(ctx, client, ['caps', 'add', '--uid', user1, '--caps', caps])
 840
 841     assert out['caps'][0]['perm'] == 'read'
 842
 843     # TESTCASE 'caps-rm', 'caps', 'rm', 'remove existing cap from user', 'succeeds'
 844     (err, out) = rgwadmin(ctx, client, ['caps', 'rm', '--uid', user1, '--caps', caps])
 845
 846     assert not out['caps']
 847
 848     # TESTCASE 'rm-user','user','rm','existing user','fails, still has buckets'
 849     bucket = connection.create_bucket(bucket_name)
 850     rl.log_and_clear("create_bucket", bucket_name, user1)
 851     key = boto.s3.key.Key(bucket)
 852
 853     (err, out) = rgwadmin(ctx, client, ['user', 'rm', '--uid', user1])
 854     assert err
 855
 856     # TESTCASE 'rm-user2', 'user', 'rm', 'user with data', 'succeeds'
 857     bucket = connection.create_bucket(bucket_name)
 858     rl.log_and_clear("create_bucket", bucket_name, user1)
 859     key = boto.s3.key.Key(bucket)
 860     key.set_contents_from_string('twelve')
 861     rl.log_and_clear("put_obj", bucket_name, user1)
 862
 863     time.sleep(35)
 864
 865     # need to wait for all usage data to get flushed, should take up to 30 seconds
 866     timestamp = time.time()
 867     while time.time() - timestamp <= (2 * 60):      # wait up to 20 minutes
 868         (err, out) = rgwadmin(ctx, client, ['usage', 'show', '--categories', 'delete_obj'])  # one of the operations we did is delete_obj, should be present.
 869         if get_user_successful_ops(out, user1) > 0:
 870             break
 871         time.sleep(1)
 872
 873     assert time.time() - timestamp <= (20 * 60)
 874
 875     # TESTCASE 'usage-show' 'usage' 'show' 'all usage' 'succeeds'
 876     (err, out) = rgwadmin(ctx, client, ['usage', 'show'], check_status=True)
 877     assert len(out['entries']) > 0
 878     assert len(out['summary']) > 0
 879
 880     r = acc.compare_results(out)
 881     if len(r) != 0:
 882         sys.stderr.write(("\n".join(r))+"\n")
 883         assert(len(r) == 0)
 884
 885     user_summary = get_user_summary(out, user1)
 886
 887     total = user_summary['total']
 888     assert total['successful_ops'] > 0
 889
 890     # TESTCASE 'usage-show2' 'usage' 'show' 'user usage' 'succeeds'
 891     (err, out) = rgwadmin(ctx, client, ['usage', 'show', '--uid', user1],
 892         check_status=True)
 893     assert len(out['entries']) > 0
 894     assert len(out['summary']) > 0
 895     user_summary = out['summary'][0]
 896     for entry in user_summary['categories']:
 897         assert entry['successful_ops'] > 0
 898     assert user_summary['user'] == user1
 899
 900     # TESTCASE 'usage-show3' 'usage' 'show' 'user usage categories' 'succeeds'
 901     test_categories = ['create_bucket', 'put_obj', 'delete_obj', 'delete_bucket']
 902     for cat in test_categories:
 903         (err, out) = rgwadmin(ctx, client, ['usage', 'show', '--uid', user1, '--categories', cat],
 904             check_status=True)
 905         assert len(out['summary']) > 0
 906         user_summary = out['summary'][0]
 907         assert user_summary['user'] == user1
 908         assert len(user_summary['categories']) == 1
 909         entry = user_summary['categories'][0]
 910         assert entry['category'] == cat
 911         assert entry['successful_ops'] > 0
 912
 913     # TESTCASE 'user-rename', 'user', 'rename', 'existing user', 'new user', 'succeeds'
 914     # create a new user user3
 915     (err, out) = rgwadmin(ctx, client, [
 916         'user', 'create',
 917         '--uid', user3,
 918         '--display-name', display_name3,
 919         '--access-key', access_key3,
 920         '--secret', secret_key3,
 921         '--max-buckets', '4'
 922         ],
 923         check_status=True)
 924
 925     # create a bucket
 926     bucket = connection3.create_bucket(bucket_name + '6')
 927
 928     rl.log_and_clear("create_bucket", bucket_name + '6', user3)
 929
 930     # create object
 931     object_name1 = 'thirteen'
 932     key1 = boto.s3.key.Key(bucket, object_name1)
 933     key1.set_contents_from_string(object_name1)
 934     rl.log_and_clear("put_obj", bucket_name + '6', user3)
 935
 936     # rename user3
 937     (err, out) = rgwadmin(ctx, client, ['user', 'rename', '--uid', user3, '--new-uid', user4], check_status=True)
 938     assert out['user_id'] == user4
 939     assert out['keys'][0]['access_key'] == access_key3
 940     assert out['keys'][0]['secret_key'] == secret_key3
 941
 942     time.sleep(5)
 943
 944     # get bucket and object to test if user keys are preserved
 945     bucket = connection3.get_bucket(bucket_name + '6')
 946     s = key1.get_contents_as_string(encoding='ascii')
 947     rl.log_and_clear("get_obj", bucket_name + '6', user4)
 948     assert s == object_name1
 949
 950     # TESTCASE 'user-rename', 'user', 'rename', 'existing user', 'another existing user', 'fails'
 951     # create a new user user2
 952     (err, out) = rgwadmin(ctx, client, [
 953         'user', 'create',
 954         '--uid', user2,
 955         '--display-name', display_name2,
 956         '--access-key', access_key2,
 957         '--secret', secret_key2,
 958         '--max-buckets', '4'
 959         ],
 960         check_status=True)
 961
 962     # create a bucket
 963     bucket = connection2.create_bucket(bucket_name + '7')
 964
 965     rl.log_and_clear("create_bucket", bucket_name + '7', user2)
 966
 967     # create object
 968     object_name2 = 'fourteen'
 969     key2 = boto.s3.key.Key(bucket, object_name2)
 970     key2.set_contents_from_string(object_name2)
 971     rl.log_and_clear("put_obj", bucket_name + '7', user2)
 972
 973     (err, out) = rgwadmin(ctx, client, ['user', 'rename', '--uid', user4, '--new-uid', user2])
 974     assert err
 975
 976     # test if user 2 and user4 can still access their bucket and objects after rename fails
 977     bucket = connection3.get_bucket(bucket_name + '6')
 978     s = key1.get_contents_as_string(encoding='ascii')
 979     rl.log_and_clear("get_obj", bucket_name + '6', user4)
 980     assert s == object_name1
 981
 982     bucket = connection2.get_bucket(bucket_name + '7')
 983     s = key2.get_contents_as_string(encoding='ascii')
 984     rl.log_and_clear("get_obj", bucket_name + '7', user2)
 985     assert s == object_name2
 986
 987     (err, out) = rgwadmin(ctx, client,
 988     ['user', 'rm', '--uid', user4, '--purge-data' ],
 989     check_status=True)
 990
 991     (err, out) = rgwadmin(ctx, client,
 992     ['user', 'rm', '--uid', user2, '--purge-data' ],
 993     check_status=True)
 994
 995     time.sleep(5)
 996
 997     # should be all through with connection. (anything using connection
 998     #  should be BEFORE the usage stuff above.)
 999     rl.log_and_clear("(before-close)", '-', '-', ignore_this_entry)
1000     connection.close()
1001     connection = None
1002
1003     # the usage flush interval is 30 seconds, wait that much an then some
1004     # to make sure everything has been flushed
1005     time.sleep(35)
1006
1007     # TESTCASE 'usage-trim' 'usage' 'trim' 'user usage' 'succeeds, usage removed'
1008     (err, out) = rgwadmin(ctx, client, ['usage', 'trim', '--uid', user1],
1009         check_status=True)
1010     (err, out) = rgwadmin(ctx, client, ['usage', 'show', '--uid', user1],
1011         check_status=True)
1012     assert len(out['entries']) == 0
1013     assert len(out['summary']) == 0
1014
1015     (err, out) = rgwadmin(ctx, client,
1016         ['user', 'rm', '--uid', user1, '--purge-data' ],
1017         check_status=True)
1018
1019     # TESTCASE 'rm-user3','user','rm','deleted user','fails'
1020     (err, out) = rgwadmin(ctx, client, ['user', 'info', '--uid', user1])
1021     assert err
1022
1023     # TESTCASE 'zone-info', 'zone', 'get', 'get zone info', 'succeeds, has default placement rule'
1024     #
1025
1026     (err, out) = rgwadmin(ctx, client, ['zone', 'get','--rgw-zone','default'])
1027     orig_placement_pools = len(out['placement_pools'])
1028
1029     # removed this test, it is not correct to assume that zone has default placement, it really
1030     # depends on how we set it up before
1031     #
1032     # assert len(out) > 0
1033     # assert len(out['placement_pools']) == 1
1034
1035     # default_rule = out['placement_pools'][0]
1036     # assert default_rule['key'] == 'default-placement'
1037
1038     rule={'key': 'new-placement', 'val': {'data_pool': '.rgw.buckets.2', 'index_pool': '.rgw.buckets.index.2'}}
1039
1040     out['placement_pools'].append(rule)
1041
1042     (err, out) = rgwadmin(ctx, client, ['zone', 'set'],
1043         stdin=BytesIO(six.ensure_binary(json.dumps(out))),
1044         check_status=True)
1045
1046     (err, out) = rgwadmin(ctx, client, ['zone', 'get'])
1047     assert len(out) > 0
1048     assert len(out['placement_pools']) == orig_placement_pools + 1
1049
1050     zonecmd = ['zone', 'placement', 'rm',
1051                '--rgw-zone', 'default',
1052                '--placement-id', 'new-placement']
1053
1054     (err, out) = rgwadmin(ctx, client, zonecmd, check_status=True)
1055
1056     # TESTCASE 'zonegroup-info', 'zonegroup', 'get', 'get zonegroup info', 'succeeds'
1057     (err, out) = rgwadmin(ctx, client, ['zonegroup', 'get'], check_status=True)
1058
1059 from teuthology.config import config
1060 from teuthology.orchestra import cluster, remote
1061 import argparse;
1062
1063 def main():
1064     if len(sys.argv) == 3:
1065         user = sys.argv[1] + "@"
1066         host = sys.argv[2]
1067     elif len(sys.argv) == 2:
1068         user = ""
1069         host = sys.argv[1]
1070     else:
1071         sys.stderr.write("usage: radosgw_admin.py [user] host\n")
1072         exit(1)
1073     client0 = remote.Remote(user + host)
1074     ctx = config
1075     ctx.cluster=cluster.Cluster(remotes=[(client0,
1076         [ 'ceph.client.rgw.%s' % (host),  ]),])
1077     ctx.rgw = argparse.Namespace()
1078     endpoints = {}
1079     endpoints['ceph.client.rgw.%s' % host] = (host, 80)
1080     ctx.rgw.role_endpoints = endpoints
1081     ctx.rgw.realm = None
1082     ctx.rgw.regions = {'region0': { 'api name': 'api1',
1083         'is master': True, 'master zone': 'r0z0',
1084         'zones': ['r0z0', 'r0z1'] }}
1085     ctx.rgw.config = {'ceph.client.rgw.%s' % host: {'system user': {'name': '%s-system-user' % host}}}
1086     task(config, None)
1087     exit()
1088
1089 if __name__ == '__main__':
1090     main()