ceph/qa/tasks/radosgw_admin.py

   1 """
   2 Rgw admin testing against a running instance
   3 """
   4 # The test cases in this file have been annotated for inventory.
   5 # To extract the inventory (in csv format) use the command:
   6 #
   7 #   grep '^ *# TESTCASE' | sed 's/^ *# TESTCASE //'
   8 #
   9 # to run this standalone:
  10 #       python qa/tasks/radosgw_admin.py [USER] HOSTNAME
  11 #
  12
  13 import json
  14 import logging
  15 import time
  16 import datetime
  17 import Queue
  18
  19 import sys
  20
  21 from cStringIO import StringIO
  22
  23 import boto.exception
  24 import boto.s3.connection
  25 import boto.s3.acl
  26
  27 import httplib2
  28
  29
  30 from util.rgw import rgwadmin, get_user_summary, get_user_successful_ops
  31
  32 log = logging.getLogger(__name__)
  33
  34 def usage_acc_findentry2(entries, user, add=True):
  35     for e in entries:
  36         if e['user'] == user:
  37             return e
  38     if not add:
  39             return None
  40     e = {'user': user, 'buckets': []}
  41     entries.append(e)
  42     return e
  43 def usage_acc_findsum2(summaries, user, add=True):
  44     for e in summaries:
  45         if e['user'] == user:
  46             return e
  47     if not add:
  48         return None
  49     e = {'user': user, 'categories': [],
  50         'total': {'bytes_received': 0,
  51             'bytes_sent': 0, 'ops': 0, 'successful_ops': 0 }}
  52     summaries.append(e)
  53     return e
  54 def usage_acc_update2(x, out, b_in, err):
  55     x['bytes_sent'] += b_in
  56     x['bytes_received'] += out
  57     x['ops'] += 1
  58     if not err:
  59         x['successful_ops'] += 1
  60 def usage_acc_validate_fields(r, x, x2, what):
  61     q=[]
  62     for field in ['bytes_sent', 'bytes_received', 'ops', 'successful_ops']:
  63         try:
  64             if x2[field] < x[field]:
  65                 q.append("field %s: %d < %d" % (field, x2[field], x[field]))
  66         except Exception as ex:
  67             r.append( "missing/bad field " + field + " in " + what + " " + str(ex))
  68             return
  69     if len(q) > 0:
  70         r.append("incomplete counts in " + what + ": " + ", ".join(q))
  71 class usage_acc:
  72     def __init__(self):
  73         self.results = {'entries': [], 'summary': []}
  74     def findentry(self, user):
  75         return usage_acc_findentry2(self.results['entries'], user)
  76     def findsum(self, user):
  77         return usage_acc_findsum2(self.results['summary'], user)
  78     def e2b(self, e, bucket, add=True):
  79         for b in e['buckets']:
  80             if b['bucket'] == bucket:
  81                 return b
  82         if not add:
  83                 return None
  84         b = {'bucket': bucket, 'categories': []}
  85         e['buckets'].append(b)
  86         return b
  87     def c2x(self, c, cat, add=True):
  88         for x in c:
  89             if x['category'] == cat:
  90                 return x
  91         if not add:
  92                 return None
  93         x = {'bytes_received': 0, 'category': cat,
  94             'bytes_sent': 0, 'ops': 0, 'successful_ops': 0 }
  95         c.append(x)
  96         return x
  97     def update(self, c, cat, user, out, b_in, err):
  98         x = self.c2x(c, cat)
  99         usage_acc_update2(x, out, b_in, err)
 100         if not err and cat == 'create_bucket' and not x.has_key('owner'):
 101             x['owner'] = user
 102     def make_entry(self, cat, bucket, user, out, b_in, err):
 103         if cat == 'create_bucket' and err:
 104                 return
 105         e = self.findentry(user)
 106         b = self.e2b(e, bucket)
 107         self.update(b['categories'], cat, user, out, b_in, err)
 108         s = self.findsum(user)
 109         x = self.c2x(s['categories'], cat)
 110         usage_acc_update2(x, out, b_in, err)
 111         x = s['total']
 112         usage_acc_update2(x, out, b_in, err)
 113     def generate_make_entry(self):
 114         return lambda cat,bucket,user,out,b_in,err: self.make_entry(cat, bucket, user, out, b_in, err)
 115     def get_usage(self):
 116         return self.results
 117     def compare_results(self, results):
 118         if not results.has_key('entries') or not results.has_key('summary'):
 119             return ['Missing entries or summary']
 120         r = []
 121         for e in self.results['entries']:
 122             try:
 123                 e2 = usage_acc_findentry2(results['entries'], e['user'], False)
 124             except Exception as ex:
 125                 r.append("malformed entry looking for user "
 126                     + e['user'] + " " + str(ex))
 127                 break
 128             if e2 == None:
 129                 r.append("missing entry for user " + e['user'])
 130                 continue
 131             for b in e['buckets']:
 132                 c = b['categories']
 133                 if b['bucket'] == 'nosuchbucket':
 134                     print("got here")
 135                 try:
 136                     b2 = self.e2b(e2, b['bucket'], False)
 137                     if b2 != None:
 138                             c2 = b2['categories']
 139                 except Exception as ex:
 140                     r.append("malformed entry looking for bucket "
 141                         + b['bucket'] + " in user " + e['user'] + " " + str(ex))
 142                     break
 143                 if b2 == None:
 144                     r.append("can't find bucket " + b['bucket']
 145                         + " in user " + e['user'])
 146                     continue
 147                 for x in c:
 148                     try:
 149                         x2 = self.c2x(c2, x['category'], False)
 150                     except Exception as ex:
 151                         r.append("malformed entry looking for "
 152                             + x['category'] + " in bucket " + b['bucket']
 153                             + " user " + e['user'] + " " + str(ex))
 154                         break
 155                     usage_acc_validate_fields(r, x, x2, "entry: category "
 156                         + x['category'] + " bucket " + b['bucket']
 157                         + " in user " + e['user'])
 158         for s in self.results['summary']:
 159             c = s['categories']
 160             try:
 161                 s2 = usage_acc_findsum2(results['summary'], s['user'], False)
 162             except Exception as ex:
 163                 r.append("malformed summary looking for user " + e['user']
 164                     + " " + str(ex))
 165                 break
 166                 if s2 == None:
 167                     r.append("missing summary for user " + e['user'] + " " + str(ex))
 168                     continue
 169             try:
 170                 c2 = s2['categories']
 171             except Exception as ex:
 172                 r.append("malformed summary missing categories for user "
 173                     + e['user'] + " " + str(ex))
 174                 break
 175             for x in c:
 176                 try:
 177                     x2 = self.c2x(c2, x['category'], False)
 178                 except Exception as ex:
 179                     r.append("malformed summary looking for "
 180                         + x['category'] + " user " + e['user'] + " " + str(ex))
 181                     break
 182                 usage_acc_validate_fields(r, x, x2, "summary: category "
 183                     + x['category'] + " in user " + e['user'])
 184             x = s['total']
 185             try:
 186                 x2 = s2['total']
 187             except Exception as ex:
 188                 r.append("malformed summary looking for totals for user "
 189                          + e['user'] + " " + str(ex))
 190                 break
 191             usage_acc_validate_fields(r, x, x2, "summary: totals for user" + e['user'])
 192         return r
 193
 194 def ignore_this_entry(cat, bucket, user, out, b_in, err):
 195     pass
 196 class requestlog_queue():
 197     def __init__(self, add):
 198         self.q = Queue.Queue(1000)
 199         self.adder = add
 200     def handle_request_data(self, request, response, error=False):
 201         now = datetime.datetime.now()
 202         if error:
 203             pass
 204         elif response.status < 200 or response.status >= 400:
 205             error = True
 206         self.q.put({'t': now, 'o': request, 'i': response, 'e': error})
 207     def clear(self):
 208         with self.q.mutex:
 209             self.q.queue.clear()
 210     def log_and_clear(self, cat, bucket, user, add_entry = None):
 211         while not self.q.empty():
 212             j = self.q.get()
 213             bytes_out = 0
 214             if 'Content-Length' in j['o'].headers:
 215                 bytes_out = int(j['o'].headers['Content-Length'])
 216             bytes_in = 0
 217             if 'content-length' in j['i'].msg.dict:
 218                 bytes_in = int(j['i'].msg.dict['content-length'])
 219             log.info('RL: %s %s %s bytes_out=%d bytes_in=%d failed=%r'
 220                      % (cat, bucket, user, bytes_out, bytes_in, j['e']))
 221             if add_entry == None:
 222                 add_entry = self.adder
 223             add_entry(cat, bucket, user, bytes_out, bytes_in, j['e'])
 224
 225 def create_presigned_url(conn, method, bucket_name, key_name, expiration):
 226     return conn.generate_url(expires_in=expiration,
 227         method=method,
 228         bucket=bucket_name,
 229         key=key_name,
 230         query_auth=True,
 231     )
 232
 233 def send_raw_http_request(conn, method, bucket_name, key_name, follow_redirects = False):
 234     url = create_presigned_url(conn, method, bucket_name, key_name, 3600)
 235     print(url)
 236     h = httplib2.Http()
 237     h.follow_redirects = follow_redirects
 238     return h.request(url, method)
 239
 240
 241 def get_acl(key):
 242     """
 243     Helper function to get the xml acl from a key, ensuring that the xml
 244     version tag is removed from the acl response
 245     """
 246     raw_acl = key.get_xml_acl()
 247
 248     def remove_version(string):
 249         return string.split(
 250             '<?xml version="1.0" encoding="UTF-8"?>'
 251         )[-1]
 252
 253     def remove_newlines(string):
 254         return string.strip('\n')
 255
 256     return remove_version(
 257         remove_newlines(raw_acl)
 258     )
 259
 260 def task(ctx, config):
 261     """
 262     Test radosgw-admin functionality against a running rgw instance.
 263     """
 264     global log
 265
 266     assert ctx.rgw.config, \
 267         "radosgw_admin task needs a config passed from the rgw task"
 268     config = ctx.rgw.config
 269     log.debug('config is: %r', config)
 270
 271     clients_from_config = config.keys()
 272
 273     # choose first client as default
 274     client = clients_from_config[0]
 275
 276     # once the client is chosen, pull the host name and  assigned port out of
 277     # the role_endpoints that were assigned by the rgw task
 278     endpoint = ctx.rgw.role_endpoints[client]
 279
 280     ##
 281     user1='foo'
 282     user2='fud'
 283     user3='bar'
 284     user4='bud'
 285     subuser1='foo:foo1'
 286     subuser2='foo:foo2'
 287     display_name1='Foo'
 288     display_name2='Fud'
 289     display_name3='Bar'
 290     email='foo@foo.com'
 291     access_key='9te6NH5mcdcq0Tc5i8i1'
 292     secret_key='Ny4IOauQoL18Gp2zM7lC1vLmoawgqcYP/YGcWfXu'
 293     access_key2='p5YnriCv1nAtykxBrupQ'
 294     secret_key2='Q8Tk6Q/27hfbFSYdSkPtUqhqx1GgzvpXa4WARozh'
 295     access_key3='NX5QOQKC6BH2IDN8HC7A'
 296     secret_key3='LnEsqNNqZIpkzauboDcLXLcYaWwLQ3Kop0zAnKIn'
 297     swift_secret1='gpS2G9RREMrnbqlp29PP2D36kgPR1tm72n5fPYfL'
 298     swift_secret2='ri2VJQcKSYATOY6uaDUX7pxgkW+W1YmC6OCxPHwy'
 299
 300     bucket_name='myfoo'
 301     bucket_name2='mybar'
 302
 303     # connect to rgw
 304     connection = boto.s3.connection.S3Connection(
 305         aws_access_key_id=access_key,
 306         aws_secret_access_key=secret_key,
 307         is_secure=False,
 308         port=endpoint.port,
 309         host=endpoint.hostname,
 310         calling_format=boto.s3.connection.OrdinaryCallingFormat(),
 311         )
 312     connection2 = boto.s3.connection.S3Connection(
 313         aws_access_key_id=access_key2,
 314         aws_secret_access_key=secret_key2,
 315         is_secure=False,
 316         port=endpoint.port,
 317         host=endpoint.hostname,
 318         calling_format=boto.s3.connection.OrdinaryCallingFormat(),
 319         )
 320     connection3 = boto.s3.connection.S3Connection(
 321         aws_access_key_id=access_key3,
 322         aws_secret_access_key=secret_key3,
 323         is_secure=False,
 324         port=endpoint.port,
 325         host=endpoint.hostname,
 326         calling_format=boto.s3.connection.OrdinaryCallingFormat(),
 327         )
 328
 329     acc = usage_acc()
 330     rl = requestlog_queue(acc.generate_make_entry())
 331     connection.set_request_hook(rl)
 332     connection2.set_request_hook(rl)
 333     connection3.set_request_hook(rl)
 334
 335     # legend (test cases can be easily grep-ed out)
 336     # TESTCASE 'testname','object','method','operation','assertion'
 337
 338     # TESTCASE 'usage-show0' 'usage' 'show' 'all usage' 'succeeds'
 339     (err, summary0) = rgwadmin(ctx, client, ['usage', 'show'], check_status=True)
 340
 341     # TESTCASE 'info-nosuch','user','info','non-existent user','fails'
 342     (err, out) = rgwadmin(ctx, client, ['user', 'info', '--uid', user1])
 343     assert err
 344
 345     # TESTCASE 'create-ok','user','create','w/all valid info','succeeds'
 346     (err, out) = rgwadmin(ctx, client, [
 347             'user', 'create',
 348             '--uid', user1,
 349             '--display-name', display_name1,
 350             '--email', email,
 351             '--access-key', access_key,
 352             '--secret', secret_key,
 353             '--max-buckets', '4'
 354             ],
 355             check_status=True)
 356
 357     # TESTCASE 'duplicate email','user','create','existing user email','fails'
 358     (err, out) = rgwadmin(ctx, client, [
 359             'user', 'create',
 360             '--uid', user2,
 361             '--display-name', display_name2,
 362             '--email', email,
 363             ])
 364     assert err
 365
 366     # TESTCASE 'info-existing','user','info','existing user','returns correct info'
 367     (err, out) = rgwadmin(ctx, client, ['user', 'info', '--uid', user1], check_status=True)
 368     assert out['user_id'] == user1
 369     assert out['email'] == email
 370     assert out['display_name'] == display_name1
 371     assert len(out['keys']) == 1
 372     assert out['keys'][0]['access_key'] == access_key
 373     assert out['keys'][0]['secret_key'] == secret_key
 374     assert not out['suspended']
 375
 376     # TESTCASE 'suspend-ok','user','suspend','active user','succeeds'
 377     (err, out) = rgwadmin(ctx, client, ['user', 'suspend', '--uid', user1],
 378         check_status=True)
 379
 380     # TESTCASE 'suspend-suspended','user','suspend','suspended user','succeeds w/advisory'
 381     (err, out) = rgwadmin(ctx, client, ['user', 'info', '--uid', user1], check_status=True)
 382     assert out['suspended']
 383
 384     # TESTCASE 're-enable','user','enable','suspended user','succeeds'
 385     (err, out) = rgwadmin(ctx, client, ['user', 'enable', '--uid', user1], check_status=True)
 386
 387     # TESTCASE 'info-re-enabled','user','info','re-enabled user','no longer suspended'
 388     (err, out) = rgwadmin(ctx, client, ['user', 'info', '--uid', user1], check_status=True)
 389     assert not out['suspended']
 390
 391     # TESTCASE 'add-keys','key','create','w/valid info','succeeds'
 392     (err, out) = rgwadmin(ctx, client, [
 393             'key', 'create', '--uid', user1,
 394             '--access-key', access_key2, '--secret', secret_key2,
 395             ], check_status=True)
 396
 397     # TESTCASE 'info-new-key','user','info','after key addition','returns all keys'
 398     (err, out) = rgwadmin(ctx, client, ['user', 'info', '--uid', user1],
 399         check_status=True)
 400     assert len(out['keys']) == 2
 401     assert out['keys'][0]['access_key'] == access_key2 or out['keys'][1]['access_key'] == access_key2
 402     assert out['keys'][0]['secret_key'] == secret_key2 or out['keys'][1]['secret_key'] == secret_key2
 403
 404     # TESTCASE 'rm-key','key','rm','newly added key','succeeds, key is removed'
 405     (err, out) = rgwadmin(ctx, client, [
 406             'key', 'rm', '--uid', user1,
 407             '--access-key', access_key2,
 408             ], check_status=True)
 409     assert len(out['keys']) == 1
 410     assert out['keys'][0]['access_key'] == access_key
 411     assert out['keys'][0]['secret_key'] == secret_key
 412
 413     # TESTCASE 'add-swift-key','key','create','swift key','succeeds'
 414     subuser_access = 'full'
 415     subuser_perm = 'full-control'
 416
 417     (err, out) = rgwadmin(ctx, client, [
 418             'subuser', 'create', '--subuser', subuser1,
 419             '--access', subuser_access
 420             ], check_status=True)
 421
 422     # TESTCASE 'add-swift-key','key','create','swift key','succeeds'
 423     (err, out) = rgwadmin(ctx, client, [
 424             'subuser', 'modify', '--subuser', subuser1,
 425             '--secret', swift_secret1,
 426             '--key-type', 'swift',
 427             ], check_status=True)
 428
 429     # TESTCASE 'subuser-perm-mask', 'subuser', 'info', 'test subuser perm mask durability', 'succeeds'
 430     (err, out) = rgwadmin(ctx, client, ['user', 'info', '--uid', user1])
 431
 432     assert out['subusers'][0]['permissions'] == subuser_perm
 433
 434     # TESTCASE 'info-swift-key','user','info','after key addition','returns all keys'
 435     (err, out) = rgwadmin(ctx, client, ['user', 'info', '--uid', user1], check_status=True)
 436     assert len(out['swift_keys']) == 1
 437     assert out['swift_keys'][0]['user'] == subuser1
 438     assert out['swift_keys'][0]['secret_key'] == swift_secret1
 439
 440     # TESTCASE 'add-swift-subuser','key','create','swift sub-user key','succeeds'
 441     (err, out) = rgwadmin(ctx, client, [
 442             'subuser', 'create', '--subuser', subuser2,
 443             '--secret', swift_secret2,
 444             '--key-type', 'swift',
 445             ], check_status=True)
 446
 447     # TESTCASE 'info-swift-subuser','user','info','after key addition','returns all sub-users/keys'
 448     (err, out) = rgwadmin(ctx, client, ['user', 'info', '--uid', user1], check_status=True)
 449     assert len(out['swift_keys']) == 2
 450     assert out['swift_keys'][0]['user'] == subuser2 or out['swift_keys'][1]['user'] == subuser2
 451     assert out['swift_keys'][0]['secret_key'] == swift_secret2 or out['swift_keys'][1]['secret_key'] == swift_secret2
 452
 453     # TESTCASE 'rm-swift-key1','key','rm','subuser','succeeds, one key is removed'
 454     (err, out) = rgwadmin(ctx, client, [
 455             'key', 'rm', '--subuser', subuser1,
 456             '--key-type', 'swift',
 457             ], check_status=True)
 458     assert len(out['swift_keys']) == 1
 459
 460     # TESTCASE 'rm-subuser','subuser','rm','subuser','success, subuser is removed'
 461     (err, out) = rgwadmin(ctx, client, [
 462             'subuser', 'rm', '--subuser', subuser1,
 463             ], check_status=True)
 464     assert len(out['subusers']) == 1
 465
 466     # TESTCASE 'rm-subuser-with-keys','subuser','rm','subuser','succeeds, second subser and key is removed'
 467     (err, out) = rgwadmin(ctx, client, [
 468             'subuser', 'rm', '--subuser', subuser2,
 469             '--key-type', 'swift', '--purge-keys',
 470             ], check_status=True)
 471     assert len(out['swift_keys']) == 0
 472     assert len(out['subusers']) == 0
 473
 474     # TESTCASE 'bucket-stats','bucket','stats','no session/buckets','succeeds, empty list'
 475     (err, out) = rgwadmin(ctx, client, ['bucket', 'stats', '--uid', user1],
 476         check_status=True)
 477     assert len(out) == 0
 478
 479     # TESTCASE 'bucket-stats2','bucket','stats','no buckets','succeeds, empty list'
 480     (err, out) = rgwadmin(ctx, client, ['bucket', 'list', '--uid', user1], check_status=True)
 481     assert len(out) == 0
 482
 483     # create a first bucket
 484     bucket = connection.create_bucket(bucket_name)
 485
 486     rl.log_and_clear("create_bucket", bucket_name, user1)
 487
 488     # TESTCASE 'bucket-list','bucket','list','one bucket','succeeds, expected list'
 489     (err, out) = rgwadmin(ctx, client, ['bucket', 'list', '--uid', user1], check_status=True)
 490     assert len(out) == 1
 491     assert out[0] == bucket_name
 492
 493     bucket_list = connection.get_all_buckets()
 494     assert len(bucket_list) == 1
 495     assert bucket_list[0].name == bucket_name
 496
 497     rl.log_and_clear("list_buckets", '', user1)
 498
 499     # TESTCASE 'bucket-list-all','bucket','list','all buckets','succeeds, expected list'
 500     (err, out) = rgwadmin(ctx, client, ['bucket', 'list'], check_status=True)
 501     assert len(out) >= 1
 502     assert bucket_name in out;
 503
 504     # TESTCASE 'max-bucket-limit,'bucket','create','4 buckets','5th bucket fails due to max buckets == 4'
 505     bucket2 = connection.create_bucket(bucket_name + '2')
 506     rl.log_and_clear("create_bucket", bucket_name + '2', user1)
 507     bucket3 = connection.create_bucket(bucket_name + '3')
 508     rl.log_and_clear("create_bucket", bucket_name + '3', user1)
 509     bucket4 = connection.create_bucket(bucket_name + '4')
 510     rl.log_and_clear("create_bucket", bucket_name + '4', user1)
 511     # the 5th should fail.
 512     failed = False
 513     try:
 514         connection.create_bucket(bucket_name + '5')
 515     except Exception:
 516         failed = True
 517     assert failed
 518     rl.log_and_clear("create_bucket", bucket_name + '5', user1)
 519
 520     # delete the buckets
 521     bucket2.delete()
 522     rl.log_and_clear("delete_bucket", bucket_name + '2', user1)
 523     bucket3.delete()
 524     rl.log_and_clear("delete_bucket", bucket_name + '3', user1)
 525     bucket4.delete()
 526     rl.log_and_clear("delete_bucket", bucket_name + '4', user1)
 527
 528     # TESTCASE 'bucket-stats3','bucket','stats','new empty bucket','succeeds, empty list'
 529     (err, out) = rgwadmin(ctx, client, [
 530             'bucket', 'stats', '--bucket', bucket_name], check_status=True)
 531     assert out['owner'] == user1
 532     bucket_id = out['id']
 533
 534     # TESTCASE 'bucket-stats4','bucket','stats','new empty bucket','succeeds, expected bucket ID'
 535     (err, out) = rgwadmin(ctx, client, ['bucket', 'stats', '--uid', user1], check_status=True)
 536     assert len(out) == 1
 537     assert out[0]['id'] == bucket_id    # does it return the same ID twice in a row?
 538
 539     # use some space
 540     key = boto.s3.key.Key(bucket)
 541     key.set_contents_from_string('one')
 542     rl.log_and_clear("put_obj", bucket_name, user1)
 543
 544     # TESTCASE 'bucket-stats5','bucket','stats','after creating key','succeeds, lists one non-empty object'
 545     (err, out) = rgwadmin(ctx, client, [
 546             'bucket', 'stats', '--bucket', bucket_name], check_status=True)
 547     assert out['id'] == bucket_id
 548     assert out['usage']['rgw.main']['num_objects'] == 1
 549     assert out['usage']['rgw.main']['size_kb'] > 0
 550
 551     #validate we have a positive user stats now
 552     (err, out) = rgwadmin(ctx, client,
 553                           ['user', 'stats','--uid', user1, '--sync-stats'],
 554                           check_status=True)
 555     assert out['stats']['size'] > 0
 556
 557     # reclaim it
 558     key.delete()
 559     rl.log_and_clear("delete_obj", bucket_name, user1)
 560
 561     # TESTCASE 'bucket unlink', 'bucket', 'unlink', 'unlink bucket from user', 'fails', 'access denied error'
 562     (err, out) = rgwadmin(ctx, client,
 563         ['bucket', 'unlink', '--uid', user1, '--bucket', bucket_name],
 564         check_status=True)
 565
 566     # create a second user to link the bucket to
 567     (err, out) = rgwadmin(ctx, client, [
 568             'user', 'create',
 569             '--uid', user2,
 570             '--display-name', display_name2,
 571             '--access-key', access_key2,
 572             '--secret', secret_key2,
 573             '--max-buckets', '1',
 574             ],
 575             check_status=True)
 576
 577     # try creating an object with the first user before the bucket is relinked
 578     denied = False
 579     key = boto.s3.key.Key(bucket)
 580
 581     try:
 582         key.set_contents_from_string('two')
 583     except boto.exception.S3ResponseError:
 584         denied = True
 585
 586     assert not denied
 587     rl.log_and_clear("put_obj", bucket_name, user1)
 588
 589     # delete the object
 590     key.delete()
 591     rl.log_and_clear("delete_obj", bucket_name, user1)
 592
 593     # link the bucket to another user
 594     (err, out) = rgwadmin(ctx, client, ['metadata', 'get', 'bucket:{n}'.format(n=bucket_name)],
 595         check_status=True)
 596
 597     bucket_data = out['data']
 598     assert bucket_data['bucket']['name'] == bucket_name
 599
 600     bucket_id = bucket_data['bucket']['bucket_id']
 601
 602     # link the bucket to another user
 603     (err, out) = rgwadmin(ctx, client, ['bucket', 'link', '--uid', user2, '--bucket', bucket_name, '--bucket-id', bucket_id],
 604         check_status=True)
 605
 606     # try to remove user, should fail (has a linked bucket)
 607     (err, out) = rgwadmin(ctx, client, ['user', 'rm', '--uid', user2])
 608     assert err
 609
 610     # TESTCASE 'bucket unlink', 'bucket', 'unlink', 'unlink bucket from user', 'succeeds, bucket unlinked'
 611     (err, out) = rgwadmin(ctx, client, ['bucket', 'unlink', '--uid', user2, '--bucket', bucket_name],
 612         check_status=True)
 613
 614     # relink the bucket to the first user and delete the second user
 615     (err, out) = rgwadmin(ctx, client,
 616         ['bucket', 'link', '--uid', user1, '--bucket', bucket_name, '--bucket-id', bucket_id],
 617         check_status=True)
 618
 619     (err, out) = rgwadmin(ctx, client, ['user', 'rm', '--uid', user2],
 620         check_status=True)
 621
 622     #TESTCASE 'bucket link', 'bucket', 'tenanted user', 'succeeds'
 623     tenant_name = "testx"
 624     # create a tenanted user to link the bucket to
 625     (err, out) = rgwadmin(ctx, client, [
 626             'user', 'create',
 627             '--tenant', tenant_name,
 628             '--uid', 'tenanteduser',
 629             '--display-name', 'tenanted-user',
 630             '--access-key', access_key2,
 631             '--secret', secret_key2,
 632             '--max-buckets', '1',
 633             ],
 634             check_status=True)
 635
 636     # link the bucket to a tenanted user
 637     (err, out) = rgwadmin(ctx, client, ['bucket', 'link', '--bucket', '/' + bucket_name, '--tenant', tenant_name, '--uid', 'tenanteduser'],
 638         check_status=True)
 639
 640     # check if the bucket name has tenant/ prefix
 641     (err, out) = rgwadmin(ctx, client, ['metadata', 'get', 'bucket:{n}'.format(n= tenant_name + '/' + bucket_name)],
 642         check_status=True)
 643
 644     bucket_data = out['data']
 645     assert bucket_data['bucket']['name'] == bucket_name
 646     assert bucket_data['bucket']['tenant'] == tenant_name
 647
 648     # relink the bucket to the first user and delete the tenanted user
 649     (err, out) = rgwadmin(ctx, client,
 650         ['bucket', 'link', '--bucket', tenant_name + '/' + bucket_name, '--uid', user1],
 651         check_status=True)
 652
 653     (err, out) = rgwadmin(ctx, client, ['user', 'rm', '--tenant', tenant_name, '--uid', 'tenanteduser'],
 654         check_status=True)
 655
 656     # TESTCASE 'object-rm', 'object', 'rm', 'remove object', 'succeeds, object is removed'
 657
 658     # upload an object
 659     object_name = 'four'
 660     key = boto.s3.key.Key(bucket, object_name)
 661     key.set_contents_from_string(object_name)
 662     rl.log_and_clear("put_obj", bucket_name, user1)
 663
 664     # fetch it too (for usage stats presently)
 665     s = key.get_contents_as_string()
 666     rl.log_and_clear("get_obj", bucket_name, user1)
 667     assert s == object_name
 668     # list bucket too (for usage stats presently)
 669     keys = list(bucket.list())
 670     rl.log_and_clear("list_bucket", bucket_name, user1)
 671     assert len(keys) == 1
 672     assert keys[0].name == object_name
 673
 674     # now delete it
 675     (err, out) = rgwadmin(ctx, client,
 676         ['object', 'rm', '--bucket', bucket_name, '--object', object_name],
 677         check_status=True)
 678
 679     # TESTCASE 'bucket-stats6','bucket','stats','after deleting key','succeeds, lists one no objects'
 680     (err, out) = rgwadmin(ctx, client, [
 681             'bucket', 'stats', '--bucket', bucket_name],
 682             check_status=True)
 683     assert out['id'] == bucket_id
 684     assert out['usage']['rgw.main']['num_objects'] == 0
 685
 686     # list log objects
 687     # TESTCASE 'log-list','log','list','after activity','succeeds, lists one no objects'
 688     (err, out) = rgwadmin(ctx, client, ['log', 'list'], check_status=True)
 689     assert len(out) > 0
 690
 691     for obj in out:
 692         # TESTCASE 'log-show','log','show','after activity','returns expected info'
 693         if obj[:4] == 'meta' or obj[:4] == 'data' or obj[:18] == 'obj_delete_at_hint':
 694             continue
 695
 696         (err, rgwlog) = rgwadmin(ctx, client, ['log', 'show', '--object', obj],
 697             check_status=True)
 698         assert len(rgwlog) > 0
 699
 700         # exempt bucket_name2 from checking as it was only used for multi-region tests
 701         assert rgwlog['bucket'].find(bucket_name) == 0 or rgwlog['bucket'].find(bucket_name2) == 0
 702         assert rgwlog['bucket'] != bucket_name or rgwlog['bucket_id'] == bucket_id
 703         assert rgwlog['bucket_owner'] == user1 or rgwlog['bucket'] == bucket_name + '5' or rgwlog['bucket'] == bucket_name2
 704         for entry in rgwlog['log_entries']:
 705             log.debug('checking log entry: ', entry)
 706             assert entry['bucket'] == rgwlog['bucket']
 707             possible_buckets = [bucket_name + '5', bucket_name2]
 708             user = entry['user']
 709             assert user == user1 or user.endswith('system-user') or \
 710                 rgwlog['bucket'] in possible_buckets
 711
 712         # TESTCASE 'log-rm','log','rm','delete log objects','succeeds'
 713         (err, out) = rgwadmin(ctx, client, ['log', 'rm', '--object', obj],
 714             check_status=True)
 715
 716     # TODO: show log by bucket+date
 717
 718     # TESTCASE 'user-suspend2','user','suspend','existing user','succeeds'
 719     (err, out) = rgwadmin(ctx, client, ['user', 'suspend', '--uid', user1],
 720         check_status=True)
 721
 722     # TESTCASE 'user-suspend3','user','suspend','suspended user','cannot write objects'
 723     denied = False
 724     try:
 725         key = boto.s3.key.Key(bucket)
 726         key.set_contents_from_string('five')
 727     except boto.exception.S3ResponseError as e:
 728         denied = True
 729         assert e.status == 403
 730
 731     assert denied
 732     rl.log_and_clear("put_obj", bucket_name, user1)
 733
 734     # TESTCASE 'user-renable2','user','enable','suspended user','succeeds'
 735     (err, out) = rgwadmin(ctx, client, ['user', 'enable', '--uid', user1],
 736         check_status=True)
 737
 738     # TESTCASE 'user-renable3','user','enable','reenabled user','can write objects'
 739     key = boto.s3.key.Key(bucket)
 740     key.set_contents_from_string('six')
 741     rl.log_and_clear("put_obj", bucket_name, user1)
 742
 743     # TESTCASE 'gc-list', 'gc', 'list', 'get list of objects ready for garbage collection'
 744
 745     # create an object large enough to be split into multiple parts
 746     test_string = 'foo'*10000000
 747
 748     big_key = boto.s3.key.Key(bucket)
 749     big_key.set_contents_from_string(test_string)
 750     rl.log_and_clear("put_obj", bucket_name, user1)
 751
 752     # now delete the head
 753     big_key.delete()
 754     rl.log_and_clear("delete_obj", bucket_name, user1)
 755
 756     # wait a bit to give the garbage collector time to cycle
 757     time.sleep(15)
 758
 759     (err, out) = rgwadmin(ctx, client, ['gc', 'list'])
 760
 761     assert len(out) > 0
 762
 763     # TESTCASE 'gc-process', 'gc', 'process', 'manually collect garbage'
 764     (err, out) = rgwadmin(ctx, client, ['gc', 'process'], check_status=True)
 765
 766     #confirm
 767     (err, out) = rgwadmin(ctx, client, ['gc', 'list'])
 768
 769     assert len(out) == 0
 770
 771     # TESTCASE 'rm-user-buckets','user','rm','existing user','fails, still has buckets'
 772     (err, out) = rgwadmin(ctx, client, ['user', 'rm', '--uid', user1])
 773     assert err
 774
 775     # delete should fail because ``key`` still exists
 776     try:
 777         bucket.delete()
 778     except boto.exception.S3ResponseError as e:
 779         assert e.status == 409
 780     rl.log_and_clear("delete_bucket", bucket_name, user1)
 781
 782     key.delete()
 783     rl.log_and_clear("delete_obj", bucket_name, user1)
 784     bucket.delete()
 785     rl.log_and_clear("delete_bucket", bucket_name, user1)
 786
 787     # TESTCASE 'policy', 'bucket', 'policy', 'get bucket policy', 'returns S3 policy'
 788     bucket = connection.create_bucket(bucket_name)
 789     rl.log_and_clear("create_bucket", bucket_name, user1)
 790
 791     # create an object
 792     key = boto.s3.key.Key(bucket)
 793     key.set_contents_from_string('seven')
 794     rl.log_and_clear("put_obj", bucket_name, user1)
 795
 796     # should be private already but guarantee it
 797     key.set_acl('private')
 798     rl.log_and_clear("put_acls", bucket_name, user1)
 799
 800     (err, out) = rgwadmin(ctx, client,
 801         ['policy', '--bucket', bucket.name, '--object', key.key],
 802         check_status=True, format='xml')
 803
 804     acl = get_acl(key)
 805     rl.log_and_clear("get_acls", bucket_name, user1)
 806
 807     assert acl == out.strip('\n')
 808
 809     # add another grantee by making the object public read
 810     key.set_acl('public-read')
 811     rl.log_and_clear("put_acls", bucket_name, user1)
 812
 813     (err, out) = rgwadmin(ctx, client,
 814         ['policy', '--bucket', bucket.name, '--object', key.key],
 815         check_status=True, format='xml')
 816
 817     acl = get_acl(key)
 818     rl.log_and_clear("get_acls", bucket_name, user1)
 819
 820     assert acl == out.strip('\n')
 821
 822     # TESTCASE 'rm-bucket', 'bucket', 'rm', 'bucket with objects', 'succeeds'
 823     bucket = connection.create_bucket(bucket_name)
 824     rl.log_and_clear("create_bucket", bucket_name, user1)
 825     key_name = ['eight', 'nine', 'ten', 'eleven']
 826     for i in range(4):
 827         key = boto.s3.key.Key(bucket)
 828         key.set_contents_from_string(key_name[i])
 829     rl.log_and_clear("put_obj", bucket_name, user1)
 830
 831     (err, out) = rgwadmin(ctx, client,
 832         ['bucket', 'rm', '--bucket', bucket_name, '--purge-objects'],
 833         check_status=True)
 834
 835     # TESTCASE 'caps-add', 'caps', 'add', 'add user cap', 'succeeds'
 836     caps='user=read'
 837     (err, out) = rgwadmin(ctx, client, ['caps', 'add', '--uid', user1, '--caps', caps])
 838
 839     assert out['caps'][0]['perm'] == 'read'
 840
 841     # TESTCASE 'caps-rm', 'caps', 'rm', 'remove existing cap from user', 'succeeds'
 842     (err, out) = rgwadmin(ctx, client, ['caps', 'rm', '--uid', user1, '--caps', caps])
 843
 844     assert not out['caps']
 845
 846     # TESTCASE 'rm-user','user','rm','existing user','fails, still has buckets'
 847     bucket = connection.create_bucket(bucket_name)
 848     rl.log_and_clear("create_bucket", bucket_name, user1)
 849     key = boto.s3.key.Key(bucket)
 850
 851     (err, out) = rgwadmin(ctx, client, ['user', 'rm', '--uid', user1])
 852     assert err
 853
 854     # TESTCASE 'rm-user2', 'user', 'rm', 'user with data', 'succeeds'
 855     bucket = connection.create_bucket(bucket_name)
 856     rl.log_and_clear("create_bucket", bucket_name, user1)
 857     key = boto.s3.key.Key(bucket)
 858     key.set_contents_from_string('twelve')
 859     rl.log_and_clear("put_obj", bucket_name, user1)
 860
 861     time.sleep(35)
 862
 863     # need to wait for all usage data to get flushed, should take up to 30 seconds
 864     timestamp = time.time()
 865     while time.time() - timestamp <= (2 * 60):      # wait up to 20 minutes
 866         (err, out) = rgwadmin(ctx, client, ['usage', 'show', '--categories', 'delete_obj'])  # one of the operations we did is delete_obj, should be present.
 867         if get_user_successful_ops(out, user1) > 0:
 868             break
 869         time.sleep(1)
 870
 871     assert time.time() - timestamp <= (20 * 60)
 872
 873     # TESTCASE 'usage-show' 'usage' 'show' 'all usage' 'succeeds'
 874     (err, out) = rgwadmin(ctx, client, ['usage', 'show'], check_status=True)
 875     assert len(out['entries']) > 0
 876     assert len(out['summary']) > 0
 877
 878     r = acc.compare_results(out)
 879     if len(r) != 0:
 880         sys.stderr.write(("\n".join(r))+"\n")
 881         assert(len(r) == 0)
 882
 883     user_summary = get_user_summary(out, user1)
 884
 885     total = user_summary['total']
 886     assert total['successful_ops'] > 0
 887
 888     # TESTCASE 'usage-show2' 'usage' 'show' 'user usage' 'succeeds'
 889     (err, out) = rgwadmin(ctx, client, ['usage', 'show', '--uid', user1],
 890         check_status=True)
 891     assert len(out['entries']) > 0
 892     assert len(out['summary']) > 0
 893     user_summary = out['summary'][0]
 894     for entry in user_summary['categories']:
 895         assert entry['successful_ops'] > 0
 896     assert user_summary['user'] == user1
 897
 898     # TESTCASE 'usage-show3' 'usage' 'show' 'user usage categories' 'succeeds'
 899     test_categories = ['create_bucket', 'put_obj', 'delete_obj', 'delete_bucket']
 900     for cat in test_categories:
 901         (err, out) = rgwadmin(ctx, client, ['usage', 'show', '--uid', user1, '--categories', cat],
 902             check_status=True)
 903         assert len(out['summary']) > 0
 904         user_summary = out['summary'][0]
 905         assert user_summary['user'] == user1
 906         assert len(user_summary['categories']) == 1
 907         entry = user_summary['categories'][0]
 908         assert entry['category'] == cat
 909         assert entry['successful_ops'] > 0
 910
 911     # TESTCASE 'user-rename', 'user', 'rename', 'existing user', 'new user', 'succeeds'
 912     # create a new user user3
 913     (err, out) = rgwadmin(ctx, client, [
 914         'user', 'create',
 915         '--uid', user3,
 916         '--display-name', display_name3,
 917         '--access-key', access_key3,
 918         '--secret', secret_key3,
 919         '--max-buckets', '4'
 920         ],
 921         check_status=True)
 922
 923     # create a bucket
 924     bucket = connection3.create_bucket(bucket_name + '6')
 925
 926     rl.log_and_clear("create_bucket", bucket_name + '6', user3)
 927
 928     # create object
 929     object_name1 = 'thirteen'
 930     key1 = boto.s3.key.Key(bucket, object_name1)
 931     key1.set_contents_from_string(object_name1)
 932     rl.log_and_clear("put_obj", bucket_name + '6', user3)
 933
 934     # rename user3
 935     (err, out) = rgwadmin(ctx, client, ['user', 'rename', '--uid', user3, '--new-uid', user4], check_status=True)
 936     assert out['user_id'] == user4
 937     assert out['keys'][0]['access_key'] == access_key3
 938     assert out['keys'][0]['secret_key'] == secret_key3
 939
 940     time.sleep(5)
 941
 942     # get bucket and object to test if user keys are preserved
 943     bucket = connection3.get_bucket(bucket_name + '6')
 944     s = key1.get_contents_as_string()
 945     rl.log_and_clear("get_obj", bucket_name + '6', user4)
 946     assert s == object_name1
 947
 948     # TESTCASE 'user-rename', 'user', 'rename', 'existing user', 'another existing user', 'fails'
 949     # create a new user user2
 950     (err, out) = rgwadmin(ctx, client, [
 951         'user', 'create',
 952         '--uid', user2,
 953         '--display-name', display_name2,
 954         '--access-key', access_key2,
 955         '--secret', secret_key2,
 956         '--max-buckets', '4'
 957         ],
 958         check_status=True)
 959
 960     # create a bucket
 961     bucket = connection2.create_bucket(bucket_name + '7')
 962
 963     rl.log_and_clear("create_bucket", bucket_name + '7', user2)
 964
 965     # create object
 966     object_name2 = 'fourteen'
 967     key2 = boto.s3.key.Key(bucket, object_name2)
 968     key2.set_contents_from_string(object_name2)
 969     rl.log_and_clear("put_obj", bucket_name + '7', user2)
 970
 971     (err, out) = rgwadmin(ctx, client, ['user', 'rename', '--uid', user4, '--new-uid', user2])
 972     assert err
 973
 974     # test if user 2 and user4 can still access their bucket and objects after rename fails
 975     bucket = connection3.get_bucket(bucket_name + '6')
 976     s = key1.get_contents_as_string()
 977     rl.log_and_clear("get_obj", bucket_name + '6', user4)
 978     assert s == object_name1
 979
 980     bucket = connection2.get_bucket(bucket_name + '7')
 981     s = key2.get_contents_as_string()
 982     rl.log_and_clear("get_obj", bucket_name + '7', user2)
 983     assert s == object_name2
 984
 985     (err, out) = rgwadmin(ctx, client,
 986     ['user', 'rm', '--uid', user4, '--purge-data' ],
 987     check_status=True)
 988
 989     (err, out) = rgwadmin(ctx, client,
 990     ['user', 'rm', '--uid', user2, '--purge-data' ],
 991     check_status=True)
 992
 993     time.sleep(5)
 994
 995     # should be all through with connection. (anything using connection
 996     #  should be BEFORE the usage stuff above.)
 997     rl.log_and_clear("(before-close)", '-', '-', ignore_this_entry)
 998     connection.close()
 999     connection = None
1000
1001     # the usage flush interval is 30 seconds, wait that much an then some
1002     # to make sure everything has been flushed
1003     time.sleep(35)
1004
1005     # TESTCASE 'usage-trim' 'usage' 'trim' 'user usage' 'succeeds, usage removed'
1006     (err, out) = rgwadmin(ctx, client, ['usage', 'trim', '--uid', user1],
1007         check_status=True)
1008     (err, out) = rgwadmin(ctx, client, ['usage', 'show', '--uid', user1],
1009         check_status=True)
1010     assert len(out['entries']) == 0
1011     assert len(out['summary']) == 0
1012
1013     (err, out) = rgwadmin(ctx, client,
1014         ['user', 'rm', '--uid', user1, '--purge-data' ],
1015         check_status=True)
1016
1017     # TESTCASE 'rm-user3','user','rm','deleted user','fails'
1018     (err, out) = rgwadmin(ctx, client, ['user', 'info', '--uid', user1])
1019     assert err
1020
1021     # TESTCASE 'zone-info', 'zone', 'get', 'get zone info', 'succeeds, has default placement rule'
1022     #
1023
1024     (err, out) = rgwadmin(ctx, client, ['zone', 'get','--rgw-zone','default'])
1025     orig_placement_pools = len(out['placement_pools'])
1026
1027     # removed this test, it is not correct to assume that zone has default placement, it really
1028     # depends on how we set it up before
1029     #
1030     # assert len(out) > 0
1031     # assert len(out['placement_pools']) == 1
1032
1033     # default_rule = out['placement_pools'][0]
1034     # assert default_rule['key'] == 'default-placement'
1035
1036     rule={'key': 'new-placement', 'val': {'data_pool': '.rgw.buckets.2', 'index_pool': '.rgw.buckets.index.2'}}
1037
1038     out['placement_pools'].append(rule)
1039
1040     (err, out) = rgwadmin(ctx, client, ['zone', 'set'],
1041         stdin=StringIO(json.dumps(out)),
1042         check_status=True)
1043
1044     (err, out) = rgwadmin(ctx, client, ['zone', 'get'])
1045     assert len(out) > 0
1046     assert len(out['placement_pools']) == orig_placement_pools + 1
1047
1048     zonecmd = ['zone', 'placement', 'rm',
1049                '--rgw-zone', 'default',
1050                '--placement-id', 'new-placement']
1051
1052     (err, out) = rgwadmin(ctx, client, zonecmd, check_status=True)
1053
1054     # TESTCASE 'zonegroup-info', 'zonegroup', 'get', 'get zonegroup info', 'succeeds'
1055     (err, out) = rgwadmin(ctx, client, ['zonegroup', 'get'], check_status=True)
1056
1057 from teuthology.config import config
1058 from teuthology.orchestra import cluster, remote
1059 import argparse;
1060
1061 def main():
1062     if len(sys.argv) == 3:
1063         user = sys.argv[1] + "@"
1064         host = sys.argv[2]
1065     elif len(sys.argv) == 2:
1066         user = ""
1067         host = sys.argv[1]
1068     else:
1069         sys.stderr.write("usage: radosgw_admin.py [user] host\n")
1070         exit(1)
1071     client0 = remote.Remote(user + host)
1072     ctx = config
1073     ctx.cluster=cluster.Cluster(remotes=[(client0,
1074      [ 'ceph.client.rgw.%s' % (host),  ]),])
1075
1076     ctx.rgw = argparse.Namespace()
1077     endpoints = {}
1078     endpoints['ceph.client.rgw.%s' % host] = (host, 80)
1079     ctx.rgw.role_endpoints = endpoints
1080     ctx.rgw.realm = None
1081     ctx.rgw.regions = {'region0': { 'api name': 'api1',
1082             'is master': True, 'master zone': 'r0z0',
1083             'zones': ['r0z0', 'r0z1'] }}
1084     ctx.rgw.config = {'ceph.client.rgw.%s' % host: {'system user': {'name': '%s-system-user' % host}}}
1085     task(config, None)
1086     exit()
1087
1088 if __name__ == '__main__':
1089     main()