ceph/qa/workunits/rgw/test_rgw_reshard.py

   1 #!/usr/bin/python3
   2
   3 import logging as log
   4 import time
   5 import subprocess
   6 import json
   7 import boto3
   8
   9 """
  10 Rgw manual and dynamic resharding  testing against a running instance
  11 """
  12 # The test cases in this file have been annotated for inventory.
  13 # To extract the inventory (in csv format) use the command:
  14 #
  15 #   grep '^ *# TESTCASE' | sed 's/^ *# TESTCASE //'
  16 #
  17 #
  18
  19 log.basicConfig(level=log.DEBUG)
  20 log.getLogger('botocore').setLevel(log.CRITICAL)
  21 log.getLogger('boto3').setLevel(log.CRITICAL)
  22 log.getLogger('urllib3').setLevel(log.CRITICAL)
  23
  24 """ Constants """
  25 USER = 'tester'
  26 DISPLAY_NAME = 'Testing'
  27 ACCESS_KEY = 'NX5QOQKC6BH2IDN8HC7A'
  28 SECRET_KEY = 'LnEsqNNqZIpkzauboDcLXLcYaWwLQ3Kop0zAnKIn'
  29 BUCKET_NAME1 = 'myfoo'
  30 BUCKET_NAME2 = 'mybar'
  31 VER_BUCKET_NAME = 'myver'
  32
  33
  34 def exec_cmd(cmd):
  35     try:
  36         proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
  37         out, err = proc.communicate()
  38         if proc.returncode == 0:
  39             log.info('command succeeded')
  40             if out is not None: log.info(out)
  41             return out
  42         else:
  43             raise Exception("error: %s \nreturncode: %s" % (err, proc.returncode))
  44     except Exception as e:
  45         log.error('command failed')
  46         log.error(e)
  47         return False
  48
  49
  50 def get_radosgw_port():
  51     out = exec_cmd('sudo netstat -nltp | grep radosgw')
  52     log.debug('output: %s' % out)
  53     x = out.decode('utf8').split(" ")
  54     port = [i for i in x if ':' in i][0].split(':')[1]
  55     log.info('radosgw port: %s' % port)
  56     return port
  57
  58
  59 class BucketStats:
  60     def __init__(self, bucket_name, bucket_id, num_objs=0, size_kb=0, num_shards=0):
  61         self.bucket_name = bucket_name
  62         self.bucket_id = bucket_id
  63         self.num_objs = num_objs
  64         self.size_kb = size_kb
  65         self.num_shards = num_shards if num_shards > 0 else 1
  66
  67     def get_num_shards(self):
  68         self.num_shards = get_bucket_num_shards(self.bucket_name, self.bucket_id)
  69
  70
  71 def get_bucket_stats(bucket_name):
  72     """
  73     function to get bucket stats
  74     """
  75     cmd = exec_cmd("radosgw-admin bucket stats --bucket %s" % bucket_name)
  76     json_op = json.loads(cmd)
  77     bucket_id = json_op['id']
  78     num_shards_op = json_op['num_shards']
  79     if len(json_op['usage']) > 0:
  80         num_objects = json_op['usage']['rgw.main']['num_objects']
  81         size_kb = json_op['usage']['rgw.main']['size_kb']
  82     else:
  83         num_objects = 0
  84         size_kb = 0
  85     log.debug("bucket %s id %s num_objects %d size_kb %d num_shards %d", bucket_name, bucket_id,
  86               num_objects, size_kb, num_shards_op)
  87     return BucketStats(bucket_name, bucket_id, num_objects, size_kb, num_shards_op)
  88
  89
  90 def get_bucket_num_shards(bucket_name, bucket_id):
  91     """
  92     function to get bucket num shards
  93     """
  94     metadata = 'bucket.instance:' + bucket_name + ':' + bucket_id
  95     log.debug("metadata %s", metadata)
  96     cmd = exec_cmd('radosgw-admin metadata get %s' % metadata)
  97     json_op = json.loads(cmd)
  98     num_shards = json_op['data']['bucket_info']['num_shards']
  99     log.debug("bucket %s id %s num_shards %d", bucket_name, bucket_id, num_shards)
 100     return num_shards
 101
 102
 103 def main():
 104     """
 105     execute manual and dynamic resharding commands
 106     """
 107     # create user
 108     exec_cmd('radosgw-admin user create --uid %s --display-name %s --access-key %s --secret %s'
 109                    % (USER, DISPLAY_NAME, ACCESS_KEY, SECRET_KEY))
 110
 111     def boto_connect(portnum, ssl, proto):
 112         endpoint = proto + '://localhost:' + portnum
 113         conn = boto3.resource('s3',
 114                               aws_access_key_id=ACCESS_KEY,
 115                               aws_secret_access_key=SECRET_KEY,
 116                               use_ssl=ssl,
 117                               endpoint_url=endpoint,
 118                               verify=False,
 119                               config=None,
 120                               )
 121         return conn
 122
 123     port = get_radosgw_port()
 124
 125     if port == '80':
 126         connection = boto_connect(port, ssl=False, proto='http')
 127     elif port == '443':
 128         connection = boto_connect(port, ssl=True, proto='https')
 129
 130     # create a bucket
 131     bucket1 = connection.create_bucket(Bucket=BUCKET_NAME1)
 132     bucket2 = connection.create_bucket(Bucket=BUCKET_NAME2)
 133     ver_bucket = connection.create_bucket(Bucket=VER_BUCKET_NAME)
 134     connection.BucketVersioning('ver_bucket')
 135
 136     bucket_stats1 = get_bucket_stats(BUCKET_NAME1)
 137     bucket_stats2 = get_bucket_stats(BUCKET_NAME2)
 138     ver_bucket_stats = get_bucket_stats(VER_BUCKET_NAME)
 139
 140     bucket1_acl = connection.BucketAcl(BUCKET_NAME1).load()
 141     bucket2_acl = connection.BucketAcl(BUCKET_NAME2).load()
 142     ver_bucket_acl = connection.BucketAcl(VER_BUCKET_NAME).load()
 143
 144     # TESTCASE 'reshard-add','reshard','add','add bucket to resharding queue','succeeds'
 145     log.debug(' test: reshard add')
 146     num_shards_expected = bucket_stats1.num_shards + 1
 147     cmd = exec_cmd('radosgw-admin reshard add --bucket %s --num-shards %s' % (BUCKET_NAME1, num_shards_expected))
 148     cmd = exec_cmd('radosgw-admin reshard list')
 149     json_op = json.loads(cmd)
 150     log.debug('bucket name %s', json_op[0]['bucket_name'])
 151     assert json_op[0]['bucket_name'] == BUCKET_NAME1
 152     assert json_op[0]['new_num_shards'] == num_shards_expected
 153
 154     # TESTCASE 'reshard-process','reshard','','process bucket resharding','succeeds'
 155     log.debug(' test: reshard process')
 156     cmd = exec_cmd('radosgw-admin reshard process')
 157     time.sleep(5)
 158     # check bucket shards num
 159     bucket_stats1 = get_bucket_stats(BUCKET_NAME1)
 160     bucket_stats1.get_num_shards()
 161     if bucket_stats1.num_shards != num_shards_expected:
 162         log.error("Resharding failed on bucket %s. Expected number of shards are not created" % BUCKET_NAME1)
 163
 164     # TESTCASE 'reshard-add','reshard','add','add non empty bucket to resharding queue','succeeds'
 165     log.debug(' test: reshard add non empty bucket')
 166     # create objs
 167     num_objs = 8
 168     for i in range(0, num_objs):
 169         connection.Object(BUCKET_NAME1, ('key'+str(i))).put(Body=b"some_data")
 170
 171     num_shards_expected = bucket_stats1.num_shards + 1
 172     cmd = exec_cmd('radosgw-admin reshard add --bucket %s --num-shards %s' % (BUCKET_NAME1, num_shards_expected))
 173     cmd = exec_cmd('radosgw-admin reshard list')
 174     json_op = json.loads(cmd)
 175     log.debug('bucket name %s', json_op[0]['bucket_name'])
 176     assert json_op[0]['bucket_name'] == BUCKET_NAME1
 177     assert json_op[0]['new_num_shards'] == num_shards_expected
 178
 179     # TESTCASE 'reshard process ,'reshard','process','reshard non empty bucket','succeeds'
 180     log.debug(' test: reshard process non empty bucket')
 181     cmd = exec_cmd('radosgw-admin reshard process')
 182     # check bucket shards num
 183     bucket_stats1 = get_bucket_stats(BUCKET_NAME1)
 184     bucket_stats1.get_num_shards()
 185     if bucket_stats1.num_shards != num_shards_expected:
 186         log.error("Resharding failed on bucket %s. Expected number of shards are not created" % BUCKET_NAME1)
 187
 188     # TESTCASE 'manual resharding','bucket', 'reshard','','manual bucket resharding','succeeds'
 189     log.debug(' test: manual reshard bucket')
 190     # create objs
 191     num_objs = 11
 192     for i in range(0, num_objs):
 193         connection.Object(BUCKET_NAME2, ('key' + str(i))).put(Body=b"some_data")
 194
 195     time.sleep(10)
 196     num_shards_expected = bucket_stats2.num_shards + 1
 197     cmd = exec_cmd('radosgw-admin bucket reshard --bucket %s --num-shards %s' % (BUCKET_NAME2,
 198                                                                                  num_shards_expected))
 199     # check bucket shards num
 200     bucket_stats2 = get_bucket_stats(BUCKET_NAME2)
 201     bucket_stats2.get_num_shards()
 202     if bucket_stats2.num_shards != num_shards_expected:
 203         log.error("Resharding failed on bucket %s. Expected number of shards are not created" % BUCKET_NAME2)
 204
 205     # TESTCASE 'versioning reshard-','bucket', reshard','versioning reshard','succeeds'
 206     log.debug(' test: reshard versioned bucket')
 207     num_shards_expected = ver_bucket_stats.num_shards + 1
 208     cmd = exec_cmd('radosgw-admin bucket reshard --bucket %s --num-shards %s' % (VER_BUCKET_NAME,
 209                                                                                  num_shards_expected))
 210     # check bucket shards num
 211     ver_bucket_stats = get_bucket_stats(VER_BUCKET_NAME)
 212     assert ver_bucket_stats.num_shards == num_shards_expected
 213
 214     # TESTCASE 'check acl'
 215     new_bucket1_acl = connection.BucketAcl(BUCKET_NAME1).load()
 216     assert new_bucket1_acl == bucket1_acl
 217     new_bucket2_acl = connection.BucketAcl(BUCKET_NAME2).load()
 218     assert new_bucket2_acl == bucket2_acl
 219     new_ver_bucket_acl = connection.BucketAcl(VER_BUCKET_NAME).load()
 220     assert new_ver_bucket_acl == ver_bucket_acl
 221
 222     # Clean up
 223     log.debug("Deleting bucket %s", BUCKET_NAME1)
 224     bucket1.objects.all().delete()
 225     bucket1.delete()
 226     log.debug("Deleting bucket %s", BUCKET_NAME2)
 227     bucket2.objects.all().delete()
 228     bucket2.delete()
 229     log.debug("Deleting bucket %s", VER_BUCKET_NAME)
 230     ver_bucket.delete()
 231
 232
 233 main()
 234 log.info("Completed resharding tests")