ceph/src/test/rgw/test_multi.py

   1 import subprocess
   2 import os
   3 import random
   4 import string
   5 import argparse
   6 import sys
   7 import logging
   8 try:
   9     import configparser
  10 except ImportError:
  11     import ConfigParser as configparser
  12
  13 import nose.core
  14
  15 from rgw_multi import multisite
  16 # make tests from rgw_multi.tests available to nose
  17 from rgw_multi.tests import *
  18
  19 mstart_path = os.getenv('MSTART_PATH')
  20 if mstart_path is None:
  21     mstart_path = os.path.normpath(os.path.dirname(os.path.realpath(__file__)) + '/../..') + '/'
  22
  23 test_path = os.path.normpath(os.path.dirname(os.path.realpath(__file__))) + '/'
  24
  25 # configure logging for the tests module
  26 log = logging.getLogger('rgw_multi.tests')
  27
  28 def bash(cmd, **kwargs):
  29     log.debug('running cmd: %s', ' '.join(cmd))
  30     check_retcode = kwargs.pop('check_retcode', True)
  31     kwargs['stdout'] = subprocess.PIPE
  32     process = subprocess.Popen(cmd, **kwargs)
  33     s = process.communicate()[0]
  34     log.debug('command returned status=%d stdout=%s', process.returncode, s.decode('utf-8'))
  35     if check_retcode:
  36         assert(process.returncode == 0)
  37     return (s, process.returncode)
  38
  39 class Cluster(multisite.Cluster):
  40     """ cluster implementation based on mstart/mrun scripts """
  41     def __init__(self, cluster_id):
  42         super(Cluster, self).__init__()
  43         self.cluster_id = cluster_id
  44         self.needs_reset = True
  45
  46     def admin(self, args = None, **kwargs):
  47         """ radosgw-admin command """
  48         cmd = [test_path + 'test-rgw-call.sh', 'call_rgw_admin', self.cluster_id]
  49         if args:
  50             cmd += args
  51         if kwargs.pop('read_only', False):
  52             cmd += ['--rgw-cache-enabled', 'false']
  53         return bash(cmd, **kwargs)
  54
  55     def start(self):
  56         cmd = [mstart_path + 'mstart.sh', self.cluster_id]
  57         if self.needs_reset:
  58             cmd += ['-n', '--mds_num', '0']
  59         bash(cmd)
  60         self.needs_reset = False
  61
  62     def stop(self):
  63         cmd = [mstart_path + 'mstop.sh', self.cluster_id]
  64         bash(cmd)
  65
  66 class Gateway(multisite.Gateway):
  67     """ gateway implementation based on mrgw/mstop scripts """
  68     def __init__(self, client_id = None, *args, **kwargs):
  69         super(Gateway, self).__init__(*args, **kwargs)
  70         self.id = client_id
  71
  72     def start(self, args = None):
  73         """ start the gateway """
  74         assert(self.cluster)
  75         cmd = [mstart_path + 'mrgw.sh', self.cluster.cluster_id, str(self.port)]
  76         if self.id:
  77             cmd += ['-i', self.id]
  78         cmd += ['--debug-rgw=20', '--debug-ms=1']
  79         if args:
  80             cmd += args
  81         bash(cmd)
  82
  83     def stop(self):
  84         """ stop the gateway """
  85         assert(self.cluster)
  86         cmd = [mstart_path + 'mstop.sh', self.cluster.cluster_id, 'radosgw', self.id]
  87         bash(cmd)
  88
  89 def gen_access_key():
  90     return ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(16))
  91
  92 def gen_secret():
  93     return ''.join(random.choice(string.ascii_uppercase + string.ascii_lowercase + string.digits) for _ in range(32))
  94
  95 def gen_credentials():
  96     return multisite.Credentials(gen_access_key(), gen_secret())
  97
  98 def cluster_name(cluster_num):
  99     return 'c' + str(cluster_num)
 100
 101 def zonegroup_name(zonegroup_num):
 102     return string.ascii_lowercase[zonegroup_num]
 103
 104 def zone_name(zonegroup_num, zone_num):
 105     return zonegroup_name(zonegroup_num) + str(zone_num + 1)
 106
 107 def gateway_port(zonegroup_num, gateway_num):
 108     return 8000 + 100 * zonegroup_num + gateway_num
 109
 110 def gateway_name(zonegroup_num, zone_num, gateway_num):
 111     return zone_name(zonegroup_num, zone_num) + '-' + str(gateway_num + 1)
 112
 113 def zone_endpoints(zonegroup_num, zone_num, gateways_per_zone):
 114     endpoints = []
 115     base = gateway_port(zonegroup_num, zone_num * gateways_per_zone)
 116     for i in range(0, gateways_per_zone):
 117         endpoints.append('http://localhost:' + str(base + i))
 118     return endpoints
 119
 120 def get_log_level(log_level):
 121     if log_level >= 20:
 122         return logging.DEBUG
 123     if log_level >= 10:
 124         return logging.INFO
 125     if log_level >= 5:
 126         return logging.WARN
 127     if log_level >= 1:
 128         return logging.ERROR
 129     return logging.CRITICAL
 130
 131 def setup_logging(log_level_console, log_file, log_level_file):
 132     if log_file:
 133         formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s')
 134         fh = logging.FileHandler(log_file)
 135         fh.setFormatter(formatter)
 136         fh.setLevel(get_log_level(log_level_file))
 137         log.addHandler(fh)
 138
 139     formatter = logging.Formatter('%(levelname)s %(message)s')
 140     ch = logging.StreamHandler()
 141     ch.setFormatter(formatter)
 142     ch.setLevel(get_log_level(log_level_console))
 143     log.addHandler(ch)
 144
 145 def init(parse_args):
 146     cfg = configparser.RawConfigParser({
 147                                          'num_zonegroups': 1,
 148                                          'num_zones': 3,
 149                                          'gateways_per_zone': 2,
 150                                          'no_bootstrap': 'false',
 151                                          'log_level': 20,
 152                                          'log_file': None,
 153                                          'file_log_level': 20,
 154                                          'tenant': None,
 155                                          })
 156     try:
 157         path = os.environ['RGW_MULTI_TEST_CONF']
 158     except KeyError:
 159         path = tpath('test_multi.conf')
 160
 161     try:
 162         with open(path) as f:
 163             cfg.readfp(f)
 164     except:
 165         print('WARNING: error reading test config. Path can be set through the RGW_MULTI_TEST_CONF env variable')
 166         pass
 167
 168     parser = argparse.ArgumentParser(
 169             description='Run rgw multi-site tests',
 170             usage='test_multi [--num-zonegroups <num>] [--num-zones <num>] [--no-bootstrap]')
 171
 172     section = 'DEFAULT'
 173     parser.add_argument('--num-zonegroups', type=int, default=cfg.getint(section, 'num_zonegroups'))
 174     parser.add_argument('--num-zones', type=int, default=cfg.getint(section, 'num_zones'))
 175     parser.add_argument('--gateways-per-zone', type=int, default=cfg.getint(section, 'gateways_per_zone'))
 176     parser.add_argument('--no-bootstrap', action='store_true', default=cfg.getboolean(section, 'no_bootstrap'))
 177     parser.add_argument('--log-level', type=int, default=cfg.getint(section, 'log_level'))
 178     parser.add_argument('--log-file', type=str, default=cfg.get(section, 'log_file'))
 179     parser.add_argument('--file-log-level', type=int, default=cfg.getint(section, 'file_log_level'))
 180     parser.add_argument('--tenant', type=str, default=cfg.get(section, 'tenant'))
 181
 182     argv = []
 183
 184     if parse_args:
 185         argv = sys.argv[1:]
 186
 187     args = parser.parse_args(argv)
 188     bootstrap = not args.no_bootstrap
 189
 190     setup_logging(args.log_level, args.log_file, args.file_log_level)
 191
 192     # start first cluster
 193     c1 = Cluster(cluster_name(1))
 194     if bootstrap:
 195         c1.start()
 196     clusters = []
 197     clusters.append(c1)
 198
 199     admin_creds = gen_credentials()
 200     admin_user = multisite.User('zone.user')
 201
 202     user_creds = gen_credentials()
 203     user = multisite.User('tester')
 204
 205     realm = multisite.Realm('r')
 206     if bootstrap:
 207         # create the realm on c1
 208         realm.create(c1)
 209     else:
 210         realm.get(c1)
 211     period = multisite.Period(realm=realm)
 212     realm.current_period = period
 213
 214     for zg in range(0, args.num_zonegroups):
 215         zonegroup = multisite.ZoneGroup(zonegroup_name(zg), period)
 216         period.zonegroups.append(zonegroup)
 217
 218         is_master_zg = zg == 0
 219         if is_master_zg:
 220             period.master_zonegroup = zonegroup
 221
 222         for z in range(0, args.num_zones):
 223             is_master = z == 0
 224             # start a cluster, or use c1 for first zone
 225             cluster = None
 226             if is_master_zg and is_master:
 227                 cluster = c1
 228             else:
 229                 cluster = Cluster(cluster_name(len(clusters) + 1))
 230                 clusters.append(cluster)
 231                 if bootstrap:
 232                     cluster.start()
 233                     # pull realm configuration from the master's gateway
 234                     gateway = realm.meta_master_zone().gateways[0]
 235                     realm.pull(cluster, gateway, admin_creds)
 236
 237             endpoints = zone_endpoints(zg, z, args.gateways_per_zone)
 238             if is_master:
 239                 if bootstrap:
 240                     # create the zonegroup on its first zone's cluster
 241                     arg = []
 242                     if is_master_zg:
 243                         arg += ['--master']
 244                     if len(endpoints): # use master zone's endpoints
 245                         arg += ['--endpoints', ','.join(endpoints)]
 246                     zonegroup.create(cluster, arg)
 247                 else:
 248                     zonegroup.get(cluster)
 249
 250             # create the zone in its zonegroup
 251             zone = multisite.Zone(zone_name(zg, z), zonegroup, cluster)
 252             if bootstrap:
 253                 arg = admin_creds.credential_args()
 254                 if is_master:
 255                     arg += ['--master']
 256                 if len(endpoints):
 257                     arg += ['--endpoints', ','.join(endpoints)]
 258                 zone.create(cluster, arg)
 259             else:
 260                 zone.get(cluster)
 261             zonegroup.zones.append(zone)
 262             if is_master:
 263                 zonegroup.master_zone = zone
 264
 265             # update/commit the period
 266             if bootstrap:
 267                 period.update(zone, commit=True)
 268
 269             # start the gateways
 270             for g in range(0, args.gateways_per_zone):
 271                 port = gateway_port(zg, g + z * args.gateways_per_zone)
 272                 client_id = gateway_name(zg, z, g)
 273                 gateway = Gateway(client_id, 'localhost', port, cluster, zone)
 274                 if bootstrap:
 275                     gateway.start()
 276                 zone.gateways.append(gateway)
 277
 278             if is_master_zg and is_master:
 279                 if bootstrap:
 280                     # create admin user
 281                     arg = ['--display-name', '"Zone User"', '--system']
 282                     arg += admin_creds.credential_args()
 283                     admin_user.create(zone, arg)
 284                     # create test user
 285                     arg = ['--display-name', '"Test User"']
 286                     arg += user_creds.credential_args()
 287                     if args.tenant:
 288                         cmd += ['--tenant', args.tenant]
 289                     user.create(zone, arg)
 290                 else:
 291                     # read users and update keys
 292                     admin_user.info(zone)
 293                     admin_creds = admin_user.credentials[0]
 294                     user.info(zone)
 295                     user_creds = user.credentials[0]
 296
 297     if not bootstrap:
 298         period.get(c1)
 299
 300     init_multi(realm, user)
 301
 302 def setup_module():
 303     init(False)
 304
 305 if __name__ == "__main__":
 306     init(True)