[ceph.git] / ceph / src / test / rgw / test_multi.py

import subprocess
import os
import random
import string
import argparse
import sys
import logging
try:
    import configparser
except ImportError:
    import ConfigParser as configparser

import nose.core

from rgw_multi import multisite
from rgw_multi.zone_rados import RadosZone as RadosZone
from rgw_multi.zone_es  import ESZone as ESZone

# make tests from rgw_multi.tests available to nose
from rgw_multi.tests import *
from rgw_multi.tests_es import *

mstart_path = os.getenv('MSTART_PATH')
if mstart_path is None:
    mstart_path = os.path.normpath(os.path.dirname(os.path.realpath(__file__)) + '/../..') + '/'

test_path = os.path.normpath(os.path.dirname(os.path.realpath(__file__))) + '/'

# configure logging for the tests module
log = logging.getLogger('rgw_multi.tests')

def bash(cmd, **kwargs):
    log.debug('running cmd: %s', ' '.join(cmd))
    check_retcode = kwargs.pop('check_retcode', True)
    kwargs['stdout'] = subprocess.PIPE
    process = subprocess.Popen(cmd, **kwargs)
    s = process.communicate()[0]
    log.debug('command returned status=%d stdout=%s', process.returncode, s.decode('utf-8'))
    if check_retcode:
        assert(process.returncode == 0)
    return (s, process.returncode)

class Cluster(multisite.Cluster):
    """ cluster implementation based on mstart/mrun scripts """
    def __init__(self, cluster_id):
        super(Cluster, self).__init__()
        self.cluster_id = cluster_id
        self.needs_reset = True

    def admin(self, args = None, **kwargs):
        """ radosgw-admin command """
        cmd = [test_path + 'test-rgw-call.sh', 'call_rgw_admin', self.cluster_id]
        if args:
            cmd += args
        cmd += ['--debug-rgw', str(kwargs.pop('debug_rgw', 0))]
        cmd += ['--debug-ms', str(kwargs.pop('debug_ms', 0))]
        if kwargs.pop('read_only', False):
            cmd += ['--rgw-cache-enabled', 'false']
        return bash(cmd, **kwargs)

    def start(self):
        cmd = [mstart_path + 'mstart.sh', self.cluster_id]
        if self.needs_reset:
            cmd += ['-n', '--mds_num', '0']
        bash(cmd)
        self.needs_reset = False

    def stop(self):
        cmd = [mstart_path + 'mstop.sh', self.cluster_id]
        bash(cmd)

class Gateway(multisite.Gateway):
    """ gateway implementation based on mrgw/mstop scripts """
    def __init__(self, client_id = None, *args, **kwargs):
        super(Gateway, self).__init__(*args, **kwargs)
        self.id = client_id

    def start(self, args = None):
        """ start the gateway """
        assert(self.cluster)
        cmd = [mstart_path + 'mrgw.sh', self.cluster.cluster_id, str(self.port)]
        if self.id:
            cmd += ['-i', self.id]
        cmd += ['--debug-rgw=20', '--debug-ms=1']
        if args:
            cmd += args
        bash(cmd)

    def stop(self):
        """ stop the gateway """
        assert(self.cluster)
        cmd = [mstart_path + 'mstop.sh', self.cluster.cluster_id, 'radosgw', self.id]
        bash(cmd)

def gen_access_key():
    return ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(16))

def gen_secret():
    return ''.join(random.choice(string.ascii_uppercase + string.ascii_lowercase + string.digits) for _ in range(32))

def gen_credentials():
    return multisite.Credentials(gen_access_key(), gen_secret())

def cluster_name(cluster_num):
    return 'c' + str(cluster_num)

def zonegroup_name(zonegroup_num):
    return string.ascii_lowercase[zonegroup_num]

def zone_name(zonegroup_num, zone_num):
    return zonegroup_name(zonegroup_num) + str(zone_num + 1)

def gateway_port(zonegroup_num, gateway_num):
    return 8000 + 100 * zonegroup_num + gateway_num

def gateway_name(zonegroup_num, zone_num, gateway_num):
    return zone_name(zonegroup_num, zone_num) + '-' + str(gateway_num + 1)

def zone_endpoints(zonegroup_num, zone_num, gateways_per_zone):
    endpoints = []
    base = gateway_port(zonegroup_num, zone_num * gateways_per_zone)
    for i in range(0, gateways_per_zone):
        endpoints.append('http://localhost:' + str(base + i))
    return endpoints

def get_log_level(log_level):
    if log_level >= 20:
        return logging.DEBUG
    if log_level >= 10:
        return logging.INFO
    if log_level >= 5:
        return logging.WARN
    if log_level >= 1:
        return logging.ERROR
    return logging.CRITICAL

def setup_logging(log_level_console, log_file, log_level_file):
    if log_file:
        formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s')
        fh = logging.FileHandler(log_file)
        fh.setFormatter(formatter)
        fh.setLevel(get_log_level(log_level_file))
        log.addHandler(fh)

    formatter = logging.Formatter('%(levelname)s %(message)s')
    ch = logging.StreamHandler()
    ch.setFormatter(formatter)
    ch.setLevel(get_log_level(log_level_console))
    log.addHandler(ch)

def init(parse_args):
    cfg = configparser.RawConfigParser({
                                         'num_zonegroups': 1,
                                         'num_zones': 3,
                                         'num_es_zones': 0,
                                         'gateways_per_zone': 2,
                                         'no_bootstrap': 'false',
                                         'log_level': 20,
                                         'log_file': None,
                                         'file_log_level': 20,
                                         'tenant': None,
                                         'checkpoint_retries': 60,
                                         'checkpoint_delay': 5,
                                         'reconfigure_delay': 5,
                                         'es_endpoint': None,
                                         })
    try:
        path = os.environ['RGW_MULTI_TEST_CONF']
    except KeyError:
        path = test_path + 'test_multi.conf'

    try:
        with open(path) as f:
            cfg.readfp(f)
    except:
        print('WARNING: error reading test config. Path can be set through the RGW_MULTI_TEST_CONF env variable')
        pass

    parser = argparse.ArgumentParser(
            description='Run rgw multi-site tests',
            usage='test_multi [--num-zonegroups <num>] [--num-zones <num>] [--no-bootstrap]')

    section = 'DEFAULT'
    parser.add_argument('--num-zonegroups', type=int, default=cfg.getint(section, 'num_zonegroups'))
    parser.add_argument('--num-zones', type=int, default=cfg.getint(section, 'num_zones'))
    parser.add_argument('--num-es-zones', type=int, default=cfg.getint(section, 'num_es_zones'))
    parser.add_argument('--gateways-per-zone', type=int, default=cfg.getint(section, 'gateways_per_zone'))
    parser.add_argument('--no-bootstrap', action='store_true', default=cfg.getboolean(section, 'no_bootstrap'))
    parser.add_argument('--log-level', type=int, default=cfg.getint(section, 'log_level'))
    parser.add_argument('--log-file', type=str, default=cfg.get(section, 'log_file'))
    parser.add_argument('--file-log-level', type=int, default=cfg.getint(section, 'file_log_level'))
    parser.add_argument('--tenant', type=str, default=cfg.get(section, 'tenant'))
    parser.add_argument('--checkpoint-retries', type=int, default=cfg.getint(section, 'checkpoint_retries'))
    parser.add_argument('--checkpoint-delay', type=int, default=cfg.getint(section, 'checkpoint_delay'))
    parser.add_argument('--reconfigure-delay', type=int, default=cfg.getint(section, 'reconfigure_delay'))
    parser.add_argument('--es-endpoint', type=str, default=cfg.get(section, 'es_endpoint'))

    argv = []

    if parse_args:
        argv = sys.argv[1:]

    args = parser.parse_args(argv)
    bootstrap = not args.no_bootstrap

    # if num_es_zones is defined, need to have es_endpoint defined too
    assert(args.num_es_zones == 0 or args.es_endpoint)

    setup_logging(args.log_level, args.log_file, args.file_log_level)

    # start first cluster
    c1 = Cluster(cluster_name(1))
    if bootstrap:
        c1.start()
    clusters = []
    clusters.append(c1)

    admin_creds = gen_credentials()
    admin_user = multisite.User('zone.user')

    user_creds = gen_credentials()
    user = multisite.User('tester')

    realm = multisite.Realm('r')
    if bootstrap:
        # create the realm on c1
        realm.create(c1)
    else:
        realm.get(c1)
    period = multisite.Period(realm=realm)
    realm.current_period = period

    num_zones = args.num_zones + args.num_es_zones

    for zg in range(0, args.num_zonegroups):
        zonegroup = multisite.ZoneGroup(zonegroup_name(zg), period)
        period.zonegroups.append(zonegroup)

        is_master_zg = zg == 0
        if is_master_zg:
            period.master_zonegroup = zonegroup

        for z in range(0, num_zones):
            is_master = z == 0
            # start a cluster, or use c1 for first zone
            cluster = None
            if is_master_zg and is_master:
                cluster = c1
            else:
                cluster = Cluster(cluster_name(len(clusters) + 1))
                clusters.append(cluster)
                if bootstrap:
                    cluster.start()
                    # pull realm configuration from the master's gateway
                    gateway = realm.meta_master_zone().gateways[0]
                    realm.pull(cluster, gateway, admin_creds)

            endpoints = zone_endpoints(zg, z, args.gateways_per_zone)
            if is_master:
                if bootstrap:
                    # create the zonegroup on its first zone's cluster
                    arg = []
                    if is_master_zg:
                        arg += ['--master']
                    if len(endpoints): # use master zone's endpoints
                        arg += ['--endpoints', ','.join(endpoints)]
                    zonegroup.create(cluster, arg)
                else:
                    zonegroup.get(cluster)

            es_zone = (z >= args.num_zones)

            # create the zone in its zonegroup
            zone = multisite.Zone(zone_name(zg, z), zonegroup, cluster)
            if es_zone:
                zone = ESZone(zone_name(zg, z), args.es_endpoint, zonegroup, cluster)
            else:
                zone = RadosZone(zone_name(zg, z), zonegroup, cluster)

            if bootstrap:
                arg = admin_creds.credential_args()
                if is_master:
                    arg += ['--master']
                if len(endpoints):
                    arg += ['--endpoints', ','.join(endpoints)]
                zone.create(cluster, arg)
            else:
                zone.get(cluster)
            zonegroup.zones.append(zone)
            if is_master:
                zonegroup.master_zone = zone

            zonegroup.zones_by_type.setdefault(zone.tier_type(), []).append(zone)

            if zone.is_read_only():
                zonegroup.ro_zones.append(zone)
            else:
                zonegroup.rw_zones.append(zone)

            # update/commit the period
            if bootstrap:
                period.update(zone, commit=True)

            # start the gateways
            for g in range(0, args.gateways_per_zone):
                port = gateway_port(zg, g + z * args.gateways_per_zone)
                client_id = gateway_name(zg, z, g)
                gateway = Gateway(client_id, 'localhost', port, cluster, zone)
                if bootstrap:
                    gateway.start()
                zone.gateways.append(gateway)

            if is_master_zg and is_master:
                if bootstrap:
                    # create admin user
                    arg = ['--display-name', '"Zone User"', '--system']
                    arg += admin_creds.credential_args()
                    admin_user.create(zone, arg)
                    # create test user
                    arg = ['--display-name', '"Test User"']
                    arg += user_creds.credential_args()
                    if args.tenant:
                        cmd += ['--tenant', args.tenant]
                    user.create(zone, arg)
                else:
                    # read users and update keys
                    admin_user.info(zone)
                    admin_creds = admin_user.credentials[0]
                    user.info(zone)
                    user_creds = user.credentials[0]

    if not bootstrap:
        period.get(c1)

    config = Config(checkpoint_retries=args.checkpoint_retries,
                    checkpoint_delay=args.checkpoint_delay,
                    reconfigure_delay=args.reconfigure_delay)
    init_multi(realm, user, config)

def setup_module():
    init(False)

if __name__ == "__main__":
    init(True)
Commit	Line	Data
7c673cae FG	1	import subprocess
	2	import os
	3	import random
	4	import string
	5	import argparse
	6	import sys
	7	import logging
	8	try:
	9	import configparser
	10	except ImportError:
	11	import ConfigParser as configparser
	12
	13	import nose.core
	14
	15	from rgw_multi import multisite
31f18b77 FG	16	from rgw_multi.zone_rados import RadosZone as RadosZone
	17	from rgw_multi.zone_es import ESZone as ESZone
	18
7c673cae FG	19	# make tests from rgw_multi.tests available to nose
7c673cae FG	20	from rgw_multi.tests import *
31f18b77	21	from rgw_multi.tests_es import *
7c673cae FG	22
	23	mstart_path = os.getenv('MSTART_PATH')
	24	if mstart_path is None:
	25	mstart_path = os.path.normpath(os.path.dirname(os.path.realpath(__file__)) + '/../..') + '/'
	26
	27	test_path = os.path.normpath(os.path.dirname(os.path.realpath(__file__))) + '/'
	28
	29	# configure logging for the tests module
	30	log = logging.getLogger('rgw_multi.tests')
	31
	32	def bash(cmd, **kwargs):
	33	log.debug('running cmd: %s', ' '.join(cmd))
	34	check_retcode = kwargs.pop('check_retcode', True)
	35	kwargs['stdout'] = subprocess.PIPE
	36	process = subprocess.Popen(cmd, **kwargs)
	37	s = process.communicate()[0]
	38	log.debug('command returned status=%d stdout=%s', process.returncode, s.decode('utf-8'))
	39	if check_retcode:
	40	assert(process.returncode == 0)
	41	return (s, process.returncode)
	42
	43	class Cluster(multisite.Cluster):
	44	""" cluster implementation based on mstart/mrun scripts """
	45	def __init__(self, cluster_id):
	46	super(Cluster, self).__init__()
	47	self.cluster_id = cluster_id
	48	self.needs_reset = True
	49
	50	def admin(self, args = None, **kwargs):
	51	""" radosgw-admin command """
	52	cmd = [test_path + 'test-rgw-call.sh', 'call_rgw_admin', self.cluster_id]
	53	if args:
	54	cmd += args
b32b8144 FG	55	cmd += ['--debug-rgw', str(kwargs.pop('debug_rgw', 0))]
b32b8144 FG	56	cmd += ['--debug-ms', str(kwargs.pop('debug_ms', 0))]
7c673cae FG	57	if kwargs.pop('read_only', False):
	58	cmd += ['--rgw-cache-enabled', 'false']
	59	return bash(cmd, **kwargs)
	60
	61	def start(self):
	62	cmd = [mstart_path + 'mstart.sh', self.cluster_id]
	63	if self.needs_reset:
	64	cmd += ['-n', '--mds_num', '0']
	65	bash(cmd)
	66	self.needs_reset = False
	67
	68	def stop(self):
	69	cmd = [mstart_path + 'mstop.sh', self.cluster_id]
	70	bash(cmd)
	71
	72	class Gateway(multisite.Gateway):
	73	""" gateway implementation based on mrgw/mstop scripts """
	74	def __init__(self, client_id = None, args, *kwargs):
	75	super(Gateway, self).__init__(args, *kwargs)
	76	self.id = client_id
	77
	78	def start(self, args = None):
	79	""" start the gateway """
	80	assert(self.cluster)
	81	cmd = [mstart_path + 'mrgw.sh', self.cluster.cluster_id, str(self.port)]
	82	if self.id:
	83	cmd += ['-i', self.id]
	84	cmd += ['--debug-rgw=20', '--debug-ms=1']
	85	if args:
	86	cmd += args
	87	bash(cmd)
	88
	89	def stop(self):
	90	""" stop the gateway """
	91	assert(self.cluster)
	92	cmd = [mstart_path + 'mstop.sh', self.cluster.cluster_id, 'radosgw', self.id]
	93	bash(cmd)
	94
	95	def gen_access_key():
	96	return ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(16))
	97
	98	def gen_secret():
	99	return ''.join(random.choice(string.ascii_uppercase + string.ascii_lowercase + string.digits) for _ in range(32))
	100
	101	def gen_credentials():
	102	return multisite.Credentials(gen_access_key(), gen_secret())
	103
	104	def cluster_name(cluster_num):
	105	return 'c' + str(cluster_num)
	106
	107	def zonegroup_name(zonegroup_num):
	108	return string.ascii_lowercase[zonegroup_num]
	109
	110	def zone_name(zonegroup_num, zone_num):
	111	return zonegroup_name(zonegroup_num) + str(zone_num + 1)
	112
	113	def gateway_port(zonegroup_num, gateway_num):
	114	return 8000 + 100 * zonegroup_num + gateway_num
	115
	116	def gateway_name(zonegroup_num, zone_num, gateway_num):
	117	return zone_name(zonegroup_num, zone_num) + '-' + str(gateway_num + 1)
	118
	119	def zone_endpoints(zonegroup_num, zone_num, gateways_per_zone):
	120	endpoints = []
121	base = gateway_port(zonegroup_num, zone_num * gateways_per_zone)
122	for i in range(0, gateways_per_zone):
123	endpoints.append('http://localhost:' + str(base + i))
124	return endpoints
125
126	def get_log_level(log_level):
127	if log_level >= 20:
128	return logging.DEBUG
129	if log_level >= 10:
130	return logging.INFO
131	if log_level >= 5:
132	return logging.WARN
133	if log_level >= 1:
134	return logging.ERROR
135	return logging.CRITICAL
136
137	def setup_logging(log_level_console, log_file, log_level_file):
138	if log_file:
139	formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s')
140	fh = logging.FileHandler(log_file)
141	fh.setFormatter(formatter)
142	fh.setLevel(get_log_level(log_level_file))
143	log.addHandler(fh)
144
145	formatter = logging.Formatter('%(levelname)s %(message)s')
146	ch = logging.StreamHandler()
147	ch.setFormatter(formatter)
148	ch.setLevel(get_log_level(log_level_console))
149	log.addHandler(ch)
150
151	def init(parse_args):
152	cfg = configparser.RawConfigParser({
153	'num_zonegroups': 1,
154	'num_zones': 3,
31f18b77	155	'num_es_zones': 0,
7c673cae FG	156	'gateways_per_zone': 2,
	157	'no_bootstrap': 'false',
	158	'log_level': 20,
	159	'log_file': None,
	160	'file_log_level': 20,
	161	'tenant': None,
31f18b77 FG	162	'checkpoint_retries': 60,
	163	'checkpoint_delay': 5,
	164	'reconfigure_delay': 5,
	165	'es_endpoint': None,
7c673cae FG	166	})
	167	try:
	168	path = os.environ['RGW_MULTI_TEST_CONF']
	169	except KeyError:
31f18b77	170	path = test_path + 'test_multi.conf'
7c673cae FG	171
	172	try:
	173	with open(path) as f:
	174	cfg.readfp(f)
	175	except:
	176	print('WARNING: error reading test config. Path can be set through the RGW_MULTI_TEST_CONF env variable')
	177	pass
	178
	179	parser = argparse.ArgumentParser(
	180	description='Run rgw multi-site tests',
	181	usage='test_multi [--num-zonegroups <num>] [--num-zones <num>] [--no-bootstrap]')
	182
	183	section = 'DEFAULT'
	184	parser.add_argument('--num-zonegroups', type=int, default=cfg.getint(section, 'num_zonegroups'))
	185	parser.add_argument('--num-zones', type=int, default=cfg.getint(section, 'num_zones'))
31f18b77	186	parser.add_argument('--num-es-zones', type=int, default=cfg.getint(section, 'num_es_zones'))
7c673cae FG	187	parser.add_argument('--gateways-per-zone', type=int, default=cfg.getint(section, 'gateways_per_zone'))
	188	parser.add_argument('--no-bootstrap', action='store_true', default=cfg.getboolean(section, 'no_bootstrap'))
	189	parser.add_argument('--log-level', type=int, default=cfg.getint(section, 'log_level'))
	190	parser.add_argument('--log-file', type=str, default=cfg.get(section, 'log_file'))
	191	parser.add_argument('--file-log-level', type=int, default=cfg.getint(section, 'file_log_level'))
	192	parser.add_argument('--tenant', type=str, default=cfg.get(section, 'tenant'))
31f18b77 FG	193	parser.add_argument('--checkpoint-retries', type=int, default=cfg.getint(section, 'checkpoint_retries'))
	194	parser.add_argument('--checkpoint-delay', type=int, default=cfg.getint(section, 'checkpoint_delay'))
	195	parser.add_argument('--reconfigure-delay', type=int, default=cfg.getint(section, 'reconfigure_delay'))
	196	parser.add_argument('--es-endpoint', type=str, default=cfg.get(section, 'es_endpoint'))
7c673cae FG	197
	198	argv = []
	199
	200	if parse_args:
	201	argv = sys.argv[1:]
	202
	203	args = parser.parse_args(argv)
	204	bootstrap = not args.no_bootstrap
	205
31f18b77 FG	206	# if num_es_zones is defined, need to have es_endpoint defined too
	207	assert(args.num_es_zones == 0 or args.es_endpoint)
	208
7c673cae FG	209	setup_logging(args.log_level, args.log_file, args.file_log_level)
	210
	211	# start first cluster
	212	c1 = Cluster(cluster_name(1))
	213	if bootstrap:
	214	c1.start()
	215	clusters = []
	216	clusters.append(c1)
	217
	218	admin_creds = gen_credentials()
	219	admin_user = multisite.User('zone.user')
	220
	221	user_creds = gen_credentials()
	222	user = multisite.User('tester')
	223
	224	realm = multisite.Realm('r')
	225	if bootstrap:
	226	# create the realm on c1
	227	realm.create(c1)
	228	else:
	229	realm.get(c1)
	230	period = multisite.Period(realm=realm)
	231	realm.current_period = period
	232
31f18b77 FG	233	num_zones = args.num_zones + args.num_es_zones
31f18b77 FG	234
7c673cae FG	235	for zg in range(0, args.num_zonegroups):
	236	zonegroup = multisite.ZoneGroup(zonegroup_name(zg), period)
	237	period.zonegroups.append(zonegroup)
	238
	239	is_master_zg = zg == 0
	240	if is_master_zg:
	241	period.master_zonegroup = zonegroup
	242
31f18b77	243	for z in range(0, num_zones):
7c673cae FG	244	is_master = z == 0
	245	# start a cluster, or use c1 for first zone
	246	cluster = None
	247	if is_master_zg and is_master:
	248	cluster = c1
	249	else:
	250	cluster = Cluster(cluster_name(len(clusters) + 1))
	251	clusters.append(cluster)
	252	if bootstrap:
	253	cluster.start()
	254	# pull realm configuration from the master's gateway
	255	gateway = realm.meta_master_zone().gateways[0]
	256	realm.pull(cluster, gateway, admin_creds)
	257
	258	endpoints = zone_endpoints(zg, z, args.gateways_per_zone)
	259	if is_master:
	260	if bootstrap:
	261	# create the zonegroup on its first zone's cluster
	262	arg = []
	263	if is_master_zg:
	264	arg += ['--master']
	265	if len(endpoints): # use master zone's endpoints
	266	arg += ['--endpoints', ','.join(endpoints)]
	267	zonegroup.create(cluster, arg)
	268	else:
	269	zonegroup.get(cluster)
	270
31f18b77 FG	271	es_zone = (z >= args.num_zones)
31f18b77 FG	272
7c673cae FG	273	# create the zone in its zonegroup
7c673cae FG	274	zone = multisite.Zone(zone_name(zg, z), zonegroup, cluster)
31f18b77 FG	275	if es_zone:
	276	zone = ESZone(zone_name(zg, z), args.es_endpoint, zonegroup, cluster)
	277	else:
	278	zone = RadosZone(zone_name(zg, z), zonegroup, cluster)
	279
7c673cae FG	280	if bootstrap:
	281	arg = admin_creds.credential_args()
	282	if is_master:
	283	arg += ['--master']
	284	if len(endpoints):
	285	arg += ['--endpoints', ','.join(endpoints)]
	286	zone.create(cluster, arg)
	287	else:
	288	zone.get(cluster)
	289	zonegroup.zones.append(zone)
	290	if is_master:
	291	zonegroup.master_zone = zone
	292
31f18b77 FG	293	zonegroup.zones_by_type.setdefault(zone.tier_type(), []).append(zone)
	294
	295	if zone.is_read_only():
	296	zonegroup.ro_zones.append(zone)
	297	else:
	298	zonegroup.rw_zones.append(zone)
	299
7c673cae FG	300	# update/commit the period
	301	if bootstrap:
	302	period.update(zone, commit=True)
	303
	304	# start the gateways
	305	for g in range(0, args.gateways_per_zone):
	306	port = gateway_port(zg, g + z * args.gateways_per_zone)
	307	client_id = gateway_name(zg, z, g)
	308	gateway = Gateway(client_id, 'localhost', port, cluster, zone)
	309	if bootstrap:
	310	gateway.start()
	311	zone.gateways.append(gateway)
	312
	313	if is_master_zg and is_master:
	314	if bootstrap:
	315	# create admin user
	316	arg = ['--display-name', '"Zone User"', '--system']
	317	arg += admin_creds.credential_args()
	318	admin_user.create(zone, arg)
	319	# create test user
	320	arg = ['--display-name', '"Test User"']
	321	arg += user_creds.credential_args()
	322	if args.tenant:
	323	cmd += ['--tenant', args.tenant]
	324	user.create(zone, arg)
	325	else:
	326	# read users and update keys
	327	admin_user.info(zone)
	328	admin_creds = admin_user.credentials[0]
	329	user.info(zone)
	330	user_creds = user.credentials[0]
	331
	332	if not bootstrap:
	333	period.get(c1)
	334
31f18b77 FG	335	config = Config(checkpoint_retries=args.checkpoint_retries,
	336	checkpoint_delay=args.checkpoint_delay,
	337	reconfigure_delay=args.reconfigure_delay)
	338	init_multi(realm, user, config)
7c673cae FG	339
	340	def setup_module():
	341	init(False)
	342
	343	if __name__ == "__main__":
	344	init(True)