]>
Commit | Line | Data |
---|---|---|
7c673cae FG |
1 | import subprocess |
2 | import os | |
3 | import random | |
4 | import string | |
5 | import argparse | |
6 | import sys | |
7 | import logging | |
8 | try: | |
9 | import configparser | |
10 | except ImportError: | |
11 | import ConfigParser as configparser | |
12 | ||
13 | import nose.core | |
14 | ||
15 | from rgw_multi import multisite | |
31f18b77 | 16 | from rgw_multi.zone_rados import RadosZone as RadosZone |
11fdf7f2 TL |
17 | from rgw_multi.zone_es import ESZone as ESZone |
18 | from rgw_multi.zone_es import ESZoneConfig as ESZoneConfig | |
19 | from rgw_multi.zone_cloud import CloudZone as CloudZone | |
20 | from rgw_multi.zone_cloud import CloudZoneConfig as CloudZoneConfig | |
21 | from rgw_multi.zone_ps import PSZone as PSZone | |
eafe8130 | 22 | from rgw_multi.zone_ps import PSZoneConfig as PSZoneConfig |
31f18b77 | 23 | |
7c673cae FG |
24 | # make tests from rgw_multi.tests available to nose |
25 | from rgw_multi.tests import * | |
31f18b77 | 26 | from rgw_multi.tests_es import * |
11fdf7f2 | 27 | from rgw_multi.tests_ps import * |
7c673cae FG |
28 | |
29 | mstart_path = os.getenv('MSTART_PATH') | |
30 | if mstart_path is None: | |
31 | mstart_path = os.path.normpath(os.path.dirname(os.path.realpath(__file__)) + '/../..') + '/' | |
32 | ||
33 | test_path = os.path.normpath(os.path.dirname(os.path.realpath(__file__))) + '/' | |
34 | ||
35 | # configure logging for the tests module | |
36 | log = logging.getLogger('rgw_multi.tests') | |
37 | ||
38 | def bash(cmd, **kwargs): | |
39 | log.debug('running cmd: %s', ' '.join(cmd)) | |
40 | check_retcode = kwargs.pop('check_retcode', True) | |
41 | kwargs['stdout'] = subprocess.PIPE | |
42 | process = subprocess.Popen(cmd, **kwargs) | |
43 | s = process.communicate()[0] | |
44 | log.debug('command returned status=%d stdout=%s', process.returncode, s.decode('utf-8')) | |
45 | if check_retcode: | |
46 | assert(process.returncode == 0) | |
47 | return (s, process.returncode) | |
48 | ||
49 | class Cluster(multisite.Cluster): | |
50 | """ cluster implementation based on mstart/mrun scripts """ | |
51 | def __init__(self, cluster_id): | |
52 | super(Cluster, self).__init__() | |
53 | self.cluster_id = cluster_id | |
54 | self.needs_reset = True | |
55 | ||
56 | def admin(self, args = None, **kwargs): | |
57 | """ radosgw-admin command """ | |
58 | cmd = [test_path + 'test-rgw-call.sh', 'call_rgw_admin', self.cluster_id] | |
59 | if args: | |
60 | cmd += args | |
b32b8144 FG |
61 | cmd += ['--debug-rgw', str(kwargs.pop('debug_rgw', 0))] |
62 | cmd += ['--debug-ms', str(kwargs.pop('debug_ms', 0))] | |
7c673cae FG |
63 | if kwargs.pop('read_only', False): |
64 | cmd += ['--rgw-cache-enabled', 'false'] | |
65 | return bash(cmd, **kwargs) | |
66 | ||
67 | def start(self): | |
68 | cmd = [mstart_path + 'mstart.sh', self.cluster_id] | |
11fdf7f2 | 69 | env = None |
7c673cae | 70 | if self.needs_reset: |
11fdf7f2 TL |
71 | env = os.environ.copy() |
72 | env['CEPH_NUM_MDS'] = '0' | |
73 | cmd += ['-n'] | |
eafe8130 TL |
74 | # cmd += ['-o'] |
75 | # cmd += ['rgw_cache_enabled=false'] | |
11fdf7f2 | 76 | bash(cmd, env=env) |
7c673cae FG |
77 | self.needs_reset = False |
78 | ||
79 | def stop(self): | |
80 | cmd = [mstart_path + 'mstop.sh', self.cluster_id] | |
81 | bash(cmd) | |
82 | ||
83 | class Gateway(multisite.Gateway): | |
84 | """ gateway implementation based on mrgw/mstop scripts """ | |
85 | def __init__(self, client_id = None, *args, **kwargs): | |
86 | super(Gateway, self).__init__(*args, **kwargs) | |
87 | self.id = client_id | |
88 | ||
89 | def start(self, args = None): | |
90 | """ start the gateway """ | |
91 | assert(self.cluster) | |
eafe8130 TL |
92 | env = os.environ.copy() |
93 | # to change frontend, set RGW_FRONTEND env variable | |
94 | # e.g. RGW_FRONTEND=civetweb | |
95 | # to run test under valgrind memcheck, set RGW_VALGRIND to 'yes' | |
96 | # e.g. RGW_VALGRIND=yes | |
7c673cae FG |
97 | cmd = [mstart_path + 'mrgw.sh', self.cluster.cluster_id, str(self.port)] |
98 | if self.id: | |
99 | cmd += ['-i', self.id] | |
100 | cmd += ['--debug-rgw=20', '--debug-ms=1'] | |
101 | if args: | |
102 | cmd += args | |
eafe8130 | 103 | bash(cmd, env=env) |
7c673cae FG |
104 | |
105 | def stop(self): | |
106 | """ stop the gateway """ | |
107 | assert(self.cluster) | |
108 | cmd = [mstart_path + 'mstop.sh', self.cluster.cluster_id, 'radosgw', self.id] | |
109 | bash(cmd) | |
110 | ||
111 | def gen_access_key(): | |
112 | return ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(16)) | |
113 | ||
114 | def gen_secret(): | |
115 | return ''.join(random.choice(string.ascii_uppercase + string.ascii_lowercase + string.digits) for _ in range(32)) | |
116 | ||
117 | def gen_credentials(): | |
118 | return multisite.Credentials(gen_access_key(), gen_secret()) | |
119 | ||
120 | def cluster_name(cluster_num): | |
121 | return 'c' + str(cluster_num) | |
122 | ||
123 | def zonegroup_name(zonegroup_num): | |
124 | return string.ascii_lowercase[zonegroup_num] | |
125 | ||
126 | def zone_name(zonegroup_num, zone_num): | |
127 | return zonegroup_name(zonegroup_num) + str(zone_num + 1) | |
128 | ||
129 | def gateway_port(zonegroup_num, gateway_num): | |
130 | return 8000 + 100 * zonegroup_num + gateway_num | |
131 | ||
132 | def gateway_name(zonegroup_num, zone_num, gateway_num): | |
133 | return zone_name(zonegroup_num, zone_num) + '-' + str(gateway_num + 1) | |
134 | ||
135 | def zone_endpoints(zonegroup_num, zone_num, gateways_per_zone): | |
136 | endpoints = [] | |
137 | base = gateway_port(zonegroup_num, zone_num * gateways_per_zone) | |
138 | for i in range(0, gateways_per_zone): | |
139 | endpoints.append('http://localhost:' + str(base + i)) | |
140 | return endpoints | |
141 | ||
142 | def get_log_level(log_level): | |
143 | if log_level >= 20: | |
144 | return logging.DEBUG | |
145 | if log_level >= 10: | |
146 | return logging.INFO | |
147 | if log_level >= 5: | |
148 | return logging.WARN | |
149 | if log_level >= 1: | |
150 | return logging.ERROR | |
151 | return logging.CRITICAL | |
152 | ||
153 | def setup_logging(log_level_console, log_file, log_level_file): | |
154 | if log_file: | |
155 | formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s') | |
156 | fh = logging.FileHandler(log_file) | |
157 | fh.setFormatter(formatter) | |
158 | fh.setLevel(get_log_level(log_level_file)) | |
159 | log.addHandler(fh) | |
160 | ||
161 | formatter = logging.Formatter('%(levelname)s %(message)s') | |
162 | ch = logging.StreamHandler() | |
163 | ch.setFormatter(formatter) | |
164 | ch.setLevel(get_log_level(log_level_console)) | |
165 | log.addHandler(ch) | |
11fdf7f2 | 166 | log.setLevel(get_log_level(log_level_console)) |
7c673cae FG |
167 | |
168 | def init(parse_args): | |
169 | cfg = configparser.RawConfigParser({ | |
170 | 'num_zonegroups': 1, | |
171 | 'num_zones': 3, | |
11fdf7f2 | 172 | 'num_ps_zones': 0, |
eafe8130 | 173 | 'num_az_zones': 0, |
7c673cae FG |
174 | 'gateways_per_zone': 2, |
175 | 'no_bootstrap': 'false', | |
176 | 'log_level': 20, | |
177 | 'log_file': None, | |
178 | 'file_log_level': 20, | |
179 | 'tenant': None, | |
31f18b77 FG |
180 | 'checkpoint_retries': 60, |
181 | 'checkpoint_delay': 5, | |
182 | 'reconfigure_delay': 5, | |
7c673cae FG |
183 | }) |
184 | try: | |
185 | path = os.environ['RGW_MULTI_TEST_CONF'] | |
186 | except KeyError: | |
31f18b77 | 187 | path = test_path + 'test_multi.conf' |
7c673cae FG |
188 | |
189 | try: | |
190 | with open(path) as f: | |
191 | cfg.readfp(f) | |
192 | except: | |
193 | print('WARNING: error reading test config. Path can be set through the RGW_MULTI_TEST_CONF env variable') | |
194 | pass | |
195 | ||
196 | parser = argparse.ArgumentParser( | |
197 | description='Run rgw multi-site tests', | |
198 | usage='test_multi [--num-zonegroups <num>] [--num-zones <num>] [--no-bootstrap]') | |
199 | ||
200 | section = 'DEFAULT' | |
201 | parser.add_argument('--num-zonegroups', type=int, default=cfg.getint(section, 'num_zonegroups')) | |
202 | parser.add_argument('--num-zones', type=int, default=cfg.getint(section, 'num_zones')) | |
203 | parser.add_argument('--gateways-per-zone', type=int, default=cfg.getint(section, 'gateways_per_zone')) | |
204 | parser.add_argument('--no-bootstrap', action='store_true', default=cfg.getboolean(section, 'no_bootstrap')) | |
205 | parser.add_argument('--log-level', type=int, default=cfg.getint(section, 'log_level')) | |
206 | parser.add_argument('--log-file', type=str, default=cfg.get(section, 'log_file')) | |
207 | parser.add_argument('--file-log-level', type=int, default=cfg.getint(section, 'file_log_level')) | |
208 | parser.add_argument('--tenant', type=str, default=cfg.get(section, 'tenant')) | |
31f18b77 FG |
209 | parser.add_argument('--checkpoint-retries', type=int, default=cfg.getint(section, 'checkpoint_retries')) |
210 | parser.add_argument('--checkpoint-delay', type=int, default=cfg.getint(section, 'checkpoint_delay')) | |
211 | parser.add_argument('--reconfigure-delay', type=int, default=cfg.getint(section, 'reconfigure_delay')) | |
11fdf7f2 TL |
212 | parser.add_argument('--num-ps-zones', type=int, default=cfg.getint(section, 'num_ps_zones')) |
213 | ||
eafe8130 | 214 | |
11fdf7f2 TL |
215 | es_cfg = [] |
216 | cloud_cfg = [] | |
eafe8130 TL |
217 | ps_cfg = [] |
218 | az_cfg = [] | |
11fdf7f2 TL |
219 | |
220 | for s in cfg.sections(): | |
221 | if s.startswith('elasticsearch'): | |
222 | es_cfg.append(ESZoneConfig(cfg, s)) | |
223 | elif s.startswith('cloud'): | |
224 | cloud_cfg.append(CloudZoneConfig(cfg, s)) | |
eafe8130 TL |
225 | elif s.startswith('pubsub'): |
226 | ps_cfg.append(PSZoneConfig(cfg, s)) | |
7c673cae FG |
227 | |
228 | argv = [] | |
229 | ||
230 | if parse_args: | |
231 | argv = sys.argv[1:] | |
232 | ||
233 | args = parser.parse_args(argv) | |
234 | bootstrap = not args.no_bootstrap | |
235 | ||
236 | setup_logging(args.log_level, args.log_file, args.file_log_level) | |
237 | ||
238 | # start first cluster | |
239 | c1 = Cluster(cluster_name(1)) | |
240 | if bootstrap: | |
241 | c1.start() | |
242 | clusters = [] | |
243 | clusters.append(c1) | |
244 | ||
245 | admin_creds = gen_credentials() | |
246 | admin_user = multisite.User('zone.user') | |
247 | ||
248 | user_creds = gen_credentials() | |
eafe8130 | 249 | user = multisite.User('tester', tenant=args.tenant) |
7c673cae FG |
250 | |
251 | realm = multisite.Realm('r') | |
252 | if bootstrap: | |
253 | # create the realm on c1 | |
254 | realm.create(c1) | |
255 | else: | |
256 | realm.get(c1) | |
257 | period = multisite.Period(realm=realm) | |
258 | realm.current_period = period | |
259 | ||
11fdf7f2 TL |
260 | num_es_zones = len(es_cfg) |
261 | num_cloud_zones = len(cloud_cfg) | |
eafe8130 TL |
262 | num_ps_zones_from_conf = len(ps_cfg) |
263 | num_ps_zones = args.num_ps_zones if num_ps_zones_from_conf == 0 else num_ps_zones_from_conf | |
264 | print 'num_ps_zones = ' + str(num_ps_zones) | |
11fdf7f2 TL |
265 | |
266 | num_zones = args.num_zones + num_es_zones + num_cloud_zones + args.num_ps_zones | |
31f18b77 | 267 | |
7c673cae FG |
268 | for zg in range(0, args.num_zonegroups): |
269 | zonegroup = multisite.ZoneGroup(zonegroup_name(zg), period) | |
270 | period.zonegroups.append(zonegroup) | |
271 | ||
272 | is_master_zg = zg == 0 | |
273 | if is_master_zg: | |
274 | period.master_zonegroup = zonegroup | |
275 | ||
31f18b77 | 276 | for z in range(0, num_zones): |
7c673cae FG |
277 | is_master = z == 0 |
278 | # start a cluster, or use c1 for first zone | |
279 | cluster = None | |
280 | if is_master_zg and is_master: | |
281 | cluster = c1 | |
282 | else: | |
283 | cluster = Cluster(cluster_name(len(clusters) + 1)) | |
284 | clusters.append(cluster) | |
285 | if bootstrap: | |
286 | cluster.start() | |
287 | # pull realm configuration from the master's gateway | |
288 | gateway = realm.meta_master_zone().gateways[0] | |
289 | realm.pull(cluster, gateway, admin_creds) | |
290 | ||
291 | endpoints = zone_endpoints(zg, z, args.gateways_per_zone) | |
292 | if is_master: | |
293 | if bootstrap: | |
294 | # create the zonegroup on its first zone's cluster | |
295 | arg = [] | |
296 | if is_master_zg: | |
297 | arg += ['--master'] | |
298 | if len(endpoints): # use master zone's endpoints | |
299 | arg += ['--endpoints', ','.join(endpoints)] | |
300 | zonegroup.create(cluster, arg) | |
301 | else: | |
302 | zonegroup.get(cluster) | |
303 | ||
11fdf7f2 TL |
304 | es_zone = (z >= args.num_zones and z < args.num_zones + num_es_zones) |
305 | cloud_zone = (z >= args.num_zones + num_es_zones and z < args.num_zones + num_es_zones + num_cloud_zones) | |
306 | ps_zone = (z >= args.num_zones + num_es_zones + num_cloud_zones) | |
31f18b77 | 307 | |
7c673cae FG |
308 | # create the zone in its zonegroup |
309 | zone = multisite.Zone(zone_name(zg, z), zonegroup, cluster) | |
31f18b77 | 310 | if es_zone: |
11fdf7f2 TL |
311 | zone_index = z - args.num_zones |
312 | zone = ESZone(zone_name(zg, z), es_cfg[zone_index].endpoint, zonegroup, cluster) | |
313 | elif cloud_zone: | |
314 | zone_index = z - args.num_zones - num_es_zones | |
315 | ccfg = cloud_cfg[zone_index] | |
316 | zone = CloudZone(zone_name(zg, z), ccfg.endpoint, ccfg.credentials, ccfg.source_bucket, | |
317 | ccfg.target_path, zonegroup, cluster) | |
318 | elif ps_zone: | |
319 | zone_index = z - args.num_zones - num_es_zones - num_cloud_zones | |
eafe8130 TL |
320 | if num_ps_zones_from_conf == 0: |
321 | zone = PSZone(zone_name(zg, z), zonegroup, cluster) | |
322 | else: | |
323 | pscfg = ps_cfg[zone_index] | |
324 | zone = PSZone(zone_name(zg, z), zonegroup, cluster, | |
325 | full_sync=pscfg.full_sync, retention_days=pscfg.retention_days) | |
31f18b77 FG |
326 | else: |
327 | zone = RadosZone(zone_name(zg, z), zonegroup, cluster) | |
328 | ||
7c673cae FG |
329 | if bootstrap: |
330 | arg = admin_creds.credential_args() | |
331 | if is_master: | |
332 | arg += ['--master'] | |
333 | if len(endpoints): | |
334 | arg += ['--endpoints', ','.join(endpoints)] | |
335 | zone.create(cluster, arg) | |
336 | else: | |
337 | zone.get(cluster) | |
338 | zonegroup.zones.append(zone) | |
339 | if is_master: | |
340 | zonegroup.master_zone = zone | |
341 | ||
31f18b77 FG |
342 | zonegroup.zones_by_type.setdefault(zone.tier_type(), []).append(zone) |
343 | ||
344 | if zone.is_read_only(): | |
345 | zonegroup.ro_zones.append(zone) | |
346 | else: | |
347 | zonegroup.rw_zones.append(zone) | |
348 | ||
7c673cae FG |
349 | # update/commit the period |
350 | if bootstrap: | |
351 | period.update(zone, commit=True) | |
352 | ||
353 | # start the gateways | |
354 | for g in range(0, args.gateways_per_zone): | |
355 | port = gateway_port(zg, g + z * args.gateways_per_zone) | |
356 | client_id = gateway_name(zg, z, g) | |
357 | gateway = Gateway(client_id, 'localhost', port, cluster, zone) | |
358 | if bootstrap: | |
359 | gateway.start() | |
360 | zone.gateways.append(gateway) | |
361 | ||
362 | if is_master_zg and is_master: | |
363 | if bootstrap: | |
364 | # create admin user | |
365 | arg = ['--display-name', '"Zone User"', '--system'] | |
366 | arg += admin_creds.credential_args() | |
367 | admin_user.create(zone, arg) | |
368 | # create test user | |
369 | arg = ['--display-name', '"Test User"'] | |
370 | arg += user_creds.credential_args() | |
7c673cae FG |
371 | user.create(zone, arg) |
372 | else: | |
373 | # read users and update keys | |
374 | admin_user.info(zone) | |
375 | admin_creds = admin_user.credentials[0] | |
eafe8130 TL |
376 | arg = [] |
377 | user.info(zone, arg) | |
7c673cae FG |
378 | user_creds = user.credentials[0] |
379 | ||
380 | if not bootstrap: | |
381 | period.get(c1) | |
382 | ||
31f18b77 FG |
383 | config = Config(checkpoint_retries=args.checkpoint_retries, |
384 | checkpoint_delay=args.checkpoint_delay, | |
eafe8130 TL |
385 | reconfigure_delay=args.reconfigure_delay, |
386 | tenant=args.tenant) | |
31f18b77 | 387 | init_multi(realm, user, config) |
7c673cae FG |
388 | |
389 | def setup_module(): | |
390 | init(False) | |
391 | ||
392 | if __name__ == "__main__": | |
393 | init(True) | |
11fdf7f2 | 394 |