]>
Commit | Line | Data |
---|---|---|
7c673cae FG |
1 | import subprocess |
2 | import os | |
3 | import random | |
4 | import string | |
5 | import argparse | |
6 | import sys | |
7 | import logging | |
8 | try: | |
9 | import configparser | |
10 | except ImportError: | |
11 | import ConfigParser as configparser | |
12 | ||
13 | import nose.core | |
14 | ||
15 | from rgw_multi import multisite | |
31f18b77 FG |
16 | from rgw_multi.zone_rados import RadosZone as RadosZone |
17 | from rgw_multi.zone_es import ESZone as ESZone | |
18 | ||
7c673cae FG |
19 | # make tests from rgw_multi.tests available to nose |
20 | from rgw_multi.tests import * | |
31f18b77 | 21 | from rgw_multi.tests_es import * |
7c673cae FG |
22 | |
23 | mstart_path = os.getenv('MSTART_PATH') | |
24 | if mstart_path is None: | |
25 | mstart_path = os.path.normpath(os.path.dirname(os.path.realpath(__file__)) + '/../..') + '/' | |
26 | ||
27 | test_path = os.path.normpath(os.path.dirname(os.path.realpath(__file__))) + '/' | |
28 | ||
29 | # configure logging for the tests module | |
30 | log = logging.getLogger('rgw_multi.tests') | |
31 | ||
32 | def bash(cmd, **kwargs): | |
33 | log.debug('running cmd: %s', ' '.join(cmd)) | |
34 | check_retcode = kwargs.pop('check_retcode', True) | |
35 | kwargs['stdout'] = subprocess.PIPE | |
36 | process = subprocess.Popen(cmd, **kwargs) | |
37 | s = process.communicate()[0] | |
38 | log.debug('command returned status=%d stdout=%s', process.returncode, s.decode('utf-8')) | |
39 | if check_retcode: | |
40 | assert(process.returncode == 0) | |
41 | return (s, process.returncode) | |
42 | ||
43 | class Cluster(multisite.Cluster): | |
44 | """ cluster implementation based on mstart/mrun scripts """ | |
45 | def __init__(self, cluster_id): | |
46 | super(Cluster, self).__init__() | |
47 | self.cluster_id = cluster_id | |
48 | self.needs_reset = True | |
49 | ||
50 | def admin(self, args = None, **kwargs): | |
51 | """ radosgw-admin command """ | |
52 | cmd = [test_path + 'test-rgw-call.sh', 'call_rgw_admin', self.cluster_id] | |
53 | if args: | |
54 | cmd += args | |
55 | if kwargs.pop('read_only', False): | |
56 | cmd += ['--rgw-cache-enabled', 'false'] | |
57 | return bash(cmd, **kwargs) | |
58 | ||
59 | def start(self): | |
60 | cmd = [mstart_path + 'mstart.sh', self.cluster_id] | |
61 | if self.needs_reset: | |
62 | cmd += ['-n', '--mds_num', '0'] | |
63 | bash(cmd) | |
64 | self.needs_reset = False | |
65 | ||
66 | def stop(self): | |
67 | cmd = [mstart_path + 'mstop.sh', self.cluster_id] | |
68 | bash(cmd) | |
69 | ||
70 | class Gateway(multisite.Gateway): | |
71 | """ gateway implementation based on mrgw/mstop scripts """ | |
72 | def __init__(self, client_id = None, *args, **kwargs): | |
73 | super(Gateway, self).__init__(*args, **kwargs) | |
74 | self.id = client_id | |
75 | ||
76 | def start(self, args = None): | |
77 | """ start the gateway """ | |
78 | assert(self.cluster) | |
79 | cmd = [mstart_path + 'mrgw.sh', self.cluster.cluster_id, str(self.port)] | |
80 | if self.id: | |
81 | cmd += ['-i', self.id] | |
82 | cmd += ['--debug-rgw=20', '--debug-ms=1'] | |
83 | if args: | |
84 | cmd += args | |
85 | bash(cmd) | |
86 | ||
87 | def stop(self): | |
88 | """ stop the gateway """ | |
89 | assert(self.cluster) | |
90 | cmd = [mstart_path + 'mstop.sh', self.cluster.cluster_id, 'radosgw', self.id] | |
91 | bash(cmd) | |
92 | ||
93 | def gen_access_key(): | |
94 | return ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(16)) | |
95 | ||
96 | def gen_secret(): | |
97 | return ''.join(random.choice(string.ascii_uppercase + string.ascii_lowercase + string.digits) for _ in range(32)) | |
98 | ||
99 | def gen_credentials(): | |
100 | return multisite.Credentials(gen_access_key(), gen_secret()) | |
101 | ||
102 | def cluster_name(cluster_num): | |
103 | return 'c' + str(cluster_num) | |
104 | ||
105 | def zonegroup_name(zonegroup_num): | |
106 | return string.ascii_lowercase[zonegroup_num] | |
107 | ||
108 | def zone_name(zonegroup_num, zone_num): | |
109 | return zonegroup_name(zonegroup_num) + str(zone_num + 1) | |
110 | ||
111 | def gateway_port(zonegroup_num, gateway_num): | |
112 | return 8000 + 100 * zonegroup_num + gateway_num | |
113 | ||
114 | def gateway_name(zonegroup_num, zone_num, gateway_num): | |
115 | return zone_name(zonegroup_num, zone_num) + '-' + str(gateway_num + 1) | |
116 | ||
117 | def zone_endpoints(zonegroup_num, zone_num, gateways_per_zone): | |
118 | endpoints = [] | |
119 | base = gateway_port(zonegroup_num, zone_num * gateways_per_zone) | |
120 | for i in range(0, gateways_per_zone): | |
121 | endpoints.append('http://localhost:' + str(base + i)) | |
122 | return endpoints | |
123 | ||
124 | def get_log_level(log_level): | |
125 | if log_level >= 20: | |
126 | return logging.DEBUG | |
127 | if log_level >= 10: | |
128 | return logging.INFO | |
129 | if log_level >= 5: | |
130 | return logging.WARN | |
131 | if log_level >= 1: | |
132 | return logging.ERROR | |
133 | return logging.CRITICAL | |
134 | ||
135 | def setup_logging(log_level_console, log_file, log_level_file): | |
136 | if log_file: | |
137 | formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s') | |
138 | fh = logging.FileHandler(log_file) | |
139 | fh.setFormatter(formatter) | |
140 | fh.setLevel(get_log_level(log_level_file)) | |
141 | log.addHandler(fh) | |
142 | ||
143 | formatter = logging.Formatter('%(levelname)s %(message)s') | |
144 | ch = logging.StreamHandler() | |
145 | ch.setFormatter(formatter) | |
146 | ch.setLevel(get_log_level(log_level_console)) | |
147 | log.addHandler(ch) | |
148 | ||
149 | def init(parse_args): | |
150 | cfg = configparser.RawConfigParser({ | |
151 | 'num_zonegroups': 1, | |
152 | 'num_zones': 3, | |
31f18b77 | 153 | 'num_es_zones': 0, |
7c673cae FG |
154 | 'gateways_per_zone': 2, |
155 | 'no_bootstrap': 'false', | |
156 | 'log_level': 20, | |
157 | 'log_file': None, | |
158 | 'file_log_level': 20, | |
159 | 'tenant': None, | |
31f18b77 FG |
160 | 'checkpoint_retries': 60, |
161 | 'checkpoint_delay': 5, | |
162 | 'reconfigure_delay': 5, | |
163 | 'es_endpoint': None, | |
7c673cae FG |
164 | }) |
165 | try: | |
166 | path = os.environ['RGW_MULTI_TEST_CONF'] | |
167 | except KeyError: | |
31f18b77 | 168 | path = test_path + 'test_multi.conf' |
7c673cae FG |
169 | |
170 | try: | |
171 | with open(path) as f: | |
172 | cfg.readfp(f) | |
173 | except: | |
174 | print('WARNING: error reading test config. Path can be set through the RGW_MULTI_TEST_CONF env variable') | |
175 | pass | |
176 | ||
177 | parser = argparse.ArgumentParser( | |
178 | description='Run rgw multi-site tests', | |
179 | usage='test_multi [--num-zonegroups <num>] [--num-zones <num>] [--no-bootstrap]') | |
180 | ||
181 | section = 'DEFAULT' | |
182 | parser.add_argument('--num-zonegroups', type=int, default=cfg.getint(section, 'num_zonegroups')) | |
183 | parser.add_argument('--num-zones', type=int, default=cfg.getint(section, 'num_zones')) | |
31f18b77 | 184 | parser.add_argument('--num-es-zones', type=int, default=cfg.getint(section, 'num_es_zones')) |
7c673cae FG |
185 | parser.add_argument('--gateways-per-zone', type=int, default=cfg.getint(section, 'gateways_per_zone')) |
186 | parser.add_argument('--no-bootstrap', action='store_true', default=cfg.getboolean(section, 'no_bootstrap')) | |
187 | parser.add_argument('--log-level', type=int, default=cfg.getint(section, 'log_level')) | |
188 | parser.add_argument('--log-file', type=str, default=cfg.get(section, 'log_file')) | |
189 | parser.add_argument('--file-log-level', type=int, default=cfg.getint(section, 'file_log_level')) | |
190 | parser.add_argument('--tenant', type=str, default=cfg.get(section, 'tenant')) | |
31f18b77 FG |
191 | parser.add_argument('--checkpoint-retries', type=int, default=cfg.getint(section, 'checkpoint_retries')) |
192 | parser.add_argument('--checkpoint-delay', type=int, default=cfg.getint(section, 'checkpoint_delay')) | |
193 | parser.add_argument('--reconfigure-delay', type=int, default=cfg.getint(section, 'reconfigure_delay')) | |
194 | parser.add_argument('--es-endpoint', type=str, default=cfg.get(section, 'es_endpoint')) | |
7c673cae FG |
195 | |
196 | argv = [] | |
197 | ||
198 | if parse_args: | |
199 | argv = sys.argv[1:] | |
200 | ||
201 | args = parser.parse_args(argv) | |
202 | bootstrap = not args.no_bootstrap | |
203 | ||
31f18b77 FG |
204 | # if num_es_zones is defined, need to have es_endpoint defined too |
205 | assert(args.num_es_zones == 0 or args.es_endpoint) | |
206 | ||
7c673cae FG |
207 | setup_logging(args.log_level, args.log_file, args.file_log_level) |
208 | ||
209 | # start first cluster | |
210 | c1 = Cluster(cluster_name(1)) | |
211 | if bootstrap: | |
212 | c1.start() | |
213 | clusters = [] | |
214 | clusters.append(c1) | |
215 | ||
216 | admin_creds = gen_credentials() | |
217 | admin_user = multisite.User('zone.user') | |
218 | ||
219 | user_creds = gen_credentials() | |
220 | user = multisite.User('tester') | |
221 | ||
222 | realm = multisite.Realm('r') | |
223 | if bootstrap: | |
224 | # create the realm on c1 | |
225 | realm.create(c1) | |
226 | else: | |
227 | realm.get(c1) | |
228 | period = multisite.Period(realm=realm) | |
229 | realm.current_period = period | |
230 | ||
31f18b77 FG |
231 | num_zones = args.num_zones + args.num_es_zones |
232 | ||
7c673cae FG |
233 | for zg in range(0, args.num_zonegroups): |
234 | zonegroup = multisite.ZoneGroup(zonegroup_name(zg), period) | |
235 | period.zonegroups.append(zonegroup) | |
236 | ||
237 | is_master_zg = zg == 0 | |
238 | if is_master_zg: | |
239 | period.master_zonegroup = zonegroup | |
240 | ||
31f18b77 | 241 | for z in range(0, num_zones): |
7c673cae FG |
242 | is_master = z == 0 |
243 | # start a cluster, or use c1 for first zone | |
244 | cluster = None | |
245 | if is_master_zg and is_master: | |
246 | cluster = c1 | |
247 | else: | |
248 | cluster = Cluster(cluster_name(len(clusters) + 1)) | |
249 | clusters.append(cluster) | |
250 | if bootstrap: | |
251 | cluster.start() | |
252 | # pull realm configuration from the master's gateway | |
253 | gateway = realm.meta_master_zone().gateways[0] | |
254 | realm.pull(cluster, gateway, admin_creds) | |
255 | ||
256 | endpoints = zone_endpoints(zg, z, args.gateways_per_zone) | |
257 | if is_master: | |
258 | if bootstrap: | |
259 | # create the zonegroup on its first zone's cluster | |
260 | arg = [] | |
261 | if is_master_zg: | |
262 | arg += ['--master'] | |
263 | if len(endpoints): # use master zone's endpoints | |
264 | arg += ['--endpoints', ','.join(endpoints)] | |
265 | zonegroup.create(cluster, arg) | |
266 | else: | |
267 | zonegroup.get(cluster) | |
268 | ||
31f18b77 FG |
269 | es_zone = (z >= args.num_zones) |
270 | ||
7c673cae FG |
271 | # create the zone in its zonegroup |
272 | zone = multisite.Zone(zone_name(zg, z), zonegroup, cluster) | |
31f18b77 FG |
273 | if es_zone: |
274 | zone = ESZone(zone_name(zg, z), args.es_endpoint, zonegroup, cluster) | |
275 | else: | |
276 | zone = RadosZone(zone_name(zg, z), zonegroup, cluster) | |
277 | ||
7c673cae FG |
278 | if bootstrap: |
279 | arg = admin_creds.credential_args() | |
280 | if is_master: | |
281 | arg += ['--master'] | |
282 | if len(endpoints): | |
283 | arg += ['--endpoints', ','.join(endpoints)] | |
284 | zone.create(cluster, arg) | |
285 | else: | |
286 | zone.get(cluster) | |
287 | zonegroup.zones.append(zone) | |
288 | if is_master: | |
289 | zonegroup.master_zone = zone | |
290 | ||
31f18b77 FG |
291 | zonegroup.zones_by_type.setdefault(zone.tier_type(), []).append(zone) |
292 | ||
293 | if zone.is_read_only(): | |
294 | zonegroup.ro_zones.append(zone) | |
295 | else: | |
296 | zonegroup.rw_zones.append(zone) | |
297 | ||
7c673cae FG |
298 | # update/commit the period |
299 | if bootstrap: | |
300 | period.update(zone, commit=True) | |
301 | ||
302 | # start the gateways | |
303 | for g in range(0, args.gateways_per_zone): | |
304 | port = gateway_port(zg, g + z * args.gateways_per_zone) | |
305 | client_id = gateway_name(zg, z, g) | |
306 | gateway = Gateway(client_id, 'localhost', port, cluster, zone) | |
307 | if bootstrap: | |
308 | gateway.start() | |
309 | zone.gateways.append(gateway) | |
310 | ||
311 | if is_master_zg and is_master: | |
312 | if bootstrap: | |
313 | # create admin user | |
314 | arg = ['--display-name', '"Zone User"', '--system'] | |
315 | arg += admin_creds.credential_args() | |
316 | admin_user.create(zone, arg) | |
317 | # create test user | |
318 | arg = ['--display-name', '"Test User"'] | |
319 | arg += user_creds.credential_args() | |
320 | if args.tenant: | |
321 | cmd += ['--tenant', args.tenant] | |
322 | user.create(zone, arg) | |
323 | else: | |
324 | # read users and update keys | |
325 | admin_user.info(zone) | |
326 | admin_creds = admin_user.credentials[0] | |
327 | user.info(zone) | |
328 | user_creds = user.credentials[0] | |
329 | ||
330 | if not bootstrap: | |
331 | period.get(c1) | |
332 | ||
31f18b77 FG |
333 | config = Config(checkpoint_retries=args.checkpoint_retries, |
334 | checkpoint_delay=args.checkpoint_delay, | |
335 | reconfigure_delay=args.reconfigure_delay) | |
336 | init_multi(realm, user, config) | |
7c673cae FG |
337 | |
338 | def setup_module(): | |
339 | init(False) | |
340 | ||
341 | if __name__ == "__main__": | |
342 | init(True) |