]>
Commit | Line | Data |
---|---|---|
7c673cae FG |
1 | """ |
2 | Execute ceph-deploy as a task | |
3 | """ | |
4 | from cStringIO import StringIO | |
5 | ||
6 | import contextlib | |
7 | import os | |
8 | import time | |
9 | import logging | |
10 | import traceback | |
11 | ||
12 | from teuthology import misc as teuthology | |
13 | from teuthology import contextutil | |
14 | from teuthology.config import config as teuth_config | |
15 | from teuthology.task import install as install_fn | |
16 | from teuthology.orchestra import run | |
17 | from tasks.cephfs.filesystem import Filesystem | |
18 | ||
19 | log = logging.getLogger(__name__) | |
20 | ||
21 | ||
22 | @contextlib.contextmanager | |
23 | def download_ceph_deploy(ctx, config): | |
24 | """ | |
25 | Downloads ceph-deploy from the ceph.com git mirror and (by default) | |
26 | switches to the master branch. If the `ceph-deploy-branch` is specified, it | |
27 | will use that instead. The `bootstrap` script is ran, with the argument | |
28 | obtained from `python_version`, if specified. | |
29 | """ | |
30 | ceph_admin = ctx.cluster.only(teuthology.get_first_mon(ctx, config)) | |
31 | ||
32 | try: | |
33 | py_ver = str(config['python_version']) | |
34 | except KeyError: | |
35 | pass | |
36 | else: | |
37 | supported_versions = ['2', '3'] | |
38 | if py_ver not in supported_versions: | |
39 | raise ValueError("python_version must be: {}, not {}".format( | |
40 | ' or '.join(supported_versions), py_ver | |
41 | )) | |
42 | ||
43 | log.info("Installing Python") | |
44 | for admin in ceph_admin.remotes: | |
45 | system_type = teuthology.get_system_type(admin) | |
46 | ||
47 | if system_type == 'rpm': | |
48 | package = 'python34' if py_ver == '3' else 'python' | |
49 | ctx.cluster.run(args=[ | |
50 | 'sudo', 'yum', '-y', 'install', | |
51 | package, 'python-virtualenv' | |
52 | ]) | |
53 | else: | |
54 | package = 'python3' if py_ver == '3' else 'python' | |
55 | ctx.cluster.run(args=[ | |
56 | 'sudo', 'apt-get', '-y', '--force-yes', 'install', | |
57 | package, 'python-virtualenv' | |
58 | ]) | |
59 | ||
60 | log.info('Downloading ceph-deploy...') | |
61 | testdir = teuthology.get_testdir(ctx) | |
62 | ceph_deploy_branch = config.get('ceph-deploy-branch', 'master') | |
63 | ||
64 | ceph_admin.run( | |
65 | args=[ | |
66 | 'git', 'clone', '-b', ceph_deploy_branch, | |
67 | teuth_config.ceph_git_base_url + 'ceph-deploy.git', | |
68 | '{tdir}/ceph-deploy'.format(tdir=testdir), | |
69 | ], | |
70 | ) | |
71 | args = [ | |
72 | 'cd', | |
73 | '{tdir}/ceph-deploy'.format(tdir=testdir), | |
74 | run.Raw('&&'), | |
75 | './bootstrap', | |
76 | ] | |
77 | try: | |
78 | args.append(str(config['python_version'])) | |
79 | except KeyError: | |
80 | pass | |
81 | ceph_admin.run(args=args) | |
82 | ||
83 | try: | |
84 | yield | |
85 | finally: | |
86 | log.info('Removing ceph-deploy ...') | |
87 | ceph_admin.run( | |
88 | args=[ | |
89 | 'rm', | |
90 | '-rf', | |
91 | '{tdir}/ceph-deploy'.format(tdir=testdir), | |
92 | ], | |
93 | ) | |
94 | ||
95 | ||
96 | def is_healthy(ctx, config): | |
97 | """Wait until a Ceph cluster is healthy.""" | |
98 | testdir = teuthology.get_testdir(ctx) | |
99 | ceph_admin = teuthology.get_first_mon(ctx, config) | |
100 | (remote,) = ctx.cluster.only(ceph_admin).remotes.keys() | |
101 | max_tries = 90 # 90 tries * 10 secs --> 15 minutes | |
102 | tries = 0 | |
103 | while True: | |
104 | tries += 1 | |
105 | if tries >= max_tries: | |
106 | msg = "ceph health was unable to get 'HEALTH_OK' after waiting 15 minutes" | |
107 | remote.run( | |
108 | args=[ | |
109 | 'cd', | |
110 | '{tdir}'.format(tdir=testdir), | |
111 | run.Raw('&&'), | |
112 | 'sudo', 'ceph', | |
113 | 'report', | |
114 | ], | |
115 | ) | |
116 | raise RuntimeError(msg) | |
117 | ||
118 | r = remote.run( | |
119 | args=[ | |
120 | 'cd', | |
121 | '{tdir}'.format(tdir=testdir), | |
122 | run.Raw('&&'), | |
123 | 'sudo', 'ceph', | |
124 | 'health', | |
125 | ], | |
126 | stdout=StringIO(), | |
127 | logger=log.getChild('health'), | |
128 | ) | |
129 | out = r.stdout.getvalue() | |
130 | log.info('Ceph health: %s', out.rstrip('\n')) | |
131 | if out.split(None, 1)[0] == 'HEALTH_OK': | |
132 | break | |
133 | time.sleep(10) | |
134 | ||
135 | ||
136 | def get_nodes_using_role(ctx, target_role): | |
137 | """ | |
138 | Extract the names of nodes that match a given role from a cluster, and modify the | |
139 | cluster's service IDs to match the resulting node-based naming scheme that ceph-deploy | |
140 | uses, such that if "mon.a" is on host "foo23", it'll be renamed to "mon.foo23". | |
141 | """ | |
142 | ||
143 | # Nodes containing a service of the specified role | |
144 | nodes_of_interest = [] | |
145 | ||
146 | # Prepare a modified version of cluster.remotes with ceph-deploy-ized names | |
147 | modified_remotes = {} | |
148 | ||
149 | for _remote, roles_for_host in ctx.cluster.remotes.iteritems(): | |
150 | modified_remotes[_remote] = [] | |
151 | for svc_id in roles_for_host: | |
152 | if svc_id.startswith("{0}.".format(target_role)): | |
153 | fqdn = str(_remote).split('@')[-1] | |
154 | nodename = str(str(_remote).split('.')[0]).split('@')[1] | |
155 | if target_role == 'mon': | |
156 | nodes_of_interest.append(fqdn) | |
157 | else: | |
158 | nodes_of_interest.append(nodename) | |
159 | ||
160 | modified_remotes[_remote].append( | |
161 | "{0}.{1}".format(target_role, nodename)) | |
162 | else: | |
163 | modified_remotes[_remote].append(svc_id) | |
164 | ||
165 | ctx.cluster.remotes = modified_remotes | |
166 | ||
167 | return nodes_of_interest | |
168 | ||
169 | ||
170 | def get_dev_for_osd(ctx, config): | |
171 | """Get a list of all osd device names.""" | |
172 | osd_devs = [] | |
173 | for remote, roles_for_host in ctx.cluster.remotes.iteritems(): | |
174 | host = remote.name.split('@')[-1] | |
175 | shortname = host.split('.')[0] | |
176 | devs = teuthology.get_scratch_devices(remote) | |
177 | num_osd_per_host = list( | |
178 | teuthology.roles_of_type( | |
179 | roles_for_host, 'osd')) | |
180 | num_osds = len(num_osd_per_host) | |
181 | if config.get('separate_journal_disk') is not None: | |
182 | num_devs_reqd = 2 * num_osds | |
183 | assert num_devs_reqd <= len( | |
184 | devs), 'fewer data and journal disks than required ' + shortname | |
185 | for dindex in range(0, num_devs_reqd, 2): | |
186 | jd_index = dindex + 1 | |
187 | dev_short = devs[dindex].split('/')[-1] | |
188 | jdev_short = devs[jd_index].split('/')[-1] | |
189 | osd_devs.append((shortname, dev_short, jdev_short)) | |
190 | else: | |
191 | assert num_osds <= len(devs), 'fewer disks than osds ' + shortname | |
192 | for dev in devs[:num_osds]: | |
193 | dev_short = dev.split('/')[-1] | |
194 | osd_devs.append((shortname, dev_short)) | |
195 | return osd_devs | |
196 | ||
197 | ||
198 | def get_all_nodes(ctx, config): | |
199 | """Return a string of node names separated by blanks""" | |
200 | nodelist = [] | |
201 | for t, k in ctx.config['targets'].iteritems(): | |
202 | host = t.split('@')[-1] | |
203 | simple_host = host.split('.')[0] | |
204 | nodelist.append(simple_host) | |
205 | nodelist = " ".join(nodelist) | |
206 | return nodelist | |
207 | ||
208 | ||
209 | @contextlib.contextmanager | |
210 | def build_ceph_cluster(ctx, config): | |
211 | """Build a ceph cluster""" | |
212 | ||
213 | # Expect to find ceph_admin on the first mon by ID, same place that the download task | |
214 | # puts it. Remember this here, because subsequently IDs will change from those in | |
215 | # the test config to those that ceph-deploy invents. | |
216 | (ceph_admin,) = ctx.cluster.only( | |
217 | teuthology.get_first_mon(ctx, config)).remotes.iterkeys() | |
218 | ||
219 | def execute_ceph_deploy(cmd): | |
220 | """Remotely execute a ceph_deploy command""" | |
221 | return ceph_admin.run( | |
222 | args=[ | |
223 | 'cd', | |
224 | '{tdir}/ceph-deploy'.format(tdir=testdir), | |
225 | run.Raw('&&'), | |
226 | run.Raw(cmd), | |
227 | ], | |
228 | check_status=False, | |
229 | ).exitstatus | |
230 | ||
231 | try: | |
232 | log.info('Building ceph cluster using ceph-deploy...') | |
233 | testdir = teuthology.get_testdir(ctx) | |
234 | ceph_branch = None | |
235 | if config.get('branch') is not None: | |
236 | cbranch = config.get('branch') | |
237 | for var, val in cbranch.iteritems(): | |
238 | ceph_branch = '--{var}={val}'.format(var=var, val=val) | |
239 | all_nodes = get_all_nodes(ctx, config) | |
240 | mds_nodes = get_nodes_using_role(ctx, 'mds') | |
241 | mds_nodes = " ".join(mds_nodes) | |
242 | mon_node = get_nodes_using_role(ctx, 'mon') | |
243 | mon_nodes = " ".join(mon_node) | |
224ce89b WB |
244 | mgr_nodes = get_nodes_using_role(ctx, 'mgr') |
245 | mgr_nodes = " ".join(mgr_nodes) | |
7c673cae | 246 | new_mon = './ceph-deploy new' + " " + mon_nodes |
224ce89b | 247 | mgr_create = './ceph-deploy mgr create' + " " + mgr_nodes |
7c673cae FG |
248 | mon_hostname = mon_nodes.split(' ')[0] |
249 | mon_hostname = str(mon_hostname) | |
250 | gather_keys = './ceph-deploy gatherkeys' + " " + mon_hostname | |
251 | deploy_mds = './ceph-deploy mds create' + " " + mds_nodes | |
252 | no_of_osds = 0 | |
253 | ||
254 | if mon_nodes is None: | |
255 | raise RuntimeError("no monitor nodes in the config file") | |
256 | ||
257 | estatus_new = execute_ceph_deploy(new_mon) | |
258 | if estatus_new != 0: | |
259 | raise RuntimeError("ceph-deploy: new command failed") | |
260 | ||
261 | log.info('adding config inputs...') | |
262 | testdir = teuthology.get_testdir(ctx) | |
263 | conf_path = '{tdir}/ceph-deploy/ceph.conf'.format(tdir=testdir) | |
264 | ||
265 | if config.get('conf') is not None: | |
266 | confp = config.get('conf') | |
267 | for section, keys in confp.iteritems(): | |
268 | lines = '[{section}]\n'.format(section=section) | |
269 | teuthology.append_lines_to_file(ceph_admin, conf_path, lines, | |
270 | sudo=True) | |
271 | for key, value in keys.iteritems(): | |
272 | log.info("[%s] %s = %s" % (section, key, value)) | |
273 | lines = '{key} = {value}\n'.format(key=key, value=value) | |
274 | teuthology.append_lines_to_file( | |
275 | ceph_admin, conf_path, lines, sudo=True) | |
276 | ||
277 | # install ceph | |
278 | dev_branch = ctx.config['branch'] | |
279 | branch = '--dev={branch}'.format(branch=dev_branch) | |
280 | if ceph_branch: | |
281 | option = ceph_branch | |
282 | else: | |
283 | option = branch | |
284 | install_nodes = './ceph-deploy install ' + option + " " + all_nodes | |
285 | estatus_install = execute_ceph_deploy(install_nodes) | |
286 | if estatus_install != 0: | |
287 | raise RuntimeError("ceph-deploy: Failed to install ceph") | |
288 | # install ceph-test package too | |
289 | install_nodes2 = './ceph-deploy install --tests ' + option + \ | |
290 | " " + all_nodes | |
291 | estatus_install = execute_ceph_deploy(install_nodes2) | |
292 | if estatus_install != 0: | |
293 | raise RuntimeError("ceph-deploy: Failed to install ceph-test") | |
294 | ||
295 | mon_create_nodes = './ceph-deploy mon create-initial' | |
296 | # If the following fails, it is OK, it might just be that the monitors | |
297 | # are taking way more than a minute/monitor to form quorum, so lets | |
298 | # try the next block which will wait up to 15 minutes to gatherkeys. | |
299 | execute_ceph_deploy(mon_create_nodes) | |
224ce89b | 300 | execute_ceph_deploy(mgr_create) |
7c673cae FG |
301 | |
302 | # create-keys is explicit now | |
303 | # http://tracker.ceph.com/issues/16036 | |
304 | mons = ctx.cluster.only(teuthology.is_type('mon')) | |
305 | for remote in mons.remotes.iterkeys(): | |
306 | remote.run(args=['sudo', 'ceph-create-keys', '--cluster', 'ceph', | |
307 | '--id', remote.shortname]) | |
308 | ||
309 | estatus_gather = execute_ceph_deploy(gather_keys) | |
310 | if mds_nodes: | |
311 | estatus_mds = execute_ceph_deploy(deploy_mds) | |
312 | if estatus_mds != 0: | |
313 | raise RuntimeError("ceph-deploy: Failed to deploy mds") | |
314 | ||
315 | if config.get('test_mon_destroy') is not None: | |
316 | for d in range(1, len(mon_node)): | |
317 | mon_destroy_nodes = './ceph-deploy mon destroy' + \ | |
318 | " " + mon_node[d] | |
319 | estatus_mon_d = execute_ceph_deploy(mon_destroy_nodes) | |
320 | if estatus_mon_d != 0: | |
321 | raise RuntimeError("ceph-deploy: Failed to delete monitor") | |
322 | ||
323 | node_dev_list = get_dev_for_osd(ctx, config) | |
324 | for d in node_dev_list: | |
325 | node = d[0] | |
326 | for disk in d[1:]: | |
327 | zap = './ceph-deploy disk zap ' + node + ':' + disk | |
328 | estatus = execute_ceph_deploy(zap) | |
329 | if estatus != 0: | |
330 | raise RuntimeError("ceph-deploy: Failed to zap osds") | |
331 | osd_create_cmd = './ceph-deploy osd create ' | |
332 | if config.get('dmcrypt') is not None: | |
333 | osd_create_cmd += '--dmcrypt ' | |
334 | osd_create_cmd += ":".join(d) | |
335 | estatus_osd = execute_ceph_deploy(osd_create_cmd) | |
336 | if estatus_osd == 0: | |
337 | log.info('successfully created osd') | |
338 | no_of_osds += 1 | |
339 | else: | |
340 | raise RuntimeError("ceph-deploy: Failed to create osds") | |
341 | ||
342 | if config.get('wait-for-healthy', True) and no_of_osds >= 2: | |
343 | is_healthy(ctx=ctx, config=None) | |
344 | ||
345 | log.info('Setting up client nodes...') | |
346 | conf_path = '/etc/ceph/ceph.conf' | |
347 | admin_keyring_path = '/etc/ceph/ceph.client.admin.keyring' | |
348 | first_mon = teuthology.get_first_mon(ctx, config) | |
349 | (mon0_remote,) = ctx.cluster.only(first_mon).remotes.keys() | |
350 | conf_data = teuthology.get_file( | |
351 | remote=mon0_remote, | |
352 | path=conf_path, | |
353 | sudo=True, | |
354 | ) | |
355 | admin_keyring = teuthology.get_file( | |
356 | remote=mon0_remote, | |
357 | path=admin_keyring_path, | |
358 | sudo=True, | |
359 | ) | |
360 | ||
361 | clients = ctx.cluster.only(teuthology.is_type('client')) | |
362 | for remot, roles_for_host in clients.remotes.iteritems(): | |
363 | for id_ in teuthology.roles_of_type(roles_for_host, 'client'): | |
364 | client_keyring = \ | |
365 | '/etc/ceph/ceph.client.{id}.keyring'.format(id=id_) | |
366 | mon0_remote.run( | |
367 | args=[ | |
368 | 'cd', | |
369 | '{tdir}'.format(tdir=testdir), | |
370 | run.Raw('&&'), | |
371 | 'sudo', 'bash', '-c', | |
372 | run.Raw('"'), 'ceph', | |
373 | 'auth', | |
374 | 'get-or-create', | |
375 | 'client.{id}'.format(id=id_), | |
376 | 'mds', 'allow', | |
377 | 'mon', 'allow *', | |
378 | 'osd', 'allow *', | |
379 | run.Raw('>'), | |
380 | client_keyring, | |
381 | run.Raw('"'), | |
382 | ], | |
383 | ) | |
384 | key_data = teuthology.get_file( | |
385 | remote=mon0_remote, | |
386 | path=client_keyring, | |
387 | sudo=True, | |
388 | ) | |
389 | teuthology.sudo_write_file( | |
390 | remote=remot, | |
391 | path=client_keyring, | |
392 | data=key_data, | |
393 | perms='0644' | |
394 | ) | |
395 | teuthology.sudo_write_file( | |
396 | remote=remot, | |
397 | path=admin_keyring_path, | |
398 | data=admin_keyring, | |
399 | perms='0644' | |
400 | ) | |
401 | teuthology.sudo_write_file( | |
402 | remote=remot, | |
403 | path=conf_path, | |
404 | data=conf_data, | |
405 | perms='0644' | |
406 | ) | |
407 | ||
408 | if mds_nodes: | |
409 | log.info('Configuring CephFS...') | |
410 | ceph_fs = Filesystem(ctx, create=True) | |
411 | elif not config.get('only_mon'): | |
412 | raise RuntimeError( | |
413 | "The cluster is NOT operational due to insufficient OSDs") | |
414 | yield | |
415 | ||
416 | except Exception: | |
417 | log.info( | |
418 | "Error encountered, logging exception before tearing down ceph-deploy") | |
419 | log.info(traceback.format_exc()) | |
420 | raise | |
421 | finally: | |
422 | if config.get('keep_running'): | |
423 | return | |
424 | log.info('Stopping ceph...') | |
425 | ctx.cluster.run(args=['sudo', 'stop', 'ceph-all', run.Raw('||'), | |
426 | 'sudo', 'service', 'ceph', 'stop', run.Raw('||'), | |
427 | 'sudo', 'systemctl', 'stop', 'ceph.target']) | |
428 | ||
429 | # Are you really not running anymore? | |
430 | # try first with the init tooling | |
431 | # ignoring the status so this becomes informational only | |
432 | ctx.cluster.run( | |
433 | args=[ | |
434 | 'sudo', 'status', 'ceph-all', run.Raw('||'), | |
435 | 'sudo', 'service', 'ceph', 'status', run.Raw('||'), | |
436 | 'sudo', 'systemctl', 'status', 'ceph.target'], | |
437 | check_status=False) | |
438 | ||
439 | # and now just check for the processes themselves, as if upstart/sysvinit | |
440 | # is lying to us. Ignore errors if the grep fails | |
441 | ctx.cluster.run(args=['sudo', 'ps', 'aux', run.Raw('|'), | |
442 | 'grep', '-v', 'grep', run.Raw('|'), | |
443 | 'grep', 'ceph'], check_status=False) | |
444 | ||
445 | if ctx.archive is not None: | |
446 | # archive mon data, too | |
447 | log.info('Archiving mon data...') | |
448 | path = os.path.join(ctx.archive, 'data') | |
449 | os.makedirs(path) | |
450 | mons = ctx.cluster.only(teuthology.is_type('mon')) | |
451 | for remote, roles in mons.remotes.iteritems(): | |
452 | for role in roles: | |
453 | if role.startswith('mon.'): | |
454 | teuthology.pull_directory_tarball( | |
455 | remote, | |
456 | '/var/lib/ceph/mon', | |
457 | path + '/' + role + '.tgz') | |
458 | ||
459 | log.info('Compressing logs...') | |
460 | run.wait( | |
461 | ctx.cluster.run( | |
462 | args=[ | |
463 | 'sudo', | |
464 | 'find', | |
465 | '/var/log/ceph', | |
466 | '-name', | |
467 | '*.log', | |
468 | '-print0', | |
469 | run.Raw('|'), | |
470 | 'sudo', | |
471 | 'xargs', | |
472 | '-0', | |
473 | '--no-run-if-empty', | |
474 | '--', | |
475 | 'gzip', | |
476 | '--', | |
477 | ], | |
478 | wait=False, | |
479 | ), | |
480 | ) | |
481 | ||
482 | log.info('Archiving logs...') | |
483 | path = os.path.join(ctx.archive, 'remote') | |
484 | os.makedirs(path) | |
485 | for remote in ctx.cluster.remotes.iterkeys(): | |
486 | sub = os.path.join(path, remote.shortname) | |
487 | os.makedirs(sub) | |
488 | teuthology.pull_directory(remote, '/var/log/ceph', | |
489 | os.path.join(sub, 'log')) | |
490 | ||
491 | # Prevent these from being undefined if the try block fails | |
492 | all_nodes = get_all_nodes(ctx, config) | |
493 | purge_nodes = './ceph-deploy purge' + " " + all_nodes | |
494 | purgedata_nodes = './ceph-deploy purgedata' + " " + all_nodes | |
495 | ||
496 | log.info('Purging package...') | |
497 | execute_ceph_deploy(purge_nodes) | |
498 | log.info('Purging data...') | |
499 | execute_ceph_deploy(purgedata_nodes) | |
500 | ||
501 | ||
502 | @contextlib.contextmanager | |
503 | def cli_test(ctx, config): | |
504 | """ | |
505 | ceph-deploy cli to exercise most commonly use cli's and ensure | |
506 | all commands works and also startup the init system. | |
507 | ||
508 | """ | |
509 | log.info('Ceph-deploy Test') | |
510 | if config is None: | |
511 | config = {} | |
512 | test_branch = '' | |
513 | conf_dir = teuthology.get_testdir(ctx) + "/cdtest" | |
514 | ||
515 | def execute_cdeploy(admin, cmd, path): | |
516 | """Execute ceph-deploy commands """ | |
517 | """Either use git path or repo path """ | |
518 | args = ['cd', conf_dir, run.Raw(';')] | |
519 | if path: | |
520 | args.append('{path}/ceph-deploy/ceph-deploy'.format(path=path)); | |
521 | else: | |
522 | args.append('ceph-deploy') | |
523 | args.append(run.Raw(cmd)) | |
524 | ec = admin.run(args=args, check_status=False).exitstatus | |
525 | if ec != 0: | |
526 | raise RuntimeError( | |
527 | "failed during ceph-deploy cmd: {cmd} , ec={ec}".format(cmd=cmd, ec=ec)) | |
528 | ||
529 | if config.get('rhbuild'): | |
530 | path = None | |
531 | else: | |
532 | path = teuthology.get_testdir(ctx) | |
533 | # test on branch from config eg: wip-* , master or next etc | |
534 | # packages for all distro's should exist for wip* | |
535 | if ctx.config.get('branch'): | |
536 | branch = ctx.config.get('branch') | |
537 | test_branch = ' --dev={branch} '.format(branch=branch) | |
538 | mons = ctx.cluster.only(teuthology.is_type('mon')) | |
539 | for node, role in mons.remotes.iteritems(): | |
540 | admin = node | |
541 | admin.run(args=['mkdir', conf_dir], check_status=False) | |
542 | nodename = admin.shortname | |
543 | system_type = teuthology.get_system_type(admin) | |
544 | if config.get('rhbuild'): | |
545 | admin.run(args=['sudo', 'yum', 'install', 'ceph-deploy', '-y']) | |
546 | log.info('system type is %s', system_type) | |
547 | osds = ctx.cluster.only(teuthology.is_type('osd')) | |
548 | ||
549 | for remote, roles in osds.remotes.iteritems(): | |
550 | devs = teuthology.get_scratch_devices(remote) | |
551 | log.info("roles %s", roles) | |
552 | if (len(devs) < 3): | |
553 | log.error( | |
554 | 'Test needs minimum of 3 devices, only found %s', | |
555 | str(devs)) | |
556 | raise RuntimeError("Needs minimum of 3 devices ") | |
557 | ||
558 | conf_path = '{conf_dir}/ceph.conf'.format(conf_dir=conf_dir) | |
559 | new_cmd = 'new ' + nodename | |
560 | execute_cdeploy(admin, new_cmd, path) | |
561 | if config.get('conf') is not None: | |
562 | confp = config.get('conf') | |
563 | for section, keys in confp.iteritems(): | |
564 | lines = '[{section}]\n'.format(section=section) | |
565 | teuthology.append_lines_to_file(admin, conf_path, lines, | |
566 | sudo=True) | |
567 | for key, value in keys.iteritems(): | |
568 | log.info("[%s] %s = %s" % (section, key, value)) | |
569 | lines = '{key} = {value}\n'.format(key=key, value=value) | |
570 | teuthology.append_lines_to_file(admin, conf_path, lines, | |
571 | sudo=True) | |
572 | new_mon_install = 'install {branch} --mon '.format( | |
573 | branch=test_branch) + nodename | |
574 | new_mgr_install = 'install {branch} --mgr '.format( | |
575 | branch=test_branch) + nodename | |
576 | new_osd_install = 'install {branch} --osd '.format( | |
577 | branch=test_branch) + nodename | |
578 | new_admin = 'install {branch} --cli '.format(branch=test_branch) + nodename | |
579 | create_initial = 'mon create-initial ' | |
580 | # either use create-keys or push command | |
581 | push_keys = 'admin ' + nodename | |
582 | execute_cdeploy(admin, new_mon_install, path) | |
583 | execute_cdeploy(admin, new_mgr_install, path) | |
584 | execute_cdeploy(admin, new_osd_install, path) | |
585 | execute_cdeploy(admin, new_admin, path) | |
586 | execute_cdeploy(admin, create_initial, path) | |
587 | execute_cdeploy(admin, push_keys, path) | |
588 | ||
589 | for i in range(3): | |
590 | zap_disk = 'disk zap ' + "{n}:{d}".format(n=nodename, d=devs[i]) | |
591 | prepare = 'osd prepare ' + "{n}:{d}".format(n=nodename, d=devs[i]) | |
592 | execute_cdeploy(admin, zap_disk, path) | |
593 | execute_cdeploy(admin, prepare, path) | |
594 | ||
595 | log.info("list files for debugging purpose to check file permissions") | |
596 | admin.run(args=['ls', run.Raw('-lt'), conf_dir]) | |
597 | remote.run(args=['sudo', 'ceph', '-s'], check_status=False) | |
598 | r = remote.run(args=['sudo', 'ceph', 'health'], stdout=StringIO()) | |
599 | out = r.stdout.getvalue() | |
600 | log.info('Ceph health: %s', out.rstrip('\n')) | |
601 | log.info("Waiting for cluster to become healthy") | |
602 | with contextutil.safe_while(sleep=10, tries=6, | |
603 | action='check health') as proceed: | |
604 | while proceed(): | |
605 | r = remote.run(args=['sudo', 'ceph', 'health'], stdout=StringIO()) | |
606 | out = r.stdout.getvalue() | |
607 | if (out.split(None,1)[0] == 'HEALTH_OK'): | |
608 | break | |
609 | rgw_install = 'install {branch} --rgw {node}'.format( | |
610 | branch=test_branch, | |
611 | node=nodename, | |
612 | ) | |
613 | rgw_create = 'rgw create ' + nodename | |
614 | execute_cdeploy(admin, rgw_install, path) | |
615 | execute_cdeploy(admin, rgw_create, path) | |
616 | log.info('All ceph-deploy cli tests passed') | |
617 | try: | |
618 | yield | |
619 | finally: | |
620 | log.info("cleaning up") | |
621 | ctx.cluster.run(args=['sudo', 'stop', 'ceph-all', run.Raw('||'), | |
622 | 'sudo', 'service', 'ceph', 'stop', run.Raw('||'), | |
623 | 'sudo', 'systemctl', 'stop', 'ceph.target'], | |
624 | check_status=False) | |
625 | time.sleep(4) | |
626 | for i in range(3): | |
627 | umount_dev = "{d}1".format(d=devs[i]) | |
628 | r = remote.run(args=['sudo', 'umount', run.Raw(umount_dev)]) | |
629 | cmd = 'purge ' + nodename | |
630 | execute_cdeploy(admin, cmd, path) | |
631 | cmd = 'purgedata ' + nodename | |
632 | execute_cdeploy(admin, cmd, path) | |
633 | log.info("Removing temporary dir") | |
634 | admin.run( | |
635 | args=[ | |
636 | 'rm', | |
637 | run.Raw('-rf'), | |
638 | run.Raw(conf_dir)], | |
639 | check_status=False) | |
640 | if config.get('rhbuild'): | |
641 | admin.run(args=['sudo', 'yum', 'remove', 'ceph-deploy', '-y']) | |
642 | ||
643 | ||
644 | @contextlib.contextmanager | |
645 | def single_node_test(ctx, config): | |
646 | """ | |
647 | - ceph-deploy.single_node_test: null | |
648 | ||
649 | #rhbuild testing | |
650 | - ceph-deploy.single_node_test: | |
651 | rhbuild: 1.2.3 | |
652 | ||
653 | """ | |
654 | log.info("Testing ceph-deploy on single node") | |
655 | if config is None: | |
656 | config = {} | |
657 | overrides = ctx.config.get('overrides', {}) | |
658 | teuthology.deep_merge(config, overrides.get('ceph-deploy', {})) | |
659 | ||
660 | if config.get('rhbuild'): | |
661 | log.info("RH Build, Skip Download") | |
662 | with contextutil.nested( | |
663 | lambda: cli_test(ctx=ctx, config=config), | |
664 | ): | |
665 | yield | |
666 | else: | |
667 | with contextutil.nested( | |
668 | lambda: install_fn.ship_utilities(ctx=ctx, config=None), | |
669 | lambda: download_ceph_deploy(ctx=ctx, config=config), | |
670 | lambda: cli_test(ctx=ctx, config=config), | |
671 | ): | |
672 | yield | |
673 | ||
674 | ||
675 | @contextlib.contextmanager | |
676 | def task(ctx, config): | |
677 | """ | |
678 | Set up and tear down a Ceph cluster. | |
679 | ||
680 | For example:: | |
681 | ||
682 | tasks: | |
683 | - install: | |
684 | extras: yes | |
685 | - ssh_keys: | |
686 | - ceph-deploy: | |
687 | branch: | |
688 | stable: bobtail | |
689 | mon_initial_members: 1 | |
690 | only_mon: true | |
691 | keep_running: true | |
692 | ||
693 | tasks: | |
694 | - install: | |
695 | extras: yes | |
696 | - ssh_keys: | |
697 | - ceph-deploy: | |
698 | branch: | |
699 | dev: master | |
700 | conf: | |
701 | mon: | |
702 | debug mon = 20 | |
703 | ||
704 | tasks: | |
705 | - install: | |
706 | extras: yes | |
707 | - ssh_keys: | |
708 | - ceph-deploy: | |
709 | branch: | |
710 | testing: | |
711 | dmcrypt: yes | |
712 | separate_journal_disk: yes | |
713 | ||
714 | """ | |
715 | if config is None: | |
716 | config = {} | |
717 | ||
718 | assert isinstance(config, dict), \ | |
719 | "task ceph-deploy only supports a dictionary for configuration" | |
720 | ||
721 | overrides = ctx.config.get('overrides', {}) | |
722 | teuthology.deep_merge(config, overrides.get('ceph-deploy', {})) | |
723 | ||
724 | if config.get('branch') is not None: | |
725 | assert isinstance( | |
726 | config['branch'], dict), 'branch must be a dictionary' | |
727 | ||
728 | log.info('task ceph-deploy with config ' + str(config)) | |
729 | ||
730 | with contextutil.nested( | |
731 | lambda: install_fn.ship_utilities(ctx=ctx, config=None), | |
732 | lambda: download_ceph_deploy(ctx=ctx, config=config), | |
733 | lambda: build_ceph_cluster(ctx=ctx, config=config), | |
734 | ): | |
735 | yield |