]>
git.proxmox.com Git - ceph.git/blob - ceph/qa/tasks/ceph_deploy.py
2 Execute ceph-deploy as a task
11 from teuthology
import misc
as teuthology
12 from teuthology
import contextutil
13 from teuthology
.config
import config
as teuth_config
14 from teuthology
.task
import install
as install_fn
15 from teuthology
.orchestra
import run
16 from tasks
.cephfs
.filesystem
import Filesystem
17 from teuthology
.misc
import wait_until_healthy
19 log
= logging
.getLogger(__name__
)
22 @contextlib.contextmanager
23 def download_ceph_deploy(ctx
, config
):
25 Downloads ceph-deploy from the ceph.com git mirror and (by default)
26 switches to the master branch. If the `ceph-deploy-branch` is specified, it
27 will use that instead. The `bootstrap` script is ran, with the argument
28 obtained from `python_version`, if specified.
30 # use mon.a for ceph_admin
31 (ceph_admin
,) = ctx
.cluster
.only('mon.a').remotes
.keys()
34 py_ver
= str(config
['python_version'])
38 supported_versions
= ['2', '3']
39 if py_ver
not in supported_versions
:
40 raise ValueError("python_version must be: {}, not {}".format(
41 ' or '.join(supported_versions
), py_ver
44 log
.info("Installing Python")
45 system_type
= teuthology
.get_system_type(ceph_admin
)
47 if system_type
== 'rpm':
48 package
= 'python36' if py_ver
== '3' else 'python'
49 ctx
.cluster
.run(args
=[
50 'sudo', 'yum', '-y', 'install',
51 package
, 'python-virtualenv'
54 package
= 'python3' if py_ver
== '3' else 'python'
55 ctx
.cluster
.run(args
=[
56 'sudo', 'apt-get', '-y', '--force-yes', 'install',
57 package
, 'python-virtualenv'
60 log
.info('Downloading ceph-deploy...')
61 testdir
= teuthology
.get_testdir(ctx
)
62 ceph_deploy_branch
= config
.get('ceph-deploy-branch', 'master')
66 'git', 'clone', '-b', ceph_deploy_branch
,
67 teuth_config
.ceph_git_base_url
+ 'ceph-deploy.git',
68 '{tdir}/ceph-deploy'.format(tdir
=testdir
),
73 '{tdir}/ceph-deploy'.format(tdir
=testdir
),
78 args
.append(str(config
['python_version']))
81 ceph_admin
.run(args
=args
)
86 log
.info('Removing ceph-deploy ...')
91 '{tdir}/ceph-deploy'.format(tdir
=testdir
),
96 def is_healthy(ctx
, config
):
97 """Wait until a Ceph cluster is healthy."""
98 testdir
= teuthology
.get_testdir(ctx
)
99 ceph_admin
= teuthology
.get_first_mon(ctx
, config
)
100 (remote
,) = ctx
.cluster
.only(ceph_admin
).remotes
.keys()
101 max_tries
= 90 # 90 tries * 10 secs --> 15 minutes
105 if tries
>= max_tries
:
106 msg
= "ceph health was unable to get 'HEALTH_OK' after waiting 15 minutes"
110 '{tdir}'.format(tdir
=testdir
),
116 raise RuntimeError(msg
)
121 '{tdir}'.format(tdir
=testdir
),
126 logger
=log
.getChild('health'),
128 log
.info('Ceph health: %s', out
.rstrip('\n'))
129 if out
.split(None, 1)[0] == 'HEALTH_OK':
134 def get_nodes_using_role(ctx
, target_role
):
136 Extract the names of nodes that match a given role from a cluster, and modify the
137 cluster's service IDs to match the resulting node-based naming scheme that ceph-deploy
138 uses, such that if "mon.a" is on host "foo23", it'll be renamed to "mon.foo23".
141 # Nodes containing a service of the specified role
142 nodes_of_interest
= []
144 # Prepare a modified version of cluster.remotes with ceph-deploy-ized names
145 modified_remotes
= {}
146 ceph_deploy_mapped
= dict()
147 for _remote
, roles_for_host
in ctx
.cluster
.remotes
.items():
148 modified_remotes
[_remote
] = []
149 for svc_id
in roles_for_host
:
150 if svc_id
.startswith("{0}.".format(target_role
)):
151 fqdn
= str(_remote
).split('@')[-1]
152 nodename
= str(str(_remote
).split('.')[0]).split('@')[1]
153 if target_role
== 'mon':
154 nodes_of_interest
.append(fqdn
)
156 nodes_of_interest
.append(nodename
)
157 mapped_role
= "{0}.{1}".format(target_role
, nodename
)
158 modified_remotes
[_remote
].append(mapped_role
)
159 # keep dict of mapped role for later use by tasks
160 # eg. mon.a => mon.node1
161 ceph_deploy_mapped
[svc_id
] = mapped_role
163 modified_remotes
[_remote
].append(svc_id
)
165 ctx
.cluster
.remotes
= modified_remotes
166 # since the function is called multiple times for target roles
167 # append new mapped roles
168 if not hasattr(ctx
.cluster
, 'mapped_role'):
169 ctx
.cluster
.mapped_role
= ceph_deploy_mapped
171 ctx
.cluster
.mapped_role
.update(ceph_deploy_mapped
)
172 log
.info("New mapped_role={mr}".format(mr
=ctx
.cluster
.mapped_role
))
173 return nodes_of_interest
176 def get_dev_for_osd(ctx
, config
):
177 """Get a list of all osd device names."""
179 for remote
, roles_for_host
in ctx
.cluster
.remotes
.items():
180 host
= remote
.name
.split('@')[-1]
181 shortname
= host
.split('.')[0]
182 devs
= teuthology
.get_scratch_devices(remote
)
183 num_osd_per_host
= list(
184 teuthology
.roles_of_type(
185 roles_for_host
, 'osd'))
186 num_osds
= len(num_osd_per_host
)
187 if config
.get('separate_journal_disk') is not None:
188 num_devs_reqd
= 2 * num_osds
189 assert num_devs_reqd
<= len(
190 devs
), 'fewer data and journal disks than required ' + shortname
191 for dindex
in range(0, num_devs_reqd
, 2):
192 jd_index
= dindex
+ 1
193 dev_short
= devs
[dindex
].split('/')[-1]
194 jdev_short
= devs
[jd_index
].split('/')[-1]
195 osd_devs
.append((shortname
, dev_short
, jdev_short
))
197 assert num_osds
<= len(devs
), 'fewer disks than osds ' + shortname
198 for dev
in devs
[:num_osds
]:
199 dev_short
= dev
.split('/')[-1]
200 osd_devs
.append((shortname
, dev_short
))
204 def get_all_nodes(ctx
, config
):
205 """Return a string of node names separated by blanks"""
207 for t
, k
in ctx
.config
['targets'].items():
208 host
= t
.split('@')[-1]
209 simple_host
= host
.split('.')[0]
210 nodelist
.append(simple_host
)
211 nodelist
= " ".join(nodelist
)
214 @contextlib.contextmanager
215 def build_ceph_cluster(ctx
, config
):
216 """Build a ceph cluster"""
218 # Expect to find ceph_admin on the first mon by ID, same place that the download task
219 # puts it. Remember this here, because subsequently IDs will change from those in
220 # the test config to those that ceph-deploy invents.
222 (ceph_admin
,) = ctx
.cluster
.only('mon.a').remotes
.keys()
224 def execute_ceph_deploy(cmd
):
225 """Remotely execute a ceph_deploy command"""
226 return ceph_admin
.run(
229 '{tdir}/ceph-deploy'.format(tdir
=testdir
),
236 def ceph_disk_osd_create(ctx
, config
):
237 node_dev_list
= get_dev_for_osd(ctx
, config
)
239 for d
in node_dev_list
:
242 zap
= './ceph-deploy disk zap ' + node
+ ' ' + disk
243 estatus
= execute_ceph_deploy(zap
)
245 raise RuntimeError("ceph-deploy: Failed to zap osds")
246 osd_create_cmd
= './ceph-deploy osd create '
247 # first check for filestore, default is bluestore with ceph-deploy
248 if config
.get('filestore') is not None:
249 osd_create_cmd
+= '--filestore '
250 elif config
.get('bluestore') is not None:
251 osd_create_cmd
+= '--bluestore '
252 if config
.get('dmcrypt') is not None:
253 osd_create_cmd
+= '--dmcrypt '
254 osd_create_cmd
+= ":".join(d
)
255 estatus_osd
= execute_ceph_deploy(osd_create_cmd
)
257 log
.info('successfully created osd')
260 raise RuntimeError("ceph-deploy: Failed to create osds")
263 def ceph_volume_osd_create(ctx
, config
):
264 osds
= ctx
.cluster
.only(teuthology
.is_type('osd'))
266 for remote
in osds
.remotes
.keys():
267 # all devs should be lvm
268 osd_create_cmd
= './ceph-deploy osd create --debug ' + remote
.shortname
+ ' '
269 # default is bluestore so we just need config item for filestore
270 roles
= ctx
.cluster
.remotes
[remote
]
271 dev_needed
= len([role
for role
in roles
272 if role
.startswith('osd')])
273 all_devs
= teuthology
.get_scratch_devices(remote
)
274 log
.info("node={n}, need_devs={d}, available={a}".format(
279 devs
= all_devs
[0:dev_needed
]
280 # rest of the devices can be used for journal if required
283 device_split
= device
.split('/')
284 lv_device
= device_split
[-2] + '/' + device_split
[-1]
285 if config
.get('filestore') is not None:
286 osd_create_cmd
+= '--filestore --data ' + lv_device
+ ' '
287 # filestore with ceph-volume also needs journal disk
289 jdevice
= all_devs
.pop(jdevs
)
291 raise RuntimeError("No device available for \
292 journal configuration")
293 jdevice_split
= jdevice
.split('/')
294 j_lv
= jdevice_split
[-2] + '/' + jdevice_split
[-1]
295 osd_create_cmd
+= '--journal ' + j_lv
297 osd_create_cmd
+= ' --data ' + lv_device
298 estatus_osd
= execute_ceph_deploy(osd_create_cmd
)
300 log
.info('successfully created osd')
303 raise RuntimeError("ceph-deploy: Failed to create osds")
307 log
.info('Building ceph cluster using ceph-deploy...')
308 testdir
= teuthology
.get_testdir(ctx
)
310 if config
.get('branch') is not None:
311 cbranch
= config
.get('branch')
312 for var
, val
in cbranch
.items():
313 ceph_branch
= '--{var}={val}'.format(var
=var
, val
=val
)
314 all_nodes
= get_all_nodes(ctx
, config
)
315 mds_nodes
= get_nodes_using_role(ctx
, 'mds')
316 mds_nodes
= " ".join(mds_nodes
)
317 mon_node
= get_nodes_using_role(ctx
, 'mon')
318 mon_nodes
= " ".join(mon_node
)
319 # skip mgr based on config item
320 # this is needed when test uses latest code to install old ceph
322 skip_mgr
= config
.get('skip-mgr', False)
324 mgr_nodes
= get_nodes_using_role(ctx
, 'mgr')
325 mgr_nodes
= " ".join(mgr_nodes
)
326 new_mon
= './ceph-deploy new' + " " + mon_nodes
328 mgr_create
= './ceph-deploy mgr create' + " " + mgr_nodes
329 mon_hostname
= mon_nodes
.split(' ')[0]
330 mon_hostname
= str(mon_hostname
)
331 gather_keys
= './ceph-deploy gatherkeys' + " " + mon_hostname
332 deploy_mds
= './ceph-deploy mds create' + " " + mds_nodes
334 if mon_nodes
is None:
335 raise RuntimeError("no monitor nodes in the config file")
337 estatus_new
= execute_ceph_deploy(new_mon
)
339 raise RuntimeError("ceph-deploy: new command failed")
341 log
.info('adding config inputs...')
342 testdir
= teuthology
.get_testdir(ctx
)
343 conf_path
= '{tdir}/ceph-deploy/ceph.conf'.format(tdir
=testdir
)
345 if config
.get('conf') is not None:
346 confp
= config
.get('conf')
347 for section
, keys
in confp
.items():
348 lines
= '[{section}]\n'.format(section
=section
)
349 ceph_admin
.sudo_write_file(conf_path
, lines
, append
=True)
350 for key
, value
in keys
.items():
351 log
.info("[%s] %s = %s" % (section
, key
, value
))
352 lines
= '{key} = {value}\n'.format(key
=key
, value
=value
)
353 ceph_admin
.sudo_write_file(conf_path
, lines
, append
=True)
356 dev_branch
= ctx
.config
['branch']
357 branch
= '--dev={branch}'.format(branch
=dev_branch
)
362 install_nodes
= './ceph-deploy install ' + option
+ " " + all_nodes
363 estatus_install
= execute_ceph_deploy(install_nodes
)
364 if estatus_install
!= 0:
365 raise RuntimeError("ceph-deploy: Failed to install ceph")
366 # install ceph-test package too
367 install_nodes2
= './ceph-deploy install --tests ' + option
+ \
369 estatus_install
= execute_ceph_deploy(install_nodes2
)
370 if estatus_install
!= 0:
371 raise RuntimeError("ceph-deploy: Failed to install ceph-test")
373 mon_create_nodes
= './ceph-deploy mon create-initial'
374 # If the following fails, it is OK, it might just be that the monitors
375 # are taking way more than a minute/monitor to form quorum, so lets
376 # try the next block which will wait up to 15 minutes to gatherkeys.
377 execute_ceph_deploy(mon_create_nodes
)
379 estatus_gather
= execute_ceph_deploy(gather_keys
)
380 if estatus_gather
!= 0:
381 raise RuntimeError("ceph-deploy: Failed during gather keys")
383 # install admin key on mons (ceph-create-keys doesn't do this any more)
384 mons
= ctx
.cluster
.only(teuthology
.is_type('mon'))
385 for remote
in mons
.remotes
.keys():
386 execute_ceph_deploy('./ceph-deploy admin ' + remote
.shortname
)
389 if config
.get('use-ceph-volume', False):
390 no_of_osds
= ceph_volume_osd_create(ctx
, config
)
392 # this method will only work with ceph-deploy v1.5.39 or older
393 no_of_osds
= ceph_disk_osd_create(ctx
, config
)
396 execute_ceph_deploy(mgr_create
)
399 estatus_mds
= execute_ceph_deploy(deploy_mds
)
401 raise RuntimeError("ceph-deploy: Failed to deploy mds")
403 if config
.get('test_mon_destroy') is not None:
404 for d
in range(1, len(mon_node
)):
405 mon_destroy_nodes
= './ceph-deploy mon destroy' + \
407 estatus_mon_d
= execute_ceph_deploy(mon_destroy_nodes
)
408 if estatus_mon_d
!= 0:
409 raise RuntimeError("ceph-deploy: Failed to delete monitor")
413 if config
.get('wait-for-healthy', True) and no_of_osds
>= 2:
414 is_healthy(ctx
=ctx
, config
=None)
416 log
.info('Setting up client nodes...')
417 conf_path
= '/etc/ceph/ceph.conf'
418 admin_keyring_path
= '/etc/ceph/ceph.client.admin.keyring'
419 first_mon
= teuthology
.get_first_mon(ctx
, config
)
420 (mon0_remote
,) = ctx
.cluster
.only(first_mon
).remotes
.keys()
421 conf_data
= mon0_remote
.read_file(conf_path
, sudo
=True)
422 admin_keyring
= mon0_remote
.read_file(admin_keyring_path
, sudo
=True)
424 clients
= ctx
.cluster
.only(teuthology
.is_type('client'))
425 for remote
, roles_for_host
in clients
.remotes
.items():
426 for id_
in teuthology
.roles_of_type(roles_for_host
, 'client'):
428 '/etc/ceph/ceph.client.{id}.keyring'.format(id=id_
)
432 '{tdir}'.format(tdir
=testdir
),
434 'sudo', 'bash', '-c',
435 run
.Raw('"'), 'ceph',
438 'client.{id}'.format(id=id_
),
447 key_data
= mon0_remote
.read_file(
451 remote
.sudo_write_file(
456 remote
.sudo_write_file(
457 path
=admin_keyring_path
,
461 remote
.sudo_write_file(
468 log
.info('Configuring CephFS...')
469 Filesystem(ctx
, create
=True)
470 elif not config
.get('only_mon'):
472 "The cluster is NOT operational due to insufficient OSDs")
476 'sudo', 'ceph', '--cluster', 'ceph',
477 'osd', 'pool', 'create', 'rbd', '128', '128'],
481 'sudo', 'ceph', '--cluster', 'ceph',
482 'osd', 'pool', 'application', 'enable',
483 'rbd', 'rbd', '--yes-i-really-mean-it'
490 "Error encountered, logging exception before tearing down ceph-deploy")
491 log
.info(traceback
.format_exc())
494 if config
.get('keep_running'):
496 log
.info('Stopping ceph...')
497 ctx
.cluster
.run(args
=['sudo', 'systemctl', 'stop', 'ceph.target'],
501 # and now just check for the processes themselves, as if upstart/sysvinit
502 # is lying to us. Ignore errors if the grep fails
503 ctx
.cluster
.run(args
=['sudo', 'ps', 'aux', run
.Raw('|'),
504 'grep', '-v', 'grep', run
.Raw('|'),
505 'grep', 'ceph'], check_status
=False)
506 ctx
.cluster
.run(args
=['sudo', 'systemctl', run
.Raw('|'),
507 'grep', 'ceph'], check_status
=False)
509 if ctx
.archive
is not None:
510 # archive mon data, too
511 log
.info('Archiving mon data...')
512 path
= os
.path
.join(ctx
.archive
, 'data')
514 mons
= ctx
.cluster
.only(teuthology
.is_type('mon'))
515 for remote
, roles
in mons
.remotes
.items():
517 if role
.startswith('mon.'):
518 teuthology
.pull_directory_tarball(
521 path
+ '/' + role
+ '.tgz')
523 log
.info('Compressing logs...')
546 log
.info('Archiving logs...')
547 path
= os
.path
.join(ctx
.archive
, 'remote')
549 for remote
in ctx
.cluster
.remotes
.keys():
550 sub
= os
.path
.join(path
, remote
.shortname
)
552 teuthology
.pull_directory(remote
, '/var/log/ceph',
553 os
.path
.join(sub
, 'log'))
555 # Prevent these from being undefined if the try block fails
556 all_nodes
= get_all_nodes(ctx
, config
)
557 purge_nodes
= './ceph-deploy purge' + " " + all_nodes
558 purgedata_nodes
= './ceph-deploy purgedata' + " " + all_nodes
560 log
.info('Purging package...')
561 execute_ceph_deploy(purge_nodes
)
562 log
.info('Purging data...')
563 execute_ceph_deploy(purgedata_nodes
)
566 @contextlib.contextmanager
567 def cli_test(ctx
, config
):
569 ceph-deploy cli to exercise most commonly use cli's and ensure
570 all commands works and also startup the init system.
573 log
.info('Ceph-deploy Test')
577 conf_dir
= teuthology
.get_testdir(ctx
) + "/cdtest"
579 def execute_cdeploy(admin
, cmd
, path
):
580 """Execute ceph-deploy commands """
581 """Either use git path or repo path """
582 args
= ['cd', conf_dir
, run
.Raw(';')]
584 args
.append('{path}/ceph-deploy/ceph-deploy'.format(path
=path
))
586 args
.append('ceph-deploy')
587 args
.append(run
.Raw(cmd
))
588 ec
= admin
.run(args
=args
, check_status
=False).exitstatus
591 "failed during ceph-deploy cmd: {cmd} , ec={ec}".format(cmd
=cmd
, ec
=ec
))
593 if config
.get('rhbuild'):
596 path
= teuthology
.get_testdir(ctx
)
597 # test on branch from config eg: wip-* , master or next etc
598 # packages for all distro's should exist for wip*
599 if ctx
.config
.get('branch'):
600 branch
= ctx
.config
.get('branch')
601 test_branch
= ' --dev={branch} '.format(branch
=branch
)
602 mons
= ctx
.cluster
.only(teuthology
.is_type('mon'))
603 for node
, role
in mons
.remotes
.items():
605 admin
.run(args
=['mkdir', conf_dir
], check_status
=False)
606 nodename
= admin
.shortname
607 system_type
= teuthology
.get_system_type(admin
)
608 if config
.get('rhbuild'):
609 admin
.run(args
=['sudo', 'yum', 'install', 'ceph-deploy', '-y'])
610 log
.info('system type is %s', system_type
)
611 osds
= ctx
.cluster
.only(teuthology
.is_type('osd'))
613 for remote
, roles
in osds
.remotes
.items():
614 devs
= teuthology
.get_scratch_devices(remote
)
615 log
.info("roles %s", roles
)
618 'Test needs minimum of 3 devices, only found %s',
620 raise RuntimeError("Needs minimum of 3 devices ")
622 conf_path
= '{conf_dir}/ceph.conf'.format(conf_dir
=conf_dir
)
623 new_cmd
= 'new ' + nodename
624 execute_cdeploy(admin
, new_cmd
, path
)
625 if config
.get('conf') is not None:
626 confp
= config
.get('conf')
627 for section
, keys
in confp
.items():
628 lines
= '[{section}]\n'.format(section
=section
)
629 admin
.sudo_write_file(conf_path
, lines
, append
=True)
630 for key
, value
in keys
.items():
631 log
.info("[%s] %s = %s" % (section
, key
, value
))
632 lines
= '{key} = {value}\n'.format(key
=key
, value
=value
)
633 admin
.sudo_write_file(conf_path
, lines
, append
=True)
634 new_mon_install
= 'install {branch} --mon '.format(
635 branch
=test_branch
) + nodename
636 new_mgr_install
= 'install {branch} --mgr '.format(
637 branch
=test_branch
) + nodename
638 new_osd_install
= 'install {branch} --osd '.format(
639 branch
=test_branch
) + nodename
640 new_admin
= 'install {branch} --cli '.format(branch
=test_branch
) + nodename
641 create_initial
= 'mon create-initial '
642 mgr_create
= 'mgr create ' + nodename
643 # either use create-keys or push command
644 push_keys
= 'admin ' + nodename
645 execute_cdeploy(admin
, new_mon_install
, path
)
646 execute_cdeploy(admin
, new_mgr_install
, path
)
647 execute_cdeploy(admin
, new_osd_install
, path
)
648 execute_cdeploy(admin
, new_admin
, path
)
649 execute_cdeploy(admin
, create_initial
, path
)
650 execute_cdeploy(admin
, mgr_create
, path
)
651 execute_cdeploy(admin
, push_keys
, path
)
654 zap_disk
= 'disk zap ' + "{n}:{d}".format(n
=nodename
, d
=devs
[i
])
655 prepare
= 'osd prepare ' + "{n}:{d}".format(n
=nodename
, d
=devs
[i
])
656 execute_cdeploy(admin
, zap_disk
, path
)
657 execute_cdeploy(admin
, prepare
, path
)
659 log
.info("list files for debugging purpose to check file permissions")
660 admin
.run(args
=['ls', run
.Raw('-lt'), conf_dir
])
661 remote
.run(args
=['sudo', 'ceph', '-s'], check_status
=False)
662 out
= remote
.sh('sudo ceph health')
663 log
.info('Ceph health: %s', out
.rstrip('\n'))
664 log
.info("Waiting for cluster to become healthy")
665 with contextutil
.safe_while(sleep
=10, tries
=6,
666 action
='check health') as proceed
:
668 out
= remote
.sh('sudo ceph health')
669 if (out
.split(None, 1)[0] == 'HEALTH_OK'):
671 rgw_install
= 'install {branch} --rgw {node}'.format(
675 rgw_create
= 'rgw create ' + nodename
676 execute_cdeploy(admin
, rgw_install
, path
)
677 execute_cdeploy(admin
, rgw_create
, path
)
678 log
.info('All ceph-deploy cli tests passed')
682 log
.info("cleaning up")
683 ctx
.cluster
.run(args
=['sudo', 'systemctl', 'stop', 'ceph.target'],
687 umount_dev
= "{d}1".format(d
=devs
[i
])
688 remote
.run(args
=['sudo', 'umount', run
.Raw(umount_dev
)])
689 cmd
= 'purge ' + nodename
690 execute_cdeploy(admin
, cmd
, path
)
691 cmd
= 'purgedata ' + nodename
692 execute_cdeploy(admin
, cmd
, path
)
693 log
.info("Removing temporary dir")
700 if config
.get('rhbuild'):
701 admin
.run(args
=['sudo', 'yum', 'remove', 'ceph-deploy', '-y'])
704 @contextlib.contextmanager
705 def single_node_test(ctx
, config
):
707 - ceph-deploy.single_node_test: null
710 - ceph-deploy.single_node_test:
714 log
.info("Testing ceph-deploy on single node")
717 overrides
= ctx
.config
.get('overrides', {})
718 teuthology
.deep_merge(config
, overrides
.get('ceph-deploy', {}))
720 if config
.get('rhbuild'):
721 log
.info("RH Build, Skip Download")
722 with contextutil
.nested(
723 lambda: cli_test(ctx
=ctx
, config
=config
),
727 with contextutil
.nested(
728 lambda: install_fn
.ship_utilities(ctx
=ctx
, config
=None),
729 lambda: download_ceph_deploy(ctx
=ctx
, config
=config
),
730 lambda: cli_test(ctx
=ctx
, config
=config
),
735 @contextlib.contextmanager
736 def upgrade(ctx
, config
):
738 Upgrade using ceph-deploy
741 # to upgrade to specific branch, use
744 # to setup mgr node, use
746 # to wait for cluster to be healthy after all upgrade, use
747 wait-for-healthy: True
748 role: (upgrades the below roles serially)
753 roles
= config
.get('roles')
754 # get the roles that are mapped as per ceph-deploy
755 # roles are mapped for mon/mds eg: mon.a => mon.host_short_name
756 mapped_role
= ctx
.cluster
.mapped_role
757 log
.info("roles={r}, mapped_roles={mr}".format(r
=roles
, mr
=mapped_role
))
758 if config
.get('branch'):
759 branch
= config
.get('branch')
760 (var
, val
) = branch
.items()[0]
761 ceph_branch
= '--{var}={val}'.format(var
=var
, val
=val
)
763 # default to wip-branch under test
764 dev_branch
= ctx
.config
['branch']
765 ceph_branch
= '--dev={branch}'.format(branch
=dev_branch
)
766 # get the node used for initial deployment which is mon.a
767 mon_a
= mapped_role
.get('mon.a')
768 (ceph_admin
,) = ctx
.cluster
.only(mon_a
).remotes
.keys()
769 testdir
= teuthology
.get_testdir(ctx
)
770 cmd
= './ceph-deploy install ' + ceph_branch
772 # check if this role is mapped (mon or mds)
773 if mapped_role
.get(role
):
774 role
= mapped_role
.get(role
)
775 remotes_and_roles
= ctx
.cluster
.only(role
).remotes
776 for remote
, roles
in remotes_and_roles
.items():
777 nodename
= remote
.shortname
778 cmd
= cmd
+ ' ' + nodename
779 log
.info("Upgrading ceph on %s", nodename
)
783 '{tdir}/ceph-deploy'.format(tdir
=testdir
),
788 # restart all ceph services, ideally upgrade should but it does not
791 'sudo', 'systemctl', 'restart', 'ceph.target'
794 ceph_admin
.run(args
=['sudo', 'ceph', '-s'])
796 # workaround for http://tracker.ceph.com/issues/20950
797 # write the correct mgr key to disk
798 if config
.get('setup-mgr-node', None):
799 mons
= ctx
.cluster
.only(teuthology
.is_type('mon'))
800 for remote
, roles
in mons
.remotes
.items():
803 run
.Raw('sudo ceph auth get client.bootstrap-mgr'),
806 run
.Raw('/var/lib/ceph/bootstrap-mgr/ceph.keyring')
810 if config
.get('setup-mgr-node', None):
811 mgr_nodes
= get_nodes_using_role(ctx
, 'mgr')
812 mgr_nodes
= " ".join(mgr_nodes
)
813 mgr_install
= './ceph-deploy install --mgr ' + ceph_branch
+ " " + mgr_nodes
814 mgr_create
= './ceph-deploy mgr create' + " " + mgr_nodes
819 '{tdir}/ceph-deploy'.format(tdir
=testdir
),
821 run
.Raw(mgr_install
),
828 '{tdir}/ceph-deploy'.format(tdir
=testdir
),
833 ceph_admin
.run(args
=['sudo', 'ceph', '-s'])
834 if config
.get('wait-for-healthy', None):
835 wait_until_healthy(ctx
, ceph_admin
, use_sudo
=True)
839 @contextlib.contextmanager
840 def task(ctx
, config
):
842 Set up and tear down a Ceph cluster.
853 mon_initial_members: 1
854 ceph-deploy-branch: my-ceph-deploy-branch
857 # either choose bluestore or filestore, default is bluestore
861 # skip install of mgr for old release using below flag
862 skip-mgr: True ( default is False )
863 # to use ceph-volume instead of ceph-disk
864 # ceph-disk can only be used with old ceph-deploy release from pypi
865 use-ceph-volume: true
886 separate_journal_disk: yes
892 assert isinstance(config
, dict), \
893 "task ceph-deploy only supports a dictionary for configuration"
895 overrides
= ctx
.config
.get('overrides', {})
896 teuthology
.deep_merge(config
, overrides
.get('ceph-deploy', {}))
898 if config
.get('branch') is not None:
900 config
['branch'], dict), 'branch must be a dictionary'
902 log
.info('task ceph-deploy with config ' + str(config
))
904 # we need to use 1.5.39-stable for testing jewel or master branch with
906 if config
.get('use-ceph-volume', False) is False:
907 # check we are not testing specific branch
908 if config
.get('ceph-deploy-branch', False) is False:
909 config
['ceph-deploy-branch'] = '1.5.39-stable'
911 with contextutil
.nested(
912 lambda: install_fn
.ship_utilities(ctx
=ctx
, config
=None),
913 lambda: download_ceph_deploy(ctx
=ctx
, config
=config
),
914 lambda: build_ceph_cluster(ctx
=ctx
, config
=config
),