3 # Copyright (C) 2015, 2016, 2017 Red Hat <contact@redhat.com>
4 # Copyright (C) 2014 Inktank <info@inktank.com>
5 # Copyright (C) 2014 Cloudwatt <libre.licensing@cloudwatt.com>
6 # Copyright (C) 2014 Catalyst.net Ltd
8 # Author: Loic Dachary <loic@dachary.org>
10 # This program is free software; you can redistribute it and/or modify
11 # it under the terms of the GNU Library Public License as published by
12 # the Free Software Foundation; either version 2, or (at your option)
15 # This program is distributed in the hope that it will be useful,
16 # but WITHOUT ANY WARRANTY; without even the implied warranty of
17 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18 # GNU Library Public License for more details.
21 from __future__
import print_function
45 CEPH_OSD_ONDISK_MAGIC
= 'ceph osd volume v026'
46 CEPH_LOCKBOX_ONDISK_MAGIC
= 'ceph lockbox volume v001'
48 KEY_MANAGEMENT_MODE_V1
= 'ceph-mon v1'
53 # identical because creating a journal is atomic
54 'ready': '45b0969e-9b03-4f30-b4c6-b4b80ceff106',
55 'tobe': '45b0969e-9b03-4f30-b4c6-b4b80ceff106',
58 # identical because creating a block is atomic
59 'ready': 'cafecafe-9b03-4f30-b4c6-b4b80ceff106',
60 'tobe': 'cafecafe-9b03-4f30-b4c6-b4b80ceff106',
63 # identical because creating a block is atomic
64 'ready': '30cd0809-c2b2-499c-8879-2d6b78529876',
65 'tobe': '30cd0809-c2b2-499c-8879-2d6b785292be',
68 # identical because creating a block is atomic
69 'ready': '5ce17fce-4087-4169-b7ff-056cc58473f9',
70 'tobe': '5ce17fce-4087-4169-b7ff-056cc58472be',
73 'ready': '4fbd7e29-9d25-41b8-afd0-062c0ceff05d',
74 'tobe': '89c57f98-2fe5-4dc0-89c1-f3ad0ceff2be',
77 'ready': 'fb3aabf9-d25f-47cc-bf5e-721d1816496b',
78 'tobe': 'fb3aabf9-d25f-47cc-bf5e-721d181642be',
83 'ready': '45b0969e-9b03-4f30-b4c6-35865ceff106',
84 'tobe': '89c57f98-2fe5-4dc0-89c1-35865ceff2be',
87 'ready': 'cafecafe-9b03-4f30-b4c6-35865ceff106',
88 'tobe': '89c57f98-2fe5-4dc0-89c1-35865ceff2be',
91 'ready': '166418da-c469-4022-adf4-b30afd37f176',
92 'tobe': '7521c784-4626-4260-bc8d-ba77a0f5f2be',
95 'ready': '86a32090-3647-40b9-bbbd-38d8c573aa86',
96 'tobe': '92dad30f-175b-4d40-a5b0-5c0a258b42be',
99 'ready': '4fbd7e29-9d25-41b8-afd0-35865ceff05d',
100 'tobe': '89c57f98-2fe5-4dc0-89c1-5ec00ceff2be',
105 'ready': '45b0969e-9b03-4f30-b4c6-5ec00ceff106',
106 'tobe': '89c57f98-2fe5-4dc0-89c1-35865ceff2be',
109 'ready': 'cafecafe-9b03-4f30-b4c6-5ec00ceff106',
110 'tobe': '89c57f98-2fe5-4dc0-89c1-35865ceff2be',
113 'ready': '93b0052d-02d9-4d8a-a43b-33a3ee4dfbc3',
114 'tobe': '69d17c68-3e58-4399-aff0-b68265f2e2be',
117 'ready': '306e8683-4fe2-4330-b7c0-00a917c16966',
118 'tobe': 'f2d89683-a621-4063-964a-eb1f7863a2be',
121 'ready': '4fbd7e29-9d25-41b8-afd0-5ec00ceff05d',
122 'tobe': '89c57f98-2fe5-4dc0-89c1-5ec00ceff2be',
127 'ready': '45b0969e-8ae0-4982-bf9d-5a8d867af560',
128 'tobe': '45b0969e-8ae0-4982-bf9d-5a8d867af560',
131 'ready': 'cafecafe-8ae0-4982-bf9d-5a8d867af560',
132 'tobe': 'cafecafe-8ae0-4982-bf9d-5a8d867af560',
135 'ready': 'ec6d6385-e346-45dc-be91-da2a7c8b3261',
136 'tobe': 'ec6d6385-e346-45dc-be91-da2a7c8b32be',
139 'ready': '01b41e1b-002a-453c-9f17-88793989ff8f',
140 'tobe': '01b41e1b-002a-453c-9f17-88793989f2be',
143 'ready': '4fbd7e29-8ae0-4982-bf9d-5a8d867af560',
144 'tobe': '89c57f98-8ae0-4982-bf9d-5a8d867af560',
147 'ready': '7f4a666a-16f3-47a2-8445-152ef4d03f6c',
148 'tobe': '7f4a666a-16f3-47a2-8445-152ef4d032be',
157 def get_ready_by_type(what
):
158 return [x
['ready'] for x
in PTYPE
[what
].values()]
161 def get_ready_by_name(name
):
162 return [x
[name
]['ready'] for x
in PTYPE
.values() if name
in x
]
165 def is_regular_space(ptype
):
166 return Ptype
.is_what_space('regular', ptype
)
169 def is_mpath_space(ptype
):
170 return Ptype
.is_what_space('mpath', ptype
)
173 def is_plain_space(ptype
):
174 return Ptype
.is_what_space('plain', ptype
)
177 def is_luks_space(ptype
):
178 return Ptype
.is_what_space('luks', ptype
)
181 def is_what_space(what
, ptype
):
182 for name
in Space
.NAMES
:
183 if ptype
== PTYPE
[what
][name
]['ready']:
188 def space_ptype_to_name(ptype
):
189 for what
in PTYPE
.values():
190 for name
in Space
.NAMES
:
191 if ptype
== what
[name
]['ready']:
193 raise ValueError('ptype ' + ptype
+ ' not found')
196 def is_dmcrypt_space(ptype
):
197 for name
in Space
.NAMES
:
198 if Ptype
.is_dmcrypt(ptype
, name
):
203 def is_dmcrypt(ptype
, name
):
204 for what
in ('plain', 'luks'):
205 if ptype
== PTYPE
[what
][name
]['ready']:
212 if platform
.system() == 'FreeBSD':
214 DEFAULT_FS_TYPE
= 'zfs'
215 PROCDIR
= '/compat/linux/proc'
216 # FreeBSD does not have blockdevices any more
221 DEFAULT_FS_TYPE
= 'xfs'
223 BLOCKDIR
= '/sys/block'
227 OSD STATUS Definition
229 OSD_STATUS_OUT_DOWN
= 0
230 OSD_STATUS_OUT_UP
= 1
231 OSD_STATUS_IN_DOWN
= 2
234 MOUNT_OPTIONS
= dict(
235 btrfs
='noatime,user_subvol_rm_allowed',
236 # user_xattr is default ever since linux 2.6.39 / 3.0, but we'll
237 # delay a moment before removing it fully because we did have some
238 # issues with ext4 before the xatts-in-leveldb work, and it seemed
239 # that user_xattr helped
240 ext4
='noatime,user_xattr',
241 xfs
='noatime,inode64',
246 # btrfs requires -f, for the same reason as xfs (see comment below)
253 # xfs insists on not overwriting previous fs; even if we wipe
254 # partition table, we often recreate it exactly the same way,
255 # so we'll see ghosts of filesystems past
274 STATEDIR
= '/var/lib/ceph'
276 SYSCONFDIR
= '/etc/ceph'
280 SUPPRESS_PREFIX
= None
282 # only warn once about some things
285 # Nuke the TERM variable to avoid confusing any subprocesses we call.
286 # For example, libreadline will print weird control sequences for some
288 if 'TERM' in os
.environ
:
289 del os
.environ
['TERM']
292 if LOG_NAME
== '__main__':
293 LOG_NAME
= os
.path
.basename(sys
.argv
[0])
294 LOG
= logging
.getLogger(LOG_NAME
)
296 # Allow user-preferred values for subprocess user and group
297 CEPH_PREF_USER
= None
298 CEPH_PREF_GROUP
= None
301 class FileLock(object):
302 def __init__(self
, fn
):
308 self
.fd
= os
.open(self
.fn
, os
.O_WRONLY | os
.O_CREAT
)
309 fcntl
.lockf(self
.fd
, fcntl
.LOCK_EX
)
311 def __exit__(self
, exc_type
, exc_val
, exc_tb
):
313 fcntl
.lockf(self
.fd
, fcntl
.LOCK_UN
)
318 class Error(Exception):
324 doc
= _bytes2str(self
.__doc
__.strip())
326 str_type
= basestring
329 args
= [a
if isinstance(a
, str_type
) else str(a
) for a
in self
.args
]
330 return ': '.join([doc
] + [_bytes2str(a
) for a
in args
])
333 class MountError(Error
):
335 Mounting filesystem failed
339 class UnmountError(Error
):
341 Unmounting filesystem failed
345 class BadMagicError(Error
):
347 Does not look like a Ceph OSD, or incompatible version
351 class TruncatedLineError(Error
):
357 class TooManyLinesError(Error
):
363 class FilesystemTypeError(Error
):
365 Cannot discover filesystem type
369 class CephDiskException(Exception):
371 A base exception for ceph-disk to provide custom (ad-hoc) messages that
372 will be caught and dealt with when main() is executed
377 class ExecutableNotFound(CephDiskException
):
379 Exception to report on executables not available in PATH
386 Detect whether systemd is running
388 with
open(PROCDIR
+ '/1/comm', 'r') as f
:
389 return 'systemd' in f
.read()
394 Detect whether upstart is running
396 (out
, err
, _
) = command(['init', '--version'])
397 return 'upstart' in out
400 def maybe_mkdir(*a
, **kw
):
402 Creates a new directory if it doesn't exist, removes
403 existing symlink before creating the directory.
405 # remove any symlink, if it is there..
406 if os
.path
.exists(*a
) and stat
.S_ISLNK(os
.lstat(*a
).st_mode
):
407 LOG
.debug('Removing old symlink at %s', *a
)
412 if e
.errno
== errno
.EEXIST
:
418 def which(executable
):
419 """find the location of an executable"""
420 envpath
= os
.environ
.get('PATH') or os
.defpath
421 PATH
= envpath
.split(os
.pathsep
)
432 for location
in locations
:
433 executable_path
= os
.path
.join(location
, executable
)
434 if (os
.path
.isfile(executable_path
) and
435 os
.access(executable_path
, os
.X_OK
)):
436 return executable_path
439 def _get_command_executable(arguments
):
441 Return the full path for an executable, raise if the executable is not
442 found. If the executable has already a full path do not perform any checks.
444 if os
.path
.isabs(arguments
[0]): # an absolute path
446 executable
= which(arguments
[0])
448 command_msg
= 'Could not run command: %s' % ' '.join(arguments
)
449 executable_msg
= '%s not in path.' % arguments
[0]
450 raise ExecutableNotFound('%s %s' % (executable_msg
, command_msg
))
452 # swap the old executable for the new one
453 arguments
[0] = executable
457 def command(arguments
, **kwargs
):
459 Safely execute a ``subprocess.Popen`` call making sure that the
460 executable exists and raising a helpful error message
463 .. note:: This should be the preferred way of calling ``subprocess.Popen``
464 since it provides the caller with the safety net of making sure that
465 executables *will* be found and will error nicely otherwise.
467 This returns the output of the command and the return code of the
468 process in a tuple: (stdout, stderr, returncode).
471 arguments
= list(map(_bytes2str
, _get_command_executable(arguments
)))
473 LOG
.info('Running command: %s' % ' '.join(arguments
))
474 process
= subprocess
.Popen(
476 stdout
=subprocess
.PIPE
,
477 stderr
=subprocess
.PIPE
,
479 out
, err
= process
.communicate()
481 return _bytes2str(out
), _bytes2str(err
), process
.returncode
484 def command_with_stdin(arguments
, stdin
):
485 LOG
.info("Running command with stdin: " + " ".join(arguments
))
486 process
= subprocess
.Popen(
488 stdin
=subprocess
.PIPE
,
489 stdout
=subprocess
.PIPE
,
490 stderr
=subprocess
.PIPE
)
491 out
, err
= process
.communicate(stdin
)
493 if process
.returncode
!= 0:
496 "'{cmd}' failed with status code {returncode}".format(
498 returncode
=process
.returncode
,
504 def _bytes2str(string
):
505 return string
.decode('utf-8') if isinstance(string
, bytes
) else string
508 def command_init(arguments
, **kwargs
):
510 Safely execute a non-blocking ``subprocess.Popen`` call
511 making sure that the executable exists and raising a helpful
512 error message if it does not.
514 .. note:: This should be the preferred way of calling ``subprocess.Popen``
515 since it provides the caller with the safety net of making sure that
516 executables *will* be found and will error nicely otherwise.
518 This returns the process.
521 arguments
= list(map(_bytes2str
, _get_command_executable(arguments
)))
523 LOG
.info('Running command: %s' % ' '.join(arguments
))
524 process
= subprocess
.Popen(
526 stdout
=subprocess
.PIPE
,
527 stderr
=subprocess
.PIPE
,
532 def command_wait(process
):
534 Wait for the process finish and parse its output.
537 out
, err
= process
.communicate()
539 return _bytes2str(out
), _bytes2str(err
), process
.returncode
542 def command_check_call(arguments
, exit
=False):
544 Safely execute a ``subprocess.check_call`` call making sure that the
545 executable exists and raising a helpful error message if it does not.
547 When ``exit`` is set to ``True`` this helper will do a clean (sans
548 traceback) system exit.
549 .. note:: This should be the preferred way of calling
550 ``subprocess.check_call`` since it provides the caller with the safety net
551 of making sure that executables *will* be found and will error nicely
554 arguments
= _get_command_executable(arguments
)
555 command
= ' '.join(arguments
)
556 LOG
.info('Running command: %s', command
)
558 return subprocess
.check_call(arguments
)
559 except subprocess
.CalledProcessError
as error
:
562 LOG
.error(error
.output
)
564 "'{cmd}' failed with status code {returncode}".format(
566 returncode
=error
.returncode
,
573 # An alternative block_path implementation would be
575 # name = basename(dev)
576 # return /sys/devices/virtual/block/$name
578 # It is however more fragile because it relies on the fact
579 # that the basename of the device the user will use always
580 # matches the one the driver will use. On Ubuntu 14.04, for
581 # instance, when multipath creates a partition table on
583 # /dev/mapper/353333330000007d0 -> ../dm-0
585 # it will create partition devices named
587 # /dev/mapper/353333330000007d0-part1
589 # which is the same device as /dev/dm-1 but not a symbolic
592 # ubuntu@other:~$ ls -l /dev/mapper /dev/dm-1
593 # brw-rw---- 1 root disk 252, 1 Aug 15 17:52 /dev/dm-1
594 # lrwxrwxrwx 1 root root 7 Aug 15 17:52 353333330000007d0 -> ../dm-0
595 # brw-rw---- 1 root disk 252, 1 Aug 15 17:52 353333330000007d0-part1
597 # Using the basename in this case fails.
604 path
= os
.path
.realpath(dev
)
605 rdev
= os
.stat(path
).st_rdev
606 (M
, m
) = (os
.major(rdev
), os
.minor(rdev
))
607 return "{sysfs}/dev/block/{M}:{m}".format(sysfs
=SYSFS
, M
=M
, m
=m
)
610 def get_dm_uuid(dev
):
611 uuid_path
= os
.path
.join(block_path(dev
), 'dm', 'uuid')
612 LOG
.debug("get_dm_uuid " + dev
+ " uuid path is " + uuid_path
)
613 if not os
.path
.exists(uuid_path
):
615 uuid
= open(uuid_path
, 'r').read()
616 LOG
.debug("get_dm_uuid " + dev
+ " uuid is " + uuid
)
622 True if the path is managed by multipath
626 uuid
= get_dm_uuid(dev
)
628 (re
.match('part\d+-mpath-', uuid
) or
629 re
.match('mpath-', uuid
)))
632 def get_dev_name(path
):
634 get device name from path. e.g.::
636 /dev/sda -> sda, /dev/cciss/c0d1 -> cciss!c0d1
638 a device "name" is something like::
644 assert path
.startswith('/dev/')
646 return base
.replace('/', '!')
649 def get_dev_path(name
):
651 get a path (/dev/...) from a name (cciss!c0d1)
652 a device "path" is something like::
658 return '/dev/' + name
.replace('!', '/')
661 def get_dev_relpath(name
):
663 get a relative path to /dev from a name (cciss!c0d1)
665 return name
.replace('!', '/')
668 def get_dev_size(dev
, size
='megabytes'):
670 Attempt to get the size of a device so that we can prevent errors
671 from actions to devices that are smaller, and improve error reporting.
673 Because we want to avoid breakage in case this approach is not robust, we
674 will issue a warning if we failed to get the size.
676 :param size: bytes or megabytes
677 :param dev: the device to calculate the size
679 fd
= os
.open(dev
, os
.O_RDONLY
)
680 dividers
= {'bytes': 1, 'megabytes': 1024 * 1024}
682 device_size
= os
.lseek(fd
, 0, os
.SEEK_END
)
683 divider
= dividers
.get(size
, 1024 * 1024) # default to megabytes
684 return device_size
// divider
685 except Exception as error
:
686 LOG
.warning('failed to get size of %s: %s' % (dev
, str(error
)))
691 def stmode_is_diskdevice(dmode
):
692 if stat
.S_ISBLK(dmode
):
695 # FreeBSD does not have block devices
696 # All disks are character devices
697 return FREEBSD
and stat
.S_ISCHR(dmode
)
700 def dev_is_diskdevice(dev
):
701 dmode
= os
.stat(dev
).st_mode
702 return stmode_is_diskdevice(dmode
)
705 def ldev_is_diskdevice(dev
):
706 dmode
= os
.lstat(dev
).st_mode
707 return stmode_is_diskdevice(dmode
)
710 def path_is_diskdevice(path
):
711 dev
= os
.path
.realpath(path
)
712 return dev_is_diskdevice(dev
)
715 def get_partition_mpath(dev
, pnum
):
716 part_re
= "part{pnum}-mpath-".format(pnum
=pnum
)
717 partitions
= list_partitions_mpath(dev
, part_re
)
724 def get_partition_dev(dev
, pnum
):
726 get the device name for a partition
728 assume that partitions are named like the base dev,
729 with a number, and optionally
730 some intervening characters (like 'p'). e.g.,
733 cciss/c0d1 1 -> cciss!c0d1p1
736 for retry
in range(0, max_retry
+ 1):
740 partname
= get_partition_mpath(dev
, pnum
)
742 name
= get_dev_name(os
.path
.realpath(dev
))
743 sys_entry
= os
.path
.join(BLOCKDIR
, name
)
744 error_msg
= " in %s" % sys_entry
745 for f
in os
.listdir(sys_entry
):
746 if f
.startswith(name
) and f
.endswith(str(pnum
)):
747 # we want the shortest name that starts with the base name
748 # and ends with the partition number
749 if not partname
or len(f
) < len(partname
):
753 LOG
.info('Found partition %d for %s after %d tries' %
755 return get_dev_path(partname
)
757 if retry
< max_retry
:
758 LOG
.info('Try %d/%d : partition %d for %s does not exist%s' %
759 (retry
+ 1, max_retry
, pnum
, dev
, error_msg
))
763 raise Error('partition %d for %s does not appear to exist%s' %
764 (pnum
, dev
, error_msg
))
767 def list_all_partitions():
769 Return a list of devices and partitions
772 names
= os
.listdir(BLOCKDIR
)
775 # /dev/fd0 may hang http://tracker.ceph.com/issues/6827
776 if re
.match(r
'^fd\d$', name
):
778 dev_part_list
[name
] = list_partitions(get_dev_path(name
))
780 with
open(os
.path
.join(PROCDIR
, "partitions")) as partitions
:
781 for line
in partitions
:
782 columns
= line
.split()
783 if len(columns
) >= 4:
785 dev_part_list
[name
] = list_partitions(get_dev_path(name
))
789 def list_partitions(dev
):
790 dev
= os
.path
.realpath(dev
)
792 return list_partitions_mpath(dev
)
794 return list_partitions_device(dev
)
797 def list_partitions_mpath(dev
, part_re
="part\d+-mpath-"):
800 holders
= os
.path
.join(p
, 'holders')
801 for holder
in os
.listdir(holders
):
802 uuid_path
= os
.path
.join(holders
, holder
, 'dm', 'uuid')
803 uuid
= open(uuid_path
, 'r').read()
804 LOG
.debug("list_partitions_mpath: " + uuid_path
+ " uuid = " + uuid
)
805 if re
.match(part_re
, uuid
):
806 partitions
.append(holder
)
810 def list_partitions_device(dev
):
812 Return a list of partitions on the given device name
815 basename
= get_dev_name(dev
)
816 for name
in os
.listdir(block_path(dev
)):
817 if name
.startswith(basename
):
818 partitions
.append(name
)
822 def get_partition_base(dev
):
824 Get the base device for a partition
826 dev
= os
.path
.realpath(dev
)
827 if not ldev_is_diskdevice(dev
):
828 raise Error('not a block device', dev
)
830 name
= get_dev_name(dev
)
831 if os
.path
.exists(os
.path
.join('/sys/block', name
)):
832 raise Error('not a partition', dev
)
835 for basename
in os
.listdir('/sys/block'):
836 if os
.path
.exists(os
.path
.join('/sys/block', basename
, name
)):
837 return get_dev_path(basename
)
838 raise Error('no parent device for partition', dev
)
841 def is_partition_mpath(dev
):
842 uuid
= get_dm_uuid(dev
)
843 return bool(re
.match('part\d+-mpath-', uuid
))
846 def partnum_mpath(dev
):
847 uuid
= get_dm_uuid(dev
)
848 return re
.findall('part(\d+)-mpath-', uuid
)[0]
851 def get_partition_base_mpath(dev
):
852 slave_path
= os
.path
.join(block_path(dev
), 'slaves')
853 slaves
= os
.listdir(slave_path
)
855 name_path
= os
.path
.join(slave_path
, slaves
[0], 'dm', 'name')
856 name
= open(name_path
, 'r').read().strip()
857 return os
.path
.join('/dev/mapper', name
)
860 def is_partition(dev
):
862 Check whether a given device path is a partition or a full disk.
865 return is_partition_mpath(dev
)
867 dev
= os
.path
.realpath(dev
)
869 if not stmode_is_diskdevice(st
.st_mode
):
870 raise Error('not a block device', dev
)
872 name
= get_dev_name(dev
)
873 if os
.path
.exists(os
.path
.join(BLOCKDIR
, name
)):
876 # make sure it is a partition of something else
877 major
= os
.major(st
.st_rdev
)
878 minor
= os
.minor(st
.st_rdev
)
879 if os
.path
.exists('/sys/dev/block/%d:%d/partition' % (major
, minor
)):
882 raise Error('not a disk or partition', dev
)
887 Check if the given device is mounted.
889 dev
= os
.path
.realpath(dev
)
890 with
open(PROCDIR
+ '/mounts', 'rb') as proc_mounts
:
891 for line
in proc_mounts
:
892 fields
= line
.split()
895 mounts_dev
= fields
[0]
897 if os
.path
.isabs(mounts_dev
) and os
.path
.exists(mounts_dev
):
898 mounts_dev
= os
.path
.realpath(mounts_dev
)
899 if mounts_dev
== dev
:
900 return _bytes2str(path
)
906 Check if a device is held by another device (e.g., a dm-crypt mapping)
908 assert os
.path
.exists(dev
)
912 dev
= os
.path
.realpath(dev
)
913 base
= get_dev_name(dev
)
916 directory
= '/sys/block/{base}/holders'.format(base
=base
)
917 if os
.path
.exists(directory
):
918 return os
.listdir(directory
)
923 directory
= '/sys/block/{base}/{part}/holders'.format(
924 part
=part
, base
=base
)
925 if os
.path
.exists(directory
):
926 return os
.listdir(directory
)
931 def verify_not_in_use(dev
, check_partitions
=False):
933 Verify if a given device (path) is in use (e.g. mounted or
934 in use by device-mapper).
936 :raises: Error if device is in use.
938 assert os
.path
.exists(dev
)
940 raise Error('Device is mounted', dev
)
941 holders
= is_held(dev
)
943 raise Error('Device %s is in use by a device-mapper '
944 'mapping (dm-crypt?)' % dev
, ','.join(holders
))
946 if check_partitions
and not is_partition(dev
):
947 for partname
in list_partitions(dev
):
948 partition
= get_dev_path(partname
)
949 if is_mounted(partition
):
950 raise Error('Device is mounted', partition
)
951 holders
= is_held(partition
)
953 raise Error('Device %s is in use by a device-mapper '
954 'mapping (dm-crypt?)'
955 % partition
, ','.join(holders
))
958 def must_be_one_line(line
):
960 Checks if given line is really one single line.
962 :raises: TruncatedLineError or TooManyLinesError
963 :return: Content of the line, or None if line isn't valid.
965 line
= _bytes2str(line
)
967 if line
[-1:] != '\n':
968 raise TruncatedLineError(line
)
971 raise TooManyLinesError(line
)
975 def read_one_line(parent
, name
):
977 Read a file whose sole contents are a single line.
981 :return: Contents of the line, or None if file did not exist.
983 path
= os
.path
.join(parent
, name
)
985 line
= open(path
, 'rb').read()
987 if e
.errno
== errno
.ENOENT
:
993 line
= must_be_one_line(line
)
994 except (TruncatedLineError
, TooManyLinesError
) as e
:
996 'File is corrupt: {path}: {msg}'.format(
1004 def write_one_line(parent
, name
, text
):
1006 Write a file whose sole contents are a single line.
1010 path
= os
.path
.join(parent
, name
)
1011 tmp
= '{path}.{pid}.tmp'.format(path
=path
, pid
=os
.getpid())
1012 with
open(tmp
, 'wb') as tmp_file
:
1013 tmp_file
.write(text
.encode('utf-8') + b
'\n')
1014 os
.fsync(tmp_file
.fileno())
1015 path_set_context(tmp
)
1016 os
.rename(tmp
, path
)
1021 Get a init system using 'ceph-detect-init'
1023 init
= _check_output(
1026 '--default', 'sysvinit',
1029 init
= must_be_one_line(init
)
1033 def check_osd_magic(path
):
1035 Check that this path has the Ceph OSD magic.
1037 :raises: BadMagicError if this does not look like a Ceph OSD data
1040 magic
= read_one_line(path
, 'magic')
1042 # probably not mkfs'ed yet
1043 raise BadMagicError(path
)
1044 if magic
!= CEPH_OSD_ONDISK_MAGIC
:
1045 raise BadMagicError(path
)
1048 def check_osd_id(osd_id
):
1050 Ensures osd id is numeric.
1052 if not re
.match(r
'^[0-9]+$', osd_id
):
1053 raise Error('osd id is not numeric', osd_id
)
1056 def allocate_osd_id(
1063 Allocates an OSD id on the given cluster.
1065 :raises: Error if the call to allocate the OSD id fails.
1066 :return: The allocated OSD id.
1068 lockbox_path
= os
.path
.join(STATEDIR
, 'osd-lockbox', fsid
)
1069 lockbox_osd_id
= read_one_line(lockbox_path
, 'whoami')
1070 osd_keyring
= os
.path
.join(path
, 'keyring')
1072 LOG
.debug('Getting OSD id from Lockbox...')
1073 osd_id
= lockbox_osd_id
1074 shutil
.move(os
.path
.join(lockbox_path
, 'osd_keyring'),
1076 path_set_context(osd_keyring
)
1077 os
.unlink(os
.path
.join(lockbox_path
, 'whoami'))
1080 LOG
.debug('Allocating OSD id...')
1083 wanttobe
= read_one_line(path
, 'wanttobe')
1084 if os
.path
.exists(os
.path
.join(path
, 'wanttobe')):
1085 os
.unlink(os
.path
.join(path
, 'wanttobe'))
1086 id_arg
= wanttobe
and [wanttobe
] or []
1087 osd_id
= command_with_stdin(
1090 '--cluster', cluster
,
1091 '--name', 'client.bootstrap-osd',
1092 '--keyring', keyring
,
1099 except subprocess
.CalledProcessError
as e
:
1100 raise Error('ceph osd create failed', e
, e
.output
)
1101 osd_id
= must_be_one_line(osd_id
)
1102 check_osd_id(osd_id
)
1103 secrets
.write_osd_keyring(osd_keyring
, osd_id
)
1107 def get_osd_id(path
):
1109 Gets the OSD id of the OSD at the given path.
1111 osd_id
= read_one_line(path
, 'whoami')
1112 if osd_id
is not None:
1113 check_osd_id(osd_id
)
1117 def get_ceph_user():
1118 global CEPH_PREF_USER
1120 if CEPH_PREF_USER
is not None:
1122 pwd
.getpwnam(CEPH_PREF_USER
)
1123 return CEPH_PREF_USER
1125 print("No such user:", CEPH_PREF_USER
)
1129 pwd
.getpwnam('ceph')
1135 def get_ceph_group():
1136 global CEPH_PREF_GROUP
1138 if CEPH_PREF_GROUP
is not None:
1140 grp
.getgrnam(CEPH_PREF_GROUP
)
1141 return CEPH_PREF_GROUP
1143 print("No such group:", CEPH_PREF_GROUP
)
1147 grp
.getgrnam('ceph')
1153 def path_set_context(path
):
1154 # restore selinux context to default policy values
1155 if which('restorecon'):
1156 command(['restorecon', '-R', path
])
1158 # if ceph user exists, set owner to ceph
1159 if get_ceph_user() == 'ceph':
1160 command(['chown', '-R', 'ceph:ceph', path
])
1163 def _check_output(args
=None, **kwargs
):
1164 out
, err
, ret
= command(args
, **kwargs
)
1167 error
= subprocess
.CalledProcessError(ret
, cmd
)
1168 error
.output
= out
+ err
1170 return _bytes2str(out
)
1173 def get_conf(cluster
, variable
):
1175 Get the value of the given configuration variable from the
1178 :raises: Error if call to ceph-conf fails.
1179 :return: The variable value or None.
1182 out
, err
, ret
= command(
1185 '--cluster={cluster}'.format(
1194 except OSError as e
:
1195 raise Error('error executing ceph-conf', e
, err
)
1197 # config entry not found
1200 raise Error('getting variable from configuration failed')
1201 value
= out
.split('\n', 1)[0]
1202 # don't differentiate between "var=" and no var set
1208 def get_conf_with_default(cluster
, variable
):
1210 Get a config value that is known to the C++ code.
1212 This will fail if called on variables that are not defined in
1213 common config options.
1216 out
= _check_output(
1219 '--cluster={cluster}'.format(
1222 '--show-config-value={variable}'.format(
1228 except subprocess
.CalledProcessError
as e
:
1230 'getting variable from configuration failed',
1234 value
= str(out
).split('\n', 1)[0]
1238 def get_fsid(cluster
):
1240 Get the fsid of the cluster.
1242 :return: The fsid or raises Error.
1244 fsid
= get_conf_with_default(cluster
=cluster
, variable
='fsid')
1246 raise Error('getting cluster uuid from configuration failed')
1250 def get_dmcrypt_key_path(
1256 Get path to dmcrypt key file.
1258 :return: Path to the dmcrypt key file, callers should check for existence.
1261 path
= os
.path
.join(key_dir
, _uuid
+ ".luks.key")
1263 path
= os
.path
.join(key_dir
, _uuid
)
1268 def get_dmcrypt_key(
1273 legacy_path
= get_dmcrypt_key_path(_uuid
, key_dir
, luks
)
1274 if os
.path
.exists(legacy_path
):
1275 return (legacy_path
,)
1276 path
= os
.path
.join(STATEDIR
, 'osd-lockbox', _uuid
)
1277 if os
.path
.exists(path
):
1278 mode
= get_oneliner(path
, 'key-management-mode')
1279 osd_uuid
= get_oneliner(path
, 'osd-uuid')
1280 ceph_fsid
= read_one_line(path
, 'ceph_fsid')
1281 if ceph_fsid
is None:
1282 LOG
.warning("no `ceph_fsid` found falling back to 'ceph' "
1286 cluster
= find_cluster_by_uuid(ceph_fsid
)
1288 raise Error('No cluster conf found in ' + SYSCONFDIR
+
1289 ' with fsid %s' % ceph_fsid
)
1291 if mode
== KEY_MANAGEMENT_MODE_V1
:
1292 key
, stderr
, ret
= command(
1295 '--cluster', cluster
,
1297 'client.osd-lockbox.' + osd_uuid
,
1299 os
.path
.join(path
, 'keyring'),
1302 'dm-crypt/osd/' + osd_uuid
+ '/luks',
1305 LOG
.debug("stderr " + stderr
)
1307 return base64
.b64decode(key
)
1309 raise Error('unknown key-management-mode ' + str(mode
))
1310 raise Error('unable to read dm-crypt key', path
, legacy_path
)
1317 cryptsetup_parameters
,
1321 dev
= dmcrypt_is_mapped(_uuid
)
1325 if isinstance(key
, tuple):
1326 # legacy, before lockbox
1327 assert os
.path
.exists(key
[0])
1332 dev
= '/dev/mapper/' + _uuid
1340 ] + cryptsetup_parameters
1358 ] + cryptsetup_parameters
1363 command_with_stdin(luksFormat_args
, key
)
1364 command_with_stdin(luksOpen_args
, key
)
1366 # Plain mode has no format function, nor any validation
1367 # that the key is correct.
1368 command_with_stdin(create_args
, key
)
1369 # set proper ownership of mapped device
1370 command_check_call(['chown', 'ceph:ceph', dev
])
1373 except subprocess
.CalledProcessError
as e
:
1374 raise Error('unable to map device', rawdev
, e
)
1380 if not os
.path
.exists('/dev/mapper/' + _uuid
):
1385 command_check_call(['cryptsetup', 'remove', _uuid
])
1387 except subprocess
.CalledProcessError
as e
:
1389 raise Error('unable to unmap device', _uuid
, e
)
1391 time
.sleep(0.5 + retries
* 1.0)
1401 Mounts a device with given filessystem type and
1402 mount options to a tempfile path under /var/lib/ceph/tmp.
1404 # sanity check: none of the arguments are None
1406 raise ValueError('dev may not be None')
1408 raise ValueError('fstype may not be None')
1410 # pick best-of-breed mount options based on fs type
1412 options
= MOUNT_OPTIONS
.get(fstype
, '')
1414 myTemp
= STATEDIR
+ '/tmp'
1415 # mkdtemp expect 'dir' to be existing on the system
1416 # Let's be sure it's always the case
1417 if not os
.path
.exists(myTemp
):
1421 path
= tempfile
.mkdtemp(
1426 LOG
.debug('Mounting %s on %s with options %s', dev
, path
, options
)
1437 if which('restorecon'):
1444 except subprocess
.CalledProcessError
as e
:
1447 except (OSError, IOError):
1459 Unmount and removes the given mount point.
1464 LOG
.debug('Unmounting %s', path
)
1473 except subprocess
.CalledProcessError
as e
:
1474 # on failure, retry 3 times with incremental backoff
1476 raise UnmountError(e
)
1478 time
.sleep(0.5 + retries
* 1.0)
1485 ###########################################
1487 def extract_parted_partition_numbers(partitions
):
1488 numbers_as_strings
= re
.findall('^\d+', partitions
, re
.MULTILINE
)
1489 return map(int, numbers_as_strings
)
1492 def get_free_partition_index(dev
):
1494 Get the next free partition index on a given device.
1496 :return: Index number (> 1 if there is already a partition on the device)
1497 or 1 if there is no partition table.
1500 lines
= _check_output(
1509 except subprocess
.CalledProcessError
as e
:
1510 LOG
.info('cannot read partition index; assume it '
1511 'isn\'t present\n (Error: %s)' % e
)
1515 raise Error('parted failed to output anything')
1516 LOG
.debug('get_free_partition_index: analyzing ' + lines
)
1517 if ('CHS;' not in lines
and
1518 'CYL;' not in lines
and
1519 'BYT;' not in lines
):
1520 raise Error('parted output expected to contain one of ' +
1521 'CHH; CYL; or BYT; : ' + lines
)
1522 if os
.path
.realpath(dev
) not in lines
:
1523 raise Error('parted output expected to contain ' + dev
+ ': ' + lines
)
1524 _
, partitions
= lines
.split(os
.path
.realpath(dev
))
1525 partition_numbers
= extract_parted_partition_numbers(partitions
)
1526 if partition_numbers
:
1527 return max(partition_numbers
) + 1
1532 def check_journal_reqs(args
):
1533 _
, _
, allows_journal
= command([
1534 'ceph-osd', '--check-allows-journal',
1536 '--log-file', '$run_dir/$cluster-osd-check.log',
1537 '--cluster', args
.cluster
,
1538 '--setuser', get_ceph_user(),
1539 '--setgroup', get_ceph_group(),
1541 _
, _
, wants_journal
= command([
1542 'ceph-osd', '--check-wants-journal',
1544 '--log-file', '$run_dir/$cluster-osd-check.log',
1545 '--cluster', args
.cluster
,
1546 '--setuser', get_ceph_user(),
1547 '--setgroup', get_ceph_group(),
1549 _
, _
, needs_journal
= command([
1550 'ceph-osd', '--check-needs-journal',
1552 '--log-file', '$run_dir/$cluster-osd-check.log',
1553 '--cluster', args
.cluster
,
1554 '--setuser', get_ceph_user(),
1555 '--setgroup', get_ceph_group(),
1557 return (not allows_journal
, not wants_journal
, not needs_journal
)
1560 def update_partition(dev
, description
):
1562 Must be called after modifying a partition table so the kernel
1563 know about the change and fire udev events accordingly. A side
1564 effect of partprobe is to remove partitions and add them again.
1565 The first udevadm settle waits for ongoing udev events to
1566 complete, just in case one of them rely on an existing partition
1567 on dev. The second udevadm settle guarantees to the caller that
1568 all udev events related to the partition table change have been
1569 processed, i.e. the 95-ceph-osd.rules actions and mode changes,
1570 group changes etc. are complete.
1572 LOG
.debug('Calling partprobe on %s device %s', description
, dev
)
1573 partprobe_ok
= False
1574 error
= 'unknown error'
1575 partprobe
= _get_command_executable(['partprobe'])[0]
1577 command_check_call(['udevadm', 'settle', '--timeout=600'])
1579 _check_output(['flock', '-s', dev
, partprobe
, dev
])
1582 except subprocess
.CalledProcessError
as e
:
1584 if ('unable to inform the kernel' not in error
and
1585 'Device or resource busy' not in error
):
1587 LOG
.debug('partprobe %s failed : %s (ignored, waiting 60s)'
1590 if not partprobe_ok
:
1591 raise Error('partprobe %s failed : %s' % (dev
, error
))
1592 command_check_call(['udevadm', 'settle', '--timeout=600'])
1597 # Thoroughly wipe all partitions of any traces of
1598 # Filesystems or OSD Journals
1600 # In addition we need to write 10M of data to each partition
1601 # to make sure that after re-creating the same partition
1602 # there is no trace left of any previous Filesystem or OSD
1605 LOG
.debug('Writing zeros to existing partitions on %s', dev
)
1607 for partname
in list_partitions(dev
):
1608 partition
= get_dev_path(partname
)
1621 'of={path}'.format(path
=partition
),
1627 LOG
.debug('Zapping partition table on %s', dev
)
1629 # try to wipe out any GPT partition table backups. sgdisk
1630 # isn't too thorough.
1632 size
= 33 * lba_size
1633 with
open(dev
, 'wb') as dev_file
:
1634 dev_file
.seek(-size
, os
.SEEK_END
)
1635 dev_file
.write(size
* b
'\0')
1654 update_partition(dev
, 'zapped')
1656 except subprocess
.CalledProcessError
as e
:
1660 def zap_freebsd(dev
):
1662 # For FreeBSD we just need to zap the partition.
1672 except subprocess
.CalledProcessError
as e
:
1678 Destroy the partition table and content of a given disk.
1680 dev
= os
.path
.realpath(dev
)
1681 dmode
= os
.stat(dev
).st_mode
1682 if not stat
.S_ISBLK(dmode
) or is_partition(dev
):
1683 raise Error('not full block device; cannot zap', dev
)
1690 def adjust_symlink(target
, path
):
1692 if os
.path
.lexists(path
):
1694 mode
= os
.lstat(path
).st_mode
1695 if stat
.S_ISREG(mode
):
1696 LOG
.debug('Removing old file %s', path
)
1698 elif stat
.S_ISLNK(mode
):
1699 old
= os
.readlink(path
)
1701 LOG
.debug('Removing old symlink %s -> %s', path
, old
)
1706 raise Error('unable to remove (or adjust) old file (symlink)',
1709 LOG
.debug('Creating symlink %s -> %s', path
, target
)
1711 os
.symlink(target
, path
)
1713 raise Error('unable to create symlink %s -> %s' % (path
, target
))
1716 def get_mount_options(cluster
, fs_type
):
1717 mount_options
= get_conf(
1719 variable
='osd_mount_options_{fstype}'.format(
1723 if mount_options
is None:
1724 mount_options
= get_conf(
1726 variable
='osd_fs_mount_options_{fstype}'.format(
1731 # remove whitespaces
1732 mount_options
= "".join(mount_options
.split())
1733 return mount_options
1736 class Device(object):
1738 def __init__(self
, path
, args
):
1741 self
.dev_size
= None
1742 self
.partitions
= {}
1743 self
.ptype_map
= None
1744 assert not is_partition(self
.path
)
1746 def create_partition(self
, uuid
, name
, size
=0, num
=0):
1747 ptype
= self
.ptype_tobe_for_name(name
)
1749 num
= get_free_partition_index(dev
=self
.path
)
1751 new
= '--new={num}:0:+{size}M'.format(num
=num
, size
=size
)
1752 if size
> self
.get_dev_size():
1753 LOG
.error('refusing to create %s on %s' % (name
, self
.path
))
1754 LOG
.error('%s size (%sM) is bigger than device (%sM)'
1755 % (name
, size
, self
.get_dev_size()))
1756 raise Error('%s device size (%sM) is not big enough for %s'
1757 % (self
.path
, self
.get_dev_size(), name
))
1759 new
= '--largest-new={num}'.format(num
=num
)
1761 LOG
.debug('Creating %s partition num %d size %d on %s',
1762 name
, num
, size
, self
.path
)
1767 '--change-name={num}:ceph {name}'.format(num
=num
, name
=name
),
1768 '--partition-guid={num}:{uuid}'.format(num
=num
, uuid
=uuid
),
1769 '--typecode={num}:{uuid}'.format(num
=num
, uuid
=ptype
),
1776 update_partition(self
.path
, 'created')
1779 def ptype_tobe_for_name(self
, name
):
1780 LOG
.debug("name = " + name
)
1783 if name
== 'lockbox':
1784 if is_mpath(self
.path
):
1785 return PTYPE
['mpath']['lockbox']['tobe']
1787 return PTYPE
['regular']['lockbox']['tobe']
1788 if self
.ptype_map
is None:
1789 partition
= DevicePartition
.factory(
1790 path
=self
.path
, dev
=None, args
=self
.args
)
1791 self
.ptype_map
= partition
.ptype_map
1792 return self
.ptype_map
[name
]['tobe']
1794 def get_partition(self
, num
):
1795 if num
not in self
.partitions
:
1796 dev
= get_partition_dev(self
.path
, num
)
1797 partition
= DevicePartition
.factory(
1798 path
=self
.path
, dev
=dev
, args
=self
.args
)
1799 partition
.set_partition_number(num
)
1800 self
.partitions
[num
] = partition
1801 return self
.partitions
[num
]
1803 def get_dev_size(self
):
1804 if self
.dev_size
is None:
1805 self
.dev_size
= get_dev_size(self
.path
)
1806 return self
.dev_size
1809 def factory(path
, args
):
1810 return Device(path
, args
)
1813 class DevicePartition(object):
1815 def __init__(self
, args
):
1821 self
.ptype_map
= None
1823 self
.set_variables_ptype()
1826 if self
.uuid
is None:
1827 self
.uuid
= get_partition_uuid(self
.rawdev
)
1830 def get_ptype(self
):
1831 if self
.ptype
is None:
1832 self
.ptype
= get_partition_type(self
.rawdev
)
1835 def set_partition_number(self
, num
):
1838 def get_partition_number(self
):
1841 def set_dev(self
, dev
):
1848 def get_rawdev(self
):
1851 def set_variables_ptype(self
):
1852 self
.ptype_map
= PTYPE
['regular']
1854 def ptype_for_name(self
, name
):
1855 return self
.ptype_map
[name
]['ready']
1858 def factory(path
, dev
, args
):
1859 dmcrypt_type
= CryptHelpers
.get_dmcrypt_type(args
)
1860 if ((path
is not None and is_mpath(path
)) or
1861 (dev
is not None and is_mpath(dev
))):
1862 partition
= DevicePartitionMultipath(args
)
1863 elif dmcrypt_type
== 'luks':
1864 partition
= DevicePartitionCryptLuks(args
)
1865 elif dmcrypt_type
== 'plain':
1866 partition
= DevicePartitionCryptPlain(args
)
1868 partition
= DevicePartition(args
)
1869 partition
.set_dev(dev
)
1873 class DevicePartitionMultipath(DevicePartition
):
1875 def set_variables_ptype(self
):
1876 self
.ptype_map
= PTYPE
['mpath']
1879 class DevicePartitionCrypt(DevicePartition
):
1881 def __init__(self
, args
):
1882 super(DevicePartitionCrypt
, self
).__init
__(args
)
1883 self
.osd_dm_key
= None
1884 self
.cryptsetup_parameters
= CryptHelpers
.get_cryptsetup_parameters(
1886 self
.dmcrypt_type
= CryptHelpers
.get_dmcrypt_type(self
.args
)
1887 self
.dmcrypt_keysize
= CryptHelpers
.get_dmcrypt_keysize(self
.args
)
1889 def setup_crypt(self
):
1894 self
.dev
= _dmcrypt_map(
1896 key
=self
.osd_dm_key
,
1897 _uuid
=self
.get_uuid(),
1898 cryptsetup_parameters
=self
.cryptsetup_parameters
,
1905 dmcrypt_unmap(self
.get_uuid())
1906 self
.dev
= self
.rawdev
1913 class DevicePartitionCryptPlain(DevicePartitionCrypt
):
1918 def setup_crypt(self
):
1919 if self
.osd_dm_key
is not None:
1922 self
.cryptsetup_parameters
+= ['--key-size', str(self
.dmcrypt_keysize
)]
1924 self
.osd_dm_key
= get_dmcrypt_key(
1925 self
.get_uuid(), self
.args
.dmcrypt_key_dir
,
1928 def set_variables_ptype(self
):
1929 self
.ptype_map
= PTYPE
['plain']
1932 class DevicePartitionCryptLuks(DevicePartitionCrypt
):
1937 def setup_crypt(self
):
1938 if self
.osd_dm_key
is not None:
1941 if self
.dmcrypt_keysize
== 1024:
1942 # We don't force this into the cryptsetup_parameters,
1943 # as we want the cryptsetup defaults
1944 # to prevail for the actual LUKS key lengths.
1947 self
.cryptsetup_parameters
+= ['--key-size',
1948 str(self
.dmcrypt_keysize
)]
1950 self
.osd_dm_key
= get_dmcrypt_key(
1951 self
.get_uuid(), self
.args
.dmcrypt_key_dir
,
1954 def set_variables_ptype(self
):
1955 self
.ptype_map
= PTYPE
['luks']
1958 class Prepare(object):
1960 def __init__(self
, args
):
1965 parser
= argparse
.ArgumentParser(add_help
=False)
1966 parser
.add_argument(
1970 help='cluster name to assign this disk to',
1972 parser
.add_argument(
1975 help='cluster uuid to assign this disk to',
1977 parser
.add_argument(
1980 help='unique OSD uuid to assign this disk to',
1982 parser
.add_argument(
1985 help='unique OSD id to assign this disk to',
1987 parser
.add_argument(
1988 '--crush-device-class',
1989 help='crush device class to assign this disk to',
1991 parser
.add_argument(
1993 action
='store_true', default
=None,
1994 help='encrypt DATA and/or JOURNAL devices with dm-crypt',
1996 parser
.add_argument(
1997 '--dmcrypt-key-dir',
1999 default
='/etc/ceph/dmcrypt-keys',
2000 help='directory where dm-crypt keys are stored',
2002 parser
.add_argument(
2005 help='bootstrap-osd keyring path template (%(default)s)',
2006 default
='{statedir}/bootstrap-osd/{cluster}.keyring',
2007 dest
='prepare_key_template',
2009 parser
.add_argument(
2011 action
='store_true', default
=None,
2012 help='let many prepare\'s run in parallel',
2017 def set_subparser(subparsers
):
2020 PrepareData
.parser(),
2023 parents
.extend(PrepareFilestore
.parent_parsers())
2024 parents
.extend(PrepareBluestore
.parent_parsers())
2025 parser
= subparsers
.add_parser(
2028 formatter_class
=argparse
.RawDescriptionHelpFormatter
,
2029 description
=textwrap
.fill(textwrap
.dedent("""\
2030 If the --bluestore argument is given, a bluestore objectstore
2031 will be created. If --filestore is provided, a legacy FileStore
2032 objectstore will be created. If neither is specified, we default
2035 When an entire device is prepared for bluestore, two
2036 partitions are created. The first partition is for metadata,
2037 the second partition is for blocks that contain data.
2039 Unless explicitly specified with --block.db or
2040 --block.wal, the bluestore DB and WAL data is stored on
2041 the main block device. For instance:
2043 ceph-disk prepare --bluestore /dev/sdc
2047 /dev/sdc1 for osd metadata
2048 /dev/sdc2 for block, db, and wal data (the rest of the disk)
2051 If either --block.db or --block.wal are specified to be
2052 the same whole device, they will be created as partition
2053 three and four respectively. For instance:
2055 ceph-disk prepare --bluestore \\
2056 --block.db /dev/sdc \\
2057 --block.wal /dev/sdc \\
2062 /dev/sdc1 for osd metadata
2063 /dev/sdc2 for block (the rest of the disk)
2068 help='Prepare a directory or disk for a Ceph OSD',
2070 parser
.set_defaults(
2076 if self
.args
.no_locking
:
2085 return PrepareBluestore(args
)
2087 return PrepareFilestore(args
)
2091 Prepare
.factory(args
).prepare()
2094 class PrepareFilestore(Prepare
):
2096 def __init__(self
, args
):
2097 super(PrepareFilestore
, self
).__init
__(args
)
2099 self
.lockbox
= Lockbox(args
)
2100 self
.data
= PrepareFilestoreData(args
)
2101 self
.journal
= PrepareJournal(args
)
2104 def parent_parsers():
2106 PrepareJournal
.parser(),
2110 if self
.data
.args
.dmcrypt
:
2111 self
.lockbox
.prepare()
2112 self
.data
.prepare(self
.journal
)
2115 class PrepareBluestore(Prepare
):
2117 def __init__(self
, args
):
2118 super(PrepareBluestore
, self
).__init
__(args
)
2120 self
.lockbox
= Lockbox(args
)
2121 self
.data
= PrepareBluestoreData(args
)
2122 self
.block
= PrepareBluestoreBlock(args
)
2123 self
.blockdb
= PrepareBluestoreBlockDB(args
)
2124 self
.blockwal
= PrepareBluestoreBlockWAL(args
)
2128 parser
= argparse
.ArgumentParser(add_help
=False)
2129 parser
.add_argument(
2132 action
='store_true', default
=True,
2133 help='bluestore objectstore',
2135 parser
.add_argument(
2138 action
='store_false',
2139 help='filestore objectstore',
2144 def parent_parsers():
2146 PrepareBluestore
.parser(),
2147 PrepareBluestoreBlock
.parser(),
2148 PrepareBluestoreBlockDB
.parser(),
2149 PrepareBluestoreBlockWAL
.parser(),
2153 if self
.data
.args
.dmcrypt
:
2154 self
.lockbox
.prepare()
2155 to_prepare_list
= []
2156 if getattr(self
.data
.args
, 'block.db'):
2157 to_prepare_list
.append(self
.blockdb
)
2158 if getattr(self
.data
.args
, 'block.wal'):
2159 to_prepare_list
.append(self
.blockwal
)
2160 to_prepare_list
.append(self
.block
)
2161 self
.data
.prepare(*to_prepare_list
)
2164 class Space(object):
2166 NAMES
= ('block', 'journal', 'block.db', 'block.wal')
2169 class PrepareSpace(object):
2175 def __init__(self
, args
):
2178 self
.space_size
= self
.get_space_size()
2179 if getattr(self
.args
, self
.name
+ '_uuid') is None:
2180 setattr(self
.args
, self
.name
+ '_uuid', str(uuid
.uuid4()))
2181 self
.space_symlink
= None
2182 self
.space_dmcrypt
= None
2187 if (self
.wants_space() and
2188 dev_is_diskdevice(args
.data
) and
2189 not is_partition(args
.data
) and
2190 getattr(args
, name
) is None and
2191 getattr(args
, name
+ '_file') is None):
2192 LOG
.info('Will colocate %s with data on %s',
2194 setattr(args
, name
, args
.data
)
2196 if getattr(args
, name
) is None:
2197 if getattr(args
, name
+ '_dev'):
2198 raise Error('%s is unspecified; not a block device' %
2199 name
.capitalize(), getattr(args
, name
))
2200 self
.type = self
.NONE
2203 if not os
.path
.exists(getattr(args
, name
)):
2204 if getattr(args
, name
+ '_dev'):
2205 raise Error('%s does not exist; not a block device' %
2206 name
.capitalize(), getattr(args
, name
))
2207 self
.type = self
.FILE
2210 mode
= os
.stat(getattr(args
, name
)).st_mode
2211 if stmode_is_diskdevice(mode
):
2212 if getattr(args
, name
+ '_file'):
2213 raise Error('%s is not a regular file' % name
.capitalize
,
2214 getattr(args
, name
))
2215 self
.type = self
.DEVICE
2218 if stat
.S_ISREG(mode
):
2219 if getattr(args
, name
+ '_dev'):
2220 raise Error('%s is not a block device' % name
.capitalize
,
2221 getattr(args
, name
))
2222 self
.type = self
.FILE
2225 raise Error('%s %s is neither a block device nor regular file' %
2226 (name
.capitalize
, getattr(args
, name
)))
2229 return self
.type == self
.NONE
2232 return self
.type == self
.FILE
2234 def is_device(self
):
2235 return self
.type == self
.DEVICE
2238 def parser(name
, positional
=True):
2239 parser
= argparse
.ArgumentParser(add_help
=False)
2240 parser
.add_argument(
2243 help='unique uuid to assign to the %s' % name
,
2245 parser
.add_argument(
2247 action
='store_true', default
=None,
2248 help='verify that %s is a file' % name
.upper(),
2250 parser
.add_argument(
2252 action
='store_true', default
=None,
2253 help='verify that %s is a block device' % name
.upper(),
2257 parser
.add_argument(
2259 metavar
=name
.upper(),
2261 help=('path to OSD %s disk block device;' % name
+
2262 ' leave out to store %s in file' % name
),
2266 def wants_space(self
):
2269 def populate_data_path(self
, path
):
2270 if self
.type == self
.DEVICE
:
2271 self
.populate_data_path_device(path
)
2272 elif self
.type == self
.FILE
:
2273 self
.populate_data_path_file(path
)
2274 elif self
.type == self
.NONE
:
2277 raise Error('unexpected type ', self
.type)
2279 def populate_data_path_file(self
, path
):
2280 space_uuid
= self
.name
+ '_uuid'
2281 if getattr(self
.args
, space_uuid
) is not None:
2282 write_one_line(path
, space_uuid
,
2283 getattr(self
.args
, space_uuid
))
2284 if self
.space_symlink
is not None:
2285 adjust_symlink(self
.space_symlink
,
2286 os
.path
.join(path
, self
.name
))
2288 def populate_data_path_device(self
, path
):
2289 self
.populate_data_path_file(path
)
2291 if self
.space_dmcrypt
is not None:
2292 adjust_symlink(self
.space_dmcrypt
,
2293 os
.path
.join(path
, self
.name
+ '_dmcrypt'))
2296 os
.unlink(os
.path
.join(path
, self
.name
+ '_dmcrypt'))
2301 if self
.type == self
.DEVICE
:
2302 self
.prepare_device()
2303 elif self
.type == self
.FILE
:
2305 elif self
.type == self
.NONE
:
2308 raise Error('unexpected type ', self
.type)
2310 def prepare_file(self
):
2311 space_filename
= getattr(self
.args
, self
.name
)
2312 if not os
.path
.exists(space_filename
):
2313 LOG
.debug('Creating %s file %s with size 0'
2314 ' (ceph-osd will resize and allocate)',
2317 space_file
= open(space_filename
, 'wb')
2319 path_set_context(space_filename
)
2321 LOG
.debug('%s is file %s',
2322 self
.name
.capitalize(),
2324 LOG
.warning('OSD will not be hot-swappable if %s is '
2325 'not the same device as the osd data' %
2327 self
.space_symlink
= space_filename
2329 def prepare_device(self
):
2330 reusing_partition
= False
2332 if is_partition(getattr(self
.args
, self
.name
)):
2333 LOG
.debug('%s %s is a partition',
2334 self
.name
.capitalize(), getattr(self
.args
, self
.name
))
2335 partition
= DevicePartition
.factory(
2336 path
=None, dev
=getattr(self
.args
, self
.name
), args
=self
.args
)
2337 if isinstance(partition
, DevicePartitionCrypt
):
2338 raise Error(getattr(self
.args
, self
.name
) +
2339 ' partition already exists'
2340 ' and --dmcrypt specified')
2341 LOG
.warning('OSD will not be hot-swappable' +
2342 ' if ' + self
.name
+ ' is not' +
2343 ' the same device as the osd data')
2344 if partition
.get_ptype() == partition
.ptype_for_name(self
.name
):
2345 LOG
.debug('%s %s was previously prepared with '
2346 'ceph-disk. Reusing it.',
2347 self
.name
.capitalize(),
2348 getattr(self
.args
, self
.name
))
2349 reusing_partition
= True
2350 # Read and reuse the partition uuid from this journal's
2351 # previous life. We reuse the uuid instead of changing it
2352 # because udev does not reliably notice changes to an
2353 # existing partition's GUID. See
2354 # http://tracker.ceph.com/issues/10146
2355 setattr(self
.args
, self
.name
+ '_uuid', partition
.get_uuid())
2356 LOG
.debug('Reusing %s with uuid %s',
2358 getattr(self
.args
, self
.name
+ '_uuid'))
2360 LOG
.warning('%s %s was not prepared with '
2361 'ceph-disk. Symlinking directly.',
2362 self
.name
.capitalize(),
2363 getattr(self
.args
, self
.name
))
2364 self
.space_symlink
= getattr(self
.args
, self
.name
)
2367 self
.space_symlink
= '/dev/disk/by-partuuid/{uuid}'.format(
2368 uuid
=getattr(self
.args
, self
.name
+ '_uuid'))
2370 if self
.args
.dmcrypt
:
2371 self
.space_dmcrypt
= self
.space_symlink
2372 self
.space_symlink
= '/dev/mapper/{uuid}'.format(
2373 uuid
=getattr(self
.args
, self
.name
+ '_uuid'))
2375 if reusing_partition
:
2376 # confirm that the space_symlink exists. It should since
2377 # this was an active space
2378 # in the past. Continuing otherwise would be futile.
2379 assert os
.path
.exists(self
.space_symlink
)
2382 num
= self
.desired_partition_number()
2385 LOG
.warning('OSD will not be hot-swappable if %s '
2386 'is not the same device as the osd data',
2389 device
= Device
.factory(getattr(self
.args
, self
.name
), self
.args
)
2390 num
= device
.create_partition(
2391 uuid
=getattr(self
.args
, self
.name
+ '_uuid'),
2393 size
=self
.space_size
,
2396 partition
= device
.get_partition(num
)
2398 LOG
.debug('%s is GPT partition %s',
2399 self
.name
.capitalize(),
2402 if isinstance(partition
, DevicePartitionCrypt
):
2409 '--typecode={num}:{uuid}'.format(
2411 uuid
=partition
.ptype_for_name(self
.name
),
2414 getattr(self
.args
, self
.name
),
2417 update_partition(getattr(self
.args
, self
.name
), 'prepared')
2419 LOG
.debug('%s is GPT partition %s',
2420 self
.name
.capitalize(),
2424 class PrepareJournal(PrepareSpace
):
2426 def __init__(self
, args
):
2427 self
.name
= 'journal'
2428 (self
.allows_journal
,
2430 self
.needs_journal
) = check_journal_reqs(args
)
2432 if args
.journal
and not self
.allows_journal
:
2433 raise Error('journal specified but not allowed by osd backend')
2435 super(PrepareJournal
, self
).__init
__(args
)
2437 def wants_space(self
):
2438 return self
.wants_journal
2440 def get_space_size(self
):
2441 return int(get_conf_with_default(
2442 cluster
=self
.args
.cluster
,
2443 variable
='osd_journal_size',
2446 def desired_partition_number(self
):
2447 if self
.args
.journal
== self
.args
.data
:
2448 # we're sharing the disk between osd data and journal;
2449 # make journal be partition number 2
2457 return PrepareSpace
.parser('journal')
2460 class PrepareBluestoreBlock(PrepareSpace
):
2462 def __init__(self
, args
):
2464 super(PrepareBluestoreBlock
, self
).__init
__(args
)
2466 def get_space_size(self
):
2467 block_size
= get_conf(
2468 cluster
=self
.args
.cluster
,
2469 variable
='bluestore_block_size',
2472 if block_size
is None:
2473 return 0 # get as much space as possible
2475 return int(block_size
) / 1048576 # MB
2477 def desired_partition_number(self
):
2478 if self
.args
.block
== self
.args
.data
:
2486 return PrepareSpace
.parser('block')
2489 class PrepareBluestoreBlockDB(PrepareSpace
):
2491 def __init__(self
, args
):
2492 self
.name
= 'block.db'
2493 super(PrepareBluestoreBlockDB
, self
).__init
__(args
)
2495 def get_space_size(self
):
2496 block_db_size
= get_conf(
2497 cluster
=self
.args
.cluster
,
2498 variable
='bluestore_block_db_size',
2501 if block_db_size
is None or int(block_db_size
) == 0:
2502 block_size
= get_conf(
2503 cluster
=self
.args
.cluster
,
2504 variable
='bluestore_block_size',
2506 if block_size
is None:
2508 size
= int(block_size
) / 100 / 1048576
2509 return max(size
, 1024) # MB
2511 return int(block_db_size
) / 1048576 # MB
2513 def desired_partition_number(self
):
2514 if getattr(self
.args
, 'block.db') == self
.args
.data
:
2520 def wants_space(self
):
2525 parser
= PrepareSpace
.parser('block.db', positional
=False)
2526 parser
.add_argument(
2529 help='path to the device or file for bluestore block.db',
2534 class PrepareBluestoreBlockWAL(PrepareSpace
):
2536 def __init__(self
, args
):
2537 self
.name
= 'block.wal'
2538 super(PrepareBluestoreBlockWAL
, self
).__init
__(args
)
2540 def get_space_size(self
):
2541 block_size
= get_conf(
2542 cluster
=self
.args
.cluster
,
2543 variable
='bluestore_block_wal_size',
2546 if block_size
is None:
2547 return 576 # MB, default value
2549 return int(block_size
) / 1048576 # MB
2551 def desired_partition_number(self
):
2552 if getattr(self
.args
, 'block.wal') == self
.args
.data
:
2558 def wants_space(self
):
2563 parser
= PrepareSpace
.parser('block.wal', positional
=False)
2564 parser
.add_argument(
2567 help='path to the device or file for bluestore block.wal',
2572 class CryptHelpers(object):
2575 def get_cryptsetup_parameters(args
):
2576 cryptsetup_parameters_str
= get_conf(
2577 cluster
=args
.cluster
,
2578 variable
='osd_cryptsetup_parameters',
2580 if cryptsetup_parameters_str
is None:
2583 return shlex
.split(cryptsetup_parameters_str
)
2586 def get_dmcrypt_keysize(args
):
2587 dmcrypt_keysize_str
= get_conf(
2588 cluster
=args
.cluster
,
2589 variable
='osd_dmcrypt_key_size',
2591 dmcrypt_type
= CryptHelpers
.get_dmcrypt_type(args
)
2592 if dmcrypt_type
== 'luks':
2593 if dmcrypt_keysize_str
is None:
2594 # As LUKS will hash the 'passphrase' in .luks.key
2595 # into a key, set a large default
2596 # so if not updated for some time, it is still a
2601 return int(dmcrypt_keysize_str
)
2602 elif dmcrypt_type
== 'plain':
2603 if dmcrypt_keysize_str
is None:
2604 # This value is hard-coded in the udev script
2607 LOG
.warning('ensure the 95-ceph-osd.rules file has '
2608 'been copied to /etc/udev/rules.d '
2609 'and modified to call cryptsetup '
2610 'with --key-size=%s' % dmcrypt_keysize_str
)
2611 return int(dmcrypt_keysize_str
)
2616 def get_dmcrypt_type(args
):
2617 if hasattr(args
, 'dmcrypt') and args
.dmcrypt
:
2618 dmcrypt_type
= get_conf(
2619 cluster
=args
.cluster
,
2620 variable
='osd_dmcrypt_type',
2623 if dmcrypt_type
is None or dmcrypt_type
== 'luks':
2625 elif dmcrypt_type
== 'plain':
2628 raise Error('invalid osd_dmcrypt_type parameter '
2629 '(must be luks or plain): ', dmcrypt_type
)
2634 class Secrets(object):
2637 secret
, stderr
, ret
= command(['ceph-authtool', '--gen-print-key'])
2638 LOG
.debug("stderr " + stderr
)
2641 'cephx_secret': secret
.strip(),
2644 def write_osd_keyring(self
, keyring
, osd_id
):
2647 'ceph-authtool', keyring
,
2649 '--name', 'osd.' + str(osd_id
),
2650 '--add-key', self
.keys
['cephx_secret'],
2652 path_set_context(keyring
)
2655 return bytearray(json
.dumps(self
.keys
), 'ascii')
2658 class LockboxSecrets(Secrets
):
2660 def __init__(self
, args
):
2661 super(LockboxSecrets
, self
).__init
__()
2663 key_size
= CryptHelpers
.get_dmcrypt_keysize(args
)
2664 key
= open('/dev/urandom', 'rb').read(key_size
/ 8)
2665 base64_key
= base64
.b64encode(key
).decode('ascii')
2667 secret
, stderr
, ret
= command(['ceph-authtool', '--gen-print-key'])
2668 LOG
.debug("stderr " + stderr
)
2672 'dmcrypt_key': base64_key
,
2673 'cephx_lockbox_secret': secret
.strip(),
2676 def write_lockbox_keyring(self
, path
, osd_uuid
):
2677 keyring
= os
.path
.join(path
, 'keyring')
2680 'ceph-authtool', keyring
,
2682 '--name', 'client.osd-lockbox.' + osd_uuid
,
2683 '--add-key', self
.keys
['cephx_lockbox_secret'],
2685 path_set_context(keyring
)
2688 class Lockbox(object):
2690 def __init__(self
, args
):
2692 self
.partition
= None
2695 if hasattr(self
.args
, 'lockbox') and self
.args
.lockbox
is None:
2696 self
.args
.lockbox
= self
.args
.data
2698 def set_partition(self
, partition
):
2699 self
.partition
= partition
2703 parser
= argparse
.ArgumentParser(add_help
=False)
2704 parser
.add_argument(
2706 help='path to the device to store the lockbox',
2708 parser
.add_argument(
2711 help='unique lockbox uuid',
2715 def create_partition(self
):
2716 self
.device
= Device
.factory(self
.args
.lockbox
, argparse
.Namespace())
2717 partition_number
= 5
2718 self
.device
.create_partition(uuid
=self
.args
.lockbox_uuid
,
2720 num
=partition_number
,
2722 return self
.device
.get_partition(partition_number
)
2724 def set_or_create_partition(self
):
2725 if is_partition(self
.args
.lockbox
):
2726 LOG
.debug('OSD lockbox device %s is a partition',
2728 self
.partition
= DevicePartition
.factory(
2729 path
=None, dev
=self
.args
.lockbox
, args
=self
.args
)
2730 ptype
= self
.partition
.get_ptype()
2731 ready
= Ptype
.get_ready_by_name('lockbox')
2732 if ptype
not in ready
:
2733 LOG
.warning('incorrect partition UUID: %s, expected %s'
2734 % (ptype
, str(ready
)))
2736 LOG
.debug('Creating osd partition on %s',
2738 self
.partition
= self
.create_partition()
2740 def create_key(self
):
2741 cluster
= self
.args
.cluster
2742 bootstrap
= self
.args
.prepare_key_template
.format(cluster
=cluster
,
2744 path
= self
.get_mount_point()
2745 secrets
= LockboxSecrets(self
.args
)
2746 id_arg
= self
.args
.osd_id
and [self
.args
.osd_id
] or []
2747 osd_id
= command_with_stdin(
2750 '--cluster', cluster
,
2751 '--name', 'client.bootstrap-osd',
2752 '--keyring', bootstrap
,
2754 'osd', 'new', self
.args
.osd_uuid
,
2758 secrets
.write_lockbox_keyring(path
, self
.args
.osd_uuid
)
2759 osd_id
= must_be_one_line(osd_id
)
2760 check_osd_id(osd_id
)
2761 write_one_line(path
, 'whoami', osd_id
)
2762 secrets
.write_osd_keyring(os
.path
.join(path
, 'osd_keyring'), osd_id
)
2763 write_one_line(path
, 'key-management-mode', KEY_MANAGEMENT_MODE_V1
)
2765 def symlink_spaces(self
, path
):
2766 target
= self
.get_mount_point()
2767 for name
in Space
.NAMES
:
2768 if (hasattr(self
.args
, name
+ '_uuid') and
2769 getattr(self
.args
, name
+ '_uuid')):
2770 uuid
= getattr(self
.args
, name
+ '_uuid')
2771 symlink
= os
.path
.join(STATEDIR
, 'osd-lockbox', uuid
)
2772 adjust_symlink(target
, symlink
)
2773 write_one_line(path
, name
+ '-uuid', uuid
)
2776 maybe_mkdir(os
.path
.join(STATEDIR
, 'osd-lockbox'))
2777 args
= ['mkfs', '-t', 'ext4', self
.partition
.get_dev()]
2778 LOG
.debug('Creating lockbox fs on %s: ' + str(" ".join(args
)))
2779 command_check_call(args
)
2780 path
= self
.get_mount_point()
2782 args
= ['mount', '-t', 'ext4', self
.partition
.get_dev(), path
]
2783 LOG
.debug('Mounting lockbox ' + str(" ".join(args
)))
2784 command_check_call(args
)
2785 write_one_line(path
, 'osd-uuid', self
.args
.osd_uuid
)
2786 if self
.args
.cluster_uuid
is None:
2787 self
.args
.cluster_uuid
= get_fsid(cluster
=self
.args
.cluster
)
2788 write_one_line(path
, 'ceph_fsid', self
.args
.cluster_uuid
)
2790 self
.symlink_spaces(path
)
2791 write_one_line(path
, 'magic', CEPH_LOCKBOX_ONDISK_MAGIC
)
2792 if self
.device
is not None:
2796 '--typecode={num}:{uuid}'.format(
2797 num
=self
.partition
.get_partition_number(),
2798 uuid
=self
.partition
.ptype_for_name('lockbox'),
2801 get_partition_base(self
.partition
.get_dev()),
2805 def get_mount_point(self
):
2806 return os
.path
.join(STATEDIR
, 'osd-lockbox', self
.args
.osd_uuid
)
2808 def get_osd_uuid(self
):
2809 return self
.args
.osd_uuid
2812 path
= is_mounted(self
.partition
.get_dev())
2814 LOG
.info("Lockbox already mounted at " + path
)
2817 path
= tempfile
.mkdtemp(
2819 dir=STATEDIR
+ '/tmp',
2821 args
= ['mount', '-t', 'ext4', '-o', 'ro',
2822 self
.partition
.get_dev(),
2824 LOG
.debug('Mounting lockbox temporarily ' + str(" ".join(args
)))
2825 command_check_call(args
)
2826 self
.args
.osd_uuid
= get_oneliner(path
, 'osd-uuid')
2827 command_check_call(['umount', path
])
2828 LOG
.debug('Mounting lockbox readonly ' + str(" ".join(args
)))
2829 args
= ['mount', '-t', 'ext4', '-o', 'ro',
2830 self
.partition
.get_dev(),
2831 self
.get_mount_point()]
2832 command_check_call(args
)
2833 for name
in Space
.NAMES
+ ('osd',):
2834 uuid_path
= os
.path
.join(self
.get_mount_point(), name
+ '-uuid')
2835 if os
.path
.exists(uuid_path
):
2836 uuid
= get_oneliner(self
.get_mount_point(), name
+ '-uuid')
2837 dev
= os
.path
.join('/dev/disk/by-partuuid/', uuid
.lower())
2838 args
= ['ceph-disk', 'trigger', dev
]
2839 command_check_call(args
)
2842 verify_not_in_use(self
.args
.lockbox
, check_partitions
=True)
2843 self
.set_or_create_partition()
2847 class PrepareData(object):
2852 def __init__(self
, args
):
2855 self
.partition
= None
2857 if self
.args
.cluster_uuid
is None:
2858 self
.args
.cluster_uuid
= get_fsid(cluster
=self
.args
.cluster
)
2860 if self
.args
.osd_uuid
is None:
2861 self
.args
.osd_uuid
= str(uuid
.uuid4())
2864 dmode
= os
.stat(self
.args
.data
).st_mode
2866 if stat
.S_ISDIR(dmode
):
2867 self
.type = self
.FILE
2868 elif stmode_is_diskdevice(dmode
):
2869 self
.type = self
.DEVICE
2871 raise Error('not a dir or block device', self
.args
.data
)
2874 return self
.type == self
.FILE
2876 def is_device(self
):
2877 return self
.type == self
.DEVICE
2881 parser
= argparse
.ArgumentParser(add_help
=False)
2882 parser
.add_argument(
2884 help='file system type to use (e.g. "ext4")',
2886 parser
.add_argument(
2888 action
='store_true', default
=None,
2889 help='destroy the partition table (and content) of a disk',
2891 parser
.add_argument(
2893 action
='store_true', default
=None,
2894 help='verify that DATA is a dir',
2896 parser
.add_argument(
2898 action
='store_true', default
=None,
2899 help='verify that DATA is a block device',
2901 parser
.add_argument(
2904 help='path to OSD data (a disk block device or directory)',
2908 def populate_data_path_file(self
, path
, *to_prepare_list
):
2909 self
.populate_data_path(path
, *to_prepare_list
)
2911 def populate_data_path(self
, path
, *to_prepare_list
):
2912 if os
.path
.exists(os
.path
.join(path
, 'magic')):
2913 LOG
.debug('Data dir %s already exists', path
)
2916 LOG
.debug('Preparing osd data dir %s', path
)
2918 if self
.args
.osd_uuid
is None:
2919 self
.args
.osd_uuid
= str(uuid
.uuid4())
2921 write_one_line(path
, 'ceph_fsid', self
.args
.cluster_uuid
)
2922 write_one_line(path
, 'fsid', self
.args
.osd_uuid
)
2923 if self
.args
.osd_id
:
2924 write_one_line(path
, 'wanttobe', self
.args
.osd_id
)
2925 if self
.args
.crush_device_class
:
2926 write_one_line(path
, 'crush_device_class',
2927 self
.args
.crush_device_class
)
2928 write_one_line(path
, 'magic', CEPH_OSD_ONDISK_MAGIC
)
2930 for to_prepare
in to_prepare_list
:
2931 to_prepare
.populate_data_path(path
)
2933 def prepare(self
, *to_prepare_list
):
2934 if self
.type == self
.DEVICE
:
2935 self
.prepare_device(*to_prepare_list
)
2936 elif self
.type == self
.FILE
:
2937 self
.prepare_file(*to_prepare_list
)
2939 raise Error('unexpected type ', self
.type)
2941 def prepare_file(self
, *to_prepare_list
):
2943 if not os
.path
.exists(self
.args
.data
):
2944 raise Error('data path for directory does not exist',
2947 if self
.args
.data_dev
:
2948 raise Error('data path is not a block device', self
.args
.data
)
2950 for to_prepare
in to_prepare_list
:
2951 to_prepare
.prepare()
2953 self
.populate_data_path_file(self
.args
.data
, *to_prepare_list
)
2955 def sanity_checks(self
):
2956 if not os
.path
.exists(self
.args
.data
):
2957 raise Error('data path for device does not exist',
2959 verify_not_in_use(self
.args
.data
,
2960 check_partitions
=not self
.args
.dmcrypt
)
2962 def set_variables(self
):
2963 if self
.args
.fs_type
is None:
2964 self
.args
.fs_type
= get_conf(
2965 cluster
=self
.args
.cluster
,
2966 variable
='osd_mkfs_type',
2968 if self
.args
.fs_type
is None:
2969 self
.args
.fs_type
= get_conf(
2970 cluster
=self
.args
.cluster
,
2971 variable
='osd_fs_type',
2973 if self
.args
.fs_type
is None:
2974 self
.args
.fs_type
= DEFAULT_FS_TYPE
2976 self
.mkfs_args
= get_conf(
2977 cluster
=self
.args
.cluster
,
2978 variable
='osd_mkfs_options_{fstype}'.format(
2979 fstype
=self
.args
.fs_type
,
2982 if self
.mkfs_args
is None:
2983 self
.mkfs_args
= get_conf(
2984 cluster
=self
.args
.cluster
,
2985 variable
='osd_fs_mkfs_options_{fstype}'.format(
2986 fstype
=self
.args
.fs_type
,
2990 self
.mount_options
= get_mount_options(cluster
=self
.args
.cluster
,
2991 fs_type
=self
.args
.fs_type
)
2993 if self
.args
.osd_uuid
is None:
2994 self
.args
.osd_uuid
= str(uuid
.uuid4())
2996 def prepare_device(self
, *to_prepare_list
):
2997 self
.sanity_checks()
2998 self
.set_variables()
2999 if self
.args
.zap_disk
is not None:
3002 def create_data_partition(self
):
3003 device
= Device
.factory(self
.args
.data
, self
.args
)
3004 partition_number
= 1
3005 device
.create_partition(uuid
=self
.args
.osd_uuid
,
3007 num
=partition_number
,
3008 size
=self
.get_space_size())
3009 return device
.get_partition(partition_number
)
3011 def set_data_partition(self
):
3012 if is_partition(self
.args
.data
):
3013 LOG
.debug('OSD data device %s is a partition',
3015 self
.partition
= DevicePartition
.factory(
3016 path
=None, dev
=self
.args
.data
, args
=self
.args
)
3017 ptype
= self
.partition
.get_ptype()
3018 ready
= Ptype
.get_ready_by_name('osd')
3019 if ptype
not in ready
:
3020 LOG
.warning('incorrect partition UUID: %s, expected %s'
3021 % (ptype
, str(ready
)))
3023 LOG
.debug('Creating osd partition on %s',
3025 self
.partition
= self
.create_data_partition()
3027 def populate_data_path_device(self
, *to_prepare_list
):
3028 partition
= self
.partition
3030 if isinstance(partition
, DevicePartitionCrypt
):
3039 if self
.mkfs_args
is not None:
3040 args
.extend(self
.mkfs_args
.split())
3041 if self
.args
.fs_type
== 'xfs':
3042 args
.extend(['-f']) # always force
3044 args
.extend(MKFS_ARGS
.get(self
.args
.fs_type
, []))
3047 partition
.get_dev(),
3049 LOG
.debug('Creating %s fs on %s',
3050 self
.args
.fs_type
, partition
.get_dev())
3051 command_check_call(args
, exit
=True)
3053 path
= mount(dev
=partition
.get_dev(),
3054 fstype
=self
.args
.fs_type
,
3055 options
=self
.mount_options
)
3058 self
.populate_data_path(path
, *to_prepare_list
)
3060 path_set_context(path
)
3063 if isinstance(partition
, DevicePartitionCrypt
):
3066 if not is_partition(self
.args
.data
):
3070 '--typecode=%d:%s' % (partition
.get_partition_number(),
3071 partition
.ptype_for_name('osd')),
3077 update_partition(self
.args
.data
, 'prepared')
3078 command_check_call(['udevadm', 'trigger',
3081 os
.path
.basename(partition
.rawdev
)])
3084 class PrepareFilestoreData(PrepareData
):
3086 def get_space_size(self
):
3087 return 0 # get as much space as possible
3089 def prepare_device(self
, *to_prepare_list
):
3090 super(PrepareFilestoreData
, self
).prepare_device(*to_prepare_list
)
3091 for to_prepare
in to_prepare_list
:
3092 to_prepare
.prepare()
3093 self
.set_data_partition()
3094 self
.populate_data_path_device(*to_prepare_list
)
3096 def populate_data_path(self
, path
, *to_prepare_list
):
3097 super(PrepareFilestoreData
, self
).populate_data_path(path
,
3099 write_one_line(path
, 'type', 'filestore')
3102 class PrepareBluestoreData(PrepareData
):
3104 def get_space_size(self
):
3107 def prepare_device(self
, *to_prepare_list
):
3108 super(PrepareBluestoreData
, self
).prepare_device(*to_prepare_list
)
3109 self
.set_data_partition()
3110 for to_prepare
in to_prepare_list
:
3111 to_prepare
.prepare()
3112 self
.populate_data_path_device(*to_prepare_list
)
3114 def populate_data_path(self
, path
, *to_prepare_list
):
3115 super(PrepareBluestoreData
, self
).populate_data_path(path
,
3117 write_one_line(path
, 'type', 'bluestore')
3127 monmap
= os
.path
.join(path
, 'activate.monmap')
3131 '--cluster', cluster
,
3132 '--name', 'client.bootstrap-osd',
3133 '--keyring', keyring
,
3134 'mon', 'getmap', '-o', monmap
,
3138 osd_type
= read_one_line(path
, 'type')
3140 if osd_type
== 'bluestore':
3144 '--cluster', cluster
,
3150 '--setuser', get_ceph_user(),
3151 '--setgroup', get_ceph_group(),
3154 elif osd_type
== 'filestore':
3158 '--cluster', cluster
,
3163 '--osd-journal', os
.path
.join(path
, 'journal'),
3165 '--setuser', get_ceph_user(),
3166 '--setgroup', get_ceph_group(),
3170 raise Error('unrecognized objectstore type %s' % osd_type
)
3173 def get_mount_point(cluster
, osd_id
):
3174 parent
= STATEDIR
+ '/osd'
3175 return os
.path
.join(
3177 '{cluster}-{osd_id}'.format(cluster
=cluster
, osd_id
=osd_id
),
3189 LOG
.debug('Moving mount to final location...')
3190 osd_data
= get_mount_point(cluster
, osd_id
)
3191 maybe_mkdir(osd_data
)
3193 # pick best-of-breed mount options based on fs type
3194 if mount_options
is None:
3195 mount_options
= MOUNT_OPTIONS
.get(fstype
, '')
3197 # we really want to mount --move, but that is not supported when
3198 # the parent mount is shared, as it is by default on RH, Fedora,
3199 # and probably others. Also, --bind doesn't properly manipulate
3200 # /etc/mtab, which *still* isn't a symlink to /proc/mounts despite
3201 # this being 2013. Instead, mount the original device at the final
3216 '-l', # lazy, in case someone else is peeking at the
3225 # For upgrade purposes, to make sure there are no competing units,
3226 # both --runtime unit and the default should be disabled. There can be
3227 # two units at the same time: one with --runtime and another without
3228 # it. If, for any reason (manual or ceph-disk) the two units co-exist
3229 # they will compete with each other.
3231 def systemd_disable(
3235 # ensure there is no duplicate ceph-osd@.service
3236 for style
in ([], ['--runtime']):
3241 'ceph-osd@{osd_id}'.format(osd_id
=osd_id
),
3250 systemd_disable(path
, osd_id
)
3251 if is_mounted(path
):
3252 style
= ['--runtime']
3259 'ceph-osd@{osd_id}'.format(osd_id
=osd_id
),
3266 'ceph-osd@{osd_id}'.format(osd_id
=osd_id
),
3275 systemd_disable(path
, osd_id
)
3280 'ceph-osd@{osd_id}'.format(osd_id
=osd_id
),
3289 LOG
.debug('Starting %s osd.%s...', cluster
, osd_id
)
3291 path
= (STATEDIR
+ '/osd/{cluster}-{osd_id}').format(
3292 cluster
=cluster
, osd_id
=osd_id
)
3295 if os
.path
.exists(os
.path
.join(path
, 'upstart')):
3299 # use emit, not start, because start would fail if the
3300 # instance was already running
3302 # since the daemon starting doesn't guarantee much about
3303 # the service being operational anyway, don't bother
3308 'cluster={cluster}'.format(cluster
=cluster
),
3309 'id={osd_id}'.format(osd_id
=osd_id
),
3312 elif os
.path
.exists(os
.path
.join(path
, 'sysvinit')):
3313 if os
.path
.exists('/usr/sbin/service'):
3314 svc
= '/usr/sbin/service'
3316 svc
= '/sbin/service'
3322 '{cluster}'.format(cluster
=cluster
),
3324 'osd.{osd_id}'.format(osd_id
=osd_id
),
3327 elif os
.path
.exists(os
.path
.join(path
, 'systemd')):
3328 systemd_start(path
, osd_id
)
3329 elif os
.path
.exists(os
.path
.join(path
, 'openrc')):
3330 base_script
= '/etc/init.d/ceph-osd'
3331 osd_script
= '{base}.{osd_id}'.format(
3335 if not os
.path
.exists(osd_script
):
3336 os
.symlink(base_script
, osd_script
)
3343 elif os
.path
.exists(os
.path
.join(path
, 'bsdrc')):
3346 '/usr/sbin/service', 'ceph', 'start',
3347 'osd.{osd_id}'.format(osd_id
=osd_id
),
3351 raise Error('{cluster} osd.{osd_id} '
3352 'is not tagged with an init system'
3357 except subprocess
.CalledProcessError
as e
:
3358 raise Error('ceph osd start failed', e
)
3365 LOG
.debug('Stoping %s osd.%s...', cluster
, osd_id
)
3367 path
= (STATEDIR
+ '/osd/{cluster}-{osd_id}').format(
3368 cluster
=cluster
, osd_id
=osd_id
)
3371 if os
.path
.exists(os
.path
.join(path
, 'upstart')):
3377 'cluster={cluster}'.format(cluster
=cluster
),
3378 'id={osd_id}'.format(osd_id
=osd_id
),
3381 elif os
.path
.exists(os
.path
.join(path
, 'sysvinit')):
3382 svc
= which('service')
3388 '{cluster}'.format(cluster
=cluster
),
3390 'osd.{osd_id}'.format(osd_id
=osd_id
),
3393 elif os
.path
.exists(os
.path
.join(path
, 'systemd')):
3394 systemd_stop(path
, osd_id
)
3395 elif os
.path
.exists(os
.path
.join(path
, 'openrc')):
3398 '/etc/init.d/ceph-osd.{osd_id}'.format(osd_id
=osd_id
),
3402 elif os
.path
.exists(os
.path
.join(path
, 'bsdrc')):
3405 '/usr/local/etc/rc.d/ceph stop osd.{osd_id}'
3406 .format(osd_id
=osd_id
),
3410 raise Error('{cluster} osd.{osd_id} '
3411 'is not tagged with an init system'
3412 .format(cluster
=cluster
, osd_id
=osd_id
))
3413 except subprocess
.CalledProcessError
as e
:
3414 raise Error('ceph osd stop failed', e
)
3417 def detect_fstype(dev
):
3419 fstype
= _check_output(
3427 fstype
= _check_output(
3430 # we don't want stale cached results
3438 fstype
= must_be_one_line(fstype
)
3442 def dmcrypt_is_mapped(uuid
):
3443 path
= os
.path
.join('/dev/mapper', uuid
)
3444 if os
.path
.exists(path
):
3450 def dmcrypt_map(dev
, dmcrypt_key_dir
):
3451 ptype
= get_partition_type(dev
)
3452 if ptype
in Ptype
.get_ready_by_type('plain'):
3454 cryptsetup_parameters
= ['--key-size', '256']
3455 elif ptype
in Ptype
.get_ready_by_type('luks'):
3457 cryptsetup_parameters
= []
3459 raise Error('--dmcrypt called for dev %s with invalid ptype %s'
3461 part_uuid
= get_partition_uuid(dev
)
3462 dmcrypt_key
= get_dmcrypt_key(part_uuid
, dmcrypt_key_dir
, luks
)
3463 return _dmcrypt_map(
3467 cryptsetup_parameters
=cryptsetup_parameters
,
3475 activate_key_template
,
3483 part_uuid
= get_partition_uuid(dev
)
3484 dev
= dmcrypt_map(dev
, dmcrypt_key_dir
)
3486 fstype
= detect_fstype(dev
=dev
)
3487 except (subprocess
.CalledProcessError
,
3489 TooManyLinesError
) as e
:
3490 raise FilesystemTypeError(
3491 'device {dev}'.format(dev
=dev
),
3495 # TODO always using mount options from cluster=ceph for
3496 # now; see http://tracker.newdream.net/issues/3253
3497 mount_options
= get_mount_options(cluster
='ceph', fs_type
=fstype
)
3499 path
= mount(dev
=dev
, fstype
=fstype
, options
=mount_options
)
3501 # check if the disk is deactive, change the journal owner, group
3502 # mode for correct user and group.
3503 if os
.path
.exists(os
.path
.join(path
, 'deactive')):
3504 # logging to syslog will help us easy to know udev triggered failure
3507 # we need to unmap again because dmcrypt map will create again
3508 # on bootup stage (due to deactivate)
3509 if '/dev/mapper/' in dev
:
3510 part_uuid
= dev
.replace('/dev/mapper/', '')
3511 dmcrypt_unmap(part_uuid
)
3512 LOG
.info('OSD deactivated! reactivate with: --reactivate')
3513 raise Error('OSD deactivated! reactivate with: --reactivate')
3514 # flag to activate a deactive osd.
3522 (osd_id
, cluster
) = activate(path
, activate_key_template
, init
)
3524 # Now active successfully
3525 # If we got reactivate and deactive, remove the deactive file
3526 if deactive
and reactivate
:
3527 os
.remove(os
.path
.join(path
, 'deactive'))
3528 LOG
.info('Remove `deactive` file.')
3530 # check if the disk is already active, or if something else is already
3534 src_dev
= os
.stat(path
).st_dev
3536 dst_dev
= os
.stat((STATEDIR
+ '/osd/{cluster}-{osd_id}').format(
3538 osd_id
=osd_id
)).st_dev
3539 if src_dev
== dst_dev
:
3542 parent_dev
= os
.stat(STATEDIR
+ '/osd').st_dev
3543 if dst_dev
!= parent_dev
:
3545 elif os
.listdir(get_mount_point(cluster
, osd_id
)):
3546 LOG
.info(get_mount_point(cluster
, osd_id
) +
3547 " is not empty, won't override")
3554 LOG
.info('%s osd.%s already mounted in position; unmounting ours.'
3555 % (cluster
, osd_id
))
3558 raise Error('another %s osd.%s already mounted in position '
3559 '(old/different cluster instance?); unmounting ours.'
3560 % (cluster
, osd_id
))
3568 mount_options
=mount_options
,
3570 return cluster
, osd_id
3573 LOG
.error('Failed to activate')
3577 # remove our temp dir
3578 if os
.path
.exists(path
):
3584 activate_key_template
,
3588 if not os
.path
.exists(path
):
3590 'directory %s does not exist' % path
3593 (osd_id
, cluster
) = activate(path
, activate_key_template
, init
)
3595 if init
not in (None, 'none'):
3596 canonical
= (STATEDIR
+ '/osd/{cluster}-{osd_id}').format(
3599 if path
!= canonical
:
3600 # symlink it from the proper location
3602 if os
.path
.lexists(canonical
):
3603 old
= os
.readlink(canonical
)
3605 LOG
.debug('Removing old symlink %s -> %s', canonical
, old
)
3607 os
.unlink(canonical
)
3609 raise Error('unable to remove old symlink', canonical
)
3613 LOG
.debug('Creating symlink %s -> %s', canonical
, path
)
3615 os
.symlink(path
, canonical
)
3617 raise Error('unable to create symlink %s -> %s'
3618 % (canonical
, path
))
3620 return cluster
, osd_id
3623 def find_cluster_by_uuid(_uuid
):
3625 Find a cluster name by searching /etc/ceph/*.conf for a conf file
3626 with the right uuid.
3628 _uuid
= _uuid
.lower()
3630 if not os
.path
.exists(SYSCONFDIR
):
3632 for conf_file
in os
.listdir(SYSCONFDIR
):
3633 if not conf_file
.endswith('.conf'):
3635 cluster
= conf_file
[:-5]
3637 fsid
= get_fsid(cluster
)
3639 if 'getting cluster uuid from configuration failed' not in str(e
):
3641 no_fsid
.append(cluster
)
3645 # be tolerant of /etc/ceph/ceph.conf without an fsid defined.
3646 if len(no_fsid
) == 1 and no_fsid
[0] == 'ceph':
3647 LOG
.warning('No fsid defined in ' + SYSCONFDIR
+
3648 '/ceph.conf; using anyway')
3655 activate_key_template
,
3659 check_osd_magic(path
)
3661 ceph_fsid
= read_one_line(path
, 'ceph_fsid')
3662 if ceph_fsid
is None:
3663 raise Error('No cluster uuid assigned.')
3664 LOG
.debug('Cluster uuid is %s', ceph_fsid
)
3666 cluster
= find_cluster_by_uuid(ceph_fsid
)
3668 raise Error('No cluster conf found in ' + SYSCONFDIR
+
3669 ' with fsid %s' % ceph_fsid
)
3670 LOG
.debug('Cluster name is %s', cluster
)
3672 fsid
= read_one_line(path
, 'fsid')
3674 raise Error('No OSD uuid assigned.')
3675 LOG
.debug('OSD uuid is %s', fsid
)
3677 keyring
= activate_key_template
.format(cluster
=cluster
,
3680 osd_id
= get_osd_id(path
)
3682 osd_id
= allocate_osd_id(
3688 write_one_line(path
, 'whoami', osd_id
)
3689 LOG
.debug('OSD id is %s', osd_id
)
3691 if not os
.path
.exists(os
.path
.join(path
, 'ready')):
3692 LOG
.debug('Initializing OSD...')
3693 # re-running mkfs is safe, so just run until it completes
3702 if init
not in (None, 'none'):
3704 conf_val
= get_conf(
3708 if conf_val
is not None:
3713 LOG
.debug('Marking with init system %s', init
)
3714 init_path
= os
.path
.join(path
, init
)
3715 with
open(init_path
, 'w'):
3716 path_set_context(init_path
)
3718 # remove markers for others, just in case.
3719 for other
in INIT_SYSTEMS
:
3722 os
.unlink(os
.path
.join(path
, other
))
3726 if not os
.path
.exists(os
.path
.join(path
, 'active')):
3727 write_one_line(path
, 'active', 'ok')
3728 LOG
.debug('%s osd.%s data dir is ready at %s', cluster
, osd_id
, path
)
3729 return (osd_id
, cluster
)
3732 def main_activate(args
):
3736 LOG
.info('path = ' + str(args
.path
))
3737 if not os
.path
.exists(args
.path
):
3738 raise Error('%s does not exist' % args
.path
)
3740 if is_suppressed(args
.path
):
3741 LOG
.info('suppressed activate request on %s', args
.path
)
3745 mode
= os
.stat(args
.path
).st_mode
3746 if stmode_is_diskdevice(mode
):
3747 if (is_partition(args
.path
) and
3748 (get_partition_type(args
.path
) ==
3749 PTYPE
['mpath']['osd']['ready']) and
3750 not is_mpath(args
.path
)):
3751 raise Error('%s is not a multipath block device' %
3753 (cluster
, osd_id
) = mount_activate(
3755 activate_key_template
=args
.activate_key_template
,
3756 init
=args
.mark_init
,
3757 dmcrypt
=args
.dmcrypt
,
3758 dmcrypt_key_dir
=args
.dmcrypt_key_dir
,
3759 reactivate
=args
.reactivate
,
3761 osd_data
= get_mount_point(cluster
, osd_id
)
3763 elif stat
.S_ISDIR(mode
):
3764 (cluster
, osd_id
) = activate_dir(
3766 activate_key_template
=args
.activate_key_template
,
3767 init
=args
.mark_init
,
3769 osd_data
= args
.path
3772 raise Error('%s is not a directory or block device' % args
.path
)
3774 # exit with 0 if the journal device is not up, yet
3775 # journal device will do the activation
3776 osd_journal
= '{path}/journal'.format(path
=osd_data
)
3777 if os
.path
.islink(osd_journal
) and not os
.access(osd_journal
, os
.F_OK
):
3778 LOG
.info("activate: Journal not present, not starting, yet")
3781 if (not args
.no_start_daemon
and args
.mark_init
== 'none'):
3785 '--cluster={cluster}'.format(cluster
=cluster
),
3786 '--id={osd_id}'.format(osd_id
=osd_id
),
3787 '--osd-data={path}'.format(path
=osd_data
),
3788 '--osd-journal={journal}'.format(journal
=osd_journal
),
3792 if (not args
.no_start_daemon
and
3793 args
.mark_init
not in (None, 'none')):
3801 def main_activate_lockbox(args
):
3803 main_activate_lockbox_protected(args
)
3806 def main_activate_lockbox_protected(args
):
3807 partition
= DevicePartition
.factory(
3808 path
=None, dev
=args
.path
, args
=args
)
3810 lockbox
= Lockbox(args
)
3811 lockbox
.set_partition(partition
)
3815 ###########################
3817 def _mark_osd_out(cluster
, osd_id
):
3818 LOG
.info('Prepare to mark osd.%d out...', osd_id
)
3827 def _check_osd_status(cluster
, osd_id
):
3829 report the osd status:
3830 00(0) : means OSD OUT AND DOWN
3831 01(1) : means OSD OUT AND UP
3832 10(2) : means OSD IN AND DOWN
3833 11(3) : means OSD IN AND UP
3835 LOG
.info("Checking osd id: %s ..." % osd_id
)
3838 out
, err
, ret
= command([
3842 '--cluster={cluster}'.format(
3848 out_json
= json
.loads(out
)
3849 for item
in out_json
[u
'osds']:
3850 if item
.get(u
'osd') == int(osd_id
):
3852 if item
.get(u
'in') is 1:
3854 if item
.get(u
'up') is 1:
3857 raise Error('Could not osd.%s in osd tree!' % osd_id
)
3861 def _remove_osd_directory_files(mounted_path
, cluster
):
3863 To remove the 'ready', 'active', INIT-specific files.
3865 if os
.path
.exists(os
.path
.join(mounted_path
, 'ready')):
3866 os
.remove(os
.path
.join(mounted_path
, 'ready'))
3867 LOG
.info('Remove `ready` file.')
3869 LOG
.info('`ready` file is already removed.')
3871 if os
.path
.exists(os
.path
.join(mounted_path
, 'active')):
3872 os
.remove(os
.path
.join(mounted_path
, 'active'))
3873 LOG
.info('Remove `active` file.')
3875 LOG
.info('`active` file is already removed.')
3877 # Just check `upstart` and `sysvinit` directly if filename is init-spec.
3878 conf_val
= get_conf(
3882 if conf_val
is not None:
3886 os
.remove(os
.path
.join(mounted_path
, init
))
3887 LOG
.info('Remove `%s` file.', init
)
3891 def main_deactivate(args
):
3893 main_deactivate_locked(args
)
3896 def main_deactivate_locked(args
):
3897 osd_id
= args
.deactivate_by_id
3901 devices
= list_devices()
3903 # list all devices and found we need
3904 for device
in devices
:
3905 if 'partitions' in device
:
3906 for dev_part
in device
.get('partitions'):
3908 'whoami' in dev_part
and
3909 dev_part
['whoami'] == osd_id
):
3910 target_dev
= dev_part
3912 'path' in dev_part
and
3913 dev_part
['path'] == path
):
3914 target_dev
= dev_part
3916 raise Error('Cannot find any match device!!')
3918 # set up all we need variable
3919 osd_id
= target_dev
['whoami']
3920 part_type
= target_dev
['ptype']
3921 mounted_path
= target_dev
['mount']
3922 if Ptype
.is_dmcrypt(part_type
, 'osd'):
3925 # Do not do anything if osd is already down.
3926 status_code
= _check_osd_status(args
.cluster
, osd_id
)
3927 if status_code
== OSD_STATUS_IN_UP
:
3928 if args
.mark_out
is True:
3929 _mark_osd_out(args
.cluster
, int(osd_id
))
3930 stop_daemon(args
.cluster
, osd_id
)
3931 elif status_code
== OSD_STATUS_IN_DOWN
:
3932 if args
.mark_out
is True:
3933 _mark_osd_out(args
.cluster
, int(osd_id
))
3934 LOG
.info("OSD already out/down. Do not do anything now.")
3936 elif status_code
== OSD_STATUS_OUT_UP
:
3937 stop_daemon(args
.cluster
, osd_id
)
3938 elif status_code
== OSD_STATUS_OUT_DOWN
:
3939 LOG
.info("OSD already out/down. Do not do anything now.")
3943 # remove 'ready', 'active', and INIT-specific files.
3944 _remove_osd_directory_files(mounted_path
, args
.cluster
)
3946 # Write deactivate to osd directory!
3947 with
open(os
.path
.join(mounted_path
, 'deactive'), 'w'):
3948 path_set_context(os
.path
.join(mounted_path
, 'deactive'))
3950 unmount(mounted_path
, do_rm
=not args
.once
)
3951 LOG
.info("Umount `%s` successfully.", mounted_path
)
3954 lockbox
= os
.path
.join(STATEDIR
, 'osd-lockbox')
3955 command(['umount', os
.path
.join(lockbox
, target_dev
['uuid'])])
3957 dmcrypt_unmap(target_dev
['uuid'])
3958 for name
in Space
.NAMES
:
3959 if name
+ '_uuid' in target_dev
:
3960 dmcrypt_unmap(target_dev
[name
+ '_uuid'])
3962 ###########################
3965 def _remove_lockbox(uuid
, cluster
):
3966 lockbox
= os
.path
.join(STATEDIR
, 'osd-lockbox')
3967 if not os
.path
.exists(lockbox
):
3969 canonical
= os
.path
.join(lockbox
, uuid
)
3970 command(['umount', canonical
])
3971 for name
in os
.listdir(lockbox
):
3972 path
= os
.path
.join(lockbox
, name
)
3973 if os
.path
.islink(path
) and os
.readlink(path
) == canonical
:
3977 def destroy_lookup_device(args
, predicate
, description
):
3978 devices
= list_devices()
3979 for device
in devices
:
3980 for partition
in device
.get('partitions', []):
3981 if partition
['type'] == 'lockbox':
3982 if not is_mounted(partition
['path']):
3983 main_activate_lockbox_protected(
3984 argparse
.Namespace(verbose
=args
.verbose
,
3985 path
=partition
['path']))
3986 for device
in devices
:
3987 for partition
in device
.get('partitions', []):
3988 if partition
['dmcrypt']:
3989 dmcrypt_path
= dmcrypt_is_mapped(partition
['uuid'])
3993 dmcrypt_path
= dmcrypt_map(partition
['path'],
3994 args
.dmcrypt_key_dir
)
3996 list_dev_osd(dmcrypt_path
, {}, partition
)
3998 dmcrypt_unmap(partition
['uuid'])
4002 if predicate(partition
):
4003 return dmcrypt
, partition
4004 raise Error('found no device matching ', description
)
4007 def main_destroy(args
):
4009 main_destroy_locked(args
)
4012 def main_destroy_locked(args
):
4013 osd_id
= args
.destroy_by_id
4018 if not is_partition(path
):
4019 raise Error(path
+ " must be a partition device")
4020 path
= os
.path
.realpath(path
)
4023 (dmcrypt
, target_dev
) = destroy_lookup_device(
4024 args
, lambda x
: x
.get('path') == path
,
4027 (dmcrypt
, target_dev
) = destroy_lookup_device(
4028 args
, lambda x
: x
.get('whoami') == osd_id
,
4029 'osd id ' + str(osd_id
))
4031 osd_id
= target_dev
['whoami']
4032 dev_path
= target_dev
['path']
4033 if target_dev
['ptype'] == PTYPE
['mpath']['osd']['ready']:
4034 base_dev
= get_partition_base_mpath(dev_path
)
4036 base_dev
= get_partition_base(dev_path
)
4038 # Before osd deactivate, we cannot destroy it
4039 status_code
= _check_osd_status(args
.cluster
, osd_id
)
4040 if status_code
!= OSD_STATUS_OUT_DOWN
and \
4041 status_code
!= OSD_STATUS_IN_DOWN
:
4042 raise Error("Could not destroy the active osd. (osd-id: %s)" %
4049 LOG
.info("Prepare to %s osd.%s" % (action
, osd_id
))
4055 '--yes-i-really-mean-it',
4058 # we remove the crypt map and device mapper (if dmcrypt is True)
4060 for name
in Space
.NAMES
:
4061 if target_dev
.get(name
+ '_uuid'):
4062 dmcrypt_unmap(target_dev
[name
+ '_uuid'])
4063 _remove_lockbox(target_dev
['uuid'], args
.cluster
)
4065 # Check zap flag. If we found zap flag, we need to find device for
4066 # destroy this osd data.
4067 if args
.zap
is True:
4068 # erase the osd data
4069 LOG
.info("Prepare to zap the device %s" % base_dev
)
4073 def get_space_osd_uuid(name
, path
):
4074 if not os
.path
.exists(path
):
4075 raise Error('%s does not exist' % path
)
4077 if not path_is_diskdevice(path
):
4078 raise Error('%s is not a block device' % path
)
4080 if (is_partition(path
) and
4081 get_partition_type(path
) in (PTYPE
['mpath']['journal']['ready'],
4082 PTYPE
['mpath']['block']['ready']) and
4083 not is_mpath(path
)):
4084 raise Error('%s is not a multipath block device' %
4088 out
= _check_output(
4091 '--get-device-fsid',
4096 except subprocess
.CalledProcessError
as e
:
4098 'failed to get osd uuid/fsid from %s' % name
,
4101 value
= str(out
).split('\n', 1)[0]
4102 LOG
.debug('%s %s has OSD UUID %s', name
.capitalize(), path
, value
)
4106 def main_activate_space(name
, args
):
4107 if not os
.path
.exists(args
.dev
):
4108 raise Error('%s does not exist' % args
.dev
)
4110 if is_suppressed(args
.dev
):
4111 LOG
.info('suppressed activate request on space %s', args
.dev
)
4120 dev
= dmcrypt_map(args
.dev
, args
.dmcrypt_key_dir
)
4123 # FIXME: For an encrypted journal dev, does this return the
4124 # cyphertext or plaintext dev uuid!? Also, if the journal is
4125 # encrypted, is the data partition also always encrypted, or
4126 # are mixed pairs supported!?
4127 osd_uuid
= get_space_osd_uuid(name
, dev
)
4128 path
= os
.path
.join('/dev/disk/by-partuuid/', osd_uuid
.lower())
4130 if is_suppressed(path
):
4131 LOG
.info('suppressed activate request on %s', path
)
4134 # warn and exit with 0 if the data device is not up, yet
4135 # data device will do the activation
4136 if not os
.access(path
, os
.F_OK
):
4137 LOG
.info("activate: OSD device not present, not starting, yet")
4140 (cluster
, osd_id
) = mount_activate(
4142 activate_key_template
=args
.activate_key_template
,
4143 init
=args
.mark_init
,
4144 dmcrypt
=args
.dmcrypt
,
4145 dmcrypt_key_dir
=args
.dmcrypt_key_dir
,
4146 reactivate
=args
.reactivate
,
4155 ###########################
4158 def main_activate_all(args
):
4159 dir = '/dev/disk/by-parttypeuuid'
4160 LOG
.debug('Scanning %s', dir)
4161 if not os
.path
.exists(dir):
4164 for name
in os
.listdir(dir):
4165 if name
.find('.') < 0:
4167 (tag
, uuid
) = name
.split('.')
4169 if tag
in Ptype
.get_ready_by_name('osd'):
4171 if Ptype
.is_dmcrypt(tag
, 'osd'):
4172 path
= os
.path
.join('/dev/mapper', uuid
)
4174 path
= os
.path
.join(dir, name
)
4176 if is_suppressed(path
):
4177 LOG
.info('suppressed activate request on %s', path
)
4180 LOG
.info('Activating %s', path
)
4183 # never map dmcrypt cyphertext devices
4184 (cluster
, osd_id
) = mount_activate(
4186 activate_key_template
=args
.activate_key_template
,
4187 init
=args
.mark_init
,
4196 except Exception as e
:
4198 '{prog}: {msg}'.format(prog
=args
.prog
, msg
=e
),
4205 raise Error('One or more partitions failed to activate')
4208 ###########################
4211 dev
= os
.path
.realpath(dev
)
4212 with
open(PROCDIR
+ '/swaps', 'rb') as proc_swaps
:
4213 for line
in proc_swaps
.readlines()[1:]:
4214 fields
= line
.split()
4217 swaps_dev
= fields
[0]
4218 if os
.path
.isabs(swaps_dev
) and os
.path
.exists(swaps_dev
):
4219 swaps_dev
= os
.path
.realpath(swaps_dev
)
4220 if swaps_dev
== dev
:
4225 def get_oneliner(base
, name
):
4226 path
= os
.path
.join(base
, name
)
4227 if os
.path
.isfile(path
):
4228 with
open(path
, 'rb') as _file
:
4229 return _bytes2str(_file
.readline().rstrip())
4233 def get_dev_fs(dev
):
4235 fstype
, _
, ret
= command(
4245 fscheck
, _
, _
= command(
4253 if 'TYPE' in fscheck
:
4254 fstype
= fscheck
.split()[1].split('"')[1]
4259 def split_dev_base_partnum(dev
):
4261 partnum
= partnum_mpath(dev
)
4262 base
= get_partition_base_mpath(dev
)
4265 partnum
= open(os
.path
.join(b
, 'partition')).read().strip()
4266 base
= get_partition_base(dev
)
4267 return base
, partnum
4270 def get_partition_type(part
):
4271 return get_blkid_partition_info(part
, 'ID_PART_ENTRY_TYPE')
4274 def get_partition_uuid(part
):
4275 return get_blkid_partition_info(part
, 'ID_PART_ENTRY_UUID')
4278 def get_blkid_partition_info(dev
, what
=None):
4279 out
, _
, _
= command(
4289 for line
in out
.splitlines():
4290 (key
, value
) = line
.split('=')
4298 def more_osd_info(path
, uuid_map
, desc
):
4299 desc
['ceph_fsid'] = get_oneliner(path
, 'ceph_fsid')
4300 if desc
['ceph_fsid']:
4301 desc
['cluster'] = find_cluster_by_uuid(desc
['ceph_fsid'])
4302 desc
['whoami'] = get_oneliner(path
, 'whoami')
4303 for name
in Space
.NAMES
:
4304 uuid
= get_oneliner(path
, name
+ '_uuid')
4306 desc
[name
+ '_uuid'] = uuid
.lower()
4307 if desc
[name
+ '_uuid'] in uuid_map
:
4308 desc
[name
+ '_dev'] = uuid_map
[desc
[name
+ '_uuid']]
4311 def list_dev_osd(dev
, uuid_map
, desc
):
4312 desc
['mount'] = is_mounted(dev
)
4313 desc
['fs_type'] = get_dev_fs(dev
)
4314 desc
['state'] = 'unprepared'
4316 desc
['state'] = 'active'
4317 more_osd_info(desc
['mount'], uuid_map
, desc
)
4318 elif desc
['fs_type']:
4320 tpath
= mount(dev
=dev
, fstype
=desc
['fs_type'], options
='')
4323 magic
= get_oneliner(tpath
, 'magic')
4324 if magic
is not None:
4325 desc
['magic'] = magic
4326 desc
['state'] = 'prepared'
4327 more_osd_info(tpath
, uuid_map
, desc
)
4334 def list_dev_lockbox(dev
, uuid_map
, desc
):
4335 desc
['mount'] = is_mounted(dev
)
4336 desc
['fs_type'] = get_dev_fs(dev
)
4337 desc
['state'] = 'unprepared'
4339 desc
['state'] = 'active'
4340 desc
['osd_uuid'] = get_oneliner(desc
['mount'], 'osd-uuid')
4341 elif desc
['fs_type']:
4343 tpath
= tempfile
.mkdtemp(prefix
='mnt.', dir=STATEDIR
+ '/tmp')
4344 args
= ['mount', '-t', 'ext4', dev
, tpath
]
4345 LOG
.debug('Mounting lockbox ' + str(" ".join(args
)))
4346 command_check_call(args
)
4347 magic
= get_oneliner(tpath
, 'magic')
4348 if magic
is not None:
4349 desc
['magic'] = magic
4350 desc
['state'] = 'prepared'
4351 desc
['osd_uuid'] = get_oneliner(tpath
, 'osd-uuid')
4353 except subprocess
.CalledProcessError
:
4355 if desc
.get('osd_uuid') in uuid_map
:
4356 desc
['lockbox_for'] = uuid_map
[desc
['osd_uuid']]
4359 def list_format_lockbox_plain(dev
):
4361 if dev
.get('lockbox_for'):
4362 desc
.append('for ' + dev
['lockbox_for'])
4363 elif dev
.get('osd_uuid'):
4364 desc
.append('for osd ' + dev
['osd_uuid'])
4368 def list_format_more_osd_info_plain(dev
):
4370 if dev
.get('ceph_fsid'):
4371 if dev
.get('cluster'):
4372 desc
.append('cluster ' + dev
['cluster'])
4374 desc
.append('unknown cluster ' + dev
['ceph_fsid'])
4375 if dev
.get('whoami'):
4376 desc
.append('osd.%s' % dev
['whoami'])
4377 for name
in Space
.NAMES
:
4378 if dev
.get(name
+ '_dev'):
4379 desc
.append(name
+ ' %s' % dev
[name
+ '_dev'])
4383 def list_format_dev_plain(dev
, prefix
=''):
4385 if dev
['ptype'] == PTYPE
['regular']['osd']['ready']:
4386 desc
= (['ceph data', dev
['state']] +
4387 list_format_more_osd_info_plain(dev
))
4388 elif dev
['ptype'] in (PTYPE
['regular']['lockbox']['ready'],
4389 PTYPE
['mpath']['lockbox']['ready']):
4390 desc
= (['ceph lockbox', dev
['state']] +
4391 list_format_lockbox_plain(dev
))
4392 elif Ptype
.is_dmcrypt(dev
['ptype'], 'osd'):
4393 dmcrypt
= dev
['dmcrypt']
4394 if not dmcrypt
['holders']:
4395 desc
= ['ceph data (dmcrypt %s)' % dmcrypt
['type'],
4396 'not currently mapped']
4397 elif len(dmcrypt
['holders']) == 1:
4398 holder
= get_dev_path(dmcrypt
['holders'][0])
4399 desc
= ['ceph data (dmcrypt %s %s)' %
4400 (dmcrypt
['type'], holder
)]
4401 desc
+= list_format_more_osd_info_plain(dev
)
4403 desc
= ['ceph data (dmcrypt %s)' % dmcrypt
['type'],
4404 'holders: ' + ','.join(dmcrypt
['holders'])]
4405 elif Ptype
.is_regular_space(dev
['ptype']):
4406 name
= Ptype
.space_ptype_to_name(dev
['ptype'])
4407 desc
.append('ceph ' + name
)
4408 if dev
.get(name
+ '_for'):
4409 desc
.append('for %s' % dev
[name
+ '_for'])
4410 elif Ptype
.is_dmcrypt_space(dev
['ptype']):
4411 name
= Ptype
.space_ptype_to_name(dev
['ptype'])
4412 dmcrypt
= dev
['dmcrypt']
4413 if dmcrypt
['holders'] and len(dmcrypt
['holders']) == 1:
4414 holder
= get_dev_path(dmcrypt
['holders'][0])
4415 desc
= ['ceph ' + name
+ ' (dmcrypt %s %s)' %
4416 (dmcrypt
['type'], holder
)]
4418 desc
= ['ceph ' + name
+ ' (dmcrypt %s)' % dmcrypt
['type']]
4419 if dev
.get(name
+ '_for'):
4420 desc
.append('for %s' % dev
[name
+ '_for'])
4422 desc
.append(dev
['type'])
4423 if dev
.get('fs_type'):
4424 desc
.append(dev
['fs_type'])
4425 elif dev
.get('ptype'):
4426 desc
.append(dev
['ptype'])
4427 if dev
.get('mount'):
4428 desc
.append('mounted on %s' % dev
['mount'])
4429 return '%s%s %s' % (prefix
, dev
['path'], ', '.join(desc
))
4432 def list_format_plain(devices
):
4434 for device
in devices
:
4435 if device
.get('partitions'):
4436 lines
.append('%s :' % device
['path'])
4437 for p
in sorted(device
['partitions'], key
=lambda x
: x
['path']):
4438 lines
.append(list_format_dev_plain(dev
=p
,
4441 lines
.append(list_format_dev_plain(dev
=device
,
4443 return "\n".join(lines
)
4446 def list_dev(dev
, uuid_map
, space_map
):
4452 info
['is_partition'] = is_partition(dev
)
4453 if info
['is_partition']:
4454 ptype
= get_partition_type(dev
)
4455 info
['uuid'] = get_partition_uuid(dev
)
4458 info
['ptype'] = ptype
4459 LOG
.info("list_dev(dev = " + dev
+ ", ptype = " + str(ptype
) + ")")
4460 if ptype
in (PTYPE
['regular']['osd']['ready'],
4461 PTYPE
['mpath']['osd']['ready']):
4462 info
['type'] = 'data'
4463 if ptype
== PTYPE
['mpath']['osd']['ready']:
4464 info
['multipath'] = True
4465 list_dev_osd(dev
, uuid_map
, info
)
4466 elif ptype
in (PTYPE
['regular']['lockbox']['ready'],
4467 PTYPE
['mpath']['lockbox']['ready']):
4468 info
['type'] = 'lockbox'
4469 if ptype
== PTYPE
['mpath']['osd']['ready']:
4470 info
['multipath'] = True
4471 list_dev_lockbox(dev
, uuid_map
, info
)
4472 elif ptype
== PTYPE
['plain']['osd']['ready']:
4473 holders
= is_held(dev
)
4474 info
['type'] = 'data'
4475 info
['dmcrypt']['holders'] = holders
4476 info
['dmcrypt']['type'] = 'plain'
4477 if len(holders
) == 1:
4478 list_dev_osd(get_dev_path(holders
[0]), uuid_map
, info
)
4479 elif ptype
== PTYPE
['luks']['osd']['ready']:
4480 holders
= is_held(dev
)
4481 info
['type'] = 'data'
4482 info
['dmcrypt']['holders'] = holders
4483 info
['dmcrypt']['type'] = 'LUKS'
4484 if len(holders
) == 1:
4485 list_dev_osd(get_dev_path(holders
[0]), uuid_map
, info
)
4486 elif Ptype
.is_regular_space(ptype
) or Ptype
.is_mpath_space(ptype
):
4487 name
= Ptype
.space_ptype_to_name(ptype
)
4489 if ptype
== PTYPE
['mpath'][name
]['ready']:
4490 info
['multipath'] = True
4491 if info
.get('uuid') in space_map
:
4492 info
[name
+ '_for'] = space_map
[info
['uuid']]
4493 elif Ptype
.is_plain_space(ptype
):
4494 name
= Ptype
.space_ptype_to_name(ptype
)
4495 holders
= is_held(dev
)
4497 info
['dmcrypt']['type'] = 'plain'
4498 info
['dmcrypt']['holders'] = holders
4499 if info
.get('uuid') in space_map
:
4500 info
[name
+ '_for'] = space_map
[info
['uuid']]
4501 elif Ptype
.is_luks_space(ptype
):
4502 name
= Ptype
.space_ptype_to_name(ptype
)
4503 holders
= is_held(dev
)
4505 info
['dmcrypt']['type'] = 'LUKS'
4506 info
['dmcrypt']['holders'] = holders
4507 if info
.get('uuid') in space_map
:
4508 info
[name
+ '_for'] = space_map
[info
['uuid']]
4510 path
= is_mounted(dev
)
4511 fs_type
= get_dev_fs(dev
)
4513 info
['type'] = 'swap'
4515 info
['type'] = 'other'
4517 info
['fs_type'] = fs_type
4519 info
['mount'] = path
4525 partmap
= list_all_partitions()
4529 for base
, parts
in sorted(partmap
.items()):
4531 dev
= get_dev_path(p
)
4532 part_uuid
= get_partition_uuid(dev
)
4534 uuid_map
[part_uuid
] = dev
4535 ptype
= get_partition_type(dev
)
4536 LOG
.debug("main_list: " + dev
+
4537 " ptype = " + str(ptype
) +
4538 " uuid = " + str(part_uuid
))
4539 if ptype
in Ptype
.get_ready_by_name('osd'):
4540 if Ptype
.is_dmcrypt(ptype
, 'osd'):
4541 holders
= is_held(dev
)
4542 if len(holders
) != 1:
4544 dev_to_mount
= get_dev_path(holders
[0])
4548 fs_type
= get_dev_fs(dev_to_mount
)
4549 if fs_type
is not None:
4550 mount_options
= get_mount_options(cluster
='ceph',
4553 tpath
= mount(dev
=dev_to_mount
,
4554 fstype
=fs_type
, options
=mount_options
)
4556 for name
in Space
.NAMES
:
4557 space_uuid
= get_oneliner(tpath
,
4560 space_map
[space_uuid
.lower()] = dev
4566 LOG
.debug("main_list: " + str(partmap
) + ", uuid_map = " +
4567 str(uuid_map
) + ", space_map = " + str(space_map
))
4570 for base
, parts
in sorted(partmap
.items()):
4572 disk
= {'path': get_dev_path(base
)}
4574 for p
in sorted(parts
):
4575 partitions
.append(list_dev(get_dev_path(p
),
4578 disk
['partitions'] = partitions
4579 devices
.append(disk
)
4581 device
= list_dev(get_dev_path(base
), uuid_map
, space_map
)
4582 device
['path'] = get_dev_path(base
)
4583 devices
.append(device
)
4584 LOG
.debug("list_devices: " + str(devices
))
4590 out
, err
, ret
= command(
4594 '-o', 'name,mountpoint'
4597 except subprocess
.CalledProcessError
as e
:
4598 LOG
.info('zfs list -o name,mountpoint '
4599 'fails.\n (Error: %s)' % e
)
4601 lines
= out
.splitlines()
4602 for line
in lines
[1:]:
4603 vdevline
= line
.split()
4604 if os
.path
.exists(os
.path
.join(vdevline
[1], 'active')):
4605 elems
= os
.path
.split(vdevline
[1])
4606 print(vdevline
[0], "ceph data, active, cluster ceph,", elems
[1],
4607 "mounted on:", vdevline
[1])
4609 print(vdevline
[0] + " other, zfs, mounted on: " + vdevline
[1])
4612 def main_list(args
):
4615 main_list_freebsd(args
)
4617 main_list_protected(args
)
4620 def main_list_protected(args
):
4621 devices
= list_devices()
4624 for path
in args
.path
:
4625 if os
.path
.exists(path
):
4626 paths
.append(os
.path
.realpath(path
))
4629 selected_devices
= []
4630 for device
in devices
:
4632 if re
.search(path
+ '$', device
['path']):
4633 selected_devices
.append(device
)
4635 selected_devices
= devices
4636 if args
.format
== 'json':
4637 print(json
.dumps(selected_devices
))
4639 output
= list_format_plain(selected_devices
)
4644 def main_list_freebsd(args
):
4645 # Currently accomodate only ZFS Filestore partitions
4646 # return a list of VDEVs and mountpoints
4648 # NAME USED AVAIL REFER MOUNTPOINT
4649 # osd0 1.01G 1.32T 1.01G /var/lib/ceph/osd/osd.0
4650 # osd1 1.01G 1.32T 1.01G /var/lib/ceph/osd/osd.1
4654 ###########################
4656 # Mark devices that we want to suppress activates on with a
4659 # /var/lib/ceph/tmp/suppress-activate.sdb
4661 # where the last bit is the sanitized device name (/dev/X without the
4662 # /dev/ prefix) and the is_suppress() check matches a prefix. That
4663 # means suppressing sdb will stop activate on sdb1, sdb2, etc.
4666 def is_suppressed(path
):
4667 disk
= os
.path
.realpath(path
)
4669 if (not disk
.startswith('/dev/') or
4670 not ldev_is_diskdevice(disk
)):
4672 base
= get_dev_name(disk
)
4674 if os
.path
.exists(SUPPRESS_PREFIX
+ base
): # noqa
4681 def set_suppress(path
):
4682 disk
= os
.path
.realpath(path
)
4683 if not os
.path
.exists(disk
):
4684 raise Error('does not exist', path
)
4685 if not ldev_is_diskdevice(path
):
4686 raise Error('not a block device', path
)
4687 base
= get_dev_name(disk
)
4689 with
open(SUPPRESS_PREFIX
+ base
, 'w') as f
: # noqa
4691 LOG
.info('set suppress flag on %s', base
)
4694 def unset_suppress(path
):
4695 disk
= os
.path
.realpath(path
)
4696 if not os
.path
.exists(disk
):
4697 raise Error('does not exist', path
)
4698 if not ldev_is_diskdevice(path
):
4699 raise Error('not a block device', path
)
4700 assert disk
.startswith('/dev/')
4701 base
= get_dev_name(disk
)
4703 fn
= SUPPRESS_PREFIX
+ base
# noqa
4704 if not os
.path
.exists(fn
):
4705 raise Error('not marked as suppressed', path
)
4709 LOG
.info('unset suppress flag on %s', base
)
4710 except OSError as e
:
4711 raise Error('failed to unsuppress', e
)
4714 def main_suppress(args
):
4715 set_suppress(args
.path
)
4718 def main_unsuppress(args
):
4719 unset_suppress(args
.path
)
4723 for dev
in args
.dev
:
4727 def main_trigger(args
):
4728 LOG
.debug("main_trigger: " + str(args
))
4729 if is_systemd() and not args
.sync
:
4730 # http://www.freedesktop.org/software/systemd/man/systemd-escape.html
4731 escaped_dev
= args
.dev
[1:].replace('-', '\\x2d')
4732 service
= 'ceph-disk@{dev}.service'.format(dev
=escaped_dev
)
4733 LOG
.info('systemd detected, triggering %s' % service
)
4743 if is_upstart() and not args
.sync
:
4744 LOG
.info('upstart detected, triggering ceph-disk task')
4750 'dev={dev}'.format(dev
=args
.dev
),
4751 'pid={pid}'.format(pid
=os
.getpid()),
4756 if get_ceph_user() == 'ceph':
4757 command_check_call(['chown', 'ceph:ceph', args
.dev
])
4758 parttype
= get_partition_type(args
.dev
)
4759 partid
= get_partition_uuid(args
.dev
)
4761 LOG
.info('trigger {dev} parttype {parttype} uuid {partid}'.format(
4767 ceph_disk
= ['ceph-disk']
4769 ceph_disk
.append('--verbose')
4771 if parttype
in (PTYPE
['regular']['osd']['ready'],
4772 PTYPE
['mpath']['osd']['ready']):
4773 out
, err
, ret
= command(
4781 elif parttype
in (PTYPE
['plain']['osd']['ready'],
4782 PTYPE
['luks']['osd']['ready']):
4783 out
, err
, ret
= command(
4792 elif parttype
in (PTYPE
['regular']['journal']['ready'],
4793 PTYPE
['mpath']['journal']['ready']):
4794 out
, err
, ret
= command(
4802 elif parttype
in (PTYPE
['plain']['journal']['ready'],
4803 PTYPE
['luks']['journal']['ready']):
4804 out
, err
, ret
= command(
4813 elif parttype
in (PTYPE
['regular']['block']['ready'],
4814 PTYPE
['regular']['block.db']['ready'],
4815 PTYPE
['regular']['block.wal']['ready'],
4816 PTYPE
['mpath']['block']['ready'],
4817 PTYPE
['mpath']['block.db']['ready'],
4818 PTYPE
['mpath']['block.wal']['ready']):
4819 out
, err
, ret
= command(
4827 elif parttype
in (PTYPE
['plain']['block']['ready'],
4828 PTYPE
['plain']['block.db']['ready'],
4829 PTYPE
['plain']['block.wal']['ready'],
4830 PTYPE
['luks']['block']['ready'],
4831 PTYPE
['luks']['block.db']['ready'],
4832 PTYPE
['luks']['block.wal']['ready']):
4833 out
, err
, ret
= command(
4842 elif parttype
in (PTYPE
['regular']['lockbox']['ready'],
4843 PTYPE
['mpath']['lockbox']['ready']):
4844 out
, err
, ret
= command(
4853 raise Error('unrecognized partition type %s' % parttype
)
4858 raise Error('return code ' + str(ret
))
4865 # A hash table containing 'path': ('uid', 'gid', blocking, recursive)
4867 ('/usr/bin/ceph-mon', 'root', ROOTGROUP
, True, False),
4868 ('/usr/bin/ceph-mds', 'root', ROOTGROUP
, True, False),
4869 ('/usr/bin/ceph-osd', 'root', ROOTGROUP
, True, False),
4870 ('/usr/bin/radosgw', 'root', ROOTGROUP
, True, False),
4871 ('/etc/ceph', 'root', ROOTGROUP
, True, True),
4872 ('/var/run/ceph', 'ceph', 'ceph', True, True),
4873 ('/var/log/ceph', 'ceph', 'ceph', True, True),
4874 ('/var/log/radosgw', 'ceph', 'ceph', True, True),
4875 ('/var/lib/ceph', 'ceph', 'ceph', True, False),
4878 # Relabel/chown all files under /var/lib/ceph/ recursively (except for osd)
4879 for directory
in glob
.glob('/var/lib/ceph/*'):
4880 if directory
== '/var/lib/ceph/osd':
4881 fix_table
.append((directory
, 'ceph', 'ceph', True, False))
4883 fix_table
.append((directory
, 'ceph', 'ceph', True, True))
4885 # Relabel/chown the osds recursively and in parallel
4886 for directory
in glob
.glob('/var/lib/ceph/osd/*'):
4887 fix_table
.append((directory
, 'ceph', 'ceph', False, True))
4889 LOG
.debug("fix_table: " + str(fix_table
))
4891 # The lists of background processes
4893 permissions_processes
= []
4894 selinux_processes
= []
4896 # Preliminary checks
4897 if args
.selinux
or args
.all
:
4898 out
, err
, ret
= command(['selinuxenabled'])
4900 LOG
.error('SELinux is not enabled, please enable it, first.')
4901 raise Error('no SELinux')
4903 for daemon
in ['ceph-mon', 'ceph-osd', 'ceph-mds', 'radosgw', 'ceph-mgr']:
4904 out
, err
, ret
= command(['pgrep', daemon
])
4906 LOG
.error(daemon
+ ' is running, please stop it, first')
4907 raise Error(daemon
+ ' running')
4909 # Relabel the basic system data without the ceph files
4910 if args
.system
or args
.all
:
4911 c
= ['restorecon', '-R', '/']
4912 for directory
, _
, _
, _
, _
in fix_table
:
4913 # Skip /var/lib/ceph subdirectories
4914 if directory
.startswith('/var/lib/ceph/'):
4919 out
, err
, ret
= command(c
)
4922 LOG
.error("Failed to restore labels of the underlying system")
4924 raise Error("basic restore failed")
4926 # Use find to relabel + chown ~simultaenously
4928 for directory
, uid
, gid
, blocking
, recursive
in fix_table
:
4929 # Skip directories/files that are not installed
4930 if not os
.access(directory
, os
.F_OK
):
4938 ':'.join((uid
, gid
)),
4947 # Just pass -maxdepth 0 for non-recursive calls
4949 c
+= ['-maxdepth', '0']
4952 out
, err
, ret
= command(c
)
4955 LOG
.error("Failed to fix " + directory
)
4957 raise Error(directory
+ " fix failed")
4959 all_processes
.append(command_init(c
))
4961 LOG
.debug("all_processes: " + str(all_processes
))
4962 for process
in all_processes
:
4963 out
, err
, ret
= command_wait(process
)
4965 LOG
.error("A background find process failed")
4967 raise Error("background failed")
4970 if args
.permissions
:
4971 for directory
, uid
, gid
, blocking
, recursive
in fix_table
:
4972 # Skip directories/files that are not installed
4973 if not os
.access(directory
, os
.F_OK
):
4980 ':'.join((uid
, gid
)),
4986 ':'.join((uid
, gid
)),
4991 out
, err
, ret
= command(c
)
4994 LOG
.error("Failed to chown " + directory
)
4996 raise Error(directory
+ " chown failed")
4998 permissions_processes
.append(command_init(c
))
5000 LOG
.debug("permissions_processes: " + str(permissions_processes
))
5001 for process
in permissions_processes
:
5002 out
, err
, ret
= command_wait(process
)
5004 LOG
.error("A background permissions process failed")
5006 raise Error("background failed")
5008 # Fix SELinux labels
5010 for directory
, uid
, gid
, blocking
, recursive
in fix_table
:
5011 # Skip directories/files that are not installed
5012 if not os
.access(directory
, os
.F_OK
):
5028 out
, err
, ret
= command(c
)
5031 LOG
.error("Failed to restore labels for " + directory
)
5033 raise Error(directory
+ " relabel failed")
5035 selinux_processes
.append(command_init(c
))
5037 LOG
.debug("selinux_processes: " + str(selinux_processes
))
5038 for process
in selinux_processes
:
5039 out
, err
, ret
= command_wait(process
)
5041 LOG
.error("A background selinux process failed")
5043 raise Error("background failed")
5046 "The ceph files has been fixed, please reboot "
5047 "the system for the changes to take effect."
5051 def setup_statedir(dir):
5052 # XXX The following use of globals makes linting
5053 # really hard. Global state in Python is iffy and
5054 # should be avoided.
5058 if not os
.path
.exists(STATEDIR
):
5060 if not os
.path
.exists(STATEDIR
+ "/tmp"):
5061 os
.mkdir(STATEDIR
+ "/tmp")
5064 prepare_lock
= FileLock(STATEDIR
+ '/tmp/ceph-disk.prepare.lock')
5066 global activate_lock
5067 activate_lock
= FileLock(STATEDIR
+ '/tmp/ceph-disk.activate.lock')
5069 global SUPPRESS_PREFIX
5070 SUPPRESS_PREFIX
= STATEDIR
+ '/tmp/suppress-activate.'
5073 def setup_sysconfdir(dir):
5078 def parse_args(argv
):
5079 parser
= argparse
.ArgumentParser(
5082 parser
.add_argument(
5084 action
='store_true', default
=None,
5085 help='be more verbose',
5087 parser
.add_argument(
5089 action
='store_true', default
=None,
5090 help='log to stdout',
5092 parser
.add_argument(
5093 '--prepend-to-path',
5096 help=('prepend PATH to $PATH for backward compatibility '
5097 '(default /usr/bin)'),
5099 parser
.add_argument(
5102 default
='/var/lib/ceph',
5103 help=('directory in which ceph state is preserved '
5104 '(default /var/lib/ceph)'),
5106 parser
.add_argument(
5109 default
='/etc/ceph',
5110 help=('directory in which ceph configuration files are found '
5111 '(default /etc/ceph)'),
5113 parser
.add_argument(
5117 help='use the given user for subprocesses, rather than ceph or root'
5119 parser
.add_argument(
5123 help='use the given group for subprocesses, rather than ceph or root'
5125 parser
.set_defaults(
5126 # we want to hold on to this, for later
5130 subparsers
= parser
.add_subparsers(
5131 title
='subcommands',
5132 description
='valid subcommands',
5133 help='sub-command help',
5136 Prepare
.set_subparser(subparsers
)
5137 make_activate_parser(subparsers
)
5138 make_activate_lockbox_parser(subparsers
)
5139 make_activate_block_parser(subparsers
)
5140 make_activate_journal_parser(subparsers
)
5141 make_activate_all_parser(subparsers
)
5142 make_list_parser(subparsers
)
5143 make_suppress_parser(subparsers
)
5144 make_deactivate_parser(subparsers
)
5145 make_destroy_parser(subparsers
)
5146 make_zap_parser(subparsers
)
5147 make_trigger_parser(subparsers
)
5148 make_fix_parser(subparsers
)
5150 args
= parser
.parse_args(argv
)
5154 def make_fix_parser(subparsers
):
5155 fix_parser
= subparsers
.add_parser(
5157 formatter_class
=argparse
.RawDescriptionHelpFormatter
,
5158 description
=textwrap
.fill(textwrap
.dedent("""\
5160 help='fix SELinux labels and/or file permissions')
5162 fix_parser
.add_argument(
5164 action
='store_true',
5166 help='fix SELinux labels for the non-ceph system data'
5168 fix_parser
.add_argument(
5170 action
='store_true',
5172 help='fix SELinux labels for ceph data'
5174 fix_parser
.add_argument(
5176 action
='store_true',
5178 help='fix file permissions for ceph data'
5180 fix_parser
.add_argument(
5182 action
='store_true',
5184 help='perform all the fix-related operations'
5186 fix_parser
.set_defaults(
5192 def make_trigger_parser(subparsers
):
5193 trigger_parser
= subparsers
.add_parser(
5195 formatter_class
=argparse
.RawDescriptionHelpFormatter
,
5196 description
=textwrap
.fill(textwrap
.dedent("""\
5197 The partition given in argument is activated. The type of the
5198 partition (data, lockbox, journal etc.) is detected by its
5199 type. If the init system is upstart or systemd, the activation is
5200 delegated to it and runs asynchronously, which
5201 helps reduce the execution time of udev actions.
5203 help='activate any device (called by udev)')
5204 trigger_parser
.add_argument(
5208 trigger_parser
.add_argument(
5212 help='cluster name to assign this disk to',
5214 trigger_parser
.add_argument(
5216 action
='store_true', default
=None,
5217 help='map devices with dm-crypt',
5219 trigger_parser
.add_argument(
5220 '--dmcrypt-key-dir',
5222 default
='/etc/ceph/dmcrypt-keys',
5223 help='directory where dm-crypt keys are stored',
5225 trigger_parser
.add_argument(
5227 action
='store_true', default
=None,
5228 help='do operation synchronously; do not trigger systemd',
5230 trigger_parser
.set_defaults(
5233 return trigger_parser
5236 def make_activate_parser(subparsers
):
5237 activate_parser
= subparsers
.add_parser(
5239 formatter_class
=argparse
.RawDescriptionHelpFormatter
,
5240 description
=textwrap
.fill(textwrap
.dedent("""\
5241 Activate the OSD found at PATH (can be a directory
5242 or a device partition, possibly encrypted). When
5243 activated for the first time, a unique OSD id is obtained
5244 from the cluster. If PATH is a directory, a symbolic
5245 link is added in {statedir}/osd/ceph-$id. If PATH is
5246 a partition, it is mounted on {statedir}/osd/ceph-$id.
5247 Finally, the OSD daemon is run.
5249 If the OSD depends on auxiliary partitions (journal, block, ...)
5250 they need to be available otherwise activation will fail. It
5251 may happen if a journal is encrypted and cryptsetup was not
5253 """.format(statedir
=STATEDIR
))),
5254 help='Activate a Ceph OSD')
5255 activate_parser
.add_argument(
5257 action
='store_true', default
=None,
5258 help='mount a block device [deprecated, ignored]',
5260 activate_parser
.add_argument(
5263 help='bootstrap-osd keyring path template (%(default)s)',
5264 dest
='activate_key_template',
5266 activate_parser
.add_argument(
5268 metavar
='INITSYSTEM',
5269 help='init system to manage this dir',
5271 choices
=INIT_SYSTEMS
,
5273 activate_parser
.add_argument(
5274 '--no-start-daemon',
5275 action
='store_true', default
=None,
5276 help='do not start the daemon',
5278 activate_parser
.add_argument(
5281 help='path to block device or directory',
5283 activate_parser
.add_argument(
5285 action
='store_true', default
=None,
5286 help='map DATA and/or JOURNAL devices with dm-crypt',
5288 activate_parser
.add_argument(
5289 '--dmcrypt-key-dir',
5291 default
='/etc/ceph/dmcrypt-keys',
5292 help='directory where dm-crypt keys are stored',
5294 activate_parser
.add_argument(
5296 action
='store_true', default
=False,
5297 help='activate the deactived OSD',
5299 activate_parser
.set_defaults(
5300 activate_key_template
='{statedir}/bootstrap-osd/{cluster}.keyring',
5303 return activate_parser
5306 def make_activate_lockbox_parser(subparsers
):
5307 parser
= subparsers
.add_parser(
5309 formatter_class
=argparse
.RawDescriptionHelpFormatter
,
5310 description
=textwrap
.fill(textwrap
.dedent("""\
5311 Mount the partition found at PATH on {statedir}/osd-lockbox/$uuid
5312 where $uuid uniquely identifies the OSD that needs this lockbox
5313 to retrieve keys from the monitor and unlock its partitions.
5315 If the OSD has one or more auxiliary devices (journal, block, ...)
5316 symbolic links are created at {statedir}/osd-lockbox/$other_uuid
5317 and point to {statedir}/osd-lockbox/$uuid. This will, for instance,
5318 allow a journal encrypted in a partition identified by $other_uuid to
5319 fetch the keys it needs from the monitor.
5321 Finally the OSD is activated, as it would be with ceph-disk activate.
5322 """.format(statedir
=STATEDIR
))),
5323 help='Activate a Ceph lockbox')
5324 parser
.add_argument(
5326 help='bootstrap-osd keyring path template (%(default)s)',
5327 dest
='activate_key_template',
5329 parser
.add_argument(
5330 '--dmcrypt-key-dir',
5332 default
='/etc/ceph/dmcrypt-keys',
5333 help='directory where dm-crypt keys are stored',
5335 parser
.add_argument(
5338 help='path to block device',
5340 parser
.set_defaults(
5341 activate_key_template
='{statedir}/bootstrap-osd/{cluster}.keyring',
5342 func
=main_activate_lockbox
,
5347 def make_activate_block_parser(subparsers
):
5348 return make_activate_space_parser('block', subparsers
)
5351 def make_activate_journal_parser(subparsers
):
5352 return make_activate_space_parser('journal', subparsers
)
5355 def make_activate_space_parser(name
, subparsers
):
5356 activate_space_parser
= subparsers
.add_parser(
5357 'activate-%s' % name
,
5358 formatter_class
=argparse
.RawDescriptionHelpFormatter
,
5359 description
=textwrap
.fill(textwrap
.dedent("""\
5360 Activating a {name} partition is only meaningfull
5361 if it is encrypted and it will map it using
5364 Finally the corresponding OSD is activated,
5365 as it would be with ceph-disk activate.
5366 """.format(name
=name
))),
5367 help='Activate an OSD via its %s device' % name
)
5368 activate_space_parser
.add_argument(
5371 help='path to %s block device' % name
,
5373 activate_space_parser
.add_argument(
5376 help='bootstrap-osd keyring path template (%(default)s)',
5377 dest
='activate_key_template',
5379 activate_space_parser
.add_argument(
5381 metavar
='INITSYSTEM',
5382 help='init system to manage this dir',
5384 choices
=INIT_SYSTEMS
,
5386 activate_space_parser
.add_argument(
5388 action
='store_true', default
=None,
5389 help=('map data and/or auxiliariy (journal, etc.) '
5390 'devices with dm-crypt'),
5392 activate_space_parser
.add_argument(
5393 '--dmcrypt-key-dir',
5395 default
='/etc/ceph/dmcrypt-keys',
5396 help='directory where dm-crypt keys are stored',
5398 activate_space_parser
.add_argument(
5400 action
='store_true', default
=False,
5401 help='activate the deactived OSD',
5403 activate_space_parser
.set_defaults(
5404 activate_key_template
='{statedir}/bootstrap-osd/{cluster}.keyring',
5405 func
=lambda args
: main_activate_space(name
, args
),
5407 return activate_space_parser
5410 def make_activate_all_parser(subparsers
):
5411 activate_all_parser
= subparsers
.add_parser(
5413 formatter_class
=argparse
.RawDescriptionHelpFormatter
,
5414 description
=textwrap
.fill(textwrap
.dedent("""\
5415 Activate all OSD partitions found in /dev/disk/by-parttypeuuid.
5416 The partitions containing auxiliary devices (journal, block, ...)
5419 help='Activate all tagged OSD partitions')
5420 activate_all_parser
.add_argument(
5423 help='bootstrap-osd keyring path template (%(default)s)',
5424 dest
='activate_key_template',
5426 activate_all_parser
.add_argument(
5428 metavar
='INITSYSTEM',
5429 help='init system to manage this dir',
5431 choices
=INIT_SYSTEMS
,
5433 activate_all_parser
.set_defaults(
5434 activate_key_template
='{statedir}/bootstrap-osd/{cluster}.keyring',
5435 func
=main_activate_all
,
5437 return activate_all_parser
5440 def make_list_parser(subparsers
):
5441 list_parser
= subparsers
.add_parser(
5443 formatter_class
=argparse
.RawDescriptionHelpFormatter
,
5444 description
=textwrap
.fill(textwrap
.dedent("""\
5445 Display all partitions on the system and their
5446 associated Ceph information, if any.
5448 help='List disks, partitions, and Ceph OSDs')
5449 list_parser
.add_argument(
5451 help='output format',
5453 choices
=['json', 'plain'],
5455 list_parser
.add_argument(
5459 help='path to block devices, relative to /sys/block',
5461 list_parser
.set_defaults(
5467 def make_suppress_parser(subparsers
):
5468 suppress_parser
= subparsers
.add_parser(
5469 'suppress-activate',
5470 formatter_class
=argparse
.RawDescriptionHelpFormatter
,
5471 description
=textwrap
.fill(textwrap
.dedent("""\
5472 Add a prefix to the list of suppressed device names
5473 so that they are ignored by all activate* subcommands.
5475 help='Suppress activate on a device (prefix)')
5476 suppress_parser
.add_argument(
5479 help='path to block device or directory',
5481 suppress_parser
.set_defaults(
5485 unsuppress_parser
= subparsers
.add_parser(
5486 'unsuppress-activate',
5487 formatter_class
=argparse
.RawDescriptionHelpFormatter
,
5488 description
=textwrap
.fill(textwrap
.dedent("""\
5489 Remove a prefix from the list of suppressed device names
5490 so that they are no longer ignored by all
5491 activate* subcommands.
5493 help='Stop suppressing activate on a device (prefix)')
5494 unsuppress_parser
.add_argument(
5497 help='path to block device or directory',
5499 unsuppress_parser
.set_defaults(
5500 func
=main_unsuppress
,
5502 return suppress_parser
5505 def make_deactivate_parser(subparsers
):
5506 deactivate_parser
= subparsers
.add_parser(
5508 formatter_class
=argparse
.RawDescriptionHelpFormatter
,
5509 description
=textwrap
.fill(textwrap
.dedent("""\
5510 Deactivate the OSD located at PATH. It stops the OSD daemon
5511 and optionally marks it out (with --mark-out). The content of
5512 the OSD is left untouched.
5514 By default, the, ready, active, INIT-specific files are
5515 removed (so that it is not automatically re-activated by the
5516 udev rules or ceph-disk trigger) and the file deactive is
5517 created to remember the OSD is deactivated.
5519 If the --once option is given, the ready, active, INIT-specific
5520 files are not removed and the OSD will reactivate whenever
5521 ceph-disk trigger is run on one of the devices (journal, data,
5522 block, lockbox, ...).
5524 If the OSD is dmcrypt, remove the data dmcrypt map. When
5525 deactivate finishes, the OSD is down.
5527 help='Deactivate a Ceph OSD')
5528 deactivate_parser
.add_argument(
5532 help='cluster name to assign this disk to',
5534 deactivate_parser
.add_argument(
5538 help='path to block device or directory',
5540 deactivate_parser
.add_argument(
5541 '--deactivate-by-id',
5543 help='ID of OSD to deactive'
5545 deactivate_parser
.add_argument(
5547 action
='store_true', default
=False,
5548 help='option to mark the osd out',
5550 deactivate_parser
.add_argument(
5552 action
='store_true', default
=False,
5553 help='does not need --reactivate to activate again',
5555 deactivate_parser
.set_defaults(
5556 func
=main_deactivate
,
5560 def make_destroy_parser(subparsers
):
5561 destroy_parser
= subparsers
.add_parser(
5563 formatter_class
=argparse
.RawDescriptionHelpFormatter
,
5564 description
=textwrap
.fill(textwrap
.dedent("""\ Destroy the OSD located at PATH. It removes the OSD from the
5565 cluster and marks it destroyed. An OSD must be down before it
5566 can be destroyed. Once it is destroyed, a new OSD can be created
5567 in its place, reusing the same OSD id and position (e.g. after
5568 a failed HDD or SSD is replaced). Alternatively, if the
5569 --purge option is also specified, the OSD is removed from the
5570 CRUSH map and the OSD id is deallocated.""")),
5571 help='Destroy a Ceph OSD')
5572 destroy_parser
.add_argument(
5576 help='cluster name to assign this disk to',
5578 destroy_parser
.add_argument(
5582 help='path to block device or directory',
5584 destroy_parser
.add_argument(
5587 help='ID of OSD to destroy'
5589 destroy_parser
.add_argument(
5590 '--dmcrypt-key-dir',
5592 default
='/etc/ceph/dmcrypt-keys',
5593 help=('directory where dm-crypt keys are stored '
5594 '(If you don\'t know how it work, '
5595 'dont use it. we have default value)'),
5597 destroy_parser
.add_argument(
5599 action
='store_true', default
=False,
5600 help='option to erase data and partition',
5602 destroy_parser
.add_argument(
5604 action
='store_true', default
=False,
5605 help='option to remove OSD from CRUSH map and deallocate the id',
5607 destroy_parser
.set_defaults(
5612 def make_zap_parser(subparsers
):
5613 zap_parser
= subparsers
.add_parser(
5615 formatter_class
=argparse
.RawDescriptionHelpFormatter
,
5616 description
=textwrap
.fill(textwrap
.dedent("""\
5617 Zap/erase/destroy a device's partition table and contents. It
5618 actually uses sgdisk and it's option --zap-all to
5619 destroy both GPT and MBR data structures so that the disk
5620 becomes suitable for repartitioning.
5622 help='Zap/erase/destroy a device\'s partition table (and contents)')
5623 zap_parser
.add_argument(
5627 help='path to block device',
5629 zap_parser
.set_defaults(
5636 args
= parse_args(argv
)
5638 setup_logging(args
.verbose
, args
.log_stdout
)
5640 if args
.prepend_to_path
!= '':
5641 path
= os
.environ
.get('PATH', os
.defpath
)
5642 os
.environ
['PATH'] = args
.prepend_to_path
+ ":" + path
5644 if args
.func
.__name
__ != 'main_trigger':
5645 # trigger may run when statedir is unavailable and does not use it
5646 setup_statedir(args
.statedir
)
5647 setup_sysconfdir(args
.sysconfdir
)
5649 global CEPH_PREF_USER
5650 CEPH_PREF_USER
= args
.setuser
5651 global CEPH_PREF_GROUP
5652 CEPH_PREF_GROUP
= args
.setgroup
5657 main_catch(args
.func
, args
)
5660 def setup_logging(verbose
, log_stdout
):
5661 loglevel
= logging
.WARNING
5663 loglevel
= logging
.DEBUG
5666 ch
= logging
.StreamHandler(stream
=sys
.stdout
)
5667 ch
.setLevel(loglevel
)
5668 formatter
= logging
.Formatter('%(funcName)s: %(message)s')
5669 ch
.setFormatter(formatter
)
5671 LOG
.setLevel(loglevel
)
5673 logging
.basicConfig(
5675 format
='%(funcName)s: %(message)s',
5679 def main_catch(func
, args
):
5686 '{prog}: {msg}'.format(
5692 except CephDiskException
as error
:
5693 exc_name
= error
.__class
__.__name
__
5695 '{prog} {exc_name}: {msg}'.format(
5707 if __name__
== '__main__':