3 # Copyright (C) 2015, 2016, 2017 Red Hat <contact@redhat.com>
4 # Copyright (C) 2014 Inktank <info@inktank.com>
5 # Copyright (C) 2014 Cloudwatt <libre.licensing@cloudwatt.com>
6 # Copyright (C) 2014 Catalyst.net Ltd
8 # Author: Loic Dachary <loic@dachary.org>
10 # This program is free software; you can redistribute it and/or modify
11 # it under the terms of the GNU Library Public License as published by
12 # the Free Software Foundation; either version 2, or (at your option)
15 # This program is distributed in the hope that it will be useful,
16 # but WITHOUT ANY WARRANTY; without even the implied warranty of
17 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18 # GNU Library Public License for more details.
21 from __future__
import print_function
45 CEPH_OSD_ONDISK_MAGIC
= 'ceph osd volume v026'
46 CEPH_LOCKBOX_ONDISK_MAGIC
= 'ceph lockbox volume v001'
48 KEY_MANAGEMENT_MODE_V1
= 'ceph-mon v1'
53 # identical because creating a journal is atomic
54 'ready': '45b0969e-9b03-4f30-b4c6-b4b80ceff106',
55 'tobe': '45b0969e-9b03-4f30-b4c6-b4b80ceff106',
58 # identical because creating a block is atomic
59 'ready': 'cafecafe-9b03-4f30-b4c6-b4b80ceff106',
60 'tobe': 'cafecafe-9b03-4f30-b4c6-b4b80ceff106',
63 # identical because creating a block is atomic
64 'ready': '30cd0809-c2b2-499c-8879-2d6b78529876',
65 'tobe': '30cd0809-c2b2-499c-8879-2d6b785292be',
68 # identical because creating a block is atomic
69 'ready': '5ce17fce-4087-4169-b7ff-056cc58473f9',
70 'tobe': '5ce17fce-4087-4169-b7ff-056cc58472be',
73 'ready': '4fbd7e29-9d25-41b8-afd0-062c0ceff05d',
74 'tobe': '89c57f98-2fe5-4dc0-89c1-f3ad0ceff2be',
77 'ready': 'fb3aabf9-d25f-47cc-bf5e-721d1816496b',
78 'tobe': 'fb3aabf9-d25f-47cc-bf5e-721d181642be',
83 'ready': '45b0969e-9b03-4f30-b4c6-35865ceff106',
84 'tobe': '89c57f98-2fe5-4dc0-89c1-35865ceff2be',
87 'ready': 'cafecafe-9b03-4f30-b4c6-35865ceff106',
88 'tobe': '89c57f98-2fe5-4dc0-89c1-35865ceff2be',
91 'ready': '166418da-c469-4022-adf4-b30afd37f176',
92 'tobe': '7521c784-4626-4260-bc8d-ba77a0f5f2be',
95 'ready': '86a32090-3647-40b9-bbbd-38d8c573aa86',
96 'tobe': '92dad30f-175b-4d40-a5b0-5c0a258b42be',
99 'ready': '4fbd7e29-9d25-41b8-afd0-35865ceff05d',
100 'tobe': '89c57f98-2fe5-4dc0-89c1-5ec00ceff2be',
105 'ready': '45b0969e-9b03-4f30-b4c6-5ec00ceff106',
106 'tobe': '89c57f98-2fe5-4dc0-89c1-35865ceff2be',
109 'ready': 'cafecafe-9b03-4f30-b4c6-5ec00ceff106',
110 'tobe': '89c57f98-2fe5-4dc0-89c1-35865ceff2be',
113 'ready': '93b0052d-02d9-4d8a-a43b-33a3ee4dfbc3',
114 'tobe': '69d17c68-3e58-4399-aff0-b68265f2e2be',
117 'ready': '306e8683-4fe2-4330-b7c0-00a917c16966',
118 'tobe': 'f2d89683-a621-4063-964a-eb1f7863a2be',
121 'ready': '4fbd7e29-9d25-41b8-afd0-5ec00ceff05d',
122 'tobe': '89c57f98-2fe5-4dc0-89c1-5ec00ceff2be',
127 'ready': '45b0969e-8ae0-4982-bf9d-5a8d867af560',
128 'tobe': '45b0969e-8ae0-4982-bf9d-5a8d867af560',
131 'ready': 'cafecafe-8ae0-4982-bf9d-5a8d867af560',
132 'tobe': 'cafecafe-8ae0-4982-bf9d-5a8d867af560',
135 'ready': 'ec6d6385-e346-45dc-be91-da2a7c8b3261',
136 'tobe': 'ec6d6385-e346-45dc-be91-da2a7c8b32be',
139 'ready': '01b41e1b-002a-453c-9f17-88793989ff8f',
140 'tobe': '01b41e1b-002a-453c-9f17-88793989f2be',
143 'ready': '4fbd7e29-8ae0-4982-bf9d-5a8d867af560',
144 'tobe': '89c57f98-8ae0-4982-bf9d-5a8d867af560',
147 'ready': '7f4a666a-16f3-47a2-8445-152ef4d03f6c',
148 'tobe': '7f4a666a-16f3-47a2-8445-152ef4d032be',
157 def get_ready_by_type(what
):
158 return [x
['ready'] for x
in PTYPE
[what
].values()]
161 def get_ready_by_name(name
):
162 return [x
[name
]['ready'] for x
in PTYPE
.values() if name
in x
]
165 def is_regular_space(ptype
):
166 return Ptype
.is_what_space('regular', ptype
)
169 def is_mpath_space(ptype
):
170 return Ptype
.is_what_space('mpath', ptype
)
173 def is_plain_space(ptype
):
174 return Ptype
.is_what_space('plain', ptype
)
177 def is_luks_space(ptype
):
178 return Ptype
.is_what_space('luks', ptype
)
181 def is_what_space(what
, ptype
):
182 for name
in Space
.NAMES
:
183 if ptype
== PTYPE
[what
][name
]['ready']:
188 def space_ptype_to_name(ptype
):
189 for what
in PTYPE
.values():
190 for name
in Space
.NAMES
:
191 if ptype
== what
[name
]['ready']:
193 raise ValueError('ptype ' + ptype
+ ' not found')
196 def is_dmcrypt_space(ptype
):
197 for name
in Space
.NAMES
:
198 if Ptype
.is_dmcrypt(ptype
, name
):
203 def is_dmcrypt(ptype
, name
):
204 for what
in ('plain', 'luks'):
205 if ptype
== PTYPE
[what
][name
]['ready']:
212 if platform
.system() == 'FreeBSD':
214 DEFAULT_FS_TYPE
= 'zfs'
215 PROCDIR
= '/compat/linux/proc'
216 # FreeBSD does not have blockdevices any more
221 DEFAULT_FS_TYPE
= 'xfs'
223 BLOCKDIR
= '/sys/block'
227 OSD STATUS Definition
229 OSD_STATUS_OUT_DOWN
= 0
230 OSD_STATUS_OUT_UP
= 1
231 OSD_STATUS_IN_DOWN
= 2
234 MOUNT_OPTIONS
= dict(
235 btrfs
='noatime,user_subvol_rm_allowed',
236 # user_xattr is default ever since linux 2.6.39 / 3.0, but we'll
237 # delay a moment before removing it fully because we did have some
238 # issues with ext4 before the xatts-in-leveldb work, and it seemed
239 # that user_xattr helped
240 ext4
='noatime,user_xattr',
241 xfs
='noatime,inode64',
246 # btrfs requires -f, for the same reason as xfs (see comment below)
253 # xfs insists on not overwriting previous fs; even if we wipe
254 # partition table, we often recreate it exactly the same way,
255 # so we'll see ghosts of filesystems past
274 STATEDIR
= '/var/lib/ceph'
276 SYSCONFDIR
= '/etc/ceph'
280 SUPPRESS_PREFIX
= None
282 # only warn once about some things
285 # Nuke the TERM variable to avoid confusing any subprocesses we call.
286 # For example, libreadline will print weird control sequences for some
288 if 'TERM' in os
.environ
:
289 del os
.environ
['TERM']
292 if LOG_NAME
== '__main__':
293 LOG_NAME
= os
.path
.basename(sys
.argv
[0])
294 LOG
= logging
.getLogger(LOG_NAME
)
296 # Allow user-preferred values for subprocess user and group
297 CEPH_PREF_USER
= None
298 CEPH_PREF_GROUP
= None
301 class FileLock(object):
302 def __init__(self
, fn
):
308 self
.fd
= os
.open(self
.fn
, os
.O_WRONLY | os
.O_CREAT
)
309 fcntl
.lockf(self
.fd
, fcntl
.LOCK_EX
)
311 def __exit__(self
, exc_type
, exc_val
, exc_tb
):
313 fcntl
.lockf(self
.fd
, fcntl
.LOCK_UN
)
318 class Error(Exception):
324 doc
= _bytes2str(self
.__doc
__.strip())
326 str_type
= basestring
329 args
= [a
if isinstance(a
, str_type
) else str(a
) for a
in self
.args
]
330 return ': '.join([doc
] + [_bytes2str(a
) for a
in args
])
333 class MountError(Error
):
335 Mounting filesystem failed
339 class UnmountError(Error
):
341 Unmounting filesystem failed
345 class BadMagicError(Error
):
347 Does not look like a Ceph OSD, or incompatible version
351 class TruncatedLineError(Error
):
357 class TooManyLinesError(Error
):
363 class FilesystemTypeError(Error
):
365 Cannot discover filesystem type
369 class CephDiskException(Exception):
371 A base exception for ceph-disk to provide custom (ad-hoc) messages that
372 will be caught and dealt with when main() is executed
377 class ExecutableNotFound(CephDiskException
):
379 Exception to report on executables not available in PATH
386 Detect whether systemd is running
388 with
open(PROCDIR
+ '/1/comm', 'r') as f
:
389 return 'systemd' in f
.read()
394 Detect whether upstart is running
396 (out
, err
, _
) = command(['init', '--version'])
397 return 'upstart' in out
400 def maybe_mkdir(*a
, **kw
):
402 Creates a new directory if it doesn't exist, removes
403 existing symlink before creating the directory.
405 # remove any symlink, if it is there..
406 if os
.path
.exists(*a
) and stat
.S_ISLNK(os
.lstat(*a
).st_mode
):
407 LOG
.debug('Removing old symlink at %s', *a
)
412 if e
.errno
== errno
.EEXIST
:
418 def which(executable
):
419 """find the location of an executable"""
420 envpath
= os
.environ
.get('PATH') or os
.defpath
421 PATH
= envpath
.split(os
.pathsep
)
432 for location
in locations
:
433 executable_path
= os
.path
.join(location
, executable
)
434 if (os
.path
.isfile(executable_path
) and
435 os
.access(executable_path
, os
.X_OK
)):
436 return executable_path
439 def _get_command_executable(arguments
):
441 Return the full path for an executable, raise if the executable is not
442 found. If the executable has already a full path do not perform any checks.
444 if os
.path
.isabs(arguments
[0]): # an absolute path
446 executable
= which(arguments
[0])
448 command_msg
= 'Could not run command: %s' % ' '.join(arguments
)
449 executable_msg
= '%s not in path.' % arguments
[0]
450 raise ExecutableNotFound('%s %s' % (executable_msg
, command_msg
))
452 # swap the old executable for the new one
453 arguments
[0] = executable
457 def command(arguments
, **kwargs
):
459 Safely execute a ``subprocess.Popen`` call making sure that the
460 executable exists and raising a helpful error message
463 .. note:: This should be the preferred way of calling ``subprocess.Popen``
464 since it provides the caller with the safety net of making sure that
465 executables *will* be found and will error nicely otherwise.
467 This returns the output of the command and the return code of the
468 process in a tuple: (stdout, stderr, returncode).
471 arguments
= list(map(_bytes2str
, _get_command_executable(arguments
)))
473 LOG
.info('Running command: %s' % ' '.join(arguments
))
474 process
= subprocess
.Popen(
476 stdout
=subprocess
.PIPE
,
477 stderr
=subprocess
.PIPE
,
479 out
, err
= process
.communicate()
481 return _bytes2str(out
), _bytes2str(err
), process
.returncode
484 def command_with_stdin(arguments
, stdin
):
485 LOG
.info("Running command with stdin: " + " ".join(arguments
))
486 process
= subprocess
.Popen(
488 stdin
=subprocess
.PIPE
,
489 stdout
=subprocess
.PIPE
,
490 stderr
=subprocess
.PIPE
)
491 out
, err
= process
.communicate(stdin
)
493 if process
.returncode
!= 0:
496 "'{cmd}' failed with status code {returncode}".format(
498 returncode
=process
.returncode
,
504 def _bytes2str(string
):
505 return string
.decode('utf-8') if isinstance(string
, bytes
) else string
508 def command_init(arguments
, **kwargs
):
510 Safely execute a non-blocking ``subprocess.Popen`` call
511 making sure that the executable exists and raising a helpful
512 error message if it does not.
514 .. note:: This should be the preferred way of calling ``subprocess.Popen``
515 since it provides the caller with the safety net of making sure that
516 executables *will* be found and will error nicely otherwise.
518 This returns the process.
521 arguments
= list(map(_bytes2str
, _get_command_executable(arguments
)))
523 LOG
.info('Running command: %s' % ' '.join(arguments
))
524 process
= subprocess
.Popen(
526 stdout
=subprocess
.PIPE
,
527 stderr
=subprocess
.PIPE
,
532 def command_wait(process
):
534 Wait for the process finish and parse its output.
537 out
, err
= process
.communicate()
539 return _bytes2str(out
), _bytes2str(err
), process
.returncode
542 def command_check_call(arguments
, exit
=False):
544 Safely execute a ``subprocess.check_call`` call making sure that the
545 executable exists and raising a helpful error message if it does not.
547 When ``exit`` is set to ``True`` this helper will do a clean (sans
548 traceback) system exit.
549 .. note:: This should be the preferred way of calling
550 ``subprocess.check_call`` since it provides the caller with the safety net
551 of making sure that executables *will* be found and will error nicely
554 arguments
= _get_command_executable(arguments
)
555 command
= ' '.join(arguments
)
556 LOG
.info('Running command: %s', command
)
558 return subprocess
.check_call(arguments
)
559 except subprocess
.CalledProcessError
as error
:
562 LOG
.error(error
.output
)
564 "'{cmd}' failed with status code {returncode}".format(
566 returncode
=error
.returncode
,
573 # An alternative block_path implementation would be
575 # name = basename(dev)
576 # return /sys/devices/virtual/block/$name
578 # It is however more fragile because it relies on the fact
579 # that the basename of the device the user will use always
580 # matches the one the driver will use. On Ubuntu 14.04, for
581 # instance, when multipath creates a partition table on
583 # /dev/mapper/353333330000007d0 -> ../dm-0
585 # it will create partition devices named
587 # /dev/mapper/353333330000007d0-part1
589 # which is the same device as /dev/dm-1 but not a symbolic
592 # ubuntu@other:~$ ls -l /dev/mapper /dev/dm-1
593 # brw-rw---- 1 root disk 252, 1 Aug 15 17:52 /dev/dm-1
594 # lrwxrwxrwx 1 root root 7 Aug 15 17:52 353333330000007d0 -> ../dm-0
595 # brw-rw---- 1 root disk 252, 1 Aug 15 17:52 353333330000007d0-part1
597 # Using the basename in this case fails.
604 path
= os
.path
.realpath(dev
)
605 rdev
= os
.stat(path
).st_rdev
606 (M
, m
) = (os
.major(rdev
), os
.minor(rdev
))
607 return "{sysfs}/dev/block/{M}:{m}".format(sysfs
=SYSFS
, M
=M
, m
=m
)
610 def get_dm_uuid(dev
):
611 uuid_path
= os
.path
.join(block_path(dev
), 'dm', 'uuid')
612 LOG
.debug("get_dm_uuid " + dev
+ " uuid path is " + uuid_path
)
613 if not os
.path
.exists(uuid_path
):
615 uuid
= open(uuid_path
, 'r').read()
616 LOG
.debug("get_dm_uuid " + dev
+ " uuid is " + uuid
)
622 True if the path is managed by multipath
626 uuid
= get_dm_uuid(dev
)
628 (re
.match('part\d+-mpath-', uuid
) or
629 re
.match('mpath-', uuid
)))
632 def get_dev_name(path
):
634 get device name from path. e.g.::
636 /dev/sda -> sda, /dev/cciss/c0d1 -> cciss!c0d1
638 a device "name" is something like::
644 assert path
.startswith('/dev/')
646 return base
.replace('/', '!')
649 def get_dev_path(name
):
651 get a path (/dev/...) from a name (cciss!c0d1)
652 a device "path" is something like::
658 return '/dev/' + name
.replace('!', '/')
661 def get_dev_relpath(name
):
663 get a relative path to /dev from a name (cciss!c0d1)
665 return name
.replace('!', '/')
668 def get_dev_size(dev
, size
='megabytes'):
670 Attempt to get the size of a device so that we can prevent errors
671 from actions to devices that are smaller, and improve error reporting.
673 Because we want to avoid breakage in case this approach is not robust, we
674 will issue a warning if we failed to get the size.
676 :param size: bytes or megabytes
677 :param dev: the device to calculate the size
679 fd
= os
.open(dev
, os
.O_RDONLY
)
680 dividers
= {'bytes': 1, 'megabytes': 1024 * 1024}
682 device_size
= os
.lseek(fd
, 0, os
.SEEK_END
)
683 divider
= dividers
.get(size
, 1024 * 1024) # default to megabytes
684 return device_size
// divider
685 except Exception as error
:
686 LOG
.warning('failed to get size of %s: %s' % (dev
, str(error
)))
691 def stmode_is_diskdevice(dmode
):
692 if stat
.S_ISBLK(dmode
):
695 # FreeBSD does not have block devices
696 # All disks are character devices
697 return FREEBSD
and stat
.S_ISCHR(dmode
)
700 def dev_is_diskdevice(dev
):
701 dmode
= os
.stat(dev
).st_mode
702 return stmode_is_diskdevice(dmode
)
705 def ldev_is_diskdevice(dev
):
706 dmode
= os
.lstat(dev
).st_mode
707 return stmode_is_diskdevice(dmode
)
710 def path_is_diskdevice(path
):
711 dev
= os
.path
.realpath(path
)
712 return dev_is_diskdevice(dev
)
715 def get_partition_mpath(dev
, pnum
):
716 part_re
= "part{pnum}-mpath-".format(pnum
=pnum
)
717 partitions
= list_partitions_mpath(dev
, part_re
)
724 def get_partition_dev(dev
, pnum
):
726 get the device name for a partition
728 assume that partitions are named like the base dev,
729 with a number, and optionally
730 some intervening characters (like 'p'). e.g.,
733 cciss/c0d1 1 -> cciss!c0d1p1
736 for retry
in range(0, max_retry
+ 1):
740 partname
= get_partition_mpath(dev
, pnum
)
742 name
= get_dev_name(os
.path
.realpath(dev
))
743 sys_entry
= os
.path
.join(BLOCKDIR
, name
)
744 error_msg
= " in %s" % sys_entry
745 for f
in os
.listdir(sys_entry
):
746 if f
.startswith(name
) and f
.endswith(str(pnum
)):
747 # we want the shortest name that starts with the base name
748 # and ends with the partition number
749 if not partname
or len(f
) < len(partname
):
753 LOG
.info('Found partition %d for %s after %d tries' %
755 return get_dev_path(partname
)
757 if retry
< max_retry
:
758 LOG
.info('Try %d/%d : partition %d for %s does not exist%s' %
759 (retry
+ 1, max_retry
, pnum
, dev
, error_msg
))
763 raise Error('partition %d for %s does not appear to exist%s' %
764 (pnum
, dev
, error_msg
))
767 def list_all_partitions():
769 Return a list of devices and partitions
772 names
= os
.listdir(BLOCKDIR
)
775 # /dev/fd0 may hang http://tracker.ceph.com/issues/6827
776 if re
.match(r
'^fd\d$', name
):
778 dev_part_list
[name
] = list_partitions(get_dev_path(name
))
780 with
open(os
.path
.join(PROCDIR
, "partitions")) as partitions
:
781 for line
in partitions
:
782 columns
= line
.split()
783 if len(columns
) >= 4:
785 dev_part_list
[name
] = list_partitions(get_dev_path(name
))
789 def list_partitions(dev
):
790 dev
= os
.path
.realpath(dev
)
792 return list_partitions_mpath(dev
)
794 return list_partitions_device(dev
)
797 def list_partitions_mpath(dev
, part_re
="part\d+-mpath-"):
800 holders
= os
.path
.join(p
, 'holders')
801 for holder
in os
.listdir(holders
):
802 uuid_path
= os
.path
.join(holders
, holder
, 'dm', 'uuid')
803 uuid
= open(uuid_path
, 'r').read()
804 LOG
.debug("list_partitions_mpath: " + uuid_path
+ " uuid = " + uuid
)
805 if re
.match(part_re
, uuid
):
806 partitions
.append(holder
)
810 def list_partitions_device(dev
):
812 Return a list of partitions on the given device name
815 basename
= get_dev_name(dev
)
816 for name
in os
.listdir(block_path(dev
)):
817 if name
.startswith(basename
):
818 partitions
.append(name
)
822 def get_partition_base(dev
):
824 Get the base device for a partition
826 dev
= os
.path
.realpath(dev
)
827 if not ldev_is_diskdevice(dev
):
828 raise Error('not a block device', dev
)
830 name
= get_dev_name(dev
)
831 if os
.path
.exists(os
.path
.join('/sys/block', name
)):
832 raise Error('not a partition', dev
)
835 for basename
in os
.listdir('/sys/block'):
836 if os
.path
.exists(os
.path
.join('/sys/block', basename
, name
)):
837 return get_dev_path(basename
)
838 raise Error('no parent device for partition', dev
)
841 def is_partition_mpath(dev
):
842 uuid
= get_dm_uuid(dev
)
843 return bool(re
.match('part\d+-mpath-', uuid
))
846 def partnum_mpath(dev
):
847 uuid
= get_dm_uuid(dev
)
848 return re
.findall('part(\d+)-mpath-', uuid
)[0]
851 def get_partition_base_mpath(dev
):
852 slave_path
= os
.path
.join(block_path(dev
), 'slaves')
853 slaves
= os
.listdir(slave_path
)
855 name_path
= os
.path
.join(slave_path
, slaves
[0], 'dm', 'name')
856 name
= open(name_path
, 'r').read().strip()
857 return os
.path
.join('/dev/mapper', name
)
860 def is_partition(dev
):
862 Check whether a given device path is a partition or a full disk.
865 return is_partition_mpath(dev
)
867 dev
= os
.path
.realpath(dev
)
869 if not stmode_is_diskdevice(st
.st_mode
):
870 raise Error('not a block device', dev
)
872 name
= get_dev_name(dev
)
873 if os
.path
.exists(os
.path
.join(BLOCKDIR
, name
)):
876 # make sure it is a partition of something else
877 major
= os
.major(st
.st_rdev
)
878 minor
= os
.minor(st
.st_rdev
)
879 if os
.path
.exists('/sys/dev/block/%d:%d/partition' % (major
, minor
)):
882 raise Error('not a disk or partition', dev
)
887 Check if the given device is mounted.
889 dev
= os
.path
.realpath(dev
)
890 with
open(PROCDIR
+ '/mounts', 'rb') as proc_mounts
:
891 for line
in proc_mounts
:
892 fields
= line
.split()
895 mounts_dev
= fields
[0]
897 if os
.path
.isabs(mounts_dev
) and os
.path
.exists(mounts_dev
):
898 mounts_dev
= os
.path
.realpath(mounts_dev
)
899 if mounts_dev
== dev
:
900 return _bytes2str(path
)
906 Check if a device is held by another device (e.g., a dm-crypt mapping)
908 assert os
.path
.exists(dev
)
912 dev
= os
.path
.realpath(dev
)
913 base
= get_dev_name(dev
)
916 directory
= '/sys/block/{base}/holders'.format(base
=base
)
917 if os
.path
.exists(directory
):
918 return os
.listdir(directory
)
923 directory
= '/sys/block/{base}/{part}/holders'.format(
924 part
=part
, base
=base
)
925 if os
.path
.exists(directory
):
926 return os
.listdir(directory
)
931 def verify_not_in_use(dev
, check_partitions
=False):
933 Verify if a given device (path) is in use (e.g. mounted or
934 in use by device-mapper).
936 :raises: Error if device is in use.
938 assert os
.path
.exists(dev
)
940 raise Error('Device is mounted', dev
)
941 holders
= is_held(dev
)
943 raise Error('Device %s is in use by a device-mapper '
944 'mapping (dm-crypt?)' % dev
, ','.join(holders
))
946 if check_partitions
and not is_partition(dev
):
947 for partname
in list_partitions(dev
):
948 partition
= get_dev_path(partname
)
949 if is_mounted(partition
):
950 raise Error('Device is mounted', partition
)
951 holders
= is_held(partition
)
953 raise Error('Device %s is in use by a device-mapper '
954 'mapping (dm-crypt?)'
955 % partition
, ','.join(holders
))
958 def must_be_one_line(line
):
960 Checks if given line is really one single line.
962 :raises: TruncatedLineError or TooManyLinesError
963 :return: Content of the line, or None if line isn't valid.
965 line
= _bytes2str(line
)
967 if line
[-1:] != '\n':
968 raise TruncatedLineError(line
)
971 raise TooManyLinesError(line
)
975 def read_one_line(parent
, name
):
977 Read a file whose sole contents are a single line.
981 :return: Contents of the line, or None if file did not exist.
983 path
= os
.path
.join(parent
, name
)
985 line
= open(path
, 'rb').read()
987 if e
.errno
== errno
.ENOENT
:
993 line
= must_be_one_line(line
)
994 except (TruncatedLineError
, TooManyLinesError
) as e
:
996 'File is corrupt: {path}: {msg}'.format(
1004 def write_one_line(parent
, name
, text
):
1006 Write a file whose sole contents are a single line.
1010 path
= os
.path
.join(parent
, name
)
1011 tmp
= '{path}.{pid}.tmp'.format(path
=path
, pid
=os
.getpid())
1012 with
open(tmp
, 'wb') as tmp_file
:
1013 tmp_file
.write(text
.encode('utf-8') + b
'\n')
1014 os
.fsync(tmp_file
.fileno())
1015 path_set_context(tmp
)
1016 os
.rename(tmp
, path
)
1021 Get a init system using 'ceph-detect-init'
1023 init
= _check_output(
1026 '--default', 'sysvinit',
1029 init
= must_be_one_line(init
)
1033 def check_osd_magic(path
):
1035 Check that this path has the Ceph OSD magic.
1037 :raises: BadMagicError if this does not look like a Ceph OSD data
1040 magic
= read_one_line(path
, 'magic')
1042 # probably not mkfs'ed yet
1043 raise BadMagicError(path
)
1044 if magic
!= CEPH_OSD_ONDISK_MAGIC
:
1045 raise BadMagicError(path
)
1048 def check_osd_id(osd_id
):
1050 Ensures osd id is numeric.
1052 if not re
.match(r
'^[0-9]+$', osd_id
):
1053 raise Error('osd id is not numeric', osd_id
)
1056 def allocate_osd_id(
1063 Allocates an OSD id on the given cluster.
1065 :raises: Error if the call to allocate the OSD id fails.
1066 :return: The allocated OSD id.
1068 lockbox_path
= os
.path
.join(STATEDIR
, 'osd-lockbox', fsid
)
1069 lockbox_osd_id
= read_one_line(lockbox_path
, 'whoami')
1070 osd_keyring
= os
.path
.join(path
, 'keyring')
1072 LOG
.debug('Getting OSD id from Lockbox...')
1073 osd_id
= lockbox_osd_id
1074 shutil
.move(os
.path
.join(lockbox_path
, 'osd_keyring'),
1076 path_set_context(osd_keyring
)
1077 os
.unlink(os
.path
.join(lockbox_path
, 'whoami'))
1080 LOG
.debug('Allocating OSD id...')
1083 wanttobe
= read_one_line(path
, 'wanttobe')
1084 if os
.path
.exists(os
.path
.join(path
, 'wanttobe')):
1085 os
.unlink(os
.path
.join(path
, 'wanttobe'))
1086 id_arg
= wanttobe
and [wanttobe
] or []
1087 osd_id
= command_with_stdin(
1090 '--cluster', cluster
,
1091 '--name', 'client.bootstrap-osd',
1092 '--keyring', keyring
,
1099 except subprocess
.CalledProcessError
as e
:
1100 raise Error('ceph osd create failed', e
, e
.output
)
1101 osd_id
= must_be_one_line(osd_id
)
1102 check_osd_id(osd_id
)
1103 secrets
.write_osd_keyring(osd_keyring
, osd_id
)
1107 def get_osd_id(path
):
1109 Gets the OSD id of the OSD at the given path.
1111 osd_id
= read_one_line(path
, 'whoami')
1112 if osd_id
is not None:
1113 check_osd_id(osd_id
)
1117 def get_ceph_user():
1118 global CEPH_PREF_USER
1120 if CEPH_PREF_USER
is not None:
1122 pwd
.getpwnam(CEPH_PREF_USER
)
1123 return CEPH_PREF_USER
1125 print("No such user:", CEPH_PREF_USER
)
1129 pwd
.getpwnam('ceph')
1135 def get_ceph_group():
1136 global CEPH_PREF_GROUP
1138 if CEPH_PREF_GROUP
is not None:
1140 grp
.getgrnam(CEPH_PREF_GROUP
)
1141 return CEPH_PREF_GROUP
1143 print("No such group:", CEPH_PREF_GROUP
)
1147 grp
.getgrnam('ceph')
1153 def path_set_context(path
):
1154 # restore selinux context to default policy values
1155 if which('restorecon'):
1156 command(['restorecon', '-R', path
])
1158 # if ceph user exists, set owner to ceph
1159 if get_ceph_user() == 'ceph':
1160 command(['chown', '-R', 'ceph:ceph', path
])
1163 def _check_output(args
=None, **kwargs
):
1164 out
, err
, ret
= command(args
, **kwargs
)
1167 error
= subprocess
.CalledProcessError(ret
, cmd
)
1168 error
.output
= out
+ err
1170 return _bytes2str(out
)
1173 def get_conf(cluster
, variable
):
1175 Get the value of the given configuration variable from the
1178 :raises: Error if call to ceph-conf fails.
1179 :return: The variable value or None.
1182 out
, err
, ret
= command(
1185 '--cluster={cluster}'.format(
1194 except OSError as e
:
1195 raise Error('error executing ceph-conf', e
, err
)
1197 # config entry not found
1200 raise Error('getting variable from configuration failed')
1201 value
= out
.split('\n', 1)[0]
1202 # don't differentiate between "var=" and no var set
1208 def get_conf_with_default(cluster
, variable
):
1210 Get a config value that is known to the C++ code.
1212 This will fail if called on variables that are not defined in
1213 common config options.
1216 out
= _check_output(
1219 '--cluster={cluster}'.format(
1222 '--show-config-value={variable}'.format(
1228 except subprocess
.CalledProcessError
as e
:
1230 'getting variable from configuration failed',
1234 value
= str(out
).split('\n', 1)[0]
1238 def get_fsid(cluster
):
1240 Get the fsid of the cluster.
1242 :return: The fsid or raises Error.
1244 fsid
= get_conf_with_default(cluster
=cluster
, variable
='fsid')
1246 raise Error('getting cluster uuid from configuration failed')
1250 def get_dmcrypt_key_path(
1256 Get path to dmcrypt key file.
1258 :return: Path to the dmcrypt key file, callers should check for existence.
1261 path
= os
.path
.join(key_dir
, _uuid
+ ".luks.key")
1263 path
= os
.path
.join(key_dir
, _uuid
)
1268 def get_dmcrypt_key(
1273 legacy_path
= get_dmcrypt_key_path(_uuid
, key_dir
, luks
)
1274 if os
.path
.exists(legacy_path
):
1275 return (legacy_path
,)
1276 path
= os
.path
.join(STATEDIR
, 'osd-lockbox', _uuid
)
1277 if os
.path
.exists(path
):
1278 mode
= get_oneliner(path
, 'key-management-mode')
1279 osd_uuid
= get_oneliner(path
, 'osd-uuid')
1280 ceph_fsid
= read_one_line(path
, 'ceph_fsid')
1281 if ceph_fsid
is None:
1282 raise Error('No cluster uuid assigned.')
1283 cluster
= find_cluster_by_uuid(ceph_fsid
)
1285 raise Error('No cluster conf found in ' + SYSCONFDIR
+
1286 ' with fsid %s' % ceph_fsid
)
1288 if mode
== KEY_MANAGEMENT_MODE_V1
:
1289 key
, stderr
, ret
= command(
1292 '--cluster', cluster
,
1294 'client.osd-lockbox.' + osd_uuid
,
1296 os
.path
.join(path
, 'keyring'),
1299 'dm-crypt/osd/' + osd_uuid
+ '/luks',
1302 LOG
.debug("stderr " + stderr
)
1304 return base64
.b64decode(key
)
1306 raise Error('unknown key-management-mode ' + str(mode
))
1307 raise Error('unable to read dm-crypt key', path
, legacy_path
)
1314 cryptsetup_parameters
,
1318 dev
= dmcrypt_is_mapped(_uuid
)
1322 if isinstance(key
, tuple):
1323 # legacy, before lockbox
1324 assert os
.path
.exists(key
[0])
1329 dev
= '/dev/mapper/' + _uuid
1337 ] + cryptsetup_parameters
1355 ] + cryptsetup_parameters
1360 command_with_stdin(luksFormat_args
, key
)
1361 command_with_stdin(luksOpen_args
, key
)
1363 # Plain mode has no format function, nor any validation
1364 # that the key is correct.
1365 command_with_stdin(create_args
, key
)
1366 # set proper ownership of mapped device
1367 command_check_call(['chown', 'ceph:ceph', dev
])
1370 except subprocess
.CalledProcessError
as e
:
1371 raise Error('unable to map device', rawdev
, e
)
1377 if not os
.path
.exists('/dev/mapper/' + _uuid
):
1382 command_check_call(['cryptsetup', 'remove', _uuid
])
1384 except subprocess
.CalledProcessError
as e
:
1386 raise Error('unable to unmap device', _uuid
, e
)
1388 time
.sleep(0.5 + retries
* 1.0)
1398 Mounts a device with given filessystem type and
1399 mount options to a tempfile path under /var/lib/ceph/tmp.
1401 # sanity check: none of the arguments are None
1403 raise ValueError('dev may not be None')
1405 raise ValueError('fstype may not be None')
1407 # pick best-of-breed mount options based on fs type
1409 options
= MOUNT_OPTIONS
.get(fstype
, '')
1411 myTemp
= STATEDIR
+ '/tmp'
1412 # mkdtemp expect 'dir' to be existing on the system
1413 # Let's be sure it's always the case
1414 if not os
.path
.exists(myTemp
):
1418 path
= tempfile
.mkdtemp(
1423 LOG
.debug('Mounting %s on %s with options %s', dev
, path
, options
)
1434 if which('restorecon'):
1441 except subprocess
.CalledProcessError
as e
:
1444 except (OSError, IOError):
1455 Unmount and removes the given mount point.
1460 LOG
.debug('Unmounting %s', path
)
1469 except subprocess
.CalledProcessError
as e
:
1470 # on failure, retry 3 times with incremental backoff
1472 raise UnmountError(e
)
1474 time
.sleep(0.5 + retries
* 1.0)
1480 ###########################################
1482 def extract_parted_partition_numbers(partitions
):
1483 numbers_as_strings
= re
.findall('^\d+', partitions
, re
.MULTILINE
)
1484 return map(int, numbers_as_strings
)
1487 def get_free_partition_index(dev
):
1489 Get the next free partition index on a given device.
1491 :return: Index number (> 1 if there is already a partition on the device)
1492 or 1 if there is no partition table.
1495 lines
= _check_output(
1504 except subprocess
.CalledProcessError
as e
:
1505 LOG
.info('cannot read partition index; assume it '
1506 'isn\'t present\n (Error: %s)' % e
)
1510 raise Error('parted failed to output anything')
1511 LOG
.debug('get_free_partition_index: analyzing ' + lines
)
1512 if ('CHS;' not in lines
and
1513 'CYL;' not in lines
and
1514 'BYT;' not in lines
):
1515 raise Error('parted output expected to contain one of ' +
1516 'CHH; CYL; or BYT; : ' + lines
)
1517 if os
.path
.realpath(dev
) not in lines
:
1518 raise Error('parted output expected to contain ' + dev
+ ': ' + lines
)
1519 _
, partitions
= lines
.split(os
.path
.realpath(dev
))
1520 partition_numbers
= extract_parted_partition_numbers(partitions
)
1521 if partition_numbers
:
1522 return max(partition_numbers
) + 1
1527 def check_journal_reqs(args
):
1528 _
, _
, allows_journal
= command([
1529 'ceph-osd', '--check-allows-journal',
1531 '--log-file', '$run_dir/$cluster-osd-check.log',
1532 '--cluster', args
.cluster
,
1533 '--setuser', get_ceph_user(),
1534 '--setgroup', get_ceph_group(),
1536 _
, _
, wants_journal
= command([
1537 'ceph-osd', '--check-wants-journal',
1539 '--log-file', '$run_dir/$cluster-osd-check.log',
1540 '--cluster', args
.cluster
,
1541 '--setuser', get_ceph_user(),
1542 '--setgroup', get_ceph_group(),
1544 _
, _
, needs_journal
= command([
1545 'ceph-osd', '--check-needs-journal',
1547 '--log-file', '$run_dir/$cluster-osd-check.log',
1548 '--cluster', args
.cluster
,
1549 '--setuser', get_ceph_user(),
1550 '--setgroup', get_ceph_group(),
1552 return (not allows_journal
, not wants_journal
, not needs_journal
)
1555 def update_partition(dev
, description
):
1557 Must be called after modifying a partition table so the kernel
1558 know about the change and fire udev events accordingly. A side
1559 effect of partprobe is to remove partitions and add them again.
1560 The first udevadm settle waits for ongoing udev events to
1561 complete, just in case one of them rely on an existing partition
1562 on dev. The second udevadm settle guarantees to the caller that
1563 all udev events related to the partition table change have been
1564 processed, i.e. the 95-ceph-osd.rules actions and mode changes,
1565 group changes etc. are complete.
1567 LOG
.debug('Calling partprobe on %s device %s', description
, dev
)
1568 partprobe_ok
= False
1569 error
= 'unknown error'
1570 partprobe
= _get_command_executable(['partprobe'])[0]
1572 command_check_call(['udevadm', 'settle', '--timeout=600'])
1574 _check_output(['flock', '-s', dev
, partprobe
, dev
])
1577 except subprocess
.CalledProcessError
as e
:
1579 if ('unable to inform the kernel' not in error
and
1580 'Device or resource busy' not in error
):
1582 LOG
.debug('partprobe %s failed : %s (ignored, waiting 60s)'
1585 if not partprobe_ok
:
1586 raise Error('partprobe %s failed : %s' % (dev
, error
))
1587 command_check_call(['udevadm', 'settle', '--timeout=600'])
1592 # Thoroughly wipe all partitions of any traces of
1593 # Filesystems or OSD Journals
1595 # In addition we need to write 10M of data to each partition
1596 # to make sure that after re-creating the same partition
1597 # there is no trace left of any previous Filesystem or OSD
1600 LOG
.debug('Writing zeros to existing partitions on %s', dev
)
1602 for partname
in list_partitions(dev
):
1603 partition
= get_dev_path(partname
)
1616 'of={path}'.format(path
=partition
),
1622 LOG
.debug('Zapping partition table on %s', dev
)
1624 # try to wipe out any GPT partition table backups. sgdisk
1625 # isn't too thorough.
1627 size
= 33 * lba_size
1628 with
open(dev
, 'wb') as dev_file
:
1629 dev_file
.seek(-size
, os
.SEEK_END
)
1630 dev_file
.write(size
* b
'\0')
1649 update_partition(dev
, 'zapped')
1651 except subprocess
.CalledProcessError
as e
:
1655 def zap_freebsd(dev
):
1657 # For FreeBSD we just need to zap the partition.
1667 except subprocess
.CalledProcessError
as e
:
1673 Destroy the partition table and content of a given disk.
1675 dev
= os
.path
.realpath(dev
)
1676 dmode
= os
.stat(dev
).st_mode
1677 if not stat
.S_ISBLK(dmode
) or is_partition(dev
):
1678 raise Error('not full block device; cannot zap', dev
)
1685 def adjust_symlink(target
, path
):
1687 if os
.path
.lexists(path
):
1689 mode
= os
.lstat(path
).st_mode
1690 if stat
.S_ISREG(mode
):
1691 LOG
.debug('Removing old file %s', path
)
1693 elif stat
.S_ISLNK(mode
):
1694 old
= os
.readlink(path
)
1696 LOG
.debug('Removing old symlink %s -> %s', path
, old
)
1701 raise Error('unable to remove (or adjust) old file (symlink)',
1704 LOG
.debug('Creating symlink %s -> %s', path
, target
)
1706 os
.symlink(target
, path
)
1708 raise Error('unable to create symlink %s -> %s' % (path
, target
))
1711 def get_mount_options(cluster
, fs_type
):
1712 mount_options
= get_conf(
1714 variable
='osd_mount_options_{fstype}'.format(
1718 if mount_options
is None:
1719 mount_options
= get_conf(
1721 variable
='osd_fs_mount_options_{fstype}'.format(
1726 # remove whitespaces
1727 mount_options
= "".join(mount_options
.split())
1728 return mount_options
1731 class Device(object):
1733 def __init__(self
, path
, args
):
1736 self
.dev_size
= None
1737 self
.partitions
= {}
1738 self
.ptype_map
= None
1739 assert not is_partition(self
.path
)
1741 def create_partition(self
, uuid
, name
, size
=0, num
=0):
1742 ptype
= self
.ptype_tobe_for_name(name
)
1744 num
= get_free_partition_index(dev
=self
.path
)
1746 new
= '--new={num}:0:+{size}M'.format(num
=num
, size
=size
)
1747 if size
> self
.get_dev_size():
1748 LOG
.error('refusing to create %s on %s' % (name
, self
.path
))
1749 LOG
.error('%s size (%sM) is bigger than device (%sM)'
1750 % (name
, size
, self
.get_dev_size()))
1751 raise Error('%s device size (%sM) is not big enough for %s'
1752 % (self
.path
, self
.get_dev_size(), name
))
1754 new
= '--largest-new={num}'.format(num
=num
)
1756 LOG
.debug('Creating %s partition num %d size %d on %s',
1757 name
, num
, size
, self
.path
)
1762 '--change-name={num}:ceph {name}'.format(num
=num
, name
=name
),
1763 '--partition-guid={num}:{uuid}'.format(num
=num
, uuid
=uuid
),
1764 '--typecode={num}:{uuid}'.format(num
=num
, uuid
=ptype
),
1771 update_partition(self
.path
, 'created')
1774 def ptype_tobe_for_name(self
, name
):
1775 LOG
.debug("name = " + name
)
1778 if name
== 'lockbox':
1779 if is_mpath(self
.path
):
1780 return PTYPE
['mpath']['lockbox']['tobe']
1782 return PTYPE
['regular']['lockbox']['tobe']
1783 if self
.ptype_map
is None:
1784 partition
= DevicePartition
.factory(
1785 path
=self
.path
, dev
=None, args
=self
.args
)
1786 self
.ptype_map
= partition
.ptype_map
1787 return self
.ptype_map
[name
]['tobe']
1789 def get_partition(self
, num
):
1790 if num
not in self
.partitions
:
1791 dev
= get_partition_dev(self
.path
, num
)
1792 partition
= DevicePartition
.factory(
1793 path
=self
.path
, dev
=dev
, args
=self
.args
)
1794 partition
.set_partition_number(num
)
1795 self
.partitions
[num
] = partition
1796 return self
.partitions
[num
]
1798 def get_dev_size(self
):
1799 if self
.dev_size
is None:
1800 self
.dev_size
= get_dev_size(self
.path
)
1801 return self
.dev_size
1804 def factory(path
, args
):
1805 return Device(path
, args
)
1808 class DevicePartition(object):
1810 def __init__(self
, args
):
1816 self
.ptype_map
= None
1818 self
.set_variables_ptype()
1821 if self
.uuid
is None:
1822 self
.uuid
= get_partition_uuid(self
.rawdev
)
1825 def get_ptype(self
):
1826 if self
.ptype
is None:
1827 self
.ptype
= get_partition_type(self
.rawdev
)
1830 def set_partition_number(self
, num
):
1833 def get_partition_number(self
):
1836 def set_dev(self
, dev
):
1843 def get_rawdev(self
):
1846 def set_variables_ptype(self
):
1847 self
.ptype_map
= PTYPE
['regular']
1849 def ptype_for_name(self
, name
):
1850 return self
.ptype_map
[name
]['ready']
1853 def factory(path
, dev
, args
):
1854 dmcrypt_type
= CryptHelpers
.get_dmcrypt_type(args
)
1855 if ((path
is not None and is_mpath(path
)) or
1856 (dev
is not None and is_mpath(dev
))):
1857 partition
= DevicePartitionMultipath(args
)
1858 elif dmcrypt_type
== 'luks':
1859 partition
= DevicePartitionCryptLuks(args
)
1860 elif dmcrypt_type
== 'plain':
1861 partition
= DevicePartitionCryptPlain(args
)
1863 partition
= DevicePartition(args
)
1864 partition
.set_dev(dev
)
1868 class DevicePartitionMultipath(DevicePartition
):
1870 def set_variables_ptype(self
):
1871 self
.ptype_map
= PTYPE
['mpath']
1874 class DevicePartitionCrypt(DevicePartition
):
1876 def __init__(self
, args
):
1877 super(DevicePartitionCrypt
, self
).__init
__(args
)
1878 self
.osd_dm_key
= None
1879 self
.cryptsetup_parameters
= CryptHelpers
.get_cryptsetup_parameters(
1881 self
.dmcrypt_type
= CryptHelpers
.get_dmcrypt_type(self
.args
)
1882 self
.dmcrypt_keysize
= CryptHelpers
.get_dmcrypt_keysize(self
.args
)
1884 def setup_crypt(self
):
1889 self
.dev
= _dmcrypt_map(
1891 key
=self
.osd_dm_key
,
1892 _uuid
=self
.get_uuid(),
1893 cryptsetup_parameters
=self
.cryptsetup_parameters
,
1900 dmcrypt_unmap(self
.get_uuid())
1901 self
.dev
= self
.rawdev
1908 class DevicePartitionCryptPlain(DevicePartitionCrypt
):
1913 def setup_crypt(self
):
1914 if self
.osd_dm_key
is not None:
1917 self
.cryptsetup_parameters
+= ['--key-size', str(self
.dmcrypt_keysize
)]
1919 self
.osd_dm_key
= get_dmcrypt_key(
1920 self
.get_uuid(), self
.args
.dmcrypt_key_dir
,
1923 def set_variables_ptype(self
):
1924 self
.ptype_map
= PTYPE
['plain']
1927 class DevicePartitionCryptLuks(DevicePartitionCrypt
):
1932 def setup_crypt(self
):
1933 if self
.osd_dm_key
is not None:
1936 if self
.dmcrypt_keysize
== 1024:
1937 # We don't force this into the cryptsetup_parameters,
1938 # as we want the cryptsetup defaults
1939 # to prevail for the actual LUKS key lengths.
1942 self
.cryptsetup_parameters
+= ['--key-size',
1943 str(self
.dmcrypt_keysize
)]
1945 self
.osd_dm_key
= get_dmcrypt_key(
1946 self
.get_uuid(), self
.args
.dmcrypt_key_dir
,
1949 def set_variables_ptype(self
):
1950 self
.ptype_map
= PTYPE
['luks']
1953 class Prepare(object):
1955 def __init__(self
, args
):
1960 parser
= argparse
.ArgumentParser(add_help
=False)
1961 parser
.add_argument(
1965 help='cluster name to assign this disk to',
1967 parser
.add_argument(
1970 help='cluster uuid to assign this disk to',
1972 parser
.add_argument(
1975 help='unique OSD uuid to assign this disk to',
1977 parser
.add_argument(
1980 help='unique OSD id to assign this disk to',
1982 parser
.add_argument(
1983 '--crush-device-class',
1984 help='crush device class to assign this disk to',
1986 parser
.add_argument(
1988 action
='store_true', default
=None,
1989 help='encrypt DATA and/or JOURNAL devices with dm-crypt',
1991 parser
.add_argument(
1992 '--dmcrypt-key-dir',
1994 default
='/etc/ceph/dmcrypt-keys',
1995 help='directory where dm-crypt keys are stored',
1997 parser
.add_argument(
2000 help='bootstrap-osd keyring path template (%(default)s)',
2001 default
='{statedir}/bootstrap-osd/{cluster}.keyring',
2002 dest
='prepare_key_template',
2004 parser
.add_argument(
2006 action
='store_true', default
=None,
2007 help='let many prepare\'s run in parallel',
2012 def set_subparser(subparsers
):
2015 PrepareData
.parser(),
2018 parents
.extend(PrepareFilestore
.parent_parsers())
2019 parents
.extend(PrepareBluestore
.parent_parsers())
2020 parser
= subparsers
.add_parser(
2023 formatter_class
=argparse
.RawDescriptionHelpFormatter
,
2024 description
=textwrap
.fill(textwrap
.dedent("""\
2025 If the --bluestore argument is given, a bluestore objectstore
2026 will be created. If --filestore is provided, a legacy FileStore
2027 objectstore will be created. If neither is specified, we default
2030 When an entire device is prepared for bluestore, two
2031 partitions are created. The first partition is for metadata,
2032 the second partition is for blocks that contain data.
2034 Unless explicitly specified with --block.db or
2035 --block.wal, the bluestore DB and WAL data is stored on
2036 the main block device. For instance:
2038 ceph-disk prepare --bluestore /dev/sdc
2042 /dev/sdc1 for osd metadata
2043 /dev/sdc2 for block, db, and wal data (the rest of the disk)
2046 If either --block.db or --block.wal are specified to be
2047 the same whole device, they will be created as partition
2048 three and four respectively. For instance:
2050 ceph-disk prepare --bluestore \\
2051 --block.db /dev/sdc \\
2052 --block.wal /dev/sdc \\
2057 /dev/sdc1 for osd metadata
2058 /dev/sdc2 for block (the rest of the disk)
2063 help='Prepare a directory or disk for a Ceph OSD',
2065 parser
.set_defaults(
2071 if self
.args
.no_locking
:
2080 return PrepareBluestore(args
)
2082 return PrepareFilestore(args
)
2086 Prepare
.factory(args
).prepare()
2089 class PrepareFilestore(Prepare
):
2091 def __init__(self
, args
):
2092 super(PrepareFilestore
, self
).__init
__(args
)
2094 self
.lockbox
= Lockbox(args
)
2095 self
.data
= PrepareFilestoreData(args
)
2096 self
.journal
= PrepareJournal(args
)
2099 def parent_parsers():
2101 PrepareJournal
.parser(),
2105 if self
.data
.args
.dmcrypt
:
2106 self
.lockbox
.prepare()
2107 self
.data
.prepare(self
.journal
)
2110 class PrepareBluestore(Prepare
):
2112 def __init__(self
, args
):
2113 super(PrepareBluestore
, self
).__init
__(args
)
2115 self
.lockbox
= Lockbox(args
)
2116 self
.data
= PrepareBluestoreData(args
)
2117 self
.block
= PrepareBluestoreBlock(args
)
2118 self
.blockdb
= PrepareBluestoreBlockDB(args
)
2119 self
.blockwal
= PrepareBluestoreBlockWAL(args
)
2123 parser
= argparse
.ArgumentParser(add_help
=False)
2124 parser
.add_argument(
2127 action
='store_true', default
=True,
2128 help='bluestore objectstore',
2130 parser
.add_argument(
2133 action
='store_false',
2134 help='filestore objectstore',
2139 def parent_parsers():
2141 PrepareBluestore
.parser(),
2142 PrepareBluestoreBlock
.parser(),
2143 PrepareBluestoreBlockDB
.parser(),
2144 PrepareBluestoreBlockWAL
.parser(),
2148 if self
.data
.args
.dmcrypt
:
2149 self
.lockbox
.prepare()
2150 to_prepare_list
= []
2151 if getattr(self
.data
.args
, 'block.db'):
2152 to_prepare_list
.append(self
.blockdb
)
2153 if getattr(self
.data
.args
, 'block.wal'):
2154 to_prepare_list
.append(self
.blockwal
)
2155 to_prepare_list
.append(self
.block
)
2156 self
.data
.prepare(*to_prepare_list
)
2159 class Space(object):
2161 NAMES
= ('block', 'journal', 'block.db', 'block.wal')
2164 class PrepareSpace(object):
2170 def __init__(self
, args
):
2173 self
.space_size
= self
.get_space_size()
2174 if getattr(self
.args
, self
.name
+ '_uuid') is None:
2175 setattr(self
.args
, self
.name
+ '_uuid', str(uuid
.uuid4()))
2176 self
.space_symlink
= None
2177 self
.space_dmcrypt
= None
2182 if (self
.wants_space() and
2183 dev_is_diskdevice(args
.data
) and
2184 not is_partition(args
.data
) and
2185 getattr(args
, name
) is None and
2186 getattr(args
, name
+ '_file') is None):
2187 LOG
.info('Will colocate %s with data on %s',
2189 setattr(args
, name
, args
.data
)
2191 if getattr(args
, name
) is None:
2192 if getattr(args
, name
+ '_dev'):
2193 raise Error('%s is unspecified; not a block device' %
2194 name
.capitalize(), getattr(args
, name
))
2195 self
.type = self
.NONE
2198 if not os
.path
.exists(getattr(args
, name
)):
2199 if getattr(args
, name
+ '_dev'):
2200 raise Error('%s does not exist; not a block device' %
2201 name
.capitalize(), getattr(args
, name
))
2202 self
.type = self
.FILE
2205 mode
= os
.stat(getattr(args
, name
)).st_mode
2206 if stmode_is_diskdevice(mode
):
2207 if getattr(args
, name
+ '_file'):
2208 raise Error('%s is not a regular file' % name
.capitalize
,
2209 getattr(args
, name
))
2210 self
.type = self
.DEVICE
2213 if stat
.S_ISREG(mode
):
2214 if getattr(args
, name
+ '_dev'):
2215 raise Error('%s is not a block device' % name
.capitalize
,
2216 getattr(args
, name
))
2217 self
.type = self
.FILE
2220 raise Error('%s %s is neither a block device nor regular file' %
2221 (name
.capitalize
, getattr(args
, name
)))
2224 return self
.type == self
.NONE
2227 return self
.type == self
.FILE
2229 def is_device(self
):
2230 return self
.type == self
.DEVICE
2233 def parser(name
, positional
=True):
2234 parser
= argparse
.ArgumentParser(add_help
=False)
2235 parser
.add_argument(
2238 help='unique uuid to assign to the %s' % name
,
2240 parser
.add_argument(
2242 action
='store_true', default
=None,
2243 help='verify that %s is a file' % name
.upper(),
2245 parser
.add_argument(
2247 action
='store_true', default
=None,
2248 help='verify that %s is a block device' % name
.upper(),
2252 parser
.add_argument(
2254 metavar
=name
.upper(),
2256 help=('path to OSD %s disk block device;' % name
+
2257 ' leave out to store %s in file' % name
),
2261 def wants_space(self
):
2264 def populate_data_path(self
, path
):
2265 if self
.type == self
.DEVICE
:
2266 self
.populate_data_path_device(path
)
2267 elif self
.type == self
.FILE
:
2268 self
.populate_data_path_file(path
)
2269 elif self
.type == self
.NONE
:
2272 raise Error('unexpected type ', self
.type)
2274 def populate_data_path_file(self
, path
):
2275 space_uuid
= self
.name
+ '_uuid'
2276 if getattr(self
.args
, space_uuid
) is not None:
2277 write_one_line(path
, space_uuid
,
2278 getattr(self
.args
, space_uuid
))
2279 if self
.space_symlink
is not None:
2280 adjust_symlink(self
.space_symlink
,
2281 os
.path
.join(path
, self
.name
))
2283 def populate_data_path_device(self
, path
):
2284 self
.populate_data_path_file(path
)
2286 if self
.space_dmcrypt
is not None:
2287 adjust_symlink(self
.space_dmcrypt
,
2288 os
.path
.join(path
, self
.name
+ '_dmcrypt'))
2291 os
.unlink(os
.path
.join(path
, self
.name
+ '_dmcrypt'))
2296 if self
.type == self
.DEVICE
:
2297 self
.prepare_device()
2298 elif self
.type == self
.FILE
:
2300 elif self
.type == self
.NONE
:
2303 raise Error('unexpected type ', self
.type)
2305 def prepare_file(self
):
2306 space_filename
= getattr(self
.args
, self
.name
)
2307 if not os
.path
.exists(space_filename
):
2308 LOG
.debug('Creating %s file %s with size 0'
2309 ' (ceph-osd will resize and allocate)',
2312 space_file
= open(space_filename
, 'wb')
2314 path_set_context(space_filename
)
2316 LOG
.debug('%s is file %s',
2317 self
.name
.capitalize(),
2319 LOG
.warning('OSD will not be hot-swappable if %s is '
2320 'not the same device as the osd data' %
2322 self
.space_symlink
= space_filename
2324 def prepare_device(self
):
2325 reusing_partition
= False
2327 if is_partition(getattr(self
.args
, self
.name
)):
2328 LOG
.debug('%s %s is a partition',
2329 self
.name
.capitalize(), getattr(self
.args
, self
.name
))
2330 partition
= DevicePartition
.factory(
2331 path
=None, dev
=getattr(self
.args
, self
.name
), args
=self
.args
)
2332 if isinstance(partition
, DevicePartitionCrypt
):
2333 raise Error(getattr(self
.args
, self
.name
) +
2334 ' partition already exists'
2335 ' and --dmcrypt specified')
2336 LOG
.warning('OSD will not be hot-swappable' +
2337 ' if ' + self
.name
+ ' is not' +
2338 ' the same device as the osd data')
2339 if partition
.get_ptype() == partition
.ptype_for_name(self
.name
):
2340 LOG
.debug('%s %s was previously prepared with '
2341 'ceph-disk. Reusing it.',
2342 self
.name
.capitalize(),
2343 getattr(self
.args
, self
.name
))
2344 reusing_partition
= True
2345 # Read and reuse the partition uuid from this journal's
2346 # previous life. We reuse the uuid instead of changing it
2347 # because udev does not reliably notice changes to an
2348 # existing partition's GUID. See
2349 # http://tracker.ceph.com/issues/10146
2350 setattr(self
.args
, self
.name
+ '_uuid', partition
.get_uuid())
2351 LOG
.debug('Reusing %s with uuid %s',
2353 getattr(self
.args
, self
.name
+ '_uuid'))
2355 LOG
.warning('%s %s was not prepared with '
2356 'ceph-disk. Symlinking directly.',
2357 self
.name
.capitalize(),
2358 getattr(self
.args
, self
.name
))
2359 self
.space_symlink
= getattr(self
.args
, self
.name
)
2362 self
.space_symlink
= '/dev/disk/by-partuuid/{uuid}'.format(
2363 uuid
=getattr(self
.args
, self
.name
+ '_uuid'))
2365 if self
.args
.dmcrypt
:
2366 self
.space_dmcrypt
= self
.space_symlink
2367 self
.space_symlink
= '/dev/mapper/{uuid}'.format(
2368 uuid
=getattr(self
.args
, self
.name
+ '_uuid'))
2370 if reusing_partition
:
2371 # confirm that the space_symlink exists. It should since
2372 # this was an active space
2373 # in the past. Continuing otherwise would be futile.
2374 assert os
.path
.exists(self
.space_symlink
)
2377 num
= self
.desired_partition_number()
2380 LOG
.warning('OSD will not be hot-swappable if %s '
2381 'is not the same device as the osd data',
2384 device
= Device
.factory(getattr(self
.args
, self
.name
), self
.args
)
2385 num
= device
.create_partition(
2386 uuid
=getattr(self
.args
, self
.name
+ '_uuid'),
2388 size
=self
.space_size
,
2391 partition
= device
.get_partition(num
)
2393 LOG
.debug('%s is GPT partition %s',
2394 self
.name
.capitalize(),
2397 if isinstance(partition
, DevicePartitionCrypt
):
2404 '--typecode={num}:{uuid}'.format(
2406 uuid
=partition
.ptype_for_name(self
.name
),
2409 getattr(self
.args
, self
.name
),
2412 update_partition(getattr(self
.args
, self
.name
), 'prepared')
2414 LOG
.debug('%s is GPT partition %s',
2415 self
.name
.capitalize(),
2419 class PrepareJournal(PrepareSpace
):
2421 def __init__(self
, args
):
2422 self
.name
= 'journal'
2423 (self
.allows_journal
,
2425 self
.needs_journal
) = check_journal_reqs(args
)
2427 if args
.journal
and not self
.allows_journal
:
2428 raise Error('journal specified but not allowed by osd backend')
2430 super(PrepareJournal
, self
).__init
__(args
)
2432 def wants_space(self
):
2433 return self
.wants_journal
2435 def get_space_size(self
):
2436 return int(get_conf_with_default(
2437 cluster
=self
.args
.cluster
,
2438 variable
='osd_journal_size',
2441 def desired_partition_number(self
):
2442 if self
.args
.journal
== self
.args
.data
:
2443 # we're sharing the disk between osd data and journal;
2444 # make journal be partition number 2
2452 return PrepareSpace
.parser('journal')
2455 class PrepareBluestoreBlock(PrepareSpace
):
2457 def __init__(self
, args
):
2459 super(PrepareBluestoreBlock
, self
).__init
__(args
)
2461 def get_space_size(self
):
2462 block_size
= get_conf(
2463 cluster
=self
.args
.cluster
,
2464 variable
='bluestore_block_size',
2467 if block_size
is None:
2468 return 0 # get as much space as possible
2470 return int(block_size
) / 1048576 # MB
2472 def desired_partition_number(self
):
2473 if self
.args
.block
== self
.args
.data
:
2481 return PrepareSpace
.parser('block')
2484 class PrepareBluestoreBlockDB(PrepareSpace
):
2486 def __init__(self
, args
):
2487 self
.name
= 'block.db'
2488 super(PrepareBluestoreBlockDB
, self
).__init
__(args
)
2490 def get_space_size(self
):
2491 block_db_size
= get_conf(
2492 cluster
=self
.args
.cluster
,
2493 variable
='bluestore_block_db_size',
2496 if block_db_size
is None or int(block_db_size
) == 0:
2497 block_size
= get_conf(
2498 cluster
=self
.args
.cluster
,
2499 variable
='bluestore_block_size',
2501 if block_size
is None:
2503 size
= int(block_size
) / 100 / 1048576
2504 return max(size
, 1024) # MB
2506 return int(block_db_size
) / 1048576 # MB
2508 def desired_partition_number(self
):
2509 if getattr(self
.args
, 'block.db') == self
.args
.data
:
2515 def wants_space(self
):
2520 parser
= PrepareSpace
.parser('block.db', positional
=False)
2521 parser
.add_argument(
2524 help='path to the device or file for bluestore block.db',
2529 class PrepareBluestoreBlockWAL(PrepareSpace
):
2531 def __init__(self
, args
):
2532 self
.name
= 'block.wal'
2533 super(PrepareBluestoreBlockWAL
, self
).__init
__(args
)
2535 def get_space_size(self
):
2536 block_size
= get_conf(
2537 cluster
=self
.args
.cluster
,
2538 variable
='bluestore_block_wal_size',
2541 if block_size
is None:
2542 return 576 # MB, default value
2544 return int(block_size
) / 1048576 # MB
2546 def desired_partition_number(self
):
2547 if getattr(self
.args
, 'block.wal') == self
.args
.data
:
2553 def wants_space(self
):
2558 parser
= PrepareSpace
.parser('block.wal', positional
=False)
2559 parser
.add_argument(
2562 help='path to the device or file for bluestore block.wal',
2567 class CryptHelpers(object):
2570 def get_cryptsetup_parameters(args
):
2571 cryptsetup_parameters_str
= get_conf(
2572 cluster
=args
.cluster
,
2573 variable
='osd_cryptsetup_parameters',
2575 if cryptsetup_parameters_str
is None:
2578 return shlex
.split(cryptsetup_parameters_str
)
2581 def get_dmcrypt_keysize(args
):
2582 dmcrypt_keysize_str
= get_conf(
2583 cluster
=args
.cluster
,
2584 variable
='osd_dmcrypt_key_size',
2586 dmcrypt_type
= CryptHelpers
.get_dmcrypt_type(args
)
2587 if dmcrypt_type
== 'luks':
2588 if dmcrypt_keysize_str
is None:
2589 # As LUKS will hash the 'passphrase' in .luks.key
2590 # into a key, set a large default
2591 # so if not updated for some time, it is still a
2596 return int(dmcrypt_keysize_str
)
2597 elif dmcrypt_type
== 'plain':
2598 if dmcrypt_keysize_str
is None:
2599 # This value is hard-coded in the udev script
2602 LOG
.warning('ensure the 95-ceph-osd.rules file has '
2603 'been copied to /etc/udev/rules.d '
2604 'and modified to call cryptsetup '
2605 'with --key-size=%s' % dmcrypt_keysize_str
)
2606 return int(dmcrypt_keysize_str
)
2611 def get_dmcrypt_type(args
):
2612 if hasattr(args
, 'dmcrypt') and args
.dmcrypt
:
2613 dmcrypt_type
= get_conf(
2614 cluster
=args
.cluster
,
2615 variable
='osd_dmcrypt_type',
2618 if dmcrypt_type
is None or dmcrypt_type
== 'luks':
2620 elif dmcrypt_type
== 'plain':
2623 raise Error('invalid osd_dmcrypt_type parameter '
2624 '(must be luks or plain): ', dmcrypt_type
)
2629 class Secrets(object):
2632 secret
, stderr
, ret
= command(['ceph-authtool', '--gen-print-key'])
2633 LOG
.debug("stderr " + stderr
)
2636 'cephx_secret': secret
.strip(),
2639 def write_osd_keyring(self
, keyring
, osd_id
):
2642 'ceph-authtool', keyring
,
2644 '--name', 'osd.' + str(osd_id
),
2645 '--add-key', self
.keys
['cephx_secret'],
2647 path_set_context(keyring
)
2650 return bytearray(json
.dumps(self
.keys
), 'ascii')
2653 class LockboxSecrets(Secrets
):
2655 def __init__(self
, args
):
2656 super(LockboxSecrets
, self
).__init
__()
2658 key_size
= CryptHelpers
.get_dmcrypt_keysize(args
)
2659 key
= open('/dev/urandom', 'rb').read(key_size
/ 8)
2660 base64_key
= base64
.b64encode(key
).decode('ascii')
2662 secret
, stderr
, ret
= command(['ceph-authtool', '--gen-print-key'])
2663 LOG
.debug("stderr " + stderr
)
2667 'dmcrypt_key': base64_key
,
2668 'cephx_lockbox_secret': secret
.strip(),
2671 def write_lockbox_keyring(self
, path
, osd_uuid
):
2672 keyring
= os
.path
.join(path
, 'keyring')
2675 'ceph-authtool', keyring
,
2677 '--name', 'client.osd-lockbox.' + osd_uuid
,
2678 '--add-key', self
.keys
['cephx_lockbox_secret'],
2680 path_set_context(keyring
)
2683 class Lockbox(object):
2685 def __init__(self
, args
):
2687 self
.partition
= None
2690 if hasattr(self
.args
, 'lockbox') and self
.args
.lockbox
is None:
2691 self
.args
.lockbox
= self
.args
.data
2693 def set_partition(self
, partition
):
2694 self
.partition
= partition
2698 parser
= argparse
.ArgumentParser(add_help
=False)
2699 parser
.add_argument(
2701 help='path to the device to store the lockbox',
2703 parser
.add_argument(
2706 help='unique lockbox uuid',
2710 def create_partition(self
):
2711 self
.device
= Device
.factory(self
.args
.lockbox
, argparse
.Namespace())
2712 partition_number
= 5
2713 self
.device
.create_partition(uuid
=self
.args
.lockbox_uuid
,
2715 num
=partition_number
,
2717 return self
.device
.get_partition(partition_number
)
2719 def set_or_create_partition(self
):
2720 if is_partition(self
.args
.lockbox
):
2721 LOG
.debug('OSD lockbox device %s is a partition',
2723 self
.partition
= DevicePartition
.factory(
2724 path
=None, dev
=self
.args
.lockbox
, args
=self
.args
)
2725 ptype
= self
.partition
.get_ptype()
2726 ready
= Ptype
.get_ready_by_name('lockbox')
2727 if ptype
not in ready
:
2728 LOG
.warning('incorrect partition UUID: %s, expected %s'
2729 % (ptype
, str(ready
)))
2731 LOG
.debug('Creating osd partition on %s',
2733 self
.partition
= self
.create_partition()
2735 def create_key(self
):
2736 cluster
= self
.args
.cluster
2737 bootstrap
= self
.args
.prepare_key_template
.format(cluster
=cluster
,
2739 path
= self
.get_mount_point()
2740 secrets
= LockboxSecrets(self
.args
)
2741 id_arg
= self
.args
.osd_id
and [self
.args
.osd_id
] or []
2742 osd_id
= command_with_stdin(
2745 '--cluster', cluster
,
2746 '--name', 'client.bootstrap-osd',
2747 '--keyring', bootstrap
,
2749 'osd', 'new', self
.args
.osd_uuid
,
2753 secrets
.write_lockbox_keyring(path
, self
.args
.osd_uuid
)
2754 osd_id
= must_be_one_line(osd_id
)
2755 check_osd_id(osd_id
)
2756 write_one_line(path
, 'whoami', osd_id
)
2757 secrets
.write_osd_keyring(os
.path
.join(path
, 'osd_keyring'), osd_id
)
2758 write_one_line(path
, 'key-management-mode', KEY_MANAGEMENT_MODE_V1
)
2760 def symlink_spaces(self
, path
):
2761 target
= self
.get_mount_point()
2762 for name
in Space
.NAMES
:
2763 if (hasattr(self
.args
, name
+ '_uuid') and
2764 getattr(self
.args
, name
+ '_uuid')):
2765 uuid
= getattr(self
.args
, name
+ '_uuid')
2766 symlink
= os
.path
.join(STATEDIR
, 'osd-lockbox', uuid
)
2767 adjust_symlink(target
, symlink
)
2768 write_one_line(path
, name
+ '-uuid', uuid
)
2771 maybe_mkdir(os
.path
.join(STATEDIR
, 'osd-lockbox'))
2772 args
= ['mkfs', '-t', 'ext4', self
.partition
.get_dev()]
2773 LOG
.debug('Creating lockbox fs on %s: ' + str(" ".join(args
)))
2774 command_check_call(args
)
2775 path
= self
.get_mount_point()
2777 args
= ['mount', '-t', 'ext4', self
.partition
.get_dev(), path
]
2778 LOG
.debug('Mounting lockbox ' + str(" ".join(args
)))
2779 command_check_call(args
)
2780 write_one_line(path
, 'osd-uuid', self
.args
.osd_uuid
)
2781 if self
.args
.cluster_uuid
is None:
2782 self
.args
.cluster_uuid
= get_fsid(cluster
=self
.args
.cluster
)
2783 write_one_line(path
, 'ceph_fsid', self
.args
.cluster_uuid
)
2785 self
.symlink_spaces(path
)
2786 write_one_line(path
, 'magic', CEPH_LOCKBOX_ONDISK_MAGIC
)
2787 if self
.device
is not None:
2791 '--typecode={num}:{uuid}'.format(
2792 num
=self
.partition
.get_partition_number(),
2793 uuid
=self
.partition
.ptype_for_name('lockbox'),
2796 get_partition_base(self
.partition
.get_dev()),
2800 def get_mount_point(self
):
2801 return os
.path
.join(STATEDIR
, 'osd-lockbox', self
.args
.osd_uuid
)
2803 def get_osd_uuid(self
):
2804 return self
.args
.osd_uuid
2807 path
= is_mounted(self
.partition
.get_dev())
2809 LOG
.info("Lockbox already mounted at " + path
)
2812 path
= tempfile
.mkdtemp(
2814 dir=STATEDIR
+ '/tmp',
2816 args
= ['mount', '-t', 'ext4', '-o', 'ro',
2817 self
.partition
.get_dev(),
2819 LOG
.debug('Mounting lockbox temporarily ' + str(" ".join(args
)))
2820 command_check_call(args
)
2821 self
.args
.osd_uuid
= get_oneliner(path
, 'osd-uuid')
2822 command_check_call(['umount', path
])
2823 LOG
.debug('Mounting lockbox readonly ' + str(" ".join(args
)))
2824 args
= ['mount', '-t', 'ext4', '-o', 'ro',
2825 self
.partition
.get_dev(),
2826 self
.get_mount_point()]
2827 command_check_call(args
)
2828 for name
in Space
.NAMES
+ ('osd',):
2829 uuid_path
= os
.path
.join(self
.get_mount_point(), name
+ '-uuid')
2830 if os
.path
.exists(uuid_path
):
2831 uuid
= get_oneliner(self
.get_mount_point(), name
+ '-uuid')
2832 dev
= os
.path
.join('/dev/disk/by-partuuid/', uuid
.lower())
2833 args
= ['ceph-disk', 'trigger', dev
]
2834 command_check_call(args
)
2837 verify_not_in_use(self
.args
.lockbox
, check_partitions
=True)
2838 self
.set_or_create_partition()
2842 class PrepareData(object):
2847 def __init__(self
, args
):
2850 self
.partition
= None
2852 if self
.args
.cluster_uuid
is None:
2853 self
.args
.cluster_uuid
= get_fsid(cluster
=self
.args
.cluster
)
2855 if self
.args
.osd_uuid
is None:
2856 self
.args
.osd_uuid
= str(uuid
.uuid4())
2859 dmode
= os
.stat(self
.args
.data
).st_mode
2861 if stat
.S_ISDIR(dmode
):
2862 self
.type = self
.FILE
2863 elif stmode_is_diskdevice(dmode
):
2864 self
.type = self
.DEVICE
2866 raise Error('not a dir or block device', self
.args
.data
)
2869 return self
.type == self
.FILE
2871 def is_device(self
):
2872 return self
.type == self
.DEVICE
2876 parser
= argparse
.ArgumentParser(add_help
=False)
2877 parser
.add_argument(
2879 help='file system type to use (e.g. "ext4")',
2881 parser
.add_argument(
2883 action
='store_true', default
=None,
2884 help='destroy the partition table (and content) of a disk',
2886 parser
.add_argument(
2888 action
='store_true', default
=None,
2889 help='verify that DATA is a dir',
2891 parser
.add_argument(
2893 action
='store_true', default
=None,
2894 help='verify that DATA is a block device',
2896 parser
.add_argument(
2899 help='path to OSD data (a disk block device or directory)',
2903 def populate_data_path_file(self
, path
, *to_prepare_list
):
2904 self
.populate_data_path(path
, *to_prepare_list
)
2906 def populate_data_path(self
, path
, *to_prepare_list
):
2907 if os
.path
.exists(os
.path
.join(path
, 'magic')):
2908 LOG
.debug('Data dir %s already exists', path
)
2911 LOG
.debug('Preparing osd data dir %s', path
)
2913 if self
.args
.osd_uuid
is None:
2914 self
.args
.osd_uuid
= str(uuid
.uuid4())
2916 write_one_line(path
, 'ceph_fsid', self
.args
.cluster_uuid
)
2917 write_one_line(path
, 'fsid', self
.args
.osd_uuid
)
2918 if self
.args
.osd_id
:
2919 write_one_line(path
, 'wanttobe', self
.args
.osd_id
)
2920 if self
.args
.crush_device_class
:
2921 write_one_line(path
, 'crush_device_class',
2922 self
.args
.crush_device_class
)
2923 write_one_line(path
, 'magic', CEPH_OSD_ONDISK_MAGIC
)
2925 for to_prepare
in to_prepare_list
:
2926 to_prepare
.populate_data_path(path
)
2928 def prepare(self
, *to_prepare_list
):
2929 if self
.type == self
.DEVICE
:
2930 self
.prepare_device(*to_prepare_list
)
2931 elif self
.type == self
.FILE
:
2932 self
.prepare_file(*to_prepare_list
)
2934 raise Error('unexpected type ', self
.type)
2936 def prepare_file(self
, *to_prepare_list
):
2938 if not os
.path
.exists(self
.args
.data
):
2939 raise Error('data path for directory does not exist',
2942 if self
.args
.data_dev
:
2943 raise Error('data path is not a block device', self
.args
.data
)
2945 for to_prepare
in to_prepare_list
:
2946 to_prepare
.prepare()
2948 self
.populate_data_path_file(self
.args
.data
, *to_prepare_list
)
2950 def sanity_checks(self
):
2951 if not os
.path
.exists(self
.args
.data
):
2952 raise Error('data path for device does not exist',
2954 verify_not_in_use(self
.args
.data
,
2955 check_partitions
=not self
.args
.dmcrypt
)
2957 def set_variables(self
):
2958 if self
.args
.fs_type
is None:
2959 self
.args
.fs_type
= get_conf(
2960 cluster
=self
.args
.cluster
,
2961 variable
='osd_mkfs_type',
2963 if self
.args
.fs_type
is None:
2964 self
.args
.fs_type
= get_conf(
2965 cluster
=self
.args
.cluster
,
2966 variable
='osd_fs_type',
2968 if self
.args
.fs_type
is None:
2969 self
.args
.fs_type
= DEFAULT_FS_TYPE
2971 self
.mkfs_args
= get_conf(
2972 cluster
=self
.args
.cluster
,
2973 variable
='osd_mkfs_options_{fstype}'.format(
2974 fstype
=self
.args
.fs_type
,
2977 if self
.mkfs_args
is None:
2978 self
.mkfs_args
= get_conf(
2979 cluster
=self
.args
.cluster
,
2980 variable
='osd_fs_mkfs_options_{fstype}'.format(
2981 fstype
=self
.args
.fs_type
,
2985 self
.mount_options
= get_mount_options(cluster
=self
.args
.cluster
,
2986 fs_type
=self
.args
.fs_type
)
2988 if self
.args
.osd_uuid
is None:
2989 self
.args
.osd_uuid
= str(uuid
.uuid4())
2991 def prepare_device(self
, *to_prepare_list
):
2992 self
.sanity_checks()
2993 self
.set_variables()
2994 if self
.args
.zap_disk
is not None:
2997 def create_data_partition(self
):
2998 device
= Device
.factory(self
.args
.data
, self
.args
)
2999 partition_number
= 1
3000 device
.create_partition(uuid
=self
.args
.osd_uuid
,
3002 num
=partition_number
,
3003 size
=self
.get_space_size())
3004 return device
.get_partition(partition_number
)
3006 def set_data_partition(self
):
3007 if is_partition(self
.args
.data
):
3008 LOG
.debug('OSD data device %s is a partition',
3010 self
.partition
= DevicePartition
.factory(
3011 path
=None, dev
=self
.args
.data
, args
=self
.args
)
3012 ptype
= self
.partition
.get_ptype()
3013 ready
= Ptype
.get_ready_by_name('osd')
3014 if ptype
not in ready
:
3015 LOG
.warning('incorrect partition UUID: %s, expected %s'
3016 % (ptype
, str(ready
)))
3018 LOG
.debug('Creating osd partition on %s',
3020 self
.partition
= self
.create_data_partition()
3022 def populate_data_path_device(self
, *to_prepare_list
):
3023 partition
= self
.partition
3025 if isinstance(partition
, DevicePartitionCrypt
):
3034 if self
.mkfs_args
is not None:
3035 args
.extend(self
.mkfs_args
.split())
3036 if self
.args
.fs_type
== 'xfs':
3037 args
.extend(['-f']) # always force
3039 args
.extend(MKFS_ARGS
.get(self
.args
.fs_type
, []))
3042 partition
.get_dev(),
3044 LOG
.debug('Creating %s fs on %s',
3045 self
.args
.fs_type
, partition
.get_dev())
3046 command_check_call(args
, exit
=True)
3048 path
= mount(dev
=partition
.get_dev(),
3049 fstype
=self
.args
.fs_type
,
3050 options
=self
.mount_options
)
3053 self
.populate_data_path(path
, *to_prepare_list
)
3055 path_set_context(path
)
3058 if isinstance(partition
, DevicePartitionCrypt
):
3061 if not is_partition(self
.args
.data
):
3065 '--typecode=%d:%s' % (partition
.get_partition_number(),
3066 partition
.ptype_for_name('osd')),
3072 update_partition(self
.args
.data
, 'prepared')
3073 command_check_call(['udevadm', 'trigger',
3076 os
.path
.basename(partition
.rawdev
)])
3079 class PrepareFilestoreData(PrepareData
):
3081 def get_space_size(self
):
3082 return 0 # get as much space as possible
3084 def prepare_device(self
, *to_prepare_list
):
3085 super(PrepareFilestoreData
, self
).prepare_device(*to_prepare_list
)
3086 for to_prepare
in to_prepare_list
:
3087 to_prepare
.prepare()
3088 self
.set_data_partition()
3089 self
.populate_data_path_device(*to_prepare_list
)
3091 def populate_data_path(self
, path
, *to_prepare_list
):
3092 super(PrepareFilestoreData
, self
).populate_data_path(path
,
3094 write_one_line(path
, 'type', 'filestore')
3097 class PrepareBluestoreData(PrepareData
):
3099 def get_space_size(self
):
3102 def prepare_device(self
, *to_prepare_list
):
3103 super(PrepareBluestoreData
, self
).prepare_device(*to_prepare_list
)
3104 self
.set_data_partition()
3105 for to_prepare
in to_prepare_list
:
3106 to_prepare
.prepare()
3107 self
.populate_data_path_device(*to_prepare_list
)
3109 def populate_data_path(self
, path
, *to_prepare_list
):
3110 super(PrepareBluestoreData
, self
).populate_data_path(path
,
3112 write_one_line(path
, 'type', 'bluestore')
3122 monmap
= os
.path
.join(path
, 'activate.monmap')
3126 '--cluster', cluster
,
3127 '--name', 'client.bootstrap-osd',
3128 '--keyring', keyring
,
3129 'mon', 'getmap', '-o', monmap
,
3133 osd_type
= read_one_line(path
, 'type')
3135 if osd_type
== 'bluestore':
3139 '--cluster', cluster
,
3145 '--setuser', get_ceph_user(),
3146 '--setgroup', get_ceph_group(),
3149 elif osd_type
== 'filestore':
3153 '--cluster', cluster
,
3158 '--osd-journal', os
.path
.join(path
, 'journal'),
3160 '--setuser', get_ceph_user(),
3161 '--setgroup', get_ceph_group(),
3165 raise Error('unrecognized objectstore type %s' % osd_type
)
3168 def get_mount_point(cluster
, osd_id
):
3169 parent
= STATEDIR
+ '/osd'
3170 return os
.path
.join(
3172 '{cluster}-{osd_id}'.format(cluster
=cluster
, osd_id
=osd_id
),
3184 LOG
.debug('Moving mount to final location...')
3185 osd_data
= get_mount_point(cluster
, osd_id
)
3186 maybe_mkdir(osd_data
)
3188 # pick best-of-breed mount options based on fs type
3189 if mount_options
is None:
3190 mount_options
= MOUNT_OPTIONS
.get(fstype
, '')
3192 # we really want to mount --move, but that is not supported when
3193 # the parent mount is shared, as it is by default on RH, Fedora,
3194 # and probably others. Also, --bind doesn't properly manipulate
3195 # /etc/mtab, which *still* isn't a symlink to /proc/mounts despite
3196 # this being 2013. Instead, mount the original device at the final
3211 '-l', # lazy, in case someone else is peeking at the
3220 # For upgrade purposes, to make sure there are no competing units,
3221 # both --runtime unit and the default should be disabled. There can be
3222 # two units at the same time: one with --runtime and another without
3223 # it. If, for any reason (manual or ceph-disk) the two units co-exist
3224 # they will compete with each other.
3226 def systemd_disable(
3230 # ensure there is no duplicate ceph-osd@.service
3231 for style
in ([], ['--runtime']):
3236 'ceph-osd@{osd_id}'.format(osd_id
=osd_id
),
3245 systemd_disable(path
, osd_id
)
3246 if is_mounted(path
):
3247 style
= ['--runtime']
3254 'ceph-osd@{osd_id}'.format(osd_id
=osd_id
),
3261 'ceph-osd@{osd_id}'.format(osd_id
=osd_id
),
3270 systemd_disable(path
, osd_id
)
3275 'ceph-osd@{osd_id}'.format(osd_id
=osd_id
),
3284 LOG
.debug('Starting %s osd.%s...', cluster
, osd_id
)
3286 path
= (STATEDIR
+ '/osd/{cluster}-{osd_id}').format(
3287 cluster
=cluster
, osd_id
=osd_id
)
3290 if os
.path
.exists(os
.path
.join(path
, 'upstart')):
3294 # use emit, not start, because start would fail if the
3295 # instance was already running
3297 # since the daemon starting doesn't guarantee much about
3298 # the service being operational anyway, don't bother
3303 'cluster={cluster}'.format(cluster
=cluster
),
3304 'id={osd_id}'.format(osd_id
=osd_id
),
3307 elif os
.path
.exists(os
.path
.join(path
, 'sysvinit')):
3308 if os
.path
.exists('/usr/sbin/service'):
3309 svc
= '/usr/sbin/service'
3311 svc
= '/sbin/service'
3317 '{cluster}'.format(cluster
=cluster
),
3319 'osd.{osd_id}'.format(osd_id
=osd_id
),
3322 elif os
.path
.exists(os
.path
.join(path
, 'systemd')):
3323 systemd_start(path
, osd_id
)
3324 elif os
.path
.exists(os
.path
.join(path
, 'openrc')):
3325 base_script
= '/etc/init.d/ceph-osd'
3326 osd_script
= '{base}.{osd_id}'.format(
3330 if not os
.path
.exists(osd_script
):
3331 os
.symlink(base_script
, osd_script
)
3338 elif os
.path
.exists(os
.path
.join(path
, 'bsdrc')):
3341 '/usr/sbin/service', 'ceph', 'start',
3342 'osd.{osd_id}'.format(osd_id
=osd_id
),
3346 raise Error('{cluster} osd.{osd_id} '
3347 'is not tagged with an init system'
3352 except subprocess
.CalledProcessError
as e
:
3353 raise Error('ceph osd start failed', e
)
3360 LOG
.debug('Stoping %s osd.%s...', cluster
, osd_id
)
3362 path
= (STATEDIR
+ '/osd/{cluster}-{osd_id}').format(
3363 cluster
=cluster
, osd_id
=osd_id
)
3366 if os
.path
.exists(os
.path
.join(path
, 'upstart')):
3372 'cluster={cluster}'.format(cluster
=cluster
),
3373 'id={osd_id}'.format(osd_id
=osd_id
),
3376 elif os
.path
.exists(os
.path
.join(path
, 'sysvinit')):
3377 svc
= which('service')
3383 '{cluster}'.format(cluster
=cluster
),
3385 'osd.{osd_id}'.format(osd_id
=osd_id
),
3388 elif os
.path
.exists(os
.path
.join(path
, 'systemd')):
3389 systemd_stop(path
, osd_id
)
3390 elif os
.path
.exists(os
.path
.join(path
, 'openrc')):
3393 '/etc/init.d/ceph-osd.{osd_id}'.format(osd_id
=osd_id
),
3397 elif os
.path
.exists(os
.path
.join(path
, 'bsdrc')):
3400 '/usr/local/etc/rc.d/ceph stop osd.{osd_id}'
3401 .format(osd_id
=osd_id
),
3405 raise Error('{cluster} osd.{osd_id} '
3406 'is not tagged with an init system'
3407 .format(cluster
=cluster
, osd_id
=osd_id
))
3408 except subprocess
.CalledProcessError
as e
:
3409 raise Error('ceph osd stop failed', e
)
3412 def detect_fstype(dev
):
3414 fstype
= _check_output(
3422 fstype
= _check_output(
3425 # we don't want stale cached results
3433 fstype
= must_be_one_line(fstype
)
3437 def dmcrypt_is_mapped(uuid
):
3438 path
= os
.path
.join('/dev/mapper', uuid
)
3439 if os
.path
.exists(path
):
3445 def dmcrypt_map(dev
, dmcrypt_key_dir
):
3446 ptype
= get_partition_type(dev
)
3447 if ptype
in Ptype
.get_ready_by_type('plain'):
3449 cryptsetup_parameters
= ['--key-size', '256']
3450 elif ptype
in Ptype
.get_ready_by_type('luks'):
3452 cryptsetup_parameters
= []
3454 raise Error('--dmcrypt called for dev %s with invalid ptype %s'
3456 part_uuid
= get_partition_uuid(dev
)
3457 dmcrypt_key
= get_dmcrypt_key(part_uuid
, dmcrypt_key_dir
, luks
)
3458 return _dmcrypt_map(
3462 cryptsetup_parameters
=cryptsetup_parameters
,
3470 activate_key_template
,
3478 part_uuid
= get_partition_uuid(dev
)
3479 dev
= dmcrypt_map(dev
, dmcrypt_key_dir
)
3481 fstype
= detect_fstype(dev
=dev
)
3482 except (subprocess
.CalledProcessError
,
3484 TooManyLinesError
) as e
:
3485 raise FilesystemTypeError(
3486 'device {dev}'.format(dev
=dev
),
3490 # TODO always using mount options from cluster=ceph for
3491 # now; see http://tracker.newdream.net/issues/3253
3492 mount_options
= get_mount_options(cluster
='ceph', fs_type
=fstype
)
3494 path
= mount(dev
=dev
, fstype
=fstype
, options
=mount_options
)
3496 # check if the disk is deactive, change the journal owner, group
3497 # mode for correct user and group.
3498 if os
.path
.exists(os
.path
.join(path
, 'deactive')):
3499 # logging to syslog will help us easy to know udev triggered failure
3502 # we need to unmap again because dmcrypt map will create again
3503 # on bootup stage (due to deactivate)
3504 if '/dev/mapper/' in dev
:
3505 part_uuid
= dev
.replace('/dev/mapper/', '')
3506 dmcrypt_unmap(part_uuid
)
3507 LOG
.info('OSD deactivated! reactivate with: --reactivate')
3508 raise Error('OSD deactivated! reactivate with: --reactivate')
3509 # flag to activate a deactive osd.
3517 (osd_id
, cluster
) = activate(path
, activate_key_template
, init
)
3519 # Now active successfully
3520 # If we got reactivate and deactive, remove the deactive file
3521 if deactive
and reactivate
:
3522 os
.remove(os
.path
.join(path
, 'deactive'))
3523 LOG
.info('Remove `deactive` file.')
3525 # check if the disk is already active, or if something else is already
3529 src_dev
= os
.stat(path
).st_dev
3531 dst_dev
= os
.stat((STATEDIR
+ '/osd/{cluster}-{osd_id}').format(
3533 osd_id
=osd_id
)).st_dev
3534 if src_dev
== dst_dev
:
3537 parent_dev
= os
.stat(STATEDIR
+ '/osd').st_dev
3538 if dst_dev
!= parent_dev
:
3540 elif os
.listdir(get_mount_point(cluster
, osd_id
)):
3541 LOG
.info(get_mount_point(cluster
, osd_id
) +
3542 " is not empty, won't override")
3549 LOG
.info('%s osd.%s already mounted in position; unmounting ours.'
3550 % (cluster
, osd_id
))
3553 raise Error('another %s osd.%s already mounted in position '
3554 '(old/different cluster instance?); unmounting ours.'
3555 % (cluster
, osd_id
))
3563 mount_options
=mount_options
,
3565 return cluster
, osd_id
3568 LOG
.error('Failed to activate')
3572 # remove our temp dir
3573 if os
.path
.exists(path
):
3579 activate_key_template
,
3583 if not os
.path
.exists(path
):
3585 'directory %s does not exist' % path
3588 (osd_id
, cluster
) = activate(path
, activate_key_template
, init
)
3590 if init
not in (None, 'none'):
3591 canonical
= (STATEDIR
+ '/osd/{cluster}-{osd_id}').format(
3594 if path
!= canonical
:
3595 # symlink it from the proper location
3597 if os
.path
.lexists(canonical
):
3598 old
= os
.readlink(canonical
)
3600 LOG
.debug('Removing old symlink %s -> %s', canonical
, old
)
3602 os
.unlink(canonical
)
3604 raise Error('unable to remove old symlink', canonical
)
3608 LOG
.debug('Creating symlink %s -> %s', canonical
, path
)
3610 os
.symlink(path
, canonical
)
3612 raise Error('unable to create symlink %s -> %s'
3613 % (canonical
, path
))
3615 return cluster
, osd_id
3618 def find_cluster_by_uuid(_uuid
):
3620 Find a cluster name by searching /etc/ceph/*.conf for a conf file
3621 with the right uuid.
3623 _uuid
= _uuid
.lower()
3625 if not os
.path
.exists(SYSCONFDIR
):
3627 for conf_file
in os
.listdir(SYSCONFDIR
):
3628 if not conf_file
.endswith('.conf'):
3630 cluster
= conf_file
[:-5]
3632 fsid
= get_fsid(cluster
)
3634 if 'getting cluster uuid from configuration failed' not in str(e
):
3636 no_fsid
.append(cluster
)
3640 # be tolerant of /etc/ceph/ceph.conf without an fsid defined.
3641 if len(no_fsid
) == 1 and no_fsid
[0] == 'ceph':
3642 LOG
.warning('No fsid defined in ' + SYSCONFDIR
+
3643 '/ceph.conf; using anyway')
3650 activate_key_template
,
3654 check_osd_magic(path
)
3656 ceph_fsid
= read_one_line(path
, 'ceph_fsid')
3657 if ceph_fsid
is None:
3658 raise Error('No cluster uuid assigned.')
3659 LOG
.debug('Cluster uuid is %s', ceph_fsid
)
3661 cluster
= find_cluster_by_uuid(ceph_fsid
)
3663 raise Error('No cluster conf found in ' + SYSCONFDIR
+
3664 ' with fsid %s' % ceph_fsid
)
3665 LOG
.debug('Cluster name is %s', cluster
)
3667 fsid
= read_one_line(path
, 'fsid')
3669 raise Error('No OSD uuid assigned.')
3670 LOG
.debug('OSD uuid is %s', fsid
)
3672 keyring
= activate_key_template
.format(cluster
=cluster
,
3675 osd_id
= get_osd_id(path
)
3677 osd_id
= allocate_osd_id(
3683 write_one_line(path
, 'whoami', osd_id
)
3684 LOG
.debug('OSD id is %s', osd_id
)
3686 if not os
.path
.exists(os
.path
.join(path
, 'ready')):
3687 LOG
.debug('Initializing OSD...')
3688 # re-running mkfs is safe, so just run until it completes
3697 if init
not in (None, 'none'):
3699 conf_val
= get_conf(
3703 if conf_val
is not None:
3708 LOG
.debug('Marking with init system %s', init
)
3709 init_path
= os
.path
.join(path
, init
)
3710 with
open(init_path
, 'w'):
3711 path_set_context(init_path
)
3713 # remove markers for others, just in case.
3714 for other
in INIT_SYSTEMS
:
3717 os
.unlink(os
.path
.join(path
, other
))
3721 if not os
.path
.exists(os
.path
.join(path
, 'active')):
3722 write_one_line(path
, 'active', 'ok')
3723 LOG
.debug('%s osd.%s data dir is ready at %s', cluster
, osd_id
, path
)
3724 return (osd_id
, cluster
)
3727 def main_activate(args
):
3731 LOG
.info('path = ' + str(args
.path
))
3732 if not os
.path
.exists(args
.path
):
3733 raise Error('%s does not exist' % args
.path
)
3735 if is_suppressed(args
.path
):
3736 LOG
.info('suppressed activate request on %s', args
.path
)
3740 mode
= os
.stat(args
.path
).st_mode
3741 if stmode_is_diskdevice(mode
):
3742 if (is_partition(args
.path
) and
3743 (get_partition_type(args
.path
) ==
3744 PTYPE
['mpath']['osd']['ready']) and
3745 not is_mpath(args
.path
)):
3746 raise Error('%s is not a multipath block device' %
3748 (cluster
, osd_id
) = mount_activate(
3750 activate_key_template
=args
.activate_key_template
,
3751 init
=args
.mark_init
,
3752 dmcrypt
=args
.dmcrypt
,
3753 dmcrypt_key_dir
=args
.dmcrypt_key_dir
,
3754 reactivate
=args
.reactivate
,
3756 osd_data
= get_mount_point(cluster
, osd_id
)
3758 elif stat
.S_ISDIR(mode
):
3759 (cluster
, osd_id
) = activate_dir(
3761 activate_key_template
=args
.activate_key_template
,
3762 init
=args
.mark_init
,
3764 osd_data
= args
.path
3767 raise Error('%s is not a directory or block device' % args
.path
)
3769 # exit with 0 if the journal device is not up, yet
3770 # journal device will do the activation
3771 osd_journal
= '{path}/journal'.format(path
=osd_data
)
3772 if os
.path
.islink(osd_journal
) and not os
.access(osd_journal
, os
.F_OK
):
3773 LOG
.info("activate: Journal not present, not starting, yet")
3776 if (not args
.no_start_daemon
and args
.mark_init
== 'none'):
3780 '--cluster={cluster}'.format(cluster
=cluster
),
3781 '--id={osd_id}'.format(osd_id
=osd_id
),
3782 '--osd-data={path}'.format(path
=osd_data
),
3783 '--osd-journal={journal}'.format(journal
=osd_journal
),
3787 if (not args
.no_start_daemon
and
3788 args
.mark_init
not in (None, 'none')):
3796 def main_activate_lockbox(args
):
3798 main_activate_lockbox_protected(args
)
3801 def main_activate_lockbox_protected(args
):
3802 partition
= DevicePartition
.factory(
3803 path
=None, dev
=args
.path
, args
=args
)
3805 lockbox
= Lockbox(args
)
3806 lockbox
.set_partition(partition
)
3810 ###########################
3812 def _mark_osd_out(cluster
, osd_id
):
3813 LOG
.info('Prepare to mark osd.%d out...', osd_id
)
3822 def _check_osd_status(cluster
, osd_id
):
3824 report the osd status:
3825 00(0) : means OSD OUT AND DOWN
3826 01(1) : means OSD OUT AND UP
3827 10(2) : means OSD IN AND DOWN
3828 11(3) : means OSD IN AND UP
3830 LOG
.info("Checking osd id: %s ..." % osd_id
)
3833 out
, err
, ret
= command([
3837 '--cluster={cluster}'.format(
3843 out_json
= json
.loads(out
)
3844 for item
in out_json
[u
'osds']:
3845 if item
.get(u
'osd') == int(osd_id
):
3847 if item
.get(u
'in') is 1:
3849 if item
.get(u
'up') is 1:
3852 raise Error('Could not osd.%s in osd tree!' % osd_id
)
3856 def _remove_osd_directory_files(mounted_path
, cluster
):
3858 To remove the 'ready', 'active', INIT-specific files.
3860 if os
.path
.exists(os
.path
.join(mounted_path
, 'ready')):
3861 os
.remove(os
.path
.join(mounted_path
, 'ready'))
3862 LOG
.info('Remove `ready` file.')
3864 LOG
.info('`ready` file is already removed.')
3866 if os
.path
.exists(os
.path
.join(mounted_path
, 'active')):
3867 os
.remove(os
.path
.join(mounted_path
, 'active'))
3868 LOG
.info('Remove `active` file.')
3870 LOG
.info('`active` file is already removed.')
3872 # Just check `upstart` and `sysvinit` directly if filename is init-spec.
3873 conf_val
= get_conf(
3877 if conf_val
is not None:
3881 os
.remove(os
.path
.join(mounted_path
, init
))
3882 LOG
.info('Remove `%s` file.', init
)
3886 def main_deactivate(args
):
3888 main_deactivate_locked(args
)
3891 def main_deactivate_locked(args
):
3892 osd_id
= args
.deactivate_by_id
3896 devices
= list_devices()
3898 # list all devices and found we need
3899 for device
in devices
:
3900 if 'partitions' in device
:
3901 for dev_part
in device
.get('partitions'):
3903 'whoami' in dev_part
and
3904 dev_part
['whoami'] == osd_id
):
3905 target_dev
= dev_part
3907 'path' in dev_part
and
3908 dev_part
['path'] == path
):
3909 target_dev
= dev_part
3911 raise Error('Cannot find any match device!!')
3913 # set up all we need variable
3914 osd_id
= target_dev
['whoami']
3915 part_type
= target_dev
['ptype']
3916 mounted_path
= target_dev
['mount']
3917 if Ptype
.is_dmcrypt(part_type
, 'osd'):
3920 # Do not do anything if osd is already down.
3921 status_code
= _check_osd_status(args
.cluster
, osd_id
)
3922 if status_code
== OSD_STATUS_IN_UP
:
3923 if args
.mark_out
is True:
3924 _mark_osd_out(args
.cluster
, int(osd_id
))
3925 stop_daemon(args
.cluster
, osd_id
)
3926 elif status_code
== OSD_STATUS_IN_DOWN
:
3927 if args
.mark_out
is True:
3928 _mark_osd_out(args
.cluster
, int(osd_id
))
3929 LOG
.info("OSD already out/down. Do not do anything now.")
3931 elif status_code
== OSD_STATUS_OUT_UP
:
3932 stop_daemon(args
.cluster
, osd_id
)
3933 elif status_code
== OSD_STATUS_OUT_DOWN
:
3934 LOG
.info("OSD already out/down. Do not do anything now.")
3938 # remove 'ready', 'active', and INIT-specific files.
3939 _remove_osd_directory_files(mounted_path
, args
.cluster
)
3941 # Write deactivate to osd directory!
3942 with
open(os
.path
.join(mounted_path
, 'deactive'), 'w'):
3943 path_set_context(os
.path
.join(mounted_path
, 'deactive'))
3945 unmount(mounted_path
)
3946 LOG
.info("Umount `%s` successfully.", mounted_path
)
3949 lockbox
= os
.path
.join(STATEDIR
, 'osd-lockbox')
3950 command(['umount', os
.path
.join(lockbox
, target_dev
['uuid'])])
3952 dmcrypt_unmap(target_dev
['uuid'])
3953 for name
in Space
.NAMES
:
3954 if name
+ '_uuid' in target_dev
:
3955 dmcrypt_unmap(target_dev
[name
+ '_uuid'])
3957 ###########################
3960 def _remove_lockbox(uuid
, cluster
):
3961 lockbox
= os
.path
.join(STATEDIR
, 'osd-lockbox')
3962 if not os
.path
.exists(lockbox
):
3964 canonical
= os
.path
.join(lockbox
, uuid
)
3965 command(['umount', canonical
])
3966 for name
in os
.listdir(lockbox
):
3967 path
= os
.path
.join(lockbox
, name
)
3968 if os
.path
.islink(path
) and os
.readlink(path
) == canonical
:
3972 def destroy_lookup_device(args
, predicate
, description
):
3973 devices
= list_devices()
3974 for device
in devices
:
3975 for partition
in device
.get('partitions', []):
3976 if partition
['type'] == 'lockbox':
3977 if not is_mounted(partition
['path']):
3978 main_activate_lockbox_protected(
3979 argparse
.Namespace(verbose
=args
.verbose
,
3980 path
=partition
['path']))
3981 for device
in devices
:
3982 for partition
in device
.get('partitions', []):
3983 if partition
['dmcrypt']:
3984 dmcrypt_path
= dmcrypt_is_mapped(partition
['uuid'])
3988 dmcrypt_path
= dmcrypt_map(partition
['path'],
3989 args
.dmcrypt_key_dir
)
3991 list_dev_osd(dmcrypt_path
, {}, partition
)
3993 dmcrypt_unmap(partition
['uuid'])
3997 if predicate(partition
):
3998 return dmcrypt
, partition
3999 raise Error('found no device matching ', description
)
4002 def main_destroy(args
):
4004 main_destroy_locked(args
)
4007 def main_destroy_locked(args
):
4008 osd_id
= args
.destroy_by_id
4013 if not is_partition(path
):
4014 raise Error(path
+ " must be a partition device")
4015 path
= os
.path
.realpath(path
)
4018 (dmcrypt
, target_dev
) = destroy_lookup_device(
4019 args
, lambda x
: x
.get('path') == path
,
4022 (dmcrypt
, target_dev
) = destroy_lookup_device(
4023 args
, lambda x
: x
.get('whoami') == osd_id
,
4024 'osd id ' + str(osd_id
))
4026 osd_id
= target_dev
['whoami']
4027 dev_path
= target_dev
['path']
4028 if target_dev
['ptype'] == PTYPE
['mpath']['osd']['ready']:
4029 base_dev
= get_partition_base_mpath(dev_path
)
4031 base_dev
= get_partition_base(dev_path
)
4033 # Before osd deactivate, we cannot destroy it
4034 status_code
= _check_osd_status(args
.cluster
, osd_id
)
4035 if status_code
!= OSD_STATUS_OUT_DOWN
and \
4036 status_code
!= OSD_STATUS_IN_DOWN
:
4037 raise Error("Could not destroy the active osd. (osd-id: %s)" %
4044 LOG
.info("Prepare to %s osd.%s" % (action
, osd_id
))
4050 '--yes-i-really-mean-it',
4053 # we remove the crypt map and device mapper (if dmcrypt is True)
4055 for name
in Space
.NAMES
:
4056 if target_dev
.get(name
+ '_uuid'):
4057 dmcrypt_unmap(target_dev
[name
+ '_uuid'])
4058 _remove_lockbox(target_dev
['uuid'], args
.cluster
)
4060 # Check zap flag. If we found zap flag, we need to find device for
4061 # destroy this osd data.
4062 if args
.zap
is True:
4063 # erase the osd data
4064 LOG
.info("Prepare to zap the device %s" % base_dev
)
4068 def get_space_osd_uuid(name
, path
):
4069 if not os
.path
.exists(path
):
4070 raise Error('%s does not exist' % path
)
4072 if not path_is_diskdevice(path
):
4073 raise Error('%s is not a block device' % path
)
4075 if (is_partition(path
) and
4076 get_partition_type(path
) in (PTYPE
['mpath']['journal']['ready'],
4077 PTYPE
['mpath']['block']['ready']) and
4078 not is_mpath(path
)):
4079 raise Error('%s is not a multipath block device' %
4083 out
= _check_output(
4086 '--get-device-fsid',
4091 except subprocess
.CalledProcessError
as e
:
4093 'failed to get osd uuid/fsid from %s' % name
,
4096 value
= str(out
).split('\n', 1)[0]
4097 LOG
.debug('%s %s has OSD UUID %s', name
.capitalize(), path
, value
)
4101 def main_activate_space(name
, args
):
4102 if not os
.path
.exists(args
.dev
):
4103 raise Error('%s does not exist' % args
.dev
)
4105 if is_suppressed(args
.dev
):
4106 LOG
.info('suppressed activate request on space %s', args
.dev
)
4115 dev
= dmcrypt_map(args
.dev
, args
.dmcrypt_key_dir
)
4118 # FIXME: For an encrypted journal dev, does this return the
4119 # cyphertext or plaintext dev uuid!? Also, if the journal is
4120 # encrypted, is the data partition also always encrypted, or
4121 # are mixed pairs supported!?
4122 osd_uuid
= get_space_osd_uuid(name
, dev
)
4123 path
= os
.path
.join('/dev/disk/by-partuuid/', osd_uuid
.lower())
4125 if is_suppressed(path
):
4126 LOG
.info('suppressed activate request on %s', path
)
4129 # warn and exit with 0 if the data device is not up, yet
4130 # data device will do the activation
4131 if not os
.access(path
, os
.F_OK
):
4132 LOG
.info("activate: OSD device not present, not starting, yet")
4135 (cluster
, osd_id
) = mount_activate(
4137 activate_key_template
=args
.activate_key_template
,
4138 init
=args
.mark_init
,
4139 dmcrypt
=args
.dmcrypt
,
4140 dmcrypt_key_dir
=args
.dmcrypt_key_dir
,
4141 reactivate
=args
.reactivate
,
4150 ###########################
4153 def main_activate_all(args
):
4154 dir = '/dev/disk/by-parttypeuuid'
4155 LOG
.debug('Scanning %s', dir)
4156 if not os
.path
.exists(dir):
4159 for name
in os
.listdir(dir):
4160 if name
.find('.') < 0:
4162 (tag
, uuid
) = name
.split('.')
4164 if tag
in Ptype
.get_ready_by_name('osd'):
4166 if Ptype
.is_dmcrypt(tag
, 'osd'):
4167 path
= os
.path
.join('/dev/mapper', uuid
)
4169 path
= os
.path
.join(dir, name
)
4171 if is_suppressed(path
):
4172 LOG
.info('suppressed activate request on %s', path
)
4175 LOG
.info('Activating %s', path
)
4178 # never map dmcrypt cyphertext devices
4179 (cluster
, osd_id
) = mount_activate(
4181 activate_key_template
=args
.activate_key_template
,
4182 init
=args
.mark_init
,
4191 except Exception as e
:
4193 '{prog}: {msg}'.format(prog
=args
.prog
, msg
=e
),
4200 raise Error('One or more partitions failed to activate')
4203 ###########################
4206 dev
= os
.path
.realpath(dev
)
4207 with
open(PROCDIR
+ '/swaps', 'rb') as proc_swaps
:
4208 for line
in proc_swaps
.readlines()[1:]:
4209 fields
= line
.split()
4212 swaps_dev
= fields
[0]
4213 if os
.path
.isabs(swaps_dev
) and os
.path
.exists(swaps_dev
):
4214 swaps_dev
= os
.path
.realpath(swaps_dev
)
4215 if swaps_dev
== dev
:
4220 def get_oneliner(base
, name
):
4221 path
= os
.path
.join(base
, name
)
4222 if os
.path
.isfile(path
):
4223 with
open(path
, 'rb') as _file
:
4224 return _bytes2str(_file
.readline().rstrip())
4228 def get_dev_fs(dev
):
4230 fstype
, _
, ret
= command(
4240 fscheck
, _
, _
= command(
4248 if 'TYPE' in fscheck
:
4249 fstype
= fscheck
.split()[1].split('"')[1]
4254 def split_dev_base_partnum(dev
):
4256 partnum
= partnum_mpath(dev
)
4257 base
= get_partition_base_mpath(dev
)
4260 partnum
= open(os
.path
.join(b
, 'partition')).read().strip()
4261 base
= get_partition_base(dev
)
4262 return base
, partnum
4265 def get_partition_type(part
):
4266 return get_blkid_partition_info(part
, 'ID_PART_ENTRY_TYPE')
4269 def get_partition_uuid(part
):
4270 return get_blkid_partition_info(part
, 'ID_PART_ENTRY_UUID')
4273 def get_blkid_partition_info(dev
, what
=None):
4274 out
, _
, _
= command(
4284 for line
in out
.splitlines():
4285 (key
, value
) = line
.split('=')
4293 def more_osd_info(path
, uuid_map
, desc
):
4294 desc
['ceph_fsid'] = get_oneliner(path
, 'ceph_fsid')
4295 if desc
['ceph_fsid']:
4296 desc
['cluster'] = find_cluster_by_uuid(desc
['ceph_fsid'])
4297 desc
['whoami'] = get_oneliner(path
, 'whoami')
4298 for name
in Space
.NAMES
:
4299 uuid
= get_oneliner(path
, name
+ '_uuid')
4301 desc
[name
+ '_uuid'] = uuid
.lower()
4302 if desc
[name
+ '_uuid'] in uuid_map
:
4303 desc
[name
+ '_dev'] = uuid_map
[desc
[name
+ '_uuid']]
4306 def list_dev_osd(dev
, uuid_map
, desc
):
4307 desc
['mount'] = is_mounted(dev
)
4308 desc
['fs_type'] = get_dev_fs(dev
)
4309 desc
['state'] = 'unprepared'
4311 desc
['state'] = 'active'
4312 more_osd_info(desc
['mount'], uuid_map
, desc
)
4313 elif desc
['fs_type']:
4315 tpath
= mount(dev
=dev
, fstype
=desc
['fs_type'], options
='')
4318 magic
= get_oneliner(tpath
, 'magic')
4319 if magic
is not None:
4320 desc
['magic'] = magic
4321 desc
['state'] = 'prepared'
4322 more_osd_info(tpath
, uuid_map
, desc
)
4329 def list_dev_lockbox(dev
, uuid_map
, desc
):
4330 desc
['mount'] = is_mounted(dev
)
4331 desc
['fs_type'] = get_dev_fs(dev
)
4332 desc
['state'] = 'unprepared'
4334 desc
['state'] = 'active'
4335 desc
['osd_uuid'] = get_oneliner(desc
['mount'], 'osd-uuid')
4336 elif desc
['fs_type']:
4338 tpath
= tempfile
.mkdtemp(prefix
='mnt.', dir=STATEDIR
+ '/tmp')
4339 args
= ['mount', '-t', 'ext4', dev
, tpath
]
4340 LOG
.debug('Mounting lockbox ' + str(" ".join(args
)))
4341 command_check_call(args
)
4342 magic
= get_oneliner(tpath
, 'magic')
4343 if magic
is not None:
4344 desc
['magic'] = magic
4345 desc
['state'] = 'prepared'
4346 desc
['osd_uuid'] = get_oneliner(tpath
, 'osd-uuid')
4348 except subprocess
.CalledProcessError
:
4350 if desc
.get('osd_uuid') in uuid_map
:
4351 desc
['lockbox_for'] = uuid_map
[desc
['osd_uuid']]
4354 def list_format_lockbox_plain(dev
):
4356 if dev
.get('lockbox_for'):
4357 desc
.append('for ' + dev
['lockbox_for'])
4358 elif dev
.get('osd_uuid'):
4359 desc
.append('for osd ' + dev
['osd_uuid'])
4363 def list_format_more_osd_info_plain(dev
):
4365 if dev
.get('ceph_fsid'):
4366 if dev
.get('cluster'):
4367 desc
.append('cluster ' + dev
['cluster'])
4369 desc
.append('unknown cluster ' + dev
['ceph_fsid'])
4370 if dev
.get('whoami'):
4371 desc
.append('osd.%s' % dev
['whoami'])
4372 for name
in Space
.NAMES
:
4373 if dev
.get(name
+ '_dev'):
4374 desc
.append(name
+ ' %s' % dev
[name
+ '_dev'])
4378 def list_format_dev_plain(dev
, prefix
=''):
4380 if dev
['ptype'] == PTYPE
['regular']['osd']['ready']:
4381 desc
= (['ceph data', dev
['state']] +
4382 list_format_more_osd_info_plain(dev
))
4383 elif dev
['ptype'] in (PTYPE
['regular']['lockbox']['ready'],
4384 PTYPE
['mpath']['lockbox']['ready']):
4385 desc
= (['ceph lockbox', dev
['state']] +
4386 list_format_lockbox_plain(dev
))
4387 elif Ptype
.is_dmcrypt(dev
['ptype'], 'osd'):
4388 dmcrypt
= dev
['dmcrypt']
4389 if not dmcrypt
['holders']:
4390 desc
= ['ceph data (dmcrypt %s)' % dmcrypt
['type'],
4391 'not currently mapped']
4392 elif len(dmcrypt
['holders']) == 1:
4393 holder
= get_dev_path(dmcrypt
['holders'][0])
4394 desc
= ['ceph data (dmcrypt %s %s)' %
4395 (dmcrypt
['type'], holder
)]
4396 desc
+= list_format_more_osd_info_plain(dev
)
4398 desc
= ['ceph data (dmcrypt %s)' % dmcrypt
['type'],
4399 'holders: ' + ','.join(dmcrypt
['holders'])]
4400 elif Ptype
.is_regular_space(dev
['ptype']):
4401 name
= Ptype
.space_ptype_to_name(dev
['ptype'])
4402 desc
.append('ceph ' + name
)
4403 if dev
.get(name
+ '_for'):
4404 desc
.append('for %s' % dev
[name
+ '_for'])
4405 elif Ptype
.is_dmcrypt_space(dev
['ptype']):
4406 name
= Ptype
.space_ptype_to_name(dev
['ptype'])
4407 dmcrypt
= dev
['dmcrypt']
4408 if dmcrypt
['holders'] and len(dmcrypt
['holders']) == 1:
4409 holder
= get_dev_path(dmcrypt
['holders'][0])
4410 desc
= ['ceph ' + name
+ ' (dmcrypt %s %s)' %
4411 (dmcrypt
['type'], holder
)]
4413 desc
= ['ceph ' + name
+ ' (dmcrypt %s)' % dmcrypt
['type']]
4414 if dev
.get(name
+ '_for'):
4415 desc
.append('for %s' % dev
[name
+ '_for'])
4417 desc
.append(dev
['type'])
4418 if dev
.get('fs_type'):
4419 desc
.append(dev
['fs_type'])
4420 elif dev
.get('ptype'):
4421 desc
.append(dev
['ptype'])
4422 if dev
.get('mount'):
4423 desc
.append('mounted on %s' % dev
['mount'])
4424 return '%s%s %s' % (prefix
, dev
['path'], ', '.join(desc
))
4427 def list_format_plain(devices
):
4429 for device
in devices
:
4430 if device
.get('partitions'):
4431 lines
.append('%s :' % device
['path'])
4432 for p
in sorted(device
['partitions'], key
=lambda x
: x
['path']):
4433 lines
.append(list_format_dev_plain(dev
=p
,
4436 lines
.append(list_format_dev_plain(dev
=device
,
4438 return "\n".join(lines
)
4441 def list_dev(dev
, uuid_map
, space_map
):
4447 info
['is_partition'] = is_partition(dev
)
4448 if info
['is_partition']:
4449 ptype
= get_partition_type(dev
)
4450 info
['uuid'] = get_partition_uuid(dev
)
4453 info
['ptype'] = ptype
4454 LOG
.info("list_dev(dev = " + dev
+ ", ptype = " + str(ptype
) + ")")
4455 if ptype
in (PTYPE
['regular']['osd']['ready'],
4456 PTYPE
['mpath']['osd']['ready']):
4457 info
['type'] = 'data'
4458 if ptype
== PTYPE
['mpath']['osd']['ready']:
4459 info
['multipath'] = True
4460 list_dev_osd(dev
, uuid_map
, info
)
4461 elif ptype
in (PTYPE
['regular']['lockbox']['ready'],
4462 PTYPE
['mpath']['lockbox']['ready']):
4463 info
['type'] = 'lockbox'
4464 if ptype
== PTYPE
['mpath']['osd']['ready']:
4465 info
['multipath'] = True
4466 list_dev_lockbox(dev
, uuid_map
, info
)
4467 elif ptype
== PTYPE
['plain']['osd']['ready']:
4468 holders
= is_held(dev
)
4469 info
['type'] = 'data'
4470 info
['dmcrypt']['holders'] = holders
4471 info
['dmcrypt']['type'] = 'plain'
4472 if len(holders
) == 1:
4473 list_dev_osd(get_dev_path(holders
[0]), uuid_map
, info
)
4474 elif ptype
== PTYPE
['luks']['osd']['ready']:
4475 holders
= is_held(dev
)
4476 info
['type'] = 'data'
4477 info
['dmcrypt']['holders'] = holders
4478 info
['dmcrypt']['type'] = 'LUKS'
4479 if len(holders
) == 1:
4480 list_dev_osd(get_dev_path(holders
[0]), uuid_map
, info
)
4481 elif Ptype
.is_regular_space(ptype
) or Ptype
.is_mpath_space(ptype
):
4482 name
= Ptype
.space_ptype_to_name(ptype
)
4484 if ptype
== PTYPE
['mpath'][name
]['ready']:
4485 info
['multipath'] = True
4486 if info
.get('uuid') in space_map
:
4487 info
[name
+ '_for'] = space_map
[info
['uuid']]
4488 elif Ptype
.is_plain_space(ptype
):
4489 name
= Ptype
.space_ptype_to_name(ptype
)
4490 holders
= is_held(dev
)
4492 info
['dmcrypt']['type'] = 'plain'
4493 info
['dmcrypt']['holders'] = holders
4494 if info
.get('uuid') in space_map
:
4495 info
[name
+ '_for'] = space_map
[info
['uuid']]
4496 elif Ptype
.is_luks_space(ptype
):
4497 name
= Ptype
.space_ptype_to_name(ptype
)
4498 holders
= is_held(dev
)
4500 info
['dmcrypt']['type'] = 'LUKS'
4501 info
['dmcrypt']['holders'] = holders
4502 if info
.get('uuid') in space_map
:
4503 info
[name
+ '_for'] = space_map
[info
['uuid']]
4505 path
= is_mounted(dev
)
4506 fs_type
= get_dev_fs(dev
)
4508 info
['type'] = 'swap'
4510 info
['type'] = 'other'
4512 info
['fs_type'] = fs_type
4514 info
['mount'] = path
4520 partmap
= list_all_partitions()
4524 for base
, parts
in sorted(partmap
.items()):
4526 dev
= get_dev_path(p
)
4527 part_uuid
= get_partition_uuid(dev
)
4529 uuid_map
[part_uuid
] = dev
4530 ptype
= get_partition_type(dev
)
4531 LOG
.debug("main_list: " + dev
+
4532 " ptype = " + str(ptype
) +
4533 " uuid = " + str(part_uuid
))
4534 if ptype
in Ptype
.get_ready_by_name('osd'):
4535 if Ptype
.is_dmcrypt(ptype
, 'osd'):
4536 holders
= is_held(dev
)
4537 if len(holders
) != 1:
4539 dev_to_mount
= get_dev_path(holders
[0])
4543 fs_type
= get_dev_fs(dev_to_mount
)
4544 if fs_type
is not None:
4545 mount_options
= get_mount_options(cluster
='ceph',
4548 tpath
= mount(dev
=dev_to_mount
,
4549 fstype
=fs_type
, options
=mount_options
)
4551 for name
in Space
.NAMES
:
4552 space_uuid
= get_oneliner(tpath
,
4555 space_map
[space_uuid
.lower()] = dev
4561 LOG
.debug("main_list: " + str(partmap
) + ", uuid_map = " +
4562 str(uuid_map
) + ", space_map = " + str(space_map
))
4565 for base
, parts
in sorted(partmap
.items()):
4567 disk
= {'path': get_dev_path(base
)}
4569 for p
in sorted(parts
):
4570 partitions
.append(list_dev(get_dev_path(p
),
4573 disk
['partitions'] = partitions
4574 devices
.append(disk
)
4576 device
= list_dev(get_dev_path(base
), uuid_map
, space_map
)
4577 device
['path'] = get_dev_path(base
)
4578 devices
.append(device
)
4579 LOG
.debug("list_devices: " + str(devices
))
4585 out
, err
, ret
= command(
4589 '-o', 'name,mountpoint'
4592 except subprocess
.CalledProcessError
as e
:
4593 LOG
.info('zfs list -o name,mountpoint '
4594 'fails.\n (Error: %s)' % e
)
4596 lines
= out
.splitlines()
4597 for line
in lines
[1:]:
4598 vdevline
= line
.split()
4599 if os
.path
.exists(os
.path
.join(vdevline
[1], 'active')):
4600 elems
= os
.path
.split(vdevline
[1])
4601 print(vdevline
[0], "ceph data, active, cluster ceph,", elems
[1],
4602 "mounted on:", vdevline
[1])
4604 print(vdevline
[0] + " other, zfs, mounted on: " + vdevline
[1])
4607 def main_list(args
):
4610 main_list_freebsd(args
)
4612 main_list_protected(args
)
4615 def main_list_protected(args
):
4616 devices
= list_devices()
4619 for path
in args
.path
:
4620 if os
.path
.exists(path
):
4621 paths
.append(os
.path
.realpath(path
))
4624 selected_devices
= []
4625 for device
in devices
:
4627 if re
.search(path
+ '$', device
['path']):
4628 selected_devices
.append(device
)
4630 selected_devices
= devices
4631 if args
.format
== 'json':
4632 print(json
.dumps(selected_devices
))
4634 output
= list_format_plain(selected_devices
)
4639 def main_list_freebsd(args
):
4640 # Currently accomodate only ZFS Filestore partitions
4641 # return a list of VDEVs and mountpoints
4643 # NAME USED AVAIL REFER MOUNTPOINT
4644 # osd0 1.01G 1.32T 1.01G /var/lib/ceph/osd/osd.0
4645 # osd1 1.01G 1.32T 1.01G /var/lib/ceph/osd/osd.1
4649 ###########################
4651 # Mark devices that we want to suppress activates on with a
4654 # /var/lib/ceph/tmp/suppress-activate.sdb
4656 # where the last bit is the sanitized device name (/dev/X without the
4657 # /dev/ prefix) and the is_suppress() check matches a prefix. That
4658 # means suppressing sdb will stop activate on sdb1, sdb2, etc.
4661 def is_suppressed(path
):
4662 disk
= os
.path
.realpath(path
)
4664 if (not disk
.startswith('/dev/') or
4665 not ldev_is_diskdevice(disk
)):
4667 base
= get_dev_name(disk
)
4669 if os
.path
.exists(SUPPRESS_PREFIX
+ base
): # noqa
4676 def set_suppress(path
):
4677 disk
= os
.path
.realpath(path
)
4678 if not os
.path
.exists(disk
):
4679 raise Error('does not exist', path
)
4680 if not ldev_is_diskdevice(path
):
4681 raise Error('not a block device', path
)
4682 base
= get_dev_name(disk
)
4684 with
open(SUPPRESS_PREFIX
+ base
, 'w') as f
: # noqa
4686 LOG
.info('set suppress flag on %s', base
)
4689 def unset_suppress(path
):
4690 disk
= os
.path
.realpath(path
)
4691 if not os
.path
.exists(disk
):
4692 raise Error('does not exist', path
)
4693 if not ldev_is_diskdevice(path
):
4694 raise Error('not a block device', path
)
4695 assert disk
.startswith('/dev/')
4696 base
= get_dev_name(disk
)
4698 fn
= SUPPRESS_PREFIX
+ base
# noqa
4699 if not os
.path
.exists(fn
):
4700 raise Error('not marked as suppressed', path
)
4704 LOG
.info('unset suppress flag on %s', base
)
4705 except OSError as e
:
4706 raise Error('failed to unsuppress', e
)
4709 def main_suppress(args
):
4710 set_suppress(args
.path
)
4713 def main_unsuppress(args
):
4714 unset_suppress(args
.path
)
4718 for dev
in args
.dev
:
4722 def main_trigger(args
):
4723 LOG
.debug("main_trigger: " + str(args
))
4724 if is_systemd() and not args
.sync
:
4725 # http://www.freedesktop.org/software/systemd/man/systemd-escape.html
4726 escaped_dev
= args
.dev
[1:].replace('-', '\\x2d')
4727 service
= 'ceph-disk@{dev}.service'.format(dev
=escaped_dev
)
4728 LOG
.info('systemd detected, triggering %s' % service
)
4738 if is_upstart() and not args
.sync
:
4739 LOG
.info('upstart detected, triggering ceph-disk task')
4745 'dev={dev}'.format(dev
=args
.dev
),
4746 'pid={pid}'.format(pid
=os
.getpid()),
4751 if get_ceph_user() == 'ceph':
4752 command_check_call(['chown', 'ceph:ceph', args
.dev
])
4753 parttype
= get_partition_type(args
.dev
)
4754 partid
= get_partition_uuid(args
.dev
)
4756 LOG
.info('trigger {dev} parttype {parttype} uuid {partid}'.format(
4762 ceph_disk
= ['ceph-disk']
4764 ceph_disk
.append('--verbose')
4766 if parttype
in (PTYPE
['regular']['osd']['ready'],
4767 PTYPE
['mpath']['osd']['ready']):
4768 out
, err
, ret
= command(
4776 elif parttype
in (PTYPE
['plain']['osd']['ready'],
4777 PTYPE
['luks']['osd']['ready']):
4778 out
, err
, ret
= command(
4787 elif parttype
in (PTYPE
['regular']['journal']['ready'],
4788 PTYPE
['mpath']['journal']['ready']):
4789 out
, err
, ret
= command(
4797 elif parttype
in (PTYPE
['plain']['journal']['ready'],
4798 PTYPE
['luks']['journal']['ready']):
4799 out
, err
, ret
= command(
4808 elif parttype
in (PTYPE
['regular']['block']['ready'],
4809 PTYPE
['regular']['block.db']['ready'],
4810 PTYPE
['regular']['block.wal']['ready'],
4811 PTYPE
['mpath']['block']['ready'],
4812 PTYPE
['mpath']['block.db']['ready'],
4813 PTYPE
['mpath']['block.wal']['ready']):
4814 out
, err
, ret
= command(
4822 elif parttype
in (PTYPE
['plain']['block']['ready'],
4823 PTYPE
['plain']['block.db']['ready'],
4824 PTYPE
['plain']['block.wal']['ready'],
4825 PTYPE
['luks']['block']['ready'],
4826 PTYPE
['luks']['block.db']['ready'],
4827 PTYPE
['luks']['block.wal']['ready']):
4828 out
, err
, ret
= command(
4837 elif parttype
in (PTYPE
['regular']['lockbox']['ready'],
4838 PTYPE
['mpath']['lockbox']['ready']):
4839 out
, err
, ret
= command(
4848 raise Error('unrecognized partition type %s' % parttype
)
4853 raise Error('return code ' + str(ret
))
4860 # A hash table containing 'path': ('uid', 'gid', blocking, recursive)
4862 ('/usr/bin/ceph-mon', 'root', ROOTGROUP
, True, False),
4863 ('/usr/bin/ceph-mds', 'root', ROOTGROUP
, True, False),
4864 ('/usr/bin/ceph-osd', 'root', ROOTGROUP
, True, False),
4865 ('/usr/bin/radosgw', 'root', ROOTGROUP
, True, False),
4866 ('/etc/ceph', 'root', ROOTGROUP
, True, True),
4867 ('/var/run/ceph', 'ceph', 'ceph', True, True),
4868 ('/var/log/ceph', 'ceph', 'ceph', True, True),
4869 ('/var/log/radosgw', 'ceph', 'ceph', True, True),
4870 ('/var/lib/ceph', 'ceph', 'ceph', True, False),
4873 # Relabel/chown all files under /var/lib/ceph/ recursively (except for osd)
4874 for directory
in glob
.glob('/var/lib/ceph/*'):
4875 if directory
== '/var/lib/ceph/osd':
4876 fix_table
.append((directory
, 'ceph', 'ceph', True, False))
4878 fix_table
.append((directory
, 'ceph', 'ceph', True, True))
4880 # Relabel/chown the osds recursively and in parallel
4881 for directory
in glob
.glob('/var/lib/ceph/osd/*'):
4882 fix_table
.append((directory
, 'ceph', 'ceph', False, True))
4884 LOG
.debug("fix_table: " + str(fix_table
))
4886 # The lists of background processes
4888 permissions_processes
= []
4889 selinux_processes
= []
4891 # Preliminary checks
4892 if args
.selinux
or args
.all
:
4893 out
, err
, ret
= command(['selinuxenabled'])
4895 LOG
.error('SELinux is not enabled, please enable it, first.')
4896 raise Error('no SELinux')
4898 for daemon
in ['ceph-mon', 'ceph-osd', 'ceph-mds', 'radosgw', 'ceph-mgr']:
4899 out
, err
, ret
= command(['pgrep', daemon
])
4901 LOG
.error(daemon
+ ' is running, please stop it, first')
4902 raise Error(daemon
+ ' running')
4904 # Relabel the basic system data without the ceph files
4905 if args
.system
or args
.all
:
4906 c
= ['restorecon', '-R', '/']
4907 for directory
, _
, _
, _
, _
in fix_table
:
4908 # Skip /var/lib/ceph subdirectories
4909 if directory
.startswith('/var/lib/ceph/'):
4914 out
, err
, ret
= command(c
)
4917 LOG
.error("Failed to restore labels of the underlying system")
4919 raise Error("basic restore failed")
4921 # Use find to relabel + chown ~simultaenously
4923 for directory
, uid
, gid
, blocking
, recursive
in fix_table
:
4924 # Skip directories/files that are not installed
4925 if not os
.access(directory
, os
.F_OK
):
4933 ':'.join((uid
, gid
)),
4942 # Just pass -maxdepth 0 for non-recursive calls
4944 c
+= ['-maxdepth', '0']
4947 out
, err
, ret
= command(c
)
4950 LOG
.error("Failed to fix " + directory
)
4952 raise Error(directory
+ " fix failed")
4954 all_processes
.append(command_init(c
))
4956 LOG
.debug("all_processes: " + str(all_processes
))
4957 for process
in all_processes
:
4958 out
, err
, ret
= command_wait(process
)
4960 LOG
.error("A background find process failed")
4962 raise Error("background failed")
4965 if args
.permissions
:
4966 for directory
, uid
, gid
, blocking
, recursive
in fix_table
:
4967 # Skip directories/files that are not installed
4968 if not os
.access(directory
, os
.F_OK
):
4975 ':'.join((uid
, gid
)),
4981 ':'.join((uid
, gid
)),
4986 out
, err
, ret
= command(c
)
4989 LOG
.error("Failed to chown " + directory
)
4991 raise Error(directory
+ " chown failed")
4993 permissions_processes
.append(command_init(c
))
4995 LOG
.debug("permissions_processes: " + str(permissions_processes
))
4996 for process
in permissions_processes
:
4997 out
, err
, ret
= command_wait(process
)
4999 LOG
.error("A background permissions process failed")
5001 raise Error("background failed")
5003 # Fix SELinux labels
5005 for directory
, uid
, gid
, blocking
, recursive
in fix_table
:
5006 # Skip directories/files that are not installed
5007 if not os
.access(directory
, os
.F_OK
):
5023 out
, err
, ret
= command(c
)
5026 LOG
.error("Failed to restore labels for " + directory
)
5028 raise Error(directory
+ " relabel failed")
5030 selinux_processes
.append(command_init(c
))
5032 LOG
.debug("selinux_processes: " + str(selinux_processes
))
5033 for process
in selinux_processes
:
5034 out
, err
, ret
= command_wait(process
)
5036 LOG
.error("A background selinux process failed")
5038 raise Error("background failed")
5041 "The ceph files has been fixed, please reboot "
5042 "the system for the changes to take effect."
5046 def setup_statedir(dir):
5047 # XXX The following use of globals makes linting
5048 # really hard. Global state in Python is iffy and
5049 # should be avoided.
5053 if not os
.path
.exists(STATEDIR
):
5055 if not os
.path
.exists(STATEDIR
+ "/tmp"):
5056 os
.mkdir(STATEDIR
+ "/tmp")
5059 prepare_lock
= FileLock(STATEDIR
+ '/tmp/ceph-disk.prepare.lock')
5061 global activate_lock
5062 activate_lock
= FileLock(STATEDIR
+ '/tmp/ceph-disk.activate.lock')
5064 global SUPPRESS_PREFIX
5065 SUPPRESS_PREFIX
= STATEDIR
+ '/tmp/suppress-activate.'
5068 def setup_sysconfdir(dir):
5073 def parse_args(argv
):
5074 parser
= argparse
.ArgumentParser(
5077 parser
.add_argument(
5079 action
='store_true', default
=None,
5080 help='be more verbose',
5082 parser
.add_argument(
5084 action
='store_true', default
=None,
5085 help='log to stdout',
5087 parser
.add_argument(
5088 '--prepend-to-path',
5091 help=('prepend PATH to $PATH for backward compatibility '
5092 '(default /usr/bin)'),
5094 parser
.add_argument(
5097 default
='/var/lib/ceph',
5098 help=('directory in which ceph state is preserved '
5099 '(default /var/lib/ceph)'),
5101 parser
.add_argument(
5104 default
='/etc/ceph',
5105 help=('directory in which ceph configuration files are found '
5106 '(default /etc/ceph)'),
5108 parser
.add_argument(
5112 help='use the given user for subprocesses, rather than ceph or root'
5114 parser
.add_argument(
5118 help='use the given group for subprocesses, rather than ceph or root'
5120 parser
.set_defaults(
5121 # we want to hold on to this, for later
5125 subparsers
= parser
.add_subparsers(
5126 title
='subcommands',
5127 description
='valid subcommands',
5128 help='sub-command help',
5131 Prepare
.set_subparser(subparsers
)
5132 make_activate_parser(subparsers
)
5133 make_activate_lockbox_parser(subparsers
)
5134 make_activate_block_parser(subparsers
)
5135 make_activate_journal_parser(subparsers
)
5136 make_activate_all_parser(subparsers
)
5137 make_list_parser(subparsers
)
5138 make_suppress_parser(subparsers
)
5139 make_deactivate_parser(subparsers
)
5140 make_destroy_parser(subparsers
)
5141 make_zap_parser(subparsers
)
5142 make_trigger_parser(subparsers
)
5143 make_fix_parser(subparsers
)
5145 args
= parser
.parse_args(argv
)
5149 def make_fix_parser(subparsers
):
5150 fix_parser
= subparsers
.add_parser(
5152 formatter_class
=argparse
.RawDescriptionHelpFormatter
,
5153 description
=textwrap
.fill(textwrap
.dedent("""\
5155 help='fix SELinux labels and/or file permissions')
5157 fix_parser
.add_argument(
5159 action
='store_true',
5161 help='fix SELinux labels for the non-ceph system data'
5163 fix_parser
.add_argument(
5165 action
='store_true',
5167 help='fix SELinux labels for ceph data'
5169 fix_parser
.add_argument(
5171 action
='store_true',
5173 help='fix file permissions for ceph data'
5175 fix_parser
.add_argument(
5177 action
='store_true',
5179 help='perform all the fix-related operations'
5181 fix_parser
.set_defaults(
5187 def make_trigger_parser(subparsers
):
5188 trigger_parser
= subparsers
.add_parser(
5190 formatter_class
=argparse
.RawDescriptionHelpFormatter
,
5191 description
=textwrap
.fill(textwrap
.dedent("""\
5192 The partition given in argument is activated. The type of the
5193 partition (data, lockbox, journal etc.) is detected by its
5194 type. If the init system is upstart or systemd, the activation is
5195 delegated to it and runs asynchronously, which
5196 helps reduce the execution time of udev actions.
5198 help='activate any device (called by udev)')
5199 trigger_parser
.add_argument(
5203 trigger_parser
.add_argument(
5207 help='cluster name to assign this disk to',
5209 trigger_parser
.add_argument(
5211 action
='store_true', default
=None,
5212 help='map devices with dm-crypt',
5214 trigger_parser
.add_argument(
5215 '--dmcrypt-key-dir',
5217 default
='/etc/ceph/dmcrypt-keys',
5218 help='directory where dm-crypt keys are stored',
5220 trigger_parser
.add_argument(
5222 action
='store_true', default
=None,
5223 help='do operation synchronously; do not trigger systemd',
5225 trigger_parser
.set_defaults(
5228 return trigger_parser
5231 def make_activate_parser(subparsers
):
5232 activate_parser
= subparsers
.add_parser(
5234 formatter_class
=argparse
.RawDescriptionHelpFormatter
,
5235 description
=textwrap
.fill(textwrap
.dedent("""\
5236 Activate the OSD found at PATH (can be a directory
5237 or a device partition, possibly encrypted). When
5238 activated for the first time, a unique OSD id is obtained
5239 from the cluster. If PATH is a directory, a symbolic
5240 link is added in {statedir}/osd/ceph-$id. If PATH is
5241 a partition, it is mounted on {statedir}/osd/ceph-$id.
5242 Finally, the OSD daemon is run.
5244 If the OSD depends on auxiliary partitions (journal, block, ...)
5245 they need to be available otherwise activation will fail. It
5246 may happen if a journal is encrypted and cryptsetup was not
5248 """.format(statedir
=STATEDIR
))),
5249 help='Activate a Ceph OSD')
5250 activate_parser
.add_argument(
5252 action
='store_true', default
=None,
5253 help='mount a block device [deprecated, ignored]',
5255 activate_parser
.add_argument(
5258 help='bootstrap-osd keyring path template (%(default)s)',
5259 dest
='activate_key_template',
5261 activate_parser
.add_argument(
5263 metavar
='INITSYSTEM',
5264 help='init system to manage this dir',
5266 choices
=INIT_SYSTEMS
,
5268 activate_parser
.add_argument(
5269 '--no-start-daemon',
5270 action
='store_true', default
=None,
5271 help='do not start the daemon',
5273 activate_parser
.add_argument(
5276 help='path to block device or directory',
5278 activate_parser
.add_argument(
5280 action
='store_true', default
=None,
5281 help='map DATA and/or JOURNAL devices with dm-crypt',
5283 activate_parser
.add_argument(
5284 '--dmcrypt-key-dir',
5286 default
='/etc/ceph/dmcrypt-keys',
5287 help='directory where dm-crypt keys are stored',
5289 activate_parser
.add_argument(
5291 action
='store_true', default
=False,
5292 help='activate the deactived OSD',
5294 activate_parser
.set_defaults(
5295 activate_key_template
='{statedir}/bootstrap-osd/{cluster}.keyring',
5298 return activate_parser
5301 def make_activate_lockbox_parser(subparsers
):
5302 parser
= subparsers
.add_parser(
5304 formatter_class
=argparse
.RawDescriptionHelpFormatter
,
5305 description
=textwrap
.fill(textwrap
.dedent("""\
5306 Mount the partition found at PATH on {statedir}/osd-lockbox/$uuid
5307 where $uuid uniquely identifies the OSD that needs this lockbox
5308 to retrieve keys from the monitor and unlock its partitions.
5310 If the OSD has one or more auxiliary devices (journal, block, ...)
5311 symbolic links are created at {statedir}/osd-lockbox/$other_uuid
5312 and point to {statedir}/osd-lockbox/$uuid. This will, for instance,
5313 allow a journal encrypted in a partition identified by $other_uuid to
5314 fetch the keys it needs from the monitor.
5316 Finally the OSD is activated, as it would be with ceph-disk activate.
5317 """.format(statedir
=STATEDIR
))),
5318 help='Activate a Ceph lockbox')
5319 parser
.add_argument(
5321 help='bootstrap-osd keyring path template (%(default)s)',
5322 dest
='activate_key_template',
5324 parser
.add_argument(
5325 '--dmcrypt-key-dir',
5327 default
='/etc/ceph/dmcrypt-keys',
5328 help='directory where dm-crypt keys are stored',
5330 parser
.add_argument(
5333 help='path to block device',
5335 parser
.set_defaults(
5336 activate_key_template
='{statedir}/bootstrap-osd/{cluster}.keyring',
5337 func
=main_activate_lockbox
,
5342 def make_activate_block_parser(subparsers
):
5343 return make_activate_space_parser('block', subparsers
)
5346 def make_activate_journal_parser(subparsers
):
5347 return make_activate_space_parser('journal', subparsers
)
5350 def make_activate_space_parser(name
, subparsers
):
5351 activate_space_parser
= subparsers
.add_parser(
5352 'activate-%s' % name
,
5353 formatter_class
=argparse
.RawDescriptionHelpFormatter
,
5354 description
=textwrap
.fill(textwrap
.dedent("""\
5355 Activating a {name} partition is only meaningfull
5356 if it is encrypted and it will map it using
5359 Finally the corresponding OSD is activated,
5360 as it would be with ceph-disk activate.
5361 """.format(name
=name
))),
5362 help='Activate an OSD via its %s device' % name
)
5363 activate_space_parser
.add_argument(
5366 help='path to %s block device' % name
,
5368 activate_space_parser
.add_argument(
5371 help='bootstrap-osd keyring path template (%(default)s)',
5372 dest
='activate_key_template',
5374 activate_space_parser
.add_argument(
5376 metavar
='INITSYSTEM',
5377 help='init system to manage this dir',
5379 choices
=INIT_SYSTEMS
,
5381 activate_space_parser
.add_argument(
5383 action
='store_true', default
=None,
5384 help=('map data and/or auxiliariy (journal, etc.) '
5385 'devices with dm-crypt'),
5387 activate_space_parser
.add_argument(
5388 '--dmcrypt-key-dir',
5390 default
='/etc/ceph/dmcrypt-keys',
5391 help='directory where dm-crypt keys are stored',
5393 activate_space_parser
.add_argument(
5395 action
='store_true', default
=False,
5396 help='activate the deactived OSD',
5398 activate_space_parser
.set_defaults(
5399 activate_key_template
='{statedir}/bootstrap-osd/{cluster}.keyring',
5400 func
=lambda args
: main_activate_space(name
, args
),
5402 return activate_space_parser
5405 def make_activate_all_parser(subparsers
):
5406 activate_all_parser
= subparsers
.add_parser(
5408 formatter_class
=argparse
.RawDescriptionHelpFormatter
,
5409 description
=textwrap
.fill(textwrap
.dedent("""\
5410 Activate all OSD partitions found in /dev/disk/by-parttypeuuid.
5411 The partitions containing auxiliary devices (journal, block, ...)
5414 help='Activate all tagged OSD partitions')
5415 activate_all_parser
.add_argument(
5418 help='bootstrap-osd keyring path template (%(default)s)',
5419 dest
='activate_key_template',
5421 activate_all_parser
.add_argument(
5423 metavar
='INITSYSTEM',
5424 help='init system to manage this dir',
5426 choices
=INIT_SYSTEMS
,
5428 activate_all_parser
.set_defaults(
5429 activate_key_template
='{statedir}/bootstrap-osd/{cluster}.keyring',
5430 func
=main_activate_all
,
5432 return activate_all_parser
5435 def make_list_parser(subparsers
):
5436 list_parser
= subparsers
.add_parser(
5438 formatter_class
=argparse
.RawDescriptionHelpFormatter
,
5439 description
=textwrap
.fill(textwrap
.dedent("""\
5440 Display all partitions on the system and their
5441 associated Ceph information, if any.
5443 help='List disks, partitions, and Ceph OSDs')
5444 list_parser
.add_argument(
5446 help='output format',
5448 choices
=['json', 'plain'],
5450 list_parser
.add_argument(
5454 help='path to block devices, relative to /sys/block',
5456 list_parser
.set_defaults(
5462 def make_suppress_parser(subparsers
):
5463 suppress_parser
= subparsers
.add_parser(
5464 'suppress-activate',
5465 formatter_class
=argparse
.RawDescriptionHelpFormatter
,
5466 description
=textwrap
.fill(textwrap
.dedent("""\
5467 Add a prefix to the list of suppressed device names
5468 so that they are ignored by all activate* subcommands.
5470 help='Suppress activate on a device (prefix)')
5471 suppress_parser
.add_argument(
5474 help='path to block device or directory',
5476 suppress_parser
.set_defaults(
5480 unsuppress_parser
= subparsers
.add_parser(
5481 'unsuppress-activate',
5482 formatter_class
=argparse
.RawDescriptionHelpFormatter
,
5483 description
=textwrap
.fill(textwrap
.dedent("""\
5484 Remove a prefix from the list of suppressed device names
5485 so that they are no longer ignored by all
5486 activate* subcommands.
5488 help='Stop suppressing activate on a device (prefix)')
5489 unsuppress_parser
.add_argument(
5492 help='path to block device or directory',
5494 unsuppress_parser
.set_defaults(
5495 func
=main_unsuppress
,
5497 return suppress_parser
5500 def make_deactivate_parser(subparsers
):
5501 deactivate_parser
= subparsers
.add_parser(
5503 formatter_class
=argparse
.RawDescriptionHelpFormatter
,
5504 description
=textwrap
.fill(textwrap
.dedent("""\
5505 Deactivate the OSD located at PATH. It stops the OSD daemon
5506 and optionally marks it out (with --mark-out). The content of
5507 the OSD is left untouched.
5509 By default, the, ready, active, INIT-specific files are
5510 removed (so that it is not automatically re-activated by the
5511 udev rules or ceph-disk trigger) and the file deactive is
5512 created to remember the OSD is deactivated.
5514 If the --once option is given, the ready, active, INIT-specific
5515 files are not removed and the OSD will reactivate whenever
5516 ceph-disk trigger is run on one of the devices (journal, data,
5517 block, lockbox, ...).
5519 If the OSD is dmcrypt, remove the data dmcrypt map. When
5520 deactivate finishes, the OSD is down.
5522 help='Deactivate a Ceph OSD')
5523 deactivate_parser
.add_argument(
5527 help='cluster name to assign this disk to',
5529 deactivate_parser
.add_argument(
5533 help='path to block device or directory',
5535 deactivate_parser
.add_argument(
5536 '--deactivate-by-id',
5538 help='ID of OSD to deactive'
5540 deactivate_parser
.add_argument(
5542 action
='store_true', default
=False,
5543 help='option to mark the osd out',
5545 deactivate_parser
.add_argument(
5547 action
='store_true', default
=False,
5548 help='does not need --reactivate to activate again',
5550 deactivate_parser
.set_defaults(
5551 func
=main_deactivate
,
5555 def make_destroy_parser(subparsers
):
5556 destroy_parser
= subparsers
.add_parser(
5558 formatter_class
=argparse
.RawDescriptionHelpFormatter
,
5559 description
=textwrap
.fill(textwrap
.dedent("""\ Destroy the OSD located at PATH. It removes the OSD from the
5560 cluster and marks it destroyed. An OSD must be down before it
5561 can be destroyed. Once it is destroyed, a new OSD can be created
5562 in its place, reusing the same OSD id and position (e.g. after
5563 a failed HDD or SSD is replaced). Alternatively, if the
5564 --purge option is also specified, the OSD is removed from the
5565 CRUSH map and the OSD id is deallocated.""")),
5566 help='Destroy a Ceph OSD')
5567 destroy_parser
.add_argument(
5571 help='cluster name to assign this disk to',
5573 destroy_parser
.add_argument(
5577 help='path to block device or directory',
5579 destroy_parser
.add_argument(
5582 help='ID of OSD to destroy'
5584 destroy_parser
.add_argument(
5585 '--dmcrypt-key-dir',
5587 default
='/etc/ceph/dmcrypt-keys',
5588 help=('directory where dm-crypt keys are stored '
5589 '(If you don\'t know how it work, '
5590 'dont use it. we have default value)'),
5592 destroy_parser
.add_argument(
5594 action
='store_true', default
=False,
5595 help='option to erase data and partition',
5597 destroy_parser
.add_argument(
5599 action
='store_true', default
=False,
5600 help='option to remove OSD from CRUSH map and deallocate the id',
5602 destroy_parser
.set_defaults(
5607 def make_zap_parser(subparsers
):
5608 zap_parser
= subparsers
.add_parser(
5610 formatter_class
=argparse
.RawDescriptionHelpFormatter
,
5611 description
=textwrap
.fill(textwrap
.dedent("""\
5612 Zap/erase/destroy a device's partition table and contents. It
5613 actually uses sgdisk and it's option --zap-all to
5614 destroy both GPT and MBR data structures so that the disk
5615 becomes suitable for repartitioning.
5617 help='Zap/erase/destroy a device\'s partition table (and contents)')
5618 zap_parser
.add_argument(
5622 help='path to block device',
5624 zap_parser
.set_defaults(
5631 args
= parse_args(argv
)
5633 setup_logging(args
.verbose
, args
.log_stdout
)
5635 if args
.prepend_to_path
!= '':
5636 path
= os
.environ
.get('PATH', os
.defpath
)
5637 os
.environ
['PATH'] = args
.prepend_to_path
+ ":" + path
5639 if args
.func
.__name
__ != 'main_trigger':
5640 # trigger may run when statedir is unavailable and does not use it
5641 setup_statedir(args
.statedir
)
5642 setup_sysconfdir(args
.sysconfdir
)
5644 global CEPH_PREF_USER
5645 CEPH_PREF_USER
= args
.setuser
5646 global CEPH_PREF_GROUP
5647 CEPH_PREF_GROUP
= args
.setgroup
5652 main_catch(args
.func
, args
)
5655 def setup_logging(verbose
, log_stdout
):
5656 loglevel
= logging
.WARNING
5658 loglevel
= logging
.DEBUG
5661 ch
= logging
.StreamHandler(stream
=sys
.stdout
)
5662 ch
.setLevel(loglevel
)
5663 formatter
= logging
.Formatter('%(funcName)s: %(message)s')
5664 ch
.setFormatter(formatter
)
5666 LOG
.setLevel(loglevel
)
5668 logging
.basicConfig(
5670 format
='%(funcName)s: %(message)s',
5674 def main_catch(func
, args
):
5681 '{prog}: {msg}'.format(
5687 except CephDiskException
as error
:
5688 exc_name
= error
.__class
__.__name
__
5690 '{prog} {exc_name}: {msg}'.format(
5702 if __name__
== '__main__':