3 # Copyright (C) 2015, 2016, 2017 Red Hat <contact@redhat.com>
4 # Copyright (C) 2014 Inktank <info@inktank.com>
5 # Copyright (C) 2014 Cloudwatt <libre.licensing@cloudwatt.com>
6 # Copyright (C) 2014 Catalyst.net Ltd
8 # Author: Loic Dachary <loic@dachary.org>
10 # This program is free software; you can redistribute it and/or modify
11 # it under the terms of the GNU Library Public License as published by
12 # the Free Software Foundation; either version 2, or (at your option)
15 # This program is distributed in the hope that it will be useful,
16 # but WITHOUT ANY WARRANTY; without even the implied warranty of
17 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18 # GNU Library Public License for more details.
21 from __future__
import print_function
46 CEPH_OSD_ONDISK_MAGIC
= 'ceph osd volume v026'
47 CEPH_LOCKBOX_ONDISK_MAGIC
= 'ceph lockbox volume v001'
49 KEY_MANAGEMENT_MODE_V1
= 'ceph-mon v1'
54 # identical because creating a journal is atomic
55 'ready': '45b0969e-9b03-4f30-b4c6-b4b80ceff106',
56 'tobe': '45b0969e-9b03-4f30-b4c6-b4b80ceff106',
59 # identical because creating a block is atomic
60 'ready': 'cafecafe-9b03-4f30-b4c6-b4b80ceff106',
61 'tobe': 'cafecafe-9b03-4f30-b4c6-b4b80ceff106',
64 # identical because creating a block is atomic
65 'ready': '30cd0809-c2b2-499c-8879-2d6b78529876',
66 'tobe': '30cd0809-c2b2-499c-8879-2d6b785292be',
69 # identical because creating a block is atomic
70 'ready': '5ce17fce-4087-4169-b7ff-056cc58473f9',
71 'tobe': '5ce17fce-4087-4169-b7ff-056cc58472be',
74 'ready': '4fbd7e29-9d25-41b8-afd0-062c0ceff05d',
75 'tobe': '89c57f98-2fe5-4dc0-89c1-f3ad0ceff2be',
78 'ready': 'fb3aabf9-d25f-47cc-bf5e-721d1816496b',
79 'tobe': 'fb3aabf9-d25f-47cc-bf5e-721d181642be',
84 'ready': '45b0969e-9b03-4f30-b4c6-35865ceff106',
85 'tobe': '89c57f98-2fe5-4dc0-89c1-35865ceff2be',
88 'ready': 'cafecafe-9b03-4f30-b4c6-35865ceff106',
89 'tobe': '89c57f98-2fe5-4dc0-89c1-35865ceff2be',
92 'ready': '166418da-c469-4022-adf4-b30afd37f176',
93 'tobe': '7521c784-4626-4260-bc8d-ba77a0f5f2be',
96 'ready': '86a32090-3647-40b9-bbbd-38d8c573aa86',
97 'tobe': '92dad30f-175b-4d40-a5b0-5c0a258b42be',
100 'ready': '4fbd7e29-9d25-41b8-afd0-35865ceff05d',
101 'tobe': '89c57f98-2fe5-4dc0-89c1-5ec00ceff2be',
106 'ready': '45b0969e-9b03-4f30-b4c6-5ec00ceff106',
107 'tobe': '89c57f98-2fe5-4dc0-89c1-35865ceff2be',
110 'ready': 'cafecafe-9b03-4f30-b4c6-5ec00ceff106',
111 'tobe': '89c57f98-2fe5-4dc0-89c1-35865ceff2be',
114 'ready': '93b0052d-02d9-4d8a-a43b-33a3ee4dfbc3',
115 'tobe': '69d17c68-3e58-4399-aff0-b68265f2e2be',
118 'ready': '306e8683-4fe2-4330-b7c0-00a917c16966',
119 'tobe': 'f2d89683-a621-4063-964a-eb1f7863a2be',
122 'ready': '4fbd7e29-9d25-41b8-afd0-5ec00ceff05d',
123 'tobe': '89c57f98-2fe5-4dc0-89c1-5ec00ceff2be',
128 'ready': '45b0969e-8ae0-4982-bf9d-5a8d867af560',
129 'tobe': '45b0969e-8ae0-4982-bf9d-5a8d867af560',
132 'ready': 'cafecafe-8ae0-4982-bf9d-5a8d867af560',
133 'tobe': 'cafecafe-8ae0-4982-bf9d-5a8d867af560',
136 'ready': 'ec6d6385-e346-45dc-be91-da2a7c8b3261',
137 'tobe': 'ec6d6385-e346-45dc-be91-da2a7c8b32be',
140 'ready': '01b41e1b-002a-453c-9f17-88793989ff8f',
141 'tobe': '01b41e1b-002a-453c-9f17-88793989f2be',
144 'ready': '4fbd7e29-8ae0-4982-bf9d-5a8d867af560',
145 'tobe': '89c57f98-8ae0-4982-bf9d-5a8d867af560',
148 'ready': '7f4a666a-16f3-47a2-8445-152ef4d03f6c',
149 'tobe': '7f4a666a-16f3-47a2-8445-152ef4d032be',
155 # see https://bugs.python.org/issue23098
157 except OverflowError:
158 os
.major
= lambda devid
: ((devid
>> 8) & 0xfff) |
((devid
>> 32) & ~
0xfff)
159 os
.minor
= lambda devid
: (devid
& 0xff) |
((devid
>> 12) & ~
0xff)
165 def get_ready_by_type(what
):
166 return [x
['ready'] for x
in PTYPE
[what
].values()]
169 def get_ready_by_name(name
):
170 return [x
[name
]['ready'] for x
in PTYPE
.values() if name
in x
]
173 def is_regular_space(ptype
):
174 return Ptype
.is_what_space('regular', ptype
)
177 def is_mpath_space(ptype
):
178 return Ptype
.is_what_space('mpath', ptype
)
181 def is_plain_space(ptype
):
182 return Ptype
.is_what_space('plain', ptype
)
185 def is_luks_space(ptype
):
186 return Ptype
.is_what_space('luks', ptype
)
189 def is_what_space(what
, ptype
):
190 for name
in Space
.NAMES
:
191 if ptype
== PTYPE
[what
][name
]['ready']:
196 def space_ptype_to_name(ptype
):
197 for what
in PTYPE
.values():
198 for name
in Space
.NAMES
:
199 if ptype
== what
[name
]['ready']:
201 raise ValueError('ptype ' + ptype
+ ' not found')
204 def is_dmcrypt_space(ptype
):
205 for name
in Space
.NAMES
:
206 if Ptype
.is_dmcrypt(ptype
, name
):
211 def is_dmcrypt(ptype
, name
):
212 for what
in ('plain', 'luks'):
213 if ptype
== PTYPE
[what
][name
]['ready']:
220 if platform
.system() == 'FreeBSD':
222 DEFAULT_FS_TYPE
= 'zfs'
223 PROCDIR
= '/compat/linux/proc'
224 # FreeBSD does not have blockdevices any more
229 DEFAULT_FS_TYPE
= 'xfs'
231 BLOCKDIR
= '/sys/block'
235 OSD STATUS Definition
237 OSD_STATUS_OUT_DOWN
= 0
238 OSD_STATUS_OUT_UP
= 1
239 OSD_STATUS_IN_DOWN
= 2
242 MOUNT_OPTIONS
= dict(
243 btrfs
='noatime,user_subvol_rm_allowed',
244 # user_xattr is default ever since linux 2.6.39 / 3.0, but we'll
245 # delay a moment before removing it fully because we did have some
246 # issues with ext4 before the xatts-in-leveldb work, and it seemed
247 # that user_xattr helped
248 ext4
='noatime,user_xattr',
249 xfs
='noatime,inode64',
254 # btrfs requires -f, for the same reason as xfs (see comment below)
261 # xfs insists on not overwriting previous fs; even if we wipe
262 # partition table, we often recreate it exactly the same way,
263 # so we'll see ghosts of filesystems past
282 STATEDIR
= '/var/lib/ceph'
284 SYSCONFDIR
= '/etc/ceph'
288 SUPPRESS_PREFIX
= None
290 # only warn once about some things
293 # Nuke the TERM variable to avoid confusing any subprocesses we call.
294 # For example, libreadline will print weird control sequences for some
296 if 'TERM' in os
.environ
:
297 del os
.environ
['TERM']
300 if LOG_NAME
== '__main__':
301 LOG_NAME
= os
.path
.basename(sys
.argv
[0])
302 LOG
= logging
.getLogger(LOG_NAME
)
304 # Allow user-preferred values for subprocess user and group
305 CEPH_PREF_USER
= None
306 CEPH_PREF_GROUP
= None
309 class FileLock(object):
310 def __init__(self
, fn
):
316 self
.fd
= os
.open(self
.fn
, os
.O_WRONLY | os
.O_CREAT
)
317 fcntl
.lockf(self
.fd
, fcntl
.LOCK_EX
)
319 def __exit__(self
, exc_type
, exc_val
, exc_tb
):
321 fcntl
.lockf(self
.fd
, fcntl
.LOCK_UN
)
326 class Error(Exception):
332 doc
= _bytes2str(self
.__doc
__.strip())
334 str_type
= basestring
337 args
= [a
if isinstance(a
, str_type
) else str(a
) for a
in self
.args
]
338 return ': '.join([doc
] + [_bytes2str(a
) for a
in args
])
341 class MountError(Error
):
343 Mounting filesystem failed
347 class UnmountError(Error
):
349 Unmounting filesystem failed
353 class BadMagicError(Error
):
355 Does not look like a Ceph OSD, or incompatible version
359 class TruncatedLineError(Error
):
365 class TooManyLinesError(Error
):
371 class FilesystemTypeError(Error
):
373 Cannot discover filesystem type
377 class CephDiskException(Exception):
379 A base exception for ceph-disk to provide custom (ad-hoc) messages that
380 will be caught and dealt with when main() is executed
385 class ExecutableNotFound(CephDiskException
):
387 Exception to report on executables not available in PATH
394 Detect whether systemd is running
396 with
open(PROCDIR
+ '/1/comm', 'r') as f
:
397 return 'systemd' in f
.read()
402 Detect whether upstart is running
404 (out
, err
, _
) = command(['init', '--version'])
405 return 'upstart' in out
408 def maybe_mkdir(*a
, **kw
):
410 Creates a new directory if it doesn't exist, removes
411 existing symlink before creating the directory.
413 # remove any symlink, if it is there..
414 if os
.path
.exists(*a
) and stat
.S_ISLNK(os
.lstat(*a
).st_mode
):
415 LOG
.debug('Removing old symlink at %s', *a
)
420 if e
.errno
== errno
.EEXIST
:
426 def which(executable
):
427 """find the location of an executable"""
428 envpath
= os
.environ
.get('PATH') or os
.defpath
429 PATH
= envpath
.split(os
.pathsep
)
440 for location
in locations
:
441 executable_path
= os
.path
.join(location
, executable
)
442 if (os
.path
.isfile(executable_path
) and
443 os
.access(executable_path
, os
.X_OK
)):
444 return executable_path
447 def _get_command_executable(arguments
):
449 Return the full path for an executable, raise if the executable is not
450 found. If the executable has already a full path do not perform any checks.
452 if os
.path
.isabs(arguments
[0]): # an absolute path
454 executable
= which(arguments
[0])
456 command_msg
= 'Could not run command: %s' % ' '.join(arguments
)
457 executable_msg
= '%s not in path.' % arguments
[0]
458 raise ExecutableNotFound('%s %s' % (executable_msg
, command_msg
))
460 # swap the old executable for the new one
461 arguments
[0] = executable
465 def command(arguments
, **kwargs
):
467 Safely execute a ``subprocess.Popen`` call making sure that the
468 executable exists and raising a helpful error message
471 .. note:: This should be the preferred way of calling ``subprocess.Popen``
472 since it provides the caller with the safety net of making sure that
473 executables *will* be found and will error nicely otherwise.
475 This returns the output of the command and the return code of the
476 process in a tuple: (stdout, stderr, returncode).
479 arguments
= list(map(_bytes2str
, _get_command_executable(arguments
)))
481 LOG
.info('Running command: %s' % ' '.join(arguments
))
482 process
= subprocess
.Popen(
484 stdout
=subprocess
.PIPE
,
485 stderr
=subprocess
.PIPE
,
487 out
, err
= process
.communicate()
489 return _bytes2str(out
), _bytes2str(err
), process
.returncode
492 def command_with_stdin(arguments
, stdin
):
493 LOG
.info("Running command with stdin: " + " ".join(arguments
))
494 process
= subprocess
.Popen(
496 stdin
=subprocess
.PIPE
,
497 stdout
=subprocess
.PIPE
,
498 stderr
=subprocess
.PIPE
)
499 out
, err
= process
.communicate(stdin
)
501 if process
.returncode
!= 0:
504 "'{cmd}' failed with status code {returncode}".format(
506 returncode
=process
.returncode
,
512 def _bytes2str(string
):
513 return string
.decode('utf-8') if isinstance(string
, bytes
) else string
516 def command_init(arguments
, **kwargs
):
518 Safely execute a non-blocking ``subprocess.Popen`` call
519 making sure that the executable exists and raising a helpful
520 error message if it does not.
522 .. note:: This should be the preferred way of calling ``subprocess.Popen``
523 since it provides the caller with the safety net of making sure that
524 executables *will* be found and will error nicely otherwise.
526 This returns the process.
529 arguments
= list(map(_bytes2str
, _get_command_executable(arguments
)))
531 LOG
.info('Running command: %s' % ' '.join(arguments
))
532 process
= subprocess
.Popen(
534 stdout
=subprocess
.PIPE
,
535 stderr
=subprocess
.PIPE
,
540 def command_wait(process
):
542 Wait for the process finish and parse its output.
545 out
, err
= process
.communicate()
547 return _bytes2str(out
), _bytes2str(err
), process
.returncode
550 def command_check_call(arguments
, exit
=False):
552 Safely execute a ``subprocess.check_call`` call making sure that the
553 executable exists and raising a helpful error message if it does not.
555 When ``exit`` is set to ``True`` this helper will do a clean (sans
556 traceback) system exit.
557 .. note:: This should be the preferred way of calling
558 ``subprocess.check_call`` since it provides the caller with the safety net
559 of making sure that executables *will* be found and will error nicely
562 arguments
= _get_command_executable(arguments
)
563 command
= ' '.join(arguments
)
564 LOG
.info('Running command: %s', command
)
566 return subprocess
.check_call(arguments
)
567 except subprocess
.CalledProcessError
as error
:
570 LOG
.error(error
.output
)
572 "'{cmd}' failed with status code {returncode}".format(
574 returncode
=error
.returncode
,
581 # An alternative block_path implementation would be
583 # name = basename(dev)
584 # return /sys/devices/virtual/block/$name
586 # It is however more fragile because it relies on the fact
587 # that the basename of the device the user will use always
588 # matches the one the driver will use. On Ubuntu 14.04, for
589 # instance, when multipath creates a partition table on
591 # /dev/mapper/353333330000007d0 -> ../dm-0
593 # it will create partition devices named
595 # /dev/mapper/353333330000007d0-part1
597 # which is the same device as /dev/dm-1 but not a symbolic
600 # ubuntu@other:~$ ls -l /dev/mapper /dev/dm-1
601 # brw-rw---- 1 root disk 252, 1 Aug 15 17:52 /dev/dm-1
602 # lrwxrwxrwx 1 root root 7 Aug 15 17:52 353333330000007d0 -> ../dm-0
603 # brw-rw---- 1 root disk 252, 1 Aug 15 17:52 353333330000007d0-part1
605 # Using the basename in this case fails.
612 path
= os
.path
.realpath(dev
)
613 rdev
= os
.stat(path
).st_rdev
614 (M
, m
) = (os
.major(rdev
), os
.minor(rdev
))
615 return "{sysfs}/dev/block/{M}:{m}".format(sysfs
=SYSFS
, M
=M
, m
=m
)
618 def get_dm_uuid(dev
):
619 uuid_path
= os
.path
.join(block_path(dev
), 'dm', 'uuid')
620 LOG
.debug("get_dm_uuid " + dev
+ " uuid path is " + uuid_path
)
621 if not os
.path
.exists(uuid_path
):
623 uuid
= open(uuid_path
, 'r').read()
624 LOG
.debug("get_dm_uuid " + dev
+ " uuid is " + uuid
)
630 True if the path is managed by multipath
634 uuid
= get_dm_uuid(dev
)
636 (re
.match('part\d+-mpath-', uuid
) or
637 re
.match('mpath-', uuid
)))
640 def get_dev_name(path
):
642 get device name from path. e.g.::
644 /dev/sda -> sda, /dev/cciss/c0d1 -> cciss!c0d1
646 a device "name" is something like::
652 assert path
.startswith('/dev/')
654 return base
.replace('/', '!')
657 def get_dev_path(name
):
659 get a path (/dev/...) from a name (cciss!c0d1)
660 a device "path" is something like::
666 return '/dev/' + name
.replace('!', '/')
669 def get_dev_relpath(name
):
671 get a relative path to /dev from a name (cciss!c0d1)
673 return name
.replace('!', '/')
676 def get_dev_size(dev
, size
='megabytes'):
678 Attempt to get the size of a device so that we can prevent errors
679 from actions to devices that are smaller, and improve error reporting.
681 Because we want to avoid breakage in case this approach is not robust, we
682 will issue a warning if we failed to get the size.
684 :param size: bytes or megabytes
685 :param dev: the device to calculate the size
687 fd
= os
.open(dev
, os
.O_RDONLY
)
688 dividers
= {'bytes': 1, 'megabytes': 1024 * 1024}
690 device_size
= os
.lseek(fd
, 0, os
.SEEK_END
)
691 divider
= dividers
.get(size
, 1024 * 1024) # default to megabytes
692 return device_size
// divider
693 except Exception as error
:
694 LOG
.warning('failed to get size of %s: %s' % (dev
, str(error
)))
699 def stmode_is_diskdevice(dmode
):
700 if stat
.S_ISBLK(dmode
):
703 # FreeBSD does not have block devices
704 # All disks are character devices
705 return FREEBSD
and stat
.S_ISCHR(dmode
)
708 def dev_is_diskdevice(dev
):
709 dmode
= os
.stat(dev
).st_mode
710 return stmode_is_diskdevice(dmode
)
713 def ldev_is_diskdevice(dev
):
714 dmode
= os
.lstat(dev
).st_mode
715 return stmode_is_diskdevice(dmode
)
718 def path_is_diskdevice(path
):
719 dev
= os
.path
.realpath(path
)
720 return dev_is_diskdevice(dev
)
723 def get_partition_mpath(dev
, pnum
):
724 part_re
= "part{pnum}-mpath-".format(pnum
=pnum
)
725 partitions
= list_partitions_mpath(dev
, part_re
)
732 def retry(on_error
=Exception, max_tries
=10, wait
=0.2, backoff
=0):
734 @functools.wraps(func
)
735 def repeat(*args
, **kwargs
):
736 for tries
in range(max_tries
- 1):
738 return func(*args
, **kwargs
)
740 time
.sleep(wait
+ backoff
* tries
)
741 return func(*args
, **kwargs
)
747 def get_partition_dev(dev
, pnum
):
749 get the device name for a partition
751 assume that partitions are named like the base dev,
752 with a number, and optionally
753 some intervening characters (like 'p'). e.g.,
756 cciss/c0d1 1 -> cciss!c0d1p1
761 partname
= get_partition_mpath(dev
, pnum
)
763 name
= get_dev_name(os
.path
.realpath(dev
))
764 sys_entry
= os
.path
.join(BLOCKDIR
, name
)
765 error_msg
= " in %s" % sys_entry
766 for f
in os
.listdir(sys_entry
):
767 if f
.startswith(name
) and f
.endswith(str(pnum
)):
768 # we want the shortest name that starts with the base name
769 # and ends with the partition number
770 if not partname
or len(f
) < len(partname
):
773 return get_dev_path(partname
)
775 raise Error('partition %d for %s does not appear to exist%s' %
776 (pnum
, dev
, error_msg
))
779 def list_all_partitions():
781 Return a list of devices and partitions
784 names
= os
.listdir(BLOCKDIR
)
787 # /dev/fd0 may hang http://tracker.ceph.com/issues/6827
788 if re
.match(r
'^fd\d$', name
):
790 dev_part_list
[name
] = list_partitions(get_dev_path(name
))
792 with
open(os
.path
.join(PROCDIR
, "partitions")) as partitions
:
793 for line
in partitions
:
794 columns
= line
.split()
795 if len(columns
) >= 4:
797 dev_part_list
[name
] = list_partitions(get_dev_path(name
))
801 def list_partitions(dev
):
802 dev
= os
.path
.realpath(dev
)
804 return list_partitions_mpath(dev
)
806 return list_partitions_device(dev
)
809 def list_partitions_mpath(dev
, part_re
="part\d+-mpath-"):
812 holders
= os
.path
.join(p
, 'holders')
813 for holder
in os
.listdir(holders
):
814 uuid_path
= os
.path
.join(holders
, holder
, 'dm', 'uuid')
815 uuid
= open(uuid_path
, 'r').read()
816 LOG
.debug("list_partitions_mpath: " + uuid_path
+ " uuid = " + uuid
)
817 if re
.match(part_re
, uuid
):
818 partitions
.append(holder
)
822 def list_partitions_device(dev
):
824 Return a list of partitions on the given device name
827 basename
= get_dev_name(dev
)
828 for name
in os
.listdir(block_path(dev
)):
829 if name
.startswith(basename
):
830 partitions
.append(name
)
834 def get_partition_base(dev
):
836 Get the base device for a partition
838 dev
= os
.path
.realpath(dev
)
839 if not ldev_is_diskdevice(dev
):
840 raise Error('not a block device', dev
)
842 name
= get_dev_name(dev
)
843 if os
.path
.exists(os
.path
.join('/sys/block', name
)):
844 raise Error('not a partition', dev
)
847 for basename
in os
.listdir('/sys/block'):
848 if os
.path
.exists(os
.path
.join('/sys/block', basename
, name
)):
849 return get_dev_path(basename
)
850 raise Error('no parent device for partition', dev
)
853 def is_partition_mpath(dev
):
854 uuid
= get_dm_uuid(dev
)
855 return bool(re
.match('part\d+-mpath-', uuid
))
858 def partnum_mpath(dev
):
859 uuid
= get_dm_uuid(dev
)
860 return re
.findall('part(\d+)-mpath-', uuid
)[0]
863 def get_partition_base_mpath(dev
):
864 slave_path
= os
.path
.join(block_path(dev
), 'slaves')
865 slaves
= os
.listdir(slave_path
)
867 name_path
= os
.path
.join(slave_path
, slaves
[0], 'dm', 'name')
868 name
= open(name_path
, 'r').read().strip()
869 return os
.path
.join('/dev/mapper', name
)
872 def is_partition(dev
):
874 Check whether a given device path is a partition or a full disk.
877 return is_partition_mpath(dev
)
879 dev
= os
.path
.realpath(dev
)
881 if not stmode_is_diskdevice(st
.st_mode
):
882 raise Error('not a block device', dev
)
884 name
= get_dev_name(dev
)
885 if os
.path
.exists(os
.path
.join(BLOCKDIR
, name
)):
888 # make sure it is a partition of something else
889 major
= os
.major(st
.st_rdev
)
890 minor
= os
.minor(st
.st_rdev
)
891 if os
.path
.exists('/sys/dev/block/%d:%d/partition' % (major
, minor
)):
894 raise Error('not a disk or partition', dev
)
899 Check if the given device is mounted.
901 dev
= os
.path
.realpath(dev
)
902 with
open(PROCDIR
+ '/mounts', 'rb') as proc_mounts
:
903 for line
in proc_mounts
:
904 fields
= line
.split()
907 mounts_dev
= fields
[0]
909 if os
.path
.isabs(mounts_dev
) and os
.path
.exists(mounts_dev
):
910 mounts_dev
= os
.path
.realpath(mounts_dev
)
911 if mounts_dev
== dev
:
912 return _bytes2str(path
)
918 Check if a device is held by another device (e.g., a dm-crypt mapping)
920 assert os
.path
.exists(dev
)
924 dev
= os
.path
.realpath(dev
)
925 base
= get_dev_name(dev
)
928 directory
= '/sys/block/{base}/holders'.format(base
=base
)
929 if os
.path
.exists(directory
):
930 return os
.listdir(directory
)
935 directory
= '/sys/block/{base}/{part}/holders'.format(
936 part
=part
, base
=base
)
937 if os
.path
.exists(directory
):
938 return os
.listdir(directory
)
943 def verify_not_in_use(dev
, check_partitions
=False):
945 Verify if a given device (path) is in use (e.g. mounted or
946 in use by device-mapper).
948 :raises: Error if device is in use.
950 assert os
.path
.exists(dev
)
952 raise Error('Device is mounted', dev
)
953 holders
= is_held(dev
)
955 raise Error('Device %s is in use by a device-mapper '
956 'mapping (dm-crypt?)' % dev
, ','.join(holders
))
958 if check_partitions
and not is_partition(dev
):
959 for partname
in list_partitions(dev
):
960 partition
= get_dev_path(partname
)
961 if is_mounted(partition
):
962 raise Error('Device is mounted', partition
)
963 holders
= is_held(partition
)
965 raise Error('Device %s is in use by a device-mapper '
966 'mapping (dm-crypt?)'
967 % partition
, ','.join(holders
))
970 def must_be_one_line(line
):
972 Checks if given line is really one single line.
974 :raises: TruncatedLineError or TooManyLinesError
975 :return: Content of the line, or None if line isn't valid.
977 line
= _bytes2str(line
)
979 if line
[-1:] != '\n':
980 raise TruncatedLineError(line
)
983 raise TooManyLinesError(line
)
987 def read_one_line(parent
, name
):
989 Read a file whose sole contents are a single line.
993 :return: Contents of the line, or None if file did not exist.
995 path
= os
.path
.join(parent
, name
)
997 line
= open(path
, 'rb').read()
999 if e
.errno
== errno
.ENOENT
:
1005 line
= must_be_one_line(line
)
1006 except (TruncatedLineError
, TooManyLinesError
) as e
:
1008 'File is corrupt: {path}: {msg}'.format(
1016 def write_one_line(parent
, name
, text
):
1018 Write a file whose sole contents are a single line.
1022 path
= os
.path
.join(parent
, name
)
1023 tmp
= '{path}.{pid}.tmp'.format(path
=path
, pid
=os
.getpid())
1024 with
open(tmp
, 'wb') as tmp_file
:
1025 tmp_file
.write(text
.encode('utf-8') + b
'\n')
1026 os
.fsync(tmp_file
.fileno())
1027 path_set_context(tmp
)
1028 os
.rename(tmp
, path
)
1033 Get a init system using 'ceph-detect-init'
1035 init
= _check_output(
1038 '--default', 'sysvinit',
1041 init
= must_be_one_line(init
)
1045 def check_osd_magic(path
):
1047 Check that this path has the Ceph OSD magic.
1049 :raises: BadMagicError if this does not look like a Ceph OSD data
1052 magic
= read_one_line(path
, 'magic')
1054 # probably not mkfs'ed yet
1055 raise BadMagicError(path
)
1056 if magic
!= CEPH_OSD_ONDISK_MAGIC
:
1057 raise BadMagicError(path
)
1060 def check_osd_id(osd_id
):
1062 Ensures osd id is numeric.
1064 if not re
.match(r
'^[0-9]+$', osd_id
):
1065 raise Error('osd id is not numeric', osd_id
)
1068 def allocate_osd_id(
1075 Allocates an OSD id on the given cluster.
1077 :raises: Error if the call to allocate the OSD id fails.
1078 :return: The allocated OSD id.
1080 lockbox_path
= os
.path
.join(STATEDIR
, 'osd-lockbox', fsid
)
1081 lockbox_osd_id
= read_one_line(lockbox_path
, 'whoami')
1082 osd_keyring
= os
.path
.join(path
, 'keyring')
1084 LOG
.debug('Getting OSD id from Lockbox...')
1085 osd_id
= lockbox_osd_id
1086 shutil
.move(os
.path
.join(lockbox_path
, 'osd_keyring'),
1088 path_set_context(osd_keyring
)
1089 os
.unlink(os
.path
.join(lockbox_path
, 'whoami'))
1092 LOG
.debug('Allocating OSD id...')
1095 wanttobe
= read_one_line(path
, 'wanttobe')
1096 if os
.path
.exists(os
.path
.join(path
, 'wanttobe')):
1097 os
.unlink(os
.path
.join(path
, 'wanttobe'))
1098 id_arg
= wanttobe
and [wanttobe
] or []
1099 osd_id
= command_with_stdin(
1102 '--cluster', cluster
,
1103 '--name', 'client.bootstrap-osd',
1104 '--keyring', keyring
,
1111 except subprocess
.CalledProcessError
as e
:
1112 raise Error('ceph osd create failed', e
, e
.output
)
1113 osd_id
= must_be_one_line(osd_id
)
1114 check_osd_id(osd_id
)
1115 secrets
.write_osd_keyring(osd_keyring
, osd_id
)
1119 def get_osd_id(path
):
1121 Gets the OSD id of the OSD at the given path.
1123 osd_id
= read_one_line(path
, 'whoami')
1124 if osd_id
is not None:
1125 check_osd_id(osd_id
)
1129 def get_ceph_user():
1130 global CEPH_PREF_USER
1132 if CEPH_PREF_USER
is not None:
1134 pwd
.getpwnam(CEPH_PREF_USER
)
1135 return CEPH_PREF_USER
1137 print("No such user:", CEPH_PREF_USER
)
1141 pwd
.getpwnam('ceph')
1147 def get_ceph_group():
1148 global CEPH_PREF_GROUP
1150 if CEPH_PREF_GROUP
is not None:
1152 grp
.getgrnam(CEPH_PREF_GROUP
)
1153 return CEPH_PREF_GROUP
1155 print("No such group:", CEPH_PREF_GROUP
)
1159 grp
.getgrnam('ceph')
1165 def path_set_context(path
):
1166 # restore selinux context to default policy values
1167 if which('restorecon'):
1168 command(['restorecon', '-R', path
])
1170 # if ceph user exists, set owner to ceph
1171 if get_ceph_user() == 'ceph':
1172 command(['chown', '-R', 'ceph:ceph', path
])
1175 def _check_output(args
=None, **kwargs
):
1176 out
, err
, ret
= command(args
, **kwargs
)
1179 error
= subprocess
.CalledProcessError(ret
, cmd
)
1180 error
.output
= out
+ err
1182 return _bytes2str(out
)
1185 def get_conf(cluster
, variable
):
1187 Get the value of the given configuration variable from the
1190 :raises: Error if call to ceph-conf fails.
1191 :return: The variable value or None.
1194 out
, err
, ret
= command(
1197 '--cluster={cluster}'.format(
1206 except OSError as e
:
1207 raise Error('error executing ceph-conf', e
, err
)
1209 # config entry not found
1212 raise Error('getting variable from configuration failed')
1213 value
= out
.split('\n', 1)[0]
1214 # don't differentiate between "var=" and no var set
1220 def get_conf_with_default(cluster
, variable
):
1222 Get a config value that is known to the C++ code.
1224 This will fail if called on variables that are not defined in
1225 common config options.
1228 out
= _check_output(
1231 '--cluster={cluster}'.format(
1234 '--show-config-value={variable}'.format(
1240 except subprocess
.CalledProcessError
as e
:
1242 'getting variable from configuration failed',
1246 value
= str(out
).split('\n', 1)[0]
1250 def get_fsid(cluster
):
1252 Get the fsid of the cluster.
1254 :return: The fsid or raises Error.
1256 fsid
= get_conf_with_default(cluster
=cluster
, variable
='fsid')
1257 # uuids from boost always default to 'the empty uuid'
1258 if fsid
== '00000000-0000-0000-0000-000000000000':
1259 raise Error('getting cluster uuid from configuration failed')
1263 def get_dmcrypt_key_path(
1269 Get path to dmcrypt key file.
1271 :return: Path to the dmcrypt key file, callers should check for existence.
1274 path
= os
.path
.join(key_dir
, _uuid
+ ".luks.key")
1276 path
= os
.path
.join(key_dir
, _uuid
)
1281 def get_dmcrypt_key(
1286 legacy_path
= get_dmcrypt_key_path(_uuid
, key_dir
, luks
)
1287 if os
.path
.exists(legacy_path
):
1288 return (legacy_path
,)
1289 path
= os
.path
.join(STATEDIR
, 'osd-lockbox', _uuid
)
1290 if os
.path
.exists(path
):
1291 mode
= get_oneliner(path
, 'key-management-mode')
1292 osd_uuid
= get_oneliner(path
, 'osd-uuid')
1293 ceph_fsid
= read_one_line(path
, 'ceph_fsid')
1294 if ceph_fsid
is None:
1295 LOG
.warning("no `ceph_fsid` found falling back to 'ceph' "
1299 cluster
= find_cluster_by_uuid(ceph_fsid
)
1301 raise Error('No cluster conf found in ' + SYSCONFDIR
+
1302 ' with fsid %s' % ceph_fsid
)
1304 if mode
== KEY_MANAGEMENT_MODE_V1
:
1305 key
, stderr
, ret
= command(
1308 '--cluster', cluster
,
1310 'client.osd-lockbox.' + osd_uuid
,
1312 os
.path
.join(path
, 'keyring'),
1315 'dm-crypt/osd/' + osd_uuid
+ '/luks',
1318 LOG
.debug("stderr " + stderr
)
1320 return base64
.b64decode(key
)
1322 raise Error('unknown key-management-mode ' + str(mode
))
1323 raise Error('unable to read dm-crypt key', path
, legacy_path
)
1330 cryptsetup_parameters
,
1334 dev
= dmcrypt_is_mapped(_uuid
)
1338 if isinstance(key
, tuple):
1339 # legacy, before lockbox
1340 assert os
.path
.exists(key
[0])
1345 dev
= '/dev/mapper/' + _uuid
1353 ] + cryptsetup_parameters
1371 ] + cryptsetup_parameters
1376 command_with_stdin(luksFormat_args
, key
)
1377 command_with_stdin(luksOpen_args
, key
)
1379 # Plain mode has no format function, nor any validation
1380 # that the key is correct.
1381 command_with_stdin(create_args
, key
)
1382 # set proper ownership of mapped device
1383 command_check_call(['chown', 'ceph:ceph', dev
])
1386 except subprocess
.CalledProcessError
as e
:
1387 raise Error('unable to map device', rawdev
, e
)
1390 @retry(Error
, max_tries
=10, wait
=0.5, backoff
=1.0)
1391 def dmcrypt_unmap(_uuid
):
1392 if not os
.path
.exists('/dev/mapper/' + _uuid
):
1395 command_check_call(['cryptsetup', 'remove', _uuid
])
1396 except subprocess
.CalledProcessError
as e
:
1397 raise Error('unable to unmap device', _uuid
, e
)
1406 Mounts a device with given filessystem type and
1407 mount options to a tempfile path under /var/lib/ceph/tmp.
1409 # sanity check: none of the arguments are None
1411 raise ValueError('dev may not be None')
1413 raise ValueError('fstype may not be None')
1415 # pick best-of-breed mount options based on fs type
1417 options
= MOUNT_OPTIONS
.get(fstype
, '')
1419 myTemp
= STATEDIR
+ '/tmp'
1420 # mkdtemp expect 'dir' to be existing on the system
1421 # Let's be sure it's always the case
1422 if not os
.path
.exists(myTemp
):
1426 path
= tempfile
.mkdtemp(
1431 LOG
.debug('Mounting %s on %s with options %s', dev
, path
, options
)
1442 if which('restorecon'):
1449 except subprocess
.CalledProcessError
as e
:
1452 except (OSError, IOError):
1459 @retry(UnmountError
, max_tries
=3, wait
=0.5, backoff
=1.0)
1465 Unmount and removes the given mount point.
1468 LOG
.debug('Unmounting %s', path
)
1476 except subprocess
.CalledProcessError
as e
:
1477 raise UnmountError(e
)
1483 ###########################################
1485 def extract_parted_partition_numbers(partitions
):
1486 numbers_as_strings
= re
.findall('^\d+', partitions
, re
.MULTILINE
)
1487 return map(int, numbers_as_strings
)
1490 def get_free_partition_index(dev
):
1492 Get the next free partition index on a given device.
1494 :return: Index number (> 1 if there is already a partition on the device)
1495 or 1 if there is no partition table.
1498 lines
= _check_output(
1507 except subprocess
.CalledProcessError
as e
:
1508 LOG
.info('cannot read partition index; assume it '
1509 'isn\'t present\n (Error: %s)' % e
)
1513 raise Error('parted failed to output anything')
1514 LOG
.debug('get_free_partition_index: analyzing ' + lines
)
1515 if ('CHS;' not in lines
and
1516 'CYL;' not in lines
and
1517 'BYT;' not in lines
):
1518 raise Error('parted output expected to contain one of ' +
1519 'CHH; CYL; or BYT; : ' + lines
)
1520 if os
.path
.realpath(dev
) not in lines
:
1521 raise Error('parted output expected to contain ' + dev
+ ': ' + lines
)
1522 _
, partitions
= lines
.split(os
.path
.realpath(dev
))
1523 partition_numbers
= extract_parted_partition_numbers(partitions
)
1524 if partition_numbers
:
1525 return max(partition_numbers
) + 1
1530 def check_journal_reqs(args
):
1531 _
, _
, allows_journal
= command([
1532 'ceph-osd', '--check-allows-journal',
1534 '--log-file', '$run_dir/$cluster-osd-check.log',
1535 '--cluster', args
.cluster
,
1536 '--setuser', get_ceph_user(),
1537 '--setgroup', get_ceph_group(),
1539 _
, _
, wants_journal
= command([
1540 'ceph-osd', '--check-wants-journal',
1542 '--log-file', '$run_dir/$cluster-osd-check.log',
1543 '--cluster', args
.cluster
,
1544 '--setuser', get_ceph_user(),
1545 '--setgroup', get_ceph_group(),
1547 _
, _
, needs_journal
= command([
1548 'ceph-osd', '--check-needs-journal',
1550 '--log-file', '$run_dir/$cluster-osd-check.log',
1551 '--cluster', args
.cluster
,
1552 '--setuser', get_ceph_user(),
1553 '--setgroup', get_ceph_group(),
1555 return (not allows_journal
, not wants_journal
, not needs_journal
)
1558 def update_partition(dev
, description
):
1560 Must be called after modifying a partition table so the kernel
1561 know about the change and fire udev events accordingly. A side
1562 effect of partprobe is to remove partitions and add them again.
1563 The first udevadm settle waits for ongoing udev events to
1564 complete, just in case one of them rely on an existing partition
1565 on dev. The second udevadm settle guarantees to the caller that
1566 all udev events related to the partition table change have been
1567 processed, i.e. the 95-ceph-osd.rules actions and mode changes,
1568 group changes etc. are complete.
1570 LOG
.debug('Calling partprobe on %s device %s', description
, dev
)
1571 partprobe_ok
= False
1572 error
= 'unknown error'
1573 partprobe
= _get_command_executable(['partprobe'])[0]
1575 command_check_call(['udevadm', 'settle', '--timeout=600'])
1577 _check_output(['flock', '-s', dev
, partprobe
, dev
])
1580 except subprocess
.CalledProcessError
as e
:
1582 if ('unable to inform the kernel' not in error
and
1583 'Device or resource busy' not in error
):
1585 LOG
.debug('partprobe %s failed : %s (ignored, waiting 60s)'
1588 if not partprobe_ok
:
1589 raise Error('partprobe %s failed : %s' % (dev
, error
))
1590 command_check_call(['udevadm', 'settle', '--timeout=600'])
1595 # Thoroughly wipe all partitions of any traces of
1596 # Filesystems or OSD Journals
1598 # In addition we need to write 10M of data to each partition
1599 # to make sure that after re-creating the same partition
1600 # there is no trace left of any previous Filesystem or OSD
1603 LOG
.debug('Writing zeros to existing partitions on %s', dev
)
1605 for partname
in list_partitions(dev
):
1606 partition
= get_dev_path(partname
)
1619 'of={path}'.format(path
=partition
),
1625 LOG
.debug('Zapping partition table on %s', dev
)
1627 # try to wipe out any GPT partition table backups. sgdisk
1628 # isn't too thorough.
1630 size
= 33 * lba_size
1631 with
open(dev
, 'wb') as dev_file
:
1632 dev_file
.seek(-size
, os
.SEEK_END
)
1633 dev_file
.write(size
* b
'\0')
1652 update_partition(dev
, 'zapped')
1654 except subprocess
.CalledProcessError
as e
:
1658 def zap_freebsd(dev
):
1660 # For FreeBSD we just need to zap the partition.
1670 except subprocess
.CalledProcessError
as e
:
1676 Destroy the partition table and content of a given disk.
1678 dev
= os
.path
.realpath(dev
)
1679 dmode
= os
.stat(dev
).st_mode
1680 if not stat
.S_ISBLK(dmode
) or is_partition(dev
):
1681 raise Error('not full block device; cannot zap', dev
)
1688 def adjust_symlink(target
, path
):
1690 if os
.path
.lexists(path
):
1692 mode
= os
.lstat(path
).st_mode
1693 if stat
.S_ISREG(mode
):
1694 LOG
.debug('Removing old file %s', path
)
1696 elif stat
.S_ISLNK(mode
):
1697 old
= os
.readlink(path
)
1699 LOG
.debug('Removing old symlink %s -> %s', path
, old
)
1704 raise Error('unable to remove (or adjust) old file (symlink)',
1707 LOG
.debug('Creating symlink %s -> %s', path
, target
)
1709 os
.symlink(target
, path
)
1711 raise Error('unable to create symlink %s -> %s' % (path
, target
))
1714 def get_mount_options(cluster
, fs_type
):
1715 mount_options
= get_conf(
1717 variable
='osd_mount_options_{fstype}'.format(
1721 if mount_options
is None:
1722 mount_options
= get_conf(
1724 variable
='osd_fs_mount_options_{fstype}'.format(
1729 # remove whitespaces
1730 mount_options
= "".join(mount_options
.split())
1731 return mount_options
1734 class Device(object):
1736 def __init__(self
, path
, args
):
1739 self
.dev_size
= None
1740 self
.partitions
= {}
1741 self
.ptype_map
= None
1742 assert not is_partition(self
.path
)
1744 def create_partition(self
, uuid
, name
, size
=0, num
=0):
1745 ptype
= self
.ptype_tobe_for_name(name
)
1747 num
= get_free_partition_index(dev
=self
.path
)
1749 new
= '--new={num}:0:+{size}M'.format(num
=num
, size
=size
)
1750 if size
> self
.get_dev_size():
1751 LOG
.error('refusing to create %s on %s' % (name
, self
.path
))
1752 LOG
.error('%s size (%sM) is bigger than device (%sM)'
1753 % (name
, size
, self
.get_dev_size()))
1754 raise Error('%s device size (%sM) is not big enough for %s'
1755 % (self
.path
, self
.get_dev_size(), name
))
1757 new
= '--largest-new={num}'.format(num
=num
)
1759 LOG
.debug('Creating %s partition num %d size %d on %s',
1760 name
, num
, size
, self
.path
)
1765 '--change-name={num}:ceph {name}'.format(num
=num
, name
=name
),
1766 '--partition-guid={num}:{uuid}'.format(num
=num
, uuid
=uuid
),
1767 '--typecode={num}:{uuid}'.format(num
=num
, uuid
=ptype
),
1774 update_partition(self
.path
, 'created')
1777 def ptype_tobe_for_name(self
, name
):
1778 LOG
.debug("name = " + name
)
1781 if name
== 'lockbox':
1782 if is_mpath(self
.path
):
1783 return PTYPE
['mpath']['lockbox']['tobe']
1785 return PTYPE
['regular']['lockbox']['tobe']
1786 if self
.ptype_map
is None:
1787 partition
= DevicePartition
.factory(
1788 path
=self
.path
, dev
=None, args
=self
.args
)
1789 self
.ptype_map
= partition
.ptype_map
1790 return self
.ptype_map
[name
]['tobe']
1792 def get_partition(self
, num
):
1793 if num
not in self
.partitions
:
1794 dev
= get_partition_dev(self
.path
, num
)
1795 partition
= DevicePartition
.factory(
1796 path
=self
.path
, dev
=dev
, args
=self
.args
)
1797 partition
.set_partition_number(num
)
1798 self
.partitions
[num
] = partition
1799 return self
.partitions
[num
]
1801 def get_dev_size(self
):
1802 if self
.dev_size
is None:
1803 self
.dev_size
= get_dev_size(self
.path
)
1804 return self
.dev_size
1807 def factory(path
, args
):
1808 return Device(path
, args
)
1811 class DevicePartition(object):
1813 def __init__(self
, args
):
1819 self
.ptype_map
= None
1821 self
.set_variables_ptype()
1824 if self
.uuid
is None:
1825 self
.uuid
= get_partition_uuid(self
.rawdev
)
1828 def get_ptype(self
):
1829 if self
.ptype
is None:
1830 self
.ptype
= get_partition_type(self
.rawdev
)
1833 def set_partition_number(self
, num
):
1836 def get_partition_number(self
):
1839 def set_dev(self
, dev
):
1846 def get_rawdev(self
):
1849 def set_variables_ptype(self
):
1850 self
.ptype_map
= PTYPE
['regular']
1852 def ptype_for_name(self
, name
):
1853 return self
.ptype_map
[name
]['ready']
1857 def factory(path
, dev
, args
):
1858 dmcrypt_type
= CryptHelpers
.get_dmcrypt_type(args
)
1859 if ((path
is not None and is_mpath(path
)) or
1860 (dev
is not None and is_mpath(dev
))):
1861 partition
= DevicePartitionMultipath(args
)
1862 elif dmcrypt_type
== 'luks':
1863 partition
= DevicePartitionCryptLuks(args
)
1864 elif dmcrypt_type
== 'plain':
1865 partition
= DevicePartitionCryptPlain(args
)
1867 partition
= DevicePartition(args
)
1868 partition
.set_dev(dev
)
1872 class DevicePartitionMultipath(DevicePartition
):
1874 def set_variables_ptype(self
):
1875 self
.ptype_map
= PTYPE
['mpath']
1878 class DevicePartitionCrypt(DevicePartition
):
1880 def __init__(self
, args
):
1881 super(DevicePartitionCrypt
, self
).__init
__(args
)
1882 self
.osd_dm_key
= None
1883 self
.cryptsetup_parameters
= CryptHelpers
.get_cryptsetup_parameters(
1885 self
.dmcrypt_type
= CryptHelpers
.get_dmcrypt_type(self
.args
)
1886 self
.dmcrypt_keysize
= CryptHelpers
.get_dmcrypt_keysize(self
.args
)
1888 def setup_crypt(self
):
1893 self
.dev
= _dmcrypt_map(
1895 key
=self
.osd_dm_key
,
1896 _uuid
=self
.get_uuid(),
1897 cryptsetup_parameters
=self
.cryptsetup_parameters
,
1904 dmcrypt_unmap(self
.get_uuid())
1905 self
.dev
= self
.rawdev
1912 class DevicePartitionCryptPlain(DevicePartitionCrypt
):
1917 def setup_crypt(self
):
1918 if self
.osd_dm_key
is not None:
1921 self
.cryptsetup_parameters
+= ['--key-size', str(self
.dmcrypt_keysize
)]
1923 self
.osd_dm_key
= get_dmcrypt_key(
1924 self
.get_uuid(), self
.args
.dmcrypt_key_dir
,
1927 def set_variables_ptype(self
):
1928 self
.ptype_map
= PTYPE
['plain']
1931 class DevicePartitionCryptLuks(DevicePartitionCrypt
):
1936 def setup_crypt(self
):
1937 if self
.osd_dm_key
is not None:
1940 if self
.dmcrypt_keysize
== 1024:
1941 # We don't force this into the cryptsetup_parameters,
1942 # as we want the cryptsetup defaults
1943 # to prevail for the actual LUKS key lengths.
1946 self
.cryptsetup_parameters
+= ['--key-size',
1947 str(self
.dmcrypt_keysize
)]
1949 self
.osd_dm_key
= get_dmcrypt_key(
1950 self
.get_uuid(), self
.args
.dmcrypt_key_dir
,
1953 def set_variables_ptype(self
):
1954 self
.ptype_map
= PTYPE
['luks']
1957 class Prepare(object):
1959 def __init__(self
, args
):
1964 parser
= argparse
.ArgumentParser(add_help
=False)
1965 parser
.add_argument(
1969 help='cluster name to assign this disk to',
1971 parser
.add_argument(
1974 help='cluster uuid to assign this disk to',
1976 parser
.add_argument(
1979 help='unique OSD uuid to assign this disk to',
1981 parser
.add_argument(
1984 help='unique OSD id to assign this disk to',
1986 parser
.add_argument(
1987 '--crush-device-class',
1988 help='crush device class to assign this disk to',
1990 parser
.add_argument(
1992 action
='store_true', default
=None,
1993 help='encrypt DATA and/or JOURNAL devices with dm-crypt',
1995 parser
.add_argument(
1996 '--dmcrypt-key-dir',
1998 default
='/etc/ceph/dmcrypt-keys',
1999 help='directory where dm-crypt keys are stored',
2001 parser
.add_argument(
2004 help='bootstrap-osd keyring path template (%(default)s)',
2005 default
='{statedir}/bootstrap-osd/{cluster}.keyring',
2006 dest
='prepare_key_template',
2008 parser
.add_argument(
2010 action
='store_true', default
=None,
2011 help='let many prepare\'s run in parallel',
2016 def set_subparser(subparsers
):
2019 PrepareData
.parser(),
2022 parents
.extend(PrepareFilestore
.parent_parsers())
2023 parents
.extend(PrepareBluestore
.parent_parsers())
2024 parser
= subparsers
.add_parser(
2027 formatter_class
=argparse
.RawDescriptionHelpFormatter
,
2028 description
=textwrap
.fill(textwrap
.dedent("""\
2029 If the --bluestore argument is given, a bluestore objectstore
2030 will be created. If --filestore is provided, a legacy FileStore
2031 objectstore will be created. If neither is specified, we default
2034 When an entire device is prepared for bluestore, two
2035 partitions are created. The first partition is for metadata,
2036 the second partition is for blocks that contain data.
2038 Unless explicitly specified with --block.db or
2039 --block.wal, the bluestore DB and WAL data is stored on
2040 the main block device. For instance:
2042 ceph-disk prepare --bluestore /dev/sdc
2046 /dev/sdc1 for osd metadata
2047 /dev/sdc2 for block, db, and wal data (the rest of the disk)
2050 If either --block.db or --block.wal are specified to be
2051 the same whole device, they will be created as partition
2052 three and four respectively. For instance:
2054 ceph-disk prepare --bluestore \\
2055 --block.db /dev/sdc \\
2056 --block.wal /dev/sdc \\
2061 /dev/sdc1 for osd metadata
2062 /dev/sdc2 for block (the rest of the disk)
2067 help='Prepare a directory or disk for a Ceph OSD',
2069 parser
.set_defaults(
2075 if self
.args
.no_locking
:
2084 return PrepareBluestore(args
)
2086 return PrepareFilestore(args
)
2090 Prepare
.factory(args
).prepare()
2093 class PrepareFilestore(Prepare
):
2095 def __init__(self
, args
):
2096 super(PrepareFilestore
, self
).__init
__(args
)
2098 self
.lockbox
= Lockbox(args
)
2099 self
.data
= PrepareFilestoreData(args
)
2100 self
.journal
= PrepareJournal(args
)
2103 def parent_parsers():
2105 PrepareJournal
.parser(),
2109 if self
.data
.args
.dmcrypt
:
2110 self
.lockbox
.prepare()
2111 self
.data
.prepare(self
.journal
)
2114 class PrepareBluestore(Prepare
):
2116 def __init__(self
, args
):
2117 super(PrepareBluestore
, self
).__init
__(args
)
2119 self
.lockbox
= Lockbox(args
)
2120 self
.data
= PrepareBluestoreData(args
)
2121 self
.block
= PrepareBluestoreBlock(args
)
2122 self
.blockdb
= PrepareBluestoreBlockDB(args
)
2123 self
.blockwal
= PrepareBluestoreBlockWAL(args
)
2127 parser
= argparse
.ArgumentParser(add_help
=False)
2128 parser
.add_argument(
2131 action
='store_true', default
=True,
2132 help='bluestore objectstore',
2134 parser
.add_argument(
2137 action
='store_false',
2138 help='filestore objectstore',
2143 def parent_parsers():
2145 PrepareBluestore
.parser(),
2146 PrepareBluestoreBlock
.parser(),
2147 PrepareBluestoreBlockDB
.parser(),
2148 PrepareBluestoreBlockWAL
.parser(),
2152 if self
.data
.args
.dmcrypt
:
2153 self
.lockbox
.prepare()
2154 to_prepare_list
= []
2155 if getattr(self
.data
.args
, 'block.db'):
2156 to_prepare_list
.append(self
.blockdb
)
2157 if getattr(self
.data
.args
, 'block.wal'):
2158 to_prepare_list
.append(self
.blockwal
)
2159 to_prepare_list
.append(self
.block
)
2160 self
.data
.prepare(*to_prepare_list
)
2163 class Space(object):
2165 NAMES
= ('block', 'journal', 'block.db', 'block.wal')
2168 class PrepareSpace(object):
2174 def __init__(self
, args
):
2177 self
.space_size
= self
.get_space_size()
2178 if getattr(self
.args
, self
.name
+ '_uuid') is None:
2179 setattr(self
.args
, self
.name
+ '_uuid', str(uuid
.uuid4()))
2180 self
.space_symlink
= None
2181 self
.space_dmcrypt
= None
2186 if (self
.wants_space() and
2187 dev_is_diskdevice(args
.data
) and
2188 not is_partition(args
.data
) and
2189 getattr(args
, name
) is None and
2190 getattr(args
, name
+ '_file') is None):
2191 LOG
.info('Will colocate %s with data on %s',
2193 setattr(args
, name
, args
.data
)
2195 if getattr(args
, name
) is None:
2196 if getattr(args
, name
+ '_dev'):
2197 raise Error('%s is unspecified; not a block device' %
2198 name
.capitalize(), getattr(args
, name
))
2199 self
.type = self
.NONE
2202 if not os
.path
.exists(getattr(args
, name
)):
2203 if getattr(args
, name
+ '_dev'):
2204 raise Error('%s does not exist; not a block device' %
2205 name
.capitalize(), getattr(args
, name
))
2206 self
.type = self
.FILE
2209 mode
= os
.stat(getattr(args
, name
)).st_mode
2210 if stmode_is_diskdevice(mode
):
2211 if getattr(args
, name
+ '_file'):
2212 raise Error('%s is not a regular file' % name
.capitalize
,
2213 getattr(args
, name
))
2214 self
.type = self
.DEVICE
2217 if stat
.S_ISREG(mode
):
2218 if getattr(args
, name
+ '_dev'):
2219 raise Error('%s is not a block device' % name
.capitalize
,
2220 getattr(args
, name
))
2221 self
.type = self
.FILE
2224 raise Error('%s %s is neither a block device nor regular file' %
2225 (name
.capitalize
, getattr(args
, name
)))
2228 return self
.type == self
.NONE
2231 return self
.type == self
.FILE
2233 def is_device(self
):
2234 return self
.type == self
.DEVICE
2237 def parser(name
, positional
=True):
2238 parser
= argparse
.ArgumentParser(add_help
=False)
2239 parser
.add_argument(
2242 help='unique uuid to assign to the %s' % name
,
2244 parser
.add_argument(
2246 action
='store_true', default
=None,
2247 help='verify that %s is a file' % name
.upper(),
2249 parser
.add_argument(
2251 action
='store_true', default
=None,
2252 help='verify that %s is a block device' % name
.upper(),
2256 parser
.add_argument(
2258 metavar
=name
.upper(),
2260 help=('path to OSD %s disk block device;' % name
+
2261 ' leave out to store %s in file' % name
),
2265 def wants_space(self
):
2268 def populate_data_path(self
, path
):
2269 if self
.type == self
.DEVICE
:
2270 self
.populate_data_path_device(path
)
2271 elif self
.type == self
.FILE
:
2272 self
.populate_data_path_file(path
)
2273 elif self
.type == self
.NONE
:
2276 raise Error('unexpected type ', self
.type)
2278 def populate_data_path_file(self
, path
):
2279 space_uuid
= self
.name
+ '_uuid'
2280 if getattr(self
.args
, space_uuid
) is not None:
2281 write_one_line(path
, space_uuid
,
2282 getattr(self
.args
, space_uuid
))
2283 if self
.space_symlink
is not None:
2284 adjust_symlink(self
.space_symlink
,
2285 os
.path
.join(path
, self
.name
))
2287 def populate_data_path_device(self
, path
):
2288 self
.populate_data_path_file(path
)
2290 if self
.space_dmcrypt
is not None:
2291 adjust_symlink(self
.space_dmcrypt
,
2292 os
.path
.join(path
, self
.name
+ '_dmcrypt'))
2295 os
.unlink(os
.path
.join(path
, self
.name
+ '_dmcrypt'))
2300 if self
.type == self
.DEVICE
:
2301 self
.prepare_device()
2302 elif self
.type == self
.FILE
:
2304 elif self
.type == self
.NONE
:
2307 raise Error('unexpected type ', self
.type)
2309 def prepare_file(self
):
2310 space_filename
= getattr(self
.args
, self
.name
)
2311 if not os
.path
.exists(space_filename
):
2312 LOG
.debug('Creating %s file %s with size 0'
2313 ' (ceph-osd will resize and allocate)',
2316 space_file
= open(space_filename
, 'wb')
2318 path_set_context(space_filename
)
2320 LOG
.debug('%s is file %s',
2321 self
.name
.capitalize(),
2323 LOG
.warning('OSD will not be hot-swappable if %s is '
2324 'not the same device as the osd data' %
2326 self
.space_symlink
= space_filename
2328 def prepare_device(self
):
2329 reusing_partition
= False
2331 if is_partition(getattr(self
.args
, self
.name
)):
2332 LOG
.debug('%s %s is a partition',
2333 self
.name
.capitalize(), getattr(self
.args
, self
.name
))
2334 partition
= DevicePartition
.factory(
2335 path
=None, dev
=getattr(self
.args
, self
.name
), args
=self
.args
)
2336 if isinstance(partition
, DevicePartitionCrypt
):
2337 raise Error(getattr(self
.args
, self
.name
) +
2338 ' partition already exists'
2339 ' and --dmcrypt specified')
2340 LOG
.warning('OSD will not be hot-swappable' +
2341 ' if ' + self
.name
+ ' is not' +
2342 ' the same device as the osd data')
2343 if partition
.get_ptype() == partition
.ptype_for_name(self
.name
):
2344 LOG
.debug('%s %s was previously prepared with '
2345 'ceph-disk. Reusing it.',
2346 self
.name
.capitalize(),
2347 getattr(self
.args
, self
.name
))
2348 reusing_partition
= True
2349 # Read and reuse the partition uuid from this journal's
2350 # previous life. We reuse the uuid instead of changing it
2351 # because udev does not reliably notice changes to an
2352 # existing partition's GUID. See
2353 # http://tracker.ceph.com/issues/10146
2354 setattr(self
.args
, self
.name
+ '_uuid', partition
.get_uuid())
2355 LOG
.debug('Reusing %s with uuid %s',
2357 getattr(self
.args
, self
.name
+ '_uuid'))
2359 LOG
.warning('%s %s was not prepared with '
2360 'ceph-disk. Symlinking directly.',
2361 self
.name
.capitalize(),
2362 getattr(self
.args
, self
.name
))
2363 self
.space_symlink
= getattr(self
.args
, self
.name
)
2366 self
.space_symlink
= '/dev/disk/by-partuuid/{uuid}'.format(
2367 uuid
=getattr(self
.args
, self
.name
+ '_uuid'))
2369 if self
.args
.dmcrypt
:
2370 self
.space_dmcrypt
= self
.space_symlink
2371 self
.space_symlink
= '/dev/mapper/{uuid}'.format(
2372 uuid
=getattr(self
.args
, self
.name
+ '_uuid'))
2374 if reusing_partition
:
2375 # confirm that the space_symlink exists. It should since
2376 # this was an active space
2377 # in the past. Continuing otherwise would be futile.
2378 assert os
.path
.exists(self
.space_symlink
)
2381 num
= self
.desired_partition_number()
2384 LOG
.warning('OSD will not be hot-swappable if %s '
2385 'is not the same device as the osd data',
2388 device
= Device
.factory(getattr(self
.args
, self
.name
), self
.args
)
2389 num
= device
.create_partition(
2390 uuid
=getattr(self
.args
, self
.name
+ '_uuid'),
2392 size
=self
.space_size
,
2395 partition
= device
.get_partition(num
)
2397 LOG
.debug('%s is GPT partition %s',
2398 self
.name
.capitalize(),
2401 if isinstance(partition
, DevicePartitionCrypt
):
2408 '--typecode={num}:{uuid}'.format(
2410 uuid
=partition
.ptype_for_name(self
.name
),
2413 getattr(self
.args
, self
.name
),
2416 update_partition(getattr(self
.args
, self
.name
), 'prepared')
2418 LOG
.debug('%s is GPT partition %s',
2419 self
.name
.capitalize(),
2423 class PrepareJournal(PrepareSpace
):
2425 def __init__(self
, args
):
2426 self
.name
= 'journal'
2427 (self
.allows_journal
,
2429 self
.needs_journal
) = check_journal_reqs(args
)
2431 if args
.journal
and not self
.allows_journal
:
2432 raise Error('journal specified but not allowed by osd backend')
2434 super(PrepareJournal
, self
).__init
__(args
)
2436 def wants_space(self
):
2437 return self
.wants_journal
2439 def get_space_size(self
):
2440 return int(get_conf_with_default(
2441 cluster
=self
.args
.cluster
,
2442 variable
='osd_journal_size',
2445 def desired_partition_number(self
):
2446 if self
.args
.journal
== self
.args
.data
:
2447 # we're sharing the disk between osd data and journal;
2448 # make journal be partition number 2
2456 return PrepareSpace
.parser('journal')
2459 class PrepareBluestoreBlock(PrepareSpace
):
2461 def __init__(self
, args
):
2463 super(PrepareBluestoreBlock
, self
).__init
__(args
)
2465 def get_space_size(self
):
2466 block_size
= get_conf(
2467 cluster
=self
.args
.cluster
,
2468 variable
='bluestore_block_size',
2471 if block_size
is None:
2472 return 0 # get as much space as possible
2474 return int(block_size
) / 1048576 # MB
2476 def desired_partition_number(self
):
2477 if self
.args
.block
== self
.args
.data
:
2485 return PrepareSpace
.parser('block')
2488 class PrepareBluestoreBlockDB(PrepareSpace
):
2490 def __init__(self
, args
):
2491 self
.name
= 'block.db'
2492 super(PrepareBluestoreBlockDB
, self
).__init
__(args
)
2494 def get_space_size(self
):
2495 block_db_size
= get_conf(
2496 cluster
=self
.args
.cluster
,
2497 variable
='bluestore_block_db_size',
2500 if block_db_size
is None or int(block_db_size
) == 0:
2501 block_size
= get_conf(
2502 cluster
=self
.args
.cluster
,
2503 variable
='bluestore_block_size',
2505 if block_size
is None:
2507 size
= int(block_size
) / 100 / 1048576
2508 return max(size
, 1024) # MB
2510 return int(block_db_size
) / 1048576 # MB
2512 def desired_partition_number(self
):
2513 if getattr(self
.args
, 'block.db') == self
.args
.data
:
2519 def wants_space(self
):
2524 parser
= PrepareSpace
.parser('block.db', positional
=False)
2525 parser
.add_argument(
2528 help='path to the device or file for bluestore block.db',
2533 class PrepareBluestoreBlockWAL(PrepareSpace
):
2535 def __init__(self
, args
):
2536 self
.name
= 'block.wal'
2537 super(PrepareBluestoreBlockWAL
, self
).__init
__(args
)
2539 def get_space_size(self
):
2540 block_size
= get_conf(
2541 cluster
=self
.args
.cluster
,
2542 variable
='bluestore_block_wal_size',
2545 if block_size
is None:
2546 return 576 # MB, default value
2548 return int(block_size
) / 1048576 # MB
2550 def desired_partition_number(self
):
2551 if getattr(self
.args
, 'block.wal') == self
.args
.data
:
2557 def wants_space(self
):
2562 parser
= PrepareSpace
.parser('block.wal', positional
=False)
2563 parser
.add_argument(
2566 help='path to the device or file for bluestore block.wal',
2571 class CryptHelpers(object):
2574 def get_cryptsetup_parameters(args
):
2575 cryptsetup_parameters_str
= get_conf(
2576 cluster
=args
.cluster
,
2577 variable
='osd_cryptsetup_parameters',
2579 if cryptsetup_parameters_str
is None:
2582 return shlex
.split(cryptsetup_parameters_str
)
2585 def get_dmcrypt_keysize(args
):
2586 dmcrypt_keysize_str
= get_conf(
2587 cluster
=args
.cluster
,
2588 variable
='osd_dmcrypt_key_size',
2590 dmcrypt_type
= CryptHelpers
.get_dmcrypt_type(args
)
2591 if dmcrypt_type
== 'luks':
2592 if dmcrypt_keysize_str
is None:
2593 # As LUKS will hash the 'passphrase' in .luks.key
2594 # into a key, set a large default
2595 # so if not updated for some time, it is still a
2600 return int(dmcrypt_keysize_str
)
2601 elif dmcrypt_type
== 'plain':
2602 if dmcrypt_keysize_str
is None:
2603 # This value is hard-coded in the udev script
2606 LOG
.warning('ensure the 95-ceph-osd.rules file has '
2607 'been copied to /etc/udev/rules.d '
2608 'and modified to call cryptsetup '
2609 'with --key-size=%s' % dmcrypt_keysize_str
)
2610 return int(dmcrypt_keysize_str
)
2615 def get_dmcrypt_type(args
):
2616 if hasattr(args
, 'dmcrypt') and args
.dmcrypt
:
2617 dmcrypt_type
= get_conf(
2618 cluster
=args
.cluster
,
2619 variable
='osd_dmcrypt_type',
2622 if dmcrypt_type
is None or dmcrypt_type
== 'luks':
2624 elif dmcrypt_type
== 'plain':
2627 raise Error('invalid osd_dmcrypt_type parameter '
2628 '(must be luks or plain): ', dmcrypt_type
)
2633 class Secrets(object):
2636 secret
, stderr
, ret
= command(['ceph-authtool', '--gen-print-key'])
2637 LOG
.debug("stderr " + stderr
)
2640 'cephx_secret': secret
.strip(),
2643 def write_osd_keyring(self
, keyring
, osd_id
):
2646 'ceph-authtool', keyring
,
2648 '--name', 'osd.' + str(osd_id
),
2649 '--add-key', self
.keys
['cephx_secret'],
2651 path_set_context(keyring
)
2654 return bytearray(json
.dumps(self
.keys
), 'ascii')
2657 class LockboxSecrets(Secrets
):
2659 def __init__(self
, args
):
2660 super(LockboxSecrets
, self
).__init
__()
2662 key_size
= CryptHelpers
.get_dmcrypt_keysize(args
)
2663 key
= open('/dev/urandom', 'rb').read(key_size
/ 8)
2664 base64_key
= base64
.b64encode(key
).decode('ascii')
2666 secret
, stderr
, ret
= command(['ceph-authtool', '--gen-print-key'])
2667 LOG
.debug("stderr " + stderr
)
2671 'dmcrypt_key': base64_key
,
2672 'cephx_lockbox_secret': secret
.strip(),
2675 def write_lockbox_keyring(self
, path
, osd_uuid
):
2676 keyring
= os
.path
.join(path
, 'keyring')
2679 'ceph-authtool', keyring
,
2681 '--name', 'client.osd-lockbox.' + osd_uuid
,
2682 '--add-key', self
.keys
['cephx_lockbox_secret'],
2684 path_set_context(keyring
)
2687 class Lockbox(object):
2689 def __init__(self
, args
):
2691 self
.partition
= None
2694 if hasattr(self
.args
, 'lockbox') and self
.args
.lockbox
is None:
2695 self
.args
.lockbox
= self
.args
.data
2697 def set_partition(self
, partition
):
2698 self
.partition
= partition
2702 parser
= argparse
.ArgumentParser(add_help
=False)
2703 parser
.add_argument(
2705 help='path to the device to store the lockbox',
2707 parser
.add_argument(
2710 help='unique lockbox uuid',
2714 def create_partition(self
):
2715 self
.device
= Device
.factory(self
.args
.lockbox
, argparse
.Namespace())
2716 partition_number
= 5
2717 self
.device
.create_partition(uuid
=self
.args
.lockbox_uuid
,
2719 num
=partition_number
,
2721 return self
.device
.get_partition(partition_number
)
2723 def set_or_create_partition(self
):
2724 if is_partition(self
.args
.lockbox
):
2725 LOG
.debug('OSD lockbox device %s is a partition',
2727 self
.partition
= DevicePartition
.factory(
2728 path
=None, dev
=self
.args
.lockbox
, args
=self
.args
)
2729 ptype
= self
.partition
.get_ptype()
2730 ready
= Ptype
.get_ready_by_name('lockbox')
2731 if ptype
not in ready
:
2732 LOG
.warning('incorrect partition UUID: %s, expected %s'
2733 % (ptype
, str(ready
)))
2735 LOG
.debug('Creating osd partition on %s',
2737 self
.partition
= self
.create_partition()
2739 def create_key(self
):
2740 cluster
= self
.args
.cluster
2741 bootstrap
= self
.args
.prepare_key_template
.format(cluster
=cluster
,
2743 path
= self
.get_mount_point()
2744 secrets
= LockboxSecrets(self
.args
)
2745 id_arg
= self
.args
.osd_id
and [self
.args
.osd_id
] or []
2746 osd_id
= command_with_stdin(
2749 '--cluster', cluster
,
2750 '--name', 'client.bootstrap-osd',
2751 '--keyring', bootstrap
,
2753 'osd', 'new', self
.args
.osd_uuid
,
2757 secrets
.write_lockbox_keyring(path
, self
.args
.osd_uuid
)
2758 osd_id
= must_be_one_line(osd_id
)
2759 check_osd_id(osd_id
)
2760 write_one_line(path
, 'whoami', osd_id
)
2761 secrets
.write_osd_keyring(os
.path
.join(path
, 'osd_keyring'), osd_id
)
2762 write_one_line(path
, 'key-management-mode', KEY_MANAGEMENT_MODE_V1
)
2764 def symlink_spaces(self
, path
):
2765 target
= self
.get_mount_point()
2766 for name
in Space
.NAMES
:
2767 if (hasattr(self
.args
, name
+ '_uuid') and
2768 getattr(self
.args
, name
+ '_uuid')):
2769 uuid
= getattr(self
.args
, name
+ '_uuid')
2770 symlink
= os
.path
.join(STATEDIR
, 'osd-lockbox', uuid
)
2771 adjust_symlink(target
, symlink
)
2772 write_one_line(path
, name
+ '-uuid', uuid
)
2775 maybe_mkdir(os
.path
.join(STATEDIR
, 'osd-lockbox'))
2776 args
= ['mkfs', '-t', 'ext4', self
.partition
.get_dev()]
2777 LOG
.debug('Creating lockbox fs on %s: ' + str(" ".join(args
)))
2778 command_check_call(args
)
2779 path
= self
.get_mount_point()
2781 args
= ['mount', '-t', 'ext4', self
.partition
.get_dev(), path
]
2782 LOG
.debug('Mounting lockbox ' + str(" ".join(args
)))
2783 command_check_call(args
)
2784 write_one_line(path
, 'osd-uuid', self
.args
.osd_uuid
)
2785 if self
.args
.cluster_uuid
is None:
2786 self
.args
.cluster_uuid
= get_fsid(cluster
=self
.args
.cluster
)
2787 write_one_line(path
, 'ceph_fsid', self
.args
.cluster_uuid
)
2789 self
.symlink_spaces(path
)
2790 write_one_line(path
, 'magic', CEPH_LOCKBOX_ONDISK_MAGIC
)
2791 if self
.device
is not None:
2795 '--typecode={num}:{uuid}'.format(
2796 num
=self
.partition
.get_partition_number(),
2797 uuid
=self
.partition
.ptype_for_name('lockbox'),
2800 get_partition_base(self
.partition
.get_dev()),
2804 def get_mount_point(self
):
2805 return os
.path
.join(STATEDIR
, 'osd-lockbox', self
.args
.osd_uuid
)
2807 def get_osd_uuid(self
):
2808 return self
.args
.osd_uuid
2811 path
= is_mounted(self
.partition
.get_dev())
2813 LOG
.info("Lockbox already mounted at " + path
)
2816 path
= tempfile
.mkdtemp(
2818 dir=STATEDIR
+ '/tmp',
2820 args
= ['mount', '-t', 'ext4', '-o', 'ro',
2821 self
.partition
.get_dev(),
2823 LOG
.debug('Mounting lockbox temporarily ' + str(" ".join(args
)))
2824 command_check_call(args
)
2825 self
.args
.osd_uuid
= get_oneliner(path
, 'osd-uuid')
2826 command_check_call(['umount', path
])
2827 LOG
.debug('Mounting lockbox readonly ' + str(" ".join(args
)))
2828 args
= ['mount', '-t', 'ext4', '-o', 'ro',
2829 self
.partition
.get_dev(),
2830 self
.get_mount_point()]
2831 command_check_call(args
)
2832 for name
in Space
.NAMES
+ ('osd',):
2833 uuid_path
= os
.path
.join(self
.get_mount_point(), name
+ '-uuid')
2834 if os
.path
.exists(uuid_path
):
2835 uuid
= get_oneliner(self
.get_mount_point(), name
+ '-uuid')
2836 dev
= os
.path
.join('/dev/disk/by-partuuid/', uuid
.lower())
2837 args
= ['ceph-disk', 'trigger', dev
]
2838 command_check_call(args
)
2841 verify_not_in_use(self
.args
.lockbox
, check_partitions
=True)
2842 self
.set_or_create_partition()
2846 class PrepareData(object):
2851 def __init__(self
, args
):
2854 self
.partition
= None
2856 if self
.args
.cluster_uuid
is None:
2857 self
.args
.cluster_uuid
= get_fsid(cluster
=self
.args
.cluster
)
2859 if self
.args
.osd_uuid
is None:
2860 self
.args
.osd_uuid
= str(uuid
.uuid4())
2863 dmode
= os
.stat(self
.args
.data
).st_mode
2865 if stat
.S_ISDIR(dmode
):
2866 self
.type = self
.FILE
2867 elif stmode_is_diskdevice(dmode
):
2868 self
.type = self
.DEVICE
2870 raise Error('not a dir or block device', self
.args
.data
)
2873 return self
.type == self
.FILE
2875 def is_device(self
):
2876 return self
.type == self
.DEVICE
2880 parser
= argparse
.ArgumentParser(add_help
=False)
2881 parser
.add_argument(
2883 help='file system type to use (e.g. "ext4")',
2885 parser
.add_argument(
2887 action
='store_true', default
=None,
2888 help='destroy the partition table (and content) of a disk',
2890 parser
.add_argument(
2892 action
='store_true', default
=None,
2893 help='verify that DATA is a dir',
2895 parser
.add_argument(
2897 action
='store_true', default
=None,
2898 help='verify that DATA is a block device',
2900 parser
.add_argument(
2903 help='path to OSD data (a disk block device or directory)',
2907 def populate_data_path_file(self
, path
, *to_prepare_list
):
2908 self
.populate_data_path(path
, *to_prepare_list
)
2910 def populate_data_path(self
, path
, *to_prepare_list
):
2911 if os
.path
.exists(os
.path
.join(path
, 'magic')):
2912 LOG
.debug('Data dir %s already exists', path
)
2915 LOG
.debug('Preparing osd data dir %s', path
)
2917 if self
.args
.osd_uuid
is None:
2918 self
.args
.osd_uuid
= str(uuid
.uuid4())
2920 write_one_line(path
, 'ceph_fsid', self
.args
.cluster_uuid
)
2921 write_one_line(path
, 'fsid', self
.args
.osd_uuid
)
2922 if self
.args
.osd_id
:
2923 write_one_line(path
, 'wanttobe', self
.args
.osd_id
)
2924 if self
.args
.crush_device_class
:
2925 write_one_line(path
, 'crush_device_class',
2926 self
.args
.crush_device_class
)
2927 write_one_line(path
, 'magic', CEPH_OSD_ONDISK_MAGIC
)
2929 for to_prepare
in to_prepare_list
:
2930 to_prepare
.populate_data_path(path
)
2932 def prepare(self
, *to_prepare_list
):
2933 if self
.type == self
.DEVICE
:
2934 self
.prepare_device(*to_prepare_list
)
2935 elif self
.type == self
.FILE
:
2936 self
.prepare_file(*to_prepare_list
)
2938 raise Error('unexpected type ', self
.type)
2940 def prepare_file(self
, *to_prepare_list
):
2942 if not os
.path
.exists(self
.args
.data
):
2943 raise Error('data path for directory does not exist',
2946 if self
.args
.data_dev
:
2947 raise Error('data path is not a block device', self
.args
.data
)
2949 for to_prepare
in to_prepare_list
:
2950 to_prepare
.prepare()
2952 self
.populate_data_path_file(self
.args
.data
, *to_prepare_list
)
2954 def sanity_checks(self
):
2955 if not os
.path
.exists(self
.args
.data
):
2956 raise Error('data path for device does not exist',
2958 verify_not_in_use(self
.args
.data
,
2959 check_partitions
=not self
.args
.dmcrypt
)
2961 def set_variables(self
):
2962 if self
.args
.fs_type
is None:
2963 self
.args
.fs_type
= get_conf(
2964 cluster
=self
.args
.cluster
,
2965 variable
='osd_mkfs_type',
2967 if self
.args
.fs_type
is None:
2968 self
.args
.fs_type
= get_conf(
2969 cluster
=self
.args
.cluster
,
2970 variable
='osd_fs_type',
2972 if self
.args
.fs_type
is None:
2973 self
.args
.fs_type
= DEFAULT_FS_TYPE
2975 self
.mkfs_args
= get_conf(
2976 cluster
=self
.args
.cluster
,
2977 variable
='osd_mkfs_options_{fstype}'.format(
2978 fstype
=self
.args
.fs_type
,
2981 if self
.mkfs_args
is None:
2982 self
.mkfs_args
= get_conf(
2983 cluster
=self
.args
.cluster
,
2984 variable
='osd_fs_mkfs_options_{fstype}'.format(
2985 fstype
=self
.args
.fs_type
,
2989 self
.mount_options
= get_mount_options(cluster
=self
.args
.cluster
,
2990 fs_type
=self
.args
.fs_type
)
2992 if self
.args
.osd_uuid
is None:
2993 self
.args
.osd_uuid
= str(uuid
.uuid4())
2995 def prepare_device(self
, *to_prepare_list
):
2996 self
.sanity_checks()
2997 self
.set_variables()
2998 if self
.args
.zap_disk
is not None:
3001 def create_data_partition(self
):
3002 device
= Device
.factory(self
.args
.data
, self
.args
)
3003 partition_number
= 1
3004 device
.create_partition(uuid
=self
.args
.osd_uuid
,
3006 num
=partition_number
,
3007 size
=self
.get_space_size())
3008 return device
.get_partition(partition_number
)
3010 def set_data_partition(self
):
3011 if is_partition(self
.args
.data
):
3012 LOG
.debug('OSD data device %s is a partition',
3014 self
.partition
= DevicePartition
.factory(
3015 path
=None, dev
=self
.args
.data
, args
=self
.args
)
3016 ptype
= self
.partition
.get_ptype()
3017 ready
= Ptype
.get_ready_by_name('osd')
3018 if ptype
not in ready
:
3019 LOG
.warning('incorrect partition UUID: %s, expected %s'
3020 % (ptype
, str(ready
)))
3022 LOG
.debug('Creating osd partition on %s',
3024 self
.partition
= self
.create_data_partition()
3026 def populate_data_path_device(self
, *to_prepare_list
):
3027 partition
= self
.partition
3029 if isinstance(partition
, DevicePartitionCrypt
):
3038 if self
.mkfs_args
is not None:
3039 args
.extend(self
.mkfs_args
.split())
3040 if self
.args
.fs_type
== 'xfs':
3041 args
.extend(['-f']) # always force
3043 args
.extend(MKFS_ARGS
.get(self
.args
.fs_type
, []))
3046 partition
.get_dev(),
3048 LOG
.debug('Creating %s fs on %s',
3049 self
.args
.fs_type
, partition
.get_dev())
3050 command_check_call(args
, exit
=True)
3052 path
= mount(dev
=partition
.get_dev(),
3053 fstype
=self
.args
.fs_type
,
3054 options
=self
.mount_options
)
3057 self
.populate_data_path(path
, *to_prepare_list
)
3059 path_set_context(path
)
3062 if isinstance(partition
, DevicePartitionCrypt
):
3065 if not is_partition(self
.args
.data
):
3069 '--typecode=%d:%s' % (partition
.get_partition_number(),
3070 partition
.ptype_for_name('osd')),
3076 update_partition(self
.args
.data
, 'prepared')
3077 command_check_call(['udevadm', 'trigger',
3080 os
.path
.basename(partition
.rawdev
)])
3083 class PrepareFilestoreData(PrepareData
):
3085 def get_space_size(self
):
3086 return 0 # get as much space as possible
3088 def prepare_device(self
, *to_prepare_list
):
3089 super(PrepareFilestoreData
, self
).prepare_device(*to_prepare_list
)
3090 for to_prepare
in to_prepare_list
:
3091 to_prepare
.prepare()
3092 self
.set_data_partition()
3093 self
.populate_data_path_device(*to_prepare_list
)
3095 def populate_data_path(self
, path
, *to_prepare_list
):
3096 super(PrepareFilestoreData
, self
).populate_data_path(path
,
3098 write_one_line(path
, 'type', 'filestore')
3101 class PrepareBluestoreData(PrepareData
):
3103 def get_space_size(self
):
3106 def prepare_device(self
, *to_prepare_list
):
3107 super(PrepareBluestoreData
, self
).prepare_device(*to_prepare_list
)
3108 self
.set_data_partition()
3109 for to_prepare
in to_prepare_list
:
3110 to_prepare
.prepare()
3111 self
.populate_data_path_device(*to_prepare_list
)
3113 def populate_data_path(self
, path
, *to_prepare_list
):
3114 super(PrepareBluestoreData
, self
).populate_data_path(path
,
3116 write_one_line(path
, 'type', 'bluestore')
3126 monmap
= os
.path
.join(path
, 'activate.monmap')
3130 '--cluster', cluster
,
3131 '--name', 'client.bootstrap-osd',
3132 '--keyring', keyring
,
3133 'mon', 'getmap', '-o', monmap
,
3137 osd_type
= read_one_line(path
, 'type')
3139 if osd_type
== 'bluestore':
3143 '--cluster', cluster
,
3149 '--setuser', get_ceph_user(),
3150 '--setgroup', get_ceph_group(),
3153 elif osd_type
== 'filestore':
3157 '--cluster', cluster
,
3162 '--osd-journal', os
.path
.join(path
, 'journal'),
3164 '--setuser', get_ceph_user(),
3165 '--setgroup', get_ceph_group(),
3169 raise Error('unrecognized objectstore type %s' % osd_type
)
3172 def get_mount_point(cluster
, osd_id
):
3173 parent
= STATEDIR
+ '/osd'
3174 return os
.path
.join(
3176 '{cluster}-{osd_id}'.format(cluster
=cluster
, osd_id
=osd_id
),
3188 LOG
.debug('Moving mount to final location...')
3189 osd_data
= get_mount_point(cluster
, osd_id
)
3190 maybe_mkdir(osd_data
)
3192 # pick best-of-breed mount options based on fs type
3193 if mount_options
is None:
3194 mount_options
= MOUNT_OPTIONS
.get(fstype
, '')
3196 # we really want to mount --move, but that is not supported when
3197 # the parent mount is shared, as it is by default on RH, Fedora,
3198 # and probably others. Also, --bind doesn't properly manipulate
3199 # /etc/mtab, which *still* isn't a symlink to /proc/mounts despite
3200 # this being 2013. Instead, mount the original device at the final
3215 '-l', # lazy, in case someone else is peeking at the
3224 # For upgrade purposes, to make sure there are no competing units,
3225 # both --runtime unit and the default should be disabled. There can be
3226 # two units at the same time: one with --runtime and another without
3227 # it. If, for any reason (manual or ceph-disk) the two units co-exist
3228 # they will compete with each other.
3230 def systemd_disable(
3234 # ensure there is no duplicate ceph-osd@.service
3235 for style
in ([], ['--runtime']):
3240 'ceph-osd@{osd_id}'.format(osd_id
=osd_id
),
3249 systemd_disable(path
, osd_id
)
3250 if os
.path
.ismount(path
):
3251 style
= ['--runtime']
3258 'ceph-osd@{osd_id}'.format(osd_id
=osd_id
),
3265 'ceph-osd@{osd_id}'.format(osd_id
=osd_id
),
3274 systemd_disable(path
, osd_id
)
3279 'ceph-osd@{osd_id}'.format(osd_id
=osd_id
),
3288 LOG
.debug('Starting %s osd.%s...', cluster
, osd_id
)
3290 path
= (STATEDIR
+ '/osd/{cluster}-{osd_id}').format(
3291 cluster
=cluster
, osd_id
=osd_id
)
3294 if os
.path
.exists(os
.path
.join(path
, 'upstart')):
3298 # use emit, not start, because start would fail if the
3299 # instance was already running
3301 # since the daemon starting doesn't guarantee much about
3302 # the service being operational anyway, don't bother
3307 'cluster={cluster}'.format(cluster
=cluster
),
3308 'id={osd_id}'.format(osd_id
=osd_id
),
3311 elif os
.path
.exists(os
.path
.join(path
, 'sysvinit')):
3312 if os
.path
.exists('/usr/sbin/service'):
3313 svc
= '/usr/sbin/service'
3315 svc
= '/sbin/service'
3321 '{cluster}'.format(cluster
=cluster
),
3323 'osd.{osd_id}'.format(osd_id
=osd_id
),
3326 elif os
.path
.exists(os
.path
.join(path
, 'systemd')):
3327 systemd_start(path
, osd_id
)
3328 elif os
.path
.exists(os
.path
.join(path
, 'openrc')):
3329 base_script
= '/etc/init.d/ceph-osd'
3330 osd_script
= '{base}.{osd_id}'.format(
3334 if not os
.path
.exists(osd_script
):
3335 os
.symlink(base_script
, osd_script
)
3342 elif os
.path
.exists(os
.path
.join(path
, 'bsdrc')):
3345 '/usr/sbin/service', 'ceph', 'start',
3346 'osd.{osd_id}'.format(osd_id
=osd_id
),
3350 raise Error('{cluster} osd.{osd_id} '
3351 'is not tagged with an init system'
3356 except subprocess
.CalledProcessError
as e
:
3357 raise Error('ceph osd start failed', e
)
3364 LOG
.debug('Stoping %s osd.%s...', cluster
, osd_id
)
3366 path
= (STATEDIR
+ '/osd/{cluster}-{osd_id}').format(
3367 cluster
=cluster
, osd_id
=osd_id
)
3370 if os
.path
.exists(os
.path
.join(path
, 'upstart')):
3376 'cluster={cluster}'.format(cluster
=cluster
),
3377 'id={osd_id}'.format(osd_id
=osd_id
),
3380 elif os
.path
.exists(os
.path
.join(path
, 'sysvinit')):
3381 svc
= which('service')
3387 '{cluster}'.format(cluster
=cluster
),
3389 'osd.{osd_id}'.format(osd_id
=osd_id
),
3392 elif os
.path
.exists(os
.path
.join(path
, 'systemd')):
3393 systemd_stop(path
, osd_id
)
3394 elif os
.path
.exists(os
.path
.join(path
, 'openrc')):
3397 '/etc/init.d/ceph-osd.{osd_id}'.format(osd_id
=osd_id
),
3401 elif os
.path
.exists(os
.path
.join(path
, 'bsdrc')):
3404 '/usr/local/etc/rc.d/ceph stop osd.{osd_id}'
3405 .format(osd_id
=osd_id
),
3409 raise Error('{cluster} osd.{osd_id} '
3410 'is not tagged with an init system'
3411 .format(cluster
=cluster
, osd_id
=osd_id
))
3412 except subprocess
.CalledProcessError
as e
:
3413 raise Error('ceph osd stop failed', e
)
3416 def detect_fstype(dev
):
3418 fstype
= _check_output(
3426 fstype
= _check_output(
3429 # we don't want stale cached results
3437 fstype
= must_be_one_line(fstype
)
3441 def dmcrypt_is_mapped(uuid
):
3442 path
= os
.path
.join('/dev/mapper', uuid
)
3443 if os
.path
.exists(path
):
3449 def dmcrypt_map(dev
, dmcrypt_key_dir
):
3450 ptype
= get_partition_type(dev
)
3451 if ptype
in Ptype
.get_ready_by_type('plain'):
3453 cryptsetup_parameters
= ['--key-size', '256']
3454 elif ptype
in Ptype
.get_ready_by_type('luks'):
3456 cryptsetup_parameters
= []
3458 raise Error('--dmcrypt called for dev %s with invalid ptype %s'
3460 part_uuid
= get_partition_uuid(dev
)
3461 dmcrypt_key
= get_dmcrypt_key(part_uuid
, dmcrypt_key_dir
, luks
)
3462 return _dmcrypt_map(
3466 cryptsetup_parameters
=cryptsetup_parameters
,
3474 activate_key_template
,
3482 part_uuid
= get_partition_uuid(dev
)
3483 dev
= dmcrypt_map(dev
, dmcrypt_key_dir
)
3485 fstype
= detect_fstype(dev
=dev
)
3486 except (subprocess
.CalledProcessError
,
3488 TooManyLinesError
) as e
:
3489 raise FilesystemTypeError(
3490 'device {dev}'.format(dev
=dev
),
3494 # TODO always using mount options from cluster=ceph for
3495 # now; see http://tracker.newdream.net/issues/3253
3496 mount_options
= get_mount_options(cluster
='ceph', fs_type
=fstype
)
3498 path
= mount(dev
=dev
, fstype
=fstype
, options
=mount_options
)
3500 # check if the disk is deactive, change the journal owner, group
3501 # mode for correct user and group.
3502 if os
.path
.exists(os
.path
.join(path
, 'deactive')):
3503 # logging to syslog will help us easy to know udev triggered failure
3506 # we need to unmap again because dmcrypt map will create again
3507 # on bootup stage (due to deactivate)
3508 if '/dev/mapper/' in dev
:
3509 part_uuid
= dev
.replace('/dev/mapper/', '')
3510 dmcrypt_unmap(part_uuid
)
3511 LOG
.info('OSD deactivated! reactivate with: --reactivate')
3512 raise Error('OSD deactivated! reactivate with: --reactivate')
3513 # flag to activate a deactive osd.
3521 (osd_id
, cluster
) = activate(path
, activate_key_template
, init
)
3523 # Now active successfully
3524 # If we got reactivate and deactive, remove the deactive file
3525 if deactive
and reactivate
:
3526 os
.remove(os
.path
.join(path
, 'deactive'))
3527 LOG
.info('Remove `deactive` file.')
3529 # check if the disk is already active, or if something else is already
3533 src_dev
= os
.stat(path
).st_dev
3535 dst_dev
= os
.stat((STATEDIR
+ '/osd/{cluster}-{osd_id}').format(
3537 osd_id
=osd_id
)).st_dev
3538 if src_dev
== dst_dev
:
3541 parent_dev
= os
.stat(STATEDIR
+ '/osd').st_dev
3542 if dst_dev
!= parent_dev
:
3544 elif os
.listdir(get_mount_point(cluster
, osd_id
)):
3545 LOG
.info(get_mount_point(cluster
, osd_id
) +
3546 " is not empty, won't override")
3553 LOG
.info('%s osd.%s already mounted in position; unmounting ours.'
3554 % (cluster
, osd_id
))
3557 raise Error('another %s osd.%s already mounted in position '
3558 '(old/different cluster instance?); unmounting ours.'
3559 % (cluster
, osd_id
))
3567 mount_options
=mount_options
,
3569 return cluster
, osd_id
3572 LOG
.error('Failed to activate')
3576 # remove our temp dir
3577 if os
.path
.exists(path
):
3583 activate_key_template
,
3587 if not os
.path
.exists(path
):
3589 'directory %s does not exist' % path
3592 (osd_id
, cluster
) = activate(path
, activate_key_template
, init
)
3594 if init
not in (None, 'none'):
3595 canonical
= (STATEDIR
+ '/osd/{cluster}-{osd_id}').format(
3598 if path
!= canonical
:
3599 # symlink it from the proper location
3601 if os
.path
.lexists(canonical
):
3602 old
= os
.readlink(canonical
)
3604 LOG
.debug('Removing old symlink %s -> %s', canonical
, old
)
3606 os
.unlink(canonical
)
3608 raise Error('unable to remove old symlink', canonical
)
3612 LOG
.debug('Creating symlink %s -> %s', canonical
, path
)
3614 os
.symlink(path
, canonical
)
3616 raise Error('unable to create symlink %s -> %s'
3617 % (canonical
, path
))
3619 return cluster
, osd_id
3622 def find_cluster_by_uuid(_uuid
):
3624 Find a cluster name by searching /etc/ceph/*.conf for a conf file
3625 with the right uuid.
3627 _uuid
= _uuid
.lower()
3629 if not os
.path
.exists(SYSCONFDIR
):
3631 for conf_file
in os
.listdir(SYSCONFDIR
):
3632 if not conf_file
.endswith('.conf'):
3634 cluster
= conf_file
[:-5]
3636 fsid
= get_fsid(cluster
)
3638 if 'getting cluster uuid from configuration failed' not in str(e
):
3640 no_fsid
.append(cluster
)
3644 # be tolerant of /etc/ceph/ceph.conf without an fsid defined.
3645 if len(no_fsid
) == 1 and no_fsid
[0] == 'ceph':
3646 LOG
.warning('No fsid defined in ' + SYSCONFDIR
+
3647 '/ceph.conf; using anyway')
3654 activate_key_template
,
3658 check_osd_magic(path
)
3660 ceph_fsid
= read_one_line(path
, 'ceph_fsid')
3661 if ceph_fsid
is None:
3662 raise Error('No cluster uuid assigned.')
3663 LOG
.debug('Cluster uuid is %s', ceph_fsid
)
3665 cluster
= find_cluster_by_uuid(ceph_fsid
)
3667 raise Error('No cluster conf found in ' + SYSCONFDIR
+
3668 ' with fsid %s' % ceph_fsid
)
3669 LOG
.debug('Cluster name is %s', cluster
)
3671 fsid
= read_one_line(path
, 'fsid')
3673 raise Error('No OSD uuid assigned.')
3674 LOG
.debug('OSD uuid is %s', fsid
)
3676 keyring
= activate_key_template
.format(cluster
=cluster
,
3679 osd_id
= get_osd_id(path
)
3681 osd_id
= allocate_osd_id(
3687 write_one_line(path
, 'whoami', osd_id
)
3688 LOG
.debug('OSD id is %s', osd_id
)
3690 if not os
.path
.exists(os
.path
.join(path
, 'ready')):
3691 LOG
.debug('Initializing OSD...')
3692 # re-running mkfs is safe, so just run until it completes
3701 if init
not in (None, 'none'):
3703 conf_val
= get_conf(
3707 if conf_val
is not None:
3712 LOG
.debug('Marking with init system %s', init
)
3713 init_path
= os
.path
.join(path
, init
)
3714 with
open(init_path
, 'w'):
3715 path_set_context(init_path
)
3717 # remove markers for others, just in case.
3718 for other
in INIT_SYSTEMS
:
3721 os
.unlink(os
.path
.join(path
, other
))
3725 if not os
.path
.exists(os
.path
.join(path
, 'active')):
3726 write_one_line(path
, 'active', 'ok')
3727 LOG
.debug('%s osd.%s data dir is ready at %s', cluster
, osd_id
, path
)
3728 return (osd_id
, cluster
)
3731 def main_activate(args
):
3735 LOG
.info('path = ' + str(args
.path
))
3736 if not os
.path
.exists(args
.path
):
3737 raise Error('%s does not exist' % args
.path
)
3739 if is_suppressed(args
.path
):
3740 LOG
.info('suppressed activate request on %s', args
.path
)
3744 mode
= os
.stat(args
.path
).st_mode
3745 if stmode_is_diskdevice(mode
):
3746 if (is_partition(args
.path
) and
3747 (get_partition_type(args
.path
) ==
3748 PTYPE
['mpath']['osd']['ready']) and
3749 not is_mpath(args
.path
)):
3750 raise Error('%s is not a multipath block device' %
3752 (cluster
, osd_id
) = mount_activate(
3754 activate_key_template
=args
.activate_key_template
,
3755 init
=args
.mark_init
,
3756 dmcrypt
=args
.dmcrypt
,
3757 dmcrypt_key_dir
=args
.dmcrypt_key_dir
,
3758 reactivate
=args
.reactivate
,
3760 osd_data
= get_mount_point(cluster
, osd_id
)
3762 args
.cluster
= cluster
3764 for name
in Space
.NAMES
:
3765 # Check if encrypted device in journal
3766 dev_path
= os
.path
.join(osd_data
, name
+ '_dmcrypt')
3767 if not os
.path
.exists(dev_path
):
3769 partition
= DevicePartition
.factory(
3773 partition
.rawdev
= args
.path
3776 elif stat
.S_ISDIR(mode
):
3777 (cluster
, osd_id
) = activate_dir(
3779 activate_key_template
=args
.activate_key_template
,
3780 init
=args
.mark_init
,
3782 osd_data
= args
.path
3785 raise Error('%s is not a directory or block device' % args
.path
)
3787 # exit with 0 if the journal device is not up, yet
3788 # journal device will do the activation
3789 osd_journal
= '{path}/journal'.format(path
=osd_data
)
3790 if os
.path
.islink(osd_journal
) and not os
.access(osd_journal
, os
.F_OK
):
3791 LOG
.info("activate: Journal not present, not starting, yet")
3794 if (not args
.no_start_daemon
and args
.mark_init
== 'none'):
3798 '--cluster={cluster}'.format(cluster
=cluster
),
3799 '--id={osd_id}'.format(osd_id
=osd_id
),
3800 '--osd-data={path}'.format(path
=osd_data
),
3801 '--osd-journal={journal}'.format(journal
=osd_journal
),
3805 if (not args
.no_start_daemon
and
3806 args
.mark_init
not in (None, 'none')):
3814 def main_activate_lockbox(args
):
3816 main_activate_lockbox_protected(args
)
3819 def main_activate_lockbox_protected(args
):
3820 partition
= DevicePartition
.factory(
3821 path
=None, dev
=args
.path
, args
=args
)
3823 lockbox
= Lockbox(args
)
3824 lockbox
.set_partition(partition
)
3828 ###########################
3830 def _mark_osd_out(cluster
, osd_id
):
3831 LOG
.info('Prepare to mark osd.%d out...', osd_id
)
3840 def _check_osd_status(cluster
, osd_id
):
3842 report the osd status:
3843 00(0) : means OSD OUT AND DOWN
3844 01(1) : means OSD OUT AND UP
3845 10(2) : means OSD IN AND DOWN
3846 11(3) : means OSD IN AND UP
3848 LOG
.info("Checking osd id: %s ..." % osd_id
)
3851 out
, err
, ret
= command([
3855 '--cluster={cluster}'.format(
3861 out_json
= json
.loads(out
)
3862 for item
in out_json
[u
'osds']:
3863 if item
.get(u
'osd') == int(osd_id
):
3865 if item
.get(u
'in') is 1:
3867 if item
.get(u
'up') is 1:
3870 raise Error('Could not osd.%s in osd tree!' % osd_id
)
3874 def _remove_osd_directory_files(mounted_path
, cluster
):
3876 To remove the 'ready', 'active', INIT-specific files.
3878 if os
.path
.exists(os
.path
.join(mounted_path
, 'ready')):
3879 os
.remove(os
.path
.join(mounted_path
, 'ready'))
3880 LOG
.info('Remove `ready` file.')
3882 LOG
.info('`ready` file is already removed.')
3884 if os
.path
.exists(os
.path
.join(mounted_path
, 'active')):
3885 os
.remove(os
.path
.join(mounted_path
, 'active'))
3886 LOG
.info('Remove `active` file.')
3888 LOG
.info('`active` file is already removed.')
3890 # Just check `upstart` and `sysvinit` directly if filename is init-spec.
3891 conf_val
= get_conf(
3895 if conf_val
is not None:
3899 os
.remove(os
.path
.join(mounted_path
, init
))
3900 LOG
.info('Remove `%s` file.', init
)
3904 def main_deactivate(args
):
3906 main_deactivate_locked(args
)
3909 def main_deactivate_locked(args
):
3910 osd_id
= args
.deactivate_by_id
3914 devices
= list_devices()
3916 # list all devices and found we need
3917 for device
in devices
:
3918 if 'partitions' in device
:
3919 for dev_part
in device
.get('partitions'):
3921 'whoami' in dev_part
and
3922 dev_part
['whoami'] == osd_id
):
3923 target_dev
= dev_part
3925 'path' in dev_part
and
3926 dev_part
['path'] == path
):
3927 target_dev
= dev_part
3929 raise Error('Cannot find any match device!!')
3931 # set up all we need variable
3932 osd_id
= target_dev
['whoami']
3933 part_type
= target_dev
['ptype']
3934 mounted_path
= target_dev
['mount']
3935 if Ptype
.is_dmcrypt(part_type
, 'osd'):
3938 # Do not do anything if osd is already down.
3939 status_code
= _check_osd_status(args
.cluster
, osd_id
)
3940 if status_code
== OSD_STATUS_IN_UP
:
3941 if args
.mark_out
is True:
3942 _mark_osd_out(args
.cluster
, int(osd_id
))
3943 stop_daemon(args
.cluster
, osd_id
)
3944 elif status_code
== OSD_STATUS_IN_DOWN
:
3945 if args
.mark_out
is True:
3946 _mark_osd_out(args
.cluster
, int(osd_id
))
3947 LOG
.info("OSD already out/down. Do not do anything now.")
3949 elif status_code
== OSD_STATUS_OUT_UP
:
3950 stop_daemon(args
.cluster
, osd_id
)
3951 elif status_code
== OSD_STATUS_OUT_DOWN
:
3952 LOG
.info("OSD already out/down. Do not do anything now.")
3956 # remove 'ready', 'active', and INIT-specific files.
3957 _remove_osd_directory_files(mounted_path
, args
.cluster
)
3959 # Write deactivate to osd directory!
3960 with
open(os
.path
.join(mounted_path
, 'deactive'), 'w'):
3961 path_set_context(os
.path
.join(mounted_path
, 'deactive'))
3963 unmount(mounted_path
, do_rm
=not args
.once
)
3964 LOG
.info("Umount `%s` successfully.", mounted_path
)
3967 lockbox
= os
.path
.join(STATEDIR
, 'osd-lockbox')
3968 command(['umount', os
.path
.join(lockbox
, target_dev
['uuid'])])
3970 dmcrypt_unmap(target_dev
['uuid'])
3971 for name
in Space
.NAMES
:
3972 if name
+ '_uuid' in target_dev
:
3973 dmcrypt_unmap(target_dev
[name
+ '_uuid'])
3975 ###########################
3978 def _remove_lockbox(uuid
, cluster
):
3979 lockbox
= os
.path
.join(STATEDIR
, 'osd-lockbox')
3980 if not os
.path
.exists(lockbox
):
3982 canonical
= os
.path
.join(lockbox
, uuid
)
3983 command(['umount', canonical
])
3984 for name
in os
.listdir(lockbox
):
3985 path
= os
.path
.join(lockbox
, name
)
3986 if os
.path
.islink(path
) and os
.readlink(path
) == canonical
:
3990 def destroy_lookup_device(args
, predicate
, description
):
3991 devices
= list_devices()
3992 for device
in devices
:
3993 for partition
in device
.get('partitions', []):
3994 if partition
['type'] == 'lockbox':
3995 if not is_mounted(partition
['path']):
3996 main_activate_lockbox_protected(
3997 argparse
.Namespace(verbose
=args
.verbose
,
3998 path
=partition
['path']))
3999 for device
in devices
:
4000 for partition
in device
.get('partitions', []):
4001 if partition
['dmcrypt']:
4002 dmcrypt_path
= dmcrypt_is_mapped(partition
['uuid'])
4006 dmcrypt_path
= dmcrypt_map(partition
['path'],
4007 args
.dmcrypt_key_dir
)
4009 list_dev_osd(dmcrypt_path
, {}, partition
)
4011 dmcrypt_unmap(partition
['uuid'])
4015 if predicate(partition
):
4016 return dmcrypt
, partition
4017 raise Error('found no device matching ', description
)
4020 def main_destroy(args
):
4022 main_destroy_locked(args
)
4025 def main_destroy_locked(args
):
4026 osd_id
= args
.destroy_by_id
4031 if not is_partition(path
):
4032 raise Error(path
+ " must be a partition device")
4033 path
= os
.path
.realpath(path
)
4036 (dmcrypt
, target_dev
) = destroy_lookup_device(
4037 args
, lambda x
: x
.get('path') == path
,
4040 (dmcrypt
, target_dev
) = destroy_lookup_device(
4041 args
, lambda x
: x
.get('whoami') == osd_id
,
4042 'osd id ' + str(osd_id
))
4044 osd_id
= target_dev
['whoami']
4045 dev_path
= target_dev
['path']
4046 if target_dev
['ptype'] == PTYPE
['mpath']['osd']['ready']:
4047 base_dev
= get_partition_base_mpath(dev_path
)
4049 base_dev
= get_partition_base(dev_path
)
4051 # Before osd deactivate, we cannot destroy it
4052 status_code
= _check_osd_status(args
.cluster
, osd_id
)
4053 if status_code
!= OSD_STATUS_OUT_DOWN
and \
4054 status_code
!= OSD_STATUS_IN_DOWN
:
4055 raise Error("Could not destroy the active osd. (osd-id: %s)" %
4062 LOG
.info("Prepare to %s osd.%s" % (action
, osd_id
))
4068 '--yes-i-really-mean-it',
4071 # we remove the crypt map and device mapper (if dmcrypt is True)
4073 for name
in Space
.NAMES
:
4074 if target_dev
.get(name
+ '_uuid'):
4075 dmcrypt_unmap(target_dev
[name
+ '_uuid'])
4076 _remove_lockbox(target_dev
['uuid'], args
.cluster
)
4078 # Check zap flag. If we found zap flag, we need to find device for
4079 # destroy this osd data.
4080 if args
.zap
is True:
4081 # erase the osd data
4082 LOG
.info("Prepare to zap the device %s" % base_dev
)
4086 def get_space_osd_uuid(name
, path
):
4087 if not os
.path
.exists(path
):
4088 raise Error('%s does not exist' % path
)
4090 if not path_is_diskdevice(path
):
4091 raise Error('%s is not a block device' % path
)
4093 if (is_partition(path
) and
4094 get_partition_type(path
) in (PTYPE
['mpath']['journal']['ready'],
4095 PTYPE
['mpath']['block']['ready']) and
4096 not is_mpath(path
)):
4097 raise Error('%s is not a multipath block device' %
4101 out
= _check_output(
4104 '--get-device-fsid',
4109 except subprocess
.CalledProcessError
as e
:
4111 'failed to get osd uuid/fsid from %s' % name
,
4114 value
= str(out
).split('\n', 1)[0]
4115 LOG
.debug('%s %s has OSD UUID %s', name
.capitalize(), path
, value
)
4119 def main_activate_space(name
, args
):
4120 if not os
.path
.exists(args
.dev
):
4121 raise Error('%s does not exist' % args
.dev
)
4123 if is_suppressed(args
.dev
):
4124 LOG
.info('suppressed activate request on space %s', args
.dev
)
4133 dev
= dmcrypt_map(args
.dev
, args
.dmcrypt_key_dir
)
4136 # FIXME: For an encrypted journal dev, does this return the
4137 # cyphertext or plaintext dev uuid!? Also, if the journal is
4138 # encrypted, is the data partition also always encrypted, or
4139 # are mixed pairs supported!?
4140 osd_uuid
= get_space_osd_uuid(name
, dev
)
4141 path
= os
.path
.join('/dev/disk/by-partuuid/', osd_uuid
.lower())
4143 if is_suppressed(path
):
4144 LOG
.info('suppressed activate request on %s', path
)
4147 # warn and exit with 0 if the data device is not up, yet
4148 # data device will do the activation
4149 if not os
.access(path
, os
.F_OK
):
4150 LOG
.info("activate: OSD device not present, not starting, yet")
4153 (cluster
, osd_id
) = mount_activate(
4155 activate_key_template
=args
.activate_key_template
,
4156 init
=args
.mark_init
,
4157 dmcrypt
=args
.dmcrypt
,
4158 dmcrypt_key_dir
=args
.dmcrypt_key_dir
,
4159 reactivate
=args
.reactivate
,
4168 ###########################
4171 def main_activate_all(args
):
4172 dir = '/dev/disk/by-parttypeuuid'
4173 LOG
.debug('Scanning %s', dir)
4174 if not os
.path
.exists(dir):
4177 for name
in os
.listdir(dir):
4178 if name
.find('.') < 0:
4180 (tag
, uuid
) = name
.split('.')
4182 if tag
in Ptype
.get_ready_by_name('osd'):
4184 if Ptype
.is_dmcrypt(tag
, 'osd'):
4185 path
= os
.path
.join('/dev/mapper', uuid
)
4187 path
= os
.path
.join(dir, name
)
4189 if is_suppressed(path
):
4190 LOG
.info('suppressed activate request on %s', path
)
4193 LOG
.info('Activating %s', path
)
4196 # never map dmcrypt cyphertext devices
4197 (cluster
, osd_id
) = mount_activate(
4199 activate_key_template
=args
.activate_key_template
,
4200 init
=args
.mark_init
,
4209 except Exception as e
:
4211 '{prog}: {msg}'.format(prog
=args
.prog
, msg
=e
),
4218 raise Error('One or more partitions failed to activate')
4221 ###########################
4224 dev
= os
.path
.realpath(dev
)
4225 with
open(PROCDIR
+ '/swaps', 'rb') as proc_swaps
:
4226 for line
in proc_swaps
.readlines()[1:]:
4227 fields
= line
.split()
4230 swaps_dev
= fields
[0]
4231 if os
.path
.isabs(swaps_dev
) and os
.path
.exists(swaps_dev
):
4232 swaps_dev
= os
.path
.realpath(swaps_dev
)
4233 if swaps_dev
== dev
:
4238 def get_oneliner(base
, name
):
4239 path
= os
.path
.join(base
, name
)
4240 if os
.path
.isfile(path
):
4241 with
open(path
, 'rb') as _file
:
4242 return _bytes2str(_file
.readline().rstrip())
4246 def get_dev_fs(dev
):
4248 fstype
, _
, ret
= command(
4258 fscheck
, _
, _
= command(
4266 if 'TYPE' in fscheck
:
4267 fstype
= fscheck
.split()[1].split('"')[1]
4272 def split_dev_base_partnum(dev
):
4274 partnum
= partnum_mpath(dev
)
4275 base
= get_partition_base_mpath(dev
)
4278 partnum
= open(os
.path
.join(b
, 'partition')).read().strip()
4279 base
= get_partition_base(dev
)
4280 return base
, partnum
4283 def get_partition_type(part
):
4284 return get_blkid_partition_info(part
, 'ID_PART_ENTRY_TYPE')
4287 def get_partition_uuid(part
):
4288 return get_blkid_partition_info(part
, 'ID_PART_ENTRY_UUID')
4291 def get_blkid_partition_info(dev
, what
=None):
4292 out
, _
, _
= command(
4302 for line
in out
.splitlines():
4303 (key
, value
) = line
.split('=')
4311 def more_osd_info(path
, uuid_map
, desc
):
4312 desc
['ceph_fsid'] = get_oneliner(path
, 'ceph_fsid')
4313 if desc
['ceph_fsid']:
4314 desc
['cluster'] = find_cluster_by_uuid(desc
['ceph_fsid'])
4315 desc
['whoami'] = get_oneliner(path
, 'whoami')
4316 for name
in Space
.NAMES
:
4317 uuid
= get_oneliner(path
, name
+ '_uuid')
4319 desc
[name
+ '_uuid'] = uuid
.lower()
4320 if desc
[name
+ '_uuid'] in uuid_map
:
4321 desc
[name
+ '_dev'] = uuid_map
[desc
[name
+ '_uuid']]
4324 def list_dev_osd(dev
, uuid_map
, desc
):
4325 desc
['mount'] = is_mounted(dev
)
4326 desc
['fs_type'] = get_dev_fs(dev
)
4327 desc
['state'] = 'unprepared'
4329 desc
['state'] = 'active'
4330 more_osd_info(desc
['mount'], uuid_map
, desc
)
4331 elif desc
['fs_type']:
4333 tpath
= mount(dev
=dev
, fstype
=desc
['fs_type'], options
='')
4336 magic
= get_oneliner(tpath
, 'magic')
4337 if magic
is not None:
4338 desc
['magic'] = magic
4339 desc
['state'] = 'prepared'
4340 more_osd_info(tpath
, uuid_map
, desc
)
4347 def list_dev_lockbox(dev
, uuid_map
, desc
):
4348 desc
['mount'] = is_mounted(dev
)
4349 desc
['fs_type'] = get_dev_fs(dev
)
4350 desc
['state'] = 'unprepared'
4352 desc
['state'] = 'active'
4353 desc
['osd_uuid'] = get_oneliner(desc
['mount'], 'osd-uuid')
4354 elif desc
['fs_type']:
4356 tpath
= tempfile
.mkdtemp(prefix
='mnt.', dir=STATEDIR
+ '/tmp')
4357 args
= ['mount', '-t', 'ext4', dev
, tpath
]
4358 LOG
.debug('Mounting lockbox ' + str(" ".join(args
)))
4359 command_check_call(args
)
4360 magic
= get_oneliner(tpath
, 'magic')
4361 if magic
is not None:
4362 desc
['magic'] = magic
4363 desc
['state'] = 'prepared'
4364 desc
['osd_uuid'] = get_oneliner(tpath
, 'osd-uuid')
4366 except subprocess
.CalledProcessError
:
4368 if desc
.get('osd_uuid') in uuid_map
:
4369 desc
['lockbox_for'] = uuid_map
[desc
['osd_uuid']]
4372 def list_format_lockbox_plain(dev
):
4374 if dev
.get('lockbox_for'):
4375 desc
.append('for ' + dev
['lockbox_for'])
4376 elif dev
.get('osd_uuid'):
4377 desc
.append('for osd ' + dev
['osd_uuid'])
4381 def list_format_more_osd_info_plain(dev
):
4383 if dev
.get('ceph_fsid'):
4384 if dev
.get('cluster'):
4385 desc
.append('cluster ' + dev
['cluster'])
4387 desc
.append('unknown cluster ' + dev
['ceph_fsid'])
4388 if dev
.get('whoami'):
4389 desc
.append('osd.%s' % dev
['whoami'])
4390 for name
in Space
.NAMES
:
4391 if dev
.get(name
+ '_dev'):
4392 desc
.append(name
+ ' %s' % dev
[name
+ '_dev'])
4396 def list_format_dev_plain(dev
, prefix
=''):
4398 if dev
['ptype'] == PTYPE
['regular']['osd']['ready']:
4399 desc
= (['ceph data', dev
['state']] +
4400 list_format_more_osd_info_plain(dev
))
4401 elif dev
['ptype'] in (PTYPE
['regular']['lockbox']['ready'],
4402 PTYPE
['mpath']['lockbox']['ready']):
4403 desc
= (['ceph lockbox', dev
['state']] +
4404 list_format_lockbox_plain(dev
))
4405 elif Ptype
.is_dmcrypt(dev
['ptype'], 'osd'):
4406 dmcrypt
= dev
['dmcrypt']
4407 if not dmcrypt
['holders']:
4408 desc
= ['ceph data (dmcrypt %s)' % dmcrypt
['type'],
4409 'not currently mapped']
4410 elif len(dmcrypt
['holders']) == 1:
4411 holder
= get_dev_path(dmcrypt
['holders'][0])
4412 desc
= ['ceph data (dmcrypt %s %s)' %
4413 (dmcrypt
['type'], holder
)]
4414 desc
+= list_format_more_osd_info_plain(dev
)
4416 desc
= ['ceph data (dmcrypt %s)' % dmcrypt
['type'],
4417 'holders: ' + ','.join(dmcrypt
['holders'])]
4418 elif Ptype
.is_regular_space(dev
['ptype']):
4419 name
= Ptype
.space_ptype_to_name(dev
['ptype'])
4420 desc
.append('ceph ' + name
)
4421 if dev
.get(name
+ '_for'):
4422 desc
.append('for %s' % dev
[name
+ '_for'])
4423 elif Ptype
.is_dmcrypt_space(dev
['ptype']):
4424 name
= Ptype
.space_ptype_to_name(dev
['ptype'])
4425 dmcrypt
= dev
['dmcrypt']
4426 if dmcrypt
['holders'] and len(dmcrypt
['holders']) == 1:
4427 holder
= get_dev_path(dmcrypt
['holders'][0])
4428 desc
= ['ceph ' + name
+ ' (dmcrypt %s %s)' %
4429 (dmcrypt
['type'], holder
)]
4431 desc
= ['ceph ' + name
+ ' (dmcrypt %s)' % dmcrypt
['type']]
4432 if dev
.get(name
+ '_for'):
4433 desc
.append('for %s' % dev
[name
+ '_for'])
4435 desc
.append(dev
['type'])
4436 if dev
.get('fs_type'):
4437 desc
.append(dev
['fs_type'])
4438 elif dev
.get('ptype'):
4439 desc
.append(dev
['ptype'])
4440 if dev
.get('mount'):
4441 desc
.append('mounted on %s' % dev
['mount'])
4442 return '%s%s %s' % (prefix
, dev
['path'], ', '.join(desc
))
4445 def list_format_plain(devices
):
4447 for device
in devices
:
4448 if device
.get('partitions'):
4449 lines
.append('%s :' % device
['path'])
4450 for p
in sorted(device
['partitions'], key
=lambda x
: x
['path']):
4451 lines
.append(list_format_dev_plain(dev
=p
,
4454 lines
.append(list_format_dev_plain(dev
=device
,
4456 return "\n".join(lines
)
4459 def list_dev(dev
, uuid_map
, space_map
):
4465 info
['is_partition'] = is_partition(dev
)
4466 if info
['is_partition']:
4467 ptype
= get_partition_type(dev
)
4468 info
['uuid'] = get_partition_uuid(dev
)
4471 info
['ptype'] = ptype
4472 LOG
.info("list_dev(dev = " + dev
+ ", ptype = " + str(ptype
) + ")")
4473 if ptype
in (PTYPE
['regular']['osd']['ready'],
4474 PTYPE
['mpath']['osd']['ready']):
4475 info
['type'] = 'data'
4476 if ptype
== PTYPE
['mpath']['osd']['ready']:
4477 info
['multipath'] = True
4478 list_dev_osd(dev
, uuid_map
, info
)
4479 elif ptype
in (PTYPE
['regular']['lockbox']['ready'],
4480 PTYPE
['mpath']['lockbox']['ready']):
4481 info
['type'] = 'lockbox'
4482 if ptype
== PTYPE
['mpath']['osd']['ready']:
4483 info
['multipath'] = True
4484 list_dev_lockbox(dev
, uuid_map
, info
)
4485 elif ptype
== PTYPE
['plain']['osd']['ready']:
4486 holders
= is_held(dev
)
4487 info
['type'] = 'data'
4488 info
['dmcrypt']['holders'] = holders
4489 info
['dmcrypt']['type'] = 'plain'
4490 if len(holders
) == 1:
4491 list_dev_osd(get_dev_path(holders
[0]), uuid_map
, info
)
4492 elif ptype
== PTYPE
['luks']['osd']['ready']:
4493 holders
= is_held(dev
)
4494 info
['type'] = 'data'
4495 info
['dmcrypt']['holders'] = holders
4496 info
['dmcrypt']['type'] = 'LUKS'
4497 if len(holders
) == 1:
4498 list_dev_osd(get_dev_path(holders
[0]), uuid_map
, info
)
4499 elif Ptype
.is_regular_space(ptype
) or Ptype
.is_mpath_space(ptype
):
4500 name
= Ptype
.space_ptype_to_name(ptype
)
4502 if ptype
== PTYPE
['mpath'][name
]['ready']:
4503 info
['multipath'] = True
4504 if info
.get('uuid') in space_map
:
4505 info
[name
+ '_for'] = space_map
[info
['uuid']]
4506 elif Ptype
.is_plain_space(ptype
):
4507 name
= Ptype
.space_ptype_to_name(ptype
)
4508 holders
= is_held(dev
)
4510 info
['dmcrypt']['type'] = 'plain'
4511 info
['dmcrypt']['holders'] = holders
4512 if info
.get('uuid') in space_map
:
4513 info
[name
+ '_for'] = space_map
[info
['uuid']]
4514 elif Ptype
.is_luks_space(ptype
):
4515 name
= Ptype
.space_ptype_to_name(ptype
)
4516 holders
= is_held(dev
)
4518 info
['dmcrypt']['type'] = 'LUKS'
4519 info
['dmcrypt']['holders'] = holders
4520 if info
.get('uuid') in space_map
:
4521 info
[name
+ '_for'] = space_map
[info
['uuid']]
4523 path
= is_mounted(dev
)
4524 fs_type
= get_dev_fs(dev
)
4526 info
['type'] = 'swap'
4528 info
['type'] = 'other'
4530 info
['fs_type'] = fs_type
4532 info
['mount'] = path
4538 partmap
= list_all_partitions()
4542 for base
, parts
in sorted(partmap
.items()):
4544 dev
= get_dev_path(p
)
4545 part_uuid
= get_partition_uuid(dev
)
4547 uuid_map
[part_uuid
] = dev
4548 ptype
= get_partition_type(dev
)
4549 LOG
.debug("main_list: " + dev
+
4550 " ptype = " + str(ptype
) +
4551 " uuid = " + str(part_uuid
))
4552 if ptype
in Ptype
.get_ready_by_name('osd'):
4553 if Ptype
.is_dmcrypt(ptype
, 'osd'):
4554 holders
= is_held(dev
)
4555 if len(holders
) != 1:
4557 dev_to_mount
= get_dev_path(holders
[0])
4561 fs_type
= get_dev_fs(dev_to_mount
)
4562 if fs_type
is not None:
4563 mount_options
= get_mount_options(cluster
='ceph',
4566 tpath
= mount(dev
=dev_to_mount
,
4567 fstype
=fs_type
, options
=mount_options
)
4569 for name
in Space
.NAMES
:
4570 space_uuid
= get_oneliner(tpath
,
4573 space_map
[space_uuid
.lower()] = dev
4579 LOG
.debug("main_list: " + str(partmap
) + ", uuid_map = " +
4580 str(uuid_map
) + ", space_map = " + str(space_map
))
4583 for base
, parts
in sorted(partmap
.items()):
4585 disk
= {'path': get_dev_path(base
)}
4587 for p
in sorted(parts
):
4588 partitions
.append(list_dev(get_dev_path(p
),
4591 disk
['partitions'] = partitions
4592 devices
.append(disk
)
4594 device
= list_dev(get_dev_path(base
), uuid_map
, space_map
)
4595 device
['path'] = get_dev_path(base
)
4596 devices
.append(device
)
4597 LOG
.debug("list_devices: " + str(devices
))
4603 out
, err
, ret
= command(
4607 '-o', 'name,mountpoint'
4610 except subprocess
.CalledProcessError
as e
:
4611 LOG
.info('zfs list -o name,mountpoint '
4612 'fails.\n (Error: %s)' % e
)
4614 lines
= out
.splitlines()
4615 for line
in lines
[1:]:
4616 vdevline
= line
.split()
4617 if os
.path
.exists(os
.path
.join(vdevline
[1], 'active')):
4618 elems
= os
.path
.split(vdevline
[1])
4619 print(vdevline
[0], "ceph data, active, cluster ceph,", elems
[1],
4620 "mounted on:", vdevline
[1])
4622 print(vdevline
[0] + " other, zfs, mounted on: " + vdevline
[1])
4625 def main_list(args
):
4628 main_list_freebsd(args
)
4630 main_list_protected(args
)
4633 def main_list_protected(args
):
4634 devices
= list_devices()
4637 for path
in args
.path
:
4638 if os
.path
.exists(path
):
4639 paths
.append(os
.path
.realpath(path
))
4642 selected_devices
= []
4643 for device
in devices
:
4645 if re
.search(path
+ '$', device
['path']):
4646 selected_devices
.append(device
)
4648 selected_devices
= devices
4649 if args
.format
== 'json':
4650 print(json
.dumps(selected_devices
))
4652 output
= list_format_plain(selected_devices
)
4657 def main_list_freebsd(args
):
4658 # Currently accomodate only ZFS Filestore partitions
4659 # return a list of VDEVs and mountpoints
4661 # NAME USED AVAIL REFER MOUNTPOINT
4662 # osd0 1.01G 1.32T 1.01G /var/lib/ceph/osd/osd.0
4663 # osd1 1.01G 1.32T 1.01G /var/lib/ceph/osd/osd.1
4667 ###########################
4669 # Mark devices that we want to suppress activates on with a
4672 # /var/lib/ceph/tmp/suppress-activate.sdb
4674 # where the last bit is the sanitized device name (/dev/X without the
4675 # /dev/ prefix) and the is_suppress() check matches a prefix. That
4676 # means suppressing sdb will stop activate on sdb1, sdb2, etc.
4679 def is_suppressed(path
):
4680 disk
= os
.path
.realpath(path
)
4682 if (not disk
.startswith('/dev/') or
4683 not ldev_is_diskdevice(disk
)):
4685 base
= get_dev_name(disk
)
4687 if os
.path
.exists(SUPPRESS_PREFIX
+ base
): # noqa
4694 def set_suppress(path
):
4695 disk
= os
.path
.realpath(path
)
4696 if not os
.path
.exists(disk
):
4697 raise Error('does not exist', path
)
4698 if not ldev_is_diskdevice(path
):
4699 raise Error('not a block device', path
)
4700 base
= get_dev_name(disk
)
4702 with
open(SUPPRESS_PREFIX
+ base
, 'w') as f
: # noqa
4704 LOG
.info('set suppress flag on %s', base
)
4707 def unset_suppress(path
):
4708 disk
= os
.path
.realpath(path
)
4709 if not os
.path
.exists(disk
):
4710 raise Error('does not exist', path
)
4711 if not ldev_is_diskdevice(path
):
4712 raise Error('not a block device', path
)
4713 assert disk
.startswith('/dev/')
4714 base
= get_dev_name(disk
)
4716 fn
= SUPPRESS_PREFIX
+ base
# noqa
4717 if not os
.path
.exists(fn
):
4718 raise Error('not marked as suppressed', path
)
4722 LOG
.info('unset suppress flag on %s', base
)
4723 except OSError as e
:
4724 raise Error('failed to unsuppress', e
)
4727 def main_suppress(args
):
4728 set_suppress(args
.path
)
4731 def main_unsuppress(args
):
4732 unset_suppress(args
.path
)
4736 for dev
in args
.dev
:
4740 def main_trigger(args
):
4741 LOG
.debug("main_trigger: " + str(args
))
4742 if is_systemd() and not args
.sync
:
4743 # http://www.freedesktop.org/software/systemd/man/systemd-escape.html
4744 escaped_dev
= args
.dev
[1:].replace('-', '\\x2d')
4745 service
= 'ceph-disk@{dev}.service'.format(dev
=escaped_dev
)
4746 LOG
.info('systemd detected, triggering %s' % service
)
4756 if is_upstart() and not args
.sync
:
4757 LOG
.info('upstart detected, triggering ceph-disk task')
4763 'dev={dev}'.format(dev
=args
.dev
),
4764 'pid={pid}'.format(pid
=os
.getpid()),
4769 if get_ceph_user() == 'ceph':
4770 command_check_call(['chown', 'ceph:ceph', args
.dev
])
4771 parttype
= get_partition_type(args
.dev
)
4772 partid
= get_partition_uuid(args
.dev
)
4774 LOG
.info('trigger {dev} parttype {parttype} uuid {partid}'.format(
4780 ceph_disk
= ['ceph-disk']
4782 ceph_disk
.append('--verbose')
4784 if parttype
in (PTYPE
['regular']['osd']['ready'],
4785 PTYPE
['mpath']['osd']['ready']):
4786 out
, err
, ret
= command(
4794 elif parttype
in (PTYPE
['plain']['osd']['ready'],
4795 PTYPE
['luks']['osd']['ready']):
4796 out
, err
, ret
= command(
4805 elif parttype
in (PTYPE
['regular']['journal']['ready'],
4806 PTYPE
['mpath']['journal']['ready']):
4807 out
, err
, ret
= command(
4815 elif parttype
in (PTYPE
['plain']['journal']['ready'],
4816 PTYPE
['luks']['journal']['ready']):
4817 out
, err
, ret
= command(
4826 elif parttype
in (PTYPE
['regular']['block']['ready'],
4827 PTYPE
['regular']['block.db']['ready'],
4828 PTYPE
['regular']['block.wal']['ready'],
4829 PTYPE
['mpath']['block']['ready'],
4830 PTYPE
['mpath']['block.db']['ready'],
4831 PTYPE
['mpath']['block.wal']['ready']):
4832 out
, err
, ret
= command(
4840 elif parttype
in (PTYPE
['plain']['block']['ready'],
4841 PTYPE
['plain']['block.db']['ready'],
4842 PTYPE
['plain']['block.wal']['ready'],
4843 PTYPE
['luks']['block']['ready'],
4844 PTYPE
['luks']['block.db']['ready'],
4845 PTYPE
['luks']['block.wal']['ready']):
4846 out
, err
, ret
= command(
4855 elif parttype
in (PTYPE
['regular']['lockbox']['ready'],
4856 PTYPE
['mpath']['lockbox']['ready']):
4857 out
, err
, ret
= command(
4866 raise Error('unrecognized partition type %s' % parttype
)
4871 raise Error('return code ' + str(ret
))
4878 # A hash table containing 'path': ('uid', 'gid', blocking, recursive)
4880 ('/usr/bin/ceph-mon', 'root', ROOTGROUP
, True, False),
4881 ('/usr/bin/ceph-mds', 'root', ROOTGROUP
, True, False),
4882 ('/usr/bin/ceph-osd', 'root', ROOTGROUP
, True, False),
4883 ('/usr/bin/radosgw', 'root', ROOTGROUP
, True, False),
4884 ('/etc/ceph', 'root', ROOTGROUP
, True, True),
4885 ('/var/run/ceph', 'ceph', 'ceph', True, True),
4886 ('/var/log/ceph', 'ceph', 'ceph', True, True),
4887 ('/var/log/radosgw', 'ceph', 'ceph', True, True),
4888 ('/var/lib/ceph', 'ceph', 'ceph', True, False),
4891 # Relabel/chown all files under /var/lib/ceph/ recursively (except for osd)
4892 for directory
in glob
.glob('/var/lib/ceph/*'):
4893 if directory
== '/var/lib/ceph/osd':
4894 fix_table
.append((directory
, 'ceph', 'ceph', True, False))
4896 fix_table
.append((directory
, 'ceph', 'ceph', True, True))
4898 # Relabel/chown the osds recursively and in parallel
4899 for directory
in glob
.glob('/var/lib/ceph/osd/*'):
4900 fix_table
.append((directory
, 'ceph', 'ceph', False, True))
4902 LOG
.debug("fix_table: " + str(fix_table
))
4904 # The lists of background processes
4906 permissions_processes
= []
4907 selinux_processes
= []
4909 # Preliminary checks
4910 if args
.selinux
or args
.all
:
4911 out
, err
, ret
= command(['selinuxenabled'])
4913 LOG
.error('SELinux is not enabled, please enable it, first.')
4914 raise Error('no SELinux')
4916 for daemon
in ['ceph-mon', 'ceph-osd', 'ceph-mds', 'radosgw', 'ceph-mgr']:
4917 out
, err
, ret
= command(['pgrep', daemon
])
4919 LOG
.error(daemon
+ ' is running, please stop it, first')
4920 raise Error(daemon
+ ' running')
4922 # Relabel the basic system data without the ceph files
4923 if args
.system
or args
.all
:
4924 c
= ['restorecon', '-R', '/']
4925 for directory
, _
, _
, _
, _
in fix_table
:
4926 # Skip /var/lib/ceph subdirectories
4927 if directory
.startswith('/var/lib/ceph/'):
4932 out
, err
, ret
= command(c
)
4935 LOG
.error("Failed to restore labels of the underlying system")
4937 raise Error("basic restore failed")
4939 # Use find to relabel + chown ~simultaenously
4941 for directory
, uid
, gid
, blocking
, recursive
in fix_table
:
4942 # Skip directories/files that are not installed
4943 if not os
.access(directory
, os
.F_OK
):
4951 ':'.join((uid
, gid
)),
4960 # Just pass -maxdepth 0 for non-recursive calls
4962 c
+= ['-maxdepth', '0']
4965 out
, err
, ret
= command(c
)
4968 LOG
.error("Failed to fix " + directory
)
4970 raise Error(directory
+ " fix failed")
4972 all_processes
.append(command_init(c
))
4974 LOG
.debug("all_processes: " + str(all_processes
))
4975 for process
in all_processes
:
4976 out
, err
, ret
= command_wait(process
)
4978 LOG
.error("A background find process failed")
4980 raise Error("background failed")
4983 if args
.permissions
:
4984 for directory
, uid
, gid
, blocking
, recursive
in fix_table
:
4985 # Skip directories/files that are not installed
4986 if not os
.access(directory
, os
.F_OK
):
4993 ':'.join((uid
, gid
)),
4999 ':'.join((uid
, gid
)),
5004 out
, err
, ret
= command(c
)
5007 LOG
.error("Failed to chown " + directory
)
5009 raise Error(directory
+ " chown failed")
5011 permissions_processes
.append(command_init(c
))
5013 LOG
.debug("permissions_processes: " + str(permissions_processes
))
5014 for process
in permissions_processes
:
5015 out
, err
, ret
= command_wait(process
)
5017 LOG
.error("A background permissions process failed")
5019 raise Error("background failed")
5021 # Fix SELinux labels
5023 for directory
, uid
, gid
, blocking
, recursive
in fix_table
:
5024 # Skip directories/files that are not installed
5025 if not os
.access(directory
, os
.F_OK
):
5041 out
, err
, ret
= command(c
)
5044 LOG
.error("Failed to restore labels for " + directory
)
5046 raise Error(directory
+ " relabel failed")
5048 selinux_processes
.append(command_init(c
))
5050 LOG
.debug("selinux_processes: " + str(selinux_processes
))
5051 for process
in selinux_processes
:
5052 out
, err
, ret
= command_wait(process
)
5054 LOG
.error("A background selinux process failed")
5056 raise Error("background failed")
5059 "The ceph files has been fixed, please reboot "
5060 "the system for the changes to take effect."
5064 def setup_statedir(dir):
5065 # XXX The following use of globals makes linting
5066 # really hard. Global state in Python is iffy and
5067 # should be avoided.
5071 if not os
.path
.exists(STATEDIR
):
5073 if not os
.path
.exists(STATEDIR
+ "/tmp"):
5074 os
.mkdir(STATEDIR
+ "/tmp")
5077 prepare_lock
= FileLock(STATEDIR
+ '/tmp/ceph-disk.prepare.lock')
5079 global activate_lock
5080 activate_lock
= FileLock(STATEDIR
+ '/tmp/ceph-disk.activate.lock')
5082 global SUPPRESS_PREFIX
5083 SUPPRESS_PREFIX
= STATEDIR
+ '/tmp/suppress-activate.'
5086 def setup_sysconfdir(dir):
5091 def parse_args(argv
):
5092 parser
= argparse
.ArgumentParser(
5095 parser
.add_argument(
5097 action
='store_true', default
=None,
5098 help='be more verbose',
5100 parser
.add_argument(
5102 action
='store_true', default
=None,
5103 help='log to stdout',
5105 parser
.add_argument(
5106 '--prepend-to-path',
5109 help=('prepend PATH to $PATH for backward compatibility '
5110 '(default /usr/bin)'),
5112 parser
.add_argument(
5115 default
='/var/lib/ceph',
5116 help=('directory in which ceph state is preserved '
5117 '(default /var/lib/ceph)'),
5119 parser
.add_argument(
5122 default
='/etc/ceph',
5123 help=('directory in which ceph configuration files are found '
5124 '(default /etc/ceph)'),
5126 parser
.add_argument(
5130 help='use the given user for subprocesses, rather than ceph or root'
5132 parser
.add_argument(
5136 help='use the given group for subprocesses, rather than ceph or root'
5138 parser
.set_defaults(
5139 # we want to hold on to this, for later
5143 subparsers
= parser
.add_subparsers(
5144 title
='subcommands',
5145 description
='valid subcommands',
5146 help='sub-command help',
5149 Prepare
.set_subparser(subparsers
)
5150 make_activate_parser(subparsers
)
5151 make_activate_lockbox_parser(subparsers
)
5152 make_activate_block_parser(subparsers
)
5153 make_activate_journal_parser(subparsers
)
5154 make_activate_all_parser(subparsers
)
5155 make_list_parser(subparsers
)
5156 make_suppress_parser(subparsers
)
5157 make_deactivate_parser(subparsers
)
5158 make_destroy_parser(subparsers
)
5159 make_zap_parser(subparsers
)
5160 make_trigger_parser(subparsers
)
5161 make_fix_parser(subparsers
)
5163 args
= parser
.parse_args(argv
)
5167 def make_fix_parser(subparsers
):
5168 fix_parser
= subparsers
.add_parser(
5170 formatter_class
=argparse
.RawDescriptionHelpFormatter
,
5171 description
=textwrap
.fill(textwrap
.dedent("""\
5173 help='fix SELinux labels and/or file permissions')
5175 fix_parser
.add_argument(
5177 action
='store_true',
5179 help='fix SELinux labels for the non-ceph system data'
5181 fix_parser
.add_argument(
5183 action
='store_true',
5185 help='fix SELinux labels for ceph data'
5187 fix_parser
.add_argument(
5189 action
='store_true',
5191 help='fix file permissions for ceph data'
5193 fix_parser
.add_argument(
5195 action
='store_true',
5197 help='perform all the fix-related operations'
5199 fix_parser
.set_defaults(
5205 def make_trigger_parser(subparsers
):
5206 trigger_parser
= subparsers
.add_parser(
5208 formatter_class
=argparse
.RawDescriptionHelpFormatter
,
5209 description
=textwrap
.fill(textwrap
.dedent("""\
5210 The partition given in argument is activated. The type of the
5211 partition (data, lockbox, journal etc.) is detected by its
5212 type. If the init system is upstart or systemd, the activation is
5213 delegated to it and runs asynchronously, which
5214 helps reduce the execution time of udev actions.
5216 help='activate any device (called by udev)')
5217 trigger_parser
.add_argument(
5221 trigger_parser
.add_argument(
5225 help='cluster name to assign this disk to',
5227 trigger_parser
.add_argument(
5229 action
='store_true', default
=None,
5230 help='map devices with dm-crypt',
5232 trigger_parser
.add_argument(
5233 '--dmcrypt-key-dir',
5235 default
='/etc/ceph/dmcrypt-keys',
5236 help='directory where dm-crypt keys are stored',
5238 trigger_parser
.add_argument(
5240 action
='store_true', default
=None,
5241 help='do operation synchronously; do not trigger systemd',
5243 trigger_parser
.set_defaults(
5246 return trigger_parser
5249 def make_activate_parser(subparsers
):
5250 activate_parser
= subparsers
.add_parser(
5252 formatter_class
=argparse
.RawDescriptionHelpFormatter
,
5253 description
=textwrap
.fill(textwrap
.dedent("""\
5254 Activate the OSD found at PATH (can be a directory
5255 or a device partition, possibly encrypted). When
5256 activated for the first time, a unique OSD id is obtained
5257 from the cluster. If PATH is a directory, a symbolic
5258 link is added in {statedir}/osd/ceph-$id. If PATH is
5259 a partition, it is mounted on {statedir}/osd/ceph-$id.
5260 Finally, the OSD daemon is run.
5262 If the OSD depends on auxiliary partitions (journal, block, ...)
5263 they need to be available otherwise activation will fail. It
5264 may happen if a journal is encrypted and cryptsetup was not
5266 """.format(statedir
=STATEDIR
))),
5267 help='Activate a Ceph OSD')
5268 activate_parser
.add_argument(
5270 action
='store_true', default
=None,
5271 help='mount a block device [deprecated, ignored]',
5273 activate_parser
.add_argument(
5276 help='bootstrap-osd keyring path template (%(default)s)',
5277 dest
='activate_key_template',
5279 activate_parser
.add_argument(
5281 metavar
='INITSYSTEM',
5282 help='init system to manage this dir',
5284 choices
=INIT_SYSTEMS
,
5286 activate_parser
.add_argument(
5287 '--no-start-daemon',
5288 action
='store_true', default
=None,
5289 help='do not start the daemon',
5291 activate_parser
.add_argument(
5294 help='path to block device or directory',
5296 activate_parser
.add_argument(
5298 action
='store_true', default
=None,
5299 help='map DATA and/or JOURNAL devices with dm-crypt',
5301 activate_parser
.add_argument(
5302 '--dmcrypt-key-dir',
5304 default
='/etc/ceph/dmcrypt-keys',
5305 help='directory where dm-crypt keys are stored',
5307 activate_parser
.add_argument(
5309 action
='store_true', default
=False,
5310 help='activate the deactived OSD',
5312 activate_parser
.set_defaults(
5313 activate_key_template
='{statedir}/bootstrap-osd/{cluster}.keyring',
5316 return activate_parser
5319 def make_activate_lockbox_parser(subparsers
):
5320 parser
= subparsers
.add_parser(
5322 formatter_class
=argparse
.RawDescriptionHelpFormatter
,
5323 description
=textwrap
.fill(textwrap
.dedent("""\
5324 Mount the partition found at PATH on {statedir}/osd-lockbox/$uuid
5325 where $uuid uniquely identifies the OSD that needs this lockbox
5326 to retrieve keys from the monitor and unlock its partitions.
5328 If the OSD has one or more auxiliary devices (journal, block, ...)
5329 symbolic links are created at {statedir}/osd-lockbox/$other_uuid
5330 and point to {statedir}/osd-lockbox/$uuid. This will, for instance,
5331 allow a journal encrypted in a partition identified by $other_uuid to
5332 fetch the keys it needs from the monitor.
5334 Finally the OSD is activated, as it would be with ceph-disk activate.
5335 """.format(statedir
=STATEDIR
))),
5336 help='Activate a Ceph lockbox')
5337 parser
.add_argument(
5339 help='bootstrap-osd keyring path template (%(default)s)',
5340 dest
='activate_key_template',
5342 parser
.add_argument(
5343 '--dmcrypt-key-dir',
5345 default
='/etc/ceph/dmcrypt-keys',
5346 help='directory where dm-crypt keys are stored',
5348 parser
.add_argument(
5351 help='path to block device',
5353 parser
.set_defaults(
5354 activate_key_template
='{statedir}/bootstrap-osd/{cluster}.keyring',
5355 func
=main_activate_lockbox
,
5360 def make_activate_block_parser(subparsers
):
5361 return make_activate_space_parser('block', subparsers
)
5364 def make_activate_journal_parser(subparsers
):
5365 return make_activate_space_parser('journal', subparsers
)
5368 def make_activate_space_parser(name
, subparsers
):
5369 activate_space_parser
= subparsers
.add_parser(
5370 'activate-%s' % name
,
5371 formatter_class
=argparse
.RawDescriptionHelpFormatter
,
5372 description
=textwrap
.fill(textwrap
.dedent("""\
5373 Activating a {name} partition is only meaningfull
5374 if it is encrypted and it will map it using
5377 Finally the corresponding OSD is activated,
5378 as it would be with ceph-disk activate.
5379 """.format(name
=name
))),
5380 help='Activate an OSD via its %s device' % name
)
5381 activate_space_parser
.add_argument(
5384 help='path to %s block device' % name
,
5386 activate_space_parser
.add_argument(
5389 help='bootstrap-osd keyring path template (%(default)s)',
5390 dest
='activate_key_template',
5392 activate_space_parser
.add_argument(
5394 metavar
='INITSYSTEM',
5395 help='init system to manage this dir',
5397 choices
=INIT_SYSTEMS
,
5399 activate_space_parser
.add_argument(
5401 action
='store_true', default
=None,
5402 help=('map data and/or auxiliariy (journal, etc.) '
5403 'devices with dm-crypt'),
5405 activate_space_parser
.add_argument(
5406 '--dmcrypt-key-dir',
5408 default
='/etc/ceph/dmcrypt-keys',
5409 help='directory where dm-crypt keys are stored',
5411 activate_space_parser
.add_argument(
5413 action
='store_true', default
=False,
5414 help='activate the deactived OSD',
5416 activate_space_parser
.set_defaults(
5417 activate_key_template
='{statedir}/bootstrap-osd/{cluster}.keyring',
5418 func
=lambda args
: main_activate_space(name
, args
),
5420 return activate_space_parser
5423 def make_activate_all_parser(subparsers
):
5424 activate_all_parser
= subparsers
.add_parser(
5426 formatter_class
=argparse
.RawDescriptionHelpFormatter
,
5427 description
=textwrap
.fill(textwrap
.dedent("""\
5428 Activate all OSD partitions found in /dev/disk/by-parttypeuuid.
5429 The partitions containing auxiliary devices (journal, block, ...)
5432 help='Activate all tagged OSD partitions')
5433 activate_all_parser
.add_argument(
5436 help='bootstrap-osd keyring path template (%(default)s)',
5437 dest
='activate_key_template',
5439 activate_all_parser
.add_argument(
5441 metavar
='INITSYSTEM',
5442 help='init system to manage this dir',
5444 choices
=INIT_SYSTEMS
,
5446 activate_all_parser
.set_defaults(
5447 activate_key_template
='{statedir}/bootstrap-osd/{cluster}.keyring',
5448 func
=main_activate_all
,
5450 return activate_all_parser
5453 def make_list_parser(subparsers
):
5454 list_parser
= subparsers
.add_parser(
5456 formatter_class
=argparse
.RawDescriptionHelpFormatter
,
5457 description
=textwrap
.fill(textwrap
.dedent("""\
5458 Display all partitions on the system and their
5459 associated Ceph information, if any.
5461 help='List disks, partitions, and Ceph OSDs')
5462 list_parser
.add_argument(
5464 help='output format',
5466 choices
=['json', 'plain'],
5468 list_parser
.add_argument(
5472 help='path to block devices, relative to /sys/block',
5474 list_parser
.set_defaults(
5480 def make_suppress_parser(subparsers
):
5481 suppress_parser
= subparsers
.add_parser(
5482 'suppress-activate',
5483 formatter_class
=argparse
.RawDescriptionHelpFormatter
,
5484 description
=textwrap
.fill(textwrap
.dedent("""\
5485 Add a prefix to the list of suppressed device names
5486 so that they are ignored by all activate* subcommands.
5488 help='Suppress activate on a device (prefix)')
5489 suppress_parser
.add_argument(
5492 help='path to block device or directory',
5494 suppress_parser
.set_defaults(
5498 unsuppress_parser
= subparsers
.add_parser(
5499 'unsuppress-activate',
5500 formatter_class
=argparse
.RawDescriptionHelpFormatter
,
5501 description
=textwrap
.fill(textwrap
.dedent("""\
5502 Remove a prefix from the list of suppressed device names
5503 so that they are no longer ignored by all
5504 activate* subcommands.
5506 help='Stop suppressing activate on a device (prefix)')
5507 unsuppress_parser
.add_argument(
5510 help='path to block device or directory',
5512 unsuppress_parser
.set_defaults(
5513 func
=main_unsuppress
,
5515 return suppress_parser
5518 def make_deactivate_parser(subparsers
):
5519 deactivate_parser
= subparsers
.add_parser(
5521 formatter_class
=argparse
.RawDescriptionHelpFormatter
,
5522 description
=textwrap
.fill(textwrap
.dedent("""\
5523 Deactivate the OSD located at PATH. It stops the OSD daemon
5524 and optionally marks it out (with --mark-out). The content of
5525 the OSD is left untouched.
5527 By default, the, ready, active, INIT-specific files are
5528 removed (so that it is not automatically re-activated by the
5529 udev rules or ceph-disk trigger) and the file deactive is
5530 created to remember the OSD is deactivated.
5532 If the --once option is given, the ready, active, INIT-specific
5533 files are not removed and the OSD will reactivate whenever
5534 ceph-disk trigger is run on one of the devices (journal, data,
5535 block, lockbox, ...).
5537 If the OSD is dmcrypt, remove the data dmcrypt map. When
5538 deactivate finishes, the OSD is down.
5540 help='Deactivate a Ceph OSD')
5541 deactivate_parser
.add_argument(
5545 help='cluster name to assign this disk to',
5547 deactivate_parser
.add_argument(
5551 help='path to block device or directory',
5553 deactivate_parser
.add_argument(
5554 '--deactivate-by-id',
5556 help='ID of OSD to deactive'
5558 deactivate_parser
.add_argument(
5560 action
='store_true', default
=False,
5561 help='option to mark the osd out',
5563 deactivate_parser
.add_argument(
5565 action
='store_true', default
=False,
5566 help='does not need --reactivate to activate again',
5568 deactivate_parser
.set_defaults(
5569 func
=main_deactivate
,
5573 def make_destroy_parser(subparsers
):
5574 destroy_parser
= subparsers
.add_parser(
5576 formatter_class
=argparse
.RawDescriptionHelpFormatter
,
5577 description
=textwrap
.fill(textwrap
.dedent("""\ Destroy the OSD located at PATH. It removes the OSD from the
5578 cluster and marks it destroyed. An OSD must be down before it
5579 can be destroyed. Once it is destroyed, a new OSD can be created
5580 in its place, reusing the same OSD id and position (e.g. after
5581 a failed HDD or SSD is replaced). Alternatively, if the
5582 --purge option is also specified, the OSD is removed from the
5583 CRUSH map and the OSD id is deallocated.""")),
5584 help='Destroy a Ceph OSD')
5585 destroy_parser
.add_argument(
5589 help='cluster name to assign this disk to',
5591 destroy_parser
.add_argument(
5595 help='path to block device or directory',
5597 destroy_parser
.add_argument(
5600 help='ID of OSD to destroy'
5602 destroy_parser
.add_argument(
5603 '--dmcrypt-key-dir',
5605 default
='/etc/ceph/dmcrypt-keys',
5606 help=('directory where dm-crypt keys are stored '
5607 '(If you don\'t know how it work, '
5608 'dont use it. we have default value)'),
5610 destroy_parser
.add_argument(
5612 action
='store_true', default
=False,
5613 help='option to erase data and partition',
5615 destroy_parser
.add_argument(
5617 action
='store_true', default
=False,
5618 help='option to remove OSD from CRUSH map and deallocate the id',
5620 destroy_parser
.set_defaults(
5625 def make_zap_parser(subparsers
):
5626 zap_parser
= subparsers
.add_parser(
5628 formatter_class
=argparse
.RawDescriptionHelpFormatter
,
5629 description
=textwrap
.fill(textwrap
.dedent("""\
5630 Zap/erase/destroy a device's partition table and contents. It
5631 actually uses sgdisk and it's option --zap-all to
5632 destroy both GPT and MBR data structures so that the disk
5633 becomes suitable for repartitioning.
5635 help='Zap/erase/destroy a device\'s partition table (and contents)')
5636 zap_parser
.add_argument(
5640 help='path to block device',
5642 zap_parser
.set_defaults(
5649 args
= parse_args(argv
)
5651 setup_logging(args
.verbose
, args
.log_stdout
)
5653 if args
.prepend_to_path
!= '':
5654 path
= os
.environ
.get('PATH', os
.defpath
)
5655 os
.environ
['PATH'] = args
.prepend_to_path
+ ":" + path
5657 if args
.func
.__name
__ != 'main_trigger':
5658 # trigger may run when statedir is unavailable and does not use it
5659 setup_statedir(args
.statedir
)
5660 setup_sysconfdir(args
.sysconfdir
)
5662 global CEPH_PREF_USER
5663 CEPH_PREF_USER
= args
.setuser
5664 global CEPH_PREF_GROUP
5665 CEPH_PREF_GROUP
= args
.setgroup
5670 main_catch(args
.func
, args
)
5673 def setup_logging(verbose
, log_stdout
):
5674 loglevel
= logging
.WARNING
5676 loglevel
= logging
.DEBUG
5679 ch
= logging
.StreamHandler(stream
=sys
.stdout
)
5680 ch
.setLevel(loglevel
)
5681 formatter
= logging
.Formatter('%(funcName)s: %(message)s')
5682 ch
.setFormatter(formatter
)
5684 LOG
.setLevel(loglevel
)
5686 logging
.basicConfig(
5688 format
='%(funcName)s: %(message)s',
5692 def main_catch(func
, args
):
5699 '{prog}: {msg}'.format(
5705 except CephDiskException
as error
:
5706 exc_name
= error
.__class
__.__name
__
5708 '{prog} {exc_name}: {msg}'.format(
5720 if __name__
== '__main__':